diff --git a/ceph_osd_df b/ceph_osd_df new file mode 100644 index 0000000000000000000000000000000000000000..1d55b9346a30cca72f9bb8284085405b42311dee --- /dev/null +++ b/ceph_osd_df @@ -0,0 +1,63 @@ +ID CLASS WEIGHT REWEIGHT SIZE USE AVAIL %USE VAR PGS + 0 hdd 3.68689 1.00000 3725G 999G 2725G 26.83 1.00 103 + 2 hdd 3.68689 1.00000 3725G 1081G 2643G 29.04 1.09 111 + 3 hdd 3.68689 1.00000 3725G 1052G 2673G 28.24 1.06 108 + 4 hdd 3.68689 1.00000 3725G 991G 2733G 26.62 1.00 102 + 5 hdd 3.68689 1.00000 3725G 932G 2793G 25.02 0.94 96 + 6 hdd 3.68689 1.00000 3725G 1049G 2676G 28.16 1.05 108 + 7 hdd 3.68689 1.00000 3725G 883G 2842G 23.71 0.89 91 + 8 hdd 3.68689 1.00000 3725G 1030G 2695G 27.66 1.03 106 + 9 hdd 3.68689 1.00000 3725G 964G 2760G 25.89 0.97 99 +54 hdd 3.68689 1.00000 3725G 906G 2819G 24.33 0.91 93 +55 hdd 3.68689 1.00000 3725G 1121G 2604G 30.10 1.13 115 +56 hdd 3.68689 1.00000 3725G 805G 2919G 21.62 0.81 83 + 1 hdd 3.68689 1.00000 3725G 913G 2812G 24.52 0.92 94 +10 hdd 3.68689 1.00000 3725G 1040G 2684G 27.94 1.04 107 +11 hdd 3.68689 1.00000 3725G 923G 2802G 24.78 0.93 95 +12 hdd 3.68689 1.00000 3725G 778G 2947G 20.89 0.78 80 +13 hdd 3.68689 1.00000 3725G 1078G 2646G 28.96 1.08 111 +14 hdd 3.68689 1.00000 3725G 999G 2725G 26.84 1.00 103 +15 hdd 3.68689 1.00000 3725G 1080G 2644G 29.00 1.08 111 +16 hdd 3.68689 1.00000 3725G 997G 2728G 26.76 1.00 102 +17 hdd 3.68689 1.00000 3725G 1186G 2538G 31.86 1.19 122 +18 hdd 3.68689 1.00000 3725G 828G 2896G 22.24 0.83 85 +19 hdd 3.68689 1.00000 3725G 1252G 2472G 33.63 1.26 129 +57 hdd 3.68689 1.00000 3725G 850G 2875G 22.83 0.85 87 +20 hdd 3.68689 1.00000 3725G 1042G 2682G 27.99 1.05 107 +21 hdd 3.68689 1.00000 3725G 1119G 2605G 30.05 1.12 115 +22 hdd 3.68689 1.00000 3725G 1071G 2653G 28.76 1.08 110 +23 hdd 3.68689 1.00000 3725G 971G 2753G 26.08 0.98 100 +24 hdd 3.68689 1.00000 3725G 983G 2741G 26.41 0.99 101 +25 hdd 3.68689 1.00000 3725G 1018G 2706G 27.35 1.02 105 +26 hdd 3.68689 1.00000 3725G 962G 2762G 25.85 0.97 99 +27 hdd 3.68689 1.00000 3725G 1044G 2680G 28.04 1.05 107 +28 hdd 3.68689 1.00000 3725G 905G 2820G 24.30 0.91 93 +29 hdd 3.68689 1.00000 3725G 955G 2769G 25.66 0.96 98 +58 hdd 3.68689 1.00000 3725G 980G 2744G 26.33 0.98 101 +59 hdd 3.68689 1.00000 3725G 806G 2918G 21.65 0.81 83 +30 hdd 3.68689 1.00000 3725G 1056G 2668G 28.36 1.06 109 +31 hdd 3.68689 1.00000 3725G 1117G 2608G 29.99 1.12 115 +32 hdd 3.68689 1.00000 3725G 924G 2801G 24.80 0.93 95 +33 hdd 3.68689 1.00000 3725G 836G 2888G 22.45 0.84 86 +34 hdd 3.68689 1.00000 3725G 1130G 2594G 30.34 1.13 116 +35 hdd 3.68689 1.00000 3725G 1068G 2656G 28.69 1.07 110 +36 hdd 3.68689 1.00000 3725G 1138G 2587G 30.55 1.14 117 +37 hdd 3.68689 1.00000 3725G 1091G 2633G 29.30 1.10 112 +38 hdd 3.68689 1.00000 3725G 1138G 2586G 30.56 1.14 117 +39 hdd 3.68689 1.00000 3725G 917G 2808G 24.62 0.92 94 +40 hdd 3.68689 1.00000 3725G 845G 2880G 22.69 0.85 87 +41 hdd 3.68689 1.00000 3725G 886G 2838G 23.81 0.89 91 +42 hdd 3.68689 1.00000 3725G 1063G 2661G 28.55 1.07 109 +43 hdd 3.68689 1.00000 3725G 954G 2771G 25.62 0.96 98 +44 hdd 3.68689 1.00000 3725G 1014G 2710G 27.25 1.02 104 +45 hdd 3.68689 1.00000 3725G 930G 2794G 24.98 0.93 96 +46 hdd 3.68689 1.00000 3725G 1024G 2701G 27.50 1.03 105 +47 hdd 3.68689 1.00000 3725G 990G 2734G 26.60 0.99 102 +48 hdd 3.68689 1.00000 3725G 853G 2871G 22.92 0.86 88 +49 hdd 3.68689 1.00000 3725G 970G 2754G 26.06 0.97 100 +50 hdd 3.68689 1.00000 3725G 1045G 2680G 28.06 1.05 107 +51 hdd 3.68689 1.00000 3725G 1169G 2556G 31.39 1.17 120 +52 hdd 3.68689 1.00000 3725G 1031G 2693G 27.70 1.04 106 +53 hdd 3.68689 1.00000 3725G 970G 2754G 26.06 0.97 100 + TOTAL 218T 59785G 159T 26.75 +MIN/MAX VAR: 0.78/1.26 STDDEV: 2.74 diff --git a/ceph_osd_tree b/ceph_osd_tree new file mode 100644 index 0000000000000000000000000000000000000000..db1f6e8da49a257fe013ba1d19556854430987cc --- /dev/null +++ b/ceph_osd_tree @@ -0,0 +1,67 @@ +ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF +-1 221.21338 root default +-2 44.24268 host pm0 + 0 hdd 3.68689 osd.0 up 1.00000 1.00000 + 2 hdd 3.68689 osd.2 up 1.00000 1.00000 + 3 hdd 3.68689 osd.3 up 1.00000 1.00000 + 4 hdd 3.68689 osd.4 up 1.00000 1.00000 + 5 hdd 3.68689 osd.5 up 1.00000 1.00000 + 6 hdd 3.68689 osd.6 up 1.00000 1.00000 + 7 hdd 3.68689 osd.7 up 1.00000 1.00000 + 8 hdd 3.68689 osd.8 up 1.00000 1.00000 + 9 hdd 3.68689 osd.9 up 1.00000 1.00000 +54 hdd 3.68689 osd.54 up 1.00000 1.00000 +55 hdd 3.68689 osd.55 up 1.00000 1.00000 +56 hdd 3.68689 osd.56 up 1.00000 1.00000 +-3 44.24268 host pm1 + 1 hdd 3.68689 osd.1 up 1.00000 1.00000 +10 hdd 3.68689 osd.10 up 1.00000 1.00000 +11 hdd 3.68689 osd.11 up 1.00000 1.00000 +12 hdd 3.68689 osd.12 up 1.00000 1.00000 +13 hdd 3.68689 osd.13 up 1.00000 1.00000 +14 hdd 3.68689 osd.14 up 1.00000 1.00000 +15 hdd 3.68689 osd.15 up 1.00000 1.00000 +16 hdd 3.68689 osd.16 up 1.00000 1.00000 +17 hdd 3.68689 osd.17 up 1.00000 1.00000 +18 hdd 3.68689 osd.18 up 1.00000 1.00000 +19 hdd 3.68689 osd.19 up 1.00000 1.00000 +57 hdd 3.68689 osd.57 up 1.00000 1.00000 +-4 44.24268 host pm2 +20 hdd 3.68689 osd.20 up 1.00000 1.00000 +21 hdd 3.68689 osd.21 up 1.00000 1.00000 +22 hdd 3.68689 osd.22 up 1.00000 1.00000 +23 hdd 3.68689 osd.23 up 1.00000 1.00000 +24 hdd 3.68689 osd.24 up 1.00000 1.00000 +25 hdd 3.68689 osd.25 up 1.00000 1.00000 +26 hdd 3.68689 osd.26 up 1.00000 1.00000 +27 hdd 3.68689 osd.27 up 1.00000 1.00000 +28 hdd 3.68689 osd.28 up 1.00000 1.00000 +29 hdd 3.68689 osd.29 up 1.00000 1.00000 +58 hdd 3.68689 osd.58 up 1.00000 1.00000 +59 hdd 3.68689 osd.59 up 1.00000 1.00000 +-5 44.24268 host pm3 +30 hdd 3.68689 osd.30 up 1.00000 1.00000 +31 hdd 3.68689 osd.31 up 1.00000 1.00000 +32 hdd 3.68689 osd.32 up 1.00000 1.00000 +33 hdd 3.68689 osd.33 up 1.00000 1.00000 +34 hdd 3.68689 osd.34 up 1.00000 1.00000 +35 hdd 3.68689 osd.35 up 1.00000 1.00000 +36 hdd 3.68689 osd.36 up 1.00000 1.00000 +37 hdd 3.68689 osd.37 up 1.00000 1.00000 +38 hdd 3.68689 osd.38 up 1.00000 1.00000 +39 hdd 3.68689 osd.39 up 1.00000 1.00000 +40 hdd 3.68689 osd.40 up 1.00000 1.00000 +41 hdd 3.68689 osd.41 up 1.00000 1.00000 +-6 44.24268 host pm4 +42 hdd 3.68689 osd.42 up 1.00000 1.00000 +43 hdd 3.68689 osd.43 up 1.00000 1.00000 +44 hdd 3.68689 osd.44 up 1.00000 1.00000 +45 hdd 3.68689 osd.45 up 1.00000 1.00000 +46 hdd 3.68689 osd.46 up 1.00000 1.00000 +47 hdd 3.68689 osd.47 up 1.00000 1.00000 +48 hdd 3.68689 osd.48 up 1.00000 1.00000 +49 hdd 3.68689 osd.49 up 1.00000 1.00000 +50 hdd 3.68689 osd.50 up 1.00000 1.00000 +51 hdd 3.68689 osd.51 up 1.00000 1.00000 +52 hdd 3.68689 osd.52 up 1.00000 1.00000 +53 hdd 3.68689 osd.53 up 1.00000 1.00000 diff --git a/etc/network/interfaces b/etc/network/interfaces new file mode 100644 index 0000000000000000000000000000000000000000..2552855c8fc406a0e848b8865f50921bc5c10e8b --- /dev/null +++ b/etc/network/interfaces @@ -0,0 +1,34 @@ +# Loopback interface +auto lo +iface lo inet loopback + +# Bond two main interfaces together +allow-vmbr1 bond0 +iface bond0 inet manual + ovs_bridge vmbr1 + ovs_type OVSBond + ovs_bonds eth1 eth0 + pre-up ( ifconfig eth0 mtu 1500 && ifconfig eth1 mtu 1500 ) + ovs_options bond_mode=balance-slb + mtu 1500 + +# Bridge for our bond and vlan virtual interfaces (our VMs will +# also attach to this bridge) +auto vmbr1 +allow-ovs vmbr1 +iface vmbr1 inet manual + ovs_type OVSBridge + ovs_ports bond0 vlan888 + mtu 1500 + +allow-vmbr1 vlan888 +iface vlan888 inet static + ovs_type OVSIntPort + ovs_bridge vmbr1 + ovs_options tag=888 + ovs_extra set interface ${IFACE} external-ids:iface-id=$(hostname -s)-${IFACE}-vif + address 172.18.117.66 + netmask 255.255.255.0 + gateway 172.18.117.254 + mtu 1500 + diff --git a/etc/pve/ceph.conf b/etc/pve/ceph.conf new file mode 100755 index 0000000000000000000000000000000000000000..1277c5e388daff674d67caee6feb7e521051058b --- /dev/null +++ b/etc/pve/ceph.conf @@ -0,0 +1,89 @@ +[global] + cluster network = 172.18.117.0/24 + public network = 172.18.117.0/24 + keyring = /etc/pve/priv/$cluster.$name.keyring + auth client required = none + auth cluster required = none + auth service required = none + bluestore block db size = 53687091200 + bluestore cache size hdd = 2147483648 + filestore xattr use omap = true + fsid = 244c8dc2-6791-4bcd-b72e-0c491a90457c + mon allow pool delete = false + debug asok = 0/0 + debug auth = 0/0 + debug buffer = 0/0 + debug client = 0/0 + debug context = 0/0 + debug crush = 0/0 + debug filer = 0/0 + debug filestore = 0/0 + debug finisher = 0/0 + debug heartbeatmap = 0/0 + debug journal = 0/0 + debug journaler = 0/0 + debug lockdep = 0/0 + debug mds = 0/0 + debug mds balancer = 0/0 + debug mds locker = 0/0 + debug mds log = 0/0 + debug mds log expire = 0/0 + debug mds migrator = 0/0 + debug mon = 0/0 + debug monc = 0/0 + debug ms = 0/0 + debug objclass = 0/0 + debug objectcacher = 0/0 + debug objecter = 0/0 + debug optracker = 0/0 + debug osd = 0/0 + debug paxos = 0/0 + debug perfcounter = 0/0 + debug rados = 0/0 + debug rbd = 0/0 + debug rgw = 0/0 + debug throttle = 0/0 + debug timer = 0/0 + debug tp = 0/0 + +[osd] + keyring = /var/lib/ceph/osd/ceph-$id/keyring + osd pool default min size = 2 + osd pool default pg num = 2048 + osd pool default pgp num = 2048 + osd pool default size = 3 + osd scrub begin hour = 19 + osd scrub end hour = 6 + osd scrub sleep = 0.1 + osd client op priority = 63 + osd op threads = 4 + osd disk threads = 2 + osd recovery op priority = 1 + osd max backfills = 4 + osd recovery max active = 2 + osd recovery max single start = 1 + osd scrub min interval = 172800 + osd scrub max interval = 604800 + osd deep scrub interval = 2419200 + osd deep scrub randomize ratio = 0.5 + osd deep scrub stride = 1048576 + debug osd = 0/0 + debug journal = 0/0 + debug ms = 0/0 + +[mon.pm2] + host = pm2 + mon addr = 172.18.117.68:6789 + +[mon.pm0] + host = pm0 + mon addr = 172.18.117.66:6789 + +[mon.pm4] + host = pm4 + mon addr = 172.18.117.93:6789 + +[client] + rbd cache = true +# admin socket = /var/run/ceph/$cluster-$type.$id.$pid.$cctid.asok + log file = /var/log/ceph/client.log