RHEL 6.7下安装Ceph 0.80.11的过程

系统环境:
rhel6.7,挂载iso源
ceph0.8.11

一、安装ceph
lvextend -L +50G /dev/mapper/vg_system-lv_root
resize2fs /dev/mapper/vg_system-lv_root
yum install -y automake autoconf automake libtool make snappy snappy-devel libblkid libblkid-devel libudev libudev-devel expat expat-devel fuse fuse-devel libaio-devel xfsprogs-devel boost* ncurses-devel expect redhat-lsb
rpm -ivh leveldb-1.7.0-2.el6.x86_64.rpm
rpm -ivh leveldb-devel-1.7.0-2.el6.x86_64.rpm
rpm -ivh cryptopp-5.6.2-9.el6.x86_64.rpm
rpm -ivh cryptopp-devel-5.6.2-9.el6.x86_64.rpm
rpm -ivh libedit-devel-2.11-4.20080712cvs.1.el6.x86_64.rpm
cd /root/
tar xzf argparse-1.4.0.tar.gz
cd argparse-1.4.0
python setup.py install
cd /root/
tar xzf ceph-0.80.11.tar.gz
cd ceph-0.80.11
./autogen.sh
./configure –prefix=/usr/local/ceph –without-tcmalloc –without-libatomic-ops
make -j 32 && make install
cp -vf /usr/local/ceph/lib/python2.6/site-packages/* /usr/lib64/python2.6
echo /usr/local/ceph/lib >/etc/ld.so.conf.d/ceph.conf
ldconfig
cd /root/
/usr/local/ceph/bin/ceph –version

二、准备环境
cat >>/etc/hosts <<EOF
10.10.10.105 ceph1 ceph1.dearda
10.10.10.106 ceph2 ceph2.dearda
10.10.10.107 ceph3 ceph3.dearda
10.10.10.108 ceph4 ceph4.dearda
10.10.10.109 ceph5 ceph5.dearda
EOF

if ! grep ceph /etc/profile;then echo “export PATH=\$PATH:/usr/local/ceph/bin:/usr/local/ceph/sbin” >>/etc/profile;export PATH=$PATH:/usr/local/ceph/bin:/usr/local/ceph/sbin;fi
mkdir /etc/ceph
cd /root/ceph-0.80.11/src
cp sample.ceph.conf /etc/ceph/ceph.conf
cp sample.fetch_config /etc/ceph/fetch_config
cp init-ceph /etc/init.d/ceph

ssh-keygen -t rsa
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph1
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph2
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph3
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph4

for i in 2 3;do ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph$i;done

复制ceph.conf到所有机器
for i in 2 3;do scp /etc/ceph/ceph.conf root@ceph$i:/etc/ceph/;done

ceph.conf内容如下:
##########################################################
[global]
auth cluster required = none
auth service required = none
auth client required = none

[osd]
osd data = /ceph_osd
osd mkfs type = xfs

[mon]
mon data = /var/lib/ceph/mon/mon.a

[mon.a]
host = ceph1
mon addr = 10.10.10.105:6789

[osd.0]
host = ceph2
devs = /dev/sdb1
[osd.1]
host = ceph3
devs = /dev/sdb1

[mds.a]
host = ceph1
##########################################################

三、配置ceph

1、osd,两个节点分别操作
mkdir /ceph_osd
mkdir -p /var/lib/ceph/osd/ceph-0/ 或者 mkdir -p /var/lib/ceph/osd/ceph-1/ #这个应该ceph默认路径
mount /dev/sdb1 /ceph_osd/

2、mon
mkdir -p /var/lib/ceph/mon/mon.a

3、初始化ceph,在mon上操作
mkcephfs -a -c /etc/ceph/ceph.conf

4、启动mon节点
/etc/init.d/ceph -c /etc/ceph/ceph.conf start mon mds

5、启动osd节点
ceph osd create
/etc/init.d/ceph -c /etc/ceph/ceph.conf start osd

查看ceph状态
[root@ceph1 ~]# ceph -s
cluster bbbdca9b-5a13-4a02-bf47-a62b12d045ba
health HEALTH_WARN 384 pgs degraded; 384 pgs stuck unclean; recovery 40/60 objects degraded (66.667%)
monmap e1: 1 mons at {a=10.10.10.105:6789/0}, election epoch 2, quorum 0 a
mdsmap e4: 1/1/1 up {0=a=up:active}
osdmap e6: 2 osds: 2 up, 2 in
pgmap v10: 384 pgs, 3 pools, 1884 bytes data, 20 objects
195 GB used, 55679 GB / 55875 GB avail
40/60 objects degraded (66.667%)
384 active+degraded
client io 1505 B/s wr, 5 op/s

出现警告,因为osd节点数量少,ceph默认为3,2, osd数要>=备份数.或者可以增加一个osd试试
配置文件增加下面两行
osd pool default size = 2
osd pool default min size =1

客户端挂载,目标为mon、mds服务器
ceph-fuse -m 10.10.10.105:6789 /opt/cfs_test/

2.6.34之后内核才支持ceph
modprobe -l|grep ceph
lsmod ceph
mount -t ceph 10.10.10.105:6789:/ /opt/cfs_test

6、新增osd节点,准备条件:hosts记录、ssh免密钥
在ceph1的/etc/ceph/ceph.conf中增加以下记录:
[osd.2]
host = ceph1
devs = /dev/sdb1

同步配置文件到其他节点
scp /etc/ceph/ceph.conf ceph2:/etc/ceph/
scp /etc/ceph/ceph.conf ceph3:/etc/ceph/

在新增的osd节点ceph1上执行
mkdir -p /var/lib/ceph/osd/ceph-2/ #journal目录
ceph osd create
/etc/init.d/ceph -c /etc/ceph/ceph.conf start osd

出现报错,/ceph_osd目录确认是存在的
[root@ceph1 /]# /etc/init.d/ceph -c /etc/ceph/ceph.conf start osd
=== osd.2 ===
Mounting xfs on ceph1:/ceph_osd
create-or-move updated item name ‘osd.2’ weight 27.28 at location {host=ceph1,root=default} to crush map
Starting Ceph osd.2 on ceph1…
2016-07-03 16:33:14.124236 7fe8b3ce3760 -1 ** ERROR: unable to open OSD superblock on /ceph_osd: (2) No such file or directory
failed: ‘ulimit -n 32768; /usr/local/ceph/bin/ceph-osd -i 2 –pid-file /var/run/ceph/osd.2.pid -c /etc/ceph/ceph.conf –cluster ceph ‘

初始化osd节点,2是新增的osd节点的编号,对应ceph.conf的配置
[root@ceph1 /]# ceph-osd -i 2 –mkfs –mkkey
2016-07-03 16:36:52.296441 7fcecf79e760 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
2016-07-03 16:36:52.323883 7fcecf79e760 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
2016-07-03 16:36:52.325663 7fcecf79e760 -1 filestore(/ceph_osd) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory
2016-07-03 16:36:52.358506 7fcecf79e760 -1 created object store /ceph_osd journal /var/lib/ceph/osd/ceph-2/journal for osd.2 fsid bbbdca9b-5a13-4a02-bf47-a62b12d045ba
2016-07-03 16:36:52.358544 7fcecf79e760 -1 auth: error reading file: /ceph_osd/keyring: can’t open /ceph_osd/keyring: (2) No such file or directory
2016-07-03 16:36:52.358625 7fcecf79e760 -1 created new key in keyring /ceph_osd/keyring

启动新的osd节点,成功
[root@ceph1 /]# /etc/init.d/ceph -c /etc/ceph/ceph.conf start osd
=== osd.2 ===
Mounting xfs on ceph1:/ceph_osd
create-or-move updated item name ‘osd.2’ weight 27.28 at location {host=ceph1,root=default} to crush map
Starting Ceph osd.2 on ceph1…
starting osd.2 at :/0 osd_data /ceph_osd /var/lib/ceph/osd/ceph-2/journal

############################################

一些批量语句:

—for i in 2 3;do scp /etc/ceph/ceph.conf root@ceph$i:/etc/ceph/;done
mkdir -p /var/lib/ceph/osd/ceph-3/
ceph osd create
ceph-osd -i 3 –mkfs –mkkey
/etc/init.d/ceph -c /etc/ceph/ceph.conf start osd

—for i in `seq 4 4`
do
for j in {d..m}
do
cat >> /etc/ceph/ceph.conf << EOF
[osd.$i]
host = ceph1
devs = /dev/sd”$j””1″
EOF
mkdir -p /var/lib/ceph/osd/ceph-$i/
ceph osd create
ceph-osd -i $i –mkfs –mkkey
done
done
for i in 2 3;do scp /etc/ceph/ceph.conf root@ceph$i:/etc/ceph/;done
/etc/init.d/ceph -c /etc/ceph/ceph.conf start osd

7、删除osd节点
[root@ceph1 ~]# /etc/init.d/ceph -c /etc/ceph/ceph.conf stop osd.2
[root@ceph1 ~]# ceph osd rm 2
removed osd.2

批量删除节点
shuzu1=(4 5 6 7 8 9 10 11 12 13)
shuzu2=(d e f g h i j k l m)
for (( i = 0; i < ${#shuzu1[@]}; i++))
do
ceph osd rm ${shuzu1[$i]}
done

批量卸载
for i in {d..m};do
{
umount /ceph_osd.sd$i
}&
done

批量删除目录
shuzu1=(4 5 6 7 8 9 10 11 12 13)
shuzu2=(d e f g h i j k l m)
for (( i = 0; i < ${#shuzu1[@]}; i++))
do
rm -fr /var/lib/ceph/osd/ceph-${shuzu1[$i]}
done

批量删除crush map
shuzu1=(4 5 6 7 8 9 10 11 12 13)
shuzu2=(d e f g h i j k l m)
for (( i = 0; i < ${#shuzu1[@]}; i++))
do
ceph osd crush remove osd.${shuzu1[$i]}
done

查看节点是否完全删除
ceph osd tree

正确删除步骤
1、ceph osd out 3
2、ceph -w
你会观察到OSD的状态从”active+clean” to “active, some degraded objects”, 最终回到 “active+clean”
回到active+clean后, 说明OSD的数据已经重分布完成. 可以进行第三步了
3、sudo /etc/init.d/ceph stop osd.{osd-num} 或 ps -ewf|grep ceph; kill it
4. 从集群中移除osd daemon信息: crush map, key, data, journal, 修改剩余OSD节点的configure.
任意ceph osd/mon节点执行 :
ceph osd crush remove osd.3
ceph auth del osd.3
移除节点
ceph osd rm {osd-num}
#for example
ceph osd rm 3

ceph osd out 3
ceph -w
/etc/init.d/ceph stop osd.3
ceph osd crush remove osd.3
ceph osd rm 3

 

四、通过cobbler安装ceph

cobbler system add –name=ceph1 –mac=A4:DC:BE:10:E8:7D –ip-address=10.10.10.105 –subnet=255.255.255.0 –gateway=10.10.10.1 –interface=eth8 –static=1 –if-gateway=10.10.10.1 –profile=RedHat-6.7-x86_64-dearda –hostname=ceph1.dearda –netboot-enabled=true

cobbler system add –name=ceph2 –mac=A4:DC:BE:10:E8:C7 –ip-address=10.10.10.106 –subnet=255.255.255.0 –gateway=10.10.10.1 –interface=eth8 –static=1 –if-gateway=10.10.10.1 –profile=RedHat-6.7-x86_64-dearda –hostname=ceph2.dearda –netboot-enabled=true

cobbler system add –name=ceph3 –mac=68:A8:28:22:9B:E5 –ip-address=10.10.10.107 –subnet=255.255.255.0 –gateway=10.10.10.1 –interface=eth8 –static=1 –if-gateway=10.10.10.1 –profile=RedHat-6.7-x86_64-dearda –hostname=ceph3.dearda –netboot-enabled=true

cobbler system add –name=ceph4 –mac=68:A8:28:22:9B:ED –ip-address=10.10.10.108 –subnet=255.255.255.0 –gateway=10.10.10.1 –interface=eth8 –static=1 –if-gateway=10.10.10.1 –profile=RedHat-6.7-x86_64-dearda –hostname=ceph4.dearda –netboot-enabled=true

cobbler system add –name=ceph5 –mac=A4:DC:BE:10:E8:C9 –ip-address=10.10.10.109 –subnet=255.255.255.0 –gateway=10.10.10.1 –interface=eth8 –static=1 –if-gateway=10.10.10.1 –profile=RedHat-6.7-x86_64-dearda –hostname=ceph5.dearda –netboot-enabled=true

kickstart文件==========================================================
#platform=x86, AMD64, or Intel EM64T
# System authorization information
auth –useshadow –enablemd5
# System bootloader configuration
bootloader –location=mbr
# Partition clearing information
clearpart –all –initlabel
# Use text mode install
text
# Firewall configuration
firewall –disabled
# Run the Setup Agent on first boot
firstboot –disable
# System keyboard
keyboard us
# System language
lang en_US
# Use network installation
url –url=$tree
# If any cobbler repo definitions were referenced in the kickstart profile, include them here.
$yum_repo_stanza
# Network information
$SNIPPET(‘network_config’)
# Reboot after installation
reboot

#Root password
rootpw –iscrypted $default_password_crypted
# SELinux configuration
selinux –disabled
# Do not configure the X Window System
skipx
# System timezone
timezone Asia/Shanghai
# Install OS instead of upgrade
install
# Clear the Master Boot Record
zerombr
# Allow anaconda to partition the system as needed
#autopart
# Partition clearing information
clearpart –all
# Disk partitioning information
part /boot –fstype=ext4 –ondisk=sda –size=150
part pv.01 –ondisk=sda –size=1 –grow
volgroup vg_system pv.01
logvol / –vgname=vg_system –fstype=ext4 –size=40960 –name=lv_root
logvol /home –vgname=vg_system –fstype=ext4 –size=8192 –name=lv_home
logvol swap –vgname=vg_system –fstype=swap –size=16000 –name=lv_swap
logvol /var –vgname=vg_system –fstype=ext4 –size=16384 –name=lv_var
logvol /tmp –vgname=vg_system –fstype=ext4 –size=8192 –name=lv_tmp
logvol /usr –vgname=vg_system –fstype=ext4 –size=8192 –name=lv_usr
logvol /opt –vgname=vg_system –fstype=ext4 –size=10240 –name=lv_opt

%pre
$SNIPPET(‘log_ks_pre’)
$SNIPPET(‘kickstart_start’)
$SNIPPET(‘pre_install_network_config’)
# Enable installation monitoring
$SNIPPET(‘pre_anamon’)
%end

%packages
@base
mtools
lrzsz
ethtool
bc
lsof
sysstat
gcc*
zlib-devel
openssl-devel
pam-devel
device-mapper-multipath
$SNIPPET(‘func_install_if_enabled’)
$SNIPPET(‘puppet_install_if_enabled’)
%packages

%post –nochroot
$SNIPPET(‘log_ks_post_nochroot’)
%end

%post
$SNIPPET(‘log_ks_post’)
# Start yum configuration
$yum_config_stanza
# End yum configuration
$SNIPPET(‘post_install_kernel_options’)
$SNIPPET(‘post_install_network_config’)
$SNIPPET(‘func_register_if_enabled’)
$SNIPPET(‘puppet_register_if_enabled’)
$SNIPPET(‘download_config_files’)
$SNIPPET(‘koan_environment’)
$SNIPPET(‘redhat_register’)
$SNIPPET(‘cobbler_register’)
# Enable post-install boot notification
$SNIPPET(‘post_anamon’)
# Start final steps
$SNIPPET(‘kickstart_done’)
# End final steps

%post
yum install -y automake autoconf automake libtool make snappy snappy-devel libblkid libblkid-devel libudev libudev-devel expat expat-devel fuse fuse-devel libaio-devel xfsprogs-devel boost* ncurses-devel expect redhat-lsb
cd /root/
wget http://10.10.10.250/cobbler/ceph/leveldb-1.7.0-2.el6.x86_64.rpm
wget http://10.10.10.250/cobbler/ceph/leveldb-devel-1.7.0-2.el6.x86_64.rpm
wget http://10.10.10.250/cobbler/ceph/cryptopp-5.6.2-9.el6.x86_64.rpm
wget http://10.10.10.250/cobbler/ceph/cryptopp-devel-5.6.2-9.el6.x86_64.rpm
wget http://10.10.10.250/cobbler/ceph/libedit-devel-2.11-4.20080712cvs.1.el6.x86_64.rpm
wget http://10.10.10.250/cobbler/ceph/argparse-1.4.0.tar.gz
wget http://10.10.10.250/cobbler/ceph/ceph-0.80.11.tar.gz
rpm -ivh leveldb-1.7.0-2.el6.x86_64.rpm
rpm -ivh leveldb-devel-1.7.0-2.el6.x86_64.rpm
rpm -ivh cryptopp-5.6.2-9.el6.x86_64.rpm
rpm -ivh cryptopp-devel-5.6.2-9.el6.x86_64.rpm
rpm -ivh libedit-devel-2.11-4.20080712cvs.1.el6.x86_64.rpm
cd /root/
tar xzf argparse-1.4.0.tar.gz
cd argparse-1.4.0
python setup.py install
cd /root/
tar xzf ceph-0.80.11.tar.gz
cd ceph-0.80.11
./autogen.sh
./configure –prefix=/usr/local/ceph –without-tcmalloc –without-libatomic-ops
make -j 32 && make install
cp -vf /usr/local/ceph/lib/python2.6/site-packages/* /usr/lib64/python2.6
echo /usr/local/ceph/lib >/etc/ld.so.conf.d/ceph.conf
ldconfig
%end

==========================================================