Ceph 实践

准备工作:

  • 关闭SELinux
1
2
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
  • 打开Ceph端口
1
2
3
# firewall-cmd --zone=public --add-port=6789/tcp --permanent
# firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent
# firewall-cmd --reload
  • 安装epel源
1
# yum -y install epel-relea
  • 安装 ntp 同步时间
1
2
3
4
5
# yum -y install ntp ntpdate ntp-doc
# ntpdate 0.us.pool.ntp.org
# hwclock --systohc
# systemctl enable ntpd.service
# systemctl start ntpd.service

安装Ceph

安装Ceph源

1
2
# rpm -Uvh http://ceph.com/rpm-hammer/el7/noarch/ceph-release-1-1.el7.noarch.rpm
# yum update -y

安装ceph-deploy工具

1
# yum install ceph-deploy -y

创建安装Ceph的工作目录

1
# mkdir ~/ceph-installation

创建Monitor节点

1
#ceph-deploy new mon1 mon2 mon3

为每个节点安装Ceph包(包含Monitor节点与OSD节点)

1
# ceph-deploy install mon1 mon2 mon3 osd1

初始化Monitor节点

1
# ceph-deploy mon create-initial

安装一个OSD

  • 分区:
1
2
3
4
parted /dev/sdb
mklabel gpt
mkpart primary 0% 50GB
mkpart primary xfs 50GB 100%
  • 格式化数据分区
1
mkfs.xfs /dev/sdb2
  • 创建OSD
1
ceph-deploy osd create osd1:/dev/sdb2:/dev/sdb1
  • 激活OSD
1
ceph-deploy osd activate osd1:/dev/sdb2:/dev/sdb1

同时创建多个

1
2
3
ceph-deploy osd create controller2:/dev/sdb2:/dev/sdb1 controller2:/dev/sdd2:/dev/sdd1 controller2:/dev/sde2:/dev/sde1

ceph-deploy osd activate controller2:/dev/sdb2:/dev/sdb1 controller2:/dev/sdd2:/dev/sdd1 controller2:/dev/sde2:/dev/sde1

image

注意点:

  • 步骤1-6只需要在一台机器上,即安装节点上做

  • 只有这两步必须在各个节点做

1
2
3
4
5
格式化数据分区
mkfs.xfs /dev/sdb2

创建OSD
ceph-deploy osd create osd1:/dev/sdb2:/dev/sdb1
  • 机械硬盘

    第一个盘的第一个分区给日志分区

  • SSD

    SSD的多个分区给多个osd做log存储

基本配置与检查

1
[root@controller3 ceph]# ceph health
1
[root@controller3 ceph]# ceph -w
1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@controller2 ~(keystone_admin)]# lsblk
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda               8:0    0  2.7T  0 disk 
├─sda1            8:1    0    1M  0 part 
├─sda2            8:2    0  500M  0 part /boot
└─sda3            8:3    0  2.7T  0 part 
  ├─centos-swap 253:0    0  7.9G  0 lvm  [SWAP]
  ├─centos-root 253:1    0   50G  0 lvm  /
  └─centos-home 253:2    0  2.7T  0 lvm  /home
sdb               8:16   0  2.7T  0 disk 
sdd               8:48   0  2.7T  0 disk 
sde               8:64   0  2.7T  0 disk 
loop0             7:0    0    2G  0 loop /srv/node/swiftloopback
loop2             7:2    0 20.6G  0 loop
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@controller2 ~(keystone_admin)]# parted /dev/sdb
GNU Parted 3.1
Using /dev/sdb
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt                                                      
(parted) mkpart primary 0% 50GB                                           
(parted) mkpart primary xfs 50GB 100%                                     
(parted) p                                                                
Model: ATA HGST HUS724030AL (scsi)
Disk /dev/sdb: 3001GB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Disk Flags: 

Number  Start   End     Size    File system  Name     Flags
 1      1049kB  50.0GB  50.0GB               primary
 2      50.0GB  3001GB  2951GB               primary

(parted)   
1
2
3
4
5
6
7
8
9
10
11
12
13
[root@controller3 ceph]# ceph osd df
ID WEIGHT  REWEIGHT SIZE   USE    AVAIL  %USE VAR  
 0 2.67999  1.00000  2746G 37096k  2746G 0.00 1.03 
 7 2.67999  1.00000  2746G 35776k  2746G 0.00 0.99 
 8 2.67999  1.00000  2746G 35256k  2746G 0.00 0.98 
 1 2.67999  1.00000  2746G 36800k  2746G 0.00 1.02 
 5 2.67999  1.00000  2746G 35568k  2746G 0.00 0.99 
 6 2.67999  1.00000  2746G 35572k  2746G 0.00 0.99 
 2 2.67999  1.00000  2746G 36048k  2746G 0.00 1.00 
 3 2.67999  1.00000  2746G 36128k  2746G 0.00 1.00 
 4 2.67999  1.00000  2746G 35664k  2746G 0.00 0.99 
              TOTAL 24719G   316M 24719G 0.00      
MIN/MAX VAR: 0.98/1.03  STDDEV: 0
1
2
3
4
5
6
7
8
9
10
[root@controller3 ceph]# ceph osd pool  get rbd size


[root@controller3 ceph]# ceph osd pool set rbd pg_num 256
set pool 0 pg_num to 256
[root@controller3 ceph]# ceph osd pool set rbd pgp_num 256
Error EBUSY: currently creating pgs, wait
[root@controller3 ceph]# ceph osd pool set rbd pgp_num 256
set pool 0 pgp_num to 256
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@controller3 ceph]# ceph osd tree
ID WEIGHT   TYPE NAME            UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 24.11993 root default                                           
-2  8.03998     host controller3                                   
 0  2.67999         osd.0             up  1.00000          1.00000 
 7  2.67999         osd.7             up  1.00000          1.00000 
 8  2.67999         osd.8             up  1.00000          1.00000 
-3  8.03998     host controller1                                   
 1  2.67999         osd.1             up  1.00000          1.00000 
 5  2.67999         osd.5             up  1.00000          1.00000 
 6  2.67999         osd.6             up  1.00000          1.00000 
-4  8.03998     host controller2                                   
 2  2.67999         osd.2             up  1.00000          1.00000 
 3  2.67999         osd.3             up  1.00000          1.00000 
 4  2.67999         osd.4             up  1.00000          1.00000 

image

1
2
3
4
5
6
7
[root@controller3 ceph]# ceph df
GLOBAL:
    SIZE       AVAIL      RAW USED     %RAW USED 
    24719G     24719G         307M             0 
POOLS:
    NAME     ID     USED     %USED     MAX AVAIL     OBJECTS 
    rbd      0         0         0         8239G           0 

image

蒋暕青

蒋暕青
Lots of mountains to climb,lots of enemies to defeat.

OpenStack cinder mutil-attach技术探秘

OpenStack cinder mutil-attach技术探秘 Continue reading

OpenStack octavia简介

Published on July 18, 2018

Openstack Cyborg项目介绍

Published on July 15, 2018