01 Drbd安装

编译安装drbd

drbd可以通过包管理器来进行安装。此处，我们使用更加通过的编译安装的方式。

参考源码包中的drbd-9.1.7/docker/entry.sh脚本。

下载地址：https://github.com/LINBIT/drbd/archive/refs/tags/drbd-9.1.7.tar.gz

# 加载依赖模块。说明：下面的模块并不是必须的
# we are not too strict about these, not all are required everywhere
	#
	# libcrc32c: dependency for DRBD
	# nvmet_rdma, nvme_rdma: LINSTOR NVME layer
	# loop: LINSTOR when using loop devices as backing disks
	# dm_writecache: LINSTOR writecache layer
	# dm_cache: LINSTOR cache layer
	# dm_thin_pool: LINSTOR thinly provisioned storage
	# dm_snapshot: LINSTOR snapshotting
	# dm_crypt: LINSTOR encrypted volumes
for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt; do
	modprobe "$m" 2>/dev/null && s=success || s=failed
	echo "Loading ${m}: ${s}"
done

cd /tmp/pkg
tar xf /drbd-9.1.7.tar.gz
cd drbd-9.1.7
make
make install

modprobe drbd # usermode_helper=disabled
modprobe drbd_transport_tcp
modprobe drbd_transport_rdma 2>/dev/null || true

说明：

drbd的源码包中有个docker镜像使用的entry.sh脚本，该脚本可以实现在容器中编译drbd内核模块；

    docker run \
    -v /sys:/sys \
    -v /dev:/dev \
    -v /usr/src:/usr/src:ro \
    -v /lib/modules:/lib/modules \
    -e LB_HOW=compile \
    -e LB_INSTALL=yes \
    --privileged \
    --rm \
    piraeusdatastore/drbd9-bionic:v9.1.7

^f38541

当然我们可以使用该脚本直接在宿主机上进行编译：

wget -O /drbd.tar.gz https://github.com/LINBIT/drbd/archive/refs/tags/drbd-9.1.7.tar.gz
tar xf /drbd.tar.gz -C /
LB_HOW=compile LB_INSTALL=yes bash /drbd-9.1.7/docker/entry.sh  # 脚本中会解压/drbd.tar.gz

删除drbd模块

rmmod drbd_transport_tcp drbd
rm -rf /lib/modules/$(uname -r)/updates/drbd*

安装drbd-utils

编译安装drbd-utils

下载地址：https://github.com/LINBIT/drbd-utils/archive/refs/tags/v9.21.2.tar.gz

前提：

必须使用git clone方式下载，相关脚本会判断是否有.git目录
make过程中，会联网下载git子模块：https://github.com/LINBIT/drbd-headers.git，

root@drbd1:~# apt install -y flex xsltproc asciidoctor
root@drbd1:~# git clone https://github.com/LINBIT/drbd-utils -b 9.21.2
# git clone https://github.com/LINBIT/drbd-headers.git drbd-utils/drbd-headers
root@drbd1:~# cd drbd-utils/
root@drbd1:~/drbd-utils# bash autogen.sh  #生成configure命令
root@drbd1:~/drbd-utils# ./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --without-manual

root@drbd1:~/drbd-utils-9.21.2# make && make install

报错：

报错：configure: error: Cannot build utils without flex.
root@drbd1:~ # apt install flex

报错：make[1]: xsltproc: Command not found
root@drbd1:~ # apt install xsltproc

报错：make[1]: asciidoctor: Command not found
root@drbd1:~/drbd-utils# apt install -y asciidoctor

报错：
warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/current/manpages/tbl.xsl"
compilation error: file http://docbook.sourceforge.net/release/xsl/current/manpages/table.xsl line 24 element include
xsl:include : unable to load http://docbook.sourceforge.net/release/xsl/current/manpages/tbl.xsl
warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/current/manpages/pi.xsl"
compilation error: file http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl line 41 element include
xsl:include : unable to load http://docbook.sourceforge.net/release/xsl/current/manpages/pi.xsl
../../documentation/common/Makefile_v9_com_post:52: recipe for target 'drbdmeta.8' failed
make[1]: *** [drbdmeta.8] Error 5
make[1]: Leaving directory '/root/drbd-utils/documentation/v9'
Makefile:94: recipe for target 'doc' failed
make: *** [doc] Error 2
需要连接网络卸载doc，所以重新配置不编译手册--without-manual
root@drbd1:~/drbd-utils# ./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --without-manual

docker安装

构建镜像Dockerfile：

FROM registry-jinan-lab.inspurcloud.cn/library/os/inspur-alpine-3.12:5.1.0 as builder

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g'  /etc/apk/repositories  \
    && apk add -U --no-cache lvm2 lvm2-extra util-linux device-mapper git gcc g++ make automake autoconf flex libffi-dev openssl-dev

ADD drbd-utils-9.21.2.tgz /
#RUN git clone https://github.com/LINBIT/drbd-utils -b 9.21.2 && \
RUN cd /drbd-utils-9.21.2 \
    && sh autogen.sh \
    && ./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --without-manual \
    && make



FROM registry-jinan-lab.inspurcloud.cn/library/os/inspur-alpine-3.12:5.1.0
LABEL maintainer lijuzhang<lijuzhang@inspur.com>

COPY --from=builder /drbd-utils-9.21.2 /drbd-utils-9.21.2
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g'  /etc/apk/repositories  \
    && apk add -U --no-cache lvm2 lvm2-extra util-linux device-mapper lvm2-dmeventd make \
    && cd /drbd-utils-9.21.2 \
    && make install

运行：

echo '10.110.68.71 registry-jinan-lab.inspurcloud.cn'>>/etc/hosts

cat >/usr/local/bin/drbd-utils.sh << "EEOF"
#!/usr/bin/env bash
#
# utils for drbdadm and drbdsetup
#

#DrbdUtilsImage=\piraeusdatastore/drbd-utils:v9.21.2
DrbdUtilsImage=registry-jinan-lab.inspurcloud.cn/library/cke/piraeusdatastore/drbd-utils:v9.21.2
DEF_TIMEOUT=${DEF_TIMEOUT:-6}

arg0=$(basename $0)
if [ "$arg0" != "drbdadm" ] && [ "$arg0" != "drbdsetup" ]; then
    echo "ERROR: invalid file name \"$arg0\": this scripts must be named to \"drbdadm\" or \"drbdsetup\"" >&2
    exit 1
fi

# make sure /etc/drbd.conf available
if [ ! -f /etc/drbd.conf ]; then
    echo "Warn: drbd config file not exist (/etc/drbd.conf), applying default..."
    # edit drbd.conf
    cat >/etc/drbd.conf <<EOF
include "/etc/drbd.d/global_common.conf";
include "/etc/drbd.d/*.res";
EOF
fi

# make sure /etc/drbd.d/ config dir available
test -e /etc/drbd.d || mkdir /etc/drbd.d
if [ ! -f /etc/drbd.d/global_common.conf ]; then
    echo "Warn: drbd global common config not exists (/etc/drbd.d/global_common.conf), applying default..."
    cat >/etc/drbd.d/global_common.conf <<EOF
global {
    usage-count no;
    }
EOF
fi

# -v /var/lib/linstor.d:/var/lib/linstor.d:ro \

[ -t 1 ] && tty='-t' || tty=''

docker run --rm --privileged -i --net host \
    -v /sys:/sys \
    -v /dev:/dev \
    -v /usr/src:/usr/src:ro \
    -v /lib/modules:/lib/modules \
    -e LB_HOW=compile \
    -v /etc/drbd.conf:/etc/drbd.conf:ro \
    -v /etc/drbd.d:/etc/drbd.d:ro \
    --privileged \
    --rm \
    $tty \
    "${DrbdUtilsImage}" timeout ${DEF_TIMEOUT} $arg0 "$@"
EEOF

chmod +x /usr/local/bin/drbd-utils.sh

[ -e /usr/local/bin/drbdadm ] && rm -rf /usr/local/bin/drbdadm
ln -s /usr/local/bin/drbd-utils.sh /usr/local/bin/drbdadm
[ -e /usr/local/bin/drbdsetup ] && rm -rf /usr/local/bin/drbdsetup
ln -s /usr/local/bin/drbd-utils.sh /usr/local/bin/drbdsetup

root@worker12:~/drbd-utils# mv /usr/local/bin/drbdadm /usr/local/bin/drbdsetup /tmp;apt install -y flex xsltproc asciidoctor;./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --without-manual && make -j 4 && make install

二进制包安装drbd-utils

apt install drbd-utils

启动

主节点

root@drbd1:~# cat >/etc/drbd.d/cke_es0.res <<"EOF"
resource es0 {
  net {
      protocol C;
      allow-two-primaries no;
  }
  options {
      auto-promote no;
  }
  on drbd1 {  # on后面的名字必须为当前主机的hostname
    meta-disk internal;
    device /dev/drbd0;
    disk /dev/sdb;
    address 172.16.1.101:7789;
    node-id 1;
  }
  on drbd2 { # remote节点对名字无要求。但是一般设置为remote的hostname，这样，所有节点的配置文件就是完全一致的，可以直接拷贝。
    meta-disk internal;
    device /dev/drbd0;
    disk /dev/sdb;
    address 172.16.1.102:7789;
    node-id 2;
  }
}
EOF


# 创建元数据，并启动资源
root@drbd1:~# drbdadm create-md es0
root@drbd1:~# drbdadm up es0

# 查看角色
root@drbd1:~# drbdadm role es0
Secondary

# 查看连接状态
root@drbd1:~# drbdadm cstate es0
Connected

# 查看disk状态
root@drbd1:~# drbdadm dstate es0
UpToDate/UpToDate

备节点

root@drbd2:~# cat >/etc/drbd.d/cke_es0.res<<"EOF"
resource es0 {
  net {
      protocol C;
      allow-two-primaries no;
  }
  options {
      auto-promote no;
  }
  on drbd1 {  # on后面的名字必须为当前主机的hostname
    meta-disk internal;
    device /dev/drbd0;
    disk /dev/sdb;
    address 172.16.1.101:7789;
    node-id 1;
  }
  on drbd2 { # remote节点对名字无要求。但是一般设置为remote的hostname，这样，所有节点的配置文件就是完全一致的，可以直接拷贝。
    meta-disk internal;
    device /dev/drbd0;
    disk /dev/sdb;
    address 172.16.1.102:7789;
    node-id 2;
  }
}
EOF

# 创建元数据，并启动资源
root@drbd2:~# drbdadm create-md es0
root@drbd2:~# drbdadm up es0

# 查看角色
root@drbd2:~# drbdadm role es0
Secondary

主节点执行：

# 将主节点设置primary
root@drbd1:~# drbdadm primary es0 --force

查看状态

root@drbd1:~# drbdadm status
es0 role:Primary
  disk:UpToDate
  drbd2 role:Secondary
    replication:SyncSource peer-disk:Inconsistent done:25.76



root@drbd2:~# drbdadm status
es0 role:Secondary
  disk:Inconsistent
  drbd1 role:Primary
    replication:SyncTarget peer-disk:UpToDate done:8.03

一段时间后，再次查看状态：

root@drbd1:~# drbdadm status
es0 role:Primary
  disk:UpToDate
  drbd2 role:Secondary
    peer-disk:UpToDate


root@drbd2:~# drbdadm status
es0 role:Secondary
  disk:UpToDate
  drbd1 role:Primary
    peer-disk:UpToDate

测试

只有primary角色的节点才能对drbd设置进行挂载。文件系统只能挂载在主(Primary)节点上，因此在设置好主节点后才可以对DRBD设备进行格式化操作

验证drbd主从同步。

主节点

root@drbd1:~# mkdir /data
root@drbd1:~# mkfs.xfs /dev/drbd0   # 格式化drbd0块设备
root@drbd1:~# mount /dev/drbd0 /data

# 创建测试文件
root@drbd1:~# touch /data/file{1..3}
root@drbd1:~# dd if=/dev/zero of=/data/testfile_big bs=1M count=100

root@drbd1:~# ls -l /data/
total 102400
-rw-r--r-- 1 root root         0 Jul 14 16:09 file1
-rw-r--r-- 1 root root         0 Jul 14 16:09 file2
-rw-r--r-- 1 root root         0 Jul 14 16:09 file3
-rw-r--r-- 1 root root 104857600 Jul 14 16:10 testfile_big


# 卸载文件系统并切换为备节点
root@drbd1:~# umount /data
root@drbd1:~# drbdadm secondary es0
root@drbd1:~# drbdadm status
es0 role:Secondary
  disk:UpToDate
  drbd2 role:Secondary
    peer-disk:UpToDate

从节点

root@drbd2:~# mkdir /data
root@drbd2:~# drbdadm primary es0
root@drbd2:~# mount /dev/drbd0 /data

root@drbd2:~# ls -l /data/
total 102400
-rw-r--r-- 1 root root         0 Jul 14 16:09 file1
-rw-r--r-- 1 root root         0 Jul 14 16:09 file2
-rw-r--r-- 1 root root         0 Jul 14 16:09 file3
-rw-r--r-- 1 root root 104857600 Jul 14 16:10 testfile_big

补充

1、如果纳管的数据盘已经是格式化并存在数据的盘，drbd仍可以纳管，只不过需要另外一块盘作为元数据盘。

2、手动禁用、启用资源

[root@ha-node1 ~]# drbdadm down mysql    # 禁用资源
[root@ha-node1 ~]# drbdadm up mysql    # 启动资源

资源名称也可以使用all表示[停用|启用]所有资源

3、升级和降级资源

[root@ha-node1 ~]# drbdadm primary mysql # 升级为主节点
[root@ha-node1 ~]# drbdadm secondary mysql    # 降级为次节点

在单主模式下的DRBD，两个节点同时处于连接状态，任何一个节点都可以在特定的时间内变成主；但两个节点中只能一为主，如果已经有一个主，需先降级才可能升级；在双主模式下没有这个限制

脚本阅读

snapshot-resync-target-lvm.sh

#!/bin/bash
#
#  snapshot-resync-target-lvm.sh
#  This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
#
# The caller (drbdadm) sets for us:
# DRBD_RESOURCE, DRBD_VOLUME, DRBD_MINOR, DRBD_LL_DISK etc.  # 这些环境变量是该脚本需要的，调用的时候需要先声明
#
###########
#
# There will be no resync if this script terminates with an
# exit code != 0. So be carefull with the exit code!
#

export LC_ALL=C LANG=C

if [[ -z "$DRBD_RESOURCE" || -z "$DRBD_LL_DISK" ]]; then
	echo "DRBD_RESOURCE/DRBD_LL_DISK is not set. This script is supposed to"
	echo "get called by drbdadm as a handler script"
	exit 0
fi

PROG=$(basename $0)

redirect_to_logger()  # 实现将该脚本的日志输出到文件中。
{
	local lf=${1:-local5}
	case $lf in
	# do we want to exclude some?
	auth|authpriv|cron|daemon|ftp|kern|lpr|mail|news|syslog|user|uucp|local[0-7])
		: OK ;;
	*)
		echo >&2 "invalid logfacility: $lf"
		return
		;;
	esac

	exec > >( exec 1>&- 2>&- logger -t "$PROG[$$]" -p $lf.info ) 2>&1  # 将后面的执行日志输入到/var/log/message。类似于exec > /var/log/message 2>&1
}

if [[ $- != *x* ]]; then  # 配置了set -x，则$-会包含"x"字符。
	# you may override with --logfacility below
	redirect_to_logger local5
fi

echo "invoked for $DRBD_RESOURCE/$DRBD_VOLUME (drbd$DRBD_MINOR)"

TEMP=$(getopt -o p:a:l:nv --long percent:,additional:,logfacility:,disconnect-on-error,verbose -- "$@")

if [ $? != 0 ]; then
	echo "getopt failed"
	exit 0
fi

if BACKING_BDEV=$(drbdadm sh-ll-dev "$DRBD_RESOURCE/$DRBD_VOLUME"); then
	is_stacked=false
elif BACKING_BDEV=$(drbdadm sh-ll-dev "$(drbdadm -S sh-lr-of "$DRBD_RESOURCE")/$DRBD_VOLUME"); then
	is_stacked=true
else
	echo "Cannot determine lower level device of resource $DRBD_RESOURCE/$DRBD_VOLUME, sorry."
	exit 0
fi

set_vg_lv_size()  # 功能：声明VG_NAME、LV_NAME、LV_SIZE_K 这3个变量
{
	local X
	if ! X=$(lvs --noheadings --nosuffix --units s -o vg_name,lv_name,lv_size "$BACKING_BDEV") ; then
		# if lvs cannot tell me the info I need,
		# this is:
		echo "Cannot create snapshot of $BACKING_BDEV, apparently no LVM LV."
		return 1
	fi
	set -- $X
	VG_NAME=$1 LV_NAME=$2 LV_SIZE_K=$[$3 / 2]
	return 0
}
set_vg_lv_size || exit 0 # clean exit if not an lvm lv


SNAP_PERC=10  # 源LV的百分比大小，创建的快照卷默认为源LV大小的10%
SNAP_ADDITIONAL=10240
DISCONNECT_ON_ERROR=0
LVC_OPTIONS=""  # 定义lvcreate 命令使用的额外参数
BE_VERBOSE=0
SNAP_NAME=$LV_NAME-before-resync  #快照卷的名字
$is_stacked && SNAP_NAME=$SNAP_NAME-stacked
DEFAULTFILE="/etc/default/drbd-snapshot"

if [ -f $DEFAULTFILE ]; then
	. $DEFAULTFILE
fi

## command line parameters override default file

eval set -- "$TEMP"
while true; do
	case $1 in
	-p|--percent)
		SNAP_PERC="$2"
		shift
		;;
	-a|--additional)
		SNAP_ADDITIONAL="$2"
		shift
		;;
	-n|--disconnect-on-error)
		DISCONNECT_ON_ERROR=1
		;;
	-v|--verbose)
		BE_VERBOSE=1
		;;
	-l|--logfacility)
		redirect_to_logger $2
		shift
		;;
	--)
		break
		;;
	esac
	shift
done
shift # the --

LVC_OPTIONS="$@"  # -- 后面的参数列表，用于定义lvcreate的其他额外的参数。例如: -- -c 16k -p r

if [[ $0 == *unsnapshot* ]]; then   # 要求shell脚本名字必须含有"unsnapshot"关键字
	[ $BE_VERBOSE = 1 ] && set -x
	lvremove -f $VG_NAME/$SNAP_NAME
	exit 0
else
	(
		set -e
		[ $BE_VERBOSE = 1 ] && set -x
		case $DRBD_MINOR in
			*[!0-9]*|"")
			if $is_stacked; then
				DRBD_MINOR=$(drbdadm -S sh-minor "$DRBD_RESOURCE")
			else
				DRBD_MINOR=$(drbdadm sh-minor "$DRBD_RESOURCE")
			fi
			;;
		*)
			:;; # ok, already exported by drbdadm
		esac

		OUT_OF_SYNC=$(sed -ne "/^ *$DRBD_MINOR:/ "'{
				n;
				s/^.* oos:\([0-9]*\).*$/\1/;
				s/^$/0/; # default if not found
				p;
				q; }' < /proc/drbd) # unit KiB
		SNAP_SIZE=$((OUT_OF_SYNC + SNAP_ADDITIONAL + LV_SIZE_K * SNAP_PERC / 100))
		lvcreate -s -n $SNAP_NAME -L ${SNAP_SIZE}k $LVC_OPTIONS $VG_NAME/$LV_NAME
	)
	RV=$?
	[ $DISCONNECT_ON_ERROR = 0 ] && exit 0
	exit $RV
fi

说明：drbdadm调用该脚本时，会传递如下环境变量：

LC_ALL=C
DRBD_MY_AF=ipv4
LANG=C
DRBD_PEER_ADDRESS=172.16.2.13
DRBD_MINOR=0
DRBD_VOLUME=0
DRBD_CONF=/etc/drbd.conf
DRBD_PEER_AF=ipv4
DRBD_MY_ADDRESS=172.16.1.13
PWD=/
HOME=/
DRBD_NODE_ID_1=remote_172.16.2.13
DRBD_NODE_ID_2=worker13
DRBD_RESOURCE=mariadb
DRBD_PEER_NODE_ID=1
TERM=linux
SHLVL=1
DRBD_BACKING_DEV=/dev/vgdrbd/lvdrbd
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DRBD_LL_DISK=/dev/vgdrbd/lvdrbd
DRBD_CSTATE=Connected
DRBD_MY_NODE_ID=2
_=/usr/bin/env

但是只有DRBD_RESOURCE, DRBD_VOLUME, DRBD_MINOR, DRBD_LL_DISK这些变量会被脚本使用。

参考连接：

DRBD主从备份搭建

DRBD详细解说及配置过程记录