diff --git a/playbook/bcache/hosts.yaml b/playbook/bcache/hosts.yaml new file mode 100644 index 000000000..d19f1c019 --- /dev/null +++ b/playbook/bcache/hosts.yaml @@ -0,0 +1,42 @@ +global: + user: curve + ssh_port: 22 + private_key_file: /home/curve/.ssh/id_rsa + +hosts: + - host: server-host1 + hostname: 10.0.1.1 + labels: + - bcache + envs: + - SUDO_ALIAS=sudo + - BACKING_DEV="/dev/sdc /dev/sdd /dev/sde" + - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3" + - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3" + - CACHE_MODE=writeback + - PERF_TUNE=true + - CLEAN_DATA=true + - host: server-host2 + hostname: 10.0.1.2 + labels: + - bcache + envs: + - SUDO_ALIAS=sudo + - BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd" + - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3" + - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3" + - CACHE_MODE=writeback + - PERF_TUNE=true + - CLEAN_DATA=true + - host: server-host3 + hostname: 10.0.1.3 + labels: + - bcache + envs: + - SUDO_ALIAS=sudo + - BACKING_DEV="/dev/sdb /dev/sdc /dev/sdd" + - CACHE_DEV_0="/dev/nvme0n1p1 /dev/nvme0n1p2 /dev/nvme0n1p3" + - CACHE_DEV_1="/dev/nvme1n1p1 /dev/nvme1n1p2 /dev/nvme1n1p3" + - CACHE_MODE=writeback + - PERF_TUNE=true + - CLEAN_DATA=true diff --git a/playbook/bcache/scripts/clean.sh b/playbook/bcache/scripts/clean.sh new file mode 100644 index 000000000..6e44a3dce --- /dev/null +++ b/playbook/bcache/scripts/clean.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +g_ls="${SUDO_ALIAS} ls" +g_ps="${SUDO_ALIAS} ps" +g_cat="${SUDO_ALIAS} cat" +g_tee="${SUDO_ALIAS} tee" +g_umount="${SUDO_ALIAS} umount" +g_wipefs="${SUDO_ALIAS} wipefs" +g_bcache_super_show="${SUDO_ALIAS} bcache-super-show" +g_mdadm="${SUDO_ALIAS} mdadm" + + +set_value() +{ + local value=$1 + local path=$2 + echo ${value} | ${g_tee} ${path} &> /dev/null +} + +pre_check() +{ + #check chunkserver is running + pid=$(${g_ps} -ef | grep chunkserver | grep -v grep | awk '{print $2}') + if [ -n "${pid}" ]; then + echo "chunkserver is running, please stop it first" + exit 1 + fi + + #check bcache dirty data + for bcache in $(${g_ls} /sys/block | grep bcache) + do + if [ "$(${g_cat} /sys/block/${bcache}/bcache/dirty_data)" != "0.0k" ]; then + echo "${bcache} has dirty data, please stop chunkserver and wait it cleaned" + exit 1 + fi + done + + echo "pre_check success" +} + + +stop_bcache() +{ + ${g_umount} /data/chunkserver* &> /dev/null + ${g_umount} /data/wal/chunkserver* &> /dev/null + + bcache_devs=$(${g_ls} /sys/block | grep bcache) + for bcache in ${bcache_devs} + do + backdev=/dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name) + uuid=$(${g_bcache_super_show} ${backdev} |grep cset |awk '{print $NF}') + + set_value 1 /sys/block/${bcache}/bcache/detach + set_value 1 /sys/fs/bcache/${uuid}/unregister + set_value 1 /sys/block/${bcache}/bcache/stop + done + + set_value 1 /sys/fs/bcache/pendings_cleanup + + sleep 1 + + bcache_devs=$(${g_ls} /sys/block | grep bcache) + cache_sets=$(${g_ls} /sys/fs/bcache | grep "-") + if [ -n "${bcache_devs}" ] || [ -n "${cache_sets}" ]; then + # need retry to wait bcache stop + echo "stop bcache failed" + exit 1 + fi + echo "stop bcache success" +} + +stop_mdraid1() +{ + for cache in $(${g_ls} /dev | grep md) + do + ${g_mdadm} --stop ${cache} + if [ $? != 0 ]; then + echo "stop mdraid1 device md${i} failed" + exit 1 + fi + done +} + +clean_bcache_data() +{ + if [ x"${CLEAN_DATA}" != x"true" ]; then + echo "no need to clean data" + exit 0 + fi + + for hdd in ${BACKING_DEV} + do + ${g_wipefs} -a --force ${hdd} &> /dev/null + if [ $? != 0 ]; then + echo "wipefs backing device ${hdd} failed" + exit 1 + fi + done + + for cache in ${CACHE_DEV_0} + do + ${g_wipefs} -a --force ${cache} &> /dev/null + if [ $? != 0 ]; then + echo "wipefs cache device ${cache} failed" + exit 1 + fi + done + + for cache in ${CACHE_DEV_1} + do + ${g_wipefs} -a --force ${cache} &> /dev/null + if [ $?!= 0 ]; then + echo "wipefs cache device ${cache} failed" + exit 1 + fi + done + + echo "clean backing and cache devices data success" +} + + +pre_check +stop_bcache +clean_bcache_data + diff --git a/playbook/bcache/scripts/deploy.sh b/playbook/bcache/scripts/deploy.sh new file mode 100644 index 000000000..d0ca34190 --- /dev/null +++ b/playbook/bcache/scripts/deploy.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +g_ls="${SUDO_ALIAS} ls" +g_lsmod="${SUDO_ALIAS} lsmod" +g_modinfo="${SUDO_ALIAS} modinfo" +g_which="${SUDO_ALIAS} which" +g_tee="${SUDO_ALIAS} tee" +g_make_bcache="${SUDO_ALIAS} make-bcache" +g_bcache_super_show="${SUDO_ALIAS} bcache-super-show" +g_mdadm="${SUDO_ALIAS} mdadm" +g_yes="${SUDO_ALIAS} yes" + +defalut_cache_mode=none + + +set_value() +{ + local value=$1 + local path=$2 + echo ${value} | ${g_tee} ${path} &> /dev/null +} + +pre_check() +{ + # check bcache-tools is installed + if [ -z "$(${g_which} make-bcache)" ]; then + echo "make-bcache could not be found" + exit 1 + fi + + if [ -z "$(${g_which} bcache-super-show)" ]; then + echo "bcache-super-show could not be found" + exit 1 + fi + + # check mdadm is installed + if [ -z "$(${g_which} mdadm)" ]; then + echo "mdadm could not be found" + exit 1 + fi + + # check bcache module is exist + ${g_modinfo} bcache &> /dev/null + if [ $? != 0 ]; then + echo "bcache module not be found" + exit 1 + fi + + # check bcache device is exist + if [ -n "$(${g_ls} /sys/block | grep bcache)" ];then + echo "bcache device is exist, clean it first" + exit 1 + fi + + # check mdraid device is exist + if [ -n "$(${g_ls} /dev | grep md)" ];then + echo "mdraid device is exist, clean it first" + exit 1 + fi + + # check backend and cache device number + if [ $(echo ${BACKING_DEV} |wc -l) != $(echo ${CACHE_DEV_0} |wc -l) ];then + echo "only support one cache device with one backing device now!" + exit 1 + fi + if [ $(echo ${BACKING_DEV} |wc -l) != $(echo ${CACHE_DEV_1} |wc -l) ];then + echo "only support one cache device with one backing device now!" + exit 1 + fi + + echo "pre_check success" +} + +deploy_mdraid1() +{ + CACHE_ARRAY_0=($CACHE_DEV_0) + CACHE_ARRAY_1=($CACHE_DEV_1) + + for ((i=0; i<${#CACHE_ARRAY_0[@]}; i++)); do + cache_0=${CACHE_ARRAY_0[i]} + cache_1=${CACHE_ARRAY_1[i]} + ${g_yes} | ${g_mdadm} --create --verbose /dev/md${i} --level=1 --raid-devices=2 ${cache_0} ${cache_1} + if [ $? != 0 ]; then + echo "make mdraid1 device md${i} with ${cache_0} and ${cache_1} failed" + exit 1 + fi + done +} + +deploy_bcache() +{ + for hdd in ${BACKING_DEV} + do + ${g_make_bcache} -B --wipe-bcache ${hdd} &> /dev/null + if [ $? = 0 ]; then + set_value ${hdd} /sys/fs/bcache/register + else + echo "make bcache device ${hdd} failed" + exit 1 + fi + done + + for cache in $(${g_ls} /dev | grep md) + do + ${g_make_bcache} -C --wipe-bcache -b 262144 ${cache} &> /dev/null + if [ $? = 0 ]; then + set_value ${cache} /sys/fs/bcache/register + else + echo "make bcache device ${cache} failed" + exit 1 + fi + done + + idx=0 + for cache in $(${g_ls} /dev | grep md) + do + uuid=$(${g_bcache_super_show} ${cache} | grep cset.uuid | awk '{print $2}') + set_value ${uuid} /sys/block/bcache${idx}/bcache/attach + idx=$((idx+1)) + done + + echo "now set cache mode to ${defalut_cache_mode}" + # using none mode before chunkfilepool formated + for bcache in $(${g_ls} /sys/block | grep bcache) + do + set_value ${defalut_cache_mode} /sys/block/${bcache}/bcache/cache_mode + done + + echo "bcache deploy success, please format chunkfilepool and walfilepool manually" +} + +pre_check +deploy_mdraid1 +deploy_bcache + diff --git a/playbook/bcache/scripts/perf.sh b/playbook/bcache/scripts/perf.sh new file mode 100644 index 000000000..2c19ecaab --- /dev/null +++ b/playbook/bcache/scripts/perf.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +g_ls="${SUDO_ALIAS} ls" +g_ps="${SUDO_ALIAS} ps" +g_cat="${SUDO_ALIAS} cat" +g_tee="${SUDO_ALIAS} tee" + +if [ ${PERF_TUNE} != "true" ]; then + echo 'PERF_TUNE is not true, exit' + exit +fi + +set_value() +{ + local value=$1 + local path=$2 + echo ${value} | ${g_tee} ${path} &> /dev/null +} + +for bcache in $(${g_ls} /sys/block | grep bcache) +do + backing_dev=$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name) + backing_sectors=$(${g_cat} /sys/block/${backing_dev}/queue/max_sectors_kb) + backing_ahead=$(${g_cat} /sys/block/${backing_dev}/queue/read_ahead_kb) + + set_value ${backing_sectors} /sys/block/${bcache}/queue/max_sectors_kb + set_value ${backing_ahead} /sys/block/${bcache}/queue/read_ahead_kb + set_value ${CACHE_MODE} /sys/block/${bcache}/bcache/cache_mode + set_value 1 /sys/block/${bcache}/bcache/clear_stats + set_value 0 /sys/block/${bcache}/bcache/readahead + set_value 40 /sys/block/${bcache}/bcache/writeback_percent + set_value 10 /sys/block/${bcache}/bcache/writeback_delay + set_value 1 /sys/block/${bcache}/bcache/writeback_rate_minimum + set_value 0 /sys/block/${bcache}/bcache/cache/congested_read_threshold_us + set_value 0 /sys/block/${bcache}/bcache/cache/congested_write_threshold_us + set_value 0 /sys/block/${bcache}/bcache/sequential_cutoff + set_value lru /sys/block/${bcache}/bcache/cache/cache0/cache_replacement_policy + set_value 1 /sys/block/${bcache}/bcache/cache/internal/gc_after_writeback + +done + +echo "bcache perf tune success, cache mode is ${CACHE_MODE}" + diff --git a/playbook/bcache/scripts/show.sh b/playbook/bcache/scripts/show.sh new file mode 100644 index 000000000..b008f379d --- /dev/null +++ b/playbook/bcache/scripts/show.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +g_ls="${SUDO_ALIAS} ls" +g_cat="${SUDO_ALIAS} cat" +g_which="${SUDO_ALIAS} which" +g_readlink="${SUDO_ALIAS} readlink" + +show_bcache() +{ + if [ -n "$(${g_which} bcache-status)" ]; then + ${SUDO_ALIAS} bcache-status -s + elif [ -n "$(${g_which} bcache)" ]; then + ${SUDO_ALIAS} bcache show + else + for bcache in $(${g_ls} /sys/block | grep bcache) + do + echo "${bcache} info:" + echo "----------------------------" + echo "backing device: /dev/$(${g_cat} /sys/block/${bcache}/bcache/backing_dev_name)" + echo "cache device: /dev/$(${g_readlink} /sys/block/${bcache}/bcache/cache/cache0 |awk -F'/' '{print $(NF-1)}')" + echo "cache mode: $(${g_cat} /sys/block/${bcache}/bcache/cache_mode | grep -oP "(?<=\[)[^\]]*(?=\])")" + echo "cache state: $(${g_cat} /sys/block/${bcache}/bcache/state)" + echo + done + fi +} + +show_bcache + diff --git a/playbook/bcache/scripts/show_sync.sh b/playbook/bcache/scripts/show_sync.sh new file mode 100644 index 000000000..f2859d06d --- /dev/null +++ b/playbook/bcache/scripts/show_sync.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +g_ls="${SUDO_ALIAS} ls" +g_mdadm="${SUDO_ALIAS} mdadm" + +show_sync() +{ + for cache in $(${g_ls} /dev | grep md) + do + echo "${cache} info:" + echo "----------------------------" + ${g_mdadm} --detail ${cache} + echo + done +} + +show_sync \ No newline at end of file