#!/bin/bash -e # set -E set -o pipefail if [ ! -x "$PWD/minio" ]; then echo "minio executable binary not found in current directory" exit 1 fi WORK_DIR="$PWD/.verify-$RANDOM" MINIO_CONFIG_DIR="$WORK_DIR/.minio" MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) GOPATH=/tmp/gopath function start_minio_3_node() { export MINIO_ROOT_USER=minio export MINIO_ROOT_PASSWORD=minio123 export MINIO_ERASURE_SET_DRIVE_COUNT=6 export MINIO_CI_CD=1 first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l) start_port=$2 args="" for d in $(seq 1 3 5); do args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ " d=$((d + 1)) args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ " done "${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 & pid1=$! disown ${pid1} "${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 & pid2=$! disown $pid2 "${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 & pid3=$! disown $pid3 sleep "$1" [ ${first_time} -eq 0 ] && upload_objects $start_port if ! ps -p $pid1 1>&2 >/dev/null; then echo "server1 log:" cat "${WORK_DIR}/dist-minio-server1.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! ps -p $pid2 1>&2 >/dev/null; then echo "server2 log:" cat "${WORK_DIR}/dist-minio-server2.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! ps -p $pid3 1>&2 >/dev/null; then echo "server3 log:" cat "${WORK_DIR}/dist-minio-server3.log" echo "FAILED" purge "$WORK_DIR" exit 1 fi if ! pkill minio; then for i in $(seq 1 3); do echo "server$i log:" cat "${WORK_DIR}/dist-minio-server$i.log" done echo "FAILED" purge "$WORK_DIR" exit 1 fi sleep 1 if pgrep minio; then # forcibly killing, to proceed further properly. if ! pkill -9 minio; then echo "no minio process running anymore, proceed." fi fi } function check_heal() { if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then return 1 fi for ((i = 0; i < 20; i++)); do test -f ${WORK_DIR}/$1/1/.minio.sys/format.json v1=$? nextInES=$(($1 + 1)) && [ $nextInES -gt 3 ] && nextInES=1 foundFiles1=$(find ${WORK_DIR}/$1/1/ | grep -v .minio.sys | grep xl.meta | wc -l) foundFiles2=$(find ${WORK_DIR}/$nextInES/1/ | grep -v .minio.sys | grep xl.meta | wc -l) test $foundFiles1 -eq $foundFiles2 v2=$? [ $v1 == 0 -a $v2 == 0 ] && return 0 sleep 10 done return 1 } function purge() { rm -rf "$1" } function __init__() { echo "Initializing environment" mkdir -p "$WORK_DIR" mkdir -p "$MINIO_CONFIG_DIR" ## version is purposefully set to '3' for minio to migrate configuration file echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" if [ ! -f /tmp/mc ]; then wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x /tmp/mc fi } function upload_objects() { start_port=$1 /tmp/mc alias set myminio http://127.0.0.1:$((start_port + 1)) minio minio123 --api=s3v4 /tmp/mc ready myminio /tmp/mc mb myminio/testbucket/ for ((i = 0; i < 20; i++)); do echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i done } function perform_test() { start_port=$2 start_minio_3_node 120 $start_port echo "Testing Distributed Erasure setup healing of drives" echo "Remove the contents of the disks belonging to '${1}' node" rm -rf ${WORK_DIR}/${1}/*/ set -x start_minio_3_node 120 $start_port check_heal ${1} rv=$? if [ "$rv" == "1" ]; then for i in $(seq 1 3); do echo "server$i log:" cat "${WORK_DIR}/dist-minio-server$i.log" done pkill -9 minio echo "FAILED" purge "$WORK_DIR" exit 1 fi } function main() { # use same ports for all tests start_port=$(shuf -i 10000-65000 -n 1) perform_test "2" ${start_port} perform_test "1" ${start_port} perform_test "3" ${start_port} } (__init__ "$@" && main "$@") rv=$? purge "$WORK_DIR" exit "$rv"