diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 0d6c54b51c..72f28b6692 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -9,6 +9,15 @@ on: pull_request: jobs: + wait-storage-shell-test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run wait-storage.sh peer failover tests + run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh + build-server: # TODO: we need test & replace it to ubuntu-24.04 or ubuntu-latest runs-on: ubuntu-22.04 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh index d4e9e278f0..f64f382d72 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh @@ -97,9 +97,10 @@ if env | grep '^hugegraph\.' > /dev/null; then log() { echo '[wait-storage] '\"\$1\"; } - check_any_pd() { + check_any_pd_stores() { for peer in \$(echo \"\$PD_REST_LIST\" | tr ',' ' '); do - if curl ${PD_AUTH_ARGS} -f -s http://\${peer}/v1/health >/dev/null 2>&1; then + if curl ${PD_AUTH_ARGS} -f -s http://\${peer}/v1/stores 2>/dev/null | \ + grep -qi '\"state\"[[:space:]]*:[[:space:]]*\"Up\"'; then echo \"\$peer\" return 0 fi @@ -107,20 +108,11 @@ if env | grep '^hugegraph\.' > /dev/null; then return 1 } - until PD_REST=\$(check_any_pd); do - log 'No PD peer ready yet, retrying in 5s' - sleep 5 - done - log \"PD health check PASSED via \$PD_REST\" - - until curl ${PD_AUTH_ARGS} -f -s \ - http://\${PD_REST}/v1/stores 2>/dev/null | \ - grep -qi '\"state\"[[:space:]]*:[[:space:]]*\"Up\"'; do + until PD_REST=\$(check_any_pd_stores); do log 'No Up store yet, retrying in 5s' sleep 5 done - - log 'Store registration check PASSED' + log \"Store registration check PASSED via \$PD_REST\" log 'Storage backend is VIABLE' " || { echo "[wait-storage] ERROR: Timeout waiting for storage backend"; exit 1; } diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh new file mode 100755 index 0000000000..88b4f85814 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -uo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SOURCE_BIN="${1:-$(cd "${SCRIPT_DIR}/../static/bin" && pwd)}" +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/wait-storage-test.XXXXXX") +DIST_ROOT="${TMP_DIR}/dist" +MOCK_BIN="${TMP_DIR}/mock-bin" +CALL_LOG="${TMP_DIR}/curl-calls" +ARGS_LOG="${TMP_DIR}/curl-args" +COUNT_FILE="${TMP_DIR}/store-call-count" +TIMEOUT_LOG="${TMP_DIR}/timeout-arg" +CASE_OUTPUT="" +CASE_RC=0 + +trap 'rm -rf "${TMP_DIR}"' EXIT + +fail() { + echo "FAIL: $1" >&2 + [[ -z "${CASE_OUTPUT}" ]] || printf '%s\n' "${CASE_OUTPUT}" >&2 + exit 1 +} + +assert_equal() { + local name="$1" expected="$2" actual="$3" + [[ "${actual}" == "${expected}" ]] || \ + fail "${name}: expected '${expected}', got '${actual}'" +} + +assert_output() { + local expected="$1" + [[ "${CASE_OUTPUT}" == *"${expected}"* ]] || \ + fail "missing output '${expected}'" +} + +assert_contract() { + ! grep -q '/v1/health' "${CALL_LOG}" || \ + fail "/v1/health must not gate readiness" + [[ -s "${ARGS_LOG}" ]] || fail "curl was not called" + if grep -Fv -- '-u test-user:test-password' "${ARGS_LOG}" | grep -q .; then + fail "authentication arguments were not preserved" + fi + assert_equal "outer timeout" "300s" "$(cat "${TIMEOUT_LOG}")" +} + +run_case() { + local scenario="$1" peers="$2" abort_after="$3" + : > "${CALL_LOG}" + : > "${ARGS_LOG}" + : > "${COUNT_FILE}" + : > "${TIMEOUT_LOG}" + : > "${DIST_ROOT}/conf/graphs/hugegraph.properties" + + CASE_OUTPUT=$(env \ + PATH="${MOCK_BIN}:${PATH}" \ + MOCK_SCENARIO="${scenario}" \ + MOCK_ABORT_AFTER="${abort_after}" \ + MOCK_CALL_LOG="${CALL_LOG}" \ + MOCK_ARGS_LOG="${ARGS_LOG}" \ + MOCK_COUNT_FILE="${COUNT_FILE}" \ + MOCK_TIMEOUT_LOG="${TIMEOUT_LOG}" \ + HG_SERVER_PD_REST_ENDPOINT="${peers}" \ + PD_AUTH_USER="test-user" \ + PD_AUTH_PASSWORD="test-password" \ + 'hugegraph.backend=hstore' \ + 'hugegraph.pd.peers=config-only:8686' \ + "${DIST_ROOT}/bin/wait-storage.sh" 2>&1) + CASE_RC=$? +} + +if [[ ! -f "${SOURCE_BIN}/wait-storage.sh" || ! -f "${SOURCE_BIN}/util.sh" ]]; then + fail "wait-storage.sh or util.sh not found under ${SOURCE_BIN}" +fi + +mkdir -p "${DIST_ROOT}/bin" "${DIST_ROOT}/conf/graphs" "${MOCK_BIN}" +cp "${SOURCE_BIN}/wait-storage.sh" "${SOURCE_BIN}/util.sh" "${DIST_ROOT}/bin/" +: > "${DIST_ROOT}/conf/graphs/hugegraph.properties" + +cat > "${MOCK_BIN}/timeout" <<'EOF' +#!/bin/bash +printf '%s\n' "$1" > "${MOCK_TIMEOUT_LOG}" +shift +"$@" & +command_pid=$! +ticks=0 +while kill -0 "${command_pid}" 2>/dev/null; do + count=$(cat "${MOCK_COUNT_FILE}" 2>/dev/null || true) + count=${count:-0} + ticks=$((ticks + 1)) + if [[ "${count}" -ge "${MOCK_ABORT_AFTER}" || "${ticks}" -ge 500 ]]; then + kill -TERM "${command_pid}" 2>/dev/null || true + wait "${command_pid}" 2>/dev/null || true + exit 124 + fi + /bin/sleep 0.01 +done +wait "${command_pid}" +EOF + +cat > "${MOCK_BIN}/sleep" <<'EOF' +#!/bin/bash +/bin/sleep 0.02 +EOF + +cat > "${MOCK_BIN}/curl" <<'EOF' +#!/bin/bash +set -u +url="${!#}" +printf '%s\n' "$*" >> "${MOCK_ARGS_LOG}" +printf '%s\n' "${url}" >> "${MOCK_CALL_LOG}" + +if [[ "${url}" == */v1/health ]]; then + printf '{}\n' + exit 0 +fi + +count=$(cat "${MOCK_COUNT_FILE}" 2>/dev/null || true) +count=$((${count:-0} + 1)) +printf '%s\n' "${count}" > "${MOCK_COUNT_FILE}" + +if [[ "${MOCK_SCENARIO}" == "pd1-up" && \ + "${url}" == "http://pd1:8620/v1/stores" ]]; then + printf '{"stores":[{"state":"Up"}]}\n' +elif [[ "${MOCK_SCENARIO}" == "retry" && "${count}" -eq 3 && \ + "${url}" == "http://pd0:8620/v1/stores" ]]; then + exit 7 +elif [[ "${MOCK_SCENARIO}" == "retry" && "${count}" -eq 4 && \ + "${url}" == "http://pd1:8620/v1/stores" ]]; then + printf '{"stores":[{"state":"Up"}]}\n' +else + printf '{"stores":[]}\n' +fi +EOF + +chmod +x "${MOCK_BIN}/timeout" "${MOCK_BIN}/sleep" "${MOCK_BIN}/curl" + +PD0='http://pd0:8620/v1/stores' +PD1='http://pd1:8620/v1/stores' +TWO_CALLS="${PD0}"$'\n'"${PD1}" +FOUR_CALLS="${TWO_CALLS}"$'\n'"${TWO_CALLS}" + +echo "wait-storage.sh peer failover tests" + +run_case "pd1-up" "pd0:8620,pd1:8620" 6 +assert_equal "storeless first peer rc" "0" "${CASE_RC}" +assert_equal "configured peer order" "${TWO_CALLS}" "$(cat "${CALL_LOG}")" +assert_output "Store registration check PASSED via pd1:8620" +assert_contract +echo " PASS storeless first peer" + +run_case "pd1-up" "pd1:8620,pd0:8620" 6 +assert_equal "reversed peer order rc" "0" "${CASE_RC}" +assert_equal "stop after first Up peer" "${PD1}" "$(cat "${CALL_LOG}")" +assert_contract +echo " PASS reversed peer order" + +run_case "retry" "pd0:8620,pd1:8620" 8 +assert_equal "retry rc" "0" "${CASE_RC}" +assert_equal "complete peer rescan" "${FOUR_CALLS}" "$(cat "${CALL_LOG}")" +assert_output "Storage backend is VIABLE" +assert_contract +echo " PASS unavailable peer retry" + +run_case "none" "pd0:8620,pd1:8620" 4 +[[ "${CASE_RC}" -ne 0 ]] || fail "all-unready peers must fail closed" +assert_equal "all-unready rescan" "${FOUR_CALLS}" "$(cat "${CALL_LOG}")" +assert_output "ERROR: Timeout waiting for storage backend" +assert_contract +echo " PASS all-unready timeout" + +echo "4 passed, 0 failed"