S26-Distributed-Capstone · MaxFdev · May 20, 2026 · May 20, 2026
diff --git a/scripts/dsvclient-k8s-helpers.sh b/scripts/dsvclient-k8s-helpers.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Shared helpers for DSVClient-driven Kubernetes integration/load tests.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+REPOS_ROOT="$(cd "${ROOT}/.." && pwd)"
+
+NAMESPACE="${NAMESPACE:-dsv}"
+STATEFULSET="${STATEFULSET:-dsv-app}"
+SERVICE="${SERVICE:-dsv-app-service}"
+SERVICE_PORT="${SERVICE_PORT:-9080}"
+LOCAL_PORT="${LOCAL_PORT:-19080}"
+BASE_URL="${BASE_URL:-http://127.0.0.1:${LOCAL_PORT}}"
+DSV_CLIENT_DIR="${DSV_CLIENT_DIR:-${REPOS_ROOT}/DSVClient}"
+CLIENT_CLI="${CLIENT_CLI:-${DSV_CLIENT_DIR}/cli.py}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+RUN_ID="${RUN_ID:-$(date +%Y%m%d%H%M%S)-${RANDOM}}"
+WORK_DIR="${WORK_DIR:-${ROOT}/target/dsvclient-k8s-${RUN_ID}}"
+PORT_FORWARD_PID=""
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+TOTAL_COUNT=0
+PASS_COUNT=0
+FAIL_COUNT=0
+
+info() {
+    echo -e "${CYAN}==>${NC} $*"
+}
+
+section() {
+    echo ""
+    echo -e "${YELLOW}--- $* ---${NC}"
+}
+
+pass() {
+    TOTAL_COUNT=$((TOTAL_COUNT + 1))
+    PASS_COUNT=$((PASS_COUNT + 1))
+    echo -e "  ${GREEN}[PASS]${NC} [${TOTAL_COUNT}] $*"
+}
+
+fail() {
+    TOTAL_COUNT=$((TOTAL_COUNT + 1))
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+    echo -e "  ${RED}[FAIL]${NC} [${TOTAL_COUNT}] $*"
+}
+
+die() {
+    echo -e "${RED}ERROR:${NC} $*" >&2
+    exit 1
+}
+
+require_command() {
+    command -v "$1" >/dev/null 2>&1 || die "Required command not found: $1"
+}
+
+setup_suite() {
+    mkdir -p "$WORK_DIR"
+    require_command kubectl
+    require_command curl
+    require_command "$PYTHON_BIN"
+    [[ -f "$CLIENT_CLI" ]] || die "DSVClient CLI not found at ${CLIENT_CLI}. Set DSV_CLIENT_DIR or CLIENT_CLI."
+
+    info "Using namespace=${NAMESPACE}, service=${SERVICE}, base_url=${BASE_URL}"
+    kubectl get namespace "$NAMESPACE" >/dev/null
+    kubectl -n "$NAMESPACE" rollout status "statefulset/${STATEFULSET}" --timeout=240s
+    ensure_gateway
+    wait_for_gateway
+}
+
+ensure_gateway() {
+    if [[ -n "${NO_PORT_FORWARD:-}" || -n "${EXTERNAL_BASE_URL:-}" ]]; then
+        BASE_URL="${EXTERNAL_BASE_URL:-${BASE_URL}}"
+        return
+    fi
+
+    info "Starting kubectl port-forward svc/${SERVICE} ${LOCAL_PORT}:${SERVICE_PORT}"
+    kubectl -n "$NAMESPACE" port-forward "svc/${SERVICE}" "${LOCAL_PORT}:${SERVICE_PORT}" \
+        >"${WORK_DIR}/port-forward.log" 2>&1 &
+    PORT_FORWARD_PID="$!"
+    trap cleanup_suite EXIT
+}
+
+cleanup_suite() {
+    local status=$?
+    if [[ -n "$PORT_FORWARD_PID" ]]; then
+        kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
+        wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
+    fi
+    return "$status"
+}
+
+wait_for_gateway() {
+    local timeout="${GATEWAY_TIMEOUT_SECONDS:-180}"
+    local elapsed=0
+    while ! curl -sf --connect-timeout 2 --max-time 10 "${BASE_URL}/actuator/health" >/dev/null 2>&1; do
+        if (( elapsed >= timeout )); then
+            [[ -f "${WORK_DIR}/port-forward.log" ]] && tail -40 "${WORK_DIR}/port-forward.log" || true
+            die "Gateway ${BASE_URL} did not become healthy within ${timeout}s"
+        fi
+        sleep 3
+        elapsed=$((elapsed + 3))
+    done
+}
+
+wait_for_rollout() {
+    kubectl -n "$NAMESPACE" rollout status "statefulset/${STATEFULSET}" --timeout="${ROLLOUT_TIMEOUT:-300s}"
+    wait_for_gateway
+}
+
+client_home() {
+    local username="$1"
+    local home_dir="${WORK_DIR}/homes/${username}-$$-${RANDOM}"
+    mkdir -p "${home_dir}/.dsv_client"
+    cat > "${home_dir}/.dsv_client/config.json" <<JSON
+{
+  "base_url": "${BASE_URL}",
+  "username": "${username}"
+}
+JSON
+    printf "%s" "$home_dir"
+}
+
+dsvc() {
+    local username="$1"
+    shift
+    local home_dir
+    home_dir="$(client_home "$username")"
+    HOME="$home_dir" "$PYTHON_BIN" "$CLIENT_CLI" "$@"
+}
+
+unique_name() {
+    local prefix="${1:-secret}"
+    printf "%s-%s-%s" "$prefix" "$RUN_ID" "$RANDOM"
+}
+
+expect_output_contains() {
+    local output="$1"
+    local expected="$2"
+    local label="$3"
+    if printf "%s" "$output" | grep -Fq -- "$expected"; then
+        pass "$label"
+    else
+        fail "${label} (expected output to contain '${expected}')"
+        printf "    Output: %s\n" "$(printf "%s" "$output" | tr '\n' ' ' | cut -c 1-240)"
+    fi
+}
+
+write_status() {
+    local file="$1"
+    local status="$2"
+    printf "%s" "$status" > "$file"
+}
+
+count_status() {
+    local dir="$1"
+    local status="$2"
+    local count=0
+    local file
+    for file in "$dir"/status-*; do
+        [[ -f "$file" ]] || continue
+        [[ "$(cat "$file")" == "$status" ]] && count=$((count + 1))
+    done
+    printf "%s" "$count"
+}
+
+print_summary() {
+    echo ""
+    echo -e "${CYAN}Summary${NC}"
+    echo "  total:  ${TOTAL_COUNT}"
+    echo -e "  passed: ${GREEN}${PASS_COUNT}${NC}"
+    if (( FAIL_COUNT > 0 )); then
+        echo -e "  failed: ${RED}${FAIL_COUNT}${NC}"
+        return 1
+    fi
+    echo "  failed: 0"
+    return 0
+}
diff --git a/scripts/test-dsvclient-failure.sh b/scripts/test-dsvclient-failure.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# DSVClient Kubernetes failure test. Deletes StatefulSet pods while client traffic is active.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "${SCRIPT_DIR}/dsvclient-k8s-helpers.sh"
+
+LOAD_REQUESTS="${LOAD_REQUESTS:-120}"
+PARALLELISM="${PARALLELISM:-30}"
+FAIL_POD_INDEX="${FAIL_POD_INDEX:-2}"
+
+setup_suite
+RESULTS_DIR="${WORK_DIR}/failure"
+mkdir -p "$RESULTS_DIR"
+
+USER="failure-client-${RUN_ID}"
+BASE_SECRET="failure-baseline-${RUN_ID}"
+
+section "Seed baseline secret"
+out="$(dsvc "$USER" create "$BASE_SECRET" "baseline-value" 2>&1)" || true
+expect_output_contains "$out" "Secret created" "Created baseline secret"
+
+section "Run client traffic while one pod is deleted"
+(
+    sleep 2
+    info "Deleting pod ${STATEFULSET}-${FAIL_POD_INDEX} in namespace ${NAMESPACE}"
+    kubectl -n "$NAMESPACE" delete pod "${STATEFULSET}-${FAIL_POD_INDEX}" --wait=false
+) &
+
+for i in $(seq 1 "$LOAD_REQUESTS"); do
+    (
+        user="failure-load-$((i % 10))-${RUN_ID}"
+        name="failure-load-${RUN_ID}-${i}"
+        case $((i % 4)) in
+            0) out="$(dsvc "$USER" get "$BASE_SECRET" 2>&1)" ;;
+            1) out="$(dsvc "$USER" update "$BASE_SECRET" "baseline-update-${i}" 2>&1)" ;;
+            2) out="$(dsvc "$user" create "$name" "value-${i}" 2>&1)" ;;
+            3) out="$(dsvc "$USER" get "$BASE_SECRET" --all 2>&1)" ;;
+        esac
+        printf "%s\n" "$out" > "${RESULTS_DIR}/op-${i}.log"
+        if printf "%s" "$out" | grep -Eq "Secret created|Secret updated|baseline"; then
+            write_status "${RESULTS_DIR}/status-${i}" "PASS"
+        else
+            write_status "${RESULTS_DIR}/status-${i}" "CHECK"
+        fi
+    ) &
+    if (( i % PARALLELISM == 0 )); then
+        wait
+    fi
+done
+wait
+
+section "Wait for StatefulSet recovery"
+wait_for_rollout
+
+section "Verify baseline after recovery"
+out="$(dsvc "$USER" get "$BASE_SECRET" 2>&1)" || true
+expect_output_contains "$out" "baseline" "Baseline remains readable after pod recovery"
+
+passed="$(count_status "$RESULTS_DIR" PASS)"
+check="$(count_status "$RESULTS_DIR" CHECK)"
+if (( passed > 0 )); then
+    pass "Traffic during pod failure produced ${passed} successful client operations; review=${check}"
+else
+    fail "No successful operations during failure. Logs: ${RESULTS_DIR}"
+fi
+
+print_summary
+
diff --git a/scripts/test-dsvclient-gauntlet.sh b/scripts/test-dsvclient-gauntlet.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Full DSVClient Kubernetes gauntlet: simple load, mixed load, namespace contention, and pod failure.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+export NAMESPACE="${NAMESPACE:-dsv}"
+export RUN_ID="${RUN_ID:-gauntlet-$(date +%Y%m%d%H%M%S)-${RANDOM}}"
+export WORK_DIR="${WORK_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)/target/dsvclient-k8s-${RUN_ID}}"
+export LOCAL_PORT="${LOCAL_PORT:-19080}"
+
+echo "DSVClient Kubernetes gauntlet"
+echo "  namespace: ${NAMESPACE}"
+echo "  run id:    ${RUN_ID}"
+echo "  work dir:  ${WORK_DIR}"
+
+# Run each phase with an external base URL after one shared port-forward is established.
+source "${SCRIPT_DIR}/dsvclient-k8s-helpers.sh"
+setup_suite
+export EXTERNAL_BASE_URL="${BASE_URL}"
+export NO_PORT_FORWARD=1
+
+REQUESTS="${GAUNTLET_HIGH_REQUESTS:-250}" \
+PARALLELISM="${GAUNTLET_HIGH_PARALLELISM:-50}" \
+bash "${SCRIPT_DIR}/test-dsvclient-high-concurrency.sh"
+
+SEED_COUNT="${GAUNTLET_MIXED_SEED_COUNT:-80}" \
+ROUNDS="${GAUNTLET_MIXED_ROUNDS:-10}" \
+PARALLELISM="${GAUNTLET_MIXED_PARALLELISM:-100}" \
+bash "${SCRIPT_DIR}/test-dsvclient-mixed-concurrency.sh"
+
+USERS="${GAUNTLET_NAMESPACE_USERS:-5}" \
+KEYS="${GAUNTLET_NAMESPACE_KEYS:-12}" \
+REQUESTS_PER_USER="${GAUNTLET_NAMESPACE_REQUESTS_PER_USER:-120}" \
+PARALLELISM="${GAUNTLET_NAMESPACE_PARALLELISM:-100}" \
+bash "${SCRIPT_DIR}/test-dsvclient-same-key-namespaces.sh"
+
+LOAD_REQUESTS="${GAUNTLET_FAILURE_REQUESTS:-180}" \
+PARALLELISM="${GAUNTLET_FAILURE_PARALLELISM:-45}" \
+bash "${SCRIPT_DIR}/test-dsvclient-failure.sh"
+
+echo ""
+echo "Gauntlet completed. Logs are under ${WORK_DIR}"
diff --git a/scripts/test-dsvclient-high-concurrency.sh b/scripts/test-dsvclient-high-concurrency.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# High-concurrency DSVClient test: many independent create/get/update/get-all/delete flows.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "${SCRIPT_DIR}/dsvclient-k8s-helpers.sh"
+
+REQUESTS="${REQUESTS:-200}"
+PARALLELISM="${PARALLELISM:-40}"
+USER_COUNT="${USER_COUNT:-25}"
+
+setup_suite
+section "High concurrency independent client flows"
+
+RESULTS_DIR="${WORK_DIR}/high-concurrency"
+mkdir -p "$RESULTS_DIR"
+
+run_flow() {
+    local i="$1"
+    local user="hc-user-$((i % USER_COUNT))-${RUN_ID}"
+    local name="hc-secret-${RUN_ID}-${i}"
+    local value="hc-value-${i}"
+    local updated="hc-updated-${i}"
+    local out
+
+    out="$(dsvc "$user" create "$name" "$value" 2>&1)" || true
+    printf "%s\n" "$out" > "${RESULTS_DIR}/create-${i}.log"
+    [[ "$out" == *"Secret created"* ]] || { write_status "${RESULTS_DIR}/status-${i}" "FAIL"; return; }
+
+    out="$(dsvc "$user" get "$name" 2>&1)" || true
+    printf "%s\n" "$out" > "${RESULTS_DIR}/get1-${i}.log"
+    [[ "$out" == *"$value"* ]] || { write_status "${RESULTS_DIR}/status-${i}" "FAIL"; return; }
+
+    out="$(dsvc "$user" update "$name" "$updated" 2>&1)" || true
+    printf "%s\n" "$out" > "${RESULTS_DIR}/update-${i}.log"
+    [[ "$out" == *"Secret updated"* ]] || { write_status "${RESULTS_DIR}/status-${i}" "FAIL"; return; }
+
+    out="$(dsvc "$user" get "$name" --all 2>&1)" || true
+    printf "%s\n" "$out" > "${RESULTS_DIR}/all-${i}.log"
+    [[ "$out" == *"$value"* && "$out" == *"$updated"* ]] || { write_status "${RESULTS_DIR}/status-${i}" "FAIL"; return; }
+
+    out="$(dsvc "$user" delete "$name" 2>&1)" || true
+    printf "%s\n" "$out" > "${RESULTS_DIR}/delete-${i}.log"
+    [[ "$out" == *"Delete succeeded"* ]] || { write_status "${RESULTS_DIR}/status-${i}" "FAIL"; return; }
+
+    write_status "${RESULTS_DIR}/status-${i}" "PASS"
+}
+
+for i in $(seq 1 "$REQUESTS"); do
+    run_flow "$i" &
+    if (( i % PARALLELISM == 0 )); then
+        wait
+    fi
+done
+wait
+
+passed="$(count_status "$RESULTS_DIR" PASS)"
+failed="$(count_status "$RESULTS_DIR" FAIL)"
+if [[ "$passed" == "$REQUESTS" ]]; then
+    pass "All ${REQUESTS} concurrent client flows completed"
+else
+    fail "High concurrency flows completed ${passed}/${REQUESTS}; failed=${failed}. Logs: ${RESULTS_DIR}"
+fi
+
+print_summary
+