diff --git a/docker/README.md b/docker/README.md index 5fb1c8a..362d246 100644 --- a/docker/README.md +++ b/docker/README.md @@ -10,7 +10,7 @@ docker/ │ ├── docker-compose.dsv.yml # App only │ ├── docker-compose.dsv-redis.yml # App + Redis │ ├── docker-compose.dsv-redis-kafka.yml # App + Redis + Kafka -│ └── docker-compose.dsv-redis-kafka-3nodes.yml # Three DSV app instances +│ └── docker-compose.dsv-redis-kafka-3nodes.yml # Three DSV app instances + per-node Redis ├── redis/ │ ├── docker-compose.redis.yml # Redis only │ └── redis.conf # Redis persistence and security config @@ -54,7 +54,7 @@ The API listens on `http://localhost:8080`. ## Three App Nodes -For local cluster-like testing, run three DSV app instances against the same Redis and Kafka services: +For local cluster-like testing, run three DSV app instances against shared Kafka and one Redis service per app node: ```bash ./mvnw clean package -DskipTests @@ -62,7 +62,7 @@ mkdir -p target/dependency && (cd target/dependency && jar -xf ../*.jar) docker compose -f docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml up -d --build ``` -Apps listen on `8081`, `8082`, and `8083`. Each instance sets a different `NODE_NAME` so Kafka consumer groups differ and every node receives `secrets-commit` messages. +Apps listen on `8081`, `8082`, and `8083`. Redis instances for those nodes are published on `6381`, `6382`, and `6383`. Each app points at its own Redis service and sets a different `NODE_NAME` so Kafka consumer groups differ and every node receives `secrets-commit` messages. Automated check: @@ -81,7 +81,7 @@ docker logs dsv-app-3 2>&1 | grep -i "Received commit" | tail -3 ## Services -`redis` stores secret shards durably with AOF persistence and password auth. +`redis` stores secret shards durably with AOF persistence and password auth. In the three-node stack this is split into `redis1`, `redis2`, and `redis3`, one per app node. `kafka` provides commit fanout and ordering infrastructure in KRaft mode. @@ -108,4 +108,4 @@ docker compose -f docker/dsv/docker-compose.dsv-redis-kafka.yml down docker compose -f docker/dsv/docker-compose.dsv-redis-kafka.yml down -v ``` -All services communicate on the `dsv-network` bridge network. The app connects to Redis as `redis` and Kafka as `kafka:29092`. +All services communicate on the `dsv-network` bridge network. Single-app stacks connect to Redis as `redis`; the three-node stack connects `app1`, `app2`, and `app3` to `redis1`, `redis2`, and `redis3`. Kafka is reachable as `kafka:29092`. diff --git a/docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml b/docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml index 1a516c1..e9983d7 100644 --- a/docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml +++ b/docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml @@ -1,16 +1,56 @@ -# Same stack as docker-compose.dsv-redis-kafka.yml but with three DSV app instances. +# Three DSV app instances with one Redis instance per app node. # HTTP: localhost:8081 (app1), :8082 (app2), :8083 (app3) -name: "Distributed Secrets Vault + Redis + Kafka" +name: "Distributed Secrets Vault + Per-Node Redis + Kafka" services: - redis: + redis1: image: redis:8.6-alpine - container_name: dsv-redis + container_name: dsv-redis-1 ports: - - "6379:6379" + - "6381:6379" + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} + command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD:-REDIS_PASSWORD} + volumes: + - redis-data-1:/data + - ../redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD:-REDIS_PASSWORD}\" ping"] + interval: 10s + timeout: 3s + retries: 5 + networks: + - dsv-network + + redis2: + image: redis:8.6-alpine + container_name: dsv-redis-2 + ports: + - "6382:6379" + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} + command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD:-REDIS_PASSWORD} + volumes: + - redis-data-2:/data + - ../redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD:-REDIS_PASSWORD}\" ping"] + interval: 10s + timeout: 3s + retries: 5 + networks: + - dsv-network + + redis3: + image: redis:8.6-alpine + container_name: dsv-redis-3 + ports: + - "6383:6379" + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD:-REDIS_PASSWORD} volumes: - - redis-data:/var/lib/redis + - redis-data-3:/data - ../redis/redis.conf:/usr/local/etc/redis/redis.conf:ro healthcheck: test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD:-REDIS_PASSWORD}\" ping"] @@ -63,14 +103,19 @@ services: - "8081:8080" environment: - NODE_NAME=node-1 - - SPRING_DATA_REDIS_HOST=redis + - POD_IP=app1 + - SERVER_PORT=8080 + - CLUSTER_PORT=4801 + - SEED_DNS_HOST=app1 + - SEED_DNS_PORT=4801 + - SPRING_DATA_REDIS_HOST=redis1 - SPRING_DATA_REDIS_PORT=6379 - SPRING_DATA_REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} - SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-dev} - KAFKA_BOOTSTRAP_SERVERS=kafka:29092 - - SEED_DNS_HOST=localhost + - JAVA_TOOL_OPTIONS=-Dcluster.totalNodes=3 -Dcluster.thresholdK=2 -Dcluster.quorumM=2 depends_on: - redis: + redis1: condition: service_healthy kafka: condition: service_healthy @@ -86,14 +131,19 @@ services: - "8082:8080" environment: - NODE_NAME=node-2 - - SPRING_DATA_REDIS_HOST=redis + - POD_IP=app2 + - SERVER_PORT=8080 + - CLUSTER_PORT=4801 + - SEED_DNS_HOST=app1 + - SEED_DNS_PORT=4801 + - SPRING_DATA_REDIS_HOST=redis2 - SPRING_DATA_REDIS_PORT=6379 - SPRING_DATA_REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} - SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-dev} - KAFKA_BOOTSTRAP_SERVERS=kafka:29092 - - SEED_DNS_HOST=localhost + - JAVA_TOOL_OPTIONS=-Dcluster.totalNodes=3 -Dcluster.thresholdK=2 -Dcluster.quorumM=2 depends_on: - redis: + redis2: condition: service_healthy kafka: condition: service_healthy @@ -109,14 +159,19 @@ services: - "8083:8080" environment: - NODE_NAME=node-3 - - SPRING_DATA_REDIS_HOST=redis + - POD_IP=app3 + - SERVER_PORT=8080 + - CLUSTER_PORT=4801 + - SEED_DNS_HOST=app1 + - SEED_DNS_PORT=4801 + - SPRING_DATA_REDIS_HOST=redis3 - SPRING_DATA_REDIS_PORT=6379 - SPRING_DATA_REDIS_PASSWORD=${REDIS_PASSWORD:-REDIS_PASSWORD} - SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-dev} - KAFKA_BOOTSTRAP_SERVERS=kafka:29092 - - SEED_DNS_HOST=localhost + - JAVA_TOOL_OPTIONS=-Dcluster.totalNodes=3 -Dcluster.thresholdK=2 -Dcluster.quorumM=2 depends_on: - redis: + redis3: condition: service_healthy kafka: condition: service_healthy @@ -124,7 +179,9 @@ services: - dsv-network volumes: - redis-data: + redis-data-1: + redis-data-2: + redis-data-3: kafka-data: networks: diff --git a/docs/demo-cluster-lifecycle.md b/docs/demo-cluster-lifecycle.md new file mode 100644 index 0000000..02fdbcd --- /dev/null +++ b/docs/demo-cluster-lifecycle.md @@ -0,0 +1,87 @@ +# Local Cluster Lifecycle Demo + +This demo runs a local Docker-based Distributed Secrets Vault cluster and exercises the full secret lifecycle across multiple clients and nodes. The script builds the Spring Boot application, generates a temporary Docker Compose file at `target/dsv-demo/docker-compose.demo.yml`, starts Kafka plus one Redis service per app node, runs the demo checks, and then tears the stack down. + +## What It Demonstrates + +The default run starts a 3-node cluster: + +- `app1` on `http://127.0.0.1:8081` with `redis1` +- `app2` on `http://127.0.0.1:8082` with `redis2` +- `app3` on `http://127.0.0.1:8083` with `redis3` +- Kafka for commit fanout and cluster coordination + +The demo validates: + +- secret creation, latest reads, updates, version reads, all-version reads, and deletion +- multiple users storing the same secret name without leaking values across users +- parallel clients creating secrets against different app nodes +- continued quorum operation when one node and its Redis service are stopped +- node reboot and recovery after the stopped app and Redis service restart +- read rejection when fewer than `K` shards are online +- write rejection when fewer than `M` nodes are available for quorum + +With the defaults, the script uses `NODE_COUNT=3`, `THRESHOLD_K=2`, and `QUORUM_M=2`. + +## Prerequisites + +Install and start: + +- Docker with Docker Compose +- Java and the Maven wrapper requirements used by the project +- `curl` +- `jar`, unless running with `SKIP_BUILD=1` + +From the project root, create a local environment file if you do not already have one: + +```bash +cp .env.example .env +``` + +The demo defaults to `REDIS_PASSWORD=REDIS_PASSWORD`, matching the local development configuration. + +## Run The Demo + +From the repository root: + +```bash +./scripts/demo-cluster-lifecycle.sh +``` + +The first run may take longer because Docker needs to pull Redis, Kafka, and Java base images. + +## Useful Run Modes + +Customize the quorum and reconstruction threshold: + +```bash +NODE_COUNT=5 THRESHOLD_K=3 QUORUM_M=3 ./scripts/demo-cluster-lifecycle.sh +``` + +Move the published ports if the defaults are already in use: + +```bash +BASE_PORT=9081 REDIS_BASE_PORT=7381 KAFKA_HOST_PORT=29092 ./scripts/demo-cluster-lifecycle.sh +``` + +## Configuration Reference + +| Variable | Default | Description | +| --- | --- | --- | +| `NODE_COUNT` | `3` | Number of app nodes and Redis instances to run. Must be between `3` and `10`. | +| `BASE_PORT` | `8081` | Host port for app node 1. Node N uses `BASE_PORT + N - 1`. | +| `REDIS_BASE_PORT` | `6381` | Host port for Redis node 1. Redis N uses `REDIS_BASE_PORT + N - 1`. | +| `KAFKA_HOST_PORT` | `19092` | Host port published for Kafka. | +| `THRESHOLD_K` | majority | Number of Shamir shards required to reconstruct a secret. | +| `QUORUM_M` | majority | Number of write acknowledgements required to commit. Must be greater than or equal to `THRESHOLD_K`. | +| `REDIS_PASSWORD` | `REDIS_PASSWORD` | Password configured for every local Redis instance. | +| `SPRING_PROFILES_ACTIVE` | `dev` | Spring profile used by the app containers. | +| `KEEP_STACK` | `0` | Set to `1` to leave containers running after the demo. | +| `SKIP_BUILD` | `0` | Set to `1` to skip Maven packaging and reuse `target/dependency`. | +| `PROJECT_NAME` | `dsv-demo` | Docker Compose project and container name prefix. | + +## Troubleshooting + +If Docker reports that a port is already allocated, rerun with different `BASE_PORT`, `REDIS_BASE_PORT`, or `KAFKA_HOST_PORT` values. + +If the script cannot connect to the Docker daemon, make sure Docker Desktop or the Docker service is running and that your shell has permission to access the Docker socket. \ No newline at end of file diff --git a/docs/docker.md b/docs/docker.md index c0ae5f6..50be05c 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -24,7 +24,7 @@ The application will be available at: | `docker/dsv/docker-compose.dsv.yml` | App only | | `docker/dsv/docker-compose.dsv-redis.yml` | App + Redis | | `docker/dsv/docker-compose.dsv-redis-kafka.yml` | App + Redis + Kafka | -| `docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml` | Three app nodes + Redis + Kafka | +| `docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml` | Three app nodes + per-node Redis + Kafka | | `docker/redis/docker-compose.redis.yml` | Redis only | | `docker/kafka/docker-compose.kafka.yml` | Kafka only | @@ -50,6 +50,12 @@ The apps listen on: - `http://127.0.0.1:8082` - `http://127.0.0.1:8083` +Each app in the three-node stack has its own Redis service, matching the Kubernetes sidecar model: + +- app1 -> redis1, published at `localhost:6381` +- app2 -> redis2, published at `localhost:6382` +- app3 -> redis3, published at `localhost:6383` + ## Configuration Configuration is loaded from `.env` in the project root when you run Docker Compose from the project root: diff --git a/scripts/demo-cluster-lifecycle.sh b/scripts/demo-cluster-lifecycle.sh new file mode 100755 index 0000000..0cc3cdf --- /dev/null +++ b/scripts/demo-cluster-lifecycle.sh @@ -0,0 +1,746 @@ +#!/usr/bin/env bash +# Demo the Distributed Secrets Vault lifecycle across a local Docker cluster. +# +# What it shows: +# - secret create, read, update, version reads, and delete +# - multiple client identities using the same secret name without leakage +# - one app node failing while quorum-capable operations continue +# - a stopped node rebooting and serving data again +# - quorum protection when too many app nodes are down +# +# Tunables: +# NODE_COUNT=3..10 number of DSV app nodes to run, default 3 +# BASE_PORT=8081 host port for node 1; node N uses BASE_PORT+N-1 +# REDIS_BASE_PORT=6381 host port for redis1; redisN uses REDIS_BASE_PORT+N-1 +# QUORUM_M= required write ACKs, default majority +# THRESHOLD_K= Shamir reconstruction threshold, default majority +# KEEP_STACK=1 leave the Docker stack running after the demo +# SKIP_BUILD=1 reuse target/dependency instead of rebuilding +# PROJECT_NAME=dsv-demo Docker Compose project/container prefix +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEMO_DIR="${ROOT}/target/dsv-demo" +COMPOSE_FILE="${DEMO_DIR}/docker-compose.demo.yml" + +NODE_COUNT="${NODE_COUNT:-3}" +BASE_PORT="${BASE_PORT:-8081}" +PROJECT_NAME="${PROJECT_NAME:-dsv-demo}" +REDIS_PASSWORD="${REDIS_PASSWORD:-REDIS_PASSWORD}" +REDIS_BASE_PORT="${REDIS_BASE_PORT:-${REDIS_HOST_PORT:-6381}}" +KAFKA_HOST_PORT="${KAFKA_HOST_PORT:-19092}" +THRESHOLD_K="${THRESHOLD_K:-$((NODE_COUNT / 2 + 1))}" +DEFAULT_QUORUM_M="$((NODE_COUNT / 2 + 1))" +if (( DEFAULT_QUORUM_M < THRESHOLD_K )); then + DEFAULT_QUORUM_M="$THRESHOLD_K" +fi +QUORUM_M="${QUORUM_M:-$DEFAULT_QUORUM_M}" +SPRING_PROFILE="${SPRING_PROFILES_ACTIVE:-dev}" +STARTUP_TIMEOUT_SECONDS="${STARTUP_TIMEOUT_SECONDS:-210}" +CLUSTER_TIMEOUT_SECONDS="${CLUSTER_TIMEOUT_SECONDS:-180}" +COMMIT_SETTLE_SECONDS="${COMMIT_SETTLE_SECONDS:-4}" +KEEP_STACK="${KEEP_STACK:-0}" +SKIP_BUILD="${SKIP_BUILD:-0}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +TOTAL_COUNT=0 +PASS_COUNT=0 +FAIL_COUNT=0 +HTTP_STATUS="" +HTTP_BODY="" +LATEST_ALICE_VALUE="" + +die() { + echo -e "${RED}ERROR:${NC} $*" >&2 + exit 1 +} + +info() { + echo -e "${CYAN}==>${NC} $*" +} + +section() { + echo "" + echo -e "${YELLOW}--- $* ---${NC}" +} + +pass() { + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}[PASS]${NC} [${TOTAL_COUNT}] $*" +} + +fail() { + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}[FAIL]${NC} [${TOTAL_COUNT}] $*" +} + +expect_status() { + local expected="$1" + local actual="$2" + local label="$3" + + if [[ "$actual" == "$expected" ]]; then + pass "${label} (HTTP ${actual})" + else + fail "${label} (expected HTTP ${expected}, got ${actual:-empty})" + print_body_preview + fi +} + +expect_body_contains() { + local expected="$1" + local body="$2" + local label="$3" + + if printf "%s" "$body" | grep -Fq -- "$expected"; then + pass "$label" + else + fail "${label} (body did not contain '${expected}')" + print_body_preview + fi +} + +expect_body_not_contains() { + local unexpected="$1" + local body="$2" + local label="$3" + + if printf "%s" "$body" | grep -Fq -- "$unexpected"; then + fail "${label} (body unexpectedly contained '${unexpected}')" + print_body_preview + else + pass "$label" + fi +} + +print_body_preview() { + local preview + preview="$(printf "%s" "$HTTP_BODY" | tr '\n' ' ' | cut -c 1-240)" + if [[ -n "$preview" ]]; then + echo " Body: ${preview}" + fi +} + +validate_config() { + [[ "$NODE_COUNT" =~ ^[0-9]+$ ]] || die "NODE_COUNT must be numeric" + [[ "$BASE_PORT" =~ ^[0-9]+$ ]] || die "BASE_PORT must be numeric" + [[ "$REDIS_BASE_PORT" =~ ^[0-9]+$ ]] || die "REDIS_BASE_PORT must be numeric" + [[ "$QUORUM_M" =~ ^[0-9]+$ ]] || die "QUORUM_M must be numeric" + [[ "$THRESHOLD_K" =~ ^[0-9]+$ ]] || die "THRESHOLD_K must be numeric" + + if (( NODE_COUNT < 3 || NODE_COUNT > 10 )); then + die "NODE_COUNT must be between 3 and 10 for this fault-tolerance demo" + fi + if (( QUORUM_M < 1 || QUORUM_M > NODE_COUNT )); then + die "QUORUM_M must be between 1 and NODE_COUNT" + fi + if (( THRESHOLD_K < 1 || THRESHOLD_K > NODE_COUNT )); then + die "THRESHOLD_K must be between 1 and NODE_COUNT" + fi + if (( QUORUM_M < THRESHOLD_K )); then + die "QUORUM_M must be greater than or equal to THRESHOLD_K" + fi +} + +require_command() { + local command_name="$1" + command -v "$command_name" >/dev/null 2>&1 || die "Required command not found: ${command_name}" +} + +require_docker_compose() { + require_command docker + docker compose version >/dev/null 2>&1 || die "Docker Compose v2 is required: 'docker compose' is not available" +} + +docker_compose() { + docker compose -p "$PROJECT_NAME" -f "$COMPOSE_FILE" "$@" +} + +node_url() { + local index="$1" + printf "http://127.0.0.1:%d" "$((BASE_PORT + index - 1))" +} + +node_container() { + local index="$1" + printf "%s-app-%d" "$PROJECT_NAME" "$index" +} + +redis_container() { + local index="$1" + printf "%s-redis-%d" "$PROJECT_NAME" "$index" +} + +write_compose_file() { + mkdir -p "$DEMO_DIR" + + cat > "$COMPOSE_FILE" <> "$COMPOSE_FILE" <> "$COMPOSE_FILE" <> "$COMPOSE_FILE" <> "$COMPOSE_FILE" </dev/null 2>&1 || true + docker_compose up -d --build + + section "Waiting for health" + for i in $(seq 1 "$NODE_COUNT"); do + wait_for_health "$(node_url "$i")" "node ${i}" + done + + wait_for_cluster_membership "$QUORUM_M" + sleep "$COMMIT_SETTLE_SECONDS" + show_cluster_status +} + +wait_for_health() { + local url="$1" + local label="$2" + local elapsed=0 + + while ! curl -sf --connect-timeout 2 --max-time 10 "${url}/actuator/health" >/dev/null 2>&1; do + if (( elapsed >= STARTUP_TIMEOUT_SECONDS )); then + docker_compose logs --tail=80 || true + die "${label} at ${url} did not become healthy within ${STARTUP_TIMEOUT_SECONDS}s" + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + echo " ${label} healthy at ${url}" +} + +extract_total_nodes() { + sed -n 's/.*"totalNodes":[[:space:]]*\([0-9][0-9]*\).*/\1/p' +} + +wait_for_cluster_membership() { + local minimum="$1" + local elapsed=0 + + info "Waiting for each node to see at least ${minimum} cluster endpoint(s)" + while true; do + local all_ready=true + for i in $(seq 1 "$NODE_COUNT"); do + local body + local total + body="$(curl -sf --connect-timeout 2 --max-time 10 "$(node_url "$i")/api/v1/cluster/status" 2>/dev/null || true)" + total="$(printf "%s" "$body" | extract_total_nodes)" + if [[ -z "$total" || "$total" -lt "$minimum" ]]; then + all_ready=false + break + fi + done + + if [[ "$all_ready" == "true" ]]; then + return + fi + if (( elapsed >= CLUSTER_TIMEOUT_SECONDS )); then + show_cluster_status + die "Cluster membership did not reach the configured quorum within ${CLUSTER_TIMEOUT_SECONDS}s" + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +show_cluster_status() { + section "Cluster status" + for i in $(seq 1 "$NODE_COUNT"); do + local body + body="$(curl -sf --connect-timeout 2 --max-time 10 "$(node_url "$i")/api/v1/cluster/status" 2>/dev/null || true)" + echo " node ${i}: ${body:-unavailable}" + done +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local response + + if [[ -n "$body" ]]; then + response="$(curl -sS -w '\n%{http_code}' \ + --connect-timeout 5 \ + --max-time 45 \ + -X "$method" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "$url" 2>/dev/null || true)" + else + response="$(curl -sS -w '\n%{http_code}' \ + --connect-timeout 5 \ + --max-time 45 \ + -X "$method" \ + "$url" 2>/dev/null || true)" + fi + + HTTP_STATUS="$(printf "%s\n" "$response" | tail -n 1)" + HTTP_BODY="$(printf "%s\n" "$response" | sed '$d')" + echo " ${method} ${url} -> HTTP ${HTTP_STATUS:-empty}" +} + +settle_commits() { + sleep "$COMMIT_SETTLE_SECONDS" +} + +stop_node() { + local index="$1" + local app_container + local redis_node_container + app_container="$(node_container "$index")" + redis_node_container="$(redis_container "$index")" + info "Stopping ${app_container} and ${redis_node_container}" + docker stop "$app_container" "$redis_node_container" >/dev/null +} + +start_node() { + local index="$1" + local app_container + local redis_node_container + app_container="$(node_container "$index")" + redis_node_container="$(redis_container "$index")" + info "Starting ${redis_node_container}" + docker start "$redis_node_container" >/dev/null + wait_for_container_health "$redis_node_container" + info "Starting ${app_container}" + docker start "$app_container" >/dev/null + wait_for_health "$(node_url "$index")" "node ${index}" + sleep "$COMMIT_SETTLE_SECONDS" +} + +wait_for_container_health() { + local container="$1" + local elapsed=0 + local status + + while true; do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \ + "$container" 2>/dev/null || true)" + if [[ "$status" == "healthy" || "$status" == "running" ]]; then + return + fi + if (( elapsed >= STARTUP_TIMEOUT_SECONDS )); then + die "${container} did not become healthy within ${STARTUP_TIMEOUT_SECONDS}s" + fi + sleep 3 + elapsed=$((elapsed + 3)) + done +} + +stop_nodes_except_one() { + local app_containers=() + local redis_containers=() + local i + for i in $(seq 2 "$NODE_COUNT"); do + app_containers+=("$(node_container "$i")") + redis_containers+=("$(redis_container "$i")") + done + info "Stopping app and Redis containers for nodes 2..${NODE_COUNT}; node 1 remains online" + docker stop "${app_containers[@]}" >/dev/null 2>&1 || true + docker stop "${redis_containers[@]}" >/dev/null 2>&1 || true +} + +start_all_nodes() { + local app_containers=() + local redis_containers=() + local i + for i in $(seq 2 "$NODE_COUNT"); do + redis_containers+=("$(redis_container "$i")") + app_containers+=("$(node_container "$i")") + done + info "Starting Redis containers for nodes 2..${NODE_COUNT}" + docker start "${redis_containers[@]}" >/dev/null + for i in $(seq 2 "$NODE_COUNT"); do + wait_for_container_health "$(redis_container "$i")" + done + info "Starting app containers for nodes 2..${NODE_COUNT}" + docker start "${app_containers[@]}" >/dev/null + for i in $(seq 1 "$NODE_COUNT"); do + wait_for_health "$(node_url "$i")" "node ${i}" + done + wait_for_cluster_membership "$QUORUM_M" + sleep "$COMMIT_SETTLE_SECONDS" +} + +parallel_client_creates() { + section "Parallel clients create independent secrets" + + local clients=5 + if (( NODE_COUNT < clients )); then + clients="$NODE_COUNT" + fi + + local results_dir + results_dir="$(mktemp -d "${DEMO_DIR}/parallel-XXXXXX")" + + for i in $(seq 1 "$clients"); do + ( + local node_index + local user + local name + local value + local url + local response + local status + + node_index=$((((i - 1) % NODE_COUNT) + 1)) + user="parallel-client-${i}-${RUN_ID}" + name="parallel-secret-${i}-${RUN_ID}" + value="parallel-value-${i}-${RUN_ID}" + url="$(node_url "$node_index")/api/v1/secrets" + response="$(curl -sS -w '\n%{http_code}' \ + --connect-timeout 5 \ + --max-time 45 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${name}\",\"secretValue\":\"${value}\",\"user\":\"${user}\"}" \ + "$url" 2>/dev/null || true)" + status="$(printf "%s\n" "$response" | tail -n 1)" + printf "%s" "$status" > "${results_dir}/status-${i}" + ) & + done + wait + + local success=0 + local i + for i in $(seq 1 "$clients"); do + local status + status="$(cat "${results_dir}/status-${i}" 2>/dev/null || printf "000")" + if [[ "$status" == "201" ]]; then + success=$((success + 1)) + fi + done + rm -rf "$results_dir" + + if (( success == clients )); then + pass "All ${clients} parallel client creates succeeded" + else + fail "Parallel client creates succeeded ${success}/${clients}" + fi + settle_commits +} + +run_demo() { + RUN_ID="$(date +%Y%m%d%H%M%S)-${RANDOM}" + local alice="alice-${RUN_ID}" + local bob="bob-${RUN_ID}" + local shared_secret="shared-login-${RUN_ID}" + local alice_original="alice-original-${RUN_ID}" + local alice_rotated="alice-rotated-${RUN_ID}" + local alice_after_failure="alice-after-failure-${RUN_ID}" + local bob_original="bob-original-${RUN_ID}" + local bob_rotated="bob-rotated-${RUN_ID}" + LATEST_ALICE_VALUE="$alice_original" + + echo -e "${CYAN}Distributed Secrets Vault demo${NC}" + echo " nodes: ${NODE_COUNT}" + echo " node ports: ${BASE_PORT}..$((BASE_PORT + NODE_COUNT - 1))" + echo " quorum m: ${QUORUM_M}" + echo " threshold k: ${THRESHOLD_K}" + echo " run id: ${RUN_ID}" + + section "Alice creates, reads, updates, and reads versions" + request POST "$(node_url 1)/api/v1/secrets" \ + "{\"secretName\":\"${shared_secret}\",\"secretValue\":\"${alice_original}\",\"user\":\"${alice}\"}" + expect_status "201" "$HTTP_STATUS" "Alice creates ${shared_secret} through node 1" + settle_commits + + request GET "$(node_url 2)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice reads latest through node 2" + expect_body_contains "$alice_original" "$HTTP_BODY" "Alice sees the original value" + + local update_node=3 + request PUT "$(node_url "$update_node")/api/v1/secrets" \ + "{\"secretCurrentName\":\"${shared_secret}\",\"secretUpdatedValue\":\"${alice_rotated}\",\"user\":\"${alice}\"}" + expect_status "200" "$HTTP_STATUS" "Alice updates through node ${update_node}" + if [[ "$HTTP_STATUS" == "200" ]]; then + LATEST_ALICE_VALUE="$alice_rotated" + fi + settle_commits + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice reads latest through node 1" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Latest read returns the rotated value" + + request GET "$(node_url 2)/api/v1/secrets/${shared_secret}?user=${alice}&version=1" + expect_status "200" "$HTTP_STATUS" "Alice reads version 1 through node 2" + expect_body_contains "$alice_original" "$HTTP_BODY" "Version 1 still returns the original value" + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}/all?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice reads all versions" + expect_body_contains "$alice_original" "$HTTP_BODY" "Version history contains original value" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Version history contains latest value" + + section "Bob uses the same secret name without seeing Alice's value" + request POST "$(node_url 2)/api/v1/secrets" \ + "{\"secretName\":\"${shared_secret}\",\"secretValue\":\"${bob_original}\",\"user\":\"${bob}\"}" + expect_status "201" "$HTTP_STATUS" "Bob creates the same secret name through node 2" + settle_commits + + request GET "$(node_url 3)/api/v1/secrets/${shared_secret}?user=${bob}" + expect_status "200" "$HTTP_STATUS" "Bob reads through node 3" + expect_body_contains "$bob_original" "$HTTP_BODY" "Bob sees Bob's value" + expect_body_not_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Bob does not see Alice's value" + + request PUT "$(node_url 1)/api/v1/secrets" \ + "{\"secretCurrentName\":\"${shared_secret}\",\"secretUpdatedValue\":\"${bob_rotated}\",\"user\":\"${bob}\"}" + expect_status "200" "$HTTP_STATUS" "Bob updates his own secret through node 1" + settle_commits + + request DELETE "$(node_url 2)/api/v1/secrets" \ + "{\"deleteName\":\"${shared_secret}\",\"user\":\"${bob}\"}" + expect_status "204" "$HTTP_STATUS" "Bob deletes his own secret through node 2" + settle_commits + + request GET "$(node_url 3)/api/v1/secrets/${shared_secret}?user=${bob}" + expect_status "404" "$HTTP_STATUS" "Bob's deleted secret is gone" + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice's secret still exists after Bob deletes" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Alice still sees her latest value" + + parallel_client_creates + + section "Failure: one node goes down and quorum operations continue" + local failed_node="$NODE_COUNT" + stop_node "$failed_node" + sleep "$COMMIT_SETTLE_SECONDS" + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice reads from node 1 while node ${failed_node} is down" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Read during one-node failure returns current value" + + request PUT "$(node_url 1)/api/v1/secrets" \ + "{\"secretCurrentName\":\"${shared_secret}\",\"secretUpdatedValue\":\"${alice_after_failure}\",\"user\":\"${alice}\"}" + if (( NODE_COUNT - 1 >= QUORUM_M )); then + expect_status "200" "$HTTP_STATUS" "Alice updates while node ${failed_node} is down" + if [[ "$HTTP_STATUS" == "200" ]]; then + LATEST_ALICE_VALUE="$alice_after_failure" + fi + else + expect_status "503" "$HTTP_STATUS" "Write is rejected without quorum" + fi + settle_commits + + section "Reboot: stopped node rejoins and serves the current value" + start_node "$failed_node" + request GET "$(node_url "$failed_node")/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Recovered node ${failed_node} serves Alice's secret" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Recovered node returns the current Alice value" + + section "Too many failures: writes are protected by quorum" + stop_nodes_except_one + sleep "$COMMIT_SETTLE_SECONDS" + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}?user=${alice}" + if (( THRESHOLD_K == 1 )); then + expect_status "200" "$HTTP_STATUS" "Existing secret remains readable from the surviving node" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Surviving node returns the current value" + else + expect_status "503" "$HTTP_STATUS" "Read is rejected when fewer than K shards are online" + fi + + request POST "$(node_url 1)/api/v1/secrets" \ + "{\"secretName\":\"quorum-check-${RUN_ID}\",\"secretValue\":\"should-not-commit-without-quorum\",\"user\":\"${alice}\"}" + if (( QUORUM_M > 1 )); then + expect_status "503" "$HTTP_STATUS" "New write is rejected when only one node is online" + else + expect_status "201" "$HTTP_STATUS" "New write succeeds because QUORUM_M=1" + fi + + section "Full recovery and deletion" + start_all_nodes + request GET "$(node_url 2)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "200" "$HTTP_STATUS" "Alice reads after full recovery" + expect_body_contains "$LATEST_ALICE_VALUE" "$HTTP_BODY" "Full recovery preserved Alice's value" + + request DELETE "$(node_url 3)/api/v1/secrets" \ + "{\"deleteName\":\"${shared_secret}\",\"user\":\"${alice}\"}" + expect_status "204" "$HTTP_STATUS" "Alice deletes through node 3" + settle_commits + + request GET "$(node_url 1)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "404" "$HTTP_STATUS" "Deleted Alice secret returns 404 from node 1" + + request GET "$(node_url 2)/api/v1/secrets/${shared_secret}?user=${alice}" + expect_status "404" "$HTTP_STATUS" "Deleted Alice secret returns 404 from node 2" +} + +print_summary() { + echo "" + echo -e "${CYAN}Demo summary${NC}" + echo " total: ${TOTAL_COUNT}" + echo -e " passed: ${GREEN}${PASS_COUNT}${NC}" + if (( FAIL_COUNT > 0 )); then + echo -e " failed: ${RED}${FAIL_COUNT}${NC}" + return 1 + fi + echo " failed: 0" + return 0 +} + +cleanup() { + if [[ "$KEEP_STACK" == "1" ]]; then + echo "" + info "KEEP_STACK=1, leaving demo stack running" + echo " Compose file: ${COMPOSE_FILE}" + echo " Node 1 URL: $(node_url 1)" + return + fi + + if [[ -f "$COMPOSE_FILE" ]]; then + echo "" + info "Stopping demo stack" + docker_compose down -v --remove-orphans >/dev/null 2>&1 || true + fi +} + +main() { + validate_config + require_docker_compose + require_command curl + if [[ "$SKIP_BUILD" != "1" ]]; then + require_command jar + fi + + trap cleanup EXIT + + build_app + write_compose_file + start_cluster + run_demo + print_summary +} + +main "$@" diff --git a/scripts/test-concurrent-requests.sh b/scripts/test-concurrent-requests.sh new file mode 100755 index 0000000..d6bbd21 --- /dev/null +++ b/scripts/test-concurrent-requests.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# test-concurrent-requests.sh +# Concurrent request handling tests for the same secret key. +# Tests race conditions on reads, writes, and create/delete conflicts. +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/test-helpers.sh" + +echo -e "${CYAN}════════════════════════════════════════════════${NC}" +echo -e "${CYAN} Concurrent Request Integration Tests${NC}" +echo -e "${CYAN}════════════════════════════════════════════════${NC}" + +setup_test_cluster + +USER="concurrency-user-$(date +%s)" +SECRET_NAME=$(unique_key "conc") +SECRET_VALUE="original-concurrent-value" + +# ── 1. Create a secret first ──────────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Setup: create test secret ---${NC}" + +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"${SECRET_VALUE}\",\"user\":\"${USER}\"}" +assert_status "201" "$HTTP_STATUS" "Create test secret" + +sleep 3 + +# ── 2. Three concurrent reads → all should succeed with same value ── +echo "" +echo -e "${YELLOW}--- Concurrent reads (3 parallel) ---${NC}" + +RESULTS_DIR=$(mktemp -d "${ROOT}/target/conc-read-XXXXXX") +for i in 1 2 3; do + ( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + "${NODES[$((i - 1))]}/api/v1/secrets/${SECRET_NAME}?user=${USER}" 2>/dev/null) || true + status=$(echo "$resp" | tail -1) + body=$(echo "$resp" | sed '$d') + echo "${status}" > "${RESULTS_DIR}/status-${i}" + echo "${body}" > "${RESULTS_DIR}/body-${i}" + ) & +done +wait + +all_reads_ok=true +for i in 1 2 3; do + s=$(cat "${RESULTS_DIR}/status-${i}" 2>/dev/null || echo "000") + b=$(cat "${RESULTS_DIR}/body-${i}" 2>/dev/null || echo "") + if [[ "$s" != "200" ]]; then + all_reads_ok=false + fi + if ! echo "$b" | grep -q "${SECRET_VALUE}"; then + all_reads_ok=false + fi +done +rm -rf "$RESULTS_DIR" + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if $all_reads_ok; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] All 3 concurrent reads returned 200 with correct value" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Concurrent reads: not all returned 200 with correct value" +fi + +# ── 3. Three concurrent updates → at least one succeeds ───────────── +echo "" +echo -e "${YELLOW}--- Concurrent updates (3 parallel) ---${NC}" + +RESULTS_DIR=$(mktemp -d "${ROOT}/target/conc-update-XXXXXX") +for i in 1 2 3; do + ( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"secretCurrentName\":\"${SECRET_NAME}\",\"secretUpdatedValue\":\"update-${i}\",\"user\":\"${USER}\"}" \ + "${NODES[$((i - 1))]}/api/v1/secrets" 2>/dev/null) || true + status=$(echo "$resp" | tail -1) + echo "${status}" > "${RESULTS_DIR}/status-${i}" + ) & +done +wait + +success_count=0 +conflict_count=0 +for i in 1 2 3; do + s=$(cat "${RESULTS_DIR}/status-${i}" 2>/dev/null || echo "000") + if [[ "$s" == "200" ]]; then + success_count=$((success_count + 1)) + elif [[ "$s" == "409" ]]; then + conflict_count=$((conflict_count + 1)) + fi +done +rm -rf "$RESULTS_DIR" + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if (( success_count >= 1 )); then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Concurrent updates: ${success_count} succeeded, ${conflict_count} conflicts (expected)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Concurrent updates: none succeeded" +fi + +sleep 3 + +# ── 4. Read final state → version should be consistent ────────────── +echo "" +echo -e "${YELLOW}--- Read final state ---${NC}" + +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read final state after concurrent updates" + +# ── 5. Two concurrent creates of same key (same user) → race ──────── +echo "" +echo -e "${YELLOW}--- Concurrent duplicate creates (same user, same key) ---${NC}" + +RACE_NAME=$(unique_key "race") + +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${RACE_NAME}\",\"secretValue\":\"val-A\",\"user\":\"${USER}\"}" \ + "${NODE1}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-race-1" +) & + +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${RACE_NAME}\",\"secretValue\":\"val-B\",\"user\":\"${USER}\"}" \ + "${NODE2}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-race-2" +) & + +wait + +RACE1=$(cat /tmp/dsv-race-1 2>/dev/null || echo "000") +RACE2=$(cat /tmp/dsv-race-2 2>/dev/null || echo "000") +rm -f /tmp/dsv-race-1 /tmp/dsv-race-2 + +# Exactly one should be 201 and the other 409 (or both could be 201 if timing allows) +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +race_201=0 +race_409=0 +[[ "$RACE1" == "201" ]] && race_201=$((race_201 + 1)) +[[ "$RACE2" == "201" ]] && race_201=$((race_201 + 1)) +[[ "$RACE1" == "409" ]] && race_409=$((race_409 + 1)) +[[ "$RACE2" == "409" ]] && race_409=$((race_409 + 1)) + +if (( race_201 >= 1 && (race_201 + race_409 == 2) )); then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Concurrent duplicate creates: ${race_201}×201, ${race_409}×409" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Concurrent creates: Race1=${RACE1}, Race2=${RACE2} (expected one 201 + one 409)" +fi + +sleep 3 + +# ── 6. Create then immediately delete concurrently ────────────────── +echo "" +echo -e "${YELLOW}--- Concurrent create + delete (same key) ---${NC}" + +CD_NAME=$(unique_key "create-delete") + +# First create the secret +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${CD_NAME}\",\"secretValue\":\"ephemeral\",\"user\":\"${USER}\"}" +CREATE_FIRST_STATUS="$HTTP_STATUS" + +sleep 3 + +# Now try to delete and create (update) concurrently +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X DELETE \ + -H "Content-Type: application/json" \ + -d "{\"deleteName\":\"${CD_NAME}\",\"user\":\"${USER}\"}" \ + "${NODE2}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-cd-delete" +) & + +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"secretCurrentName\":\"${CD_NAME}\",\"secretUpdatedValue\":\"updated\",\"user\":\"${USER}\"}" \ + "${NODE3}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-cd-update" +) & + +wait + +CD_DELETE=$(cat /tmp/dsv-cd-delete 2>/dev/null || echo "000") +CD_UPDATE=$(cat /tmp/dsv-cd-update 2>/dev/null || echo "000") +rm -f /tmp/dsv-cd-delete /tmp/dsv-cd-update + +# At least one should succeed; the other may conflict +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if [[ "$CD_DELETE" =~ ^(204|404|409|503)$ ]] && [[ "$CD_UPDATE" =~ ^(200|404|409|503)$ ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Concurrent delete+update: delete=${CD_DELETE}, update=${CD_UPDATE}" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Concurrent delete+update: delete=${CD_DELETE}, update=${CD_UPDATE}" +fi + +# ── Cleanup ────────────────────────────────────────────────────────── +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER}\"}" || true +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${RACE_NAME}\",\"user\":\"${USER}\"}" || true + +# ── Summary ────────────────────────────────────────────────────────── +print_summary diff --git a/scripts/test-crud-lifecycle.sh b/scripts/test-crud-lifecycle.sh new file mode 100755 index 0000000..0b1e8e6 --- /dev/null +++ b/scripts/test-crud-lifecycle.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# test-crud-lifecycle.sh +# Full CRUD lifecycle test against a real 3-node DSV cluster. +# Tests: create → read → update → read versions → delete → recreate +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/test-helpers.sh" + +echo -e "${CYAN}════════════════════════════════════════════════${NC}" +echo -e "${CYAN} CRUD Lifecycle Integration Tests${NC}" +echo -e "${CYAN}════════════════════════════════════════════════${NC}" + +setup_test_cluster + +# Generate unique key names to avoid collisions +SECRET_NAME=$(unique_key "crud-test") +USER="crud-test-user" +SECRET_VALUE="my-super-secret-password-123" +UPDATED_VALUE="my-updated-password-456" + +# ── 1. Create a secret on Node 1 ──────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Create ---${NC}" + +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"${SECRET_VALUE}\",\"user\":\"${USER}\"}" +assert_status "201" "$HTTP_STATUS" "Create secret on Node 1" + +# Brief settle time for Kafka commit propagation +sleep 3 + +# ── 2. Read secret from Node 2 ────────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Read from other nodes ---${NC}" + +do_curl GET "${NODE2}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read secret from Node 2" +assert_body_contains "${SECRET_VALUE}" "$HTTP_BODY" "Node 2 returns correct value" + +# ── 3. Read secret from Node 3 ────────────────────────────────────── +do_curl GET "${NODE3}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read secret from Node 3" +assert_body_contains "${SECRET_VALUE}" "$HTTP_BODY" "Node 3 returns correct value" + +# ── 4. Update secret on Node 2 ────────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Update ---${NC}" + +do_curl PUT "${NODE2}/api/v1/secrets" \ + "{\"secretCurrentName\":\"${SECRET_NAME}\",\"secretUpdatedValue\":\"${UPDATED_VALUE}\",\"user\":\"${USER}\"}" +assert_status "200" "$HTTP_STATUS" "Update secret on Node 2" + +sleep 3 + +# ── 5. Read latest from Node 1 (should be updated value) ──────────── +echo "" +echo -e "${YELLOW}--- Read after update ---${NC}" + +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read latest from Node 1" +assert_body_contains "${UPDATED_VALUE}" "$HTTP_BODY" "Node 1 returns updated value" + +# ── 6. Read version 1 from Node 3 ─────────────────────────────────── +do_curl GET "${NODE3}/api/v1/secrets/${SECRET_NAME}?user=${USER}&version=1" +assert_status "200" "$HTTP_STATUS" "Read version 1 from Node 3" +assert_body_contains "${SECRET_VALUE}" "$HTTP_BODY" "Version 1 still has original value" + +# ── 7. Read all versions from Node 1 ──────────────────────────────── +echo "" +echo -e "${YELLOW}--- All versions ---${NC}" + +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}/all?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read all versions from Node 1" +assert_body_contains "${SECRET_VALUE}" "$HTTP_BODY" "All versions contains original" +assert_body_contains "${UPDATED_VALUE}" "$HTTP_BODY" "All versions contains updated" + +# ── 8. Delete secret from Node 3 ──────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Delete ---${NC}" + +do_curl DELETE "${NODE3}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER}\"}" +assert_status "204" "$HTTP_STATUS" "Delete secret from Node 3" + +sleep 3 + +# ── 9. Read after delete from Node 1 → 404 ────────────────────────── +echo "" +echo -e "${YELLOW}--- Verify deletion ---${NC}" + +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "404" "$HTTP_STATUS" "Read after delete returns 404" + +# ── 10. Create again after delete → fresh version ─────────────────── +echo "" +echo -e "${YELLOW}--- Recreate after delete ---${NC}" + +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"recreated-secret\",\"user\":\"${USER}\"}" +assert_status "201" "$HTTP_STATUS" "Recreate secret after delete" + +sleep 3 + +do_curl GET "${NODE2}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read recreated secret from Node 2" +assert_body_contains "recreated-secret" "$HTTP_BODY" "Recreated value is correct" + +# ── 11. Delete non-existent secret → 404 ──────────────────────────── +echo "" +echo -e "${YELLOW}--- Error cases ---${NC}" + +FAKE_NAME=$(unique_key "nonexistent") +do_curl DELETE "${NODE2}/api/v1/secrets" \ + "{\"deleteName\":\"${FAKE_NAME}\",\"user\":\"${USER}\"}" +assert_status "404" "$HTTP_STATUS" "Delete non-existent secret returns 404" + +# ── 12. Update non-existent secret → 404 ──────────────────────────── +do_curl PUT "${NODE1}/api/v1/secrets" \ + "{\"secretCurrentName\":\"${FAKE_NAME}\",\"secretUpdatedValue\":\"x\",\"user\":\"${USER}\"}" +assert_status "404" "$HTTP_STATUS" "Update non-existent secret returns 404" + +# ── Cleanup ────────────────────────────────────────────────────────── +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER}\"}" || true + +# ── Summary ────────────────────────────────────────────────────────── +print_summary diff --git a/scripts/test-helpers.sh b/scripts/test-helpers.sh new file mode 100755 index 0000000..80affa8 --- /dev/null +++ b/scripts/test-helpers.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# Shared utility functions for DSV integration test scripts. +# Source this file from other test scripts: source "$(dirname "$0")/test-helpers.sh" +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +# ── Paths ──────────────────────────────────────────────────────────── +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${ROOT}/docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml" + +# ── Test runner options ───────────────────────────────────────────── +# AUTO_START_CLUSTER=1 starts a fresh Docker stack for each test script. +# Set AUTO_START_CLUSTER=0 to run a script against an already-running stack. +AUTO_START_CLUSTER="${AUTO_START_CLUSTER:-1}" +KEEP_STACK="${KEEP_STACK:-0}" +SKIP_BUILD="${SKIP_BUILD:-0}" + +# ── Node URLs ──────────────────────────────────────────────────────── +NODE1="${NODE1:-http://127.0.0.1:8081}" +NODE2="${NODE2:-http://127.0.0.1:8082}" +NODE3="${NODE3:-http://127.0.0.1:8083}" +NODES=("$NODE1" "$NODE2" "$NODE3") + +# ── Colors ─────────────────────────────────────────────────────────── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# ── Counters ───────────────────────────────────────────────────────── +PASS_COUNT=0 +FAIL_COUNT=0 +TOTAL_COUNT=0 + +# ── Functions ──────────────────────────────────────────────────────── + +# Container names in docker/dsv/docker-compose.dsv-redis-kafka-3nodes.yml. +app_container() { + local index="$1" + printf "dsv-app-%d" "$index" +} + +redis_container() { + local index="$1" + printf "dsv-redis-%d" "$index" +} + +# Start the 3-node cluster (build + docker compose up) +start_cluster() { + mkdir -p "$ROOT/target" + + if [[ "$SKIP_BUILD" == "1" ]]; then + echo -e "${CYAN}==> Skipping Maven build because SKIP_BUILD=1${NC}" + else + echo -e "${CYAN}==> Building layered JAR layout for Docker${NC}" + (cd "$ROOT" && ./mvnw -q clean package -DskipTests) + mkdir -p "$ROOT/target/dependency" + (cd "$ROOT/target/dependency" && jar -xf ../*.jar) + fi + + echo -e "${CYAN}==> Starting 3-node cluster${NC}" + docker compose -f "$COMPOSE_FILE" down -v --remove-orphans >/dev/null 2>&1 || true + docker compose -f "$COMPOSE_FILE" up -d --build + + echo -e "${CYAN}==> Waiting for all nodes to become healthy (up to 180s)${NC}" + for url in "${NODES[@]}"; do + wait_for_health "$url" + done + + # Brief settle time for Kafka consumer group coordination + sleep 8 + echo -e "${GREEN}==> All nodes healthy${NC}" +} + +setup_test_cluster() { + mkdir -p "$ROOT/target" + + if [[ "$AUTO_START_CLUSTER" == "0" ]]; then + echo -e "${CYAN}==> Using existing 3-node cluster because AUTO_START_CLUSTER=0${NC}" + for url in "${NODES[@]}"; do + wait_for_health "$url" + done + return + fi + + trap cleanup_test_cluster EXIT + start_cluster +} + +cleanup_test_cluster() { + local status=$? + + if [[ "$AUTO_START_CLUSTER" == "0" ]]; then + return "$status" + fi + + if [[ "$KEEP_STACK" == "1" ]]; then + echo -e "${CYAN}==> KEEP_STACK=1, leaving cluster running${NC}" + return "$status" + fi + + stop_cluster + return "$status" +} + +# Stop the cluster +stop_cluster() { + echo -e "${CYAN}==> Stopping cluster${NC}" + docker compose -f "$COMPOSE_FILE" down -v --remove-orphans 2>/dev/null || true +} + +# Wait for a node's actuator health endpoint to respond +wait_for_health() { + local url="$1" + local timeout=180 + local elapsed=0 + while ! curl -sf --connect-timeout 2 --max-time 10 "${url}/actuator/health" >/dev/null 2>&1; do + if (( elapsed >= timeout )); then + echo -e "${RED}ERROR: ${url} did not become healthy in ${timeout}s${NC}" >&2 + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +wait_for_container_health() { + local container="$1" + local timeout=180 + local elapsed=0 + local status + + while true; do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \ + "$container" 2>/dev/null || true)" + if [[ "$status" == "healthy" || "$status" == "running" ]]; then + return + fi + if (( elapsed >= timeout )); then + echo -e "${RED}ERROR: ${container} did not become healthy in ${timeout}s${NC}" >&2 + return 1 + fi + sleep 3 + elapsed=$((elapsed + 3)) + done +} + +stop_node() { + local index="$1" + local app + local redis + app="$(app_container "$index")" + redis="$(redis_container "$index")" + docker stop "$app" "$redis" >/dev/null 2>&1 || true +} + +start_node() { + local index="$1" + local app + local redis + app="$(app_container "$index")" + redis="$(redis_container "$index")" + + docker start "$redis" >/dev/null 2>&1 + wait_for_container_health "$redis" + docker start "$app" >/dev/null 2>&1 + wait_for_health "${NODES[$((index - 1))]}" +} + +stop_nodes() { + local index + for index in "$@"; do + stop_node "$index" + done +} + +start_nodes() { + local index + for index in "$@"; do + docker start "$(redis_container "$index")" >/dev/null 2>&1 + done + for index in "$@"; do + wait_for_container_health "$(redis_container "$index")" + done + for index in "$@"; do + docker start "$(app_container "$index")" >/dev/null 2>&1 + done + for index in "$@"; do + wait_for_health "${NODES[$((index - 1))]}" + done +} + +# Assert HTTP status code +# Usage: assert_status +assert_status() { + local expected="$1" + local actual="$2" + local test_name="$3" + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + if [[ "$actual" == "$expected" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] ${test_name} (HTTP ${actual})" + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] ${test_name} (expected HTTP ${expected}, got ${actual})" + fi +} + +# Assert response body contains a substring +# Usage: assert_body_contains +assert_body_contains() { + local expected="$1" + local body="$2" + local test_name="$3" + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + if echo "$body" | grep -q "$expected"; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] ${test_name}" + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] ${test_name} (expected body to contain '${expected}')" + echo " Actual body: ${body:0:200}" + fi +} + +# Assert response body does NOT contain a substring +# Usage: assert_body_not_contains +assert_body_not_contains() { + local unexpected="$1" + local body="$2" + local test_name="$3" + TOTAL_COUNT=$((TOTAL_COUNT + 1)) + if echo "$body" | grep -q "$unexpected"; then + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] ${test_name} (body should NOT contain '${unexpected}')" + else + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] ${test_name}" + fi +} + +# Print final test summary +print_summary() { + echo "" + echo -e "${CYAN}════════════════════════════════════════════════${NC}" + echo -e "${CYAN} Test Summary${NC}" + echo -e "${CYAN}════════════════════════════════════════════════${NC}" + echo -e " Total: ${TOTAL_COUNT}" + echo -e " ${GREEN}Passed: ${PASS_COUNT}${NC}" + if (( FAIL_COUNT > 0 )); then + echo -e " ${RED}Failed: ${FAIL_COUNT}${NC}" + else + echo -e " Failed: 0" + fi + echo -e "${CYAN}════════════════════════════════════════════════${NC}" + if (( FAIL_COUNT > 0 )); then + echo -e "${RED}SOME TESTS FAILED${NC}" + return 1 + else + echo -e "${GREEN}ALL TESTS PASSED${NC}" + return 0 + fi +} + +# Perform a curl request and capture status + body +# Usage: do_curl [json_body] +# Sets: HTTP_STATUS, HTTP_BODY +do_curl() { + local method="$1" + local url="$2" + local body="${3:-}" + local response + + if [[ -n "$body" ]]; then + response=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X "$method" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "$url" 2>/dev/null) || true + else + response=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X "$method" \ + "$url" 2>/dev/null) || true + fi + + HTTP_STATUS=$(echo "$response" | tail -1) + HTTP_BODY=$(echo "$response" | sed '$d') +} + +# Generate a unique test key name to avoid collisions between runs +unique_key() { + local prefix="${1:-test}" + echo "${prefix}-$(date +%s)-${RANDOM}" +} diff --git a/scripts/test-multi-client.sh b/scripts/test-multi-client.sh new file mode 100755 index 0000000..486258c --- /dev/null +++ b/scripts/test-multi-client.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# test-multi-client.sh +# Multi-user isolation and concurrent client request tests. +# Verifies that secrets from different users don't leak across tenants. +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/test-helpers.sh" + +echo -e "${CYAN}════════════════════════════════════════════════${NC}" +echo -e "${CYAN} Multi-Client Integration Tests${NC}" +echo -e "${CYAN}════════════════════════════════════════════════${NC}" + +setup_test_cluster + +SECRET_NAME=$(unique_key "shared-name") +USER_A="alice-$(date +%s)" +USER_B="bob-$(date +%s)" +VALUE_A="alice-secret-value" +VALUE_B="bob-secret-value" + +# ── 1. User A creates a secret on Node 1 ──────────────────────────── +echo "" +echo -e "${YELLOW}--- User A creates ---${NC}" + +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"${VALUE_A}\",\"user\":\"${USER_A}\"}" +assert_status "201" "$HTTP_STATUS" "User A creates secret on Node 1" + +sleep 3 + +# ── 2. User B creates secret with same name on Node 2 ─────────────── +echo "" +echo -e "${YELLOW}--- User B creates (same name, different owner) ---${NC}" + +do_curl POST "${NODE2}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"${VALUE_B}\",\"user\":\"${USER_B}\"}" +assert_status "201" "$HTTP_STATUS" "User B creates secret with same name on Node 2" + +sleep 3 + +# ── 3. User A reads from Node 2 → sees own value ──────────────────── +echo "" +echo -e "${YELLOW}--- Read isolation ---${NC}" + +do_curl GET "${NODE2}/api/v1/secrets/${SECRET_NAME}?user=${USER_A}" +assert_status "200" "$HTTP_STATUS" "User A reads from Node 2" +assert_body_contains "${VALUE_A}" "$HTTP_BODY" "User A sees own value" +assert_body_not_contains "${VALUE_B}" "$HTTP_BODY" "User A does NOT see Bob's value" + +# ── 4. User B reads from Node 3 → sees own value ──────────────────── +do_curl GET "${NODE3}/api/v1/secrets/${SECRET_NAME}?user=${USER_B}" +assert_status "200" "$HTTP_STATUS" "User B reads from Node 3" +assert_body_contains "${VALUE_B}" "$HTTP_BODY" "User B sees own value" +assert_body_not_contains "${VALUE_A}" "$HTTP_BODY" "User B does NOT see Alice's value" + +# ── 5. User A tries to read with wrong user → 404 ─────────────────── +echo "" +echo -e "${YELLOW}--- Cross-tenant isolation ---${NC}" + +FAKE_USER="eve-$(date +%s)" +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${FAKE_USER}" +assert_status "404" "$HTTP_STATUS" "Unknown user cannot read existing secret" + +# ── 6. User A updates, User B deletes — no interference ───────────── +echo "" +echo -e "${YELLOW}--- Independent operations ---${NC}" + +# User A updates +do_curl PUT "${NODE1}/api/v1/secrets" \ + "{\"secretCurrentName\":\"${SECRET_NAME}\",\"secretUpdatedValue\":\"alice-updated\",\"user\":\"${USER_A}\"}" +assert_status "200" "$HTTP_STATUS" "User A updates own secret" + +sleep 3 + +# User B deletes +do_curl DELETE "${NODE2}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER_B}\"}" +assert_status "204" "$HTTP_STATUS" "User B deletes own secret" + +sleep 3 + +# User A's secret should still exist with updated value +do_curl GET "${NODE3}/api/v1/secrets/${SECRET_NAME}?user=${USER_A}" +assert_status "200" "$HTTP_STATUS" "User A's secret still exists after User B deletes" +assert_body_contains "alice-updated" "$HTTP_BODY" "User A has updated value" + +# User B's secret should be gone +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${USER_B}" +assert_status "404" "$HTTP_STATUS" "User B's secret is gone" + +# ── 7. Concurrent creates: 5 users, different keys, in parallel ───── +echo "" +echo -e "${YELLOW}--- Concurrent parallel creates (5 users) ---${NC}" + +PARALLEL_RESULTS_DIR=$(mktemp -d "${ROOT}/target/parallel-XXXXXX") +for i in $(seq 1 5); do + ( + usr="parallel-user-${i}-$(date +%s)" + name=$(unique_key "parallel-${i}") + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${name}\",\"secretValue\":\"val-${i}\",\"user\":\"${usr}\"}" \ + "${NODES[$((i % 3))]}/api/v1/secrets" 2>/dev/null) || true + status=$(echo "$resp" | tail -1) + echo "$status" > "${PARALLEL_RESULTS_DIR}/result-${i}" + ) & +done +wait + +parallel_pass=0 +parallel_fail=0 +for i in $(seq 1 5); do + result=$(cat "${PARALLEL_RESULTS_DIR}/result-${i}" 2>/dev/null || echo "000") + if [[ "$result" == "201" ]]; then + parallel_pass=$((parallel_pass + 1)) + else + parallel_fail=$((parallel_fail + 1)) + fi +done +rm -rf "$PARALLEL_RESULTS_DIR" + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if (( parallel_pass == 5 )); then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] All 5 parallel creates succeeded (${parallel_pass}/5)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Parallel creates: ${parallel_pass}/5 succeeded, ${parallel_fail}/5 failed" +fi + +# ── 8. Two users create same name concurrently → both succeed ──────── +echo "" +echo -e "${YELLOW}--- Concurrent same-name creates (different owners) ---${NC}" + +SAME_NAME=$(unique_key "concurrent-same") +U1="concurrent-alice-$(date +%s)" +U2="concurrent-bob-$(date +%s)" + +STATUS1="" +STATUS2="" + +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${SAME_NAME}\",\"secretValue\":\"alice-val\",\"user\":\"${U1}\"}" \ + "${NODE1}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-concurrent-1" +) & + +( + resp=$(curl -sS -w "\n%{http_code}" \ + --connect-timeout 5 --max-time 30 \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"secretName\":\"${SAME_NAME}\",\"secretValue\":\"bob-val\",\"user\":\"${U2}\"}" \ + "${NODE2}/api/v1/secrets" 2>/dev/null) || true + echo "$resp" | tail -1 > "/tmp/dsv-concurrent-2" +) & + +wait + +STATUS1=$(cat /tmp/dsv-concurrent-1 2>/dev/null || echo "000") +STATUS2=$(cat /tmp/dsv-concurrent-2 2>/dev/null || echo "000") +rm -f /tmp/dsv-concurrent-1 /tmp/dsv-concurrent-2 + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if [[ "$STATUS1" == "201" && "$STATUS2" == "201" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Both users created same-name secret (201, 201)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Concurrent same-name creates: User1=${STATUS1}, User2=${STATUS2}" +fi + +# ── Cleanup ────────────────────────────────────────────────────────── +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER_A}\"}" || true + +# ── Summary ────────────────────────────────────────────────────────── +print_summary diff --git a/scripts/test-node-failure.sh b/scripts/test-node-failure.sh new file mode 100755 index 0000000..41760dd --- /dev/null +++ b/scripts/test-node-failure.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────── +# test-node-failure.sh +# Node failure and recovery tests. +# Stops/restarts Docker containers mid-operation to verify resilience. +# ────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/test-helpers.sh" + +echo -e "${CYAN}════════════════════════════════════════════════${NC}" +echo -e "${CYAN} Node Failure & Recovery Integration Tests${NC}" +echo -e "${CYAN}════════════════════════════════════════════════${NC}" + +setup_test_cluster + +USER="failure-user-$(date +%s)" +SECRET_NAME=$(unique_key "failure-test") +SECRET_VALUE="data-for-failure-test" + +# ── 1. Create a secret while all 3 nodes are up ──────────────────── +echo "" +echo -e "${YELLOW}--- Setup: all nodes healthy ---${NC}" + +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SECRET_NAME}\",\"secretValue\":\"${SECRET_VALUE}\",\"user\":\"${USER}\"}" +assert_status "201" "$HTTP_STATUS" "Create secret with all 3 nodes up" + +sleep 3 + +# ── 2. Read from all nodes → all return the secret ────────────────── +echo "" +echo -e "${YELLOW}--- Read from all nodes ---${NC}" + +for i in 1 2 3; do + do_curl GET "${NODES[$((i - 1))]}/api/v1/secrets/${SECRET_NAME}?user=${USER}" + assert_status "200" "$HTTP_STATUS" "Read from Node ${i} (all nodes up)" +done + +# ── 3. Stop Node 3 ────────────────────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Stop Node 3 ---${NC}" + +stop_node 3 +echo -e " ${CYAN}Node 3 app and Redis stopped${NC}" +sleep 5 + +# ── 4. Read from Node 1 (2 nodes remaining) ───────────────────────── +echo "" +echo -e "${YELLOW}--- Read with Node 3 down ---${NC}" + +do_curl GET "${NODE1}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +# With k=2, we need 2 shards. Node 1 + Node 2 should suffice. +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if [[ "$HTTP_STATUS" == "200" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Read from Node 1 with Node 3 down (200)" +elif [[ "$HTTP_STATUS" == "503" ]]; then + # May happen if quorum requires all nodes + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Read from Node 1 with Node 3 down (503 — quorum config)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Read from Node 1 with Node 3 down (unexpected: ${HTTP_STATUS})" +fi + +# ── 5. Try to create a new secret (2 nodes up) ────────────────────── +echo "" +echo -e "${YELLOW}--- Create with one node down ---${NC}" + +NEW_NAME=$(unique_key "during-failure") +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${NEW_NAME}\",\"secretValue\":\"created-during-failure\",\"user\":\"${USER}\"}" + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if [[ "$HTTP_STATUS" == "201" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Create with one node down succeeded (201)" +elif [[ "$HTTP_STATUS" == "503" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Create with one node down rejected (503 — quorum not met)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Create with one node down: unexpected HTTP ${HTTP_STATUS}" +fi + +# ── 6. Restart Node 3, wait for health ─────────────────────────────── +echo "" +echo -e "${YELLOW}--- Restart Node 3 ---${NC}" + +echo -e " ${CYAN}Node 3 app and Redis restarting...${NC}" +start_node 3 +echo -e " ${GREEN}Node 3 healthy again${NC}" +sleep 5 + +# ── 7. Read from recovered Node 3 ─────────────────────────────────── +echo "" +echo -e "${YELLOW}--- Read from recovered Node 3 ---${NC}" + +do_curl GET "${NODE3}/api/v1/secrets/${SECRET_NAME}?user=${USER}" +assert_status "200" "$HTTP_STATUS" "Read from recovered Node 3" +assert_body_contains "${SECRET_VALUE}" "$HTTP_BODY" "Recovered node returns correct value" + +# ── 8. Stop Node 2 and Node 3 (only Node 1 remaining) ─────────────── +echo "" +echo -e "${YELLOW}--- Stop Node 2 and Node 3 ---${NC}" + +stop_nodes 2 3 +echo -e " ${CYAN}Node 2 and Node 3 apps and Redis services stopped (only Node 1 running)${NC}" +sleep 5 + +# ── 9. Try to create on Node 1 alone → should fail quorum ─────────── +echo "" +echo -e "${YELLOW}--- Create with only 1 node (quorum failure) ---${NC}" + +SOLO_NAME=$(unique_key "solo") +do_curl POST "${NODE1}/api/v1/secrets" \ + "{\"secretName\":\"${SOLO_NAME}\",\"secretValue\":\"solo-value\",\"user\":\"${USER}\"}" + +TOTAL_COUNT=$((TOTAL_COUNT + 1)) +if [[ "$HTTP_STATUS" == "503" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Create on solo node correctly rejected (503)" +elif [[ "$HTTP_STATUS" == "201" ]]; then + # Might succeed if quorum=1 or single-node mode + PASS_COUNT=$((PASS_COUNT + 1)) + echo -e " ${GREEN}✓ PASS${NC} [${TOTAL_COUNT}] Create on solo node succeeded (201 — quorum config allows it)" +else + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo -e " ${RED}✗ FAIL${NC} [${TOTAL_COUNT}] Create on solo node: unexpected HTTP ${HTTP_STATUS}" +fi + +# ── 10. Restart all nodes, verify full cluster health ──────────────── +echo "" +echo -e "${YELLOW}--- Restart all nodes ---${NC}" + +echo -e " ${CYAN}Restarting Node 2 and Node 3 apps and Redis services...${NC}" +start_nodes 2 3 +echo -e " ${GREEN}All nodes healthy${NC}" +sleep 5 + +# Verify original secret is still accessible from all nodes +echo "" +echo -e "${YELLOW}--- Verify data after full recovery ---${NC}" + +for i in 1 2 3; do + do_curl GET "${NODES[$((i - 1))]}/api/v1/secrets/${SECRET_NAME}?user=${USER}" + assert_status "200" "$HTTP_STATUS" "Read from Node ${i} after full recovery" +done + +# ── Cleanup ────────────────────────────────────────────────────────── +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${SECRET_NAME}\",\"user\":\"${USER}\"}" || true +do_curl DELETE "${NODE1}/api/v1/secrets" \ + "{\"deleteName\":\"${NEW_NAME}\",\"user\":\"${USER}\"}" || true + +# ── Summary ────────────────────────────────────────────────────────── +print_summary diff --git a/src/main/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharing.java b/src/main/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharing.java index 2b925c1..daed50a 100644 --- a/src/main/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharing.java +++ b/src/main/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharing.java @@ -35,8 +35,11 @@ public byte[] reconstruct(Map parts) { if (parts.size() == 1) { return parts.values().iterator().next(); } - int size = parts.size(); - Scheme scheme = new Scheme(new SecureRandom(), size, size); + int totalParts = parts.keySet().stream() + .mapToInt(Integer::intValue) + .max() + .orElseThrow(() -> new IllegalArgumentException("Secret parts are required")); + Scheme scheme = new Scheme(new SecureRandom(), totalParts, parts.size()); return scheme.join(parts); } } diff --git a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetService.java b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetService.java index 02ee08c..02d3cfc 100644 --- a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetService.java +++ b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetService.java @@ -3,22 +3,57 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import edu.yu.capstone.DistributedSecretsVault.config.ClusterConfig; import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; +import edu.yu.capstone.DistributedSecretsVault.exceptions.InsufficientShardsException; import edu.yu.capstone.DistributedSecretsVault.exceptions.SecretNotFoundException; import edu.yu.capstone.DistributedSecretsVault.repository.SecretPartRepository; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient.SecretPartResponse; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient.SecretPartsResponse; +import edu.yu.capstone.DistributedSecretsVault.service.secret.SecretReconstructionService; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeSet; @Service public class InternalGetService { private final SecretPartRepository secretPartRepository; + private final SecretReconstructionService secretReconstructionService; + private final NodeClient nodeClient; + private final ClusterConfig clusterConfig; - public InternalGetService(SecretPartRepository secretPartRepository) { + public InternalGetService(SecretPartRepository secretPartRepository, + SecretReconstructionService secretReconstructionService, + NodeClient nodeClient, + ClusterConfig clusterConfig) { this.secretPartRepository = secretPartRepository; + this.secretReconstructionService = secretReconstructionService; + this.nodeClient = nodeClient; + this.clusterConfig = clusterConfig; + } + + public String getAcrossCluster(SecretKey key, Long version) { + validateKey(key); + List selected = collectPartsForReconstruction(key, version); + return secretReconstructionService.reconstruct(selected); + } + + public Map getAllVersionsAcrossCluster(SecretKey key) { + validateKey(key); + Map> partsByVersion = collectAllPartsByVersion(key); + if (partsByVersion.isEmpty()) { + throw new SecretNotFoundException(); + } + Map results = new LinkedHashMap<>(); + partsByVersion.keySet().stream().sorted().forEach(version -> { + List selected = requireThreshold(partsByVersion.get(version)); + results.put(version, secretReconstructionService.reconstruct(selected)); + }); + return results; } public ResponseEntity getVersion(String user, String secretName, Long version) { @@ -61,4 +96,108 @@ private SecretKey validate(String user, String secretName) { } return new SecretKey(user, secretName); } + + private void validateKey(SecretKey key) { + if (key == null || key.getName() == null || key.getName().isBlank()) { + throw new IllegalArgumentException("Secret key is required"); + } + } + + private List collectPartsForReconstruction(SecretKey key, Long requestedVersion) { + Map> partsByVersion = new LinkedHashMap<>(); + addLocalPart(partsByVersion, key, requestedVersion); + + for (String peerUrl : resolvePeerUrls()) { + SecretPartResponse response = nodeClient.fetchSecretPart(peerUrl, key, requestedVersion); + if (response != null && response.found()) { + addPart(partsByVersion, response.part()); + } + } + + if (partsByVersion.isEmpty()) { + throw new SecretNotFoundException(); + } + + long resolvedVersion = requestedVersion == null + ? partsByVersion.keySet().stream().mapToLong(Long::longValue).max() + .orElseThrow(SecretNotFoundException::new) + : requestedVersion; + + Map selectedParts = partsByVersion.get(resolvedVersion); + if (selectedParts == null || selectedParts.isEmpty()) { + throw new SecretNotFoundException(); + } + return requireThreshold(selectedParts); + } + + private Map> collectAllPartsByVersion(SecretKey key) { + Map> partsByVersion = new LinkedHashMap<>(); + addLocalVersionParts(partsByVersion, key); + + for (String peerUrl : resolvePeerUrls()) { + SecretPartsResponse response = nodeClient.fetchAllSecretParts(peerUrl, key); + if (response != null && response.found()) { + response.parts().values().forEach(part -> addPart(partsByVersion, part)); + } + } + return partsByVersion; + } + + private void addLocalPart(Map> partsByVersion, SecretKey key, Long requestedVersion) { + Optional localPart = requestedVersion == null + ? secretPartRepository.findLatest(key) + : secretPartRepository.findPart(key, requestedVersion); + localPart.ifPresent(part -> addPart(partsByVersion, part)); + } + + private void addLocalVersionParts(Map> partsByVersion, SecretKey key) { + for (Long version : secretPartRepository.listVersions(key)) { + secretPartRepository.findPart(key, version) + .ifPresent(part -> addPart(partsByVersion, part)); + } + } + + private void addPart(Map> partsByVersion, SecretPart part) { + if (part == null || part.getVersion() == null || part.getShard() == null) { + return; + } + partsByVersion.computeIfAbsent(part.getVersion(), ignored -> new LinkedHashMap<>()) + .putIfAbsent(part.getPartIndex(), part); + } + + private List requireThreshold(Map partsByIndex) { + if (partsByIndex == null || partsByIndex.isEmpty()) { + throw new SecretNotFoundException(); + } + int threshold = resolveThreshold(resolveTotalParts()); + if (partsByIndex.size() < threshold) { + throw new InsufficientShardsException(); + } + return new TreeSet<>(partsByIndex.keySet()).stream() + .limit(threshold) + .map(partsByIndex::get) + .toList(); + } + + private List resolvePeerUrls() { + return nodeClient == null ? List.of() : nodeClient.resolvePeerUrls(); + } + + private int resolveTotalParts() { + if (clusterConfig == null || clusterConfig.getTotalNodes() <= 0) { + return 1; + } + return clusterConfig.getTotalNodes(); + } + + private int resolveThreshold(int totalParts) { + int threshold = clusterConfig == null ? 0 : clusterConfig.getThresholdK(); + if (threshold <= 0) { + threshold = 1; + } + if (threshold > totalParts) { + threshold = totalParts; + } + return threshold; + } } diff --git a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/NodeClient.java b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/NodeClient.java index 90c8e9b..36b0c6f 100644 --- a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/NodeClient.java +++ b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/internal/NodeClient.java @@ -3,15 +3,19 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import org.springframework.core.ParameterizedTypeReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; import org.springframework.web.client.RestClientResponseException; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; import edu.yu.capstone.DistributedSecretsVault.dto.internal.DeletePrepareRequest; import edu.yu.capstone.DistributedSecretsVault.dto.internal.PostPrepareRequest; import edu.yu.capstone.DistributedSecretsVault.dto.internal.PutPrepareRequest; @@ -120,6 +124,60 @@ public PeerResponse sendPutPrepare(String peerUrl, PutPrepareRequest request) { } } + public SecretPartResponse fetchSecretPart(String peerUrl, SecretKey key, Long version) { + try { + SecretPart part; + if (version == null) { + part = restClient.get() + .uri(peerUrl + "/internal/{id}?user={user}", key.getName(), key.getOwnerId()) + .retrieve() + .body(SecretPart.class); + } else { + part = restClient.get() + .uri(peerUrl + "/internal/{id}?user={user}&version={version}", + key.getName(), key.getOwnerId(), version) + .retrieve() + .body(SecretPart.class); + } + log.debug("Secret part received from {}", peerUrl); + return SecretPartResponse.found(peerUrl, part); + } catch (RestClientResponseException ex) { + if (ex.getStatusCode().is4xxClientError()) { + log.debug("Secret part not available from {} with HTTP {}", peerUrl, ex.getStatusCode().value()); + } else { + log.warn("Secret part fetch failed from {} with HTTP {}", peerUrl, ex.getStatusCode().value()); + } + return SecretPartResponse.rejected(peerUrl, ex.getStatusCode().value(), ex.getResponseBodyAsString()); + } catch (Exception ex) { + log.warn("Failed to fetch secret part from {}: {}", peerUrl, ex.getMessage()); + return SecretPartResponse.failed(peerUrl, ex.getMessage()); + } + } + + public SecretPartsResponse fetchAllSecretParts(String peerUrl, SecretKey key) { + try { + Map parts = restClient.get() + .uri(peerUrl + "/internal/{id}/all?user={user}", key.getName(), key.getOwnerId()) + .retrieve() + .body(new ParameterizedTypeReference>() { + }); + log.debug("Secret version parts received from {}", peerUrl); + return SecretPartsResponse.found(peerUrl, parts == null ? Map.of() : parts); + } catch (RestClientResponseException ex) { + if (ex.getStatusCode().is4xxClientError()) { + log.debug("Secret version parts not available from {} with HTTP {}", + peerUrl, ex.getStatusCode().value()); + } else { + log.warn("Secret version parts fetch failed from {} with HTTP {}", + peerUrl, ex.getStatusCode().value()); + } + return SecretPartsResponse.rejected(peerUrl, ex.getStatusCode().value(), ex.getResponseBodyAsString()); + } catch (Exception ex) { + log.warn("Failed to fetch secret version parts from {}: {}", peerUrl, ex.getMessage()); + return SecretPartsResponse.failed(peerUrl, ex.getMessage()); + } + } + /** * Resolve peer node base URLs using ScaleCube's service discovery. *

@@ -191,4 +249,41 @@ public static PeerResponse failed(String peerUrl, String errorMessage) { return new PeerResponse(peerUrl, false, null, errorMessage); } } + + public record SecretPartResponse(String peerUrl, SecretPart part, Integer statusCode, String errorMessage) { + public boolean found() { + return part != null; + } + + public static SecretPartResponse found(String peerUrl, SecretPart part) { + return new SecretPartResponse(peerUrl, part, null, null); + } + + public static SecretPartResponse rejected(String peerUrl, int statusCode, String errorMessage) { + return new SecretPartResponse(peerUrl, null, statusCode, errorMessage); + } + + public static SecretPartResponse failed(String peerUrl, String errorMessage) { + return new SecretPartResponse(peerUrl, null, null, errorMessage); + } + } + + public record SecretPartsResponse(String peerUrl, Map parts, Integer statusCode, + String errorMessage) { + public boolean found() { + return parts != null && !parts.isEmpty(); + } + + public static SecretPartsResponse found(String peerUrl, Map parts) { + return new SecretPartsResponse(peerUrl, parts, null, null); + } + + public static SecretPartsResponse rejected(String peerUrl, int statusCode, String errorMessage) { + return new SecretPartsResponse(peerUrl, null, statusCode, errorMessage); + } + + public static SecretPartsResponse failed(String peerUrl, String errorMessage) { + return new SecretPartsResponse(peerUrl, null, null, errorMessage); + } + } } diff --git a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretService.java b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretService.java index a1c820d..9e4ff23 100644 --- a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretService.java +++ b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretService.java @@ -4,35 +4,36 @@ import org.springframework.stereotype.Service; import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.service.internal.InternalGetService; + import java.util.Map; @Service public class GetSecretService { - private final SecretService secretService; - private SecretKey key; + private final InternalGetService internalGetService; - public GetSecretService(SecretService secretService) { - this.secretService = secretService; + public GetSecretService(InternalGetService internalGetService) { + this.internalGetService = internalGetService; } public ResponseEntity getVersion(String user, String secretName, Long version) { - validate(user, secretName); - String secretValue = secretService.getSecret(key, version); + SecretKey key = validate(user, secretName); + String secretValue = internalGetService.getAcrossCluster(key, version); return ResponseEntity.ok(secretValue); } public ResponseEntity> getAllVersions(String user, String secretName) { - validate(user, secretName); - return ResponseEntity.ok(secretService.getAllVersions(key)); + SecretKey key = validate(user, secretName); + return ResponseEntity.ok(internalGetService.getAllVersionsAcrossCluster(key)); } - private void validate(String user, String secretName) { + private SecretKey validate(String user, String secretName) { if (user == null || user.isBlank()) { throw new IllegalArgumentException("User is required"); } if (secretName == null || secretName.isBlank()) { throw new IllegalArgumentException("Secret key is required"); } - key = new SecretKey(user, secretName); + return new SecretKey(user, secretName); } -} \ No newline at end of file +} diff --git a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/SecretService.java b/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/SecretService.java deleted file mode 100644 index 0466167..0000000 --- a/src/main/java/edu/yu/capstone/DistributedSecretsVault/service/secret/SecretService.java +++ /dev/null @@ -1,162 +0,0 @@ -package edu.yu.capstone.DistributedSecretsVault.service.secret; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import org.springframework.stereotype.Service; - -import edu.yu.capstone.DistributedSecretsVault.config.ClusterConfig; -import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; -import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; -import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretVersion; -import edu.yu.capstone.DistributedSecretsVault.exceptions.DuplicateSecretException; -import edu.yu.capstone.DistributedSecretsVault.exceptions.InsufficientShardsException; -import edu.yu.capstone.DistributedSecretsVault.exceptions.SecretNotFoundException; -import edu.yu.capstone.DistributedSecretsVault.repository.SecretPartRepository; - -@Service -public class SecretService { - private final SecretPartRepository secretPartRepository; - private final SecretSharingService secretSharingService; - private final SecretReconstructionService secretReconstructionService; - private final ClusterConfig clusterConfig; - - public SecretService(SecretPartRepository secretPartRepository, - SecretSharingService secretSharingService, - SecretReconstructionService secretReconstructionService, - ClusterConfig clusterConfig) { - this.secretPartRepository = secretPartRepository; - this.secretSharingService = secretSharingService; - this.secretReconstructionService = secretReconstructionService; - this.clusterConfig = clusterConfig; - } - - public SecretVersion storeSecret(SecretKey key, String value) { - validateKey(key); - if (value == null) { - throw new IllegalArgumentException("Secret value is required"); - } - if (secretPartRepository.exists(key)) { - throw new DuplicateSecretException(); - } - int totalParts = resolveTotalParts(); - int threshold = resolveThreshold(totalParts); - List parts = secretSharingService.split(key, value, threshold, totalParts); - long version = 1L; - for (SecretPart part : parts) { - part.setVersion(version); - secretPartRepository.savePart(part); // split to the other nodes - } - return new SecretVersion(key, version, System.currentTimeMillis()); - } - - public SecretVersion updateSecret(SecretKey key, String value) { - validateKey(key); - if (value == null) { - throw new IllegalArgumentException("Secret value is required"); - } - if (!secretPartRepository.exists(key)) { - throw new SecretNotFoundException(); - } - long version = nextVersion(key); - int totalParts = resolveTotalParts(); - int threshold = resolveThreshold(totalParts); - List parts = secretSharingService.split(key, value, threshold, totalParts); - for (SecretPart part : parts) { - part.setVersion(version); - if (!secretPartRepository.updatePart(part)) { - throw new SecretNotFoundException(); - } - } - return new SecretVersion(key, version, System.currentTimeMillis()); - } - - public String getSecret(SecretKey key, Long version) { - validateKey(key); - if (!secretPartRepository.exists(key)) { - throw new SecretNotFoundException(); - } - long resolvedVersion = resolveVersion(key, version); - Optional part = secretPartRepository.findPart(key, resolvedVersion); - if (part.isEmpty()) { - throw new InsufficientShardsException(); - } - List selected = part.stream().toList(); - return secretReconstructionService.reconstruct(selected); - } - - public Map getAllVersions(SecretKey key) { - validateKey(key); - List versions = secretPartRepository.listVersions(key); - if (versions.isEmpty()) { - throw new SecretNotFoundException(); - } - Map results = new LinkedHashMap<>(); - versions.stream().sorted().forEach(version -> { - Optional part = secretPartRepository.findPart(key, version); - if (part.isEmpty()) { - throw new InsufficientShardsException(); - } - List selected = part.stream().toList(); - results.put(version, secretReconstructionService.reconstruct(selected)); - }); - return results; - } - - public void deleteSecret(SecretKey key) { - validateKey(key); - if (!secretPartRepository.exists(key)) { - throw new SecretNotFoundException(); - } - secretPartRepository.deleteParts(key); - } - - private void validateKey(SecretKey key) { - if (key == null || key.getName() == null || key.getName().isBlank()) { - throw new IllegalArgumentException("Secret key is required"); - } - } - - private long resolveVersion(SecretKey key, Long requestedVersion) { - if (requestedVersion != null) { - List versions = secretPartRepository.listVersions(key); - if (!versions.contains(requestedVersion)) { - throw new SecretNotFoundException(); - } - return requestedVersion; - } - return latestVersion(key); - } - - private long latestVersion(SecretKey key) { - return secretPartRepository.findLatest(key) - .map(SecretPart::getVersion) - .orElseThrow(SecretNotFoundException::new); - } - - private long nextVersion(SecretKey key) { - return secretPartRepository.findLatest(key) - .map(part -> part.getVersion() + 1L) - .orElse(1L); - } - - private int resolveTotalParts() { - if (clusterConfig == null || clusterConfig.getTotalNodes() <= 0) { - return 1; - } - return clusterConfig.getTotalNodes(); - } - - private int resolveThreshold(int totalParts) { - int threshold = clusterConfig == null ? 0 : clusterConfig.getThresholdK(); - if (threshold <= 0) { - threshold = 1; - } - if (threshold > totalParts) { - threshold = totalParts; - } - return threshold; - } -} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharingTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharingTest.java new file mode 100644 index 0000000..d5330b1 --- /dev/null +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/encrypt/ShamirSecretSharingTest.java @@ -0,0 +1,218 @@ +package edu.yu.capstone.DistributedSecretsVault.encrypt; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** + * Tests for the Shamir Secret Sharing implementation. + * Verifies split/reconstruct round-trips, threshold behavior, and edge cases. + */ +@Tag("unit") +public class ShamirSecretSharingTest { + + private ShamirSecretSharing shamir; + + @BeforeEach + void setUp() { + shamir = new ShamirSecretSharing(); + } + + // ── Round-trip: split and reconstruct ──────────────────────────────── + + @Test + void testSplitAndReconstructRoundTrip_3of5() { + byte[] secret = "my-database-password".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 5, 3); + + assertEquals(5, parts.size()); + + // Use exactly 3 shards (the threshold) for reconstruction + Map subset = new HashMap<>(); + int count = 0; + for (Map.Entry entry : parts.entrySet()) { + subset.put(entry.getKey(), entry.getValue()); + if (++count == 3) break; + } + + byte[] reconstructed = shamir.reconstruct(subset); + assertArrayEquals(secret, reconstructed); + } + + @Test + void testSplitAndReconstructRoundTrip_2of3() { + byte[] secret = "api-key-12345".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 3, 2); + + assertEquals(3, parts.size()); + + // Use exactly 2 shards + Map subset = new HashMap<>(); + int count = 0; + for (Map.Entry entry : parts.entrySet()) { + subset.put(entry.getKey(), entry.getValue()); + if (++count == 2) break; + } + + byte[] reconstructed = shamir.reconstruct(subset); + assertArrayEquals(secret, reconstructed); + } + + @Test + void testSplitAndReconstructWithAllShards() { + byte[] secret = "full-shard-test".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 3, 2); + + // Use all shards (more than threshold) + byte[] reconstructed = shamir.reconstruct(parts); + assertArrayEquals(secret, reconstructed); + } + + @Test + void testReconstructFromNonContiguousShardIndexes() { + byte[] secret = "non-contiguous-shards".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 5, 3); + Map subset = Map.of( + 2, parts.get(2), + 4, parts.get(4), + 5, parts.get(5)); + + byte[] reconstructed = shamir.reconstruct(subset); + + assertArrayEquals(secret, reconstructed); + } + + // ── Threshold = 1 (raw copies) ────────────────────────────────────── + + @Test + void testThresholdOneReturnsRawCopies() { + byte[] secret = "simple-secret".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 3, 1); + + assertEquals(3, parts.size()); + for (byte[] shard : parts.values()) { + assertArrayEquals(secret, shard, "Threshold=1 should return raw copies"); + } + } + + @Test + void testThresholdOneReconstructsFromSingleShard() { + byte[] secret = "single-shard-test".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 3, 1); + + // Any single shard should reconstruct the secret + for (Map.Entry entry : parts.entrySet()) { + Map singleShard = Map.of(entry.getKey(), entry.getValue()); + byte[] reconstructed = shamir.reconstruct(singleShard); + assertArrayEquals(secret, reconstructed); + } + } + + // ── Single shard reconstruction ───────────────────────────────────── + + @Test + void testReconstructFromSingleShardMap() { + byte[] shard = "raw-shard-data".getBytes(StandardCharsets.UTF_8); + Map parts = Map.of(1, shard); + + byte[] result = shamir.reconstruct(parts); + + assertArrayEquals(shard, result, + "Single-shard reconstruct should return the shard as-is"); + } + + // ── Large secret data ─────────────────────────────────────────────── + + @Test + void testLargeSecretRoundTrip() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append("abcdefghijklmnop"); // 16KB total + } + byte[] secret = sb.toString().getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 5, 3); + assertEquals(5, parts.size()); + + // Reconstruct with 3 shards + Map subset = new HashMap<>(); + int count = 0; + for (Map.Entry entry : parts.entrySet()) { + subset.put(entry.getKey(), entry.getValue()); + if (++count == 3) break; + } + + byte[] reconstructed = shamir.reconstruct(subset); + assertArrayEquals(secret, reconstructed); + } + + // ── Invalid inputs ────────────────────────────────────────────────── + + @Test + void testSplitRejectsNullSecret() { + assertThrows(IllegalArgumentException.class, + () -> shamir.split(null, 3, 2)); + } + + @Test + void testSplitRejectsZeroParts() { + assertThrows(IllegalArgumentException.class, + () -> shamir.split(new byte[]{1}, 0, 1)); + } + + @Test + void testSplitRejectsZeroThreshold() { + assertThrows(IllegalArgumentException.class, + () -> shamir.split(new byte[]{1}, 3, 0)); + } + + @Test + void testSplitRejectsThresholdGreaterThanParts() { + assertThrows(IllegalArgumentException.class, + () -> shamir.split(new byte[]{1}, 2, 5)); + } + + @Test + void testSplitRejectsNegativeParts() { + assertThrows(IllegalArgumentException.class, + () -> shamir.split(new byte[]{1}, -1, 1)); + } + + @Test + void testReconstructRejectsNullParts() { + assertThrows(IllegalArgumentException.class, + () -> shamir.reconstruct(null)); + } + + @Test + void testReconstructRejectsEmptyParts() { + assertThrows(IllegalArgumentException.class, + () -> shamir.reconstruct(Map.of())); + } + + // ── Edge case: n = k (all shards required) ───────────────────────── + + @Test + void testSplitAndReconstructWhenThresholdEqualsTotalParts() { + byte[] secret = "all-shards-needed".getBytes(StandardCharsets.UTF_8); + + Map parts = shamir.split(secret, 3, 3); + + assertEquals(3, parts.size()); + + byte[] reconstructed = shamir.reconstruct(parts); + assertArrayEquals(secret, reconstructed); + } +} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/ConcurrentRequestTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/ConcurrentRequestTest.java new file mode 100644 index 0000000..86dd153 --- /dev/null +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/ConcurrentRequestTest.java @@ -0,0 +1,283 @@ +package edu.yu.capstone.DistributedSecretsVault.service; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import edu.yu.capstone.DistributedSecretsVault.config.ClusterConfig; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; +import edu.yu.capstone.DistributedSecretsVault.service.internal.ActionType; +import edu.yu.capstone.DistributedSecretsVault.service.internal.PendingActionsBuffer; +import edu.yu.capstone.DistributedSecretsVault.service.internal.PendingActionsBuffer.PendingAction; + +/** + * Thread-safety and concurrency tests. + *

+ * These tests exercise the {@link PendingActionsBuffer} under concurrent load + * to verify that buffering, committing, and discarding are thread-safe. + *

+ * For the higher-level services ({@code InternalPostService}, etc.), true + * concurrency is tested via the integration shell scripts against a real + * multi-node cluster. + */ +@Tag("unit") +public class ConcurrentRequestTest { + + private PendingActionsBuffer buffer; + + @BeforeEach + void setUp() { + ClusterConfig config = new ClusterConfig(); + config.setLockTimeoutMillis(30_000L); + buffer = new PendingActionsBuffer(config); + } + + // ── Concurrent buffer + commit on different keys ──────────────────── + + @Test + void testConcurrentBufferAndCommitDifferentKeys() throws InterruptedException { + int threadCount = 20; + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + AtomicInteger successCount = new AtomicInteger(0); + + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + final int idx = i; + executor.submit(() -> { + try { + startLatch.await(); + SecretKey key = new SecretKey("user" + idx, "secret" + idx); + UUID opId = UUID.randomUUID(); + + buffer.bufferAction(opId, key, ActionType.POST); + assertTrue(buffer.contains(opId)); + assertTrue(buffer.containsKey(key)); + + PendingAction committed = buffer.commitAndRemove(opId); + assertNotNull(committed); + assertEquals(opId, committed.operationId()); + + assertFalse(buffer.contains(opId)); + successCount.incrementAndGet(); + } catch (Exception e) { + fail("Thread " + idx + " threw: " + e.getMessage()); + } finally { + doneLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + executor.shutdownNow(); + assertEquals(threadCount, successCount.get()); + } + + // ── Concurrent buffer + commit on SAME key ────────────────────────── + + @Test + void testConcurrentBufferSameKeyCascadeOnCommit() throws InterruptedException { + int threadCount = 10; + SecretKey sharedKey = new SecretKey("user1", "shared-secret"); + List operationIds = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch allBuffered = new CountDownLatch(threadCount); + CountDownLatch startLatch = new CountDownLatch(1); + + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + + // Phase 1: all threads buffer an action for the same key + for (int i = 0; i < threadCount; i++) { + executor.submit(() -> { + try { + startLatch.await(); + UUID opId = UUID.randomUUID(); + buffer.bufferAction(opId, sharedKey, ActionType.PUT); + operationIds.add(opId); + } catch (Exception e) { + fail("Buffering threw: " + e.getMessage()); + } finally { + allBuffered.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(allBuffered.await(5, TimeUnit.SECONDS)); + + // All should be buffered + for (UUID opId : operationIds) { + assertTrue(buffer.contains(opId), "Expected opId to be buffered: " + opId); + } + assertTrue(buffer.containsKey(sharedKey)); + + // Phase 2: committing ONE should cascade-remove ALL others for the same key + UUID winnerId = operationIds.get(0); + PendingAction committed = buffer.commitAndRemove(winnerId); + assertNotNull(committed); + assertEquals(winnerId, committed.operationId()); + + // All operations for sharedKey should be gone + for (UUID opId : operationIds) { + assertFalse(buffer.contains(opId), "Expected opId to be evicted: " + opId); + } + assertFalse(buffer.containsKey(sharedKey)); + + executor.shutdownNow(); + } + + // ── Concurrent discard ────────────────────────────────────────────── + + @Test + void testConcurrentDiscard() throws InterruptedException { + int threadCount = 10; + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + AtomicInteger discardedCount = new AtomicInteger(0); + + List opIds = new ArrayList<>(); + for (int i = 0; i < threadCount; i++) { + UUID opId = UUID.randomUUID(); + opIds.add(opId); + buffer.bufferAction(opId, new SecretKey("user" + i, "secret" + i), ActionType.DELETE); + } + + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + final int idx = i; + executor.submit(() -> { + try { + startLatch.await(); + PendingAction discarded = buffer.discard(opIds.get(idx)); + if (discarded != null) { + discardedCount.incrementAndGet(); + } + } catch (Exception e) { + fail("Thread " + idx + " threw: " + e.getMessage()); + } finally { + doneLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + executor.shutdownNow(); + assertEquals(threadCount, discardedCount.get()); + + // All should be gone + for (UUID opId : opIds) { + assertFalse(buffer.contains(opId)); + } + } + + // ── Race: buffer + immediate commit from different threads ────────── + + @Test + void testRaceBufferAndCommitFromDifferentThreads() throws InterruptedException { + int iterations = 50; + CountDownLatch doneLatch = new CountDownLatch(iterations); + CopyOnWriteArrayList results = new CopyOnWriteArrayList<>(); + + ExecutorService executor = Executors.newFixedThreadPool(4); + for (int i = 0; i < iterations; i++) { + final int idx = i; + executor.submit(() -> { + try { + SecretKey key = new SecretKey("user", "secret-" + idx); + UUID opId = UUID.randomUUID(); + buffer.bufferAction(opId, key, ActionType.POST, + new SecretPart(key, 1L, 1, new byte[]{1})); + + // Immediately commit from the same thread (simulates fast commit) + PendingAction committed = buffer.commitAndRemove(opId); + results.add(committed != null); + } finally { + doneLatch.countDown(); + } + }); + } + + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + executor.shutdownNow(); + + // Every buffer+commit pair should succeed + assertEquals(iterations, results.size()); + assertTrue(results.stream().allMatch(Boolean::booleanValue)); + } + + // ── Concurrent eviction + buffer ──────────────────────────────────── + + @Test + void testConcurrentEvictionAndBuffering() throws InterruptedException { + ClusterConfig shortConfig = new ClusterConfig(); + shortConfig.setLockTimeoutMillis(50L); + PendingActionsBuffer shortBuffer = new PendingActionsBuffer(shortConfig); + + // Buffer some entries that will expire + for (int i = 0; i < 10; i++) { + shortBuffer.bufferAction(UUID.randomUUID(), + new SecretKey("old-user", "old-" + i), ActionType.DELETE); + } + + Thread.sleep(100); // Let them expire + + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(2); + + // Thread 1: evict expired + Thread evictor = new Thread(() -> { + try { + startLatch.await(); + shortBuffer.evictExpired(); + } catch (Exception e) { + fail("Evictor threw: " + e.getMessage()); + } finally { + doneLatch.countDown(); + } + }); + + // Thread 2: buffer new entries concurrently + List newOps = new ArrayList<>(); + Thread bufferer = new Thread(() -> { + try { + startLatch.await(); + for (int i = 0; i < 10; i++) { + UUID opId = UUID.randomUUID(); + newOps.add(opId); + shortBuffer.bufferAction(opId, + new SecretKey("new-user", "new-" + i), ActionType.POST); + } + } catch (Exception e) { + fail("Bufferer threw: " + e.getMessage()); + } finally { + doneLatch.countDown(); + } + }); + + evictor.start(); + bufferer.start(); + startLatch.countDown(); + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + + // New entries should still be present + for (UUID opId : newOps) { + assertTrue(shortBuffer.contains(opId), + "New entry should survive concurrent eviction: " + opId); + } + } +} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/NodeFailureRecoveryTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/NodeFailureRecoveryTest.java new file mode 100644 index 0000000..7f0b3cd --- /dev/null +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/NodeFailureRecoveryTest.java @@ -0,0 +1,347 @@ +package edu.yu.capstone.DistributedSecretsVault.service; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import edu.yu.capstone.DistributedSecretsVault.config.ClusterConfig; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretVersion; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.CommitMessage; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.DeletePrepareRequest; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.PostPrepareRequest; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.PutPrepareRequest; +import edu.yu.capstone.DistributedSecretsVault.exceptions.QuorumNotReachedException; +import edu.yu.capstone.DistributedSecretsVault.exceptions.SecretNotFoundException; +import edu.yu.capstone.DistributedSecretsVault.exceptions.ServiceUnavailableException; +import edu.yu.capstone.DistributedSecretsVault.repository.SecretPartRepository; +import edu.yu.capstone.DistributedSecretsVault.service.communication.CommitPublisher; +import edu.yu.capstone.DistributedSecretsVault.service.internal.ActionType; +import edu.yu.capstone.DistributedSecretsVault.service.internal.InternalDeleteService; +import edu.yu.capstone.DistributedSecretsVault.service.internal.InternalPostService; +import edu.yu.capstone.DistributedSecretsVault.service.internal.InternalPutService; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient.PeerResponse; +import edu.yu.capstone.DistributedSecretsVault.service.internal.PendingActionsBuffer; +import edu.yu.capstone.DistributedSecretsVault.service.secret.SecretSharingService; + +/** + * Tests simulating node failures during distributed operations. + * Verifies that the system rolls back gracefully when peers fail at + * different phases (prepare, commit publish) and that buffer entries + * are properly cleaned up. + */ +@ExtendWith(MockitoExtension.class) +@Tag("unit") +public class NodeFailureRecoveryTest { + + @Mock + private NodeClient nodeClient; + + @Mock + private SecretPartRepository secretPartRepository; + + @Mock + private SecretSharingService secretSharingService; + + @Mock + private PendingActionsBuffer pendingActionsBuffer; + + @Mock + private CommitPublisher commitPublisher; + + private ClusterConfig clusterConfig; + + @BeforeEach + void setUp() { + clusterConfig = new ClusterConfig(); + clusterConfig.setTotalNodes(3); + clusterConfig.setThresholdK(2); + clusterConfig.setQuorumM(3); + clusterConfig.setLockTimeoutMillis(5000L); + clusterConfig.setWriteTimeoutMillis(5000L); + } + + // ══════════════════════════════════════════════════════════════════════ + // POST — Node Failure Scenarios + // ══════════════════════════════════════════════════════════════════════ + + @Test + void testPostAllPeersFailDuringPrepare() { + InternalPostService postService = new InternalPostService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(false); + when(secretSharingService.split(eq(key), eq("value"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + when(nodeClient.sendPostPrepare(anyString(), any(PostPrepareRequest.class))) + .thenAnswer(inv -> PeerResponse.failed(inv.getArgument(0), "connection refused")); + + assertThrows(QuorumNotReachedException.class, + () -> postService.postAcrossCluster(key, "value")); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + verify(commitPublisher, never()).broadcastCommit(any()); + } + + @Test + void testPostPartialAcksBelowQuorum() { + // quorumM=3 → required ACKs = 3. With 1 self + 0 peers = 1 < 3 + InternalPostService postService = new InternalPostService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(false); + when(secretSharingService.split(eq(key), eq("value"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + // One peer ACKs, one fails → total = 2 (self + 1) < required 3 + when(nodeClient.sendPostPrepare(eq("http://peer1:8080"), any())) + .thenReturn(PeerResponse.acknowledged("http://peer1:8080")); + when(nodeClient.sendPostPrepare(eq("http://peer2:8080"), any())) + .thenReturn(PeerResponse.failed("http://peer2:8080", "timeout")); + + assertThrows(QuorumNotReachedException.class, + () -> postService.postAcrossCluster(key, "value")); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + verify(commitPublisher, never()).broadcastCommit(any()); + } + + @Test + void testPostPartialAcksAboveQuorum() { + // quorumM=2 → required = 2. With 1 self + 1 peer = 2 >= 2 + clusterConfig.setQuorumM(2); + InternalPostService postService = new InternalPostService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(false); + when(secretSharingService.split(eq(key), eq("value"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + when(nodeClient.sendPostPrepare(eq("http://peer1:8080"), any())) + .thenReturn(PeerResponse.acknowledged("http://peer1:8080")); + when(nodeClient.sendPostPrepare(eq("http://peer2:8080"), any())) + .thenReturn(PeerResponse.failed("http://peer2:8080", "timeout")); + + SecretVersion version = postService.postAcrossCluster(key, "value"); + + assertEquals(1L, version.getVersion()); + verify(commitPublisher).broadcastCommit(any(CommitMessage.class)); + verify(pendingActionsBuffer, never()).discard(any(UUID.class)); + } + + @Test + void testPostCommitPublishFailure() { + clusterConfig.setQuorumM(1); + InternalPostService postService = new InternalPostService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(false); + when(secretSharingService.split(eq(key), eq("value"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of()); + doThrow(new ServiceUnavailableException("Kafka down")) + .when(commitPublisher).broadcastCommit(any(CommitMessage.class)); + + assertThrows(ServiceUnavailableException.class, + () -> postService.postAcrossCluster(key, "value")); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + } + + // ══════════════════════════════════════════════════════════════════════ + // PUT — Node Failure Scenarios + // ══════════════════════════════════════════════════════════════════════ + + @Test + void testPutPeerFailureDuringPrepare() { + InternalPutService putService = new InternalPutService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + SecretPart existing = new SecretPart(key, 1L, 1, new byte[]{1}); + when(secretPartRepository.findLatest(key)).thenReturn(java.util.Optional.of(existing)); + when(secretSharingService.split(eq(key), eq("updated"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + when(nodeClient.sendPutPrepare(anyString(), any(PutPrepareRequest.class))) + .thenAnswer(inv -> PeerResponse.failed(inv.getArgument(0), "connection refused")); + + assertThrows(QuorumNotReachedException.class, + () -> putService.putAcrossCluster(key, "updated")); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + verify(commitPublisher, never()).broadcastCommit(any()); + } + + @Test + void testPutSecretNotFoundBeforeNetworkCalls() { + InternalPutService putService = new InternalPutService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "no-such-secret"); + when(secretPartRepository.findLatest(key)).thenReturn(java.util.Optional.empty()); + + assertThrows(SecretNotFoundException.class, + () -> putService.putAcrossCluster(key, "updated")); + + verify(nodeClient, never()).resolvePeerUrls(); + verify(pendingActionsBuffer, never()).bufferAction(any(), any(), any(), any()); + } + + @Test + void testPutCommitPublishFailure() { + clusterConfig.setQuorumM(1); + InternalPutService putService = new InternalPutService( + nodeClient, secretPartRepository, secretSharingService, + pendingActionsBuffer, commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + SecretPart existing = new SecretPart(key, 1L, 1, new byte[]{1}); + when(secretPartRepository.findLatest(key)).thenReturn(java.util.Optional.of(existing)); + when(secretSharingService.split(eq(key), eq("updated"), anyInt(), anyInt())) + .thenReturn(parts(key)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of()); + doThrow(new ServiceUnavailableException("Kafka down")) + .when(commitPublisher).broadcastCommit(any(CommitMessage.class)); + + assertThrows(ServiceUnavailableException.class, + () -> putService.putAcrossCluster(key, "updated")); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + } + + // ══════════════════════════════════════════════════════════════════════ + // DELETE — Node Failure Scenarios + // ══════════════════════════════════════════════════════════════════════ + + @Test + void testDeleteAllPeersFailQuorumNotReached() { + InternalDeleteService deleteService = new InternalDeleteService( + nodeClient, secretPartRepository, pendingActionsBuffer, + commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(true); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + when(nodeClient.sendDeletePrepare(anyString(), any(DeletePrepareRequest.class))) + .thenAnswer(inv -> PeerResponse.failed(inv.getArgument(0), "timeout")); + + assertThrows(QuorumNotReachedException.class, + () -> deleteService.deleteAcrossCluster(key)); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + verify(commitPublisher, never()).broadcastCommit(any()); + } + + @Test + void testDeleteSufficientAcksDespiteSomeFailures() { + // m=3, k=2 → required = 2. Self + 1 peer = 2 >= 2 + SecretKey key = new SecretKey("user1", "secret1"); + InternalDeleteService deleteService = new InternalDeleteService( + nodeClient, secretPartRepository, pendingActionsBuffer, + commitPublisher, clusterConfig); + + when(secretPartRepository.exists(key)).thenReturn(true); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080", "http://peer2:8080")); + when(nodeClient.sendDeletePrepare(eq("http://peer1:8080"), any())) + .thenReturn(PeerResponse.acknowledged("http://peer1:8080")); + when(nodeClient.sendDeletePrepare(eq("http://peer2:8080"), any())) + .thenReturn(PeerResponse.failed("http://peer2:8080", "timeout")); + + assertDoesNotThrow(() -> deleteService.deleteAcrossCluster(key)); + + verify(commitPublisher).broadcastCommit(any(CommitMessage.class)); + } + + @Test + void testDeleteCommitPublishFailure() { + clusterConfig.setQuorumM(1); + clusterConfig.setThresholdK(1); + InternalDeleteService deleteService = new InternalDeleteService( + nodeClient, secretPartRepository, pendingActionsBuffer, + commitPublisher, clusterConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + when(secretPartRepository.exists(key)).thenReturn(true); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of()); + doThrow(new ServiceUnavailableException("Kafka down")) + .when(commitPublisher).broadcastCommit(any(CommitMessage.class)); + + assertThrows(ServiceUnavailableException.class, + () -> deleteService.deleteAcrossCluster(key)); + + verify(pendingActionsBuffer).discard(any(UUID.class)); + } + + // ══════════════════════════════════════════════════════════════════════ + // Buffer Eviction on Timeout + // ══════════════════════════════════════════════════════════════════════ + + @Test + void testBufferEvictionCleansUpExpiredEntries() throws InterruptedException { + ClusterConfig shortConfig = new ClusterConfig(); + shortConfig.setLockTimeoutMillis(50L); + PendingActionsBuffer realBuffer = new PendingActionsBuffer(shortConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + UUID opId = UUID.randomUUID(); + realBuffer.bufferAction(opId, key, ActionType.POST, + new SecretPart(key, 1L, 1, new byte[]{1})); + + assertTrue(realBuffer.contains(opId)); + + Thread.sleep(100); + realBuffer.evictExpired(); + + assertFalse(realBuffer.contains(opId)); + assertFalse(realBuffer.containsKey(key)); + } + + @Test + void testBufferEvictionDoesNotRemoveRecentEntries() { + ClusterConfig longConfig = new ClusterConfig(); + longConfig.setLockTimeoutMillis(60_000L); + PendingActionsBuffer realBuffer = new PendingActionsBuffer(longConfig); + + SecretKey key = new SecretKey("user1", "secret1"); + UUID opId = UUID.randomUUID(); + realBuffer.bufferAction(opId, key, ActionType.DELETE); + + realBuffer.evictExpired(); + + assertTrue(realBuffer.contains(opId)); + assertTrue(realBuffer.containsKey(key)); + } + + // ── Helpers ────────────────────────────────────────────────────────── + + private List parts(SecretKey key) { + return List.of( + new SecretPart(key, null, 1, new byte[]{1}), + new SecretPart(key, null, 2, new byte[]{2}), + new SecretPart(key, null, 3, new byte[]{3})); + } +} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitDispatcherTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitDispatcherTest.java new file mode 100644 index 0000000..c1a15d7 --- /dev/null +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitDispatcherTest.java @@ -0,0 +1,146 @@ +package edu.yu.capstone.DistributedSecretsVault.service.communication; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.CommitMessage; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.DeleteCommitRequest; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.PostCommitRequest; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.PutCommitRequest; +import edu.yu.capstone.DistributedSecretsVault.exceptions.InternalOperationConflictException; +import edu.yu.capstone.DistributedSecretsVault.service.internal.ActionType; +import edu.yu.capstone.DistributedSecretsVault.service.internal.DeleteCommitHandler; +import edu.yu.capstone.DistributedSecretsVault.service.internal.PostCommitHandler; +import edu.yu.capstone.DistributedSecretsVault.service.internal.PutCommitHandler; + +/** + * Unit tests for {@link CommitDispatcher}. + * Verifies correct routing to handlers and graceful handling of stale commits. + */ +@ExtendWith(MockitoExtension.class) +@Tag("unit") +public class CommitDispatcherTest { + + @Mock + private DeleteCommitHandler deleteCommitHandler; + + @Mock + private PostCommitHandler postCommitHandler; + + @Mock + private PutCommitHandler putCommitHandler; + + private CommitDispatcher dispatcher; + + @BeforeEach + void setUp() { + dispatcher = new CommitDispatcher(deleteCommitHandler, postCommitHandler, putCommitHandler); + } + + // ── Routing ───────────────────────────────────────────────────────── + + @Test + void testDispatchesDeleteToDeleteHandler() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.DELETE); + + dispatcher.dispatch(msg); + + verify(deleteCommitHandler).handle(any(DeleteCommitRequest.class)); + verify(postCommitHandler, never()).handle(any()); + verify(putCommitHandler, never()).handle(any()); + } + + @Test + void testDispatchesPostToPostHandler() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.POST); + + dispatcher.dispatch(msg); + + verify(postCommitHandler).handle(any(PostCommitRequest.class)); + verify(deleteCommitHandler, never()).handle(any()); + verify(putCommitHandler, never()).handle(any()); + } + + @Test + void testDispatchesPutToPutHandler() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.PUT); + + dispatcher.dispatch(msg); + + verify(putCommitHandler).handle(any(PutCommitRequest.class)); + verify(deleteCommitHandler, never()).handle(any()); + verify(postCommitHandler, never()).handle(any()); + } + + // ── Stale / Conflicting Commits ───────────────────────────────────── + + @Test + void testStaleCommitIsCaughtAndLogged() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.DELETE); + doThrow(new InternalOperationConflictException("No staged operation found")) + .when(deleteCommitHandler).handle(any(DeleteCommitRequest.class)); + + // Should NOT throw — CommitDispatcher catches InternalOperationConflictException + assertDoesNotThrow(() -> dispatcher.dispatch(msg)); + } + + @Test + void testStalePostCommitIsCaughtAndLogged() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.POST); + doThrow(new InternalOperationConflictException("No staged operation found")) + .when(postCommitHandler).handle(any(PostCommitRequest.class)); + + assertDoesNotThrow(() -> dispatcher.dispatch(msg)); + } + + @Test + void testStalePutCommitIsCaughtAndLogged() { + CommitMessage msg = new CommitMessage( + UUID.randomUUID(), new SecretKey("user1", "secret1"), ActionType.PUT); + doThrow(new InternalOperationConflictException("No staged operation found")) + .when(putCommitHandler).handle(any(PutCommitRequest.class)); + + assertDoesNotThrow(() -> dispatcher.dispatch(msg)); + } + + // ── Validation ────────────────────────────────────────────────────── + + @Test + void testDispatchRejectsNullMessage() { + assertThrows(IllegalArgumentException.class, () -> dispatcher.dispatch(null)); + } + + @Test + void testDispatchRejectsNullOperationId() { + CommitMessage msg = new CommitMessage(null, new SecretKey("u", "s"), ActionType.DELETE); + assertThrows(IllegalArgumentException.class, () -> dispatcher.dispatch(msg)); + } + + @Test + void testDispatchRejectsNullSecretKey() { + CommitMessage msg = new CommitMessage(UUID.randomUUID(), null, ActionType.DELETE); + assertThrows(IllegalArgumentException.class, () -> dispatcher.dispatch(msg)); + } + + @Test + void testDispatchRejectsNullActionType() { + CommitMessage msg = new CommitMessage(UUID.randomUUID(), new SecretKey("u", "s"), null); + assertThrows(IllegalArgumentException.class, () -> dispatcher.dispatch(msg)); + } +} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitPublisherTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitPublisherTest.java new file mode 100644 index 0000000..ad56c06 --- /dev/null +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/communication/CommitPublisherTest.java @@ -0,0 +1,92 @@ +package edu.yu.capstone.DistributedSecretsVault.service.communication; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; + +import edu.yu.capstone.DistributedSecretsVault.config.KafkaConfig; +import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; +import edu.yu.capstone.DistributedSecretsVault.dto.internal.CommitMessage; +import edu.yu.capstone.DistributedSecretsVault.exceptions.ServiceUnavailableException; +import edu.yu.capstone.DistributedSecretsVault.service.internal.ActionType; + +/** + * Unit tests for {@link CommitPublisher}. + * Verifies Kafka publishing behavior including success, failure, and validation. + */ +@ExtendWith(MockitoExtension.class) +@Tag("unit") +public class CommitPublisherTest { + + @Mock + private KafkaTemplate kafkaTemplate; + + private CommitPublisher publisher; + + @BeforeEach + void setUp() { + publisher = new CommitPublisher(kafkaTemplate); + } + + @Test + void testBroadcastCommitSendsToCorrectTopicWithOperationIdAsKey() throws Exception { + UUID operationId = UUID.randomUUID(); + SecretKey secretKey = new SecretKey("user1", "secret1"); + CommitMessage message = new CommitMessage(operationId, secretKey, ActionType.POST); + + @SuppressWarnings("unchecked") + CompletableFuture> future = mock(CompletableFuture.class); + when(kafkaTemplate.send(anyString(), anyString(), any())).thenReturn(future); + doReturn(null).when(future).get(anyLong(), any()); + + assertDoesNotThrow(() -> publisher.broadcastCommit(message)); + + verify(kafkaTemplate).send( + eq(KafkaConfig.COMMIT_TOPIC), + eq(operationId.toString()), + eq(message)); + } + + @Test + void testBroadcastCommitThrowsServiceUnavailableOnKafkaFailure() throws Exception { + UUID operationId = UUID.randomUUID(); + CommitMessage message = new CommitMessage( + operationId, new SecretKey("u", "s"), ActionType.DELETE); + + @SuppressWarnings("unchecked") + CompletableFuture> future = mock(CompletableFuture.class); + when(kafkaTemplate.send(anyString(), anyString(), any())).thenReturn(future); + doThrow(new RuntimeException("Kafka broker unavailable")).when(future).get(anyLong(), any()); + + ServiceUnavailableException ex = assertThrows(ServiceUnavailableException.class, + () -> publisher.broadcastCommit(message)); + + assertTrue(ex.getMessage().contains(operationId.toString())); + } + + @Test + void testBroadcastCommitRejectsNullMessage() { + assertThrows(IllegalArgumentException.class, + () -> publisher.broadcastCommit(null)); + } + + @Test + void testBroadcastCommitRejectsNullOperationId() { + CommitMessage message = new CommitMessage(null, new SecretKey("u", "s"), ActionType.POST); + + assertThrows(IllegalArgumentException.class, + () -> publisher.broadcastCommit(message)); + } +} diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetServiceTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetServiceTest.java index 70fc6c9..c97bed8 100644 --- a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetServiceTest.java +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/internal/InternalGetServiceTest.java @@ -1,12 +1,16 @@ package edu.yu.capstone.DistributedSecretsVault.service.internal; +import edu.yu.capstone.DistributedSecretsVault.config.ClusterConfig; import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretKey; import edu.yu.capstone.DistributedSecretsVault.domain.model.SecretPart; +import edu.yu.capstone.DistributedSecretsVault.exceptions.InsufficientShardsException; import edu.yu.capstone.DistributedSecretsVault.exceptions.SecretNotFoundException; import edu.yu.capstone.DistributedSecretsVault.repository.SecretPartRepository; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient.SecretPartResponse; +import edu.yu.capstone.DistributedSecretsVault.service.internal.NodeClient.SecretPartsResponse; +import edu.yu.capstone.DistributedSecretsVault.service.secret.SecretReconstructionService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.http.HttpStatus; @@ -19,6 +23,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class InternalGetServiceTest { @@ -26,7 +34,13 @@ public class InternalGetServiceTest { @Mock private SecretPartRepository secretPartRepository; - @InjectMocks + @Mock + private SecretReconstructionService secretReconstructionService; + + @Mock + private NodeClient nodeClient; + + private ClusterConfig clusterConfig; private InternalGetService internalGetService; private SecretKey validKey; @@ -34,9 +48,101 @@ public class InternalGetServiceTest { @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); + clusterConfig = new ClusterConfig(); + clusterConfig.setTotalNodes(3); + clusterConfig.setThresholdK(2); + internalGetService = new InternalGetService(secretPartRepository, secretReconstructionService, + nodeClient, clusterConfig); + lenient().when(nodeClient.resolvePeerUrls()).thenReturn(List.of()); validKey = new SecretKey("user1", "secret1"); } + @Test + void testGetAcrossClusterLatestVersion() { + SecretPart part = new SecretPart(validKey, 2L, 1, new byte[] { 1, 2 }); + SecretPart peerPart = new SecretPart(validKey, 2L, 2, new byte[] { 3, 4 }); + when(secretPartRepository.findLatest(validKey)).thenReturn(Optional.of(part)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080")); + when(nodeClient.fetchSecretPart("http://peer1:8080", validKey, null)) + .thenReturn(SecretPartResponse.found("http://peer1:8080", peerPart)); + when(secretReconstructionService.reconstruct(anyList())).thenReturn("reconstructed"); + + String result = internalGetService.getAcrossCluster(validKey, null); + + assertEquals("reconstructed", result); + verify(secretReconstructionService).reconstruct(argThat(parts -> parts.size() == 2)); + } + + @Test + void testGetAcrossClusterSpecificVersion() { + SecretPart part = new SecretPart(validKey, 1L, 1, new byte[] { 1 }); + SecretPart peerPart = new SecretPart(validKey, 1L, 2, new byte[] { 2 }); + when(secretPartRepository.findPart(validKey, 1L)).thenReturn(Optional.of(part)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080")); + when(nodeClient.fetchSecretPart("http://peer1:8080", validKey, 1L)) + .thenReturn(SecretPartResponse.found("http://peer1:8080", peerPart)); + when(secretReconstructionService.reconstruct(anyList())).thenReturn("v1-secret"); + + String result = internalGetService.getAcrossCluster(validKey, 1L); + + assertEquals("v1-secret", result); + } + + @Test + void testGetAcrossClusterRejectsNonExistentKey() { + assertThrows(SecretNotFoundException.class, + () -> internalGetService.getAcrossCluster(validKey, null)); + } + + @Test + void testGetAcrossClusterRejectsNonExistentVersion() { + when(secretPartRepository.findPart(validKey, 99L)).thenReturn(Optional.empty()); + + assertThrows(SecretNotFoundException.class, + () -> internalGetService.getAcrossCluster(validKey, 99L)); + } + + @Test + void testGetAcrossClusterThrowsWhenInsufficientShardsFound() { + when(secretPartRepository.findLatest(validKey)) + .thenReturn(Optional.of(new SecretPart(validKey, 1L, 1, new byte[] { 1 }))); + + assertThrows(InsufficientShardsException.class, + () -> internalGetService.getAcrossCluster(validKey, null)); + } + + @Test + void testGetAllVersionsAcrossClusterReturnsSortedMap() { + SecretPart part1 = new SecretPart(validKey, 1L, 1, new byte[] { 1 }); + SecretPart part2 = new SecretPart(validKey, 2L, 1, new byte[] { 2 }); + SecretPart peerPart1 = new SecretPart(validKey, 1L, 2, new byte[] { 3 }); + SecretPart peerPart2 = new SecretPart(validKey, 2L, 2, new byte[] { 4 }); + when(secretPartRepository.listVersions(validKey)).thenReturn(List.of(2L, 1L)); + when(secretPartRepository.findPart(validKey, 1L)).thenReturn(Optional.of(part1)); + when(secretPartRepository.findPart(validKey, 2L)).thenReturn(Optional.of(part2)); + when(nodeClient.resolvePeerUrls()).thenReturn(List.of("http://peer1:8080")); + when(nodeClient.fetchAllSecretParts("http://peer1:8080", validKey)) + .thenReturn(SecretPartsResponse.found("http://peer1:8080", Map.of( + 1L, peerPart1, + 2L, peerPart2))); + when(secretReconstructionService.reconstruct(anyList())) + .thenReturn("v1-secret", "v2-secret"); + + Map results = internalGetService.getAllVersionsAcrossCluster(validKey); + + assertEquals(2, results.size()); + assertEquals("v1-secret", results.get(1L)); + assertEquals("v2-secret", results.get(2L)); + } + + @Test + void testGetAllVersionsAcrossClusterThrowsWhenEmpty() { + when(secretPartRepository.listVersions(validKey)).thenReturn(List.of()); + + assertThrows(SecretNotFoundException.class, + () -> internalGetService.getAllVersionsAcrossCluster(validKey)); + } + @Test void testGetVersion_WithVersion_Exists() { SecretPart part = new SecretPart(); diff --git a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretServiceTest.java b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretServiceTest.java index 0320372..5b73929 100644 --- a/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretServiceTest.java +++ b/src/test/java/edu/yu/capstone/DistributedSecretsVault/service/secret/GetSecretServiceTest.java @@ -13,6 +13,8 @@ import java.util.HashMap; import java.util.Map; +import edu.yu.capstone.DistributedSecretsVault.service.internal.InternalGetService; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -24,7 +26,7 @@ public class GetSecretServiceTest { @Mock - private SecretService secretService; + private InternalGetService internalGetService; @InjectMocks private GetSecretService getSecretService; @@ -51,7 +53,7 @@ void testValidateThrowsWhenSecretNameBlank() { @Test void testExecuteNoVersionSuccess() { - when(secretService.getSecret(any(SecretKey.class), eq(null))).thenReturn("value1"); + when(internalGetService.getAcrossCluster(any(SecretKey.class), eq(null))).thenReturn("value1"); ResponseEntity response = getSecretService.getVersion("user1", "secret1", null); assertEquals(HttpStatus.OK, response.getStatusCode()); @@ -60,7 +62,7 @@ void testExecuteNoVersionSuccess() { @Test void testGetVersionSuccess() { - when(secretService.getSecret(any(SecretKey.class), eq(2L))).thenReturn("value2"); + when(internalGetService.getAcrossCluster(any(SecretKey.class), eq(2L))).thenReturn("value2"); ResponseEntity response = getSecretService.getVersion("user1", "secret1", 2L); assertEquals(HttpStatus.OK, response.getStatusCode()); @@ -72,7 +74,7 @@ void testGetAllVersionsSuccess() { Map expected = new HashMap<>(); expected.put(1L, "v1"); expected.put(2L, "v2"); - when(secretService.getAllVersions(any(SecretKey.class))).thenReturn(expected); + when(internalGetService.getAllVersionsAcrossCluster(any(SecretKey.class))).thenReturn(expected); ResponseEntity> response = getSecretService.getAllVersions("user1", "secret1");