From ae7bb716fb6e8631d75c25becf2bf56ef10f01e5 Mon Sep 17 00:00:00 2001 From: danielshih Date: Sat, 25 Apr 2026 11:17:54 +0000 Subject: [PATCH 1/5] Add chaos testing scripts and verification scenarios for SQLite and SQL Server - Introduced `scenario5_verify.sql` to verify high-volume data replication (3M rows) in SQL Server. - Created `run_chaos_tests_sqlite.sh` for executing chaos tests with SQLite as the destination. - Developed `run_chaos_tests_sqlserver.sh` for executing chaos tests with SQL Server as the destination. - Implemented `run_pgbench_chaos_test_sqlite.sh` for performance testing with pgbench on SQLite. - Implemented `run_pgbench_chaos_test_sqlserver.sh` for performance testing with pgbench on SQL Server. - Added logging and cleanup functionalities to all scripts for better traceability and resource management. --- .../chaos_integration_test_sqlite.yml | 193 +++++++++ .../chaos_integration_test_sqlserver.yml | 214 ++++++++++ .../pgbench_integration_test_sqlite.yml | 237 +++++++++++ .../pgbench_integration_test_sqlserver.yml | 265 ++++++++++++ Dockerfile | 13 +- Makefile | 231 +++++++++-- docker-compose.chaos-test-sqlite.yml | 73 ++++ docker-compose.chaos-test-sqlserver.yml | 89 +++++ env/.env.sqlite | 35 ++ env/.env.sqlserver | 37 ++ examples/Cargo.toml | 9 +- examples/scripts/init_sqlite.sql | 6 + examples/scripts/init_sqlserver.sql | 15 + .../verify_sqlite/scenario1_verify.sql | 9 + .../verify_sqlite/scenario2_verify.sql | 10 + .../verify_sqlite/scenario3_verify.sql | 9 + .../verify_sqlite/scenario4_verify.sql | 10 + .../verify_sqlite/scenario5_verify.sql | 14 + .../verify_sqlserver/scenario1_verify.sql | 10 + .../verify_sqlserver/scenario2_verify.sql | 11 + .../verify_sqlserver/scenario3_verify.sql | 10 + .../verify_sqlserver/scenario4_verify.sql | 11 + .../verify_sqlserver/scenario5_verify.sql | 15 + tests/chaos/scripts/run_chaos_tests_sqlite.sh | 284 +++++++++++++ .../scripts/run_chaos_tests_sqlserver.sh | 329 +++++++++++++++ .../scripts/run_pgbench_chaos_test_sqlite.sh | 348 ++++++++++++++++ .../run_pgbench_chaos_test_sqlserver.sh | 378 ++++++++++++++++++ 27 files changed, 2832 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/chaos_integration_test_sqlite.yml create mode 100644 .github/workflows/chaos_integration_test_sqlserver.yml create mode 100644 .github/workflows/pgbench_integration_test_sqlite.yml create mode 100644 .github/workflows/pgbench_integration_test_sqlserver.yml create mode 100644 docker-compose.chaos-test-sqlite.yml create mode 100644 docker-compose.chaos-test-sqlserver.yml create mode 100644 env/.env.sqlite create mode 100644 env/.env.sqlserver create mode 100644 examples/scripts/init_sqlite.sql create mode 100644 examples/scripts/init_sqlserver.sql create mode 100644 tests/chaos/scenarios/verify_sqlite/scenario1_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlite/scenario2_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlite/scenario3_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlite/scenario4_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlserver/scenario1_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlserver/scenario2_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlserver/scenario3_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlserver/scenario4_verify.sql create mode 100644 tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql create mode 100755 tests/chaos/scripts/run_chaos_tests_sqlite.sh create mode 100755 tests/chaos/scripts/run_chaos_tests_sqlserver.sh create mode 100755 tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh create mode 100755 tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh diff --git a/.github/workflows/chaos_integration_test_sqlite.yml b/.github/workflows/chaos_integration_test_sqlite.yml new file mode 100644 index 0000000..122bd0b --- /dev/null +++ b/.github/workflows/chaos_integration_test_sqlite.yml @@ -0,0 +1,193 @@ +name: CDC Chaos Integration Test (SQLite) + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + chaos-integration-test-sqlite: + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + compression: ["true", "false"] + fail-fast: false + + runs-on: ${{ matrix.os }} + + timeout-minutes: 360 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + + - name: Install PostgreSQL client tools + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client-common postgresql-client + + - name: Verify test structure + run: | + echo "Verifying chaos test structure..." + ls -la tests/chaos/ + ls -la tests/chaos/scripts/ + ls -la tests/chaos/scenarios/verify_sqlite/ + + - name: Build CDC application image + run: | + echo "Building CDC application image..." + docker-compose -f docker-compose.chaos-test-sqlite.yml build cdc_app + working-directory: . + + - name: Start Docker Compose services + run: | + echo "Starting PostgreSQL..." + docker-compose -f docker-compose.chaos-test-sqlite.yml up -d postgres + working-directory: . + + - name: Wait for PostgreSQL to be healthy + run: | + echo "Waiting for PostgreSQL to be ready..." + timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + working-directory: . + + - name: Verify PostgreSQL is ready + run: | + echo "Verifying PostgreSQL connection..." + docker exec cdc_postgres pg_isready -U postgres + docker exec cdc_postgres psql -U postgres -c "SELECT version();" + + - name: Start CDC application + run: | + echo "Starting CDC application..." + docker-compose -f docker-compose.chaos-test-sqlite.yml up -d cdc_app + echo "Waiting for CDC application to initialize..." + timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' || true + docker-compose -f docker-compose.chaos-test-sqlite.yml ps + working-directory: . + + - name: Initialize SQLite database schema + run: | + echo "Initializing SQLite database schema..." + docker exec cdc_application sqlite3 /app/data/cdc_target.db ".read /init/init_sqlite.sql" + echo "Verifying SQLite table..." + docker exec cdc_application sqlite3 /app/data/cdc_target.db ".tables" + + - name: Verify CDC application is running + run: | + echo "Verifying CDC application..." + docker ps -a | grep cdc_application + docker logs cdc_application | tail -n 50 + + - name: Make scripts executable + run: | + chmod +x tests/chaos/scripts/*.sh + ls -la tests/chaos/scripts/ + + - name: Run chaos integration tests + run: | + echo "==========================================" + echo "Starting CDC Chaos Integration Tests (SQLite)" + echo "==========================================" + echo "OS: ${{ matrix.os }}" + echo "PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }}" + echo "" + + cd tests/chaos/scripts + ./run_chaos_tests_sqlite.sh + env: + CDC_POSTGRES_HOST: 127.0.0.1 + CDC_POSTGRES_PORT: 5432 + CDC_POSTGRES_USER: postgres + CDC_POSTGRES_PASSWORD: test.123 + CDC_POSTGRES_DB: postgres + CDC_CONTAINER_NAME: cdc_application + PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }} + + - name: Show PostgreSQL logs on failure + if: failure() + run: | + echo "==========================================" + echo "PostgreSQL Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_postgres --tail 100 + + - name: Show container status on failure + if: failure() + run: | + echo "==========================================" + echo "Docker Container Status" + echo "==========================================" + docker-compose -f docker-compose.chaos-test-sqlite.yml ps + docker ps -a + + - name: Capture and upload CDC application logs + if: failure() + run: | + echo "Installing colorized-logs for log conversion..." + sudo apt-get update + sudo apt-get install -y colorized-logs || sudo apt-get install -y ansifilter + echo "Capturing full CDC application logs..." + if command -v ansi2txt &> /dev/null; then + docker logs cdc_application | ansi2txt > cdc_application_log.txt + elif command -v ansifilter &> /dev/null; then + docker logs cdc_application | ansifilter > cdc_application_log.txt + else + docker logs cdc_application > cdc_application_log.txt + fi + echo "Log file created: cdc_application_log.txt" + ls -lh cdc_application_log.txt + + - name: Upload CDC application logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cdc-sqlite-chaos-logs-${{ matrix.os }}-${{ matrix.compression }} + path: cdc_application_log.txt + retention-days: 30 + + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + + - name: Stop and remove containers + if: always() + run: | + echo "Cleaning up Docker containers..." + docker-compose -f docker-compose.chaos-test-sqlite.yml down -v + docker volume rm chaos_test_sqlite_data 2>/dev/null || true + docker network rm chaos_test_network 2>/dev/null || true + docker system prune -f + working-directory: . + + - name: Show disk usage + if: always() + run: | + df -h + docker system df + + notify: + needs: chaos-integration-test-sqlite + runs-on: ubuntu-latest + if: always() + steps: + - name: Test result summary + run: | + echo "==========================================" + echo "SQLite Chaos Integration Test Summary" + echo "==========================================" + echo "Job status: ${{ needs.chaos-integration-test-sqlite.result }}" + if [ "${{ needs.chaos-integration-test-sqlite.result }}" = "success" ]; then + echo "All SQLite chaos integration tests passed!" + else + echo "Some SQLite chaos integration tests failed." + fi diff --git a/.github/workflows/chaos_integration_test_sqlserver.yml b/.github/workflows/chaos_integration_test_sqlserver.yml new file mode 100644 index 0000000..a71c279 --- /dev/null +++ b/.github/workflows/chaos_integration_test_sqlserver.yml @@ -0,0 +1,214 @@ +name: CDC Chaos Integration Test (SQL Server) + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + chaos-integration-test-sqlserver: + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + compression: ["true", "false"] + fail-fast: false + + runs-on: ${{ matrix.os }} + + timeout-minutes: 480 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + + - name: Install PostgreSQL client tools + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client-common postgresql-client + + - name: Verify test structure + run: | + echo "Verifying chaos test structure..." + ls -la tests/chaos/ + ls -la tests/chaos/scripts/ + ls -la tests/chaos/scenarios/verify_sqlserver/ + + - name: Build CDC application image + run: | + echo "Building CDC application image..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml build cdc_app + working-directory: . + + - name: Start PostgreSQL and SQL Server + run: | + echo "Starting PostgreSQL and SQL Server..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d postgres sqlserver + working-directory: . + + - name: Wait for services to be healthy + run: | + echo "Waiting for PostgreSQL to be ready..." + timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + echo "Waiting for SQL Server to be ready..." + timeout 180 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + working-directory: . + + - name: Verify PostgreSQL is ready + run: | + echo "Verifying PostgreSQL connection..." + docker exec cdc_postgres pg_isready -U postgres + docker exec cdc_postgres psql -U postgres -c "SELECT version();" + + - name: Initialize SQL Server database + run: | + echo "Detecting sqlcmd path..." + if docker exec cdc_sqlserver test -x /opt/mssql-tools18/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools18/bin/sqlcmd + elif docker exec cdc_sqlserver test -x /opt/mssql-tools/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools/bin/sqlcmd + else + echo "sqlcmd not found!" && exit 1 + fi + echo "Using sqlcmd: $SQLCMD" + echo "Initializing SQL Server database..." + docker exec cdc_sqlserver $SQLCMD -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + echo "Verifying SQL Server table..." + docker exec cdc_sqlserver $SQLCMD -S localhost -U sa -P 'Test.123!' -C -d cdc_db -Q "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';" + + - name: Start CDC application + run: | + echo "Starting CDC application..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app + echo "Waiting for CDC application to initialize..." + timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' || true + working-directory: . + + - name: Verify CDC application is running + run: | + echo "Verifying CDC application..." + docker ps -a | grep cdc_application + docker logs cdc_application | tail -n 50 + + - name: Make scripts executable + run: | + chmod +x tests/chaos/scripts/*.sh + ls -la tests/chaos/scripts/ + + - name: Run chaos integration tests + run: | + echo "==========================================" + echo "Starting CDC Chaos Integration Tests (SQL Server)" + echo "==========================================" + echo "OS: ${{ matrix.os }}" + echo "PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }}" + echo "" + + cd tests/chaos/scripts + ./run_chaos_tests_sqlserver.sh + env: + CDC_POSTGRES_HOST: 127.0.0.1 + CDC_POSTGRES_PORT: 5432 + CDC_POSTGRES_USER: postgres + CDC_POSTGRES_PASSWORD: test.123 + CDC_POSTGRES_DB: postgres + CDC_SQLSERVER_CONTAINER: cdc_sqlserver + CDC_SQLSERVER_PASSWORD: "Test.123!" + CDC_SQLSERVER_DB: cdc_db + CDC_CONTAINER_NAME: cdc_application + PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }} + + - name: Show PostgreSQL logs on failure + if: failure() + run: | + echo "==========================================" + echo "PostgreSQL Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_postgres --tail 100 + + - name: Show SQL Server logs on failure + if: failure() + run: | + echo "==========================================" + echo "SQL Server Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_sqlserver --tail 100 + + - name: Show container status on failure + if: failure() + run: | + echo "==========================================" + echo "Docker Container Status" + echo "==========================================" + docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + docker ps -a + + - name: Capture and upload CDC application logs + if: failure() + run: | + echo "Installing colorized-logs for log conversion..." + sudo apt-get update + sudo apt-get install -y colorized-logs || sudo apt-get install -y ansifilter + echo "Capturing full CDC application logs..." + if command -v ansi2txt &> /dev/null; then + docker logs cdc_application | ansi2txt > cdc_application_log.txt + elif command -v ansifilter &> /dev/null; then + docker logs cdc_application | ansifilter > cdc_application_log.txt + else + docker logs cdc_application > cdc_application_log.txt + fi + echo "Log file created: cdc_application_log.txt" + ls -lh cdc_application_log.txt + + - name: Upload CDC application logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cdc-sqlserver-chaos-logs-${{ matrix.os }}-${{ matrix.compression }} + path: cdc_application_log.txt + retention-days: 30 + + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + + - name: Stop and remove containers + if: always() + run: | + echo "Cleaning up Docker containers..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml down -v + docker network rm chaos_test_network 2>/dev/null || true + docker system prune -f + working-directory: . + + - name: Show disk usage + if: always() + run: | + df -h + docker system df + + notify: + needs: chaos-integration-test-sqlserver + runs-on: ubuntu-latest + if: always() + steps: + - name: Test result summary + run: | + echo "==========================================" + echo "SQL Server Chaos Integration Test Summary" + echo "==========================================" + echo "Job status: ${{ needs.chaos-integration-test-sqlserver.result }}" + if [ "${{ needs.chaos-integration-test-sqlserver.result }}" = "success" ]; then + echo "All SQL Server chaos integration tests passed!" + else + echo "Some SQL Server chaos integration tests failed." + fi diff --git a/.github/workflows/pgbench_integration_test_sqlite.yml b/.github/workflows/pgbench_integration_test_sqlite.yml new file mode 100644 index 0000000..e8bfdb1 --- /dev/null +++ b/.github/workflows/pgbench_integration_test_sqlite.yml @@ -0,0 +1,237 @@ +name: PGBench Chaos Integration Test (SQLite) + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +jobs: + pgbench-integration-test-sqlite: + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + compression: ["true", "false"] + fail-fast: false + + runs-on: ${{ matrix.os }} + + timeout-minutes: 480 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + + - name: Install PostgreSQL client tools (for pgbench) + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client-common postgresql-client + pgbench --version + + - name: Verify test structure + run: | + echo "Verifying pgbench test structure..." + ls -la tests/chaos/scripts/ + test -f tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh || (echo "PGBench SQLite test script not found!" && exit 1) + test -f tests/chaos/scripts/chaos_script.sh || (echo "Chaos script not found!" && exit 1) + + - name: Build CDC application image + run: | + echo "Building CDC application image..." + docker-compose -f docker-compose.chaos-test-sqlite.yml build cdc_app + working-directory: . + + - name: Start PostgreSQL + run: | + echo "Starting PostgreSQL..." + docker-compose -f docker-compose.chaos-test-sqlite.yml up -d postgres + working-directory: . + + - name: Wait for PostgreSQL to be healthy + run: | + echo "Waiting for PostgreSQL to be ready..." + timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + working-directory: . + + - name: Verify PostgreSQL is ready + run: | + echo "Verifying PostgreSQL connection..." + docker exec cdc_postgres pg_isready -U postgres + docker exec cdc_postgres psql -U postgres -c "SELECT version();" + docker exec cdc_postgres psql -U postgres -c "SHOW wal_level;" + + - name: Start CDC application + run: | + echo "Starting CDC application..." + docker-compose -f docker-compose.chaos-test-sqlite.yml up -d cdc_app + echo "Waiting for CDC application to initialize..." + timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' || true + docker-compose -f docker-compose.chaos-test-sqlite.yml ps + working-directory: . + + - name: Initialize SQLite database schema + run: | + echo "Initializing SQLite database schema..." + docker exec cdc_application sqlite3 /app/data/cdc_target.db ".read /init/init_sqlite.sql" + echo "Verifying SQLite table..." + docker exec cdc_application sqlite3 /app/data/cdc_target.db ".tables" + + - name: Make scripts executable + run: | + chmod +x tests/chaos/scripts/*.sh + ls -la tests/chaos/scripts/ + + - name: Run pgbench chaos integration test + run: | + echo "==========================================" + echo "Starting PGBench Chaos Integration Test (SQLite)" + echo "==========================================" + echo "OS: ${{ matrix.os }}" + echo "PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }}" + echo "" + + cd tests/chaos/scripts + ./run_pgbench_chaos_test_sqlite.sh + env: + CDC_POSTGRES_HOST: 127.0.0.1 + CDC_POSTGRES_PORT: 5432 + CDC_POSTGRES_USER: postgres + CDC_POSTGRES_PASSWORD: test.123 + CDC_POSTGRES_DB: postgres + CDC_CONTAINER_NAME: cdc_application + PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }} + PGBENCH_SCALE: 32 + PGBENCH_CLIENTS: 100 + PGBENCH_THREADS: 8 + PGBENCH_TRANSACTIONS: 12 + + - name: Show pgbench results + if: always() + run: | + echo "==========================================" + echo "PGBench Test Results (SQLite)" + echo "==========================================" + if ls tests/chaos/scripts/pgbench_chaos_sqlite_results_*.log 1>/dev/null 2>&1; then + cat tests/chaos/scripts/pgbench_chaos_sqlite_results_*.log + else + echo "No results file found" + fi + + - name: Show chaos script logs + if: always() + run: | + echo "==========================================" + echo "Chaos Script Activity Log" + echo "==========================================" + if [ -f tests/chaos/scripts/pgbench_chaos_script.log ]; then + cat tests/chaos/scripts/pgbench_chaos_script.log + else + echo "No chaos script log found" + fi + + - name: Verify replication statistics + if: always() + run: | + echo "==========================================" + echo "Final Replication Statistics" + echo "==========================================" + echo "PostgreSQL row count:" + docker exec cdc_postgres psql -U postgres -c "SELECT COUNT(*) FROM t1;" + echo "" + echo "SQLite row count:" + docker exec cdc_application sqlite3 /app/data/cdc_target.db "SELECT COUNT(*) FROM t1;" + + - name: Show PostgreSQL logs on failure + if: failure() + run: | + echo "==========================================" + echo "PostgreSQL Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_postgres --tail 100 + + - name: Show container status on failure + if: failure() + run: | + echo "==========================================" + echo "Docker Container Status" + echo "==========================================" + docker-compose -f docker-compose.chaos-test-sqlite.yml ps + docker ps -a + + - name: Capture and upload CDC application logs + if: failure() + run: | + echo "Installing colorized-logs for log conversion..." + sudo apt-get update + sudo apt-get install -y colorized-logs || sudo apt-get install -y ansifilter + echo "Capturing full CDC application logs..." + if command -v ansi2txt &> /dev/null; then + docker logs cdc_application | ansi2txt > cdc_application_log.txt + elif command -v ansifilter &> /dev/null; then + docker logs cdc_application | ansifilter > cdc_application_log.txt + else + docker logs cdc_application > cdc_application_log.txt + fi + echo "Log file created: cdc_application_log.txt" + ls -lh cdc_application_log.txt + + - name: Upload CDC application logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cdc-pgbench-sqlite-logs-${{ matrix.os }}-${{ matrix.compression }} + path: cdc_application_log.txt + retention-days: 30 + + - name: Upload pgbench results + if: always() + uses: actions/upload-artifact@v4 + with: + name: pgbench-sqlite-results-${{ matrix.os }}-${{ matrix.compression }} + path: tests/chaos/scripts/pgbench_chaos_sqlite_results_*.log + retention-days: 30 + + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + + - name: Stop and remove containers + if: always() + run: | + echo "Cleaning up Docker containers..." + docker-compose -f docker-compose.chaos-test-sqlite.yml down -v + docker volume rm chaos_test_sqlite_data 2>/dev/null || true + docker network rm chaos_test_network 2>/dev/null || true + docker system prune -f + working-directory: . + + - name: Show disk usage + if: always() + run: | + df -h + docker system df + + notify: + needs: pgbench-integration-test-sqlite + runs-on: ubuntu-latest + if: always() + steps: + - name: Test result summary + run: | + echo "==========================================" + echo "PGBench SQLite Integration Test Summary" + echo "==========================================" + echo "Job status: ${{ needs.pgbench-integration-test-sqlite.result }}" + if [ "${{ needs.pgbench-integration-test-sqlite.result }}" = "success" ]; then + echo "PGBench SQLite chaos integration test passed!" + else + echo "PGBench SQLite chaos integration test failed." + fi diff --git a/.github/workflows/pgbench_integration_test_sqlserver.yml b/.github/workflows/pgbench_integration_test_sqlserver.yml new file mode 100644 index 0000000..9f2ea7f --- /dev/null +++ b/.github/workflows/pgbench_integration_test_sqlserver.yml @@ -0,0 +1,265 @@ +name: PGBench Chaos Integration Test (SQL Server) + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +jobs: + pgbench-integration-test-sqlserver: + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + compression: ["true", "false"] + fail-fast: false + + runs-on: ${{ matrix.os }} + + timeout-minutes: 480 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + + - name: Install PostgreSQL client tools (for pgbench) + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client-common postgresql-client + pgbench --version + + - name: Verify test structure + run: | + echo "Verifying pgbench test structure..." + ls -la tests/chaos/scripts/ + test -f tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh || (echo "PGBench SQL Server test script not found!" && exit 1) + test -f tests/chaos/scripts/chaos_script.sh || (echo "Chaos script not found!" && exit 1) + + - name: Build CDC application image + run: | + echo "Building CDC application image..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml build cdc_app + working-directory: . + + - name: Start PostgreSQL and SQL Server + run: | + echo "Starting PostgreSQL and SQL Server..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d postgres sqlserver + working-directory: . + + - name: Wait for services to be healthy + run: | + echo "Waiting for PostgreSQL to be ready..." + timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + echo "Waiting for SQL Server to be ready..." + timeout 180 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + working-directory: . + + - name: Verify PostgreSQL is ready + run: | + echo "Verifying PostgreSQL connection..." + docker exec cdc_postgres pg_isready -U postgres + docker exec cdc_postgres psql -U postgres -c "SELECT version();" + docker exec cdc_postgres psql -U postgres -c "SHOW wal_level;" + + - name: Initialize SQL Server database + run: | + echo "Detecting sqlcmd path..." + if docker exec cdc_sqlserver test -x /opt/mssql-tools18/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools18/bin/sqlcmd + elif docker exec cdc_sqlserver test -x /opt/mssql-tools/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools/bin/sqlcmd + else + echo "sqlcmd not found!" && exit 1 + fi + echo "Using sqlcmd: $SQLCMD" + echo "Initializing SQL Server database..." + docker exec cdc_sqlserver $SQLCMD -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + echo "Verifying SQL Server table..." + docker exec cdc_sqlserver $SQLCMD -S localhost -U sa -P 'Test.123!' -C -d cdc_db -Q "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';" + + - name: Start CDC application + run: | + echo "Starting CDC application..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app + echo "Waiting for CDC application to initialize..." + timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' || true + working-directory: . + + - name: Make scripts executable + run: | + chmod +x tests/chaos/scripts/*.sh + ls -la tests/chaos/scripts/ + + - name: Run pgbench chaos integration test + run: | + echo "==========================================" + echo "Starting PGBench Chaos Integration Test (SQL Server)" + echo "==========================================" + echo "OS: ${{ matrix.os }}" + echo "PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }}" + echo "" + + cd tests/chaos/scripts + ./run_pgbench_chaos_test_sqlserver.sh + env: + CDC_POSTGRES_HOST: 127.0.0.1 + CDC_POSTGRES_PORT: 5432 + CDC_POSTGRES_USER: postgres + CDC_POSTGRES_PASSWORD: test.123 + CDC_POSTGRES_DB: postgres + CDC_SQLSERVER_CONTAINER: cdc_sqlserver + CDC_SQLSERVER_PASSWORD: "Test.123!" + CDC_SQLSERVER_DB: cdc_db + CDC_CONTAINER_NAME: cdc_application + PG2ANY_ENABLE_COMPRESSION: ${{ matrix.compression }} + PGBENCH_SCALE: 32 + PGBENCH_CLIENTS: 100 + PGBENCH_THREADS: 8 + PGBENCH_TRANSACTIONS: 12 + + - name: Show pgbench results + if: always() + run: | + echo "==========================================" + echo "PGBench Test Results (SQL Server)" + echo "==========================================" + if ls tests/chaos/scripts/pgbench_chaos_sqlserver_results_*.log 1>/dev/null 2>&1; then + cat tests/chaos/scripts/pgbench_chaos_sqlserver_results_*.log + else + echo "No results file found" + fi + + - name: Show chaos script logs + if: always() + run: | + echo "==========================================" + echo "Chaos Script Activity Log" + echo "==========================================" + if [ -f tests/chaos/scripts/pgbench_chaos_script.log ]; then + cat tests/chaos/scripts/pgbench_chaos_script.log + else + echo "No chaos script log found" + fi + + - name: Verify replication statistics + if: always() + run: | + echo "==========================================" + echo "Final Replication Statistics" + echo "==========================================" + echo "PostgreSQL row count:" + docker exec cdc_postgres psql -U postgres -c "SELECT COUNT(*) FROM t1;" + echo "" + echo "SQL Server row count:" + if docker exec cdc_sqlserver test -x /opt/mssql-tools18/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools18/bin/sqlcmd + elif docker exec cdc_sqlserver test -x /opt/mssql-tools/bin/sqlcmd; then + SQLCMD=/opt/mssql-tools/bin/sqlcmd + else + echo "sqlcmd not found" && exit 0 + fi + docker exec cdc_sqlserver $SQLCMD -S localhost -U sa -P 'Test.123!' -C -d cdc_db -Q "SELECT COUNT(*) FROM dbo.t1;" + + - name: Show PostgreSQL logs on failure + if: failure() + run: | + echo "==========================================" + echo "PostgreSQL Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_postgres --tail 100 + + - name: Show SQL Server logs on failure + if: failure() + run: | + echo "==========================================" + echo "SQL Server Logs (Last 100 lines)" + echo "==========================================" + docker logs cdc_sqlserver --tail 100 + + - name: Show container status on failure + if: failure() + run: | + echo "==========================================" + echo "Docker Container Status" + echo "==========================================" + docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + docker ps -a + + - name: Capture and upload CDC application logs + if: failure() + run: | + echo "Installing colorized-logs for log conversion..." + sudo apt-get update + sudo apt-get install -y colorized-logs || sudo apt-get install -y ansifilter + echo "Capturing full CDC application logs..." + if command -v ansi2txt &> /dev/null; then + docker logs cdc_application | ansi2txt > cdc_application_log.txt + elif command -v ansifilter &> /dev/null; then + docker logs cdc_application | ansifilter > cdc_application_log.txt + else + docker logs cdc_application > cdc_application_log.txt + fi + echo "Log file created: cdc_application_log.txt" + ls -lh cdc_application_log.txt + + - name: Upload CDC application logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cdc-pgbench-sqlserver-logs-${{ matrix.os }}-${{ matrix.compression }} + path: cdc_application_log.txt + retention-days: 30 + + - name: Upload pgbench results + if: always() + uses: actions/upload-artifact@v4 + with: + name: pgbench-sqlserver-results-${{ matrix.os }}-${{ matrix.compression }} + path: tests/chaos/scripts/pgbench_chaos_sqlserver_results_*.log + retention-days: 30 + + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + + - name: Stop and remove containers + if: always() + run: | + echo "Cleaning up Docker containers..." + docker-compose -f docker-compose.chaos-test-sqlserver.yml down -v + docker network rm chaos_test_network 2>/dev/null || true + docker system prune -f + working-directory: . + + - name: Show disk usage + if: always() + run: | + df -h + docker system df + + notify: + needs: pgbench-integration-test-sqlserver + runs-on: ubuntu-latest + if: always() + steps: + - name: Test result summary + run: | + echo "==========================================" + echo "PGBench SQL Server Integration Test Summary" + echo "==========================================" + echo "Job status: ${{ needs.pgbench-integration-test-sqlserver.result }}" + if [ "${{ needs.pgbench-integration-test-sqlserver.result }}" = "success" ]; then + echo "PGBench SQL Server chaos integration test passed!" + else + echo "PGBench SQL Server chaos integration test failed." + fi diff --git a/Dockerfile b/Dockerfile index 768eedd..1f9e40e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # Use multi-stage build for smaller final image # Build stage -FROM rust:1.88-slim as builder +FROM rust:1.88-slim AS builder + +# Destination features to compile (default: mysql,metrics) +ARG DEST_FEATURES=mysql,metrics # Install build dependencies RUN apt-get update && apt-get install -y \ @@ -20,8 +23,8 @@ COPY pg2any-lib/ ./pg2any-lib/ COPY examples/ ./examples/ WORKDIR /app/examples -# Build the application -RUN cargo build --release +# Build the application with selected destination features +RUN cargo build --release --no-default-features --features "${DEST_FEATURES}" # Runtime stage FROM debian:bookworm-slim @@ -31,10 +34,14 @@ RUN apt-get update && apt-get install -y \ curl \ ca-certificates \ libpq-dev \ + sqlite3 \ && rm -rf /var/lib/apt/lists/* WORKDIR /app +# Create data directory for SQLite and other file-based destinations +RUN mkdir -p /app/data && chown 1001:root /app/data + # Create non-root user RUN useradd -r -u 1001 -g root pg2any_user diff --git a/Makefile b/Makefile index 43612c2..0932b22 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,13 @@ # PostgreSQL CDC Makefile # Provides convenient commands for development and deployment -.PHONY: help build start stop restart clean logs test check format docker-build docker-start docker-stop docker-clean chaos-test chaos-test-setup chaos-test-clean pgbench-test pgbench-test-setup pgbench-test-clean +.PHONY: help build start stop restart clean logs test check format docker-build docker-start docker-stop docker-clean \ + chaos-test-mysql-setup chaos-test-mysql chaos-test-mysql-clean chaos-test-mysql-full \ + pgbench-test-mysql-setup pgbench-test-mysql pgbench-test-mysql-clean pgbench-test-mysql-full \ + chaos-test-sqlserver-setup chaos-test-sqlserver chaos-test-sqlserver-clean chaos-test-sqlserver-full \ + pgbench-test-sqlserver-setup pgbench-test-sqlserver pgbench-test-sqlserver-clean pgbench-test-sqlserver-full \ + chaos-test-sqlite-setup chaos-test-sqlite chaos-test-sqlite-clean chaos-test-sqlite-full \ + pgbench-test-sqlite-setup pgbench-test-sqlite pgbench-test-sqlite-clean pgbench-test-sqlite-full # Default target help: @@ -29,15 +35,41 @@ help: @echo " test-data Insert test data" @echo " show-data Show data in both databases" @echo "" - @echo "Chaos Testing:" - @echo " chaos-test-setup Set up and start services for chaos testing" - @echo " chaos-test Run chaos integration tests" - @echo " chaos-test-clean Clean up after chaos testing" + @echo "MySQL Chaos Testing:" + @echo " chaos-test-mysql-setup Set up and start services for MySQL chaos testing" + @echo " chaos-test-mysql Run chaos integration tests against MySQL" + @echo " chaos-test-mysql-clean Clean up after MySQL chaos testing" + @echo " chaos-test-mysql-full Full MySQL chaos test cycle" @echo "" - @echo "PGBench Testing:" - @echo " pgbench-test-setup Set up and start services for pgbench testing" - @echo " pgbench-test Run pgbench chaos integration tests" - @echo " pgbench-test-clean Clean up after pgbench testing" + @echo "MySQL PGBench Testing:" + @echo " pgbench-test-mysql-setup Set up and start services for MySQL pgbench testing" + @echo " pgbench-test-mysql Run pgbench chaos integration tests against MySQL" + @echo " pgbench-test-mysql-clean Clean up after MySQL pgbench testing" + @echo " pgbench-test-mysql-full Full MySQL pgbench test cycle" + @echo "" + @echo "SQL Server Chaos Testing:" + @echo " chaos-test-sqlserver-setup Set up SQL Server chaos testing environment" + @echo " chaos-test-sqlserver Run chaos tests against SQL Server" + @echo " chaos-test-sqlserver-clean Clean up SQL Server chaos testing" + @echo " chaos-test-sqlserver-full Full SQL Server chaos test cycle" + @echo "" + @echo "SQL Server PGBench Testing:" + @echo " pgbench-test-sqlserver-setup Set up SQL Server pgbench testing environment" + @echo " pgbench-test-sqlserver Run pgbench tests against SQL Server" + @echo " pgbench-test-sqlserver-clean Clean up SQL Server pgbench testing" + @echo " pgbench-test-sqlserver-full Full SQL Server pgbench test cycle" + @echo "" + @echo "SQLite Chaos Testing:" + @echo " chaos-test-sqlite-setup Set up SQLite chaos testing environment" + @echo " chaos-test-sqlite Run chaos tests against SQLite" + @echo " chaos-test-sqlite-clean Clean up SQLite chaos testing" + @echo " chaos-test-sqlite-full Full SQLite chaos test cycle" + @echo "" + @echo "SQLite PGBench Testing:" + @echo " pgbench-test-sqlite-setup Set up SQLite pgbench testing environment" + @echo " pgbench-test-sqlite Run pgbench tests against SQLite" + @echo " pgbench-test-sqlite-clean Clean up SQLite pgbench testing" + @echo " pgbench-test-sqlite-full Full SQLite pgbench test cycle" @echo "" # Development commands @@ -96,57 +128,198 @@ mysql: clean: cargo clean -# Chaos Testing commands -chaos-test-setup: - @echo "Setting up chaos testing environment..." +# MySQL Chaos Testing commands +chaos-test-mysql-setup: + @echo "Setting up MySQL chaos testing environment..." @chmod +x tests/chaos/scripts/*.sh @docker-compose -f docker-compose.chaos-test.yml up --build -d @echo "Waiting for services to be healthy..." @docker-compose -f docker-compose.chaos-test.yml ps -chaos-test: - @echo "Running chaos integration tests..." +chaos-test-mysql: + @echo "Running chaos integration tests against MySQL..." @echo "This will randomly restart the CDC application to test graceful shutdown" @cd tests/chaos/scripts && ./run_chaos_tests.sh -chaos-test-clean: - @echo "Cleaning up chaos testing environment..." +chaos-test-mysql-clean: + @echo "Cleaning up MySQL chaos testing environment..." @docker-compose -f docker-compose.chaos-test.yml down -v @docker volume rm chaos_test_lsn_data 2>/dev/null || true @docker network rm chaos_test_network 2>/dev/null || true @echo "Cleanup complete." -chaos-test-full: chaos-test-setup chaos-test chaos-test-clean - @echo "Full chaos test cycle complete!" +chaos-test-mysql-full: chaos-test-mysql-setup chaos-test-mysql chaos-test-mysql-clean + @echo "Full MySQL chaos test cycle complete!" -chaos-test-logs: +chaos-test-mysql-logs: @echo "Showing CDC application logs..." @docker logs -f cdc_application -# PGBench Testing commands -pgbench-test-setup: - @echo "Setting up pgbench testing environment..." +# MySQL PGBench Testing commands +pgbench-test-mysql-setup: + @echo "Setting up MySQL pgbench testing environment..." @chmod +x tests/chaos/scripts/*.sh @docker-compose -f docker-compose.chaos-test.yml up --build -d @echo "Waiting for services to be healthy..." @docker-compose -f docker-compose.chaos-test.yml ps @sleep 10 # Wait for the CDC application to fully initialize -pgbench-test: - @echo "Running pgbench chaos integration test..." +pgbench-test-mysql: + @echo "Running pgbench chaos integration test against MySQL..." @echo "This will run pgbench while randomly restarting the CDC application" @cd tests/chaos/scripts && ./run_pgbench_chaos_test.sh -pgbench-test-clean: - @echo "Cleaning up pgbench testing environment..." +pgbench-test-mysql-clean: + @echo "Cleaning up MySQL pgbench testing environment..." @docker-compose -f docker-compose.chaos-test.yml down -v @docker volume rm chaos_test_lsn_data 2>/dev/null || true @docker network rm chaos_test_network 2>/dev/null || true @echo "Cleanup complete." -pgbench-test-full: pgbench-test-setup pgbench-test pgbench-test-clean - @echo "Full pgbench test cycle complete!" +pgbench-test-mysql-full: pgbench-test-mysql-setup pgbench-test-mysql pgbench-test-mysql-clean + @echo "Full MySQL pgbench test cycle complete!" + +pgbench-test-mysql-logs: + @echo "Showing CDC application logs..." + @docker logs -f cdc_application + +# Helper: find sqlcmd path inside SQL Server container +SQLCMD_EXEC = docker exec cdc_sqlserver bash -c 'if [ -x /opt/mssql-tools18/bin/sqlcmd ]; then /opt/mssql-tools18/bin/sqlcmd "$$@"; elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then /opt/mssql-tools/bin/sqlcmd "$$@"; else echo "sqlcmd not found" >&2; exit 1; fi' -- + +# SQL Server Chaos Testing commands +chaos-test-sqlserver-setup: + @echo "Setting up SQL Server chaos testing environment..." + @chmod +x tests/chaos/scripts/*.sh + @echo "Building CDC application image..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml build cdc_app + @echo "Starting PostgreSQL and SQL Server..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d postgres sqlserver + @echo "Waiting for PostgreSQL to be ready..." + @timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + @echo "Waiting for SQL Server to be ready..." + @timeout 120 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @echo "Initializing SQL Server database..." + @$(SQLCMD_EXEC) -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + @echo "Starting CDC application..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app + @echo "Waiting for CDC application to initialize..." + @timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + +chaos-test-sqlserver: + @echo "Running chaos integration tests against SQL Server..." + @cd tests/chaos/scripts && ./run_chaos_tests_sqlserver.sh + +chaos-test-sqlserver-clean: + @echo "Cleaning up SQL Server chaos testing environment..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml down -v + @docker network rm chaos_test_network 2>/dev/null || true + @echo "Cleanup complete." + +chaos-test-sqlserver-full: chaos-test-sqlserver-setup chaos-test-sqlserver chaos-test-sqlserver-clean + @echo "Full SQL Server chaos test cycle complete!" -pgbench-test-logs: +chaos-test-sqlserver-logs: @echo "Showing CDC application logs..." @docker logs -f cdc_application + +# SQL Server PGBench Testing commands +pgbench-test-sqlserver-setup: + @echo "Setting up SQL Server pgbench testing environment..." + @chmod +x tests/chaos/scripts/*.sh + @echo "Building CDC application image..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml build cdc_app + @echo "Starting PostgreSQL and SQL Server..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d postgres sqlserver + @echo "Waiting for PostgreSQL to be ready..." + @timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + @echo "Waiting for SQL Server to be ready..." + @timeout 120 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @echo "Initializing SQL Server database..." + @$(SQLCMD_EXEC) -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + @echo "Starting CDC application..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app + @echo "Waiting for CDC application to initialize..." + @timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @docker-compose -f docker-compose.chaos-test-sqlserver.yml ps + +pgbench-test-sqlserver: + @echo "Running pgbench chaos integration test against SQL Server..." + @cd tests/chaos/scripts && ./run_pgbench_chaos_test_sqlserver.sh + +pgbench-test-sqlserver-clean: + @echo "Cleaning up SQL Server pgbench testing environment..." + @docker-compose -f docker-compose.chaos-test-sqlserver.yml down -v + @docker network rm chaos_test_network 2>/dev/null || true + @echo "Cleanup complete." + +pgbench-test-sqlserver-full: pgbench-test-sqlserver-setup pgbench-test-sqlserver pgbench-test-sqlserver-clean + @echo "Full SQL Server pgbench test cycle complete!" + +# SQLite Chaos Testing commands +chaos-test-sqlite-setup: + @echo "Setting up SQLite chaos testing environment..." + @chmod +x tests/chaos/scripts/*.sh + @echo "Building CDC application image..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml build cdc_app + @echo "Starting PostgreSQL..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml up -d postgres + @echo "Waiting for PostgreSQL to be ready..." + @timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + @echo "Starting CDC application..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml up -d cdc_app + @echo "Waiting for CDC application to initialize..." + @timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @echo "Initializing SQLite database schema..." + @docker exec cdc_application sqlite3 /app/data/cdc_target.db ".read /init/init_sqlite.sql" + @docker-compose -f docker-compose.chaos-test-sqlite.yml ps + +chaos-test-sqlite: + @echo "Running chaos integration tests against SQLite..." + @cd tests/chaos/scripts && ./run_chaos_tests_sqlite.sh + +chaos-test-sqlite-clean: + @echo "Cleaning up SQLite chaos testing environment..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml down -v + @docker volume rm chaos_test_sqlite_data 2>/dev/null || true + @docker network rm chaos_test_network 2>/dev/null || true + @echo "Cleanup complete." + +chaos-test-sqlite-full: chaos-test-sqlite-setup chaos-test-sqlite chaos-test-sqlite-clean + @echo "Full SQLite chaos test cycle complete!" + +chaos-test-sqlite-logs: + @echo "Showing CDC application logs..." + @docker logs -f cdc_application + +# SQLite PGBench Testing commands +pgbench-test-sqlite-setup: + @echo "Setting up SQLite pgbench testing environment..." + @chmod +x tests/chaos/scripts/*.sh + @echo "Building CDC application image..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml build cdc_app + @echo "Starting PostgreSQL..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml up -d postgres + @echo "Waiting for PostgreSQL to be ready..." + @timeout 60 bash -c 'until docker exec cdc_postgres pg_isready -U postgres 2>/dev/null; do sleep 2; done' + @echo "Starting CDC application..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml up -d cdc_app + @echo "Waiting for CDC application to initialize..." + @timeout 60 bash -c 'until docker ps --filter name=cdc_application --format "{{.Status}}" | grep -q healthy; do sleep 5; done' + @echo "Initializing SQLite database schema..." + @docker exec cdc_application sqlite3 /app/data/cdc_target.db ".read /init/init_sqlite.sql" + @docker-compose -f docker-compose.chaos-test-sqlite.yml ps + +pgbench-test-sqlite: + @echo "Running pgbench chaos integration test against SQLite..." + @cd tests/chaos/scripts && ./run_pgbench_chaos_test_sqlite.sh + +pgbench-test-sqlite-clean: + @echo "Cleaning up SQLite pgbench testing environment..." + @docker-compose -f docker-compose.chaos-test-sqlite.yml down -v + @docker volume rm chaos_test_sqlite_data 2>/dev/null || true + @docker network rm chaos_test_network 2>/dev/null || true + @echo "Cleanup complete." + +pgbench-test-sqlite-full: pgbench-test-sqlite-setup pgbench-test-sqlite pgbench-test-sqlite-clean + @echo "Full SQLite pgbench test cycle complete!" diff --git a/docker-compose.chaos-test-sqlite.yml b/docker-compose.chaos-test-sqlite.yml new file mode 100644 index 0000000..fa40c9f --- /dev/null +++ b/docker-compose.chaos-test-sqlite.yml @@ -0,0 +1,73 @@ +version: '3.8' + +services: + # PostgreSQL source database + postgres: + image: postgres:15-alpine + container_name: cdc_postgres + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: test.123 + POSTGRES_INITDB_ARGS: "--encoding=UTF8" + restart: always + ports: + - "5432:5432" + volumes: + - ./examples/scripts/init_postgres.sql:/docker-entrypoint-initdb.d/init_postgres.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + command: > + postgres + -c wal_level=logical + -c max_replication_slots=10 + -c max_wal_senders=10 + -c max_connections=500 + networks: + - chaos_test_network + + cdc_app: + build: + context: . + dockerfile: Dockerfile + args: + DEST_FEATURES: sqlite,metrics + container_name: cdc_application + depends_on: + postgres: + condition: service_healthy + env_file: + - ./env/.env.sqlite + environment: + RUST_LOG: info + RUST_BACKTRACE: 1 + CDC_LAST_LSN_FILE: /app/pg2any_last_lsn.metadata + ports: + - "8080:8080" + volumes: + - sqlite_data:/app/data + - ./examples/scripts/init_sqlite.sql:/init/init_sqlite.sql:ro + - ./tests/chaos/scenarios/verify_sqlite:/verify:ro + restart: "no" + stop_grace_period: 300s + stop_signal: SIGTERM + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - chaos_test_network + +volumes: + sqlite_data: + name: chaos_test_sqlite_data + +networks: + chaos_test_network: + driver: bridge + name: chaos_test_network diff --git a/docker-compose.chaos-test-sqlserver.yml b/docker-compose.chaos-test-sqlserver.yml new file mode 100644 index 0000000..8feb0df --- /dev/null +++ b/docker-compose.chaos-test-sqlserver.yml @@ -0,0 +1,89 @@ +version: '3.8' + +services: + # PostgreSQL source database + postgres: + image: postgres:15-alpine + container_name: cdc_postgres + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: test.123 + POSTGRES_INITDB_ARGS: "--encoding=UTF8" + restart: always + ports: + - "5432:5432" + volumes: + - ./examples/scripts/init_postgres.sql:/docker-entrypoint-initdb.d/init_postgres.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + command: > + postgres + -c wal_level=logical + -c max_replication_slots=10 + -c max_wal_senders=10 + -c max_connections=500 + networks: + - chaos_test_network + + sqlserver: + image: mcr.microsoft.com/mssql/server:2022-latest + container_name: cdc_sqlserver + restart: always + environment: + ACCEPT_EULA: "Y" + SA_PASSWORD: "Test.123!" + MSSQL_PID: "Developer" + ports: + - "1433:1433" + volumes: + - ./examples/scripts/init_sqlserver.sql:/init/init_sqlserver.sql:ro + - ./tests/chaos/scenarios/verify_sqlserver:/verify:ro + healthcheck: + test: ["CMD-SHELL", "(/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$${SA_PASSWORD}\" -C -Q 'SELECT 1' -b || /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$${SA_PASSWORD}\" -Q 'SELECT 1' -b) 2>/dev/null"] + interval: 15s + timeout: 10s + retries: 15 + start_period: 40s + networks: + - chaos_test_network + + cdc_app: + build: + context: . + dockerfile: Dockerfile + args: + DEST_FEATURES: sqlserver,metrics + container_name: cdc_application + depends_on: + postgres: + condition: service_healthy + sqlserver: + condition: service_healthy + env_file: + - ./env/.env.sqlserver + environment: + RUST_LOG: info + RUST_BACKTRACE: 1 + CDC_LAST_LSN_FILE: /app/pg2any_last_lsn.metadata + ports: + - "8080:8080" + restart: "no" + stop_grace_period: 300s + stop_signal: SIGTERM + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - chaos_test_network + +networks: + chaos_test_network: + driver: bridge + name: chaos_test_network diff --git a/env/.env.sqlite b/env/.env.sqlite new file mode 100644 index 0000000..f7c31b2 --- /dev/null +++ b/env/.env.sqlite @@ -0,0 +1,35 @@ +# PostgreSQL CDC Environment Configuration - SQLite Destination + +# Rust Environment +RUST_LOG=info +RUST_BACKTRACE=1 + +# Source PostgreSQL Database Configuration +CDC_SOURCE_CONNECTION_STRING=postgresql://postgres:test.123@postgres:5432/postgres?replication=database + +# Destination Database Configuration - SQLite +CDC_DEST_TYPE=SQLite +CDC_DEST_URI=/app/data/cdc_target.db + +# No schema mapping needed for SQLite (schema is stripped automatically) + +# CDC-Specific Configuration +CDC_REPLICATION_SLOT=cdc_slot +CDC_PUBLICATION=cdc_pub +CDC_PROTOCOL_VERSION=3 +CDC_BINARY_FORMAT=true +CDC_STREAMING=true +CDC_BUFFER_SIZE=1000 +CDC_COMMIT_BATCH_SIZE=1000 + +# Timeout Configuration (in seconds) +CDC_CONNECTION_TIMEOUT=30 +CDC_QUERY_TIMEOUT=10 + +# LSN Tracking and Position Persistence +CDC_LAST_LSN_FILE=./pg2any_last_lsn.metadata + +# Monitoring Configuration +METRICS_PORT=8080 + +PG2ANY_ENABLE_COMPRESSION=true diff --git a/env/.env.sqlserver b/env/.env.sqlserver new file mode 100644 index 0000000..c9c0df1 --- /dev/null +++ b/env/.env.sqlserver @@ -0,0 +1,37 @@ +# PostgreSQL CDC Environment Configuration - SQL Server Destination + +# Rust Environment +RUST_LOG=info +RUST_BACKTRACE=1 + +# Source PostgreSQL Database Configuration +CDC_SOURCE_CONNECTION_STRING=postgresql://postgres:test.123@postgres:5432/postgres?replication=database + +# Destination Database Configuration - SQL Server +CDC_DEST_TYPE=SqlServer +CDC_DEST_URI=Server=tcp:sqlserver,1433;Database=cdc_db;User Id=sa;Password=Test.123!;TrustServerCertificate=true + +# Schema Mapping Configuration +# Maps PostgreSQL "public" schema to SQL Server "dbo" schema +CDC_SCHEMA_MAPPING=public:dbo + +# CDC-Specific Configuration +CDC_REPLICATION_SLOT=cdc_slot +CDC_PUBLICATION=cdc_pub +CDC_PROTOCOL_VERSION=3 +CDC_BINARY_FORMAT=true +CDC_STREAMING=true +CDC_BUFFER_SIZE=1000 +CDC_COMMIT_BATCH_SIZE=1000 + +# Timeout Configuration (in seconds) +CDC_CONNECTION_TIMEOUT=30 +CDC_QUERY_TIMEOUT=10 + +# LSN Tracking and Position Persistence +CDC_LAST_LSN_FILE=./pg2any_last_lsn.metadata + +# Monitoring Configuration +METRICS_PORT=8080 + +PG2ANY_ENABLE_COMPRESSION=true diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 6275f80..888236b 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -10,8 +10,15 @@ repository = "https://github.com/isdaniel/pg2any" homepage = "https://github.com/isdaniel/pg2any" documentation = "https://docs.rs/pg2any" +[features] +default = ["mysql", "metrics"] +mysql = ["pg2any_lib/mysql"] +sqlserver = ["pg2any_lib/sqlserver"] +sqlite = ["pg2any_lib/sqlite"] +metrics = ["pg2any_lib/metrics"] + [dependencies] -pg2any_lib = { path = "../pg2any-lib", default-features = false, features = ["mysql", "metrics"] } +pg2any_lib = { path = "../pg2any-lib", default-features = false } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/examples/scripts/init_sqlite.sql b/examples/scripts/init_sqlite.sql new file mode 100644 index 0000000..355addf --- /dev/null +++ b/examples/scripts/init_sqlite.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS t1 ( + ID INTEGER NOT NULL PRIMARY KEY, + val INTEGER NOT NULL, + col1 TEXT NOT NULL, + col2 TEXT NOT NULL +); diff --git a/examples/scripts/init_sqlserver.sql b/examples/scripts/init_sqlserver.sql new file mode 100644 index 0000000..c6b0054 --- /dev/null +++ b/examples/scripts/init_sqlserver.sql @@ -0,0 +1,15 @@ +IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = 'cdc_db') + CREATE DATABASE cdc_db; +GO + +USE cdc_db; +GO + +IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 't1' AND schema_id = SCHEMA_ID('dbo')) +CREATE TABLE dbo.t1 ( + ID INT NOT NULL PRIMARY KEY, + val INT NOT NULL, + col1 CHAR(36) NOT NULL, + col2 CHAR(36) NOT NULL +); +GO diff --git a/tests/chaos/scenarios/verify_sqlite/scenario1_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario1_verify.sql new file mode 100644 index 0000000..62eef88 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlite/scenario1_verify.sql @@ -0,0 +1,9 @@ +-- Scenario 1: Verification - Check if 100 records were replicated +SELECT + CASE + WHEN COUNT(*) = 100 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 100 AS expected_count +FROM t1; diff --git a/tests/chaos/scenarios/verify_sqlite/scenario2_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario2_verify.sql new file mode 100644 index 0000000..e5d0f78 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlite/scenario2_verify.sql @@ -0,0 +1,10 @@ +-- Scenario 2: Verification - Check if updates were replicated +-- Count records where val >= 10000 (updated records) +SELECT + CASE + WHEN COUNT(*) = 50 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS updated_count +FROM t1 +WHERE val >= 10000; diff --git a/tests/chaos/scenarios/verify_sqlite/scenario3_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario3_verify.sql new file mode 100644 index 0000000..81eb27e --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlite/scenario3_verify.sql @@ -0,0 +1,9 @@ +-- Scenario 3: Verification - Check if deletes were replicated +SELECT + CASE + WHEN COUNT(*) = 50000 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 50000 AS expected_count +FROM t1; diff --git a/tests/chaos/scenarios/verify_sqlite/scenario4_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario4_verify.sql new file mode 100644 index 0000000..6c6faa8 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlite/scenario4_verify.sql @@ -0,0 +1,10 @@ +-- Scenario 4: Verification - Check if mixed operations were replicated +SELECT + CASE + WHEN COUNT(*) >= 30 AND EXISTS (SELECT 1 FROM t1 WHERE val = 999999) + THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS total_count, + (SELECT COUNT(*) FROM t1 WHERE val = 999999) AS marker_count +FROM t1; diff --git a/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql new file mode 100644 index 0000000..9f92af4 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql @@ -0,0 +1,14 @@ +-- Scenario 5: Verification - Check if high-volume data (3M rows) was replicated +SELECT + CASE + WHEN COUNT(*) = 3000000 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 3000000 AS expected_count, + CASE + WHEN COUNT(*) = 3000000 THEN 'All 3M rows replicated successfully' + WHEN COUNT(*) > 0 THEN 'Partial replication: ' || CAST(COUNT(*) AS TEXT) || ' of 3M rows' + ELSE 'No data replicated' + END AS status_message +FROM t1; diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario1_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario1_verify.sql new file mode 100644 index 0000000..d1b8906 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlserver/scenario1_verify.sql @@ -0,0 +1,10 @@ +-- Scenario 1: Verification - Check if 100 records were replicated +SELECT + CASE + WHEN COUNT(*) = 100 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 100 AS expected_count +FROM dbo.t1; +GO diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario2_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario2_verify.sql new file mode 100644 index 0000000..6f14b97 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlserver/scenario2_verify.sql @@ -0,0 +1,11 @@ +-- Scenario 2: Verification - Check if updates were replicated +-- Count records where val >= 10000 (updated records) +SELECT + CASE + WHEN COUNT(*) = 50 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS updated_count +FROM dbo.t1 +WHERE val >= 10000; +GO diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario3_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario3_verify.sql new file mode 100644 index 0000000..24931da --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlserver/scenario3_verify.sql @@ -0,0 +1,10 @@ +-- Scenario 3: Verification - Check if deletes were replicated +SELECT + CASE + WHEN COUNT(*) = 50000 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 50000 AS expected_count +FROM dbo.t1; +GO diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario4_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario4_verify.sql new file mode 100644 index 0000000..70dc5ce --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlserver/scenario4_verify.sql @@ -0,0 +1,11 @@ +-- Scenario 4: Verification - Check if mixed operations were replicated +SELECT + CASE + WHEN COUNT(*) >= 30 AND EXISTS (SELECT 1 FROM dbo.t1 WHERE val = 999999) + THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS total_count, + (SELECT COUNT(*) FROM dbo.t1 WHERE val = 999999) AS marker_count +FROM dbo.t1; +GO diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql new file mode 100644 index 0000000..f61ee32 --- /dev/null +++ b/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql @@ -0,0 +1,15 @@ +-- Scenario 5: Verification - Check if high-volume data (3M rows) was replicated +SELECT + CASE + WHEN COUNT(*) = 3000000 THEN 'PASS' + ELSE 'FAIL' + END AS test_result, + COUNT(*) AS actual_count, + 3000000 AS expected_count, + CASE + WHEN COUNT(*) = 3000000 THEN 'All 3M rows replicated successfully' + WHEN COUNT(*) > 0 THEN CONCAT('Partial replication: ', CAST(COUNT(*) AS VARCHAR(20)), ' of 3M rows') + ELSE 'No data replicated' + END AS status_message +FROM dbo.t1; +GO diff --git a/tests/chaos/scripts/run_chaos_tests_sqlite.sh b/tests/chaos/scripts/run_chaos_tests_sqlite.sh new file mode 100755 index 0000000..35f3c36 --- /dev/null +++ b/tests/chaos/scripts/run_chaos_tests_sqlite.sh @@ -0,0 +1,284 @@ +#!/bin/bash +# +# Chaos Integration Test Runner - SQLite Destination +# This script runs all test scenarios with chaos testing (random container restarts) +# and verifies that CDC replication works correctly under adverse conditions. +# + +set -e + +# Load environment variables from .env file if it exists +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" + +if [ -f "$ENV_FILE" ]; then + echo "Loading environment from: $ENV_FILE" + set -a + source "$ENV_FILE" + set +a +else + echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +POSTGRES_HOST="${CDC_POSTGRES_HOST:-127.0.0.1}" +POSTGRES_PORT="${CDC_POSTGRES_PORT:-5432}" +POSTGRES_USER="${CDC_POSTGRES_USER:-postgres}" +POSTGRES_PASSWORD="${CDC_POSTGRES_PASSWORD:-test.123}" +POSTGRES_DB="${CDC_POSTGRES_DB:-postgres}" + +CDC_CONTAINER="${CDC_CONTAINER_NAME:-cdc_application}" +SQLITE_DB_PATH="/app/data/cdc_target.db" + +CONTAINER_NAME="${CDC_CONTAINER_NAME:-cdc_application}" +MAX_RETRIES=40 +RETRY_INTERVAL=45 +CHAOS_SCRIPT_PID="" + +# Script paths +SCENARIOS_DIR="$SCRIPT_DIR/../scenarios" +CHAOS_SCRIPT="$SCRIPT_DIR/chaos_script.sh" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $*" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +# Cleanup function +cleanup() { + log_info "Cleaning up..." + + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + fi + + log_info "Cleanup complete." +} + +# Set up trap for cleanup +trap cleanup EXIT INT TERM + +# Function to execute PostgreSQL SQL +execute_postgres_sql() { + local sql_file="$1" + log_info "Executing PostgreSQL SQL: $sql_file" + + PGPASSWORD="$POSTGRES_PASSWORD" psql \ + -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -f "$sql_file" \ + 2>&1 + + return ${PIPESTATUS[0]} +} + +# Function to verify test result via docker exec with sqlite3 +verify_scenario() { + local verify_file="$1" + local verify_filename=$(basename "$verify_file") + + local result=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ + ".read /verify/$verify_filename" 2>&1) + + # Check if result contains "PASS" + if echo "$result" | grep -q "PASS"; then + return 0 + else + log_warning "Verification result: $result" + return 1 + fi +} + +# Function to clean up test data +cleanup_test_data() { + log_info "Cleaning up test data from both databases..." + + # Clean PostgreSQL + PGPASSWORD="$POSTGRES_PASSWORD" psql \ + -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -c "TRUNCATE TABLE public.t1;" > /dev/null 2>&1 || true + + # Clean SQLite (DELETE instead of TRUNCATE - SQLite doesn't support TRUNCATE) + docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ + "DELETE FROM t1;" 2>&1 || true + + # Wait for truncate to replicate + sleep 5 +} + +# Function to run a single scenario +run_scenario() { + local scenario_num="$1" + local input_file="$SCENARIOS_DIR/input/scenario${scenario_num}_input.sql" + local verify_file="$SCENARIOS_DIR/verify_sqlite/scenario${scenario_num}_verify.sql" + + if [ ! -f "$input_file" ]; then + log_error "Input file not found: $input_file" + return 1 + fi + + if [ ! -f "$verify_file" ]; then + log_error "Verify file not found: $verify_file" + return 1 + fi + + log_info "========================================" + log_info "Running Scenario $scenario_num (SQLite)" + log_info "========================================" + + # Execute input SQL on PostgreSQL + if ! execute_postgres_sql "$input_file"; then + log_error "Failed to execute input SQL for scenario $scenario_num" + return 1 + fi + + log_info "Input SQL executed successfully. Waiting for replication..." + + # Retry verification until it passes + local retry_count=0 + local verification_passed=false + + while [ $retry_count -lt $MAX_RETRIES ]; do + retry_count=$((retry_count + 1)) + log_info "Verification attempt $retry_count/$MAX_RETRIES for scenario $scenario_num..." + + if verify_scenario "$verify_file"; then + log_success "Scenario $scenario_num PASSED on attempt $retry_count" + verification_passed=true + break + else + log_warning "Scenario $scenario_num verification failed (attempt $retry_count/$MAX_RETRIES)" + + if [ $retry_count -lt $MAX_RETRIES ]; then + log_info "Waiting $RETRY_INTERVAL seconds before retry..." + sleep "$RETRY_INTERVAL" + fi + fi + done + + if [ "$verification_passed" = false ]; then + log_error "Scenario $scenario_num FAILED after $MAX_RETRIES attempts" + return 1 + fi + + return 0 +} + +# Main execution +main() { + log_info "==========================================" + log_info "CDC Chaos Integration Test Suite (SQLite)" + log_info "==========================================" + log_info "Max retries per scenario: $MAX_RETRIES" + log_info "Retry interval: $RETRY_INTERVAL seconds" + log_info "Container under test: $CONTAINER_NAME" + log_info "SQLite DB path: $SQLITE_DB_PATH" + log_info "" + + # Check if chaos script exists + if [ ! -f "$CHAOS_SCRIPT" ]; then + log_error "Chaos script not found: $CHAOS_SCRIPT" + exit 1 + fi + + # Make chaos script executable + chmod +x "$CHAOS_SCRIPT" + + # Check if container exists + if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container '$CONTAINER_NAME' not found. Please start docker-compose first." + exit 1 + fi + + # Wait for services to be ready + log_info "Waiting for services to be ready..." + sleep 10 + + # Start chaos script in background + log_info "Starting chaos script for container: $CONTAINER_NAME" + "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/chaos_script.log" 2>&1 & + CHAOS_SCRIPT_PID=$! + log_info "Chaos script started with PID: $CHAOS_SCRIPT_PID" + + # Wait a moment for chaos to begin + sleep 5 + + # Find all scenario input files + local scenario_files=($(ls "$SCENARIOS_DIR"/input/scenario*_input.sql 2>/dev/null | sort)) + local total_scenarios=${#scenario_files[@]} + + if [ $total_scenarios -eq 0 ]; then + log_error "No scenario files found in $SCENARIOS_DIR/input" + exit 1 + fi + + log_info "Found $total_scenarios scenarios to run" + log_info "" + + # Run each scenario + local passed=0 + local failed=0 + + for scenario_file in "${scenario_files[@]}"; do + # Extract scenario number from filename + local scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') + + # Clean up before each scenario + cleanup_test_data + + if run_scenario "$scenario_num"; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + log_error "Scenario $scenario_num failed. Continuing to next scenario..." + fi + + log_info "" + sleep 5 + done + + # Summary + log_info "==========================================" + log_info "Test Suite Complete (SQLite)" + log_info "==========================================" + log_info "Total scenarios: $total_scenarios" + log_success "Passed: $passed" + + if [ $failed -gt 0 ]; then + log_error "Failed: $failed" + exit 1 + else + log_success "All scenarios passed!" + exit 0 + fi +} + +# Run main function +main "$@" diff --git a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh new file mode 100755 index 0000000..fb49f1c --- /dev/null +++ b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh @@ -0,0 +1,329 @@ +#!/bin/bash +# +# Chaos Integration Test Runner - SQL Server Destination +# This script runs all test scenarios with chaos testing (random container restarts) +# and verifies that CDC replication works correctly under adverse conditions. +# + +set -e + +# Load safe environment variables from .env file if it exists +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" + +if [ -f "$ENV_FILE" ]; then + echo "Loading environment from: $ENV_FILE" + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue + [[ "$value" == *" "* ]] && continue + export "$key=$value" + done < "$ENV_FILE" +else + echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +POSTGRES_HOST="${CDC_POSTGRES_HOST:-127.0.0.1}" +POSTGRES_PORT="${CDC_POSTGRES_PORT:-5432}" +POSTGRES_USER="${CDC_POSTGRES_USER:-postgres}" +POSTGRES_PASSWORD="${CDC_POSTGRES_PASSWORD:-test.123}" +POSTGRES_DB="${CDC_POSTGRES_DB:-postgres}" + +SQLSERVER_CONTAINER="${CDC_SQLSERVER_CONTAINER:-cdc_sqlserver}" +SQLSERVER_PASSWORD="${CDC_SQLSERVER_PASSWORD:-Test.123!}" +SQLSERVER_DB="${CDC_SQLSERVER_DB:-cdc_db}" + +CONTAINER_NAME="${CDC_CONTAINER_NAME:-cdc_application}" +MAX_RETRIES=40 +RETRY_INTERVAL=45 +CHAOS_SCRIPT_PID="" + +# Script paths +SCENARIOS_DIR="$SCRIPT_DIR/../scenarios" +CHAOS_SCRIPT="$SCRIPT_DIR/chaos_script.sh" + +# Detect sqlcmd path inside SQL Server container +detect_sqlcmd() { + if docker exec "$SQLSERVER_CONTAINER" test -x /opt/mssql-tools18/bin/sqlcmd 2>/dev/null; then + echo "/opt/mssql-tools18/bin/sqlcmd" + elif docker exec "$SQLSERVER_CONTAINER" test -x /opt/mssql-tools/bin/sqlcmd 2>/dev/null; then + echo "/opt/mssql-tools/bin/sqlcmd" + else + echo "" + fi +} + +SQLCMD_PATH="" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $*" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +# Cleanup function +cleanup() { + log_info "Cleaning up..." + + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + fi + + log_info "Cleanup complete." +} + +# Set up trap for cleanup +trap cleanup EXIT INT TERM + +# Function to execute PostgreSQL SQL +execute_postgres_sql() { + local sql_file="$1" + log_info "Executing PostgreSQL SQL: $sql_file" + + PGPASSWORD="$POSTGRES_PASSWORD" psql \ + -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -f "$sql_file" \ + 2>&1 + + return ${PIPESTATUS[0]} +} + +# Function to execute SQL Server SQL via docker exec +execute_sqlserver_sql() { + local sql="$1" + log_info "Executing SQL Server SQL..." + + docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ + -d "$SQLSERVER_DB" \ + -Q "$sql" \ + 2>&1 + + return ${PIPESTATUS[0]} +} + +# Function to verify test result via mounted verify scripts +verify_scenario() { + local verify_file="$1" + local verify_filename=$(basename "$verify_file") + + local result=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ + -d "$SQLSERVER_DB" \ + -i "/verify/$verify_filename" \ + -h -1 -W 2>&1) + + # Check if result contains "PASS" + if echo "$result" | grep -q "PASS"; then + return 0 + else + log_warning "Verification result: $result" + return 1 + fi +} + +# Function to clean up test data +cleanup_test_data() { + log_info "Cleaning up test data from both databases..." + + # Clean PostgreSQL + PGPASSWORD="$POSTGRES_PASSWORD" psql \ + -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -c "TRUNCATE TABLE public.t1;" > /dev/null 2>&1 || true + + # Clean SQL Server + docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ + -d "$SQLSERVER_DB" \ + -Q "TRUNCATE TABLE dbo.t1;" \ + 2>&1 || true + + # Wait for truncate to replicate + sleep 5 +} + +# Function to run a single scenario +run_scenario() { + local scenario_num="$1" + local input_file="$SCENARIOS_DIR/input/scenario${scenario_num}_input.sql" + local verify_file="$SCENARIOS_DIR/verify_sqlserver/scenario${scenario_num}_verify.sql" + + if [ ! -f "$input_file" ]; then + log_error "Input file not found: $input_file" + return 1 + fi + + if [ ! -f "$verify_file" ]; then + log_error "Verify file not found: $verify_file" + return 1 + fi + + log_info "========================================" + log_info "Running Scenario $scenario_num (SQL Server)" + log_info "========================================" + + # Execute input SQL on PostgreSQL + if ! execute_postgres_sql "$input_file"; then + log_error "Failed to execute input SQL for scenario $scenario_num" + return 1 + fi + + log_info "Input SQL executed successfully. Waiting for replication..." + + # Retry verification until it passes + local retry_count=0 + local verification_passed=false + + while [ $retry_count -lt $MAX_RETRIES ]; do + retry_count=$((retry_count + 1)) + log_info "Verification attempt $retry_count/$MAX_RETRIES for scenario $scenario_num..." + + if verify_scenario "$verify_file"; then + log_success "Scenario $scenario_num PASSED on attempt $retry_count" + verification_passed=true + break + else + log_warning "Scenario $scenario_num verification failed (attempt $retry_count/$MAX_RETRIES)" + + if [ $retry_count -lt $MAX_RETRIES ]; then + log_info "Waiting $RETRY_INTERVAL seconds before retry..." + sleep "$RETRY_INTERVAL" + fi + fi + done + + if [ "$verification_passed" = false ]; then + log_error "Scenario $scenario_num FAILED after $MAX_RETRIES attempts" + return 1 + fi + + return 0 +} + +# Main execution +main() { + log_info "==========================================" + log_info "CDC Chaos Integration Test Suite (SQL Server)" + log_info "==========================================" + + # Detect sqlcmd path + SQLCMD_PATH=$(detect_sqlcmd) + if [ -z "$SQLCMD_PATH" ]; then + log_error "sqlcmd not found in SQL Server container" + exit 1 + fi + log_info "Using sqlcmd: $SQLCMD_PATH" + + log_info "Max retries per scenario: $MAX_RETRIES" + log_info "Retry interval: $RETRY_INTERVAL seconds" + log_info "Container under test: $CONTAINER_NAME" + log_info "SQL Server container: $SQLSERVER_CONTAINER" + log_info "" + + # Check if chaos script exists + if [ ! -f "$CHAOS_SCRIPT" ]; then + log_error "Chaos script not found: $CHAOS_SCRIPT" + exit 1 + fi + + # Make chaos script executable + chmod +x "$CHAOS_SCRIPT" + + # Check if container exists + if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container '$CONTAINER_NAME' not found. Please start docker-compose first." + exit 1 + fi + + # Wait for services to be ready + log_info "Waiting for services to be ready..." + sleep 10 + + # Start chaos script in background + log_info "Starting chaos script for container: $CONTAINER_NAME" + "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/chaos_script.log" 2>&1 & + CHAOS_SCRIPT_PID=$! + log_info "Chaos script started with PID: $CHAOS_SCRIPT_PID" + + # Wait a moment for chaos to begin + sleep 5 + + # Find all scenario input files + local scenario_files=($(ls "$SCENARIOS_DIR"/input/scenario*_input.sql 2>/dev/null | sort)) + local total_scenarios=${#scenario_files[@]} + + if [ $total_scenarios -eq 0 ]; then + log_error "No scenario files found in $SCENARIOS_DIR/input" + exit 1 + fi + + log_info "Found $total_scenarios scenarios to run" + log_info "" + + # Run each scenario + local passed=0 + local failed=0 + + for scenario_file in "${scenario_files[@]}"; do + # Extract scenario number from filename + local scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') + + # Clean up before each scenario + cleanup_test_data + + if run_scenario "$scenario_num"; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + log_error "Scenario $scenario_num failed. Continuing to next scenario..." + fi + + log_info "" + sleep 5 + done + + # Summary + log_info "==========================================" + log_info "Test Suite Complete (SQL Server)" + log_info "==========================================" + log_info "Total scenarios: $total_scenarios" + log_success "Passed: $passed" + + if [ $failed -gt 0 ]; then + log_error "Failed: $failed" + exit 1 + else + log_success "All scenarios passed!" + exit 0 + fi +} + +# Run main function +main "$@" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh new file mode 100755 index 0000000..3a7479d --- /dev/null +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh @@ -0,0 +1,348 @@ +#!/bin/bash +# +# PGBench Chaos Test Runner - SQLite Destination +# This script runs pgbench performance testing while randomly restarting the CDC application +# to test graceful shutdown and recovery under load. +# + +set -e + +# Load environment variables from .env file if it exists +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" + +if [ -f "$ENV_FILE" ]; then + echo "Loading environment from: $ENV_FILE" + set -a + source "$ENV_FILE" + set +a +else + echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Configuration +POSTGRES_HOST="${CDC_POSTGRES_HOST:-localhost}" +POSTGRES_PORT="${CDC_POSTGRES_PORT:-5432}" +POSTGRES_USER="${CDC_POSTGRES_USER:-postgres}" +POSTGRES_PASSWORD="${CDC_POSTGRES_PASSWORD:-test.123}" +POSTGRES_DB="${CDC_POSTGRES_DB:-postgres}" +CONTAINER_NAME="${CDC_CONTAINER_NAME:-cdc_application}" + +CDC_CONTAINER="${CDC_CONTAINER_NAME:-cdc_application}" +SQLITE_DB_PATH="/app/data/cdc_target.db" + +# PGBench configuration +PGBENCH_SCALE="${PGBENCH_SCALE:-32}" +PGBENCH_CLIENTS="${PGBENCH_CLIENTS:-100}" +PGBENCH_THREADS="${PGBENCH_THREADS:-8}" +PGBENCH_TRANSACTIONS="${PGBENCH_TRANSACTIONS:-12}" +PGBENCH_SCRIPT="${PGBENCH_SCRIPT:-$PROJECT_ROOT/examples/scripts/pgbench_testing.sql}" + +# Verification configuration +MAX_RETRIES=60 +RETRY_INTERVAL=45 +EXPECTED_ROW_COUNT=$((3000 * PGBENCH_CLIENTS * PGBENCH_TRANSACTIONS)) + +# Connection string +POSTGRES_CONNSTRING="postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?user=${POSTGRES_USER}&password=${POSTGRES_PASSWORD}" + +# Script paths +CHAOS_SCRIPT="$SCRIPT_DIR/chaos_script.sh" +CHAOS_SCRIPT_PID="" + +# Results file +RESULTS_FILE="$SCRIPT_DIR/pgbench_chaos_sqlite_results_$(date +%Y%m%d_%H%M%S).log" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" | tee -a "$RESULTS_FILE" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $*" | tee -a "$RESULTS_FILE" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $*" | tee -a "$RESULTS_FILE" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" | tee -a "$RESULTS_FILE" >&2 +} + +log_section() { + echo -e "${CYAN}========================================${NC}" | tee -a "$RESULTS_FILE" + echo -e "${CYAN}$*${NC}" | tee -a "$RESULTS_FILE" + echo -e "${CYAN}========================================${NC}" | tee -a "$RESULTS_FILE" +} + +# Cleanup function +cleanup() { + log_info "Cleaning up..." + + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + fi + + log_info "Cleanup complete." + log_info "Results saved to: $RESULTS_FILE" +} + +# Set up trap for cleanup +trap cleanup EXIT INT TERM + +# Function to check if container exists +check_container() { + if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container '$CONTAINER_NAME' not found. Please start docker-compose first." + exit 1 + fi + + log_success "Container '$CONTAINER_NAME' found." +} + +# Function to check if pgbench is installed +check_pgbench() { + if ! command -v pgbench &> /dev/null; then + log_error "pgbench is not installed. Please install PostgreSQL client tools." + exit 1 + fi + + log_success "pgbench is installed." +} + +# Function to test PostgreSQL connection +test_postgres_connection() { + log_info "Testing PostgreSQL connection..." + + if PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 1" > /dev/null 2>&1; then + log_success "PostgreSQL connection successful." + return 0 + else + log_error "Cannot connect to PostgreSQL." + return 1 + fi +} + +# Function to test SQLite connection +test_sqlite_connection() { + log_info "Testing SQLite connection..." + + if docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" "SELECT 1;" > /dev/null 2>&1; then + log_success "SQLite connection successful." + return 0 + else + log_error "Cannot connect to SQLite database." + return 1 + fi +} + +# Function to get row count from SQLite t1 table +get_sqlite_row_count() { + local count=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ + "SELECT COUNT(*) FROM t1;" 2>/dev/null | tr -d '[:space:]') + + echo "$count" +} + +# Function to verify replication completed +verify_replication() { + local current_count=$(get_sqlite_row_count) + + if [ -z "$current_count" ]; then + log_warning "Failed to get row count from SQLite" + return 1 + fi + + log_info "Current SQLite row count: $current_count / Expected: $EXPECTED_ROW_COUNT" + + if [ "$current_count" -eq "$EXPECTED_ROW_COUNT" ] 2>/dev/null; then + return 0 + else + return 1 + fi +} + +# Function to initialize pgbench +initialize_pgbench() { + log_section "Initializing PGBench (Scale: $PGBENCH_SCALE)" + + local start_time=$(date +%s) + + if pgbench -i -s "$PGBENCH_SCALE" --unlogged-tables --foreign-keys "$POSTGRES_CONNSTRING" 2>&1 | tee -a "$RESULTS_FILE"; then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + log_success "PGBench initialization completed in ${duration}s" + return 0 + else + log_error "PGBench initialization failed" + return 1 + fi +} + +# Function to run pgbench benchmark +run_pgbench_benchmark() { + log_section "Running PGBench Benchmark" + log_info "Configuration:" + log_info " - Clients: $PGBENCH_CLIENTS" + log_info " - Threads: $PGBENCH_THREADS" + log_info " - Transactions per client: $PGBENCH_TRANSACTIONS" + log_info " - Script: $PGBENCH_SCRIPT" + echo "" | tee -a "$RESULTS_FILE" + + local start_time=$(date +%s) + + # Check if custom script exists + local pgbench_cmd="pgbench -c $PGBENCH_CLIENTS -j $PGBENCH_THREADS -t $PGBENCH_TRANSACTIONS" + + if [ -f "$PGBENCH_SCRIPT" ]; then + log_info "Using custom script: $PGBENCH_SCRIPT" + pgbench_cmd="$pgbench_cmd -f $PGBENCH_SCRIPT" + else + log_warning "Custom script not found: $PGBENCH_SCRIPT" + fi + + pgbench_cmd="$pgbench_cmd $POSTGRES_CONNSTRING" + + echo "" | tee -a "$RESULTS_FILE" + + if eval "$pgbench_cmd" 2>&1 | tee -a "$RESULTS_FILE"; then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + log_success "PGBench benchmark completed in ${duration}s" + return 0 + else + log_error "PGBench benchmark failed" + return 1 + fi +} + +# Function to start chaos testing +start_chaos_testing() { + log_section "Starting Chaos Testing" + + if [ ! -f "$CHAOS_SCRIPT" ]; then + log_error "Chaos script not found: $CHAOS_SCRIPT" + exit 1 + fi + + chmod +x "$CHAOS_SCRIPT" + + log_info "Starting chaos script for container: $CONTAINER_NAME" + "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/pgbench_chaos_script.log" 2>&1 & + CHAOS_SCRIPT_PID=$! + + log_success "Chaos script started with PID: $CHAOS_SCRIPT_PID" + log_info "Chaos script logs: $SCRIPT_DIR/pgbench_chaos_script.log" + + sleep 5 +} + +# Main execution +main() { + export PGPASSWORD=$POSTGRES_PASSWORD + log_section "PGBench Chaos Integration Test (SQLite)" + log_info "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" + log_info "Container under test: $CONTAINER_NAME" + log_info "SQLite DB path: $SQLITE_DB_PATH" + echo "" | tee -a "$RESULTS_FILE" + + # Pre-flight checks + log_info "Running pre-flight checks..." + check_pgbench + test_postgres_connection || exit 1 + test_sqlite_connection || exit 1 + check_container + echo "" | tee -a "$RESULTS_FILE" + + # Initialize pgbench + if ! initialize_pgbench; then + log_error "Failed to initialize pgbench. Exiting." + exit 1 + fi + echo "" | tee -a "$RESULTS_FILE" + + # Wait for initialization to settle + log_info "Waiting for initialization to settle..." + sleep 10 + + # Start chaos testing + start_chaos_testing + echo "" | tee -a "$RESULTS_FILE" + + # Wait for chaos to take effect + log_info "Allowing chaos script to run for a few cycles..." + sleep 15 + + # Run benchmark + local benchmark_result=0 + if ! run_pgbench_benchmark; then + log_error "Benchmark failed." + benchmark_result=1 + fi + + echo "" | tee -a "$RESULTS_FILE" + + # Verify replication with retry loop + log_section "Verifying Replication" + log_info "Expected row count: $EXPECTED_ROW_COUNT" + log_info "Max retries: $MAX_RETRIES" + log_info "Retry interval: $RETRY_INTERVAL seconds" + echo "" | tee -a "$RESULTS_FILE" + + local retry_count=0 + local verification_passed=false + + while [ $retry_count -lt $MAX_RETRIES ]; do + retry_count=$((retry_count + 1)) + log_info "Verification attempt $retry_count/$MAX_RETRIES..." + + if verify_replication; then + log_success "Replication verification PASSED on attempt $retry_count" + verification_passed=true + break + else + log_warning "Replication verification failed (attempt $retry_count/$MAX_RETRIES)" + + if [ $retry_count -lt $MAX_RETRIES ]; then + log_info "Waiting $RETRY_INTERVAL seconds before retry..." + sleep "$RETRY_INTERVAL" + fi + fi + done + + echo "" | tee -a "$RESULTS_FILE" + + # Summary + log_section "Test Complete" + log_info "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" + + if [ $benchmark_result -eq 0 ] && [ "$verification_passed" = true ]; then + log_success "PGBench chaos test completed successfully! (SQLite)" + log_success "All $EXPECTED_ROW_COUNT rows replicated to SQLite" + log_info "Check the results file for detailed metrics: $RESULTS_FILE" + exit 0 + else + if [ $benchmark_result -ne 0 ]; then + log_error "Benchmark failed!" + fi + if [ "$verification_passed" = false ]; then + log_error "Replication verification failed after $MAX_RETRIES attempts!" + fi + exit 1 + fi +} + +# Run main function +main "$@" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh new file mode 100755 index 0000000..95e8fb5 --- /dev/null +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh @@ -0,0 +1,378 @@ +#!/bin/bash +# +# PGBench Chaos Test Runner - SQL Server Destination +# This script runs pgbench performance testing while randomly restarting the CDC application +# to test graceful shutdown and recovery under load. +# + +set -e + +# Load safe environment variables from .env file if it exists +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" + +if [ -f "$ENV_FILE" ]; then + echo "Loading environment from: $ENV_FILE" + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue + [[ "$value" == *" "* ]] && continue + export "$key=$value" + done < "$ENV_FILE" +else + echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" +fi + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Configuration +POSTGRES_HOST="${CDC_POSTGRES_HOST:-localhost}" +POSTGRES_PORT="${CDC_POSTGRES_PORT:-5432}" +POSTGRES_USER="${CDC_POSTGRES_USER:-postgres}" +POSTGRES_PASSWORD="${CDC_POSTGRES_PASSWORD:-test.123}" +POSTGRES_DB="${CDC_POSTGRES_DB:-postgres}" +CONTAINER_NAME="${CDC_CONTAINER_NAME:-cdc_application}" + +SQLSERVER_CONTAINER="${CDC_SQLSERVER_CONTAINER:-cdc_sqlserver}" +SQLSERVER_PASSWORD="${CDC_SQLSERVER_PASSWORD:-Test.123!}" +SQLSERVER_DB="${CDC_SQLSERVER_DB:-cdc_db}" + +# Detect sqlcmd path inside SQL Server container +detect_sqlcmd() { + if docker exec "$SQLSERVER_CONTAINER" test -x /opt/mssql-tools18/bin/sqlcmd 2>/dev/null; then + echo "/opt/mssql-tools18/bin/sqlcmd" + elif docker exec "$SQLSERVER_CONTAINER" test -x /opt/mssql-tools/bin/sqlcmd 2>/dev/null; then + echo "/opt/mssql-tools/bin/sqlcmd" + else + echo "" + fi +} + +SQLCMD_PATH="" + +# PGBench configuration +PGBENCH_SCALE="${PGBENCH_SCALE:-32}" +PGBENCH_CLIENTS="${PGBENCH_CLIENTS:-100}" +PGBENCH_THREADS="${PGBENCH_THREADS:-8}" +PGBENCH_TRANSACTIONS="${PGBENCH_TRANSACTIONS:-12}" +PGBENCH_SCRIPT="${PGBENCH_SCRIPT:-$PROJECT_ROOT/examples/scripts/pgbench_testing.sql}" + +# Verification configuration +MAX_RETRIES=60 +RETRY_INTERVAL=45 +EXPECTED_ROW_COUNT=$((3000 * PGBENCH_CLIENTS * PGBENCH_TRANSACTIONS)) + +# Connection string +POSTGRES_CONNSTRING="postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?user=${POSTGRES_USER}&password=${POSTGRES_PASSWORD}" + +# Script paths +CHAOS_SCRIPT="$SCRIPT_DIR/chaos_script.sh" +CHAOS_SCRIPT_PID="" + +# Results file +RESULTS_FILE="$SCRIPT_DIR/pgbench_chaos_sqlserver_results_$(date +%Y%m%d_%H%M%S).log" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" | tee -a "$RESULTS_FILE" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $*" | tee -a "$RESULTS_FILE" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $*" | tee -a "$RESULTS_FILE" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" | tee -a "$RESULTS_FILE" >&2 +} + +log_section() { + echo -e "${CYAN}========================================${NC}" | tee -a "$RESULTS_FILE" + echo -e "${CYAN}$*${NC}" | tee -a "$RESULTS_FILE" + echo -e "${CYAN}========================================${NC}" | tee -a "$RESULTS_FILE" +} + +# Cleanup function +cleanup() { + log_info "Cleaning up..." + + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + fi + + log_info "Cleanup complete." + log_info "Results saved to: $RESULTS_FILE" +} + +# Set up trap for cleanup +trap cleanup EXIT INT TERM + +# Function to check if container exists +check_container() { + if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container '$CONTAINER_NAME' not found. Please start docker-compose first." + exit 1 + fi + + log_success "Container '$CONTAINER_NAME' found." +} + +# Function to check if pgbench is installed +check_pgbench() { + if ! command -v pgbench &> /dev/null; then + log_error "pgbench is not installed. Please install PostgreSQL client tools." + exit 1 + fi + + log_success "pgbench is installed." +} + +# Function to test PostgreSQL connection +test_postgres_connection() { + log_info "Testing PostgreSQL connection..." + + if PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 1" > /dev/null 2>&1; then + log_success "PostgreSQL connection successful." + return 0 + else + log_error "Cannot connect to PostgreSQL." + return 1 + fi +} + +# Function to test SQL Server connection +test_sqlserver_connection() { + log_info "Testing SQL Server connection..." + + if docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ + -Q "SELECT 1" -b > /dev/null 2>&1; then + log_success "SQL Server connection successful." + return 0 + else + log_error "Cannot connect to SQL Server." + return 1 + fi +} + +# Function to get row count from SQL Server t1 table +get_sqlserver_row_count() { + local count=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ + -d "$SQLSERVER_DB" \ + -Q "SET NOCOUNT ON; SELECT COUNT(*) FROM dbo.t1;" \ + -h -1 -W 2>/dev/null | tr -d '[:space:]') + + echo "$count" +} + +# Function to verify replication completed +verify_replication() { + local current_count=$(get_sqlserver_row_count) + + if [ -z "$current_count" ]; then + log_warning "Failed to get row count from SQL Server" + return 1 + fi + + log_info "Current SQL Server row count: $current_count / Expected: $EXPECTED_ROW_COUNT" + + if [ "$current_count" -eq "$EXPECTED_ROW_COUNT" ] 2>/dev/null; then + return 0 + else + return 1 + fi +} + +# Function to initialize pgbench +initialize_pgbench() { + log_section "Initializing PGBench (Scale: $PGBENCH_SCALE)" + + local start_time=$(date +%s) + + if pgbench -i -s "$PGBENCH_SCALE" --unlogged-tables --foreign-keys "$POSTGRES_CONNSTRING" 2>&1 | tee -a "$RESULTS_FILE"; then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + log_success "PGBench initialization completed in ${duration}s" + return 0 + else + log_error "PGBench initialization failed" + return 1 + fi +} + +# Function to run pgbench benchmark +run_pgbench_benchmark() { + log_section "Running PGBench Benchmark" + log_info "Configuration:" + log_info " - Clients: $PGBENCH_CLIENTS" + log_info " - Threads: $PGBENCH_THREADS" + log_info " - Transactions per client: $PGBENCH_TRANSACTIONS" + log_info " - Script: $PGBENCH_SCRIPT" + echo "" | tee -a "$RESULTS_FILE" + + local start_time=$(date +%s) + + # Check if custom script exists + local pgbench_cmd="pgbench -c $PGBENCH_CLIENTS -j $PGBENCH_THREADS -t $PGBENCH_TRANSACTIONS" + + if [ -f "$PGBENCH_SCRIPT" ]; then + log_info "Using custom script: $PGBENCH_SCRIPT" + pgbench_cmd="$pgbench_cmd -f $PGBENCH_SCRIPT" + else + log_warning "Custom script not found: $PGBENCH_SCRIPT" + fi + + pgbench_cmd="$pgbench_cmd $POSTGRES_CONNSTRING" + + echo "" | tee -a "$RESULTS_FILE" + + if eval "$pgbench_cmd" 2>&1 | tee -a "$RESULTS_FILE"; then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + log_success "PGBench benchmark completed in ${duration}s" + return 0 + else + log_error "PGBench benchmark failed" + return 1 + fi +} + +# Function to start chaos testing +start_chaos_testing() { + log_section "Starting Chaos Testing" + + if [ ! -f "$CHAOS_SCRIPT" ]; then + log_error "Chaos script not found: $CHAOS_SCRIPT" + exit 1 + fi + + chmod +x "$CHAOS_SCRIPT" + + log_info "Starting chaos script for container: $CONTAINER_NAME" + "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/pgbench_chaos_script.log" 2>&1 & + CHAOS_SCRIPT_PID=$! + + log_success "Chaos script started with PID: $CHAOS_SCRIPT_PID" + log_info "Chaos script logs: $SCRIPT_DIR/pgbench_chaos_script.log" + + sleep 5 +} + +# Main execution +main() { + export PGPASSWORD=$POSTGRES_PASSWORD + log_section "PGBench Chaos Integration Test (SQL Server)" + log_info "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" + log_info "Container under test: $CONTAINER_NAME" + log_info "SQL Server container: $SQLSERVER_CONTAINER" + + # Detect sqlcmd path + SQLCMD_PATH=$(detect_sqlcmd) + if [ -z "$SQLCMD_PATH" ]; then + log_error "sqlcmd not found in SQL Server container" + exit 1 + fi + log_info "Using sqlcmd: $SQLCMD_PATH" + + echo "" | tee -a "$RESULTS_FILE" + + # Pre-flight checks + log_info "Running pre-flight checks..." + check_pgbench + test_postgres_connection || exit 1 + test_sqlserver_connection || exit 1 + check_container + echo "" | tee -a "$RESULTS_FILE" + + # Initialize pgbench + if ! initialize_pgbench; then + log_error "Failed to initialize pgbench. Exiting." + exit 1 + fi + echo "" | tee -a "$RESULTS_FILE" + + # Wait for initialization to settle + log_info "Waiting for initialization to settle..." + sleep 10 + + # Start chaos testing + start_chaos_testing + echo "" | tee -a "$RESULTS_FILE" + + # Wait for chaos to take effect + log_info "Allowing chaos script to run for a few cycles..." + sleep 15 + + # Run benchmark + local benchmark_result=0 + if ! run_pgbench_benchmark; then + log_error "Benchmark failed." + benchmark_result=1 + fi + + echo "" | tee -a "$RESULTS_FILE" + + # Verify replication with retry loop + log_section "Verifying Replication" + log_info "Expected row count: $EXPECTED_ROW_COUNT" + log_info "Max retries: $MAX_RETRIES" + log_info "Retry interval: $RETRY_INTERVAL seconds" + echo "" | tee -a "$RESULTS_FILE" + + local retry_count=0 + local verification_passed=false + + while [ $retry_count -lt $MAX_RETRIES ]; do + retry_count=$((retry_count + 1)) + log_info "Verification attempt $retry_count/$MAX_RETRIES..." + + if verify_replication; then + log_success "Replication verification PASSED on attempt $retry_count" + verification_passed=true + break + else + log_warning "Replication verification failed (attempt $retry_count/$MAX_RETRIES)" + + if [ $retry_count -lt $MAX_RETRIES ]; then + log_info "Waiting $RETRY_INTERVAL seconds before retry..." + sleep "$RETRY_INTERVAL" + fi + fi + done + + echo "" | tee -a "$RESULTS_FILE" + + # Summary + log_section "Test Complete" + log_info "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" + + if [ $benchmark_result -eq 0 ] && [ "$verification_passed" = true ]; then + log_success "PGBench chaos test completed successfully! (SQL Server)" + log_success "All $EXPECTED_ROW_COUNT rows replicated to SQL Server" + log_info "Check the results file for detailed metrics: $RESULTS_FILE" + exit 0 + else + if [ $benchmark_result -ne 0 ]; then + log_error "Benchmark failed!" + fi + if [ "$verification_passed" = false ]; then + log_error "Replication verification failed after $MAX_RETRIES attempts!" + fi + exit 1 + fi +} + +# Run main function +main "$@" From 9666836a579c362f1f208e90f607a061952a1e77 Mon Sep 17 00:00:00 2001 From: danielshih Date: Sat, 25 Apr 2026 15:34:38 +0000 Subject: [PATCH 2/5] Add container wait functions and enhance verification logic in chaos test scripts --- tests/chaos/scripts/run_chaos_tests_sqlite.sh | 29 +++++++- .../scripts/run_chaos_tests_sqlserver.sh | 37 +++++++--- .../scripts/run_pgbench_chaos_test_sqlite.sh | 60 +++++++++++++++- .../run_pgbench_chaos_test_sqlserver.sh | 68 +++++++++++++++++-- 4 files changed, 174 insertions(+), 20 deletions(-) diff --git a/tests/chaos/scripts/run_chaos_tests_sqlite.sh b/tests/chaos/scripts/run_chaos_tests_sqlite.sh index 35f3c36..4d40ad4 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlite.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlite.sh @@ -80,6 +80,23 @@ cleanup() { # Set up trap for cleanup trap cleanup EXIT INT TERM +# Function to wait for container to be running +wait_for_container_running() { + local container="$1" + local max_wait="${2:-120}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then + return 0 + fi + log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + sleep 5 + waited=$((waited + 5)) + done + log_error "Container '$container' did not start within ${max_wait}s" + return 1 +} + # Function to execute PostgreSQL SQL execute_postgres_sql() { local sql_file="$1" @@ -99,12 +116,18 @@ execute_postgres_sql() { # Function to verify test result via docker exec with sqlite3 verify_scenario() { local verify_file="$1" - local verify_filename=$(basename "$verify_file") + local verify_filename + verify_filename=$(basename "$verify_file") + + if ! wait_for_container_running "$CDC_CONTAINER" 120; then + log_error "Container not running, cannot verify scenario" + return 1 + fi - local result=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ + local result + result=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ ".read /verify/$verify_filename" 2>&1) - # Check if result contains "PASS" if echo "$result" | grep -q "PASS"; then return 0 else diff --git a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh index fb49f1c..1ef54c5 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh @@ -14,11 +14,9 @@ ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - while IFS='=' read -r key value; do - [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue - [[ "$value" == *" "* ]] && continue - export "$key=$value" - done < "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi @@ -96,6 +94,23 @@ cleanup() { # Set up trap for cleanup trap cleanup EXIT INT TERM +# Function to wait for container to be running +wait_for_container_running() { + local container="$1" + local max_wait="${2:-120}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then + return 0 + fi + log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + sleep 5 + waited=$((waited + 5)) + done + log_error "Container '$container' did not start within ${max_wait}s" + return 1 +} + # Function to execute PostgreSQL SQL execute_postgres_sql() { local sql_file="$1" @@ -129,15 +144,21 @@ execute_sqlserver_sql() { # Function to verify test result via mounted verify scripts verify_scenario() { local verify_file="$1" - local verify_filename=$(basename "$verify_file") + local verify_filename + verify_filename=$(basename "$verify_file") + + if ! wait_for_container_running "$SQLSERVER_CONTAINER" 120; then + log_error "SQL Server container not running, cannot verify scenario" + return 1 + fi - local result=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + local result + result=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ -d "$SQLSERVER_DB" \ -i "/verify/$verify_filename" \ -h -1 -W 2>&1) - # Check if result contains "PASS" if echo "$result" | grep -q "PASS"; then return 0 else diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh index 3a7479d..6695cb3 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh @@ -50,7 +50,6 @@ PGBENCH_SCRIPT="${PGBENCH_SCRIPT:-$PROJECT_ROOT/examples/scripts/pgbench_testing # Verification configuration MAX_RETRIES=60 RETRY_INTERVAL=45 -EXPECTED_ROW_COUNT=$((3000 * PGBENCH_CLIENTS * PGBENCH_TRANSACTIONS)) # Connection string POSTGRES_CONNSTRING="postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?user=${POSTGRES_USER}&password=${POSTGRES_PASSWORD}" @@ -102,6 +101,43 @@ cleanup() { # Set up trap for cleanup trap cleanup EXIT INT TERM +# Function to wait for container to be running +wait_for_container_running() { + local container="$1" + local max_wait="${2:-120}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then + return 0 + fi + log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + sleep 5 + waited=$((waited + 5)) + done + log_error "Container '$container' did not start within ${max_wait}s" + return 1 +} + +# Function to stop chaos script and ensure container is healthy +stop_chaos_and_stabilize() { + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script before verification (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + CHAOS_SCRIPT_PID="" + fi + + log_info "Ensuring CDC container is running and stable..." + if ! wait_for_container_running "$CDC_CONTAINER" 120; then + log_error "CDC container failed to start after stopping chaos" + return 1 + fi + + sleep 10 + log_success "Container stabilized for verification phase" + return 0 +} + # Function to check if container exists check_container() { if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then @@ -150,7 +186,13 @@ test_sqlite_connection() { # Function to get row count from SQLite t1 table get_sqlite_row_count() { - local count=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ + if ! wait_for_container_running "$CDC_CONTAINER" 120; then + echo "" + return 1 + fi + + local count + count=$(docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ "SELECT COUNT(*) FROM t1;" 2>/dev/null | tr -d '[:space:]') echo "$count" @@ -294,6 +336,20 @@ main() { echo "" | tee -a "$RESULTS_FILE" + # Query actual row count from PG source (some transactions may have failed under load) + EXPECTED_ROW_COUNT=$(PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT COUNT(*) FROM t1;" 2>/dev/null | tr -d '[:space:]') + if [ -z "$EXPECTED_ROW_COUNT" ] || [ "$EXPECTED_ROW_COUNT" -eq 0 ] 2>/dev/null; then + log_error "Failed to get row count from PostgreSQL source or no rows found" + exit 1 + fi + log_info "Actual PostgreSQL source row count: $EXPECTED_ROW_COUNT" + + # Stop chaos and stabilize container before verification + if ! stop_chaos_and_stabilize; then + log_error "Failed to stabilize container for verification" + exit 1 + fi + # Verify replication with retry loop log_section "Verifying Replication" log_info "Expected row count: $EXPECTED_ROW_COUNT" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh index 95e8fb5..d22cab3 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh @@ -14,11 +14,9 @@ ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - while IFS='=' read -r key value; do - [[ "$key" =~ ^#.*$ || -z "$key" ]] && continue - [[ "$value" == *" "* ]] && continue - export "$key=$value" - done < "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi @@ -66,7 +64,6 @@ PGBENCH_SCRIPT="${PGBENCH_SCRIPT:-$PROJECT_ROOT/examples/scripts/pgbench_testing # Verification configuration MAX_RETRIES=60 RETRY_INTERVAL=45 -EXPECTED_ROW_COUNT=$((3000 * PGBENCH_CLIENTS * PGBENCH_TRANSACTIONS)) # Connection string POSTGRES_CONNSTRING="postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?user=${POSTGRES_USER}&password=${POSTGRES_PASSWORD}" @@ -118,6 +115,43 @@ cleanup() { # Set up trap for cleanup trap cleanup EXIT INT TERM +# Function to wait for container to be running +wait_for_container_running() { + local container="$1" + local max_wait="${2:-120}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then + return 0 + fi + log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + sleep 5 + waited=$((waited + 5)) + done + log_error "Container '$container' did not start within ${max_wait}s" + return 1 +} + +# Function to stop chaos script and ensure container is healthy +stop_chaos_and_stabilize() { + if [ -n "$CHAOS_SCRIPT_PID" ] && kill -0 "$CHAOS_SCRIPT_PID" 2>/dev/null; then + log_info "Stopping chaos script before verification (PID: $CHAOS_SCRIPT_PID)..." + kill "$CHAOS_SCRIPT_PID" 2>/dev/null || true + wait "$CHAOS_SCRIPT_PID" 2>/dev/null || true + CHAOS_SCRIPT_PID="" + fi + + log_info "Ensuring CDC container is running and stable..." + if ! wait_for_container_running "$CONTAINER_NAME" 120; then + log_error "CDC container failed to start after stopping chaos" + return 1 + fi + + sleep 10 + log_success "Container stabilized for verification phase" + return 0 +} + # Function to check if container exists check_container() { if ! docker ps -a --filter "name=^${CONTAINER_NAME}$" --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then @@ -168,7 +202,13 @@ test_sqlserver_connection() { # Function to get row count from SQL Server t1 table get_sqlserver_row_count() { - local count=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ + if ! wait_for_container_running "$SQLSERVER_CONTAINER" 120; then + echo "" + return 1 + fi + + local count + count=$(docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ -d "$SQLSERVER_DB" \ -Q "SET NOCOUNT ON; SELECT COUNT(*) FROM dbo.t1;" \ @@ -324,6 +364,20 @@ main() { echo "" | tee -a "$RESULTS_FILE" + # Query actual row count from PG source (some transactions may have failed under load) + EXPECTED_ROW_COUNT=$(PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT COUNT(*) FROM t1;" 2>/dev/null | tr -d '[:space:]') + if [ -z "$EXPECTED_ROW_COUNT" ] || [ "$EXPECTED_ROW_COUNT" -eq 0 ] 2>/dev/null; then + log_error "Failed to get row count from PostgreSQL source or no rows found" + exit 1 + fi + log_info "Actual PostgreSQL source row count: $EXPECTED_ROW_COUNT" + + # Stop chaos and stabilize container before verification + if ! stop_chaos_and_stabilize; then + log_error "Failed to stabilize container for verification" + exit 1 + fi + # Verify replication with retry loop log_section "Verifying Replication" log_info "Expected row count: $EXPECTED_ROW_COUNT" From 449b5b81f8c7b29f1c43f6aa7a6b3fa982aee943 Mon Sep 17 00:00:00 2001 From: danielshih Date: Sat, 25 Apr 2026 23:58:19 +0000 Subject: [PATCH 3/5] Update restart policy for SQLite and SQL Server services; enhance environment loading in chaos test scripts --- docker-compose.chaos-test-sqlite.yml | 2 +- docker-compose.chaos-test-sqlserver.yml | 2 +- tests/chaos/scripts/run_chaos_tests_sqlite.sh | 17 +++++++++++++---- .../chaos/scripts/run_chaos_tests_sqlserver.sh | 17 +++++++++++++---- .../scripts/run_pgbench_chaos_test_sqlite.sh | 17 +++++++++++++---- .../scripts/run_pgbench_chaos_test_sqlserver.sh | 17 +++++++++++++---- 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/docker-compose.chaos-test-sqlite.yml b/docker-compose.chaos-test-sqlite.yml index fa40c9f..6efd8f9 100644 --- a/docker-compose.chaos-test-sqlite.yml +++ b/docker-compose.chaos-test-sqlite.yml @@ -51,7 +51,7 @@ services: - sqlite_data:/app/data - ./examples/scripts/init_sqlite.sql:/init/init_sqlite.sql:ro - ./tests/chaos/scenarios/verify_sqlite:/verify:ro - restart: "no" + restart: unless-stopped stop_grace_period: 300s stop_signal: SIGTERM healthcheck: diff --git a/docker-compose.chaos-test-sqlserver.yml b/docker-compose.chaos-test-sqlserver.yml index 8feb0df..e6d0f0f 100644 --- a/docker-compose.chaos-test-sqlserver.yml +++ b/docker-compose.chaos-test-sqlserver.yml @@ -71,7 +71,7 @@ services: CDC_LAST_LSN_FILE: /app/pg2any_last_lsn.metadata ports: - "8080:8080" - restart: "no" + restart: unless-stopped stop_grace_period: 300s stop_signal: SIGTERM healthcheck: diff --git a/tests/chaos/scripts/run_chaos_tests_sqlite.sh b/tests/chaos/scripts/run_chaos_tests_sqlite.sh index 4d40ad4..8b9cd4b 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlite.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlite.sh @@ -12,11 +12,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" +load_env_file() { + local env_file="$1" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" != *=* ]] && continue + export "$line" + done < "$env_file" +} + if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - set -a - source "$ENV_FILE" - set +a + load_env_file "$ENV_FILE" else echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" fi @@ -89,7 +96,8 @@ wait_for_container_running() { if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then return 0 fi - log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + log_warning "Container '$container' is not running. Attempting to start... (${waited}s/${max_wait}s)" + docker start "$container" 2>/dev/null || true sleep 5 waited=$((waited + 5)) done @@ -246,6 +254,7 @@ main() { # Start chaos script in background log_info "Starting chaos script for container: $CONTAINER_NAME" + rm -f "$SCRIPT_DIR/chaos_script.log" "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/chaos_script.log" 2>&1 & CHAOS_SCRIPT_PID=$! log_info "Chaos script started with PID: $CHAOS_SCRIPT_PID" diff --git a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh index 1ef54c5..255faf3 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh @@ -12,11 +12,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" +load_env_file() { + local env_file="$1" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" != *=* ]] && continue + export "$line" + done < "$env_file" +} + if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - set -a - source "$ENV_FILE" - set +a + load_env_file "$ENV_FILE" else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi @@ -103,7 +110,8 @@ wait_for_container_running() { if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then return 0 fi - log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + log_warning "Container '$container' is not running. Attempting to start... (${waited}s/${max_wait}s)" + docker start "$container" 2>/dev/null || true sleep 5 waited=$((waited + 5)) done @@ -289,6 +297,7 @@ main() { # Start chaos script in background log_info "Starting chaos script for container: $CONTAINER_NAME" + rm -f "$SCRIPT_DIR/chaos_script.log" "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/chaos_script.log" 2>&1 & CHAOS_SCRIPT_PID=$! log_info "Chaos script started with PID: $CHAOS_SCRIPT_PID" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh index 6695cb3..2dc2fae 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh @@ -12,11 +12,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" +load_env_file() { + local env_file="$1" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" != *=* ]] && continue + export "$line" + done < "$env_file" +} + if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - set -a - source "$ENV_FILE" - set +a + load_env_file "$ENV_FILE" else echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" fi @@ -110,7 +117,8 @@ wait_for_container_running() { if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then return 0 fi - log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + log_warning "Container '$container' is not running. Attempting to start... (${waited}s/${max_wait}s)" + docker start "$container" 2>/dev/null || true sleep 5 waited=$((waited + 5)) done @@ -282,6 +290,7 @@ start_chaos_testing() { chmod +x "$CHAOS_SCRIPT" log_info "Starting chaos script for container: $CONTAINER_NAME" + rm -f "$SCRIPT_DIR/pgbench_chaos_script.log" "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/pgbench_chaos_script.log" 2>&1 & CHAOS_SCRIPT_PID=$! diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh index d22cab3..84fa39a 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh @@ -12,11 +12,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" +load_env_file() { + local env_file="$1" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" != *=* ]] && continue + export "$line" + done < "$env_file" +} + if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - set -a - source "$ENV_FILE" - set +a + load_env_file "$ENV_FILE" else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi @@ -124,7 +131,8 @@ wait_for_container_running() { if docker ps --filter "name=^${container}$" --format "{{.Status}}" | grep -q "^Up"; then return 0 fi - log_warning "Container '$container' is not running. Waiting... (${waited}s/${max_wait}s)" + log_warning "Container '$container' is not running. Attempting to start... (${waited}s/${max_wait}s)" + docker start "$container" 2>/dev/null || true sleep 5 waited=$((waited + 5)) done @@ -301,6 +309,7 @@ start_chaos_testing() { chmod +x "$CHAOS_SCRIPT" log_info "Starting chaos script for container: $CONTAINER_NAME" + rm -f "$SCRIPT_DIR/pgbench_chaos_script.log" "$CHAOS_SCRIPT" "$CONTAINER_NAME" > "$SCRIPT_DIR/pgbench_chaos_script.log" 2>&1 & CHAOS_SCRIPT_PID=$! From 293798ad16cb8538453fbd91b329652f2b29e80a Mon Sep 17 00:00:00 2001 From: danielshih Date: Sun, 26 Apr 2026 09:35:59 +0000 Subject: [PATCH 4/5] Refactor verification scripts for high-volume data replication; update expected row count logic and improve variable usage in chaos test scripts --- env/.env.sqlite | 2 +- .../scenarios/verify_sqlite/scenario5_verify.sql | 15 +++++++++------ .../verify_sqlserver/scenario5_verify.sql | 12 +++++++----- tests/chaos/scripts/run_chaos_tests_sqlite.sh | 3 ++- tests/chaos/scripts/run_chaos_tests_sqlserver.sh | 3 ++- .../scripts/run_pgbench_chaos_test_sqlite.sh | 3 ++- .../scripts/run_pgbench_chaos_test_sqlserver.sh | 3 ++- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/env/.env.sqlite b/env/.env.sqlite index f7c31b2..f0fc1c9 100644 --- a/env/.env.sqlite +++ b/env/.env.sqlite @@ -20,7 +20,7 @@ CDC_PROTOCOL_VERSION=3 CDC_BINARY_FORMAT=true CDC_STREAMING=true CDC_BUFFER_SIZE=1000 -CDC_COMMIT_BATCH_SIZE=1000 +CDC_COMMIT_BATCH_SIZE=500 # Timeout Configuration (in seconds) CDC_CONNECTION_TIMEOUT=30 diff --git a/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql b/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql index 9f92af4..04d621a 100644 --- a/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql +++ b/tests/chaos/scenarios/verify_sqlite/scenario5_verify.sql @@ -1,14 +1,17 @@ --- Scenario 5: Verification - Check if high-volume data (3M rows) was replicated +-- Scenario 5: Verification - Check if high-volume data was replicated +-- Expected row count must match scenario5_input.sql generate_series(1, N) +WITH expected(cnt) AS (VALUES (3000000)) SELECT CASE - WHEN COUNT(*) = 3000000 THEN 'PASS' + WHEN COUNT(*) = expected.cnt THEN 'PASS' ELSE 'FAIL' END AS test_result, COUNT(*) AS actual_count, - 3000000 AS expected_count, + expected.cnt AS expected_count, CASE - WHEN COUNT(*) = 3000000 THEN 'All 3M rows replicated successfully' - WHEN COUNT(*) > 0 THEN 'Partial replication: ' || CAST(COUNT(*) AS TEXT) || ' of 3M rows' + WHEN COUNT(*) = expected.cnt THEN 'All ' || CAST(expected.cnt AS TEXT) || ' rows replicated successfully' + WHEN COUNT(*) > 0 THEN 'Partial replication: ' || CAST(COUNT(*) AS TEXT) || ' of ' || CAST(expected.cnt AS TEXT) || ' rows' ELSE 'No data replicated' END AS status_message -FROM t1; +FROM t1, expected +GROUP BY expected.cnt; diff --git a/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql b/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql index f61ee32..3371bcc 100644 --- a/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql +++ b/tests/chaos/scenarios/verify_sqlserver/scenario5_verify.sql @@ -1,14 +1,16 @@ --- Scenario 5: Verification - Check if high-volume data (3M rows) was replicated +-- Scenario 5: Verification - Check if high-volume data was replicated +-- Expected row count must match scenario5_input.sql generate_series(1, N) +DECLARE @expected_count INT = 3000000; SELECT CASE - WHEN COUNT(*) = 3000000 THEN 'PASS' + WHEN COUNT(*) = @expected_count THEN 'PASS' ELSE 'FAIL' END AS test_result, COUNT(*) AS actual_count, - 3000000 AS expected_count, + @expected_count AS expected_count, CASE - WHEN COUNT(*) = 3000000 THEN 'All 3M rows replicated successfully' - WHEN COUNT(*) > 0 THEN CONCAT('Partial replication: ', CAST(COUNT(*) AS VARCHAR(20)), ' of 3M rows') + WHEN COUNT(*) = @expected_count THEN CONCAT('All ', @expected_count, ' rows replicated successfully') + WHEN COUNT(*) > 0 THEN CONCAT('Partial replication: ', CAST(COUNT(*) AS VARCHAR(20)), ' of ', @expected_count, ' rows') ELSE 'No data replicated' END AS status_message FROM dbo.t1; diff --git a/tests/chaos/scripts/run_chaos_tests_sqlite.sh b/tests/chaos/scripts/run_chaos_tests_sqlite.sh index 8b9cd4b..c86b2e0 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlite.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlite.sh @@ -280,7 +280,8 @@ main() { for scenario_file in "${scenario_files[@]}"; do # Extract scenario number from filename - local scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') + local scenario_num + scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') # Clean up before each scenario cleanup_test_data diff --git a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh index 255faf3..71a91fa 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh @@ -323,7 +323,8 @@ main() { for scenario_file in "${scenario_files[@]}"; do # Extract scenario number from filename - local scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') + local scenario_num + scenario_num=$(basename "$scenario_file" | sed 's/scenario\([0-9]*\)_input.sql/\1/') # Clean up before each scenario cleanup_test_data diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh index 2dc2fae..151797a 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh @@ -208,7 +208,8 @@ get_sqlite_row_count() { # Function to verify replication completed verify_replication() { - local current_count=$(get_sqlite_row_count) + local current_count + current_count=$(get_sqlite_row_count) if [ -z "$current_count" ]; then log_warning "Failed to get row count from SQLite" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh index 84fa39a..8ab725d 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh @@ -227,7 +227,8 @@ get_sqlserver_row_count() { # Function to verify replication completed verify_replication() { - local current_count=$(get_sqlserver_row_count) + local current_count + current_count=$(get_sqlserver_row_count) if [ -z "$current_count" ]; then log_warning "Failed to get row count from SQL Server" From 40614eff2d54d1e5b04000c50cfe8e6588104ef8 Mon Sep 17 00:00:00 2001 From: danielshih Date: Sun, 26 Apr 2026 10:03:54 +0000 Subject: [PATCH 5/5] Refactor environment variable loading in chaos test scripts for SQLite and SQL Server; streamline code by using 'set -a' and 'source' --- Makefile | 5 ++-- tests/chaos/scripts/run_chaos_tests_sqlite.sh | 14 +++------- .../scripts/run_chaos_tests_sqlserver.sh | 27 +++---------------- .../scripts/run_pgbench_chaos_test_sqlite.sh | 13 +++------ .../run_pgbench_chaos_test_sqlserver.sh | 13 +++------ 5 files changed, 16 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index 0932b22..4888528 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,7 @@ pgbench-test-mysql-logs: @docker logs -f cdc_application # Helper: find sqlcmd path inside SQL Server container +SQLSERVER_SA_PASSWORD ?= Test.123! SQLCMD_EXEC = docker exec cdc_sqlserver bash -c 'if [ -x /opt/mssql-tools18/bin/sqlcmd ]; then /opt/mssql-tools18/bin/sqlcmd "$$@"; elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then /opt/mssql-tools/bin/sqlcmd "$$@"; else echo "sqlcmd not found" >&2; exit 1; fi' -- # SQL Server Chaos Testing commands @@ -199,7 +200,7 @@ chaos-test-sqlserver-setup: @echo "Waiting for SQL Server to be ready..." @timeout 120 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' @echo "Initializing SQL Server database..." - @$(SQLCMD_EXEC) -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + @$(SQLCMD_EXEC) -S localhost -U sa -P '$(SQLSERVER_SA_PASSWORD)' -C -i /init/init_sqlserver.sql @echo "Starting CDC application..." @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app @echo "Waiting for CDC application to initialize..." @@ -236,7 +237,7 @@ pgbench-test-sqlserver-setup: @echo "Waiting for SQL Server to be ready..." @timeout 120 bash -c 'until docker ps --filter name=cdc_sqlserver --format "{{.Status}}" | grep -q healthy; do sleep 5; done' @echo "Initializing SQL Server database..." - @$(SQLCMD_EXEC) -S localhost -U sa -P 'Test.123!' -C -i /init/init_sqlserver.sql + @$(SQLCMD_EXEC) -S localhost -U sa -P '$(SQLSERVER_SA_PASSWORD)' -C -i /init/init_sqlserver.sql @echo "Starting CDC application..." @docker-compose -f docker-compose.chaos-test-sqlserver.yml up -d cdc_app @echo "Waiting for CDC application to initialize..." diff --git a/tests/chaos/scripts/run_chaos_tests_sqlite.sh b/tests/chaos/scripts/run_chaos_tests_sqlite.sh index c86b2e0..98f7312 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlite.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlite.sh @@ -12,18 +12,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" -load_env_file() { - local env_file="$1" - while IFS= read -r line || [[ -n "$line" ]]; do - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - [[ "$line" != *=* ]] && continue - export "$line" - done < "$env_file" -} - if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - load_env_file "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" fi @@ -157,6 +150,7 @@ cleanup_test_data() { -c "TRUNCATE TABLE public.t1;" > /dev/null 2>&1 || true # Clean SQLite (DELETE instead of TRUNCATE - SQLite doesn't support TRUNCATE) + wait_for_container_running "$CDC_CONTAINER" 60 docker exec "$CDC_CONTAINER" sqlite3 "$SQLITE_DB_PATH" \ "DELETE FROM t1;" 2>&1 || true diff --git a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh index 71a91fa..8cf2612 100755 --- a/tests/chaos/scripts/run_chaos_tests_sqlserver.sh +++ b/tests/chaos/scripts/run_chaos_tests_sqlserver.sh @@ -12,18 +12,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" -load_env_file() { - local env_file="$1" - while IFS= read -r line || [[ -n "$line" ]]; do - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - [[ "$line" != *=* ]] && continue - export "$line" - done < "$env_file" -} - if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - load_env_file "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi @@ -135,20 +128,6 @@ execute_postgres_sql() { return ${PIPESTATUS[0]} } -# Function to execute SQL Server SQL via docker exec -execute_sqlserver_sql() { - local sql="$1" - log_info "Executing SQL Server SQL..." - - docker exec "$SQLSERVER_CONTAINER" $SQLCMD_PATH \ - -S localhost -U sa -P "$SQLSERVER_PASSWORD" -C \ - -d "$SQLSERVER_DB" \ - -Q "$sql" \ - 2>&1 - - return ${PIPESTATUS[0]} -} - # Function to verify test result via mounted verify scripts verify_scenario() { local verify_file="$1" diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh index 151797a..08e9eb2 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlite.sh @@ -12,18 +12,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlite" -load_env_file() { - local env_file="$1" - while IFS= read -r line || [[ -n "$line" ]]; do - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - [[ "$line" != *=* ]] && continue - export "$line" - done < "$env_file" -} - if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - load_env_file "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlite file not found at $ENV_FILE, using defaults" fi diff --git a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh index 8ab725d..0c41322 100755 --- a/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh +++ b/tests/chaos/scripts/run_pgbench_chaos_test_sqlserver.sh @@ -12,18 +12,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ENV_FILE="$PROJECT_ROOT/env/.env.sqlserver" -load_env_file() { - local env_file="$1" - while IFS= read -r line || [[ -n "$line" ]]; do - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - [[ "$line" != *=* ]] && continue - export "$line" - done < "$env_file" -} - if [ -f "$ENV_FILE" ]; then echo "Loading environment from: $ENV_FILE" - load_env_file "$ENV_FILE" + set -a + source "$ENV_FILE" + set +a else echo "Warning: .env.sqlserver file not found at $ENV_FILE, using defaults" fi