diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b8731b2..934b4d2 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -16,13 +16,7 @@ This project follows the [Contributor Covenant](https://www.contributor-covenant ## Development Setup -The `testing/` directory contains OpenTofu modules for provisioning PostgreSQL instances on major cloud providers (AWS RDS, Aurora, GCP Cloud SQL, Azure). Use these to validate your changes against real managed database environments. - -```bash -cd testing/ -tofu init -tofu apply -``` +The `testing/` directory contains integration and pgTAP coverage used to validate pgFirstAid against live PostgreSQL environments. You can run the test suite against any database you control by setting the standard PostgreSQL connection environment variables described in `testing/integration/README.md`. ## Testing Requirements diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6168356..c3c1840 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,81 +1,26 @@ -# Manual Cloud Deploy Workflows +# Workflow Notes -This repo uses three manual deployment workflows and one reusable validation workflow: +This repo keeps `managed-db-validate.yml` as a reusable validation workflow. -- `deploy-aws-rds.yml` -- `deploy-gcp-postgres.yml` -- `azure-postgres-opentofu.yml` -- `managed-db-validate.yml` (reusable via `workflow_call`) +`managed-db-validate.yml` installs `pgFirstAid.sql`, recreates `view_pgFirstAid_managed.sql`, and runs integration tests, including the pgTAP-backed checks in the integration harness. -Deploy workflows are run manually from the Actions tab. - -AWS and GCP also support trusted PR comment triggers: - -- AWS: `/deploy-aws-rds [target|command] [command]` -- GCP: `/deploy-gcp-pg [target|command] [command]` - -## Deploy Inputs - -AWS and GCP workflows support `target` (`pg15`-`pg18` or `all`) and `command` (`plan`, `apply`, `destroy`). - -Azure workflow supports: - -- `action`: `plan`, `apply`, `destroy` -- `postgres_version`: `pg15`, `pg16`, `pg17`, `pg18` -- `personal_ip`: optional (falls back to secret) - -## Secrets - -### AWS - -- `AWS_ACCESS_KEY_ID` -- `AWS_SECRET_ACCESS_KEY` -- `AWS_ALLOWED_CIDR_BLOCK` -- `AWS_DB_PASSWORD` - -### GCP - -- `GCP_SA_KEY` -- `DEPLOY_PERSONAL_IP_CIDR` (unless provided as workflow input) -- `GCP_DB_PASSWORD` - -### Azure - -- OIDC: `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` - - or service principal JSON: `AZURE_CREDENTIALS` -- `AZURE_PERSONAL_IP` (unless provided as workflow input) -- `AZURE_DB_PASSWORD` - -### Shared deploy controls - -- `DEPLOY_TRIGGER_USER` (used by AWS/GCP manual and comment-triggered deploy checks) - -## Validation Workflow - -`managed-db-validate.yml` installs `pgFirstAid.sql`, recreates `view_pgFirstAid_managed.sql`, and runs integration tests (including pgTAP coverage through the integration test harness). - -It supports three connection modes: +## Supported connection modes - `direct`: caller passes `pg_host` - `aws`: resolves host from `aws_db_identifier` -- `gcp`: resolves host from `gcp_project_id` + `gcp_instance_name` +- `gcp`: resolves host from `gcp_project_id` and `gcp_instance_name` -Current wiring: +## Required inputs and secrets -- Azure apply calls `managed-db-validate.yml` automatically after deploy. -- AWS apply calls `managed-db-validate.yml` for each selected version after deploy. -- GCP apply calls `managed-db-validate.yml` for each selected version after deploy. +Connection details depend on the selected connection mode. The reusable workflow always requires: -## Secret Handling +- `pg_user` +- `pg_database` +- `pg_password` -- DB passwords are passed to OpenTofu as `TF_VAR_db_password`. -- Password variables in the OpenTofu stacks are marked `sensitive = true`. -- Workflows use step-level environment variables and masking for secret values used in shell steps. -- Avoid printing secret values in custom debug statements. +Provider-specific auth is optional and only needed when the workflow resolves the host automatically. -## Recommended Run Order +## Secret handling -1. Run `plan` -2. Run `apply` -3. Confirm validation results -4. Run `destroy` when done with test resources +- Passwords are passed through workflow inputs and masked by GitHub Actions +- Avoid printing secret values in custom debug statements diff --git a/.github/workflows/azure-postgres-opentofu.yml b/.github/workflows/azure-postgres-opentofu.yml deleted file mode 100644 index 32f519d..0000000 --- a/.github/workflows/azure-postgres-opentofu.yml +++ /dev/null @@ -1,154 +0,0 @@ -name: Azure PostgreSQL OpenTofu - -on: - workflow_dispatch: - inputs: - action: - description: "OpenTofu action" - required: true - type: choice - default: plan - options: - - plan - - apply - - destroy - postgres_version: - description: "Target PostgreSQL version" - required: true - type: choice - default: pg18 - options: - - pg15 - - pg16 - - pg17 - - pg18 - personal_ip: - description: "IP allowed to connect (example: 203.0.113.10). Leave blank to use AZURE_PERSONAL_IP secret." - required: false - type: string - -concurrency: - group: azure-postgres-${{ inputs.postgres_version }} - cancel-in-progress: false - -jobs: - opentofu: - name: ${{ inputs.action }} ${{ inputs.postgres_version }} - runs-on: [self-hosted, linux, pgfirstaid-ci] - outputs: - pg_host: ${{ steps.capture_connection.outputs.pg_host }} - pg_port: ${{ steps.capture_connection.outputs.pg_port }} - pg_user: ${{ steps.capture_connection.outputs.pg_user }} - pg_database: ${{ steps.capture_connection.outputs.pg_database }} - permissions: - contents: read - id-token: write - defaults: - run: - working-directory: testing/azure/deploy/${{ inputs.postgres_version }} - - env: - TF_IN_AUTOMATION: "true" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - clean: false - - - name: Azure login (OIDC) - if: ${{ secrets.AZURE_CLIENT_ID != '' && secrets.AZURE_TENANT_ID != '' && secrets.AZURE_SUBSCRIPTION_ID != '' }} - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Azure login (service principal JSON) - if: ${{ !(secrets.AZURE_CLIENT_ID != '' && secrets.AZURE_TENANT_ID != '' && secrets.AZURE_SUBSCRIPTION_ID != '') }} - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - name: Setup OpenTofu - uses: opentofu/setup-opentofu@v1 - - - name: Resolve personal IP - shell: bash - env: - AZURE_PERSONAL_IP: ${{ secrets.AZURE_PERSONAL_IP }} - run: | - PERSONAL_IP="${{ inputs.personal_ip }}" - if [ -z "$PERSONAL_IP" ]; then - PERSONAL_IP="$AZURE_PERSONAL_IP" - fi - - if [ -z "$PERSONAL_IP" ]; then - echo "::error::No personal IP provided. Set input 'personal_ip' or secret 'AZURE_PERSONAL_IP'." - exit 1 - fi - - echo "TF_VAR_personal_ip=$PERSONAL_IP" >> "$GITHUB_ENV" - - - name: Resolve DB password - shell: bash - env: - AZURE_DB_PASSWORD: ${{ secrets.AZURE_DB_PASSWORD }} - run: | - DB_PASSWORD="$AZURE_DB_PASSWORD" - - if [ -z "$DB_PASSWORD" ]; then - echo "::error::Missing secret 'AZURE_DB_PASSWORD'." - exit 1 - fi - - echo "::add-mask::$DB_PASSWORD" - echo "TF_VAR_db_password=$DB_PASSWORD" >> "$GITHUB_ENV" - - - name: OpenTofu init - run: tofu init -input=false - - - name: OpenTofu validate - run: tofu validate - - - name: OpenTofu plan - if: ${{ inputs.action == 'plan' || inputs.action == 'apply' }} - run: tofu plan -input=false -out=tfplan - - - name: OpenTofu apply - if: ${{ inputs.action == 'apply' }} - run: tofu apply -input=false -auto-approve tfplan - - - name: Show connection details - if: ${{ inputs.action == 'apply' }} - run: | - echo "Server: $(tofu output -raw server_name)" - echo "FQDN: $(tofu output -raw server_fqdn)" - echo "Database: $(tofu output -raw database_name)" - - - name: Capture connection outputs - id: capture_connection - if: ${{ inputs.action == 'apply' }} - run: | - echo "pg_host=$(tofu output -raw server_fqdn)" >> "$GITHUB_OUTPUT" - echo "pg_port=5432" >> "$GITHUB_OUTPUT" - echo "pg_user=$(tofu output -raw db_user)" >> "$GITHUB_OUTPUT" - echo "pg_database=$(tofu output -raw database_name)" >> "$GITHUB_OUTPUT" - - - name: OpenTofu destroy - if: ${{ inputs.action == 'destroy' }} - run: tofu destroy -input=false -auto-approve - - validate: - if: ${{ inputs.action == 'apply' }} - needs: opentofu - uses: ./.github/workflows/managed-db-validate.yml - with: - pg_host: ${{ needs.opentofu.outputs.pg_host }} - pg_port: ${{ needs.opentofu.outputs.pg_port }} - pg_user: ${{ needs.opentofu.outputs.pg_user }} - pg_database: ${{ needs.opentofu.outputs.pg_database }} - pg_sslmode: require - test_view_mode: managed - secrets: - pg_password: ${{ secrets.AZURE_DB_PASSWORD }} diff --git a/.github/workflows/deploy-aws-rds.yml b/.github/workflows/deploy-aws-rds.yml deleted file mode 100644 index 7ea6824..0000000 --- a/.github/workflows/deploy-aws-rds.yml +++ /dev/null @@ -1,257 +0,0 @@ -name: Deploy AWS RDS PostgreSQL - -on: - workflow_dispatch: - inputs: - target: - description: "Deploy target under testing/aws/deploy" - required: true - type: choice - options: - - all - - pg15 - - pg16 - - pg17 - - pg18 - default: all - command: - description: "OpenTofu command" - required: true - type: choice - options: - - plan - - apply - - destroy - default: plan - issue_comment: - types: [created] - -permissions: - contents: read - pull-requests: read - -jobs: - resolve: - if: | - ( - github.event_name == 'workflow_dispatch' && - github.actor == vars.DEPLOY_TRIGGER_USER - ) || - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request != null && - startsWith(github.event.comment.body, '/deploy-aws-rds') && - github.event.comment.user.login == vars.DEPLOY_TRIGGER_USER && - ( - github.event.comment.author_association == 'OWNER' || - github.event.comment.author_association == 'MEMBER' || - github.event.comment.author_association == 'COLLABORATOR' - ) - ) - name: Resolve deploy arguments - runs-on: ubuntu-latest - - outputs: - command: ${{ steps.args.outputs.command }} - versions_json: ${{ steps.args.outputs.versions_json }} - checkout_ref: ${{ steps.pr.outputs.head_sha || github.sha }} - - steps: - - name: Validate trigger user is configured - run: | - if [ -z "${{ vars.DEPLOY_TRIGGER_USER }}" ]; then - echo "::error::Repository variable DEPLOY_TRIGGER_USER is not set." - exit 1 - fi - - - name: Resolve deploy arguments - id: args - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - target="${{ inputs.target }}" - command="${{ inputs.command }}" - else - comment="${{ github.event.comment.body }}" - read -r trigger arg1 arg2 _ <<<"${comment}" - - if [ "${trigger}" != "/deploy-aws-rds" ]; then - echo "::error::Comment must start with '/deploy-aws-rds'" - exit 1 - fi - - if [ -z "${arg1}" ]; then - target="all" - command="plan" - elif [ "${arg1}" = "plan" ] || [ "${arg1}" = "apply" ] || [ "${arg1}" = "destroy" ]; then - target="all" - command="${arg1}" - else - target="${arg1}" - if [ -z "${arg2}" ]; then - command="plan" - else - command="${arg2}" - fi - fi - fi - - case "${target}" in - all|pg15|pg16|pg17|pg18) ;; - *) - echo "::error::Invalid target: ${target}" - exit 1 - ;; - esac - - case "${command}" in - plan|apply|destroy) ;; - *) - echo "::error::Invalid command: ${command}" - exit 1 - ;; - esac - - if [ "${target}" = "all" ]; then - versions_json='["pg15","pg16","pg17","pg18"]' - else - versions_json="[\"${target}\"]" - fi - - echo "target=${target}" >> "$GITHUB_OUTPUT" - echo "command=${command}" >> "$GITHUB_OUTPUT" - echo "versions_json=${versions_json}" >> "$GITHUB_OUTPUT" - - - name: Resolve pull request head - id: pr - if: github.event_name == 'issue_comment' - uses: actions/github-script@v7 - with: - script: | - const pull_number = context.payload.issue.number; - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number, - }); - - core.setOutput('head_sha', pr.head.sha); - core.setOutput('is_fork', String(pr.head.repo.fork)); - - - name: Block fork pull requests - if: github.event_name == 'issue_comment' && steps.pr.outputs.is_fork == 'true' - run: | - echo "::error::Comment-triggered deploy is blocked for fork PRs." - exit 1 - - - name: Show resolved inputs - run: | - echo "Target: ${{ steps.args.outputs.target }}" - echo "Versions: ${{ steps.args.outputs.versions_json }}" - echo "Command: ${{ steps.args.outputs.command }}" - - deploy: - name: ${{ needs.resolve.outputs.command }} ${{ matrix.postgres_version }} - runs-on: ubuntu-latest - needs: resolve - strategy: - fail-fast: false - matrix: - postgres_version: ${{ fromJSON(needs.resolve.outputs.versions_json) }} - - env: - AWS_REGION: us-west-2 - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ needs.resolve.outputs.checkout_ref }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Setup OpenTofu - uses: opentofu/setup-opentofu@v1 - with: - tofu_version: latest - - - name: Resolve OpenTofu variables - env: - AWS_ALLOWED_CIDR_BLOCK: ${{ secrets.AWS_ALLOWED_CIDR_BLOCK }} - AWS_DB_PASSWORD: ${{ secrets.AWS_DB_PASSWORD }} - run: | - if [ -z "$AWS_ALLOWED_CIDR_BLOCK" ]; then - echo "::error::Missing secret AWS_ALLOWED_CIDR_BLOCK" - exit 1 - fi - - if [ -z "$AWS_DB_PASSWORD" ]; then - echo "::error::Missing secret AWS_DB_PASSWORD" - exit 1 - fi - - echo "::add-mask::$AWS_DB_PASSWORD" - echo "TF_VAR_allowed_cidr_block=$AWS_ALLOWED_CIDR_BLOCK" >> "$GITHUB_ENV" - echo "TF_VAR_db_password=$AWS_DB_PASSWORD" >> "$GITHUB_ENV" - - - name: OpenTofu init - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: tofu init -input=false - - - name: OpenTofu validate - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: tofu validate - - - name: OpenTofu plan - if: needs.resolve.outputs.command == 'plan' || needs.resolve.outputs.command == 'apply' - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: tofu plan -input=false - - - name: OpenTofu apply - if: needs.resolve.outputs.command == 'apply' - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: tofu apply -auto-approve -input=false - - - name: OpenTofu destroy - if: needs.resolve.outputs.command == 'destroy' - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: tofu destroy -auto-approve -input=false - - - name: Show endpoint output - if: needs.resolve.outputs.command == 'apply' - working-directory: testing/aws/deploy/${{ matrix.postgres_version }} - run: | - endpoint="$(tofu output -raw endpoint)" - { - echo "### Deployment Output" - echo - echo "- Version: ${{ matrix.postgres_version }}" - echo "- Endpoint: ${endpoint}" - } >> "$GITHUB_STEP_SUMMARY" - - validate: - if: needs.resolve.outputs.command == 'apply' - name: validate ${{ matrix.postgres_version }} - needs: [resolve, deploy] - strategy: - fail-fast: false - matrix: - postgres_version: ${{ fromJSON(needs.resolve.outputs.versions_json) }} - uses: ./.github/workflows/managed-db-validate.yml - with: - cloud_provider: aws - aws_region: us-west-2 - aws_db_identifier: ${{ matrix.postgres_version }} - pg_port: "5432" - pg_user: randoneering - pg_database: pgFirstAid - pg_sslmode: require - test_view_mode: managed - secrets: - pg_password: ${{ secrets.AWS_DB_PASSWORD }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/deploy-gcp-postgres.yml b/.github/workflows/deploy-gcp-postgres.yml deleted file mode 100644 index ee72675..0000000 --- a/.github/workflows/deploy-gcp-postgres.yml +++ /dev/null @@ -1,285 +0,0 @@ -name: Deploy GCP Cloud SQL PostgreSQL - -on: - workflow_dispatch: - inputs: - target: - description: "Deploy target under testing/gcp/deploy" - required: true - type: choice - options: - - all - - pg15 - - pg16 - - pg17 - - pg18 - default: all - command: - description: "OpenTofu command" - required: true - type: choice - options: - - plan - - apply - - destroy - default: plan - personal_ip_cidr: - description: "Your CIDR for Cloud SQL access (optional). Leave blank to use secret DEPLOY_PERSONAL_IP_CIDR" - required: false - type: string - default: "" - issue_comment: - types: [created] - -permissions: - contents: read - pull-requests: read - -jobs: - resolve: - if: | - ( - github.event_name == 'workflow_dispatch' && - github.actor == vars.DEPLOY_TRIGGER_USER - ) || - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request != null && - startsWith(github.event.comment.body, '/deploy-gcp-pg') && - github.event.comment.user.login == vars.DEPLOY_TRIGGER_USER && - ( - github.event.comment.author_association == 'OWNER' || - github.event.comment.author_association == 'MEMBER' || - github.event.comment.author_association == 'COLLABORATOR' - ) - ) - name: Resolve deploy arguments - runs-on: ubuntu-latest - - outputs: - command: ${{ steps.args.outputs.command }} - versions_json: ${{ steps.args.outputs.versions_json }} - runner_ip_cidr: ${{ steps.args.outputs.runner_ip_cidr }} - personal_ip_cidr: ${{ steps.args.outputs.personal_ip_cidr }} - checkout_ref: ${{ steps.pr.outputs.head_sha || github.sha }} - - steps: - - name: Validate trigger user is configured - run: | - if [ -z "${{ vars.DEPLOY_TRIGGER_USER }}" ]; then - echo "::error::Repository variable DEPLOY_TRIGGER_USER is not set." - exit 1 - fi - - - name: Resolve deploy arguments - id: args - env: - DEPLOY_PERSONAL_IP_CIDR: ${{ secrets.DEPLOY_PERSONAL_IP_CIDR }} - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - target="${{ inputs.target }}" - command="${{ inputs.command }}" - personal_ip_input="${{ inputs.personal_ip_cidr }}" - else - comment="${{ github.event.comment.body }}" - read -r trigger arg1 arg2 _ <<<"${comment}" - - if [ "${trigger}" != "/deploy-gcp-pg" ]; then - echo "::error::Comment must start with '/deploy-gcp-pg'" - exit 1 - fi - - if [ -z "${arg1}" ]; then - target="all" - command="plan" - elif [ "${arg1}" = "plan" ] || [ "${arg1}" = "apply" ] || [ "${arg1}" = "destroy" ]; then - target="all" - command="${arg1}" - else - target="${arg1}" - if [ -z "${arg2}" ]; then - command="plan" - else - command="${arg2}" - fi - fi - - personal_ip_input="" - fi - - case "${target}" in - all|pg15|pg16|pg17|pg18) ;; - *) - echo "::error::Invalid target: ${target}" - exit 1 - ;; - esac - - case "${command}" in - plan|apply|destroy) ;; - *) - echo "::error::Invalid command: ${command}" - exit 1 - ;; - esac - - if [ "${target}" = "all" ]; then - versions_json='["pg15","pg16","pg17","pg18"]' - else - versions_json="[\"${target}\"]" - fi - - runner_ip="$(curl -fsSL https://checkip.amazonaws.com | tr -d '\n')" - runner_ip_cidr="${runner_ip}/32" - - if [ -n "${personal_ip_input}" ]; then - personal_ip_cidr="${personal_ip_input}" - elif [ -n "$DEPLOY_PERSONAL_IP_CIDR" ]; then - personal_ip_cidr="$DEPLOY_PERSONAL_IP_CIDR" - else - echo "::error::Set input personal_ip_cidr or repository secret DEPLOY_PERSONAL_IP_CIDR" - exit 1 - fi - - echo "target=${target}" >> "$GITHUB_OUTPUT" - echo "command=${command}" >> "$GITHUB_OUTPUT" - echo "versions_json=${versions_json}" >> "$GITHUB_OUTPUT" - echo "runner_ip_cidr=${runner_ip_cidr}" >> "$GITHUB_OUTPUT" - echo "personal_ip_cidr=${personal_ip_cidr}" >> "$GITHUB_OUTPUT" - - - name: Resolve pull request head - id: pr - if: github.event_name == 'issue_comment' - uses: actions/github-script@v7 - with: - script: | - const pull_number = context.payload.issue.number; - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number, - }); - - core.setOutput('head_sha', pr.head.sha); - core.setOutput('is_fork', String(pr.head.repo.fork)); - - - name: Block fork pull requests - if: github.event_name == 'issue_comment' && steps.pr.outputs.is_fork == 'true' - run: | - echo "::error::Comment-triggered deploy is blocked for fork PRs." - exit 1 - - - name: Show resolved inputs - run: | - echo "Target: ${{ steps.args.outputs.target }}" - echo "Versions: ${{ steps.args.outputs.versions_json }}" - echo "Command: ${{ steps.args.outputs.command }}" - echo "Runner IP CIDR: ${{ steps.args.outputs.runner_ip_cidr }}" - - deploy: - name: ${{ needs.resolve.outputs.command }} ${{ matrix.postgres_version }} - runs-on: ubuntu-latest - needs: resolve - strategy: - fail-fast: false - matrix: - postgres_version: ${{ fromJSON(needs.resolve.outputs.versions_json) }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ needs.resolve.outputs.checkout_ref }} - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - create_credentials_file: true - export_environment_variables: true - - - name: Setup OpenTofu - uses: opentofu/setup-opentofu@v1 - with: - tofu_version: latest - - - name: Resolve OpenTofu variables - env: - GCP_DB_PASSWORD: ${{ secrets.GCP_DB_PASSWORD }} - run: | - if [ -z "$GCP_DB_PASSWORD" ]; then - echo "::error::Missing secret GCP_DB_PASSWORD" - exit 1 - fi - - echo "::add-mask::$GCP_DB_PASSWORD" - echo "TF_VAR_db_password=$GCP_DB_PASSWORD" >> "$GITHUB_ENV" - - - name: OpenTofu init - working-directory: testing/gcp/deploy/${{ matrix.postgres_version }} - run: tofu init -input=false - - - name: Write CI tfvars - working-directory: testing/gcp/deploy/${{ matrix.postgres_version }} - run: | - cat > ci.auto.tfvars <> "$GITHUB_STEP_SUMMARY" - - validate: - if: needs.resolve.outputs.command == 'apply' - name: validate ${{ matrix.postgres_version }} - needs: [resolve, deploy] - strategy: - fail-fast: false - matrix: - postgres_version: ${{ fromJSON(needs.resolve.outputs.versions_json) }} - uses: ./.github/workflows/managed-db-validate.yml - with: - cloud_provider: gcp - gcp_project_id: pgfirstaid - gcp_instance_name: ${{ format('pgfirstaid-{0}', matrix.postgres_version) }} - pg_port: "5432" - pg_user: randoneering - pg_database: pgFirstAid - pg_sslmode: require - test_view_mode: managed - secrets: - pg_password: ${{ secrets.GCP_DB_PASSWORD }} - gcp_sa_key: ${{ secrets.GCP_SA_KEY }} diff --git a/README.md b/README.md index 1a31130..c41cc6e 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Thank you OSUOSL for providing hosting for our testing infrastructure and CI/CD - **Prioritized Results** - Issues ranked by severity (CRITICAL → HIGH → MEDIUM → LOW → INFO) - **Actionable Recommendations** - Each issue includes specific remediation steps - **Documentation Links** - Direct links to official PostgreSQL documentation for deeper learning +- **Optional `pg_stat_statements` Checks** - Runs additional query workload checks when `pg_stat_statements` is installed ## Quick Start @@ -88,6 +89,19 @@ That's it! No configuration needed. Deploy as a user with the highest possible p - **Replication Slots With High WAL Retention** - Replication slots that have 90% of max wal setting - **Long Running Queries** - Queries that have been running for 5 minutes or more - **Blocked and Blocking Queries** - Queries that are currently blocked or blocking other queries at the time you run pg_firstAid +- **Top 10 Expensive Active Queries** - Active queries running longer than 30 seconds, ordered by runtime +- **Lock-Wait-Heavy Active Queries** - Active queries waiting on locks for more than 30 seconds +- **Idle In Transaction Over 5 Minutes** - Sessions left idle in transaction for over 5 minutes +- **pg_stat_statements Extension Missing** - Reports when extension-based workload checks are unavailable and points to setup steps +- **Top 10 Queries by Total Execution Time** *(pg_stat_statements)* +- **High Mean Execution Time Queries** *(pg_stat_statements)* +- **Top 10 Queries by Temp Block Spills** *(pg_stat_statements)* +- **Low Cache Hit Ratio Queries** *(pg_stat_statements)* +- **High Runtime Variance Queries** *(pg_stat_statements)* +- **High Calls Low Value Queries** *(pg_stat_statements)* +- **High Rows Per Call Queries** *(pg_stat_statements)* +- **High Shared Block Reads Per Call Queries** *(pg_stat_statements)* +- **Top Queries by WAL Bytes Per Call** *(pg_stat_statements)* - **Tables With More Than 50 Columns** - List tables with more than 50 columns (but less than 200) - **Tables Larger Than 50GB** - Identifies tables larger than 50GB (but less than 100GB) @@ -174,6 +188,13 @@ ORDER BY MIN(CASE severity - Works with standard user permissions for most checks - Some checks may return fewer results for non-superuser accounts +**Optional: Enable pg_stat_statements For Deeper Query Checks** +- pgFirstAid keeps running without this extension and reports a setup action when it is missing. +- Self-hosted PostgreSQL: set `shared_preload_libraries = 'pg_stat_statements'`, restart PostgreSQL, then run `CREATE EXTENSION pg_stat_statements;` +- AWS RDS for PostgreSQL: add `pg_stat_statements` to the DB parameter group `shared_preload_libraries`, reboot, then run `CREATE EXTENSION pg_stat_statements;` +- GCP Cloud SQL for PostgreSQL: enable `cloudsql.enable_pg_stat_statements`, restart if required, then run `CREATE EXTENSION pg_stat_statements;` +- Azure Database for PostgreSQL: add `pg_stat_statements` to `shared_preload_libraries`, restart, then run `CREATE EXTENSION pg_stat_statements;` + ## Performance Impact pgFirstAid is designed to be lightweight and safe to run on production systems: diff --git a/testing/aws/deploy/pg16/locals.tf b/testing/aws/deploy/pg16/locals.tf index 48629e3..65f1701 100644 --- a/testing/aws/deploy/pg16/locals.tf +++ b/testing/aws/deploy/pg16/locals.tf @@ -2,7 +2,7 @@ locals { service = "pg16" database_name = "pgFirstAid" engine = "postgres" - engine_version = "" + engine_version = "16.13" engine_family = "postgres16" db_parameter_group = [ { diff --git a/testing/aws/deploy/pg18/locals.tf b/testing/aws/deploy/pg18/locals.tf index 086a71f..81b72e6 100644 --- a/testing/aws/deploy/pg18/locals.tf +++ b/testing/aws/deploy/pg18/locals.tf @@ -2,7 +2,7 @@ locals { service = "pg18" database_name = "pgFirstAid" engine = "postgres" - engine_version = "" + engine_version = "18.3" engine_family = "postgres18" db_parameter_group = [ { diff --git a/testing/aws/opentofu/modules/nonaurora/main.tf b/testing/aws/opentofu/modules/nonaurora/main.tf index a25824b..ef73b78 100644 --- a/testing/aws/opentofu/modules/nonaurora/main.tf +++ b/testing/aws/opentofu/modules/nonaurora/main.tf @@ -5,6 +5,10 @@ locals { allocated_storage = var.allocated_storage == "" ? 20 : var.allocated_storage iops = var.allocated_storage >= 100 ? 3000 : null storage_type = var.storage_type == "" ? "gp2" : var.storage_type + vpc_security_group_ids = concat( + var.vpc_security_group_ids, + aws_security_group.rds_access[*].id, + ) } @@ -12,6 +16,31 @@ data "aws_vpc" "default" { default = true } +resource "aws_security_group" "rds_access" { + count = var.allowed_cidr_block == null ? 0 : 1 + + name_prefix = "${var.service}-rds-access-" + description = "Allow PostgreSQL access to ${var.service}" + vpc_id = data.aws_vpc.default.id + + ingress { + description = "PostgreSQL" + from_port = tonumber(var.port) + to_port = tonumber(var.port) + protocol = "tcp" + cidr_blocks = [var.allowed_cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = var.required_tags +} + resource "random_password" "password" { length = 20 special = false @@ -27,7 +56,7 @@ resource "aws_db_instance" "rds_instance" { instance_class = local.instance_class parameter_group_name = aws_db_parameter_group.param_group.name publicly_accessible = true - vpc_security_group_ids = ["sg-0333981e44680b34b"] + vpc_security_group_ids = local.vpc_security_group_ids allocated_storage = local.allocated_storage apply_immediately = var.apply_immediately skip_final_snapshot = true diff --git a/testing/aws/opentofu/modules/nonaurora/variables.tf b/testing/aws/opentofu/modules/nonaurora/variables.tf index 033844f..3b4e684 100644 --- a/testing/aws/opentofu/modules/nonaurora/variables.tf +++ b/testing/aws/opentofu/modules/nonaurora/variables.tf @@ -23,6 +23,12 @@ variable "vpc_security_group_ids" { default = [] } +variable "allowed_cidr_block" { + description = "CIDR block allowed to access the RDS instance. When set, the module creates a security group rule for the database port." + type = string + default = null +} + variable "instance_class" { description = "Instance type to use" type = string