From f533584d1589c1871054f181bbcd4b5427750421 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Thu, 12 Mar 2026 16:53:56 +0300 Subject: [PATCH 01/27] refactor: replace GH_TOKEN with GitHub App token in workflow files --- .../build-citus-community-nightlies.yml | 22 +++++++++++++-- .../citus-package-all-platforms-test.yml | 18 ++++++++++-- .github/workflows/package-tests.yml | 22 +++++++++++++++ .github/workflows/packaging-methods-tests.yml | 19 +++++++++++-- .../workflows/publish-docker-image-tests.yml | 15 ++++++++++ .github/workflows/statistic-schedule.yml | 15 ++++++++-- .github/workflows/statistic-tests.yml | 23 ++++++++++++++- .github/workflows/tool-tests.yml | 28 ++++++++++++++++++- 8 files changed, 151 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index e4773508..2bc9e35a 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -4,7 +4,6 @@ env: MAIN_BRANCH: "all-citus" PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} on: @@ -32,9 +31,22 @@ jobs: - ubuntu/jammy steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools @@ -46,7 +58,10 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Clone build branch - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://gh-token:${GH_TOKEN}@github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -55,6 +70,9 @@ jobs: run: python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index a7692477..d74779f6 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -1,8 +1,6 @@ name: Citus package all platforms tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay @@ -36,8 +34,21 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -49,4 +60,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 0d401807..cfee9759 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -18,9 +18,18 @@ jobs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata - name: Checkout uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 2 - name: Package version id: get-citus-version @@ -60,8 +69,18 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.GITHUB_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -70,6 +89,9 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7d53056f..c34623f7 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -1,8 +1,5 @@ name: Packaging helper methods tests -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - on: push: branches: @@ -15,8 +12,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -25,4 +35,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c14bc3d7..c20c1e62 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -12,8 +12,20 @@ jobs: runs-on: ubuntu-latest steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{steps.app.outputs.token}} + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -22,4 +34,7 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Build and publish docker images tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 3dcb4b22..11d3b7f8 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} on: schedule: - cron: "0 16 * * *" @@ -25,8 +24,18 @@ jobs: job_name: [docker_pull_citus, github_clone_citus, homebrew_citus] steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -35,6 +44,8 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Execute 'Fetch Daily Statistics' - run: packaging_automation/bash/daily-statistics-job.sh env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" + run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 5b9eb9ea..76a201c8 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }} on: @@ -21,8 +20,18 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -31,13 +40,25 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Unit tests for "Docker statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 8759a7d5..a3d66756 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,7 +1,6 @@ name: Tool Tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay MAIN_BRANCH: all-citus @@ -27,9 +26,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + repositories: | + tools + packaging + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 0 - name: Set up Python 3.10 @@ -53,9 +64,15 @@ jobs: run: black . --check - name: Unit tests for "Common tools" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -63,10 +80,19 @@ jobs: # run: python -m pytest -q packaging_automation/tests/test_prepare_release.py - name: Unit tests for "Update Docker" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py From 7891e7cbb535531a81e558273072a5793c95eb8d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Thu, 12 Mar 2026 19:39:42 +0300 Subject: [PATCH 02/27] fix: update GitHub App token secret reference in workflow files --- .github/workflows/citus-package-all-platforms-test.yml | 2 +- .github/workflows/package-tests.yml | 2 +- .github/workflows/packaging-methods-tests.yml | 2 +- .github/workflows/publish-docker-image-tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index d74779f6..9e2cd6e7 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -39,7 +39,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata repositories: | tools diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index cfee9759..90eaab1c 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -24,7 +24,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index c34623f7..a82f1e3c 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -17,7 +17,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata repositories: | tools diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c20c1e62..41f34047 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -18,7 +18,7 @@ jobs: uses: actions/create-github-app-token@v2 with: app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} + private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - name: Checkout repository From f17e33f558cc40ded9652caef2edf8fd99557ec4 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Sat, 14 Mar 2026 18:53:44 +0300 Subject: [PATCH 03/27] feat: update workflows to use GitHub App token and improve permissions --- .../workflows/build-citus-community-nightlies.yml | 12 ++++++------ .../workflows/citus-package-all-platforms-test.yml | 9 ++++----- .github/workflows/package-tests.yml | 7 ++++++- .github/workflows/packaging-methods-tests.yml | 10 +++++----- .github/workflows/publish-docker-image-tests.yml | 7 +++++-- .github/workflows/pypi-statistics-schedule.yml | 2 +- .github/workflows/statistic-schedule.yml | 6 +++++- .github/workflows/statistic-tests.yml | 10 +++++----- .github/workflows/tool-tests.yml | 14 +++++--------- packaging_automation/common_tool_methods.py | 3 ++- packaging_automation/tests/test_citus_package.py | 6 ++++-- .../tests/test_citus_package_utils.py | 5 ++++- packaging_automation/tests/test_prepare_release.py | 3 ++- packaging_automation/tests/test_publish_docker.py | 3 ++- packaging_automation/tests/test_update_docker.py | 5 ++++- packaging_automation/tests/test_update_pgxn.py | 5 ++++- 16 files changed, 64 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 2bc9e35a..b2baa007 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -18,6 +18,10 @@ jobs: build_package: name: Build package runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -39,9 +43,6 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 @@ -49,6 +50,7 @@ jobs: token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools + submodules: true # This step is to fetch the images unanonymously to have higher bandwidth - name: Login to Docker Hub @@ -60,8 +62,7 @@ jobs: - name: Clone build branch env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://gh-token:${GH_TOKEN}@github.com/citusdata/packaging.git packaging + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://${GH_TOKEN}@github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -72,7 +73,6 @@ jobs: - name: Build packages env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 9e2cd6e7..54cfd327 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -17,6 +17,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -40,10 +44,6 @@ jobs: with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 @@ -62,5 +62,4 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 90eaab1c..a647e268 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -14,6 +14,10 @@ on: jobs: metadata: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read outputs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} @@ -31,6 +35,8 @@ jobs: with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 + submodules: true + - name: Package version id: get-citus-version run: | @@ -91,7 +97,6 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index a82f1e3c..7066b40b 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -10,7 +10,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -19,14 +22,12 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -37,5 +38,4 @@ jobs: - name: Citus package tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index 41f34047..c94abd96 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -10,7 +10,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token @@ -25,6 +28,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{steps.app.outputs.token}} + submodules: true - name: Install package dependencies @@ -36,5 +40,4 @@ jobs: - name: Build and publish docker images tests env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/pypi-statistics-schedule.yml b/.github/workflows/pypi-statistics-schedule.yml index a8562ddc..d54548d0 100644 --- a/.github/workflows/pypi-statistics-schedule.yml +++ b/.github/workflows/pypi-statistics-schedule.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 11d3b7f8..2cc15d9e 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -18,6 +18,10 @@ on: jobs: execute_job: runs-on: ubuntu-latest + permissions: + contents: write + packages: write + pull-requests: read strategy: fail-fast: false matrix: @@ -36,6 +40,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -46,6 +51,5 @@ jobs: - name: Execute 'Fetch Daily Statistics' env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 76a201c8..89360767 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -18,7 +18,10 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -32,6 +35,7 @@ jobs: uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} + submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -42,23 +46,19 @@ jobs: - name: Unit tests for "Docker statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index a3d66756..3938eacb 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -24,7 +24,10 @@ jobs: unit_test_execution: runs-on: ubuntu-latest - + permissions: + contents: write + packages: write + pull-requests: read steps: - name: Create GitHub App token id: app @@ -33,15 +36,13 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata - repositories: | - tools - packaging - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 0 + submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -66,13 +67,11 @@ jobs: - name: Unit tests for "Common tools" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -82,17 +81,14 @@ jobs: - name: Unit tests for "Update Docker" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler env: GH_TOKEN: ${{ steps.app.outputs.token }} - GITHUB_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b4b89d64..b7e30068 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -680,7 +680,8 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + GH_TOKEN = os.environ.get("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 7e62b7ce..0df48b62 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -86,9 +86,11 @@ def setup_module(): "pgxn-citus" if PLATFORM == "pgxn" else PACKAGING_BRANCH_NAME ) if not os.path.exists(PACKAGING_EXEC_FOLDER): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git" - f" {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 064bfcca..5b187281 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -45,8 +45,11 @@ def setup_module(): if not os.path.exists("packaging_test"): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 19a637ba..e75743a9 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,8 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + GH_TOKEN = os.getenv("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 740ff446..2d71475e 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -29,7 +29,8 @@ def initialize_env(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + GH_TOKEN = os.getenv("GH_TOKEN", "") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 06069dc6..1d8c510d 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,7 +34,10 @@ def setup_module(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + GH_TOKEN = os.getenv("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") + run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 572fc5c4..3fb1d62b 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -18,8 +18,11 @@ def setup_module(): if not os.path.exists("packaging_test"): + GH_TOKEN = os.environ.get("GH_TOKEN", "") + if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is not set.") run( - "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" ) From df17e17cc29309b4732d6bc16dcacd3ca13cf22d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Sat, 14 Mar 2026 19:18:46 +0300 Subject: [PATCH 04/27] feat: update workflows to use GH_TOKEN references --- .github/workflows/build-citus-community-nightlies.yml | 10 +++++++--- .github/workflows/citus-package-all-platforms-test.yml | 6 ++++++ .github/workflows/package-tests.yml | 9 ++++++++- .github/workflows/packaging-methods-tests.yml | 6 ++++++ .github/workflows/publish-docker-image-tests.yml | 6 ++++++ .github/workflows/statistic-schedule.yml | 6 ++++++ .github/workflows/statistic-tests.yml | 6 ++++++ .github/workflows/tool-tests.yml | 6 ++++++ packaging_automation/common_tool_methods.py | 2 +- packaging_automation/tests/test_citus_package.py | 5 +---- packaging_automation/tests/test_citus_package_utils.py | 5 +---- packaging_automation/tests/test_prepare_release.py | 2 +- packaging_automation/tests/test_publish_docker.py | 2 +- packaging_automation/tests/test_update_docker.py | 5 +---- packaging_automation/tests/test_update_pgxn.py | 5 +---- 15 files changed, 58 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index b2baa007..ef76878a 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -44,6 +44,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: @@ -60,9 +66,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Clone build branch - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://${GH_TOKEN}@github.com/citusdata/packaging.git packaging + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 54cfd327..5a7f8474 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -45,6 +45,12 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index a647e268..969787a8 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -30,13 +30,20 @@ jobs: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 submodules: true - + - name: Package version id: get-citus-version run: | diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7066b40b..523656e6 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -23,6 +23,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c94abd96..09a07cee 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -24,6 +24,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 2cc15d9e..5cc55c69 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -36,6 +36,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 89360767..39ae0c04 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -31,6 +31,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 3938eacb..75964152 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -37,6 +37,12 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: citusdata + - name: Setup git authentication for GitHub App + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b7e30068..b15a83c8 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -681,7 +681,7 @@ def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): GH_TOKEN = os.environ.get("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/{project_name}.git {checkout_dir}") + run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 0df48b62..d08f3398 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -86,11 +86,8 @@ def setup_module(): "pgxn-citus" if PLATFORM == "pgxn" else PACKAGING_BRANCH_NAME ) if not os.path.exists(PACKAGING_EXEC_FOLDER): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch {packaging_branch_name} https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 5b187281..064bfcca 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -45,11 +45,8 @@ def setup_module(): if not os.path.exists("packaging_test"): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch all-citus-unit-tests https://{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index e75743a9..15af96ea 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -41,7 +41,7 @@ def initialize_env() -> str: remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 2d71475e..81631179 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -30,7 +30,7 @@ def initialize_env(): if not os.path.exists("docker"): GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") + run(f"git clone https://github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 1d8c510d..de286598 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,10 +34,7 @@ def setup_module(): if not os.path.exists("docker"): - GH_TOKEN = os.getenv("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") - run(f"git clone https://{GH_TOKEN}@github.com/citusdata/docker.git") + run(f"git clone https://github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 3fb1d62b..674edaed 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -18,11 +18,8 @@ def setup_module(): if not os.path.exists("packaging_test"): - GH_TOKEN = os.environ.get("GH_TOKEN", "") - if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is not set.") run( - f"git clone --branch pgxn-citus https://{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" ) From 9fd829617c4372e6e3f6caa1b40df35aa2973543 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 15:35:26 +0300 Subject: [PATCH 05/27] refactor: update workflows to use GitHub App --- .../build-citus-community-nightlies.yml | 19 ++++------- .../citus-package-all-platforms-test.yml | 11 ++----- .../delete-packagecloud-packages.yml | 2 +- .github/workflows/package-tests.yml | 33 +++++++++---------- .github/workflows/packaging-methods-tests.yml | 21 ++++-------- .../workflows/publish-docker-image-tests.yml | 22 ++++--------- .github/workflows/statistic-schedule.yml | 22 +++++-------- .github/workflows/statistic-tests.yml | 28 +++++----------- .github/workflows/tool-tests.yml | 31 +++++------------ .../tests/test_citus_package.py | 2 +- 10 files changed, 66 insertions(+), 125 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index ef76878a..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -18,10 +18,6 @@ jobs: build_package: name: Build package runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -38,17 +34,17 @@ jobs: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 @@ -56,7 +52,6 @@ jobs: token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools - submodules: true # This step is to fetch the images unanonymously to have higher bandwidth - name: Login to Docker Hub @@ -75,8 +70,6 @@ jobs: run: python -m pip install -r tools/packaging_automation/requirements.txt - name: Build packages - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | python -m tools.packaging_automation.citus_package \ --gh_token "${GH_TOKEN}" \ diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 5a7f8474..88226ad4 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -17,10 +17,6 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -40,16 +36,17 @@ jobs: steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} - name: Setup git authentication for GitHub App env: GH_TOKEN: ${{ steps.app.outputs.token }} run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 @@ -66,6 +63,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/.github/workflows/delete-packagecloud-packages.yml b/.github/workflows/delete-packagecloud-packages.yml index c9d55c53..de8a8900 100644 --- a/.github/workflows/delete-packagecloud-packages.yml +++ b/.github/workflows/delete-packagecloud-packages.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 969787a8..3e65e82a 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -14,10 +14,6 @@ on: jobs: metadata: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read outputs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} @@ -29,20 +25,19 @@ jobs: with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 2 - submodules: true - name: Package version id: get-citus-version @@ -86,11 +81,17 @@ jobs: id: app uses: actions/create-github-app-token@v2 with: - app-id: ${{ secrets.GITHUB_APP_ID }} - private-key: ${{ secrets.GITHUB_APP_PRIVATE_KEY }} - owner: citusdata + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} - - name: Checkout repository + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + + - name: Checkout uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} @@ -102,8 +103,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | export PROJECT_VERSION="${{ github.event.inputs.project_version }}" echo "Citus Version: ${PROJECT_VERSION} " diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 523656e6..f7030f8c 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -10,30 +10,25 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -42,6 +37,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package_utils.py diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index 09a07cee..22d51c70 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -10,32 +10,26 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{steps.app.outputs.token}} - submodules: true - - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -44,6 +38,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Build and publish docker images tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_publish_docker.py diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 5cc55c69..5aad636a 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -18,10 +18,6 @@ on: jobs: execute_job: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read strategy: fail-fast: false matrix: @@ -30,23 +26,22 @@ jobs: steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -55,7 +50,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Execute 'Fetch Daily Statistics' + run: packaging_automation/bash/daily-statistics-job.sh env: - GH_TOKEN: ${{ steps.app.outputs.token }} JOB_NAME: "${{ matrix.JOB_NAME }}" - run: packaging_automation/bash/daily-statistics-job.sh diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 39ae0c04..702b1766 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -18,30 +18,26 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read + steps: - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} - submodules: true - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources @@ -50,21 +46,13 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Unit tests for "Docker statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_docker_statistics_collector.py - name: Unit tests for "Github clone statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_github_statistics_collector.py - name: Unit tests for "Packagecloud download statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_package_cloud_statistics_collector.py - name: Unit tests for "Homebrew download statistics" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_homebrew_statistics_collector.py diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 75964152..89029dda 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -19,36 +19,33 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + - name: Install all scripts run: make && sudo make install unit_test_execution: runs-on: ubuntu-latest - permissions: - contents: write - packages: write - pull-requests: read steps: + - name: Create GitHub App token id: app - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v1 with: app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_KEY }} - owner: citusdata + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV - - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} - run: | - git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Checkout repository uses: actions/checkout@v3 with: token: ${{ steps.app.outputs.token }} fetch-depth: 0 - submodules: true - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -71,13 +68,9 @@ jobs: run: black . --check - name: Unit tests for "Common tools" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_common_tool_methods.py - name: Unit tests for "Update Package Properties" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_package_properties.py # no longer viable, outdated test, skipping to not block the pipeline @@ -85,16 +78,10 @@ jobs: # run: python -m pytest -q packaging_automation/tests/test_prepare_release.py - name: Unit tests for "Update Docker" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_docker.py - name: Unit tests for "Update Pgxn" - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_update_pgxn.py - name: Packaging Warning Handler - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_packaging_warning_handler.py diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index d08f3398..4f1c0d35 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -87,7 +87,7 @@ def setup_module(): ) if not os.path.exists(PACKAGING_EXEC_FOLDER): run( - f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" ) From 1fa5d293a5ee722d17eedf15afa3986dfa748734 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 16:21:29 +0300 Subject: [PATCH 06/27] refactor: replace GITHUB_TOKEN with GH_TOKEN in workflow and test files also should refactor links if clone fails --- packaging_automation/citus_package.py | 4 ++-- packaging_automation/common_tool_methods.py | 1 - packaging_automation/tests/test_common_tool_methods.py | 6 +++--- packaging_automation/tests/test_prepare_release.py | 1 - packaging_automation/tests/test_publish_docker.py | 3 +-- packaging_automation/tests/test_update_docker.py | 2 +- .../tests/test_update_package_properties.py | 2 +- packaging_automation/tests/test_update_pgxn.py | 2 +- 8 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 06a81dc7..d6ff3955 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,7 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GITHUB_TOKEN"] = github_token + os.environ["GH_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b15a83c8..b4b89d64 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -680,7 +680,6 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - GH_TOKEN = os.environ.get("GH_TOKEN", "") run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 2cf7b98f..5f5cece5 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -46,7 +46,7 @@ str_array_to_str, ) -GITHUB_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() TEST_GPG_KEY_NAME = "Citus Data " @@ -264,7 +264,7 @@ def test_prepend_line_in_file(): def test_getprs(): # created at is not seen on Github. Should be checked on API result - g = Github(GITHUB_TOKEN) + g = Github(GH_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, @@ -277,7 +277,7 @@ def test_getprs(): def test_getprs_with_backlog_label(): - g = Github(GITHUB_TOKEN) + g = Github(GH_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 15af96ea..19a637ba 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,6 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - GH_TOKEN = os.getenv("GH_TOKEN", "") run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 81631179..740ff446 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -29,8 +29,7 @@ def initialize_env(): if not os.path.exists("docker"): - GH_TOKEN = os.getenv("GH_TOKEN", "") - run(f"git clone https://github.com/citusdata/docker.git") + run("git clone https://github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index de286598..06069dc6 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -34,7 +34,7 @@ def setup_module(): if not os.path.exists("docker"): - run(f"git clone https://github.com/citusdata/docker.git") + run("git clone https://github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index c1bcebed..804ee541 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -25,7 +25,7 @@ TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[1]) -GITHUB_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") PROJECT_VERSION = os.getenv("PROJECT_VERSION", default="10.2.4") TAG_NAME = os.getenv("TAG_NAME", default="v10.2.4") PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 674edaed..572fc5c4 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -19,7 +19,7 @@ def setup_module(): if not os.path.exists("packaging_test"): run( - f"git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" ) From 361638f29a3a276e6bceb4ac381d3c8bed3e40ab Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 16:53:57 +0300 Subject: [PATCH 07/27] update build_packages to double check token validility --- packaging_automation/citus_package.py | 1 + packaging_automation/tests/test_citus_package_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index d6ff3955..36a9d636 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -401,6 +401,7 @@ def build_packages( input_output_parameters: InputOutputParameters, is_test: bool = False, ) -> None: + os.environ["GH_TOKEN"] = github_token os_name, os_version = decode_os_and_release(platform) release_versions, nightly_versions = get_postgres_versions( platform, input_output_parameters.input_files_dir diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 064bfcca..44c8d0e4 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -46,7 +46,7 @@ def setup_module(): if not os.path.exists("packaging_test"): run( - f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) From 413ed4e3a9f4e9bbd1fe872d17928cc8527a8891 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:03:04 +0300 Subject: [PATCH 08/27] changing the way token gets passed --- .github/workflows/build-citus-community-nightlies.yml | 2 +- packaging_automation/tests/test_citus_package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 745abc19..6e1dce4b 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -72,7 +72,7 @@ jobs: - name: Build packages run: | python -m tools.packaging_automation.citus_package \ - --gh_token "${GH_TOKEN}" \ + --gh_token "${{ steps.app.outputs.token }}" \ --platform "${{ matrix.platform }}" \ --build_type "nightly" \ --secret_key "${PACKAGING_SECRET_KEY}" \ diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 4f1c0d35..6059a542 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -60,7 +60,7 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE") -GH_TOKEN = os.getenv("GH_TOKEN") +GH_TOKEN = os.environ("GH_TOKEN") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") PLATFORM = get_build_platform( From 8ad28a3bce8029c1841d1510b164fe05a84c0d11 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:06:49 +0300 Subject: [PATCH 09/27] refactor: update GH_TOKEN usage in build process and tests --- .github/workflows/build-citus-community-nightlies.yml | 2 +- packaging_automation/citus_package.py | 1 - packaging_automation/tests/test_citus_package.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 6e1dce4b..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -72,7 +72,7 @@ jobs: - name: Build packages run: | python -m tools.packaging_automation.citus_package \ - --gh_token "${{ steps.app.outputs.token }}" \ + --gh_token "${GH_TOKEN}" \ --platform "${{ matrix.platform }}" \ --build_type "nightly" \ --secret_key "${PACKAGING_SECRET_KEY}" \ diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 36a9d636..d6ff3955 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -401,7 +401,6 @@ def build_packages( input_output_parameters: InputOutputParameters, is_test: bool = False, ) -> None: - os.environ["GH_TOKEN"] = github_token os_name, os_version = decode_os_and_release(platform) release_versions, nightly_versions = get_postgres_versions( platform, input_output_parameters.input_files_dir diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 6059a542..02fb6a20 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -60,7 +60,9 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE") -GH_TOKEN = os.environ("GH_TOKEN") +GH_TOKEN = os.getenv("GH_TOKEN") +if not GH_TOKEN: + raise ValueError("GH_TOKEN environment variable is required for tests to run") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") PLATFORM = get_build_platform( From adc0d6af5e4a27fff2c1cf64380f55df2364ffa0 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 17:09:42 +0300 Subject: [PATCH 10/27] refactor: add GH_TOKEN environment variable to Citus package tests step --- .github/workflows/citus-package-all-platforms-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 88226ad4..01860d72 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -63,4 +63,6 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests + env: + GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s From c3e370db129e5c05d52f6484a1ab732ecacfa142 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 18:29:06 +0300 Subject: [PATCH 11/27] testing sh scripts --- .github/workflows/citus-package-all-platforms-test.yml | 8 ++++---- packaging_automation/citus_package.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 01860d72..7667dbf3 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -42,9 +42,11 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} + - name: Set GH_TOKEN for all steps + run: | + echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + - name: Setup git authentication for GitHub App - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: | git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" @@ -63,6 +65,4 @@ jobs: run: python -m pip install -r packaging_automation/requirements.txt - name: Citus package tests - env: - GH_TOKEN: ${{ steps.app.outputs.token }} run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index d6ff3955..06a81dc7 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,7 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GH_TOKEN"] = github_token + os.environ["GITHUB_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) From 131e05955f82052c90c41d8d38487e9f620c448d Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 17 Mar 2026 18:40:44 +0300 Subject: [PATCH 12/27] refactor: update git clone commands inside docker --- packaging_automation/common_tool_methods.py | 5 ++++- packaging_automation/tests/test_prepare_release.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b4b89d64..1b50a9c2 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -679,8 +679,11 @@ def remove_suffix(initial_str: str, suffix: str) -> str: def initialize_env(exec_path: str, project_name: str, checkout_dir: str): remove_cloned_code(f"{exec_path}/{checkout_dir}") + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + raise ValueError("GH_TOKEN environment variable is required") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + run(f"git clone https://x-access-token:{gh_token}@github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 19a637ba..0cde6ee7 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,7 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major From ff500eedaa17b4a100c7a7ccb700717a6a9c44c5 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Mon, 6 Apr 2026 14:44:04 +0300 Subject: [PATCH 13/27] replace GITHUB_TOKEN with GH_TOKEN in citus_package. testing for packaging repo --- packaging_automation/citus_package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 06a81dc7..d6ff3955 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,7 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GITHUB_TOKEN"] = github_token + os.environ["GH_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) From 20ac6710dc85f4f6857f8af1d2af4437f6aa5b51 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Mon, 6 Apr 2026 17:17:03 +0300 Subject: [PATCH 14/27] fix: add GITHUB_TOKEN environment variable for docker build process --- packaging_automation/citus_package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index d6ff3955..50712215 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -342,6 +342,7 @@ def build_package( docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" os.environ["GH_TOKEN"] = github_token + os.environ["GITHUB_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): os.makedirs(input_output_parameters.output_dir) @@ -349,7 +350,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GH_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GH_TOKEN -e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) From bf97f8e2efef5ebd9f78cd6e7fef0498fbee615f Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Fri, 17 Apr 2026 09:28:27 +0300 Subject: [PATCH 15/27] Revert changes --- .../citus-package-all-platforms-test.yml | 1 + .github/workflows/package-tests.yml | 15 --------------- .github/workflows/publish-docker-image-tests.yml | 16 ---------------- packaging_automation/citus_package.py | 3 +-- packaging_automation/common_tool_methods.py | 7 +++---- packaging_automation/tests/test_citus_package.py | 2 -- .../tests/test_citus_package_utils.py | 1 - .../tests/test_common_tool_methods.py | 6 +++--- .../tests/test_prepare_release.py | 2 +- .../tests/test_update_package_properties.py | 2 +- 10 files changed, 10 insertions(+), 45 deletions(-) diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index 7667dbf3..2689fc4b 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -45,6 +45,7 @@ jobs: - name: Set GH_TOKEN for all steps run: | echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + echo "GITHUB_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV - name: Setup git authentication for GitHub App run: | diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 3e65e82a..c5c8dff1 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -19,24 +19,9 @@ jobs: citus_version: ${{ steps.get-citus-version.outputs.citus_version }} steps: - - name: Create GitHub App token - id: app - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GH_APP_KEY }} - owner: ${{ github.repository_owner }} - - - name: Set GH_TOKEN for all steps - run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV - - - name: Configure git with x-access-token - run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - - name: Checkout uses: actions/checkout@v3 with: - token: ${{ steps.app.outputs.token }} fetch-depth: 2 - name: Package version diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index 22d51c70..70fd4dbb 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -12,24 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - - name: Create GitHub App token - id: app - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GH_APP_KEY }} - owner: ${{ github.repository_owner }} - - - name: Set GH_TOKEN for all steps - run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV - - - name: Configure git with x-access-token - run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" - - name: Checkout repository uses: actions/checkout@v3 - with: - token: ${{steps.app.outputs.token}} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 50712215..06a81dc7 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -341,7 +341,6 @@ def build_package( ): docker_image_name = "packaging" if not is_test else "packaging-test" postgres_extension = "all" if postgres_version == "all" else f"pg{postgres_version}" - os.environ["GH_TOKEN"] = github_token os.environ["GITHUB_TOKEN"] = github_token os.environ["CONTAINER_BUILD_RUN_ENABLED"] = "true" if not os.path.exists(input_output_parameters.output_dir): @@ -350,7 +349,7 @@ def build_package( docker_command = ( f"docker run --rm -v {input_output_parameters.output_dir}:/packages -v " f"{input_output_parameters.input_files_dir}:/buildfiles:ro " - f"-e GH_TOKEN -e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " + f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 1b50a9c2..91765272 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -677,11 +677,10 @@ def remove_suffix(initial_str: str, suffix: str) -> str: return result_str -def initialize_env(exec_path: str, project_name: str, checkout_dir: str): +def initialize_env(exec_path: str, project_name: str, checkout_dir: str, gh_token: str = None): + if gh_token is None: + print("GITHUB_TOKEN not found") # Needed for cloning - may fail remove_cloned_code(f"{exec_path}/{checkout_dir}") - gh_token = os.getenv("GH_TOKEN") - if not gh_token: - raise ValueError("GH_TOKEN environment variable is required") if not os.path.exists(checkout_dir): run(f"git clone https://x-access-token:{gh_token}@github.com/citusdata/{project_name}.git {checkout_dir}") diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 02fb6a20..4f1c0d35 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -61,8 +61,6 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE") GH_TOKEN = os.getenv("GH_TOKEN") -if not GH_TOKEN: - raise ValueError("GH_TOKEN environment variable is required for tests to run") PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN") REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET") PLATFORM = get_build_platform( diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 44c8d0e4..4d31aa5a 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -34,7 +34,6 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = "Citus123" GH_TOKEN = os.getenv("GH_TOKEN") - PACKAGING_SOURCE_FOLDER = "packaging_test" PACKAGING_EXEC_FOLDER = f"{TEST_BASE_PATH}/{PACKAGING_SOURCE_FOLDER}" OUTPUT_FOLDER = f"{PACKAGING_EXEC_FOLDER}/packages" diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 5f5cece5..2cf7b98f 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -46,7 +46,7 @@ str_array_to_str, ) -GH_TOKEN = os.getenv("GH_TOKEN") +GITHUB_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() TEST_GPG_KEY_NAME = "Citus Data " @@ -264,7 +264,7 @@ def test_prepend_line_in_file(): def test_getprs(): # created at is not seen on Github. Should be checked on API result - g = Github(GH_TOKEN) + g = Github(GITHUB_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, @@ -277,7 +277,7 @@ def test_getprs(): def test_getprs_with_backlog_label(): - g = Github(GH_TOKEN) + g = Github(GITHUB_TOKEN) repository = g.get_repo("citusdata/citus") prs = get_prs_for_patch_release( repository, diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 0cde6ee7..dc0cc1c9 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,7 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://x-access-token:{github_token}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_update_package_properties.py b/packaging_automation/tests/test_update_package_properties.py index 804ee541..c1bcebed 100644 --- a/packaging_automation/tests/test_update_package_properties.py +++ b/packaging_automation/tests/test_update_package_properties.py @@ -25,7 +25,7 @@ TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[1]) -GH_TOKEN = os.getenv("GH_TOKEN") +GITHUB_TOKEN = os.getenv("GH_TOKEN") PROJECT_VERSION = os.getenv("PROJECT_VERSION", default="10.2.4") TAG_NAME = os.getenv("TAG_NAME", default="v10.2.4") PROJECT_NAME = os.getenv("PROJECT_NAME", default="citus") From 75484be97966a2ee14faf6de2e8dbaace3f07456 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Fri, 17 Apr 2026 11:46:43 +0300 Subject: [PATCH 16/27] Checking token format --- .github/workflows/build-citus-community-nightlies.yml | 9 +++++++++ packaging_automation/tests/test_citus_package.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 745abc19..aa241bf8 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -40,6 +40,15 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} + - name: Check token format + run: | + TOKEN="${{ steps.app.outputs.token }}" + if [[ "$TOKEN" == eyJ* ]]; then + echo "TOKEN FORMAT: appears to be a JWT (app-level token)." + else + echo "TOKEN FORMAT: not a JWT (installation token or PAT)." + fi + - name: Set GH_TOKEN for all steps run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 4f1c0d35..80c437cb 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -87,7 +87,8 @@ def setup_module(): ) if not os.path.exists(PACKAGING_EXEC_FOLDER): run( - f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git " + f"{PACKAGING_EXEC_FOLDER}" ) From 07979504f5aace11bb3ac1f8f96eec15bd63602e Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Fri, 17 Apr 2026 11:51:38 +0300 Subject: [PATCH 17/27] Checking Token --- .github/workflows/build-citus-community-nightlies.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index aa241bf8..02055b31 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -43,11 +43,14 @@ jobs: - name: Check token format run: | TOKEN="${{ steps.app.outputs.token }}" - if [[ "$TOKEN" == eyJ* ]]; then - echo "TOKEN FORMAT: appears to be a JWT (app-level token)." - else - echo "TOKEN FORMAT: not a JWT (installation token or PAT)." + if [[ "$TOKEN" == ghu_* ]]; then + echo "GitHub App installation token" + elif [[ "$TOKEN" == eyJ* ]]; then + echo "JWT token" + elif [[ "$TOKEN" == ghp_* ]]; then + echo "Personal Access Token" fi + echo "Token preview (first 5 chars): ${TOKEN:0:5}" - name: Set GH_TOKEN for all steps run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV From 58e9c036552ac7bc65c097b978b63cce8d062bb2 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Sun, 19 Apr 2026 20:20:58 +0300 Subject: [PATCH 18/27] add GH_TOKEN to cloning steps --- .../workflows/build-citus-community-nightlies.yml | 12 ------------ packaging_automation/tests/test_publish_docker.py | 4 ++-- packaging_automation/tests/test_update_docker.py | 4 ++-- packaging_automation/tests/test_update_pgxn.py | 4 ++-- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 02055b31..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -40,18 +40,6 @@ jobs: private-key: ${{ secrets.GH_APP_KEY }} owner: ${{ github.repository_owner }} - - name: Check token format - run: | - TOKEN="${{ steps.app.outputs.token }}" - if [[ "$TOKEN" == ghu_* ]]; then - echo "GitHub App installation token" - elif [[ "$TOKEN" == eyJ* ]]; then - echo "JWT token" - elif [[ "$TOKEN" == ghp_* ]]; then - echo "Personal Access Token" - fi - echo "Token preview (first 5 chars): ${TOKEN:0:5}" - - name: Set GH_TOKEN for all steps run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 740ff446..0b190ec9 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -25,11 +25,11 @@ BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) EXEC_PATH = f"{BASE_PATH}/docker" - +GH_TOKEN = os.getenv("GH_TOKEN") def initialize_env(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 06069dc6..a8a98594 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -30,11 +30,11 @@ version_details = get_version_details(PROJECT_VERSION) TEMPLATE_PATH = f"{BASE_PATH}/packaging_automation/templates/docker" PKGVARS_FILE = f"{TEST_BASE_PATH}/pkgvars" - +GH_TOKEN = os.getenv("GH_TOKEN") def setup_module(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 572fc5c4..e37dacf3 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -14,12 +14,12 @@ PROJECT_VERSION = "10.0.3" PROJECT_NAME = "citus" TEMPLATE_PATH = f"{BASE_PATH}/packaging_automation/templates/pgxn" - +GH_TOKEN = os.getenv("GH_TOKEN") def setup_module(): if not os.path.exists("packaging_test"): run( - "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" ) From 258ceff63f37448d7f443856e82e6e341e7350ee Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 15:55:23 +0300 Subject: [PATCH 19/27] add debug steps to verify GH_TOKEN access and git authentication --- .../build-citus-community-nightlies.yml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 745abc19..7a5d46ba 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -60,9 +60,49 @@ jobs: username: ${{ secrets.DOCKERHUB_USER_NAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: "Debug: Confirm token can access repos and git auth works" + run: | + echo "GH_TOKEN present: ${GH_TOKEN:+yes}" + echo "GH_TOKEN length: $(echo -n "$GH_TOKEN" | wc -c)" + + packaging_status=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/citusdata/packaging) + echo "packaging repo status: $packaging_status" + docker_status=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/citusdata/docker) + echo "docker repo status: $docker_status" + + if [[ "$packaging_status" != "200" ]]; then + echo "ERROR: packaging repo access failed with status $packaging_status" + exit 1 + fi + if [[ "$docker_status" != "200" ]]; then + echo "ERROR: docker repo access failed with status $docker_status" + exit 1 + fi + + if git ls-remote --exit-code "https://github.com/citusdata/packaging.git" HEAD >/dev/null 2>&1; then + echo "git auth (ls-remote) succeeded" + else + echo "git auth (ls-remote) FAILED" + exit 1 + fi + + - name: "Debug: Verify git config url rewrite is active" + run: git config --global --get-regexp url.*insteadOf | sed -E 's@(https://x-access-token:)[^@]*@\1REDACTED@' || true + + - name: "Debug: Test token access from inside container" + run: | + docker run --rm -e GITHUB_TOKEN="$GH_TOKEN" citus/packaging-test:debian-trixie-all sh -c "curl -s -o /dev/null -w 'Container API status: %{http_code}\n' -H 'Authorization: token \$GITHUB_TOKEN' https://api.github.com/repos/citusdata/packaging" + - name: Clone build branch run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging + - name: "Debug: Confirm clone succeeded and files are present" + run: | + echo "packaging dir listing:" + ls -la packaging || true + echo "templates docker listing:" + find packaging -maxdepth 4 -type f -name '*docker*' -print || true + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources From 7655fc4e1bb58b518ef2baa3fd1a28a3bf069a2a Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 16:34:37 +0300 Subject: [PATCH 20/27] refactor debug steps for token verification and packaging process --- .../build-citus-community-nightlies.yml | 69 ++++++++----------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 7a5d46ba..dc960480 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -60,55 +60,44 @@ jobs: username: ${{ secrets.DOCKERHUB_USER_NAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: "Debug: Confirm token can access repos and git auth works" - run: | - echo "GH_TOKEN present: ${GH_TOKEN:+yes}" - echo "GH_TOKEN length: $(echo -n "$GH_TOKEN" | wc -c)" - - packaging_status=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/citusdata/packaging) - echo "packaging repo status: $packaging_status" - docker_status=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/citusdata/docker) - echo "docker repo status: $docker_status" - - if [[ "$packaging_status" != "200" ]]; then - echo "ERROR: packaging repo access failed with status $packaging_status" - exit 1 - fi - if [[ "$docker_status" != "200" ]]; then - echo "ERROR: docker repo access failed with status $docker_status" - exit 1 - fi - - if git ls-remote --exit-code "https://github.com/citusdata/packaging.git" HEAD >/dev/null 2>&1; then - echo "git auth (ls-remote) succeeded" - else - echo "git auth (ls-remote) FAILED" - exit 1 - fi - - - name: "Debug: Verify git config url rewrite is active" - run: git config --global --get-regexp url.*insteadOf | sed -E 's@(https://x-access-token:)[^@]*@\1REDACTED@' || true - - - name: "Debug: Test token access from inside container" - run: | - docker run --rm -e GITHUB_TOKEN="$GH_TOKEN" citus/packaging-test:debian-trixie-all sh -c "curl -s -o /dev/null -w 'Container API status: %{http_code}\n' -H 'Authorization: token \$GITHUB_TOKEN' https://api.github.com/repos/citusdata/packaging" - - name: Clone build branch run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging - - name: "Debug: Confirm clone succeeded and files are present" - run: | - echo "packaging dir listing:" - ls -la packaging || true - echo "templates docker listing:" - find packaging -maxdepth 4 -type f -name '*docker*' -print || true - - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources - name: Install python requirements run: python -m pip install -r tools/packaging_automation/requirements.txt + - name: "Debug: run packaging image the same way (capture logs)" + run: | + mkdir -p container_logs + docker run --rm \ + -v "$(pwd)/packages":/packages \ + -v "$(pwd)/packaging":/buildfiles:ro \ + -e GITHUB_TOKEN \ + -e PACKAGE_ENCRYPTION_KEY="${PACKAGING_SECRET_KEY}" \ + -e UNENCRYPTED_PACKAGE="" \ + -e CONTAINER_BUILD_RUN_ENABLED=true \ + -e MSRUSTUP_PAT \ + -e CRATES_IO_MIRROR_FEED_TOKEN \ + -e INSTALL_RUST \ + -e CI \ + -e PACKAGING_PASSPHRASE \ + citus/packaging-test:debian-trixie-all nightly \ + > container_logs/container.stdout 2> container_logs/container.stderr || true + echo "---- container.stderr (first 200 lines) ----" + sed -n '1,200p' container_logs/container.stderr || true + echo "---- container.stdout (first 200 lines) ----" + sed -n '1,200p' container_logs/container.stdout || true + + - name: "Debug: Verify packaging dir contains expected inputs" + run: | + echo "Searching for key packaging files:" + find packaging -maxdepth 4 -type f \( -name 'pkgvars' -o -name 'postgres-matrix.yml' -o -name 'supported-postgres' \) -print || true + echo "Packaging directory structure:" + ls -la packaging/ || true + - name: Build packages run: | python -m tools.packaging_automation.citus_package \ From af59f9c09b082ded9eabbd3baaaabe394faa265a Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 19:00:22 +0300 Subject: [PATCH 21/27] finding submodules --- .../build-citus-community-nightlies.yml | 85 +++++++++++++------ 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index dc960480..dd5f246a 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -69,34 +69,65 @@ jobs: - name: Install python requirements run: python -m pip install -r tools/packaging_automation/requirements.txt - - name: "Debug: run packaging image the same way (capture logs)" + - name: Debug: download and inspect GitHub tarball (host) run: | - mkdir -p container_logs - docker run --rm \ - -v "$(pwd)/packages":/packages \ - -v "$(pwd)/packaging":/buildfiles:ro \ - -e GITHUB_TOKEN \ - -e PACKAGE_ENCRYPTION_KEY="${PACKAGING_SECRET_KEY}" \ - -e UNENCRYPTED_PACKAGE="" \ - -e CONTAINER_BUILD_RUN_ENABLED=true \ - -e MSRUSTUP_PAT \ - -e CRATES_IO_MIRROR_FEED_TOKEN \ - -e INSTALL_RUST \ - -e CI \ - -e PACKAGING_PASSPHRASE \ - citus/packaging-test:debian-trixie-all nightly \ - > container_logs/container.stdout 2> container_logs/container.stderr || true - echo "---- container.stderr (first 200 lines) ----" - sed -n '1,200p' container_logs/container.stderr || true - echo "---- container.stdout (first 200 lines) ----" - sed -n '1,200p' container_logs/container.stdout || true - - - name: "Debug: Verify packaging dir contains expected inputs" - run: | - echo "Searching for key packaging files:" - find packaging -maxdepth 4 -type f \( -name 'pkgvars' -o -name 'postgres-matrix.yml' -o -name 'supported-postgres' \) -print || true - echo "Packaging directory structure:" - ls -la packaging/ || true + set -euo pipefail + + # read pkgvars from the cloned packaging repo (this is where hubproj/nightlyref come from) + echo "---- packaging/pkgvars ----" + sed -n '1,120p' packaging/pkgvars + + # load variables from pkgvars (same as container does) + set -a + source packaging/pkgvars + set +a + + pkgname="${deb_pkgname:-${pkgname}}" + hubproj="${hubproj:-${pkgname}}" + nightlyref="${nightlyref:-master}" + repopath="citusdata/${hubproj}" + + echo "DEBUG pkgname=$pkgname hubproj=$hubproj nightlyref=$nightlyref" + echo "DEBUG repopath=$repopath nightlyref=$nightlyref" + + echo "DEBUG GitHub API status for repo:" + curl -s -o /dev/null -w "%{http_code}\n" \ + -H "Authorization: token ${GH_TOKEN}" \ + "https://api.github.com/repos/${repopath}" + + # resolve gitsha the same way as fetch_and_build_deb + gitsha=$(curl -sfH "Authorization: token ${GH_TOKEN}" \ + -H "Accept: application/vnd.github.v3.sha" \ + "https://api.github.com/repos/${repopath}/commits/${nightlyref}") + echo "DEBUG gitsha=$gitsha" + + # download tarball and inspect it + curl -sfL -H "Authorization: token ${GH_TOKEN}" \ + "https://api.github.com/repos/${repopath}/tarball/${gitsha}" \ + -o /tmp/src.tgz + + echo "DEBUG tarball file:" + ls -lh /tmp/src.tgz + + echo "DEBUG does tarball contain .gitmodules?" + tar -tzf /tmp/src.tgz | grep -E '/\.gitmodules$' || echo "No .gitmodules found in tarball" + + echo "DEBUG top-level listing (first 80 entries):" + tar -tzf /tmp/src.tgz | sed -n '1,80p' + + # if it exists, print the .gitmodules content from inside the tarball + gm_path=$(tar -tzf /tmp/src.tgz | grep -E '/\.gitmodules$' | head -n 1 || true) + if [ -n "${gm_path}" ]; then + echo "DEBUG .gitmodules path in tarball: $gm_path" + tar -xOzf /tmp/src.tgz "$gm_path" | sed -n '1,120p' + fi + + - uses: actions/upload-artifact@v4 + if: always() + continue-on-error: true + with: + name: github-tarball + path: /tmp/src.tgz - name: Build packages run: | From a3f9a96508955207627bec42d0c06a6abc6882ae Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 19:12:17 +0300 Subject: [PATCH 22/27] fix typo --- .github/workflows/build-citus-community-nightlies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index dd5f246a..a8fa069c 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -69,7 +69,7 @@ jobs: - name: Install python requirements run: python -m pip install -r tools/packaging_automation/requirements.txt - - name: Debug: download and inspect GitHub tarball (host) + - name: Debug download and inspect tarball (host) run: | set -euo pipefail From 21abaa1161afdd8c5fa54923df6bb468cbe69b10 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 20:04:45 +0300 Subject: [PATCH 23/27] refactor debug step to run packaging container and capture logs --- .../build-citus-community-nightlies.yml | 97 ++++++++----------- 1 file changed, 43 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index a8fa069c..2a200705 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -69,65 +69,54 @@ jobs: - name: Install python requirements run: python -m pip install -r tools/packaging_automation/requirements.txt - - name: Debug download and inspect tarball (host) + - name: Debug run packaging container directly and capture logs + if: startsWith(matrix.platform, 'debian/') || startsWith(matrix.platform, 'ubuntu/') + env: + GITHUB_TOKEN: ${{ steps.app.outputs.token }} + CONTAINER_BUILD_RUN_ENABLED: "true" run: | set -euo pipefail - - # read pkgvars from the cloned packaging repo (this is where hubproj/nightlyref come from) - echo "---- packaging/pkgvars ----" - sed -n '1,120p' packaging/pkgvars - - # load variables from pkgvars (same as container does) - set -a - source packaging/pkgvars - set +a - - pkgname="${deb_pkgname:-${pkgname}}" - hubproj="${hubproj:-${pkgname}}" - nightlyref="${nightlyref:-master}" - repopath="citusdata/${hubproj}" - - echo "DEBUG pkgname=$pkgname hubproj=$hubproj nightlyref=$nightlyref" - echo "DEBUG repopath=$repopath nightlyref=$nightlyref" - - echo "DEBUG GitHub API status for repo:" - curl -s -o /dev/null -w "%{http_code}\n" \ - -H "Authorization: token ${GH_TOKEN}" \ - "https://api.github.com/repos/${repopath}" - - # resolve gitsha the same way as fetch_and_build_deb - gitsha=$(curl -sfH "Authorization: token ${GH_TOKEN}" \ - -H "Accept: application/vnd.github.v3.sha" \ - "https://api.github.com/repos/${repopath}/commits/${nightlyref}") - echo "DEBUG gitsha=$gitsha" - - # download tarball and inspect it - curl -sfL -H "Authorization: token ${GH_TOKEN}" \ - "https://api.github.com/repos/${repopath}/tarball/${gitsha}" \ - -o /tmp/src.tgz - - echo "DEBUG tarball file:" - ls -lh /tmp/src.tgz - - echo "DEBUG does tarball contain .gitmodules?" - tar -tzf /tmp/src.tgz | grep -E '/\.gitmodules$' || echo "No .gitmodules found in tarball" - - echo "DEBUG top-level listing (first 80 entries):" - tar -tzf /tmp/src.tgz | sed -n '1,80p' - - # if it exists, print the .gitmodules content from inside the tarball - gm_path=$(tar -tzf /tmp/src.tgz | grep -E '/\.gitmodules$' | head -n 1 || true) - if [ -n "${gm_path}" ]; then - echo "DEBUG .gitmodules path in tarball: $gm_path" - tar -xOzf /tmp/src.tgz "$gm_path" | sed -n '1,120p' - fi + mkdir -p container_logs + mkdir -p "$(pwd)/packages/${{ matrix.platform }}" + rm -f container_logs/stdout.txt container_logs/stderr.txt + + os="${{ matrix.platform }}" + os="${os%%/*}" # debian or ubuntu + rel="${{ matrix.platform }}" + rel="${rel##*/}" # bookworm/trixie/jammy/noble + + IMAGE="citus/packaging:${os}-${rel}-all" + echo "Using image: $IMAGE" + + docker run --rm \ + -v "$(pwd)/packages/${{ matrix.platform }}":/packages \ + -v "$(pwd)/packaging":/buildfiles:ro \ + -e GITHUB_TOKEN \ + -e CONTAINER_BUILD_RUN_ENABLED \ + -e PACKAGE_ENCRYPTION_KEY \ + -e UNENCRYPTED_PACKAGE \ + -e MSRUSTUP_PAT \ + -e CRATES_IO_MIRROR_FEED_TOKEN \ + -e INSTALL_RUST \ + -e CI \ + "$IMAGE" nightly \ + > container_logs/stdout.txt 2> container_logs/stderr.txt || true + + echo "===== STDERR (first 200 lines) =====" + sed -n '1,200p' container_logs/stderr.txt || true + echo "===== STDOUT (first 200 lines) =====" + sed -n '1,200p' container_logs/stdout.txt || true + + echo "===== STDERR (last 200 lines) =====" + tail -n 200 container_logs/stderr.txt || true + echo "===== STDOUT (last 200 lines) =====" + tail -n 200 container_logs/stdout.txt || true - uses: actions/upload-artifact@v4 - if: always() - continue-on-error: true + if: always() && (startsWith(matrix.platform, 'debian/') || startsWith(matrix.platform, 'ubuntu/')) with: - name: github-tarball - path: /tmp/src.tgz + name: container-logs-${{ matrix.platform }} + path: container_logs/ - name: Build packages run: | From 0c2a95cc3d8587c837dae4838885f3708a08a98a Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Tue, 21 Apr 2026 20:09:30 +0300 Subject: [PATCH 24/27] remove debug step --- .../build-citus-community-nightlies.yml | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index 2a200705..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -69,55 +69,6 @@ jobs: - name: Install python requirements run: python -m pip install -r tools/packaging_automation/requirements.txt - - name: Debug run packaging container directly and capture logs - if: startsWith(matrix.platform, 'debian/') || startsWith(matrix.platform, 'ubuntu/') - env: - GITHUB_TOKEN: ${{ steps.app.outputs.token }} - CONTAINER_BUILD_RUN_ENABLED: "true" - run: | - set -euo pipefail - mkdir -p container_logs - mkdir -p "$(pwd)/packages/${{ matrix.platform }}" - rm -f container_logs/stdout.txt container_logs/stderr.txt - - os="${{ matrix.platform }}" - os="${os%%/*}" # debian or ubuntu - rel="${{ matrix.platform }}" - rel="${rel##*/}" # bookworm/trixie/jammy/noble - - IMAGE="citus/packaging:${os}-${rel}-all" - echo "Using image: $IMAGE" - - docker run --rm \ - -v "$(pwd)/packages/${{ matrix.platform }}":/packages \ - -v "$(pwd)/packaging":/buildfiles:ro \ - -e GITHUB_TOKEN \ - -e CONTAINER_BUILD_RUN_ENABLED \ - -e PACKAGE_ENCRYPTION_KEY \ - -e UNENCRYPTED_PACKAGE \ - -e MSRUSTUP_PAT \ - -e CRATES_IO_MIRROR_FEED_TOKEN \ - -e INSTALL_RUST \ - -e CI \ - "$IMAGE" nightly \ - > container_logs/stdout.txt 2> container_logs/stderr.txt || true - - echo "===== STDERR (first 200 lines) =====" - sed -n '1,200p' container_logs/stderr.txt || true - echo "===== STDOUT (first 200 lines) =====" - sed -n '1,200p' container_logs/stdout.txt || true - - echo "===== STDERR (last 200 lines) =====" - tail -n 200 container_logs/stderr.txt || true - echo "===== STDOUT (last 200 lines) =====" - tail -n 200 container_logs/stdout.txt || true - - - uses: actions/upload-artifact@v4 - if: always() && (startsWith(matrix.platform, 'debian/') || startsWith(matrix.platform, 'ubuntu/')) - with: - name: container-logs-${{ matrix.platform }} - path: container_logs/ - - name: Build packages run: | python -m tools.packaging_automation.citus_package \ From 90ef0c726179d91b0495d6a50e449e7d6f04ad90 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Wed, 22 Apr 2026 18:10:03 +0300 Subject: [PATCH 25/27] checking error code --- packaging_automation/citus_package.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 06a81dc7..8aa81eb8 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -360,7 +360,13 @@ def build_package( if output.stdout: print("Output:" + output.stdout) if output.returncode != 0: - raise ValueError(output.stderr) + raise ValueError( + "Docker command failed.\n" + f"Command: {docker_command}\n" + f"Exit code: {output.returncode}\n" + f"--- stdout ---\n{output.stdout}\n" + f"--- stderr ---\n{output.stderr}\n" + ) if input_output_parameters.output_validation: validate_output( From cb2cf5e37e50ddbe15bfe98dd049330497f9cac5 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Wed, 22 Apr 2026 21:30:50 +0300 Subject: [PATCH 26/27] add stderr redirection to docker command in build_package function --- packaging_automation/citus_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 8aa81eb8..210486f2 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -351,7 +351,7 @@ def build_package( f"{input_output_parameters.input_files_dir}:/buildfiles:ro " f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " - f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" + f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name} 2>&1" ) print(f"Executing docker command: {docker_command}") From a07278cbd341600a8b3f75bfbb2930158296c371 Mon Sep 17 00:00:00 2001 From: BarkinKctp Date: Wed, 22 Apr 2026 21:43:15 +0300 Subject: [PATCH 27/27] debugging --- packaging_automation/citus_package.py | 9 +++------ packaging_automation/common_tool_methods.py | 8 +++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 210486f2..36f249c2 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -242,7 +242,7 @@ def sign_packages( print(f"Result:{output}") if result.returncode != 0: - raise ValueError(f"Error while signing rpm files.Err:{result.stderr}") + raise ValueError(f"Error while signing rpm files.Err:{result.stdout}") if input_output_parameters.output_validation: validate_output( output, @@ -351,21 +351,18 @@ def build_package( f"{input_output_parameters.input_files_dir}:/buildfiles:ro " f"-e GITHUB_TOKEN -e PACKAGE_ENCRYPTION_KEY -e UNENCRYPTED_PACKAGE -e CONTAINER_BUILD_RUN_ENABLED " f"-e MSRUSTUP_PAT -e CRATES_IO_MIRROR_FEED_TOKEN -e INSTALL_RUST -e CI " - f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name} 2>&1" + f"citus/{docker_image_name}:{docker_platform}-{postgres_extension} {build_type.name}" ) print(f"Executing docker command: {docker_command}") output = run_with_output(docker_command, text=True) - if output.stdout: - print("Output:" + output.stdout) if output.returncode != 0: raise ValueError( "Docker command failed.\n" f"Command: {docker_command}\n" f"Exit code: {output.returncode}\n" - f"--- stdout ---\n{output.stdout}\n" - f"--- stderr ---\n{output.stderr}\n" + f"--- combined output (stdout+stderr) ---\n{output.stdout}\n" ) if input_output_parameters.output_validation: diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 91765272..a3a312d9 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -166,7 +166,13 @@ def run_with_output(command, *args, **kwargs): # this method's main objective is to return output. Therefore it is caller's responsibility to handle # success status # pylint: disable=subprocess-run-check - result = subprocess.run(shlex.split(command), *args, capture_output=True, **kwargs) + result = subprocess.run( + shlex.split(command), + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + **kwargs + ) return result