diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index e4773508..745abc19 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -4,7 +4,6 @@ env: MAIN_BRANCH: "all-citus" PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} on: @@ -32,9 +31,25 @@ jobs: - ubuntu/jammy steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 1 path: tools @@ -46,7 +61,7 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Clone build branch - run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging + run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index a7692477..2689fc4b 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -1,8 +1,6 @@ name: Citus package all platforms tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay @@ -36,8 +34,27 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: | + echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + echo "GITHUB_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Setup git authentication for GitHub App + run: | + git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/delete-packagecloud-packages.yml b/.github/workflows/delete-packagecloud-packages.yml index c9d55c53..de8a8900 100644 --- a/.github/workflows/delete-packagecloud-packages.yml +++ b/.github/workflows/delete-packagecloud-packages.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 0d401807..c5c8dff1 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -18,10 +18,12 @@ jobs: pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} citus_version: ${{ steps.get-citus-version.outputs.citus_version }} steps: + - name: Checkout uses: actions/checkout@v3 with: fetch-depth: 2 + - name: Package version id: get-citus-version run: | @@ -60,8 +62,24 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: - - name: Checkout repository + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + + - name: Checkout uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7d53056f..f7030f8c 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -1,8 +1,5 @@ name: Packaging helper methods tests -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - on: push: branches: @@ -13,10 +10,25 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c14bc3d7..70fd4dbb 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -10,8 +10,8 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - steps: + - name: Checkout repository uses: actions/checkout@v3 diff --git a/.github/workflows/pypi-statistics-schedule.yml b/.github/workflows/pypi-statistics-schedule.yml index a8562ddc..d54548d0 100644 --- a/.github/workflows/pypi-statistics-schedule.yml +++ b/.github/workflows/pypi-statistics-schedule.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 3dcb4b22..5aad636a 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} on: schedule: - cron: "0 16 * * *" @@ -25,8 +24,24 @@ jobs: job_name: [docker_pull_citus, github_clone_citus, homebrew_citus] steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 5b9eb9ea..702b1766 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }} on: @@ -19,10 +18,26 @@ on: jobs: unit_test_execution: runs-on: ubuntu-latest - + steps: + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 + with: + token: ${{ steps.app.outputs.token }} - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 8759a7d5..89029dda 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,7 +1,6 @@ name: Tool Tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay MAIN_BRANCH: all-citus @@ -20,16 +19,32 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + - name: Install all scripts run: make && sudo make install unit_test_execution: runs-on: ubuntu-latest - steps: + + - name: Create GitHub App token + id: app + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: ${{ github.repository_owner }} + + - name: Set GH_TOKEN for all steps + run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV + + - name: Configure git with x-access-token + run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/" + - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} fetch-depth: 0 - name: Set up Python 3.10 diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 06a81dc7..36f249c2 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -242,7 +242,7 @@ def sign_packages( print(f"Result:{output}") if result.returncode != 0: - raise ValueError(f"Error while signing rpm files.Err:{result.stderr}") + raise ValueError(f"Error while signing rpm files.Err:{result.stdout}") if input_output_parameters.output_validation: validate_output( output, @@ -357,10 +357,13 @@ def build_package( print(f"Executing docker command: {docker_command}") output = run_with_output(docker_command, text=True) - if output.stdout: - print("Output:" + output.stdout) if output.returncode != 0: - raise ValueError(output.stderr) + raise ValueError( + "Docker command failed.\n" + f"Command: {docker_command}\n" + f"Exit code: {output.returncode}\n" + f"--- combined output (stdout+stderr) ---\n{output.stdout}\n" + ) if input_output_parameters.output_validation: validate_output( diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index b4b89d64..a3a312d9 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -166,7 +166,13 @@ def run_with_output(command, *args, **kwargs): # this method's main objective is to return output. Therefore it is caller's responsibility to handle # success status # pylint: disable=subprocess-run-check - result = subprocess.run(shlex.split(command), *args, capture_output=True, **kwargs) + result = subprocess.run( + shlex.split(command), + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + **kwargs + ) return result @@ -677,10 +683,12 @@ def remove_suffix(initial_str: str, suffix: str) -> str: return result_str -def initialize_env(exec_path: str, project_name: str, checkout_dir: str): +def initialize_env(exec_path: str, project_name: str, checkout_dir: str, gh_token: str = None): + if gh_token is None: + print("GITHUB_TOKEN not found") # Needed for cloning - may fail remove_cloned_code(f"{exec_path}/{checkout_dir}") if not os.path.exists(checkout_dir): - run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}") + run(f"git clone https://x-access-token:{gh_token}@github.com/citusdata/{project_name}.git {checkout_dir}") def create_pr( diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 7e62b7ce..80c437cb 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -87,8 +87,8 @@ def setup_module(): ) if not os.path.exists(PACKAGING_EXEC_FOLDER): run( - f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git" - f" {PACKAGING_EXEC_FOLDER}" + f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git " + f"{PACKAGING_EXEC_FOLDER}" ) diff --git a/packaging_automation/tests/test_citus_package_utils.py b/packaging_automation/tests/test_citus_package_utils.py index 064bfcca..4d31aa5a 100644 --- a/packaging_automation/tests/test_citus_package_utils.py +++ b/packaging_automation/tests/test_citus_package_utils.py @@ -34,7 +34,6 @@ TEST_GPG_KEY_NAME = "Citus Data " TEST_GPG_KEY_PASSPHRASE = "Citus123" GH_TOKEN = os.getenv("GH_TOKEN") - PACKAGING_SOURCE_FOLDER = "packaging_test" PACKAGING_EXEC_FOLDER = f"{TEST_BASE_PATH}/{PACKAGING_SOURCE_FOLDER}" OUTPUT_FOLDER = f"{PACKAGING_EXEC_FOLDER}/packages" @@ -46,7 +45,7 @@ def setup_module(): if not os.path.exists("packaging_test"): run( - f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" + f"git clone --branch all-citus-unit-tests https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}" ) diff --git a/packaging_automation/tests/test_prepare_release.py b/packaging_automation/tests/test_prepare_release.py index 19a637ba..dc0cc1c9 100644 --- a/packaging_automation/tests/test_prepare_release.py +++ b/packaging_automation/tests/test_prepare_release.py @@ -40,7 +40,7 @@ def initialize_env() -> str: test_base_path_major = f"{BASE_PATH}/{uuid.uuid4()}" remove_cloned_code(test_base_path_major) if not os.path.exists(test_base_path_major): - run(f"git clone https://github.com/citusdata/citus.git {test_base_path_major}") + run(f"git clone https://x-access-token:{github_token}@github.com/citusdata/citus.git {test_base_path_major}") return test_base_path_major diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 740ff446..0b190ec9 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -25,11 +25,11 @@ BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) EXEC_PATH = f"{BASE_PATH}/docker" - +GH_TOKEN = os.getenv("GH_TOKEN") def initialize_env(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/docker.git") def test_decode_triggering_event_info(): diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 06069dc6..a8a98594 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -30,11 +30,11 @@ version_details = get_version_details(PROJECT_VERSION) TEMPLATE_PATH = f"{BASE_PATH}/packaging_automation/templates/docker" PKGVARS_FILE = f"{TEST_BASE_PATH}/pkgvars" - +GH_TOKEN = os.getenv("GH_TOKEN") def setup_module(): if not os.path.exists("docker"): - run("git clone https://github.com/citusdata/docker.git") + run(f"git clone https://x-access-token:{GH_TOKEN}@github.com/citusdata/docker.git") def teardown_module(): diff --git a/packaging_automation/tests/test_update_pgxn.py b/packaging_automation/tests/test_update_pgxn.py index 572fc5c4..e37dacf3 100644 --- a/packaging_automation/tests/test_update_pgxn.py +++ b/packaging_automation/tests/test_update_pgxn.py @@ -14,12 +14,12 @@ PROJECT_VERSION = "10.0.3" PROJECT_NAME = "citus" TEMPLATE_PATH = f"{BASE_PATH}/packaging_automation/templates/pgxn" - +GH_TOKEN = os.getenv("GH_TOKEN") def setup_module(): if not os.path.exists("packaging_test"): run( - "git clone --branch pgxn-citus https://github.com/citusdata/packaging.git packaging_test" + f"git clone --branch pgxn-citus https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git packaging_test" )