From d12e4993b71553a103b0b0d843fefbfc085b5766 Mon Sep 17 00:00:00 2001 From: ay901246 Date: Fri, 17 Apr 2026 12:58:18 -0400 Subject: [PATCH 1/2] Pin and verify SHAs for CI image dependencies Made-with: Cursor --- .github/workflows/bump-binaries.yml | 81 +++++++++++++++++++++++++++++ ci/docker/Dockerfile | 42 ++++++++++----- ci/docker/dependencies.json | 26 +++++++++ ci/scripts/bump-binaries.sh | 75 ++++++++++++++++++++++++++ ci/tasks/build-docker-args/run.sh | 41 ++++++++++----- 5 files changed, 237 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/bump-binaries.yml create mode 100644 ci/docker/dependencies.json create mode 100755 ci/scripts/bump-binaries.sh diff --git a/.github/workflows/bump-binaries.yml b/.github/workflows/bump-binaries.yml new file mode 100644 index 000000000..c7a0e2e49 --- /dev/null +++ b/.github/workflows/bump-binaries.yml @@ -0,0 +1,81 @@ +name: Bump Binaries + +on: + schedule: + - cron: '0 4 * * 1' # Run every Monday at 4:00 AM UTC + workflow_dispatch: # Allow manual triggering + +jobs: + bump-binaries: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup jq + run: sudo apt-get install -y jq + + - name: Run bump script + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./ci/scripts/bump-binaries.sh + + - name: Test Docker build + run: | + # Mock the azstemcell binary for the docker build context + touch ci/docker/azstemcell + + # Extract args from dependencies.json + DEPS_FILE="ci/docker/dependencies.json" + BOSH_CLI_URL=$(jq -r '.bosh_cli.url' < $DEPS_FILE) + BOSH_CLI_SHA=$(jq -r '.bosh_cli.sha256' < $DEPS_FILE) + META4_CLI_URL=$(jq -r '.meta4_cli.url' < $DEPS_FILE) + META4_CLI_SHA=$(jq -r '.meta4_cli.sha256' < $DEPS_FILE) + YQ_CLI_URL=$(jq -r '.yq_cli.url' < $DEPS_FILE) + YQ_CLI_SHA=$(jq -r '.yq_cli.sha256' < $DEPS_FILE) + RUBY_INSTALL_URL=$(jq -r '.ruby_install.url' < $DEPS_FILE) + RUBY_INSTALL_SHA=$(jq -r '.ruby_install.sha256' < $DEPS_FILE) + GOLANGCI_LINT_URL=$(jq -r '.golangci_lint.url' < $DEPS_FILE) + GOLANGCI_LINT_SHA=$(jq -r '.golangci_lint.sha256' < $DEPS_FILE) + GOVC_URL=$(jq -r '.govc.url' < $DEPS_FILE) + GOVC_SHA=$(jq -r '.govc.sha256' < $DEPS_FILE) + + RUBY_VERSION=$(cat .ruby-version) + GEM_HOME="/usr/local/bundle" + + docker build \ + --build-arg BOSH_CLI_URL="$BOSH_CLI_URL" \ + --build-arg BOSH_CLI_SHA256="$BOSH_CLI_SHA" \ + --build-arg META4_CLI_URL="$META4_CLI_URL" \ + --build-arg META4_CLI_SHA256="$META4_CLI_SHA" \ + --build-arg YQ_CLI_URL="$YQ_CLI_URL" \ + --build-arg YQ_CLI_SHA256="$YQ_CLI_SHA" \ + --build-arg RUBY_INSTALL_URL="$RUBY_INSTALL_URL" \ + --build-arg RUBY_INSTALL_SHA256="$RUBY_INSTALL_SHA" \ + --build-arg GOLANGCI_LINT_INSTALL_URL="$GOLANGCI_LINT_URL" \ + --build-arg GOLANGCI_LINT_INSTALL_SHA256="$GOLANGCI_LINT_SHA" \ + --build-arg GOVC_INSTALL_URL="$GOVC_URL" \ + --build-arg GOVC_INSTALL_SHA256="$GOVC_SHA" \ + --build-arg RUBY_VERSION="$RUBY_VERSION" \ + --build-arg GEM_HOME="$GEM_HOME" \ + -f ci/docker/Dockerfile ci/docker/ + + - name: Create Pull Request + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update binary dependencies in dependencies.json" + title: "Update binary dependencies" + body: | + This PR automatically updates the binary dependencies in `ci/docker/dependencies.json`. + + This ensures that the CI image is always built with the latest versions of these tools while maintaining security via SHA-256 verification. + branch: "bump-binaries" + base: "windows-2019" + delete-branch: true diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 281165e57..85c57831e 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -3,12 +3,18 @@ ARG BASE_IMAGE=ubuntu:jammy FROM ${BASE_IMAGE} ARG BOSH_CLI_URL +ARG BOSH_CLI_SHA256 ARG META4_CLI_URL +ARG META4_CLI_SHA256 ARG GOLANGCI_LINT_INSTALL_URL +ARG GOLANGCI_LINT_INSTALL_SHA256 ARG GOVC_INSTALL_URL +ARG GOVC_INSTALL_SHA256 ARG YQ_CLI_URL +ARG YQ_CLI_SHA256 ARG RUBY_INSTALL_URL +ARG RUBY_INSTALL_SHA256 ARG GEM_HOME ARG RUBY_VERSION @@ -130,8 +136,10 @@ ENV PATH=/usr/local/go/bin:${PATH} ENV PATH ${GEM_HOME}/bin:${GEM_HOME}/gems/bin:${PATH} ENV BUNDLE_APP_CONFIG ${GEM_HOME} RUN cd /tmp \ - && curl --show-error -sL "${RUBY_INSTALL_URL}" \ - | tar -xzf - \ + && curl -fsSL "${RUBY_INSTALL_URL}" -o ruby-install.tar.gz \ + && echo "${RUBY_INSTALL_SHA256} ruby-install.tar.gz" | sha256sum -c - \ + && tar -xzf ruby-install.tar.gz \ + && rm ruby-install.tar.gz \ && cd ruby-install-* \ && make -s install \ && cd - \ @@ -140,38 +148,44 @@ RUN cd /tmp \ && echo 'gem: --no-document' > ~/.gemrc \ && NUM_CPUS=$(grep -c ^processor /proc/cpuinfo) \ && ruby-install --jobs=${NUM_CPUS} --cleanup --system ruby ${RUBY_VERSION} \ - -- --disable-install-doc --disable-install-rdoc \ + -- --disable-install-doc --disable-install-rdoc \ && gem update --system \ && bundle config --global path "${GEM_HOME}" \ && bundle config --global bin "${GEM_HOME}/bin" RUN bosh_cli_path="/usr/bin/bosh" \ - && curl --show-error -sL "${BOSH_CLI_URL}" \ - > "${bosh_cli_path}" \ + && curl -fsSL "${BOSH_CLI_URL}" > "${bosh_cli_path}" \ + && echo "${BOSH_CLI_SHA256} ${bosh_cli_path}" | sha256sum -c - \ && chmod +x "${bosh_cli_path}" RUN meta4_cli_path="/usr/local/bin/meta4" \ - && curl --show-error -sL "${META4_CLI_URL}" \ - > "${meta4_cli_path}" \ + && curl -fsSL "${META4_CLI_URL}" > "${meta4_cli_path}" \ + && echo "${META4_CLI_SHA256} ${meta4_cli_path}" | sha256sum -c - \ && chmod +x "${meta4_cli_path}" RUN cd /tmp \ && golangci_lint_path="/usr/local/bin/golangci-lint" \ - && curl --show-error -sL "${GOLANGCI_LINT_INSTALL_URL}" \ - | tar -xzf - \ + && curl -fsSL "${GOLANGCI_LINT_INSTALL_URL}" -o golangci-lint.tar.gz \ + && echo "${GOLANGCI_LINT_INSTALL_SHA256} golangci-lint.tar.gz" | sha256sum -c - \ + && tar -xzf golangci-lint.tar.gz \ + && rm golangci-lint.tar.gz \ && mv golangci-lint-*-linux-amd64/golangci-lint "${golangci_lint_path}" \ && rm -rf golangci-lint-*-linux-amd64 \ && chmod +x "${golangci_lint_path}" -RUN govc_path="/usr/local/bin/govc" \ - && curl --show-error -sL "${GOVC_INSTALL_URL}" \ - | tar -xOzf - govc > "${govc_path}" \ +RUN cd /tmp \ + && govc_path="/usr/local/bin/govc" \ + && curl -fsSL "${GOVC_INSTALL_URL}" -o govc.tar.gz \ + && echo "${GOVC_INSTALL_SHA256} govc.tar.gz" | sha256sum -c - \ + && tar -xzf govc.tar.gz govc \ + && rm govc.tar.gz \ + && mv govc "${govc_path}" \ && chmod +x "${govc_path}" RUN yq_cli_path="/usr/local/bin/yq" \ - && curl --show-error -sL "${YQ_CLI_URL}" \ - > "${yq_cli_path}" \ + && curl -fsSL "${YQ_CLI_URL}" > "${yq_cli_path}" \ + && echo "${YQ_CLI_SHA256} ${yq_cli_path}" | sha256sum -c - \ && chmod +x "${yq_cli_path}" diff --git a/ci/docker/dependencies.json b/ci/docker/dependencies.json new file mode 100644 index 000000000..fef24e7f8 --- /dev/null +++ b/ci/docker/dependencies.json @@ -0,0 +1,26 @@ +{ + "bosh_cli": { + "url": "https://github.com/cloudfoundry/bosh-cli/releases/download/v7.10.2/bosh-cli-7.10.2-linux-amd64", + "sha256": "d9a52693994bdefc2fc73f1fe16042ffed48c6931308c6a9ae3520413d73065c" + }, + "meta4_cli": { + "url": "https://github.com/dpb587/metalink/releases/download/v0.5.0/meta4-0.5.0-linux-amd64", + "sha256": "9f3ff22e1ac3a8b4a667a9505dce2a224e099475ab69a02b23813ad073e27e01" + }, + "yq_cli": { + "url": "https://github.com/mikefarah/yq/releases/download/v4.53.2/yq_linux_amd64", + "sha256": "d56bf5c6819e8e696340c312bd70f849dc1678a7cda9c2ad63eebd906371d56b" + }, + "ruby_install": { + "url": "https://github.com/postmodern/ruby-install/releases/download/v0.10.2/ruby-install-0.10.2.tar.gz", + "sha256": "65836158b8026992b2e96ed344f3d888112b2b105d0166ecb08ba3b4a0d91bf6" + }, + "golangci_lint": { + "url": "https://github.com/golangci/golangci-lint/releases/download/v2.11.4/golangci-lint-2.11.4-linux-amd64.tar.gz", + "sha256": "200c5b7503f67b59a6743ccf32133026c174e272b930ee79aa2aa6f37aca7ef1" + }, + "govc": { + "url": "https://github.com/vmware/govmomi/releases/download/v0.53.0/govc_Linux_x86_64.tar.gz", + "sha256": "67cd529a9a2ec4c68fd1ae99cd4dcbee086591b029e23934a50564480e86e739" + } +} diff --git a/ci/scripts/bump-binaries.sh b/ci/scripts/bump-binaries.sh new file mode 100755 index 000000000..ae8cab219 --- /dev/null +++ b/ci/scripts/bump-binaries.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Find the absolute path to the dependencies.json file +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPS_FILE="${SCRIPT_DIR}/../docker/dependencies.json" + +TMP_FILE=$(mktemp) +trap 'rm -f "${TMP_FILE}"' EXIT + +if ! command -v jq &> /dev/null; then + echo "Error: jq is required but not installed." >&2 + exit 1 +fi + +get_latest_asset() { + local repo=$1 + local pattern=$2 + + # Use GH CLI if available (handles auth automatically), otherwise fallback to curl + if command -v gh &> /dev/null; then + gh api "repos/${repo}/releases/latest" | jq -r ".assets[] | select(.name | test(\"${pattern}\")) | .browser_download_url" | head -n 1 + else + curl -s "https://api.github.com/repos/${repo}/releases/latest" | jq -r ".assets[] | select(.name | test(\"${pattern}\")) | .browser_download_url" | head -n 1 + fi +} + +get_sha256() { + local url=$1 + curl -fsSL "${url}" | sha256sum | awk '{print $1}' +} + +update_dep() { + local key=$1 + local repo=$2 + local pattern=$3 + + echo "Checking ${repo}..." + local url + url=$(get_latest_asset "${repo}" "${pattern}") + + if [[ -z "${url}" || "${url}" == "null" ]]; then + echo "Failed to find asset for ${repo} matching ${pattern}" >&2 + return 1 + fi + + local current_url + current_url=$(jq -r ".${key}.url" < "${DEPS_FILE}") + + if [[ "${url}" == "${current_url}" ]]; then + echo " Already up to date." + return 0 + fi + + echo " Found new version: ${url}" + echo " Calculating SHA256..." + local sha + sha=$(get_sha256 "${url}") + + echo " Updating ${DEPS_FILE}..." + jq ".${key}.url = \"${url}\" | .${key}.sha256 = \"${sha}\"" "${DEPS_FILE}" > "${TMP_FILE}" + mv "${TMP_FILE}" "${DEPS_FILE}" + echo " Done." +} + +echo "Bumping binaries in ${DEPS_FILE}..." + +update_dep "bosh_cli" "cloudfoundry/bosh-cli" "linux-amd64" +update_dep "meta4_cli" "dpb587/metalink" "meta4-[0-9]+.[0-9]+.[0-9]+-linux-amd64" +update_dep "yq_cli" "mikefarah/yq" "linux_amd64$" +update_dep "ruby_install" "postmodern/ruby-install" "tar.gz$" +update_dep "golangci_lint" "golangci/golangci-lint" "golangci-lint-[0-9]+.[0-9]+.[0-9]+-linux-amd64.tar.gz" +update_dep "govc" "vmware/govmomi" "govc_Linux_x86_64.tar.gz" + +echo "All binaries bumped successfully!" diff --git a/ci/tasks/build-docker-args/run.sh b/ci/tasks/build-docker-args/run.sh index a95f8713d..3b5a5b71f 100755 --- a/ci/tasks/build-docker-args/run.sh +++ b/ci/tasks/build-docker-args/run.sh @@ -5,18 +5,25 @@ set -eu -o pipefail apt-get update -y apt-get install -y ca-certificates curl jq -bosh_cli_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/cloudfoundry/bosh-cli/releases/latest \ - | jq -r '.assets[] | select(.name | contains ("linux-amd64")) | .browser_download_url')" -meta4_cli_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/dpb587/metalink/releases/latest \ - | jq -r '.assets[] | select(.name | match("meta4-[0-9]+.[0-9]+.[0-9]+-linux-amd64")) | .browser_download_url')" -yq_cli_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/mikefarah/yq/releases/latest \ - | jq -r '.assets[] | select(.name | endswith ("linux_amd64")) | .browser_download_url')" -ruby_install_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/postmodern/ruby-install/releases/latest \ - | jq -r '.assets[] | select(.name | endswith ("tar.gz")) | .browser_download_url')" -golangci_lint_install_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/golangci/golangci-lint/releases/latest \ - | jq -r '.assets[] | select(.name | match("golangci-lint-[0-9]+.[0-9]+.[0-9]+-linux-amd64.tar.gz")) | .browser_download_url')" -govc_install_url="$(curl -H "Authorization: token ${GITHUB_ACCESS_TOKEN}" -s https://api.github.com/repos/vmware/govmomi/releases/latest \ - | jq -r '.assets[] | select(.name | match("govc_Linux_x86_64.tar.gz")) | .browser_download_url')" +deps_file="bosh-windows-stemcell-builder-ci/ci/docker/dependencies.json" + +bosh_cli_url="$(jq -r '.bosh_cli.url' < "${deps_file}")" +bosh_cli_sha="$(jq -r '.bosh_cli.sha256' < "${deps_file}")" + +meta4_cli_url="$(jq -r '.meta4_cli.url' < "${deps_file}")" +meta4_cli_sha="$(jq -r '.meta4_cli.sha256' < "${deps_file}")" + +yq_cli_url="$(jq -r '.yq_cli.url' < "${deps_file}")" +yq_cli_sha="$(jq -r '.yq_cli.sha256' < "${deps_file}")" + +ruby_install_url="$(jq -r '.ruby_install.url' < "${deps_file}")" +ruby_install_sha="$(jq -r '.ruby_install.sha256' < "${deps_file}")" + +golangci_lint_install_url="$(jq -r '.golangci_lint.url' < "${deps_file}")" +golangci_lint_install_sha="$(jq -r '.golangci_lint.sha256' < "${deps_file}")" + +govc_install_url="$(jq -r '.govc.url' < "${deps_file}")" +govc_install_sha="$(jq -r '.govc.sha256' < "${deps_file}")" gem_home="/usr/local/bundle" ruby_version="$(cat bosh-windows-stemcell-builder-ci/.ruby-version)" @@ -24,12 +31,18 @@ ruby_version="$(cat bosh-windows-stemcell-builder-ci/.ruby-version)" cat << JSON > docker-build-args/docker-build-args.json { "BOSH_CLI_URL": "${bosh_cli_url}", + "BOSH_CLI_SHA256": "${bosh_cli_sha}", "META4_CLI_URL": "${meta4_cli_url}", - "GOLANGCI_LINT_INSTALL_URL":"${golangci_lint_install_url}", - "GOVC_INSTALL_URL":"${govc_install_url}", + "META4_CLI_SHA256": "${meta4_cli_sha}", + "GOLANGCI_LINT_INSTALL_URL": "${golangci_lint_install_url}", + "GOLANGCI_LINT_INSTALL_SHA256": "${golangci_lint_install_sha}", + "GOVC_INSTALL_URL": "${govc_install_url}", + "GOVC_INSTALL_SHA256": "${govc_install_sha}", "YQ_CLI_URL": "${yq_cli_url}", + "YQ_CLI_SHA256": "${yq_cli_sha}", "RUBY_INSTALL_URL": "${ruby_install_url}", + "RUBY_INSTALL_SHA256": "${ruby_install_sha}", "RUBY_VERSION": "${ruby_version}", "GEM_HOME": "${gem_home}" } From 61b500ddd7e402a16b592b6768c2d993fa0c6c73 Mon Sep 17 00:00:00 2001 From: ay901246 Date: Fri, 17 Apr 2026 13:12:08 -0400 Subject: [PATCH 2/2] Add 7-day safety buffer to dependency bumps Made-with: Cursor --- ci/scripts/bump-binaries.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ci/scripts/bump-binaries.sh b/ci/scripts/bump-binaries.sh index ae8cab219..0ea72b6f5 100755 --- a/ci/scripts/bump-binaries.sh +++ b/ci/scripts/bump-binaries.sh @@ -13,15 +13,14 @@ if ! command -v jq &> /dev/null; then exit 1 fi -get_latest_asset() { +SAFETY_DAYS=7 + +get_latest_release_json() { local repo=$1 - local pattern=$2 - - # Use GH CLI if available (handles auth automatically), otherwise fallback to curl if command -v gh &> /dev/null; then - gh api "repos/${repo}/releases/latest" | jq -r ".assets[] | select(.name | test(\"${pattern}\")) | .browser_download_url" | head -n 1 + gh api "repos/${repo}/releases/latest" else - curl -s "https://api.github.com/repos/${repo}/releases/latest" | jq -r ".assets[] | select(.name | test(\"${pattern}\")) | .browser_download_url" | head -n 1 + curl -s -H "Authorization: token ${GITHUB_TOKEN:-}" "https://api.github.com/repos/${repo}/releases/latest" fi } @@ -36,14 +35,29 @@ update_dep() { local pattern=$3 echo "Checking ${repo}..." + + local release_json + release_json=$(get_latest_release_json "${repo}") + + # 1. Check the Safety Buffer using jq + local age_days + age_days=$(echo "${release_json}" | jq -r 'if .published_at then ((now - (.published_at | fromdateiso8601)) / 86400) | floor else -1 end') + + if [[ "${age_days}" != "-1" ]] && [[ "${age_days}" -lt "${SAFETY_DAYS}" ]]; then + echo " Skipping: Release is only ${age_days} days old (requires ${SAFETY_DAYS} days of safety)." + return 0 + fi + + # 2. Extract the URL local url - url=$(get_latest_asset "${repo}" "${pattern}") + url=$(echo "${release_json}" | jq -r ".assets[] | select(.name | test(\"${pattern}\")) | .browser_download_url" | head -n 1) if [[ -z "${url}" || "${url}" == "null" ]]; then echo "Failed to find asset for ${repo} matching ${pattern}" >&2 return 1 fi + # 3. Compare with current version local current_url current_url=$(jq -r ".${key}.url" < "${DEPS_FILE}") @@ -52,6 +66,7 @@ update_dep() { return 0 fi + # 4. Apply the update echo " Found new version: ${url}" echo " Calculating SHA256..." local sha