From 4c6acdc30810cd84769bbdfec7e8b1720d8e3f7f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 9 Apr 2026 13:25:21 -0400 Subject: [PATCH 1/9] fix: adds config file and shell script to run storage cloudbuild tests --- storage/cloudbuild/run_zonal_tests.sh | 31 ++++ .../zb-system-tests-cloudbuild.yaml | 137 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 storage/cloudbuild/run_zonal_tests.sh create mode 100644 storage/cloudbuild/zb-system-tests-cloudbuild.yaml diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh new file mode 100644 index 00000000000..c54cf0ec858 --- /dev/null +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -0,0 +1,31 @@ + +set -euxo pipefail +echo '--- Installing git and cloning repository on VM ---' +sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv + +# Clone the repository and checkout the specific commit from the build trigger. +git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/python-docs-samples.git +cd python-docs-samples +git sparse-checkout set main/storage +git fetch origin "refs/pull/${_PR_NUMBER}/head" +git checkout ${COMMIT_SHA} +cd main/storage + + +echo '--- Installing Python and dependencies on VM ---' +python3 -m venv env +source env/bin/activate + +echo 'Install testing libraries explicitly, as they are not in setup.py' +pip install --upgrade pip +pip install pytest pytest-timeout pytest-subtests pytest-asyncio +pip install google-cloud-testutils google-cloud-kms +pip install -e . + +echo '--- Setting up environment variables on VM ---' +export ZONAL_BUCKET=${_ZONAL_BUCKET} +export RUN_ZONAL_SYSTEM_TESTS=True +export GCE_METADATA_MTLS_MODE=None +CURRENT_ULIMIT=$(ulimit -n) +echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT +pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py \ No newline at end of file diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml new file mode 100644 index 00000000000..b77381c011c --- /dev/null +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -0,0 +1,137 @@ +substitutions: + _REGION: "us-central1" + _ZONE: "us-central1-a" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}" + _ULIMIT: "10000" # 10k, for gRPC bidi streams + + + +steps: + # Step 0: Generate a persistent SSH key for this build run. + # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + # Generate the SSH key + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + # Save the public key content to a file for the cleanup step + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + waitFor: ["-"] + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-old-keys" + entrypoint: "bash" + args: + - "-c" + - | + #!/bin/bash + set -e + + echo "Fetching OS Login SSH keys..." + echo "Removing all keys." + echo "---------------------------------------------------------------------" + + FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \ + --format="value(fingerprint)") + + echo "Keys to delete: $$FINGERPRINTS_TO_DELETE" + + if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then + echo "No keys found to delete. Nothing to do." + exit 0 + fi + + while IFS= read -r FINGERPRINT; do + if [ -n "$$FINGERPRINT" ]; then + echo "Deleting key with fingerprint: $$FINGERPRINT" + gcloud compute os-login ssh-keys remove \ + --key="$$FINGERPRINT" \ + --quiet || true + fi + done <<< "$$FINGERPRINTS_TO_DELETE" + + echo "---------------------------------------------------------------------" + echo "Cleanup complete." + + # Step 1 Create a GCE VM to run the tests. + # The VM is created in the same zone as the buckets to test rapid storage features. + # It's given the 'cloud-platform' scope to allow it to access GCS and other services. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "${_VM_NAME}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=e2-medium" + - "--image-family=debian-13" + - "--image-project=debian-cloud" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/devstorage.read_write" + - "--metadata=enable-oslogin=TRUE" + waitFor: ["-"] + + # Step 2: Run the integration tests inside the newly created VM and cleanup. + # This step uses 'gcloud compute ssh' to execute a remote script. + # The VM is deleted after tests are run, regardless of success. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-and-delete-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready. + for i in {1..10}; do + if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $i/10)" + sleep 15 + done + # copy the script to the VM + gcloud compute scp main/storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + # Execute the script on the VM via SSH. + # Capture the exit code to ensure cleanup happens before the build fails. + set +e + gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh" + EXIT_CODE=$? + set -e + + echo "--- Deleting GCE VM ---" + gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet + + # Exit with the original exit code from the test script. + exit $$EXIT_CODE + waitFor: + - "create-vm" + - "generate-ssh-key" + - "cleanup-old-keys" + + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "--- Removing SSH key from OS Login profile to prevent accumulation ---" + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "run-tests-and-delete-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" \ No newline at end of file From b1fa4d222fc31a536fcde6553c2e6f8526cdfbdc Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 7 May 2026 16:44:22 +0530 Subject: [PATCH 2/9] Fix paths in run_zonal_tests.sh script --- storage/cloudbuild/run_zonal_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh index c54cf0ec858..51219f07111 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -1,4 +1,4 @@ - +#!/bin/bash set -euxo pipefail echo '--- Installing git and cloning repository on VM ---' sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv @@ -6,10 +6,10 @@ sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv # Clone the repository and checkout the specific commit from the build trigger. git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/python-docs-samples.git cd python-docs-samples -git sparse-checkout set main/storage +git sparse-checkout set storage git fetch origin "refs/pull/${_PR_NUMBER}/head" git checkout ${COMMIT_SHA} -cd main/storage +cd storage echo '--- Installing Python and dependencies on VM ---' @@ -28,4 +28,4 @@ export RUN_ZONAL_SYSTEM_TESTS=True export GCE_METADATA_MTLS_MODE=None CURRENT_ULIMIT=$(ulimit -n) echo '--- Running Zonal tests on VM with ulimit set to ---' $CURRENT_ULIMIT -pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py \ No newline at end of file +pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' samples/snippets/zonal_buckets/zonal_snippets_test.py From 497dd775d72ea3884e19b9a6d3f447cc84d7e237 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 7 May 2026 16:53:07 +0530 Subject: [PATCH 3/9] Fix path for copying run_zonal_tests.sh to VM --- storage/cloudbuild/zb-system-tests-cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml index b77381c011c..02e24d67fba 100644 --- a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -98,7 +98,7 @@ steps: sleep 15 done # copy the script to the VM - gcloud compute scp main/storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + gcloud compute scp storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute the script on the VM via SSH. # Capture the exit code to ensure cleanup happens before the build fails. @@ -134,4 +134,4 @@ timeout: "3600s" # 60 minutes options: logging: CLOUD_LOGGING_ONLY pool: - name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" \ No newline at end of file + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" From 1bb80b53963d8dce25517c303d7160c47c86e7f4 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 7 May 2026 19:08:24 +0530 Subject: [PATCH 4/9] Update repository URL in run_zonal_tests.sh --- storage/cloudbuild/run_zonal_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh index 51219f07111..31c61d60134 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -4,7 +4,7 @@ echo '--- Installing git and cloning repository on VM ---' sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv # Clone the repository and checkout the specific commit from the build trigger. -git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/python-docs-samples.git +git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/GoogleCloudPlatform/python-docs-samples cd python-docs-samples git sparse-checkout set storage git fetch origin "refs/pull/${_PR_NUMBER}/head" From 1e07b517379d759990d461fddb232e74fb473fb9 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 7 May 2026 19:21:09 +0530 Subject: [PATCH 5/9] Update pip install command in run_zonal_tests.sh Replace local package installation with google-cloud-storage installation. --- storage/cloudbuild/run_zonal_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh index 31c61d60134..c0303f119fd 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -20,7 +20,7 @@ echo 'Install testing libraries explicitly, as they are not in setup.py' pip install --upgrade pip pip install pytest pytest-timeout pytest-subtests pytest-asyncio pip install google-cloud-testutils google-cloud-kms -pip install -e . +pip install google-cloud-storage[grpc,testing] echo '--- Setting up environment variables on VM ---' export ZONAL_BUCKET=${_ZONAL_BUCKET} From 0a9d76f3a428dbfaebc27a75b8678b3dddb7c306 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 14 May 2026 21:14:36 +0530 Subject: [PATCH 6/9] Add copyright and license comments to run_zonal_tests.sh Add copyright notice and licensing information to the script. --- storage/cloudbuild/run_zonal_tests.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh index c0303f119fd..da0e405d740 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -1,4 +1,20 @@ #!/bin/bash + +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + set -euxo pipefail echo '--- Installing git and cloning repository on VM ---' sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv From f935cb05596a8c750247d0ed6fae31ee67b5b7e0 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Thu, 14 May 2026 21:15:26 +0530 Subject: [PATCH 7/9] Add license and copyright information Add license information and copyright notice to the cloudbuild.yaml file. --- storage/cloudbuild/zb-system-tests-cloudbuild.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml index 02e24d67fba..91a2ea11991 100644 --- a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -1,3 +1,17 @@ +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + substitutions: _REGION: "us-central1" _ZONE: "us-central1-a" From 66bbeb282e0e7431c401938852f4982fa82bfa05 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 14 May 2026 16:29:09 -0400 Subject: [PATCH 8/9] Update license to the current version from gapic generator template. --- storage/cloudbuild/zb-system-tests-cloudbuild.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml index 91a2ea11991..b968a0df18a 100644 --- a/storage/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/storage/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -1,6 +1,6 @@ -# Copyright 2026 Google Inc. All Rights Reserved. +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the 'License'); +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# substitutions: _REGION: "us-central1" From 1c5f1d6f42843926d397e3fb649d153782c4dfbd Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Thu, 14 May 2026 16:29:57 -0400 Subject: [PATCH 9/9] Update license to the version used in the gapic generator templates. --- storage/cloudbuild/run_zonal_tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/cloudbuild/run_zonal_tests.sh b/storage/cloudbuild/run_zonal_tests.sh index da0e405d740..25a8e714c7d 100644 --- a/storage/cloudbuild/run_zonal_tests.sh +++ b/storage/cloudbuild/run_zonal_tests.sh @@ -1,8 +1,8 @@ #!/bin/bash -# Copyright 2026 Google Inc. All Rights Reserved. +# Copyright 2026 Google LLC # -# Licensed under the Apache License, Version 2.0 (the 'License'); +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# set -euxo pipefail