Improve acceptance tests

.gitignore

@@ -5,7 +5,6 @@ releases/*.tgz
 releases/**/*.tgz
 ci/scripts/stemcell/*.tgz
 ci/scripts/stemcell-bionic/*.tgz
-ci/scripts/bpm/*.tgz
 dev_releases
 blobs/*
 .blobs

acceptance-tests/README.md

@@ -3,12 +3,7 @@
 ## Requirements
 
 * Docker installed locally
-* A matching Jammy stemcell tgz downloaded to `ci/scripts/stemcell`
-  * Get it from https://bosh.io/stemcells/bosh-warden-boshlite-ubuntu-jammy-go_agent
-* A matching Bionic stemcell tgz downloaded to `ci/scripts/stemcell-bionic`
-  * Get it from https://bosh.io/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent
-* A BPM release tgz downloaded to `ci/scripts/bpm`
-  * Get it from https://bosh.io/releases/github.com/cloudfoundry/bpm-release?all=1
+* x86 based computer (cloudfoundry currently doesn't build/run on arm; rosetta can't run docker in docker for x86)
 
 ## Running
 
@@ -19,6 +14,8 @@ cd acceptance-tests
 
 ### Running on Docker for Mac
 
+Unfortunately rosetta on arm based macs doesn't support docker in docker required for bosh docker-cpi used in this test setup. You will either have to run on x86 based mac or some remote x86 workstation. Virtualization using QEMU is possible, but so slow the tests fail on timeouts.
+
 The BOSH Docker CPI requires cgroups v1 to be active. Docker for Mac since 4.3.x uses cgroups v2 by default.
 
 v1 can be restored with the flag `deprecatedCgroupv1` to `true` in `~/Library/Group Containers/group.com.docker/settings.json`.
@@ -82,24 +79,40 @@ The output at the end should be:
 
 If you want to run only a specific part of the suite, you can use [focussed specs](https://onsi.github.io/ginkgo/#focused-specs)
 
-The easiest way is to just provide the name of the tests you want to run as a command line argument like so:
+The easiest way is to just provide the name of the tests you want to run as a `-F` command line argument:
 
 ```shell
-./run-local.sh "description of the test to run"
+./run-local.sh -F "description of the test to run"
 ```
 
 The argument is passed as a regular expression that will match all `Describe`, `Context` or `It` closure descriptions in the suite.
 So, e.g. if you want to run all tests that use mTLS, you can run:
 
 ```shell
-./run-local.sh mTLS
+./run-local.sh -F mTLS
 ```
 
 However, if you want to run exactly one specific test, make sure you pass the exact description of the matching `It` closure:
 
 ```shell
-./run-local.sh "Correctly terminates mTLS requests"
+./run-local.sh -F "Correctly terminates mTLS requests"
 ```
 
-Alternatively, you add an `F` to your `Describe`, `Context` or `It` closures.
-Don't forget to do a local commit before running the tests, else BOSH will fail to produce a release. The git repo must be clean before running the tests.
+Running tests in focus will also preserve the bosh container running after tests complete, so that you can easily run tests again without having to wait for bosh set-up again.
+
+### Persistent BOSH
+
+Because BOSH setup takes a while (it starts from scratch with bosh create-env), it is useful to preserve the container with bosh already configured to run tests. This can be done either by providing test focus as described above, or `-k` (keep) switch to `run-local.sh` and `run-shell.sh` scripts, e.g. `run-shell.sh -k`. Once initial setup is complete, scripts will output a message about how to get back into the running container:
+
+```text
+KEEP_RUNNING is true and bosh remains running.
+Re-enter container via: docker exec -it b7c767c5c0e4 bash
+
+Stop with: docker stop b7c767c5c0e4
+```
+
+After you have completed your work and have stopped the container, it is required you do further cleanup. In order to have a working overlay2 filesystem for docker-cpi, it is necessary to mount ext4 based storage from `/workspace/docker-in` into the bosh container. Each container gets its own temporary space, which in case of containers that keep running is not deleted when the tests complete. The scripts will output the location of this temporary storage:
+
+```text
+*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: /workspace/docker-in/scratch-19517
+```

acceptance-tests/run-local.sh

@@ -1,18 +1,32 @@
 #!/usr/bin/env bash
-FOCUS="$1"
 
 set -eu
+REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+source "${REPO_DIR}/ci/scripts/functions-ci.sh"
+KEEP_RUNNING=""
 
-SCRIPT_DIR="$(cd "$(dirname "$0")/" && pwd)"
-REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+usage() {
+    echo -e "Usage: $0 [-F <ginkgo focus target>] [-k]
 
-if ! [[ $(git status --porcelain=v1 2>/dev/null | wc -l) -eq 0 ]]; then
-    echo "You have changes in your Git repository. Commit or clean (e.g. git clean -f) before running."
-    echo "The build will fail otherwise."
-    echo "Git Status:"
-    git status
-    exit 1
-fi
+    -F      Focus on a particular test. Expects a Ginkgo test name. Keep bosh running afterwards.
+    -k      Keep bosh container running. Useful for debug." 1>&2; exit 1;
+}
+
+while getopts ":F:k" o; do
+    case "${o}" in
+        F)
+            FOCUS=${OPTARG}
+            KEEP_RUNNING=true
+            ;;
+        k)
+            KEEP_RUNNING=true
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))
 
 docker_mac_check_cgroupsv1() {
     # Force cgroups v1 on Docker for Mac
@@ -31,19 +45,14 @@ docker_mac_check_cgroupsv1() {
 check_required_files() {
   PIDS=""
   REQUIRED_FILE_PATTERNS=(
-    ../ci/scripts/stemcell/bosh-stemcell-*-ubuntu-jammy-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-jammy-go_agent
-    ../ci/scripts/stemcell-bionic/bosh-stemcell-*-ubuntu-bionic-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent
+    ci/scripts/stemcell/bosh-stemcell-*-ubuntu-jammy-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-jammy-go_agent
+    ci/scripts/stemcell-bionic/bosh-stemcell-*-ubuntu-bionic-*.tgz!https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent
   )
 
-  # BPM downloads as uuid and needs manually specifying file name
-  if [ ! -f "../ci/scripts/bpm/bpm-release-latest.tgz" ] ; then
-    curl -sL https://bosh.io/d/github.com/cloudfoundry/bpm-release -o ../ci/scripts/bpm/bpm-release-latest.tgz
-  fi
-
   for entry in "${REQUIRED_FILE_PATTERNS[@]}"; do
     pattern=$(cut -f1 -d! <<<"$entry")
     url=$(cut -f2 -d! <<<"$entry")
-    folder=$(realpath "$(dirname "$SCRIPT_DIR/$pattern")")
+    folder=$(realpath "$(dirname "$REPO_DIR/$pattern")")
     filepattern=$(basename "$pattern")
     pattern=$folder/$filepattern
 
@@ -76,14 +85,17 @@ if [ "$(uname)" == "Darwin" ]; then
     docker_mac_check_cgroupsv1
 fi
 
-# Build acceptance test image
-pushd "$SCRIPT_DIR/../ci" || exit 1
- docker build -t haproxy-boshrelease-testflight .
-popd || exit 1
+build_image "${REPO_DIR}/ci"
+prepare_docker_scratch
 
 # Run acceptance tests
-if [ -n "$FOCUS" ]; then
-  docker run --privileged -v "$REPO_DIR":/repo -e REPO_ROOT=/repo -e FOCUS="$FOCUS" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
+if [ -n "$KEEP_RUNNING" ] ; then
+  echo
+  echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}"
+  echo
+  docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="$FOCUS" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
 else
-  docker run --rm --privileged -v "$REPO_DIR":/repo -e REPO_ROOT=/repo haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
+  docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
+  echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}"
+  sudo rm -rf "${DOCKER_SCRATCH}"
 fi

acceptance-tests/run-shell.sh

@@ -1,14 +1,37 @@
 #!/usr/bin/env bash
 
 set -e
+REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+source "${REPO_DIR}/ci/scripts/functions-ci.sh"
 
-SCRIPT_DIR="$(cd "$(dirname "$0")/" && pwd)"
-REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+usage() {
+    echo -e "Usage: $0 [-k]
 
-# Build acceptance test image
-pushd "$SCRIPT_DIR/../ci" || exit 1
- docker build -t haproxy-boshrelease-testflight .
-popd || exit 1
+    -k      Keep bosh container running. Useful for debug." 1>&2; exit 1;
+}
 
-# Run local shell
-docker run -it --rm --privileged -v "$REPO_DIR":/repo -e REPO_ROOT=/repo haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./shell"
+while getopts ":k" o; do
+    case "${o}" in
+        k)
+            KEEP_RUNNING=true
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+
+build_image "${REPO_DIR}/ci"
+prepare_docker_scratch
+
+# Run local shell - start new container with bosh
+if [ -n "${KEEP_RUNNING}" ] ; then
+    echo
+    echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}"
+    echo
+    docker run -it --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e KEEP_RUNNING="${KEEP_RUNNING}" -e REPO_ROOT=/repo haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./shell ; sleep infinity"
+else
+    docker run -it --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e KEEP_RUNNING="" -e REPO_ROOT=/repo haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./shell"
+    echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}"
+    sudo rm -rf "${DOCKER_SCRATCH}"
+fi

ci/compilation.yml

@@ -0,0 +1,3 @@
+- path: /compilation/workers
+  type: replace
+  value: 2

ci/pipeline.yml

@@ -122,7 +122,6 @@ jobs:
         - { get: git, trigger: true, passed: [unit-tests] }
         - { get: stemcell }
         - { get: stemcell-bionic }
-        - { get: bpm }
         - get: haproxy-boshrelease-testflight
       - task: acceptance-tests
         privileged: true
@@ -133,7 +132,6 @@ jobs:
             - { name: git }
             - { name: stemcell }
             - { name: stemcell-bionic }
-            - { name: bpm }
           run:
             path: ./git/ci/scripts/acceptance-tests
             args: []
@@ -155,7 +153,6 @@ jobs:
       - { get: git-pull-requests, trigger: true, version: every }
       - { get: stemcell }
       - { get: stemcell-bionic }
-      - { get: bpm }
       - get: haproxy-boshrelease-testflight
       - put: git-pull-requests
         params:
@@ -173,7 +170,6 @@ jobs:
             - { name: git-pull-requests }
             - { name: stemcell }
             - { name: stemcell-bionic }
-            - { name: bpm }
           run:
             path: ./git-pull-requests/ci/scripts/acceptance-tests
             args: []
@@ -418,11 +414,6 @@ resources:
     source:
       name: bosh-warden-boshlite-ubuntu-jammy-go_agent
 
-  - name: bpm
-    type: bosh-io-release
-    source:
-      repository: cloudfoundry/bpm-release
-
   - name: version
     type: semver
     source :

ci/scripts/acceptance-tests

@@ -1,27 +1,18 @@
 #!/bin/bash
 
 set -e
-
-stemcell_jammy_path=$PWD/stemcell/*.tgz
-stemcell_bionic_path=$PWD/stemcell-bionic/*.tgz
-bpm_release_path=$PWD/bpm/*.tgz
+source "${REPO_ROOT}/ci/scripts/functions-ci.sh"
+START_DIR="${PWD}"  # Differs for CI and manual execution
 
 if [ -n "$FOCUS" ]; then
   echo "------------------------------------------------------------------"
   echo "FOCUS is set. Will only run tests matching '$FOCUS'"
-  echo "Docker won't be stopped afterwards, so you can debug the test."
   echo "------------------------------------------------------------------"
   ADDITIONAL_ARGS=("--focus" "$FOCUS")
 fi
 
-cd "${REPO_ROOT:?required}"
-echo "----- Pulling in any git submodules..."
-git config --global --add safe.directory /repo
-git config --global --add safe.directory /repo/src/ttar
-git submodule update --init --recursive --force
+git_pull
 
-# shellcheck disable=SC1091
-source "ci/scripts/skip-ci.sh"
 if [ -f ".git/resource/changed_files" ]; then
   if skip_ci "ci/.ci-ignore" ".git/resource/changed_files"; then
     echo "SKIP TEST: Only .ci-ignored changes found."
@@ -30,47 +21,8 @@ if [ -f ".git/resource/changed_files" ]; then
     echo "RUN TEST: There is at least one non-ignored change found."
   fi
 fi
-echo "----- Starting BOSH"
-
-./ci/scripts/start-bosh.sh
-
-function stop_docker() {
-  echo "----- stopping docker"
-  service docker stop
-}
-
-if [ -z "$FOCUS" ]; then
-  trap stop_docker EXIT
-fi
-
-# shellcheck disable=SC1091
-source /tmp/local-bosh/director/env
-
-echo "----- Creating candidate BOSH release..."
-bosh -n reset-release # in case dev_releases/ is in repo accidentally
-
-bosh create-release
-bosh upload-release --rebase
-release_final_version=$(spruce json dev_releases/*/index.yml | jq -r ".builds[].version" | sed -e "s%+.*%%")
-export RELEASE_VERSION="${release_final_version}.latest"
-echo "----- Created ${RELEASE_VERSION}"
-
-echo "----- Uploading Jammy stemcell"
-bosh -n upload-stemcell $stemcell_jammy_path
 
-echo "----- Uploading Bionic stemcell"
-bosh -n upload-stemcell $stemcell_bionic_path
-
-echo "----- Uploading BPM"
-bosh -n upload-release $bpm_release_path
-
-echo "----- Uploading os-conf (used for tests only)"
-bosh -n upload-release --sha1 386293038ae3d00813eaa475b4acf63f8da226ef \
-  https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=22.1.2
-
-# shellcheck disable=SC2155
-export BOSH_PATH=$(which bosh)
-export BASE_MANIFEST_PATH="$PWD/manifests/haproxy.yml"
+prepare_bosh
 
 cd "acceptance-tests"
 
@@ -82,4 +34,12 @@ echo "----- Running tests"
 
 export PATH=$PATH:$GOPATH/bin
 ginkgo version
-ginkgo -v -p -r --trace --show-node-events --randomize-all --flake-attempts 5 "${ADDITIONAL_ARGS[@]}"
+
+PARALLELISM="-p"
+if [ -n "$FOCUS" ]; then
+  PARALLELISM="--nodes=1"
+fi
+
+ginkgo -v "$PARALLELISM" -r --trace --show-node-events --randomize-all --flake-attempts 5 "${ADDITIONAL_ARGS[@]}"
+
+keep_running_info