From 1eb023d475a52cf296f4074f0ba042ff856740e0 Mon Sep 17 00:00:00 2001 From: "Patrick M. Niedzielski" Date: Thu, 16 Apr 2026 14:20:24 -0400 Subject: [PATCH 1/2] CI: Use BlazingMQ broker Docker image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch changes the test CI workflow to pull the latest BlazingMQ Docker image for running tests with, rather than running the broker that we built from scratch. There’s a few different concerns here when making this change: - Pulling the Docker images requires us to log in to GHCR. To avoid accidentally (or maliciously) pushing a Docker image while we’re logged in, we give up the write permissions from our login token. - Rather than broker logs being interleaved with test failures, we have to pull them ourselves from the Docker container. - Because we’re running exactly the broker from GHCR, and we print which version we’re running, we can pull that exact Docker image locally rather than uploading the broker binary as an artifact. - We need to update some of our test broker configuration to work nicely in the Docker image, which mostly involves changing some file paths. Signed-off-by: Patrick M. Niedzielski --- .github/workflows/build.yaml | 67 +++++++++++++++++++++-------- tests/broker-config/bmqbrkrcfg.json | 4 +- tests/broker-config/clusters.json | 4 +- 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2f4e6eb..ddb2ae0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,6 +10,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +permissions: + contents: read + packages: read + jobs: blazingmq-dependency: name: Build BlazingMQ as a dependency @@ -128,17 +132,32 @@ jobs: libfl-dev \ libbenchmark-dev \ libz-dev - - name: Run tests + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Start BlazingMQ broker + run: | + docker run -d --name bmqbrkr \ + --network host \ + -v ${{ github.workspace }}/tests/broker-config:/broker-config:ro \ + ghcr.io/bloomberg/blazingmq:latest \ + /usr/local/bin/bmqbrkr /broker-config + timeout 30 bash -c 'until nc -z localhost 30114; do sleep 1; done' \ + || (docker logs bmqbrkr; exit 1) + - name: Build and run tests env: BMQ_BROKER_URI: tcp://localhost:30114 PREFIX: blazingmq_artifacts PYTHON: ./venv/bin/python PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig run: | - mkdir -p bmq/logs - mkdir -p bmq/storage/archive - ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config & - (sleep 5; make test-build && make test-install && make check) + make test-build && make test-install && make check + - name: Stop BlazingMQ broker + if: always() + run: docker rm -f bmqbrkr lint-docs: name: Lint and Docs @@ -218,6 +237,21 @@ jobs: sudo mkdir /cores sudo chmod 777 /cores echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Start BlazingMQ broker + run: | + docker run -d --name bmqbrkr \ + --network host \ + -v ${{ github.workspace }}/tests/broker-config:/broker-config:ro \ + ghcr.io/bloomberg/blazingmq:latest \ + /usr/local/bin/bmqbrkr /broker-config + timeout 30 bash -c 'until nc -z localhost 30114; do sleep 1; done' \ + || (docker logs bmqbrkr; exit 1) - name: Run tests with coverage env: BMQ_BROKER_URI: tcp://localhost:30114 @@ -225,19 +259,24 @@ jobs: PYTHON: ./venv/bin/python PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig run: | - mkdir -p bmq/logs - mkdir -p bmq/storage/archive # Allow core dumps ulimit -c unlimited - ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config & - (sleep 5; make coverage-install && make coverage) + make coverage-install && make coverage + - name: Stop BlazingMQ broker + if: always() + run: docker rm -f bmqbrkr - name: Output code coverage summary uses: irongut/CodeCoverageSummary@v1.3.0 with: filename: coverage*.xml - name: Upload broker logs as artifacts if: failure() - uses: actions/upload-artifact@v4 + run: | + mkdir -p bmq/logs + docker logs bmqbrkr > bmq/logs/broker_stdout.log 2> bmq/logs/broker_stderr.log || true + docker cp bmqbrkr:/var/local/bmq/logs/. bmq/logs/ || true + - uses: actions/upload-artifact@v4 + if: failure() with: name: broker_logs path: ./bmq/logs @@ -250,10 +289,4 @@ jobs: name: core_dumps path: /cores retention-days: 5 - - name: Upload broker executable as artifacts to debug the core - if: failure() - uses: actions/upload-artifact@v4 - with: - name: bmqbrkr - path: ./blazingmq_artifacts/bin/bmqbrkr.tsk - retention-days: 5 + diff --git a/tests/broker-config/bmqbrkrcfg.json b/tests/broker-config/bmqbrkrcfg.json index 45fea6e..82f4153 100644 --- a/tests/broker-config/bmqbrkrcfg.json +++ b/tests/broker-config/bmqbrkrcfg.json @@ -3,7 +3,7 @@ "allocatorType": "STACKTRACETEST", "allocationLimit": 34359738368, "logController": { - "fileName": "./bmq/logs/logs.%T.%p", + "fileName": "/var/local/bmq/logs/logs.%T.%p", "fileMaxAgeDays": 10, "rotationBytes": 268435456, "logfileFormat": "%d (%t) %s %F:%l %m\n\n", @@ -66,7 +66,7 @@ "snapshotInterval": 1, "printer": { "printInterval": 60, - "file": "./bmq/logs/stat.%T.%p", + "file": "/var/local/bmq/logs/stat.%T.%p", "maxAgeDays": 3, "rotateBytes": 268435456, "rotateDays": 1 diff --git a/tests/broker-config/clusters.json b/tests/broker-config/clusters.json index ec7296d..782ecbe 100644 --- a/tests/broker-config/clusters.json +++ b/tests/broker-config/clusters.json @@ -21,7 +21,7 @@ "partitionConfig": { "name": "local", "flushAtShutdown": true, - "location": "./bmq/storage", + "location": "/var/local/bmq/storage", "maxArchivedFileSets": 0, "maxDataFileSize": 268435456, "maxJournalFileSize": 67108864, @@ -29,7 +29,7 @@ "numPartitions": 1, "preallocate": false, "prefaultPages": false, - "archiveLocation": "./bmq/storage/archive", + "archiveLocation": "/var/local/bmq/storage/archive", "syncConfig": { "fileChunkSize": 0, "masterSyncMaxDurationMs": 0, From a0f7dcbcc8087a541038c1a1eff41662e90f2e37 Mon Sep 17 00:00:00 2001 From: "Patrick M. Niedzielski" Date: Thu, 16 Apr 2026 14:36:41 -0400 Subject: [PATCH 2/2] CI: Only build libbmq, rather than bmqbrkr This patch removes bmqbrkr from our CI build, since we use the bmqbrkr docker image now. We are safe to do this even though these scripts are used when building wheels as well, since wheel builds do not need the broker either. Additionally, this patch removes mwc from the install targets, since it no longer exists. Signed-off-by: Patrick M. Niedzielski --- bin/build-macos-universal.sh | 2 +- bin/build-manylinux.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-macos-universal.sh b/bin/build-macos-universal.sh index 03efb62..298b642 100755 --- a/bin/build-macos-universal.sh +++ b/bin/build-macos-universal.sh @@ -96,7 +96,7 @@ if [ ! -e "${DIR_BUILD}/blazingmq/.complete" ]; then -DBDE_BUILD_TARGET_64=1 \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DINSTALL_TARGETS="bmqbrkr;bmq;mwc" \ + -DINSTALL_TARGETS="bmq" \ -DCMAKE_INSTALL_LIBDIR="lib" \ -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ -DCMAKE_MODULE_PATH="${DIR_ROOT}" \ diff --git a/bin/build-manylinux.sh b/bin/build-manylinux.sh index 41f1e1d..90d841f 100755 --- a/bin/build-manylinux.sh +++ b/bin/build-manylinux.sh @@ -122,7 +122,7 @@ if [ ! -e "${DIR_BUILD}/blazingmq/.complete" ]; then -DBDE_BUILD_TARGET_64=1 \ -DBDE_BUILD_TARGET_CPP17=ON \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DINSTALL_TARGETS="bmqbrkr;bmq;mwc" \ + -DINSTALL_TARGETS="bmq" \ -DCMAKE_INSTALL_LIBDIR="lib64" \ -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" \ -DCMAKE_MODULE_PATH="${DIR_ROOT}" \