diff --git a/releaser.yaml b/releaser.yaml index c2e8df0..b22d4a9 100644 --- a/releaser.yaml +++ b/releaser.yaml @@ -6,5 +6,5 @@ flavors: # renovate-uds: datasource=docker depName=bitnamilegacy/valkey extractVersion=^(?\d+\.\d+\.\d+)(-debian-.*)?$ version: 8.1.3-uds.4 - name: unicorn - # renovate-uds: datasource=docker depName=quay.io/rfcurated/valkey/valkey extractVersion=^(?\d+\.\d+\.\d+)(?:-.*)?$ - version: 8.1.6-uds.2 + # renovate-uds: datasource=docker depName=cgr.dev/defenseunicorns.com/valkey-iamguarded extractVersion=^(?\d+\.\d+\.\d+)?$ + version: 8.1.7-uds.0 diff --git a/tests/valkey/test-job.yaml b/tests/valkey/test-job.yaml index 8a0ff04..66795ba 100644 --- a/tests/valkey/test-job.yaml +++ b/tests/valkey/test-job.yaml @@ -48,7 +48,7 @@ spec: echo "All checks passed." restartPolicy: OnFailure - backoffLimit: 5 + backoffLimit: 15 --- # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial @@ -81,14 +81,51 @@ spec: command: ["/bin/sh", "-c"] args: - | - # Ask the Sentinel which node is the primary node. 'mymaster' is the default name of the primary node. - PRIMARY_ADDR="$(echo 'SENTINEL GET-PRIMARY-ADDR-BY-NAME mymaster' | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local -p 26379)" - echo "Primary ADDR is: ${PRIMARY_ADDR}" + SENTINEL_HOST="valkey.valkey-replicated-w-sentinel.svc.cluster.local" + SENTINEL_PORT="26379" - # Extract HOST and PORT using sed - HOST=$(echo "${PRIMARY_ADDR}" | sed -n '1p') - PORT=$(echo "${PRIMARY_ADDR}" | sed -n '2p') - echo "Primary is ${HOST}:${PORT}" + # Wait for Sentinel to have quorum before trusting any topology answer it gives. + # On fresh deploys or just after a failover, Sentinels need a few seconds to gossip + # and agree on the primary. Without this, GET-PRIMARY-ADDR-BY-NAME can return a + # node that is actually a replica, and writes will fail with READONLY. + echo "Waiting for Sentinel quorum on mymaster..." + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if echo "SENTINEL CKQUORUM mymaster" | valkey-cli -h ${SENTINEL_HOST} -p ${SENTINEL_PORT} | grep -q OK; then + echo "Sentinel quorum reached." + break + fi + echo "Attempt ${attempt}: Sentinel quorum not ready, retrying..." + sleep 3 + done + + # Ask Sentinel which node is the primary, then verify the node itself agrees. + # Retry the lookup if the node reports it is actually a replica (stale Sentinel view). + HOST="" + PORT="" + for attempt in 1 2 3 4 5; do + PRIMARY_ADDR="$(echo 'SENTINEL GET-PRIMARY-ADDR-BY-NAME mymaster' | valkey-cli -h ${SENTINEL_HOST} -p ${SENTINEL_PORT})" + echo "Primary ADDR is: ${PRIMARY_ADDR}" + + HOST=$(echo "${PRIMARY_ADDR}" | sed -n '1p') + PORT=$(echo "${PRIMARY_ADDR}" | sed -n '2p') + echo "Primary is ${HOST}:${PORT}" + + # ROLE returns 'master' or 'slave'. + ROLE=$(echo "ROLE" | valkey-cli -h ${HOST} -p ${PORT} | head -n 1) + if [ "${ROLE}" = "master" ]; then + echo "Confirmed ${HOST}:${PORT} is primary." + break + fi + + echo "Attempt ${attempt}: ${HOST}:${PORT} reports role=${ROLE}" + HOST="" + sleep 3 + done + + if [ -z "${HOST}" ] || [ "${ROLE}" != "master" ]; then + echo "Could not find a writable primary after retries. Last role seen: ${ROLE}" + exit 1 + fi # Check if primary responds to PING PING_OUTPUT=$(echo "ping" | valkey-cli -h ${HOST} -p ${PORT}) @@ -110,4 +147,4 @@ spec: echo "All checks passed." restartPolicy: OnFailure - backoffLimit: 5 + backoffLimit: 15 diff --git a/tests/zarf.yaml b/tests/zarf.yaml index 1595a64..e7386bb 100644 --- a/tests/zarf.yaml +++ b/tests/zarf.yaml @@ -27,4 +27,4 @@ components: after: - description: Watch test jobs and report their conditions cmd: ./tests/watch-jobs.sh - maxTotalSeconds: 120 + maxTotalSeconds: 300 diff --git a/values/unicorn-values.yaml b/values/unicorn-values.yaml index 5ca9c20..0383abf 100644 --- a/values/unicorn-values.yaml +++ b/values/unicorn-values.yaml @@ -2,18 +2,18 @@ # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial image: - registry: quay.io - repository: rfcurated/valkey/valkey - tag: 8.1.6-jammy-fips-rfcurated + registry: cgr.dev + repository: defenseunicorns.com/valkey-iamguarded + tag: 8.1.7 sentinel: image: # Unicorn flavor option DNE - registry: quay.io - repository: rfcurated/valkey-sentinel - tag: 8.1.6-jammy-bnt-fips-rfcurated + registry: cgr.dev + repository: defenseunicorns.com/valkey-sentinel-iamguarded + tag: 8.1.7 metrics: image: - registry: quay.io - repository: rfcurated/redis-exporter - tag: 1.82.0-jammy-scratch-bnt-fips-rfcurated + registry: cgr.dev + repository: defenseunicorns.com/prometheus-redis-exporter-iamguarded + tag: 1.83.0 diff --git a/values/values.yaml b/values/values.yaml index a7304f6..8331232 100644 --- a/values/values.yaml +++ b/values/values.yaml @@ -12,3 +12,6 @@ metrics: sentinel: primarySet: mymaster + automateClusterRecovery: true + downAfterMilliseconds: 2000 + failoverTimeout: 18000 diff --git a/zarf.yaml b/zarf.yaml index 1d4e7b9..23b7a01 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -41,9 +41,9 @@ components: valuesFiles: - ./values/unicorn-values.yaml images: - - quay.io/rfcurated/valkey/valkey:8.1.6-jammy-fips-rfcurated - - quay.io/rfcurated/redis-exporter:1.82.0-jammy-scratch-bnt-fips-rfcurated - - quay.io/rfcurated/valkey-sentinel:8.1.6-jammy-bnt-fips-rfcurated + - cgr.dev/defenseunicorns.com/valkey-iamguarded:8.1.7 + - cgr.dev/defenseunicorns.com/prometheus-redis-exporter-iamguarded:1.83.0 + - cgr.dev/defenseunicorns.com/valkey-sentinel-iamguarded:8.1.7 - name: valkey-redis-uri-output required: false