feat: enable connection pooling

.github/workflows/auto-update.yaml

@@ -20,5 +20,5 @@ concurrency:
 
 jobs:
   auto-update:
-    uses: defenseunicorns/uds-common/.github/workflows/callable-auto-update.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-auto-update.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     secrets: inherit # Inherits all secrets from the parent workflow.

.github/workflows/commitlint.yaml

@@ -15,4 +15,4 @@ permissions:
 
 jobs:
   validate:
-    uses: defenseunicorns/uds-common/.github/workflows/callable-commitlint.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-commitlint.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10

.github/workflows/lint.yaml

@@ -15,5 +15,5 @@ permissions:
 
 jobs:
   validate:
-    uses: defenseunicorns/uds-common/.github/workflows/callable-lint.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-lint.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     secrets: inherit

.github/workflows/release.yaml

@@ -17,8 +17,8 @@ permissions:
 jobs:
   publish:
     permissions:
-      contents: write # Allows reading the content of the repository.
-      packages: write # Allows reading the content of the repository's packages.
+      contents: write # Allows writing content to the repository.
+      packages: write # Allows writing content to the repository's packages.
       id-token: write
     strategy:
       matrix:
@@ -27,7 +27,7 @@ jobs:
         exclude:
           - flavor: registry1
             architecture: arm64
-    uses: defenseunicorns/uds-common/.github/workflows/callable-publish.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-publish.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     with:
       flavor: ${{ matrix.flavor }}
       options: --set BASE_REPO="ghcr.io/uds-packages"

.github/workflows/scan.yaml

@@ -18,5 +18,5 @@ jobs:
       packages: read # Allows reading the content of the repository's packages.
       id-token: write # Allows authentication to Chainguard via OIDC.
       pull-requests: write # Allows writing the scan results comment to the pull request.
-    uses: defenseunicorns/uds-common/.github/workflows/callable-scan.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-scan.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     secrets: inherit # Inherits all secrets from the parent workflow.

.github/workflows/scorecard.yaml

@@ -32,5 +32,5 @@ jobs:
       security-events: write
       # Used to receive a badge.
       id-token: write
-    uses: defenseunicorns/uds-common/.github/workflows/callable-scorecard.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-scorecard.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     secrets: inherit

.github/workflows/test.yaml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: test-flavor
-        uses: defenseunicorns/uds-common/.github/actions/test-flavor@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+        uses: defenseunicorns/uds-common/.github/actions/test-flavor@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
         id: test-flavor
     outputs:
       upgrade-flavors: ${{ steps.test-flavor.outputs.upgrade-flavors }}
@@ -41,7 +41,7 @@ jobs:
       matrix:
         type: [install, upgrade]
         flavor: [upstream, registry1, unicorn]
-    uses: defenseunicorns/uds-common/.github/workflows/callable-test.yaml@99fd276835257a9608656380d1d453356fe7539e # v1.24.8
+    uses: defenseunicorns/uds-common/.github/workflows/callable-test.yaml@86fcadc2845a318761276a8754e47e33c0d6ae31 # v1.24.10
     with:
       options: --set BASE_REPO="ghcr.io/uds-packages"
       upgrade-flavors: ${{ needs.check-flavor.outputs.upgrade-flavors }}

bundle/uds-bundle.yaml

@@ -24,6 +24,17 @@ packages:
                 volume:
                   size: "10Gi"
                 numberOfInstances: 2
+                enableConnectionPooler: true
+                enableReplicaConnectionPooler: true
+                connectionPooler:
+                  numberOfInstances: 1
+                  resources:
+                    requests:
+                      cpu: 100m
+                      memory: 100Mi
+                    limits:
+                      cpu: 500m
+                      memory: 500Mi
                 users:
                   gitlab.gitlab: []  # database owner
                   sonarqube.sonarqube: []  # database owner

chart/templates/_helpers.tpl

@@ -0,0 +1,24 @@
+{{/*
+Copyright 2024 Defense Unicorns
+SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+*/}}
+
+{{/*
+Network Polcies for Postgres Operator, specifically related to the addition of Connection Pooler.
+*/}}
+{{- define "uds-postgres.ingressRules" -}}
+{{- $selector := .selector -}}
+{{- if kindIs "slice" .ingress -}}
+{{- range .ingress }}
+- direction: Ingress
+  selector:
+    {{- $selector | toYaml | nindent 4 }}
+  {{- . | toYaml | nindent 2 }}
+{{- end }}
+{{- else }}
+- direction: Ingress
+  selector:
+    {{- $selector | toYaml | nindent 4 }}
+  {{- .ingress | toYaml | nindent 2 }}
+{{- end }}
+{{- end -}}

chart/templates/postgres-minimal.yaml

@@ -14,6 +14,13 @@ spec:
   volume:
     size: {{ .Values.postgresql.volume.size | quote }}
   numberOfInstances: {{ .Values.postgresql.numberOfInstances }}
+
+  enableConnectionPooler: {{ .Values.postgresql.enableConnectionPooler | default false }}
+  enableReplicaConnectionPooler: {{ .Values.postgresql.enableReplicaConnectionPooler | default false }}
+
+  connectionPooler:
+    {{- toYaml .Values.postgresql.connectionPooler | nindent 4 }}
+
   users:
     {{- toYaml .Values.postgresql.users | nindent 4 }} # database owner
   databases:

chart/templates/uds-package-postgres.yaml

@@ -25,29 +25,42 @@ spec:
       - direction: Egress
         remoteGenerated: IntraNamespace
 
-    {{- if kindIs "slice" .Values.postgresql.ingress -}}
-      {{- range .Values.postgresql.ingress }}
+      {{- include "uds-postgres.ingressRules" (dict
+          "ingress" .Values.postgresql.ingress
+          "selector" (dict "cluster-name" "pg-cluster")
+      ) | nindent 6 }}
+
       - direction: Ingress
         selector:
           cluster-name: pg-cluster
-        {{ . | toYaml | nindent 8 }}
-      {{- end }}
-    {{- else }}
-      - direction: Ingress
+        remoteNamespace: {{ .Release.Namespace }}
+        remoteSelector:
+          app.kubernetes.io/name: postgres-operator
+
+      - direction: Egress
         selector:
           cluster-name: pg-cluster
-        {{- .Values.postgresql.ingress | toYaml | nindent 8 }}
-    {{- end }}
+        remoteGenerated: KubeAPI
+
+      {{- /* Pooler (pgbouncer) traffic. Zalando labels pooler pods with
+             `application=db-connection-pooler`. IntraNamespace rules above
+             already cover pooler <-> spilo traffic within this namespace. */}}
+      {{- if or (.Values.postgresql.enableConnectionPooler | default false) (.Values.postgresql.enableReplicaConnectionPooler | default false) }}
+      {{- include "uds-postgres.ingressRules" (dict
+          "ingress" .Values.postgresql.ingress
+          "selector" (dict "application" "db-connection-pooler")
+      ) | nindent 6 }}
 
       - direction: Ingress
         selector:
-          cluster-name: pg-cluster
+          application: db-connection-pooler
         remoteNamespace: {{ .Release.Namespace }}
         remoteSelector:
           app.kubernetes.io/name: postgres-operator
 
       - direction: Egress
         selector:
-          cluster-name: pg-cluster
+          application: db-connection-pooler
         remoteGenerated: KubeAPI
+      {{- end }}
 {{- end }}

chart/templates/uds-package.yaml

@@ -18,6 +18,15 @@ spec:
         remoteNamespace: postgres
         remoteSelector:
           cluster-name: pg-cluster
+
+      {{- if or (.Values.postgresql.enableConnectionPooler | default false) (.Values.postgresql.enableReplicaConnectionPooler | default false) }}
+      - direction: Egress
+        selector:
+          app.kubernetes.io/name: postgres-operator
+        remoteNamespace: postgres
+        remoteSelector:
+          application: db-connection-pooler
+      {{- end }}
       {{- end }}
 
       - direction: Egress

chart/values.yaml

@@ -17,6 +17,12 @@ postgresql:
   additionalVolumes: []
   env: []
 
+# Connection pooler options
+# Ref: https://opensource.zalando.com/postgres-operator/docs/user.html#connection-pooler
+enableConnectionPooler: false
+enableReplicaConnectionPooler: false
+connectionPooler: {}
+
 # Example values for postgresql
 #
 # postgresql:

common/zarf.yaml

@@ -41,7 +41,7 @@ components:
             maxTotalSeconds: 300
             cmd: |
               if ./zarf tools kubectl get packages.uds.dev postgres -n postgres; then
-                 ./zarf tools wait-for packages.uds.dev postgres -n postgres '{.status.phase}'=Ready
+                 ./zarf tools wait-for resource packages.uds.dev postgres -n postgres '{.status.phase}'=Ready
               fi
           - description: Postgres Operator to be Healthy
             maxTotalSeconds: 90
@@ -55,5 +55,5 @@ components:
             maxTotalSeconds: 300
             cmd: |
               if ./zarf tools kubectl get postgresql pg-cluster -n postgres; then
-                 ./zarf tools wait-for postgresql pg-cluster -n postgres '{.status.PostgresClusterStatus}'=Running
+                 ./zarf tools wait-for resource postgresql pg-cluster -n postgres '{.status.PostgresClusterStatus}'=Running
               fi

docs/configuration.md

@@ -34,6 +34,60 @@ Postgres Operator is configured through [`acid.zalan.do/v1` `Postgresql` custom
       value: <value>
 ```
 
+## Connection Pooling
+
+Postgres Operator can deploy [pgbouncer](https://www.pgbouncer.org/) alongside the cluster to pool client connections, reducing backend churn for high-connection-count workloads. Poolers are deployed as separate `Deployment`s (`<cluster>-pooler` and/or `<cluster>-pooler-repl`) and exposed via matching `Service`s on port `5432`. Required network policies for the pooler pods are generated automatically when either flag below is enabled.
+
+- `postgresql.enableConnectionPooler`: deploy a pgbouncer pooler in front of the primary (RW traffic)
+- `postgresql.enableReplicaConnectionPooler`: deploy a pgbouncer pooler in front of the replicas (RO traffic)
+- `postgresql.connectionPooler`: optional map of pooler settings passed through to the `Postgresql` CR (e.g. `numberOfInstances`, `mode`, `resources`)
+
+Example:
+
+```yaml
+postgresql:
+  enableConnectionPooler: true
+  enableReplicaConnectionPooler: true
+  connectionPooler:
+    numberOfInstances: 2
+    mode: transaction
+```
+
+Clients connect through the pooler by pointing at `<cluster>-pooler.<namespace>.svc.cluster.local` (primary) or `<cluster>-pooler-repl.<namespace>.svc.cluster.local` (replicas) instead of the cluster Service.
+
+### Registry1 pooler shim
+
+The `registry1` flavor consumes the Iron Bank pgbouncer image, which diverges from upstream Zalando in two ways the operator does not expose via configuration:
+
+1. The `pgbouncer` user is built as UID `997`, but the Zalando operator hardcodes the pooler pod to `runAsUser: 100`. As a result the entrypoint cannot write the self-signed TLS cert to `/etc/pgbouncer/`, the log to `/var/log/pgbouncer/`, or the pidfile to `/var/run/pgbouncer/`.
+2. The baked-in `pgbouncer.ini.tmpl` hardcodes `auth_type = plain`, which fails against postgres's default `scram-sha-256` encryption (pgbouncer cannot replay SCRAM secrets returned by `auth_query` when operating in plain mode).
+
+To keep the pooler usable without rebuilding the image or adding a Pepr mutation, the `registry1` component in `zarf.yaml` runs [`tasks/registry1-pooler-patch.yaml`](../tasks/registry1-pooler-patch.yaml) as an `onDeploy.after` action. For each operator-managed pooler Deployment it strategic-merge patches in:
+
+- three in-memory `emptyDir` volumes mounted at `/etc/pgbouncer`, `/var/log/pgbouncer`, and `/var/run/pgbouncer` (writable under the pod's `fsGroup: 103`)
+- a `seed-pgbouncer-etc` init container that copies Zalando's `.tmpl` files into the emptyDir and rewrites `auth_type = plain` to `auth_type = scram-sha-256` before the main container renders them via `envsubst`
+
+The operator's pooler sync does not compare `volumes`, `volumeMounts`, or `initContainers`, so the patch survives its reconcile loop. The `upstream` flavor ships the Zalando-curated pgbouncer image and does not need the shim.
+
+### Unicorn pooler shim
+
+The `unicorn` flavor consumes `cgr.dev/defenseunicorns.com/pgbouncer`, which is a generic upstream Chainguard pgbouncer image rather than a Zalando rebuild. Two consequences the operator does not expose via configuration:
+
+1. The image's `ENTRYPOINT` is `/usr/bin/pgbouncer` with `CMD` `["--help"]` and ships no `/etc/pgbouncer/*.tmpl` files or entrypoint script. The Zalando operator leaves the pooler container's `command`/`args` unset and depends on the image entrypoint to render config from `PGHOST`/`PGPORT`/`PGUSER`/`CONNECTION_POOLER_*` env vars and exec pgbouncer; with this image the pod just runs `pgbouncer --help` and exits.
+2. The image is fully minimal (only the `pgbouncer` binary — no `sh`, `openssl`, or `envsubst`), so the rendering cannot be done inside the main container.
+
+To keep the pooler usable without rebuilding the image or adding a Pepr mutation, the `unicorn` component in `zarf.yaml` runs [`tasks/unicorn-pooler-patch.yaml`](../tasks/unicorn-pooler-patch.yaml) as an `onDeploy.after` action. For each operator-managed pooler Deployment it strategic-merge patches in:
+
+- three in-memory `emptyDir` volumes mounted at `/etc/pgbouncer`, `/var/log/pgbouncer`, and `/var/run/pgbouncer` (writable under the pod's `fsGroup: 103`)
+- a `seed-pgbouncer-etc` init container that reuses the unicorn flavor's spilo image (already pulled for the postgres pods, has `sh` + `openssl`) to render `pgbouncer.ini`, `auth_file.txt`, and a self-signed TLS cert into the emptyDir — replicating upstream Zalando's `entrypoint.sh`, with `auth_type = scram-sha-256` (PG17 default) and without the Zalando downstream-fork-only `stats_users_prefix` directive that vanilla pgbouncer rejects
+- the operator-set env block replicated onto the init container (extracted live via `kubectl jsonpath`) so the rendered config matches the configured `pool_mode`/sizes/ports; `PGUSER` and `PGPASSWORD` are preserved as `secretKeyRef`s so the password never lands in plaintext in the Deployment spec
+- a `command` override on the main container to `["/usr/bin/pgbouncer", "/etc/pgbouncer/pgbouncer.ini"]`
+
+References:
+
+- [Zalando — Connection pooler](https://opensource.zalando.com/postgres-operator/docs/user.html#connection-pooler)
+- [OneUptime — PostgreSQL with the Zalando operator](https://oneuptime.com/blog/post/2026-01-21-postgresql-zalando-operator/view)
+
 ## Postgres HugePages
 
 Postgres Operator can also support HugePages by setting the following keys appropriately for your environment.  You can learn more about HugePages in Kubernetes in their [Manage HugePages documentation](https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/#api) and learn more about these fields in the [`Postgresql` custom resource reference documentation](https://github.com/zalando/postgres-operator/blob/master/docs/reference/cluster_manifest.md#cluster-manifest-reference).

docs/pgbouncer-entrypoint.md

@@ -0,0 +1,104 @@
+# pgbouncer entrypoint script
+
+The `entrypoint.sh` script that upstream Zalando pgbouncer image uses is as follows:
+
+```bash
+#!/bin/sh
+
+set -ex
+
+if [ "$PGUSER" = "postgres" ]; then
+    echo "WARNING: pgbouncer will connect with a superuser privileges!"
+    echo "You need to fix this as soon as possible."
+fi
+
+if [ -z "${CONNECTION_POOLER_CLIENT_TLS_CRT}" ]; then
+    openssl req -nodes -new -x509 -subj /CN=spilo.dummy.org \
+        -keyout /etc/ssl/certs/pgbouncer.key \
+        -out /etc/ssl/certs/pgbouncer.crt
+else
+    ln -s ${CONNECTION_POOLER_CLIENT_TLS_CRT} /etc/ssl/certs/pgbouncer.crt
+    ln -s ${CONNECTION_POOLER_CLIENT_TLS_KEY} /etc/ssl/certs/pgbouncer.key
+    if [ ! -z "${CONNECTION_POOLER_CLIENT_CA_FILE}" ]; then
+        ln -s ${CONNECTION_POOLER_CLIENT_CA_FILE} /etc/ssl/certs/ca.crt
+    fi
+fi
+
+envsubst < /etc/pgbouncer/pgbouncer.ini.tmpl > /etc/pgbouncer/pgbouncer.ini
+envsubst < /etc/pgbouncer/auth_file.txt.tmpl > /etc/pgbouncer/auth_file.txt
+
+exec /bin/pgbouncer /etc/pgbouncer/pgbouncer.ini
+```
+
+The Chainguard image for pgbouncer only contains the binary, so we need to explictly define the `pgbouncer.ini` and `auth_file.txt`. First, let's define what the template file (`pgbouncer.ini.tmpl`) looks like:
+
+```bash
+# vim: set ft=dosini:
+
+[databases]
+* = host=$PGHOST port=$PGPORT auth_user=$PGUSER
+postgres = host=$PGHOST port=$PGPORT auth_user=$PGUSER
+
+[pgbouncer]
+pool_mode = $CONNECTION_POOLER_MODE
+listen_port = $CONNECTION_POOLER_PORT
+listen_addr = *
+auth_type = md5
+auth_file = /etc/pgbouncer/auth_file.txt
+auth_dbname = postgres
+admin_users = $PGUSER
+stats_users_prefix = robot_
+auth_query = SELECT * FROM $PGSCHEMA.user_lookup($1)
+logfile = /var/log/pgbouncer/pgbouncer.log
+pidfile = /var/run/pgbouncer/pgbouncer.pid
+
+server_tls_sslmode = require
+server_tls_ca_file = /etc/ssl/certs/pgbouncer.crt
+server_tls_protocols = secure
+client_tls_sslmode = require
+client_tls_key_file = /etc/ssl/certs/pgbouncer.key
+client_tls_cert_file = /etc/ssl/certs/pgbouncer.crt
+
+log_connections = 0
+log_disconnections = 0
+
+# How many server connections to allow per user/database pair.
+default_pool_size = $CONNECTION_POOLER_DEFAULT_SIZE
+
+# Add more server connections to pool if below this number. Improves behavior
+# when usual load comes suddenly back after period of total inactivity.
+#
+# NOTE: This value is per pool, i.e. a pair of (db, user), not a global one.
+# Which means on the higher level it has to be calculated from the max allowed
+# database connections and number of databases and users. If not taken into
+# account, then for too many users or databases PgBouncer will go crazy
+# opening/evicting connections. For now disable it.
+#
+# min_pool_size = $CONNECTION_POOLER_MIN_SIZE
+
+# How many additional connections to allow to a pool
+reserve_pool_size = $CONNECTION_POOLER_RESERVE_SIZE
+
+# Maximum number of client connections allowed.
+max_client_conn = $CONNECTION_POOLER_MAX_CLIENT_CONN
+
+# Do not allow more than this many connections per database (regardless of
+# pool, i.e. user)
+max_db_connections = $CONNECTION_POOLER_MAX_DB_CONN
+
+# If a client has been in "idle in transaction" state longer, it will be
+# disconnected. [seconds]
+idle_transaction_timeout = 600
+
+# If login failed, because of failure from connect() or authentication that
+# pooler waits this much before retrying to connect. Default is 15. [seconds]
+server_login_retry = 5
+
+# To ignore extra parameter in startup packet. By default only 'database' and
+# 'user' are allowed, all others raise error.  This is needed to tolerate
+# overenthusiastic JDBC wanting to unconditionally set 'extra_float_digits=2'
+# in startup packet.
+ignore_startup_parameters = extra_float_digits,options
+```
+
+According to this we have created a task to explictly create this file in [unicorn-pooler-patch.yaml](../tasks/unicorn-pooler-patch.yaml).