ci(pipelines): route server base images through internal mirror

server/gitrest/Dockerfile

@@ -2,10 +2,15 @@
 # Licensed under the MIT License.
 # DisableDockerDetector "No feasible secure solution for OSS repos yet"
 
-# Use the manifest digest's sha256 hash to ensure we are always using the same version of the base image.
-# That version of the base image must also be baked into the CI build agent by a repo maintainer.
-# This avoids throttling issues with Docker Hub by using an image baked into the pipeline build image.
-FROM node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
+# The node base image is pulled from public Docker Hub by default so local builds and external
+# contributors get the canonical upstream image. Our CI pipeline overrides BASE_IMAGE_REGISTRY (via
+# additionalBuildArguments in tools/pipelines/server-gitrest.yml) to point at our internal mirror,
+# because 1ES network isolation policies (CfsClean/CfsClean2) block egress to registry-1.docker.io
+# from our build pool. The mirror is byte-for-byte identical to docker.io/library, so the same
+# manifest digest pin resolves correctly against either registry. See tools/pipelines/README.md for
+# how to maintain the mirror (upgrades, additions).
+ARG BASE_IMAGE_REGISTRY=docker.io
+FROM ${BASE_IMAGE_REGISTRY}/library/node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
 
 ARG SETVERSION_VERSION=dev
 ENV SETVERSION_VERSION=$SETVERSION_VERSION

server/gitssh/Dockerfile

@@ -5,12 +5,11 @@
 
 # The alpine base image is pulled from public Docker Hub by default so local builds and external
 # contributors get the canonical upstream image. Our CI pipeline overrides BASE_IMAGE_REGISTRY (via
-# additionalBuildArguments in tools/pipelines/server-gitssh.yml) to point at Microsoft Container
-# Registry's Docker Hub mirror, because 1ES network isolation policies (CfsClean/CfsClean2) block
-# egress to registry-1.docker.io from our build pool. The mirror is byte-for-byte identical to
-# docker.io/library, so the same manifest digest pin resolves correctly against either registry.
-# MCR's mirror is frozen at alpine 3.16; mirroring a supported alpine into our internal ACR is a
-# follow-up tracked by AB#73353.
+# additionalBuildArguments in tools/pipelines/server-gitssh.yml) to point at our internal mirror,
+# because 1ES network isolation policies (CfsClean/CfsClean2) block egress to registry-1.docker.io
+# from our build pool. The mirror is byte-for-byte identical to docker.io/library, so the same
+# manifest digest pin resolves correctly against either registry. See tools/pipelines/README.md for
+# how to maintain the mirror (upgrades, additions).
 ARG BASE_IMAGE_REGISTRY=docker.io
 FROM ${BASE_IMAGE_REGISTRY}/library/alpine:3.16@sha256:452e7292acee0ee16c332324d7de05fa2c99f9994ecc9f0779c602916a672ae4
 

server/historian/Dockerfile

@@ -3,10 +3,15 @@
 
 # DisableDockerDetector "No feasible secure solution for OSS repos yet"
 
-# Use the manifest digest's sha256 hash to ensure we are always using the same version of the base image.
-# That version of the base image must also be baked into the CI build agent by a repo maintainer.
-# This avoids throttling issues with Docker Hub by using an image baked into the pipeline build image.
-FROM node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
+# The node base image is pulled from public Docker Hub by default so local builds and external
+# contributors get the canonical upstream image. Our CI pipeline overrides BASE_IMAGE_REGISTRY (via
+# additionalBuildArguments in tools/pipelines/server-historian.yml) to point at our internal mirror,
+# because 1ES network isolation policies (CfsClean/CfsClean2) block egress to registry-1.docker.io
+# from our build pool. The mirror is byte-for-byte identical to docker.io/library, so the same
+# manifest digest pin resolves correctly against either registry. See tools/pipelines/README.md for
+# how to maintain the mirror (upgrades, additions).
+ARG BASE_IMAGE_REGISTRY=docker.io
+FROM ${BASE_IMAGE_REGISTRY}/library/node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
 
 ARG SETVERSION_VERSION=dev
 ENV SETVERSION_VERSION=$SETVERSION_VERSION

server/routerlicious/Dockerfile

@@ -3,10 +3,15 @@
 
 # DisableDockerDetector "No feasible secure solution for OSS repos yet"
 
-# Use the manifest digest's sha256 hash to ensure we are always using the same version of the base image.
-# That version of the base image must also be baked into the CI build agent by a repo maintainer.
-# This avoids throttling issues with Docker Hub by using an image baked into the pipeline build image.
-FROM node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
+# The node base image is pulled from public Docker Hub by default so local builds and external
+# contributors get the canonical upstream image. Our CI pipeline overrides BASE_IMAGE_REGISTRY (via
+# additionalBuildArguments in tools/pipelines/server-routerlicious.yml) to point at our internal
+# mirror, because 1ES network isolation policies (CfsClean/CfsClean2) block egress to
+# registry-1.docker.io from our build pool. The mirror is byte-for-byte identical to
+# docker.io/library, so the same manifest digest pin resolves correctly against either registry.
+# See tools/pipelines/README.md for how to maintain the mirror (upgrades, additions).
+ARG BASE_IMAGE_REGISTRY=docker.io
+FROM ${BASE_IMAGE_REGISTRY}/library/node:22.22.2-bookworm-slim@sha256:f3a68cf41a855d227d1b0ab832bed9749469ef38cf4f58182fb8c893bc462383 AS runnerbase
 
 ARG SETVERSION_VERSION=dev
 ENV SETVERSION_VERSION=$SETVERSION_VERSION

tools/pipelines/README.md

@@ -0,0 +1,62 @@
+# Fluid Framework CI pipelines
+
+Azure Pipelines definitions and shared [`templates/`](./templates) for building, testing, and
+releasing the Fluid Framework.
+
+## Mirroring base container images for the server pipelines
+
+The `server-*` pipelines run on a 1ES build pool whose network isolation blocks egress to Docker
+Hub, so each server Dockerfile makes its base-image registry overridable and the matching pipeline
+points it at a mirrored copy on the internal container registry. Local builds default to Docker
+Hub and need no changes.
+
+```dockerfile
+ARG BASE_IMAGE_REGISTRY=docker.io
+FROM ${BASE_IMAGE_REGISTRY}/library/node:22.22.2-bookworm-slim@sha256:f3a68cf4...
+```
+
+```yaml
+additionalBuildArguments: >-
+  --build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker
+```
+
+`$(containerRegistryUrl)` comes from the `container-registry-info` variable group, loaded at the
+pipeline root. The mirror namespace `mirror/docker/library/<image>` is byte-identical to Docker
+Hub's path, so the same Dockerfile reference works against either registry.
+
+### Upgrading a pinned base image
+
+1. Resolve the new tag's Docker Hub manifest digest:
+   ```bash
+   docker buildx imagetools inspect docker.io/library/node:<new tag> \
+     --format '{{json .Manifest.Digest}}'
+   ```
+
+2. Import it into the mirror. The registry name is the value of `containerRegistryUrl` in the
+   variable group (drop the `.azurecr.io` suffix); the command requires permission to perform
+   `Microsoft.ContainerRegistry/registries/importImage/action` on the registry (held by the
+   `Contributor` role, but **not** by `AcrPush`):
+   ```bash
+   az acr import --name "$ACR_NAME" \
+     --source "docker.io/library/node@<new digest>" \
+     --image  "mirror/docker/library/node:<new tag>"
+   ```
+   (ACR's `--source` accepts a tag *or* a digest reference, but not the combined `tag@digest`
+   form — pass the digest as the source and let `--image` provide the tag on the destination.)
+
+3. Update the Dockerfile pin(s) in the same PR. The pipeline YAML doesn't change.
+
+After a green run, the previous mirrored tag can be removed with `az acr repository delete` — the
+registry has no retention policy, so old tags persist until manually deleted.
+
+### Adding a newly-mirrored base image
+
+Import as above, then in the new Dockerfile declare the build arg and prefix the upstream `FROM`:
+
+```dockerfile
+ARG BASE_IMAGE_REGISTRY=docker.io
+FROM ${BASE_IMAGE_REGISTRY}/library/<image>:<tag>@<digest>
+```
+
+In the pipeline YAML, append `--build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker`
+to `additionalBuildArguments`, using YAML's `>-` block scalar to keep multiple args readable.

tools/pipelines/server-gitrest.yml

@@ -104,7 +104,14 @@ extends:
     buildToolsVersionToInstall: ${{ parameters.buildToolsVersionToInstall }}
     dockerBuildBumpsVersion: true
     packageManagerInstallCommand: 'pnpm config set recursive-install false ; pnpm i ; pnpm config set recursive-install true'
-    additionalBuildArguments: --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+    # The first build-context is needed because the Dockerfile copies files from the repo root.
+    # The second build-arg overrides the default docker.io registry in the Dockerfile to pull the base
+    # image from our internal mirror, since 1ES network isolation policies (CfsClean/CfsClean2) block
+    # egress to registry-1.docker.io from our build pool. containerRegistryUrl comes from the
+    # container-registry-info variable group. See tools/pipelines/README.md.
+    additionalBuildArguments: >-
+      --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+      --build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker
     packageManager: pnpm
     buildDirectory: ${{ variables.FluidFrameworkDirectory }}/server/gitrest
     containerName: fluidframework/routerlicious/gitrest

tools/pipelines/server-gitssh.yml

@@ -88,7 +88,8 @@ extends:
     setVersion: false
     enableDockerImagePull: false
     tagName: gitssh
-    # Override the default Docker Hub registry in the Dockerfile to pull the base image from MCR's
-    # Docker Hub mirror instead, since 1ES network isolation policies (CfsClean/CfsClean2) block
-    # egress to registry-1.docker.io from our build pool. See AB#73353 for the broader migration.
-    additionalBuildArguments: --build-arg BASE_IMAGE_REGISTRY=mcr.microsoft.com/mirror/docker
+    # Override the default docker.io registry in the Dockerfile to pull the base image from our
+    # internal mirror, since 1ES network isolation policies (CfsClean/CfsClean2) block egress to
+    # registry-1.docker.io from our build pool. containerRegistryUrl comes from the
+    # container-registry-info variable group. See tools/pipelines/README.md.
+    additionalBuildArguments: --build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker

tools/pipelines/server-historian.yml

@@ -107,7 +107,14 @@ extends:
     # We need to install only the root dependencies; historian has native deps that don't install in CI since we use
     # Docker to do the actual build in CI. We need the root dependencies so setting package versions works.
     packageManagerInstallCommand: 'pnpm install --workspace-root'
-    additionalBuildArguments: --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+    # The first build-context is needed because the Dockerfile copies files from the repo root.
+    # The second build-arg overrides the default docker.io registry in the Dockerfile to pull the base
+    # image from our internal mirror, since 1ES network isolation policies (CfsClean/CfsClean2) block
+    # egress to registry-1.docker.io from our build pool. containerRegistryUrl comes from the
+    # container-registry-info variable group. See tools/pipelines/README.md.
+    additionalBuildArguments: >-
+      --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+      --build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker
     packageManager: pnpm
     test: test
     tagName: historian

tools/pipelines/server-routerlicious.yml

@@ -143,5 +143,12 @@ extends:
     - prettier
     - check:versions
     - generate:packageList
-    additionalBuildArguments: --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+    # The first build-context is needed because the Dockerfile copies files from the repo root.
+    # The second build-arg overrides the default docker.io registry in the Dockerfile to pull the base
+    # image from our internal mirror, since 1ES network isolation policies (CfsClean/CfsClean2) block
+    # egress to registry-1.docker.io from our build pool. containerRegistryUrl comes from the
+    # container-registry-info variable group. See tools/pipelines/README.md.
+    additionalBuildArguments: >-
+      --build-context root=$(Pipeline.Workspace)/${{ variables.FluidFrameworkDirectory }}
+      --build-arg BASE_IMAGE_REGISTRY=$(containerRegistryUrl)/mirror/docker
     enableDockerImagePull: false

tools/pipelines/templates/include-test-real-service.yml

@@ -263,7 +263,7 @@ stages:
           - task: Docker@2
             displayName: 'Login to container registry'
             inputs:
-              containerRegistry: 'Fluid Container Registry'
+              containerRegistry: $(containerRegistryConnection)
               command: 'login'
           - task: DockerCompose@1
             displayName: 'Start routerlicious environment in docker'
@@ -274,7 +274,7 @@ stages:
               dockerComposeCommand: 'up -d --wait'
               projectName: 'routerlicious'
             env:
-              REGISTRY_URL: fluidcr.azurecr.io/build
+              REGISTRY_URL: $(containerRegistryUrl)/build
               ALFRED_IMAGE_TAG: ${{ parameters.alfredImageTag }}
               HISTORIAN_IMAGE_TAG: ${{ parameters.historianImageTag }}
               GITREST_IMAGE_TAG: ${{ parameters.gitrestImageTag }}

tools/pipelines/test-real-service.yml

@@ -38,6 +38,11 @@ resources:
 variables:
 - template: /tools/pipelines/templates/include-vars-telemetry-generator.yml@self
 - group: prague-key-vault
+# Note: we tried to centralize the use of the 'container-registry-info' variable group (i.e. put it
+# inside templates we depend on) but ran into issues because if it exists in stage-level variables
+# instead of root-level pipeline variables, the value for the ACR service connection name passed
+# to Docker tasks isn't available before ADO performs checks for that task.
+- group: container-registry-info
 - name: testPackage
   value: "@fluid-private/test-end-to-end-tests"
   readonly: true
@@ -101,8 +106,6 @@ stages:
       continueOnError: true
       timeoutInMinutes: 360
       testCommand: test:realsvc:r11s:docker
-      stageVariables:
-        - group: container-registry-info
       splitTestVariants:
         - name: Non-compat
           flags: --compatVersion=0