Publish Docker image #171
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow uses actions that are not certified by GitHub. | |
| # They are provided by a third-party and are governed by | |
| # separate terms of service, privacy policy, and support | |
| # documentation. | |
| # GitHub recommends pinning actions to a commit SHA. | |
| # To get a newer version, you will need to update the SHA. | |
| # You can also reference a tag or branch, but the action may change without warning. | |
| name: Publish Docker image | |
| on: | |
| workflow_dispatch: # allows manual triggering | |
| schedule: | |
| # Rebuild daily rather than on every push because it is expensive | |
| - cron: '12 4 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| # Fine-grant permission | |
| # https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token | |
| permissions: | |
| packages: write | |
| jobs: | |
| create_tag: | |
| name: Create and push git tag | |
| runs-on: ubuntu-slim | |
| permissions: | |
| contents: write | |
| outputs: | |
| source_tag: ${{ steps.srctag.outputs.name }} | |
| steps: | |
| - name: Clone | |
| id: checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine source tag name | |
| id: srctag | |
| uses: ./.github/actions/get-tag-name | |
| env: | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| - name: Create and push git tag | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git tag ${{ steps.srctag.outputs.name }} || exit 0 | |
| git push origin ${{ steps.srctag.outputs.name }} || exit 0 | |
| prepare_matrices: | |
| name: Prepare Docker matrices | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| build_matrix: ${{ steps.matrices.outputs.build_matrix }} | |
| merge_matrix: ${{ steps.matrices.outputs.merge_matrix }} | |
| steps: | |
| - name: Generate build and merge matrices | |
| id: matrices | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Keep all build targets in one place and derive merge targets from it. | |
| cat > build-matrix.json <<'JSON' | |
| [ | |
| { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, | |
| { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x" }, | |
| { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, | |
| { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.1.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.1.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" }, | |
| { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-arm" }, | |
| { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" }, | |
| { "tag": "openvino", "dockerfile": ".devops/openvino.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" } | |
| ] | |
| JSON | |
| BUILD_MATRIX="$(jq -c . build-matrix.json)" | |
| MERGE_MATRIX="$(jq -c ' | |
| reduce .[] as $entry ({}; .[$entry.tag] |= ( | |
| . // { | |
| tag: $entry.tag, | |
| arches: [], | |
| full: false, | |
| light: false, | |
| server: false | |
| } | |
| | .full = (.full or ($entry.full // false)) | |
| | .light = (.light or ($entry.light // false)) | |
| | .server = (.server or ($entry.server // false)) | |
| | .arches += [($entry.platforms | sub("^linux/"; ""))] | |
| )) | |
| # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. | |
| | if (has("cpu") and (((.cpu.arches // []) | index("s390x")) != null)) then | |
| . + { | |
| s390x: { | |
| tag: "s390x", | |
| arches: ["s390x"], | |
| full: .cpu.full, | |
| light: .cpu.light, | |
| server: .cpu.server | |
| } | |
| } | |
| else | |
| . | |
| end | |
| | [.[] | .arches = (.arches | unique | sort | join(" "))] | |
| ' build-matrix.json)" | |
| echo "build_matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" | |
| echo "merge_matrix=$MERGE_MATRIX" >> "$GITHUB_OUTPUT" | |
| push_to_registry: | |
| name: Push Docker image to Docker Registry | |
| needs: [prepare_matrices, create_tag] | |
| runs-on: ${{ matrix.config.runs_on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJSON(needs.prepare_matrices.outputs.build_matrix) }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ needs.create_tag.outputs.source_tag }} | |
| - name: Set up QEMU | |
| if: ${{ contains(matrix.config.platforms, 'linux/amd64') }} | |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 | |
| with: | |
| image: tonistiigi/binfmt:qemu-v10.2.1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Log in to Docker Registry | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Determine image metadata | |
| id: meta | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case | |
| REPO_NAME="${{ github.event.repository.name }}" | |
| IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" | |
| PREFIX="${IMAGE_REPO}:" | |
| PLATFORM="${{ matrix.config.platforms }}" | |
| ARCH_SUFFIX="${PLATFORM#linux/}" | |
| # list all tags possible | |
| tags="${{ matrix.config.tag }}" | |
| for tag in $tags; do | |
| if [[ "$tag" == "cpu" ]]; then | |
| TYPE="" | |
| else | |
| TYPE="-$tag" | |
| fi | |
| CACHETAG="${PREFIX}buildcache${TYPE}-${ARCH_SUFFIX}" | |
| done | |
| SAFE_TAGS="$(echo "$tags" | tr ' ' '_')" | |
| echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT | |
| echo "arch_suffix=$ARCH_SUFFIX" >> $GITHUB_OUTPUT | |
| echo "cache_output_tag=$CACHETAG" >> $GITHUB_OUTPUT | |
| echo "digest_artifact_suffix=${SAFE_TAGS}-${ARCH_SUFFIX}" >> $GITHUB_OUTPUT | |
| echo "cache_output_tag=$CACHETAG" # print out for debugging | |
| env: | |
| GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}' | |
| - name: Free Disk Space (Ubuntu) | |
| if: ${{ matrix.config.free_disk_space == true }} | |
| uses: ggml-org/free-disk-space@v1.3.1 | |
| with: | |
| # this might remove tools that are actually needed, | |
| # if set to "true" but frees about 6 GB | |
| tool-cache: false | |
| # all of these default to true, but feel free to set to | |
| # "false" if necessary for your workflow | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Build and push Full Docker image by digest | |
| id: build_full | |
| if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.full == true }} | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.config.platforms }} | |
| outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true | |
| file: ${{ matrix.config.dockerfile }} | |
| target: full | |
| provenance: false | |
| build-args: | | |
| ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} | |
| ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} | |
| # using github experimental cache | |
| #cache-from: type=gha | |
| #cache-to: type=gha,mode=max | |
| # return to this if the experimental github cache is having issues | |
| #cache-to: type=local,dest=/tmp/.buildx-cache | |
| #cache-from: type=local,src=/tmp/.buildx-cache | |
| # using registry cache (no storage limit) | |
| cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} | |
| cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max | |
| - name: Build and push Light Docker image by digest | |
| id: build_light | |
| if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.light == true }} | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.config.platforms }} | |
| outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true | |
| file: ${{ matrix.config.dockerfile }} | |
| target: light | |
| provenance: false | |
| build-args: | | |
| ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} | |
| ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} | |
| # using github experimental cache | |
| #cache-from: type=gha | |
| #cache-to: type=gha,mode=max | |
| # return to this if the experimental github cache is having issues | |
| #cache-to: type=local,dest=/tmp/.buildx-cache | |
| #cache-from: type=local,src=/tmp/.buildx-cache | |
| # using registry cache (no storage limit) | |
| cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} | |
| cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max | |
| - name: Build and push Server Docker image by digest | |
| id: build_server | |
| if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.server == true }} | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.config.platforms }} | |
| outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true | |
| file: ${{ matrix.config.dockerfile }} | |
| target: server | |
| provenance: false | |
| build-args: | | |
| ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} | |
| ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} | |
| # using github experimental cache | |
| #cache-from: type=gha | |
| #cache-to: type=gha,mode=max | |
| # return to this if the experimental github cache is having issues | |
| #cache-to: type=local,dest=/tmp/.buildx-cache | |
| #cache-from: type=local,src=/tmp/.buildx-cache | |
| # using registry cache (no storage limit) | |
| cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} | |
| cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max | |
| - name: Export digest metadata | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| TAGS="${{ matrix.config.tag }}" | |
| ARCH_SUFFIX="${{ steps.meta.outputs.arch_suffix }}" | |
| DIGEST_FILE="/tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv" | |
| mkdir -p /tmp/digests | |
| add_digest_rows() { | |
| local image_type="$1" | |
| local digest="$2" | |
| if [[ -z "$digest" ]]; then | |
| echo "Missing digest for image_type=${image_type}" >&2 | |
| exit 1 | |
| fi | |
| for tag in $TAGS; do | |
| printf '%s\t%s\t%s\t%s\n' "$tag" "$ARCH_SUFFIX" "$image_type" "$digest" >> "$DIGEST_FILE" | |
| done | |
| } | |
| if [[ "${{ matrix.config.full }}" == "true" ]]; then | |
| add_digest_rows "full" "${{ steps.build_full.outputs.digest }}" | |
| fi | |
| if [[ "${{ matrix.config.light }}" == "true" ]]; then | |
| add_digest_rows "light" "${{ steps.build_light.outputs.digest }}" | |
| fi | |
| if [[ "${{ matrix.config.server }}" == "true" ]]; then | |
| add_digest_rows "server" "${{ steps.build_server.outputs.digest }}" | |
| fi | |
| - name: Upload digest metadata | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 | |
| with: | |
| name: digests-${{ steps.meta.outputs.digest_artifact_suffix }} | |
| path: /tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv | |
| if-no-files-found: error | |
| merge_arch_tags: | |
| name: Create shared tags from digests | |
| needs: [prepare_matrices, push_to_registry, create_tag] | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJSON(needs.prepare_matrices.outputs.merge_matrix) }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download digest metadata | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: digests-* | |
| path: /tmp/digests | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Log in to Docker Registry | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create tags from digests | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case | |
| REPO_NAME="${{ github.event.repository.name }}" | |
| IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" | |
| PREFIX="${IMAGE_REPO}:" | |
| SRC_TAG="${{ needs.create_tag.outputs.source_tag }}" | |
| TAGS="${{ matrix.config.tag }}" | |
| ARCHES="${{ matrix.config.arches }}" | |
| DIGEST_GLOB="/tmp/digests/*.tsv" | |
| if ! ls ${DIGEST_GLOB} >/dev/null 2>&1; then | |
| echo "No digest metadata found in /tmp/digests" >&2 | |
| exit 1 | |
| fi | |
| if [[ -z "$SRC_TAG" ]]; then | |
| echo "Missing source tag from create_tag" >&2 | |
| exit 1 | |
| fi | |
| find_digest() { | |
| local tag_name="$1" | |
| local arch="$2" | |
| local image_type="$3" | |
| local digest | |
| digest="$(awk -F '\t' -v t="$tag_name" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" | |
| # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. | |
| if [[ -z "$digest" && "$tag_name" == "s390x" && "$arch" == "s390x" ]]; then | |
| digest="$(awk -F '\t' -v t="cpu" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" | |
| fi | |
| if [[ -z "$digest" ]]; then | |
| echo "Missing digest for tag=${tag_name} arch=${arch} image_type=${image_type}" >&2 | |
| exit 1 | |
| fi | |
| echo "$digest" | |
| } | |
| create_manifest_tags() { | |
| local image_type="$1" | |
| local tag_name="$2" | |
| local suffix="$3" | |
| local merged_tag="${PREFIX}${image_type}${suffix}" | |
| local merged_versioned_tag="${merged_tag}-${SRC_TAG}" | |
| local refs=() | |
| for arch in $ARCHES; do | |
| local digest | |
| digest="$(find_digest "$tag_name" "$arch" "$image_type")" | |
| refs+=("${IMAGE_REPO}@${digest}") | |
| done | |
| echo "Creating ${merged_tag} from ${refs[*]}" | |
| docker buildx imagetools create --tag "${merged_tag}" "${refs[@]}" | |
| echo "Creating ${merged_versioned_tag} from ${refs[*]}" | |
| docker buildx imagetools create --tag "${merged_versioned_tag}" "${refs[@]}" | |
| } | |
| for tag in $TAGS; do | |
| if [[ "$tag" == "cpu" ]]; then | |
| TYPE="" | |
| else | |
| TYPE="-$tag" | |
| fi | |
| if [[ "${{ matrix.config.full }}" == "true" ]]; then | |
| create_manifest_tags "full" "$tag" "$TYPE" | |
| fi | |
| if [[ "${{ matrix.config.light }}" == "true" ]]; then | |
| create_manifest_tags "light" "$tag" "$TYPE" | |
| fi | |
| if [[ "${{ matrix.config.server }}" == "true" ]]; then | |
| create_manifest_tags "server" "$tag" "$TYPE" | |
| fi | |
| done | |
| env: | |
| GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}' |