diff --git a/.github/workflows/docker_workflow_fixed.yml b/.github/workflows/docker_workflow_fixed.yml new file mode 100644 index 00000000000..5dc4d74e7ca --- /dev/null +++ b/.github/workflows/docker_workflow_fixed.yml @@ -0,0 +1,421 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Publish Docker image + +on: + workflow_dispatch: # allows manual triggering + schedule: + # Rebuild daily rather than on every push because it is expensive + - cron: '12 4 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +# Fine-grant permission +# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token +permissions: + packages: write + +jobs: + create_tag: + name: Create and push git tag + runs-on: ubuntu-22.04 + permissions: + contents: write + outputs: + source_tag: ${{ steps.srctag.outputs.name }} + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine source tag name + id: srctag + uses: ./.github/actions/get-tag-name + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + + - name: Create and push git tag + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git tag ${{ steps.srctag.outputs.name }} || exit 0 + git push origin ${{ steps.srctag.outputs.name }} || exit 0 + + prepare_matrices: + name: Prepare Docker matrices + runs-on: ubuntu-22.04 + outputs: + build_matrix: ${{ steps.matrices.outputs.build_matrix }} + merge_matrix: ${{ steps.matrices.outputs.merge_matrix }} + + steps: + - name: Generate build and merge matrices + id: matrices + shell: bash + run: | + set -euo pipefail + + # Keep all build targets in one place and derive merge targets from it. + cat > build-matrix.json <<'JSON' + [ + { "tag": "cpu", "dockerfile": ".devops/cpu.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-22.04" }, + { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-22.04-s390x" }, + { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-22.04", "cuda_version": "12.4.0", "ubuntu_version": "22.04" }, + { "tag": "cuda13", "dockerfile": ".devops/cuda-new.Dockerfile","platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-22.04", "cuda_version": "13.1.0", "ubuntu_version": "24.04" }, + { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-22.04" }, + { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-22.04" }, + { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-22.04" }, + { "tag": "rocm", "dockerfile": ".devops/rocm.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-22.04" } + ] + JSON + + BUILD_MATRIX="$(jq -c . build-matrix.json)" + MERGE_MATRIX="$(jq -c ' + reduce .[] as $entry ({}; .[$entry.tag] |= ( + . // { + tag: $entry.tag, + arches: [], + full: false, + light: false, + server: false + } + | .full = (.full or ($entry.full // false)) + | .light = (.light or ($entry.light // false)) + | .server = (.server or ($entry.server // false)) + | .arches += [($entry.platforms | sub("^linux/"; ""))] + )) + # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. + | if (has("cpu") and (((.cpu.arches // []) | index("s390x")) != null)) then + . + { + s390x: { + tag: "s390x", + arches: ["s390x"], + full: .cpu.full, + light: .cpu.light, + server: .cpu.server + } + } + else + . + end + | [.[] | .arches = (.arches | unique | sort | join(" "))] + ' build-matrix.json)" + + echo "build_matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" + echo "merge_matrix=$MERGE_MATRIX" >> "$GITHUB_OUTPUT" + + push_to_registry: + name: Push Docker image to Docker Hub + needs: [prepare_matrices, create_tag] + + runs-on: ${{ matrix.config.runs_on }} + env: + COMMIT_SHA: ${{ github.sha }} + strategy: + fail-fast: false + matrix: + config: ${{ fromJSON(needs.prepare_matrices.outputs.build_matrix) }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # preserve git history, so we can determine the build number + ref: ${{ needs.create_tag.outputs.source_tag }} + + - name: Set up QEMU + if: ${{ matrix.config.tag != 's390x' && !contains(matrix.config.platforms, 'linux/s390x') }} + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:qemu-v7.0.0-28 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine image metadata + id: meta + shell: bash + run: | + set -euo pipefail + + REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case + REPO_NAME="${{ github.event.repository.name }}" + IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" + PREFIX="${IMAGE_REPO}:" + PLATFORM="${{ matrix.config.platforms }}" + ARCH_SUFFIX="${PLATFORM#linux/}" + + # list all tags possible + tags="${{ matrix.config.tag }}" + for tag in $tags; do + if [[ "$tag" == "cpu" ]]; then + TYPE="" + else + TYPE="-$tag" + fi + CACHETAG="${PREFIX}buildcache${TYPE}-${ARCH_SUFFIX}" + done + + SAFE_TAGS="$(echo "$tags" | tr ' ' '_')" + + echo "image_repo=$IMAGE_REPO" >> $GITHUB_OUTPUT + echo "arch_suffix=$ARCH_SUFFIX" >> $GITHUB_OUTPUT + echo "cache_output_tag=$CACHETAG" >> $GITHUB_OUTPUT + echo "digest_artifact_suffix=${SAFE_TAGS}-${ARCH_SUFFIX}" >> $GITHUB_OUTPUT + echo "cache_output_tag=$CACHETAG" # print out for debugging + env: + GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}' + + - name: Free Disk Space (Ubuntu) + if: ${{ matrix.config.free_disk_space == true }} + uses: ggml-org/free-disk-space@v1.3.1 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Build and push Full Docker image by digest + id: build_full + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.full == true }} + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true + file: ${{ matrix.config.dockerfile }} + target: full + provenance: false + build-args: | + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Build and push Light Docker image by digest + id: build_light + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.light == true }} + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true + file: ${{ matrix.config.dockerfile }} + target: light + provenance: false + build-args: | + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Build and push Server Docker image by digest + id: build_server + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.config.server == true }} + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.config.platforms }} + outputs: type=image,name=${{ steps.meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true + file: ${{ matrix.config.dockerfile }} + target: server + provenance: false + build-args: | + ${{ matrix.config.ubuntu_version && format('UBUNTU_VERSION={0}', matrix.config.ubuntu_version) || '' }} + ${{ matrix.config.cuda_version && format('CUDA_VERSION={0}', matrix.config.cuda_version) || '' }} + # using registry cache (no storage limit) + cache-from: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }} + cache-to: type=registry,ref=${{ steps.meta.outputs.cache_output_tag }},mode=max + + - name: Export digest metadata + shell: bash + run: | + set -euo pipefail + + TAGS="${{ matrix.config.tag }}" + ARCH_SUFFIX="${{ steps.meta.outputs.arch_suffix }}" + DIGEST_FILE="/tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv" + mkdir -p /tmp/digests + + add_digest_rows() { + local image_type="$1" + local digest="$2" + + if [[ -z "$digest" ]]; then + echo "Missing digest for image_type=${image_type}" >&2 + exit 1 + fi + + for tag in $TAGS; do + printf '%s\t%s\t%s\t%s\n' "$tag" "$ARCH_SUFFIX" "$image_type" "$digest" >> "$DIGEST_FILE" + done + } + + if [[ "${{ matrix.config.full }}" == "true" ]]; then + add_digest_rows "full" "${{ steps.build_full.outputs.digest }}" + fi + + if [[ "${{ matrix.config.light }}" == "true" ]]; then + add_digest_rows "light" "${{ steps.build_light.outputs.digest }}" + fi + + if [[ "${{ matrix.config.server }}" == "true" ]]; then + add_digest_rows "server" "${{ steps.build_server.outputs.digest }}" + fi + + - name: Upload digest metadata + uses: actions/upload-artifact@v4 + with: + name: digests-${{ steps.meta.outputs.digest_artifact_suffix }} + path: /tmp/digests/${{ steps.meta.outputs.digest_artifact_suffix }}.tsv + if-no-files-found: error + + merge_arch_tags: + name: Create shared tags from digests + needs: [prepare_matrices, push_to_registry, create_tag] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + config: ${{ fromJSON(needs.prepare_matrices.outputs.merge_matrix) }} + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download digest metadata + uses: actions/download-artifact@v4 + with: + pattern: digests-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create tags from digests + shell: bash + run: | + set -euo pipefail + + REPO_OWNER="${GITHUB_REPOSITORY_OWNER@L}" # to lower case + REPO_NAME="${{ github.event.repository.name }}" + IMAGE_REPO="ghcr.io/${REPO_OWNER}/${REPO_NAME}" + PREFIX="${IMAGE_REPO}:" + SRC_TAG="${{ needs.create_tag.outputs.source_tag }}" + TAGS="${{ matrix.config.tag }}" + ARCHES="${{ matrix.config.arches }}" + DIGEST_GLOB="/tmp/digests/*.tsv" + + if ! ls ${DIGEST_GLOB} >/dev/null 2>&1; then + echo "No digest metadata found in /tmp/digests" >&2 + exit 1 + fi + + if [[ -z "$SRC_TAG" ]]; then + echo "Missing source tag from create_tag" >&2 + exit 1 + fi + + find_digest() { + local tag_name="$1" + local arch="$2" + local image_type="$3" + local digest + + digest="$(awk -F '\t' -v t="$tag_name" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" + + # Backward compatibility: s390x tags are aliases of cpu for the linux/s390x platform. + if [[ -z "$digest" && "$tag_name" == "s390x" && "$arch" == "s390x" ]]; then + digest="$(awk -F '\t' -v t="cpu" -v a="$arch" -v i="$image_type" '$1 == t && $2 == a && $3 == i { print $4; exit }' ${DIGEST_GLOB})" + fi + + if [[ -z "$digest" ]]; then + echo "Missing digest for tag=${tag_name} arch=${arch} image_type=${image_type}" >&2 + exit 1 + fi + + echo "$digest" + } + + create_manifest_tags() { + local image_type="$1" + local tag_name="$2" + local suffix="$3" + + local merged_tag="${PREFIX}${image_type}${suffix}" + local merged_versioned_tag="${merged_tag}-${SRC_TAG}" + + local refs=() + + for arch in $ARCHES; do + local digest + digest="$(find_digest "$tag_name" "$arch" "$image_type")" + refs+=("${IMAGE_REPO}@${digest}") + done + + echo "Creating ${merged_tag} from ${refs[*]}" + docker buildx imagetools create --tag "${merged_tag}" "${refs[@]}" + + echo "Creating ${merged_versioned_tag} from ${refs[*]}" + docker buildx imagetools create --tag "${merged_versioned_tag}" "${refs[@]}" + } + + for tag in $TAGS; do + if [[ "$tag" == "cpu" ]]; then + TYPE="" + else + TYPE="-$tag" + fi + + if [[ "${{ matrix.config.full }}" == "true" ]]; then + create_manifest_tags "full" "$tag" "$TYPE" + fi + + if [[ "${{ matrix.config.light }}" == "true" ]]; then + create_manifest_tags "light" "$tag" "$TYPE" + fi + + if [[ "${{ matrix.config.server }}" == "true" ]]; then + create_manifest_tags "server" "$tag" "$TYPE" + fi + done + env: + GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'