From 6374c58befa0942b901776aa55168d56c036ff60 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Tue, 17 Mar 2026 17:48:11 +0100 Subject: [PATCH 01/11] Scaffold taskgraph with config, docker images, and parameters --- .taskcluster.yml | 639 +++++------------- taskcluster/code_review_taskgraph/__init__.py | 10 + .../code_review_taskgraph/parameters.py | 51 ++ .../transforms/__init__.py | 0 taskcluster/config.yml | 21 + taskcluster/docker/node/Dockerfile | 22 + taskcluster/docker/python/Dockerfile | 42 ++ taskcluster/docker/taskboot/Dockerfile | 21 + taskcluster/kinds/docker-image/kind.yml | 9 + .../test/params/github-pull-request.yml | 29 + .../test/params/github-push-master.yml | 29 + .../test/params/github-push-production.yml | 29 + taskcluster/test/params/github-push-tag.yml | 29 + .../test/params/github-push-testing.yml | 29 + 14 files changed, 504 insertions(+), 456 deletions(-) create mode 100644 taskcluster/code_review_taskgraph/__init__.py create mode 100644 taskcluster/code_review_taskgraph/parameters.py create mode 100644 taskcluster/code_review_taskgraph/transforms/__init__.py create mode 100644 taskcluster/config.yml create mode 100644 taskcluster/docker/node/Dockerfile create mode 100644 taskcluster/docker/python/Dockerfile create mode 100644 taskcluster/docker/taskboot/Dockerfile create mode 100644 taskcluster/kinds/docker-image/kind.yml create mode 100644 taskcluster/test/params/github-pull-request.yml create mode 100644 taskcluster/test/params/github-push-master.yml create mode 100644 taskcluster/test/params/github-push-production.yml create mode 100644 taskcluster/test/params/github-push-tag.yml create mode 100644 taskcluster/test/params/github-push-testing.yml diff --git a/.taskcluster.yml b/.taskcluster.yml index 9d81ef86b..8a75e4682 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,462 +1,189 @@ +# yamllint disable rule:line-length +# This file is rendered via JSON-e by +# - github events - https://github.com/taskcluster/taskcluster/tree/main/services/github +--- version: 1 +reporting: checks-v1 +autoCancelPreviousChecks: true policy: pullRequests: public -reporting: checks-v1 tasks: - $let: - head_branch: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.ref} - else: - $if: 'tasks_for == "github-push"' + - $let: + trustDomain: code-analysis + ownerEmail: + $switch: + 'tasks_for == "github-push"': "${event.pusher.email}" + 'tasks_for == "github-release"': "${event.sender.login}@users.noreply.github.com" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.user.login}@users.noreply.github.com" + baseRepoUrl: + $switch: + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.repo.html_url}" + $default: "${event.repository.html_url}" + repoUrl: + $switch: + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.head.repo.html_url}" + $default: "${event.repository.html_url}" + project: + $switch: + 'tasks_for in ["github-push", "github-release"]': "${event.repository.name}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.repo.name}" + head_ref: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.pull_request.head.ref} + 'tasks_for == "github-push"': ${event.ref} + 'tasks_for == "github-release"': ${event.release.tag_name} + base_ref: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.pull_request.base.ref} + 'tasks_for == "github-push" && event.base_ref': ${event.base_ref} + 'tasks_for == "github-push" && !(event.base_ref)': ${event.ref} + 'tasks_for == "github-release"': "" + base_sha: + $switch: + 'tasks_for == "github-push"': "${event.before}" + 'tasks_for == "github-release"': "${event.release.target_commitish}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.base.sha}" + head_sha: + $switch: + 'tasks_for == "github-push"': "${event.after}" + 'tasks_for == "github-release"': "${event.release.tag_name}" + 'tasks_for[:19] == "github-pull-request"': "${event.pull_request.head.sha}" + ownTaskId: + $eval: as_slugid("decision_task") + pullRequestAction: + $switch: + 'tasks_for[:19] == "github-pull-request"': ${event.action} + $default: "UNDEFINED" + releaseAction: + $if: 'tasks_for == "github-release"' + then: ${event.action} + else: "UNDEFINED" + isPullRequest: + $eval: 'tasks_for[:19] == "github-pull-request"' + in: + $let: + short_head_ref: + $switch: + 'head_ref[:10] == "refs/tags/"': "${head_ref[10:]}" + 'head_ref[:11] == "refs/heads/"': "${head_ref[11:]}" + $default: "${head_ref}" + level: + $if: 'tasks_for in ["github-release", "github-push"] && repoUrl == "https://github.com/mozilla/code-review"' + then: 3 + else: 1 + in: + $if: > + (tasks_for == "github-push" && short_head_ref in ["master", "production", "testing"]) + || (tasks_for == "github-release" && releaseAction == "published") + || (isPullRequest && pullRequestAction in ["opened", "reopened", "synchronize"]) then: - # Strip ref branch prefix - $if: 'event.ref[0:11] == "refs/heads/"' - then: ${event.ref[11:]} - else: ${event.ref} - else: ${event.release.target_commitish} - - head_rev: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.sha} - else: - $if: 'tasks_for == "github-push"' - then: ${event.after} - else: ${event.release.tag_name} - - repository: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.repo.html_url} - else: ${event.repository.html_url} - - channel: - $if: 'tasks_for == "github-push"' - then: - $if: 'event.ref in ["refs/heads/testing", "refs/heads/production"]' - then: ${event.ref[11:]} - else: "dev" - else: "dev" - - backend_url: - $if: 'tasks_for == "github-push"' - then: - $if: 'event.ref == "refs/heads/testing"' - then: "https://api.code-review.testing.moz.tools" - else: "https://api.code-review.moz.tools" - else: - $if: 'tasks_for == "github-pull-request"' - then: "https://api.code-review.testing.moz.tools" - else: "https://api.code-review.moz.tools" - - taskboot_image: "mozilla/taskboot:0.4.1" - - pip_install: "pip install --disable-pip-version-check --no-cache-dir --quiet" - python_version: "3.12" - - provisionerId: - $if: 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"' - then: - $if: 'tasks_for == "github-push"' - then: "code-analysis-3" - else: "code-analysis-1" - else: proj-relman - - workerType: - $if: 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"' - then: linux-gw-gcp - else: generic-worker-ubuntu-24-04 - in: - $if: '(tasks_for == "github-push" && (head_branch == "master" || head_branch == "production" || head_branch == "testing")) || (tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"])' - then: - $flatten: - $match: - # Always run those tasks - "true": - - taskId: { $eval: as_slugid("check_lint") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/bot && ${pip_install} -r requirements-dev.txt && - cd /src && pre-commit run -a --show-diff-on-failure" - metadata: - name: "Code Review Bot checks: linting" - description: Check code style with pre-commit hooks - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - # Run only on firefoxci Taskcluster - 'taskcluster_root_url == "https://firefox-ci-tc.services.mozilla.com"': - - taskId: { $eval: as_slugid("bot_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}-slim" - command: - - sh - - -lxce - - "apt-get update -q && apt-get install -q -y --no-install-recommends git && - git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - /src/tools/docker/bootstrap-mercurial.sh && - cd /src/bot && ${pip_install} . && ${pip_install} -r requirements-dev.txt && - pytest -v" - metadata: - name: "Code Review Bot checks: unit tests" - description: Check python code with pytest - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("bot_build_dind") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("bot_check_tests") } - payload: - maxRunTime: 3600 - image: "${taskboot_image}" - env: - GIT_REPOSITORY: ${repository} - GIT_REVISION: ${head_rev} - command: - - taskboot - - build - - --image - - mozilla/code-review - - --tag - - "${channel}" - - --tag - - "${head_rev}" - - --write - - /bot.tar - - bot/docker/Dockerfile - artifacts: - public/code-review-bot.tar.zst: - expires: { $fromNow: "6 months" } - path: /bot.tar.zst - type: file - routes: - $if: 'tasks_for == "github-pull-request"' + taskId: "${ownTaskId}" + taskGroupId: "${ownTaskId}" + schedulerId: "${trustDomain}-level-${level}" + + created: { $fromNow: "" } + deadline: { $fromNow: "1 day" } + expires: { $fromNow: "1 year 1 second" } + + metadata: + owner: "${ownerEmail}" + source: "${repoUrl}/raw/${head_sha}/.taskcluster.yml" + name: "Decision Task" + description: "The task that creates all of the other tasks in the task graph" + + provisionerId: "${trustDomain}-${level}" + workerType: "decision" + + tags: + createdForUser: "${ownerEmail}" + kind: decision-task + + routes: + $flatten: + - checks + - $if: 'tasks_for == "github-push"' then: - - "index.code-analysis.v2.code-review-pr.revision.${head_rev}" - - "index.code-analysis.v2.code-review-pr.branch.${head_branch}" - else: - - "index.code-analysis.v2.code-review.revision.${head_rev}" - - "index.code-analysis.v2.code-review.branch.${head_branch}" - metadata: - name: Code Review Bot docker in docker build - description: Build docker image of code review bot, using a remote docker daemon - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - # Run only on community Taskcluster - 'taskcluster_root_url == "https://community-tc.services.mozilla.com"': - - taskId: { $eval: as_slugid("backend_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/backend && ${pip_install} . && ${pip_install} -r requirements-dev.txt && - ./ci/setup_postgres.sh && - ./manage.py test && ./manage.py makemigrations --check --noinput --dry-run -v 3" - env: - DATABASE_URL: postgres://tester@127.0.0.1/code-review - metadata: - name: "Code Review Backend checks: unit tests" - description: Check python code with Django tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("frontend_build") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: node:16-alpine - env: - BACKEND_URL: "${backend_url}" - command: - - sh - - -lxce - - "apk add git lcms2-dev bash libpng-dev autoconf build-base --quiet && - git clone --quiet ${repository} /src && cd /src/frontend && git checkout ${head_rev} -b build && - npm install && npm run build" - artifacts: - public/frontend: - expires: { $fromNow: "2 weeks" } - path: /src/frontend/build - type: directory - metadata: - name: Code Review Frontend build - description: Build web single page application - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("backend_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("backend_check_tests") } - payload: - capabilities: - privileged: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - /bin/sh - - -lxce - - "git clone --quiet ${repository} /code-review && cd /code-review && git checkout ${head_rev} -b build - && backend/build.sh ${head_rev} ${head_branch} ${repository} ${channel}" - artifacts: - public/code-review-backend.tar.zst: - expires: { $fromNow: "2 weeks" } - path: /backend.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: Code Review Backend docker build - description: Build docker image of code review backend - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("integration_check_tests") } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - payload: - maxRunTime: 3600 - image: "python:${python_version}" - command: - - sh - - -lxce - - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/integration && ${pip_install} -r requirements.txt -r requirements-dev.txt && - pytest -v" - metadata: - name: "Code Review Integration checks: unit tests" - description: Check python code with pytest - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - taskId: { $eval: as_slugid("integration_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("check_lint") } - - { $eval: as_slugid("integration_check_tests") } - payload: - capabilities: - privileged: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - GIT_REPOSITORY: ${repository} - GIT_REVISION: ${head_rev} - command: - - taskboot - - build - - --image - - mozilla/code-review - - --tag - - "integration-${channel}" - - --tag - - "integration-${head_rev}" - - --write - - /integration.tar - - integration/docker/Dockerfile - artifacts: - public/code-review-integration.tar.zst: - expires: { $fromNow: "2 weeks" } - path: /integration.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: Code Review Integration docker build - description: Build docker image of code review integration tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("frontend_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("frontend_build") } - payload: - features: - # Needed for access to secret - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - command: - - taskboot - - deploy-s3 - - --artifact-folder - - public/frontend - - --bucket - - "relman-${channel}-code-review-${channel}-static-website" - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Frontend deploy (${channel})" - description: Deploy frontend build on environment - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("backend_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("backend_build") } - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - taskboot - - deploy-heroku - - --heroku-app - - "code-review-backend-${channel}" - - web:public/code-review-backend.tar.zst - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Backend deployment (${channel})" - description: Deploy docker image on Heroku - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("integration_deploy") } - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - dependencies: - - { $eval: as_slugid("integration_build") } - payload: - features: - # Needed for access to secret - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" - command: - - taskboot - - push-artifact - scopes: - - "secrets:get:project/relman/code-review/deploy-${channel}" - metadata: - name: "Code Review Integration test push (${channel})" - description: Push integration's docker image on repository - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'channel in ["testing", "production"]' - then: - taskId: { $eval: as_slugid("integration_hook") } - dependencies: - - { $eval: as_slugid("integration_deploy") } - scopes: - - "assume:hook-id:project-relman/code-review-integration-${channel}" - - "hooks:modify-hook:project-relman/code-review-integration-${channel}" - created: { $fromNow: "" } - deadline: { $fromNow: "5 hours" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - payload: - features: - # Needed for access to hook api - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - command: - - "/bin/sh" - - "-lcxe" - - "git clone --quiet ${repository} && - cd code-review && - git checkout ${head_rev} && - sed -i -e 's/CHANNEL/${channel}/g' -e 's/REVISION/${head_rev}/g' integration/taskcluster-hook.json && - taskboot --target . build-hook integration/taskcluster-hook.json project-relman code-review-integration-${channel}" - metadata: - name: "Code Review Bot integration test hook update (${channel})" - description: Update Taskcluster hook triggering the code-review integration tests - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review - - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $let: - version: { $eval: "head_branch[10:]" } - in: - taskId: { $eval: as_slugid("release") } - dependencies: - - { $eval: as_slugid("backend_build") } - - { $eval: as_slugid("frontend_build") } - - { $eval: as_slugid("integration_build") } - created: { $fromNow: "" } - deadline: { $fromNow: "2 hours" } - provisionerId: "${provisionerId}" - workerType: "${workerType}" - scopes: - - secrets:get:project/relman/code-review/release - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: project/relman/code-review/release - command: - - taskboot - - github-release - - mozilla/code-review - - "${version}" - metadata: - name: "Code Review Bot release ${version}" - description: Publish a new GitHub release for code-review platform - owner: bastien@mozilla.com - source: https://github.com/mozilla/code-review + - "index.${trustDomain}.v2.${project}.latest.taskgraph.decision" + - "index.${trustDomain}.v2.${project}.revision.${head_sha}.taskgraph.decision" + else: [] + + scopes: + $switch: + 'tasks_for == "github-push"': + - "assume:repo:${repoUrl[8:]}:branch:${short_head_ref}" + 'tasks_for == "github-release"': + - "assume:repo:${repoUrl[8:]}:release:${releaseAction}" + "isPullRequest": + - "assume:repo:github.com/${event.pull_request.base.repo.full_name}:${tasks_for[7:]}" + + dependencies: [] + requires: all-completed + + priority: + $if: 'tasks_for == "github-push" || isPullRequest' + then: very-low + else: lowest + + retries: 5 + + payload: + env: + $merge: + - CODE_REVIEW_BASE_REPOSITORY: "${baseRepoUrl}" + CODE_REVIEW_BASE_REF: "${base_ref}" + CODE_REVIEW_BASE_REV: "${base_sha}" + CODE_REVIEW_HEAD_REPOSITORY: "${repoUrl}" + CODE_REVIEW_HEAD_REF: "${head_ref}" + CODE_REVIEW_HEAD_REV: "${head_sha}" + CODE_REVIEW_REPOSITORY_TYPE: git + REPOSITORIES: { $json: { code_review: code-review } } + - $if: "isPullRequest" + then: + CODE_REVIEW_PULL_REQUEST_NUMBER: "${event.pull_request.number}" + + cache: + "${trustDomain}-level-${level}-checkouts-v2": /builds/worker/checkouts + + features: + taskclusterProxy: true + + image: mozillareleases/taskgraph:decision-v19.2.1 + + maxRunTime: 1800 + + command: + - run-task + - "--code_review-checkout=/builds/worker/checkouts/src" + - "--" + - bash + - -cx + - > + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --owner='${ownerEmail}' + --level='${level}' + --repository-type=git + --tasks-for='${tasks_for}' + --base-repository='${baseRepoUrl}' + --base-ref='${base_ref}' + --base-rev='${base_sha}' + --head-repository='${repoUrl}' + --head-ref='${head_ref}' + --head-rev='${head_sha}' + + artifacts: + "public": + type: "directory" + path: "/builds/worker/artifacts" + expires: { $fromNow: "1 year" } + "public/docker-contexts": + type: "directory" + path: "/builds/worker/checkouts/src/docker-contexts" + expires: { $fromNow: "7 day" } + + extra: + tasks_for: "${tasks_for}" diff --git a/taskcluster/code_review_taskgraph/__init__.py b/taskcluster/code_review_taskgraph/__init__.py new file mode 100644 index 000000000..ca58addde --- /dev/null +++ b/taskcluster/code_review_taskgraph/__init__.py @@ -0,0 +1,10 @@ +from importlib import import_module + + +def register(graph_config): + _import_modules(["parameters"]) + + +def _import_modules(modules): + for module in modules: + import_module(f".{module}", package=__name__) diff --git a/taskcluster/code_review_taskgraph/parameters.py b/taskcluster/code_review_taskgraph/parameters.py new file mode 100644 index 000000000..c4f161210 --- /dev/null +++ b/taskcluster/code_review_taskgraph/parameters.py @@ -0,0 +1,51 @@ +import fnmatch + +from taskgraph.parameters import extend_parameters_schema +from taskgraph.target_tasks import register_target_task +from voluptuous import Optional + +extend_parameters_schema( + { + Optional("channel"): str, + Optional("backend_url"): str, + }, +) + + +@register_target_task("default") +def target_tasks_default(full_task_graph, parameters, graph_config): + head_ref = parameters["head_ref"] + for prefix in ("refs/heads/", "refs/tags/"): + if head_ref.startswith(prefix): + head_ref = head_ref[len(prefix) :] + break + + def should_run(task): + if parameters["tasks_for"] not in task.attributes.get( + "run-on-tasks-for", [parameters["tasks_for"]] + ): + return False + run_on_branches = task.attributes.get("run-on-git-branches") + if run_on_branches is not None: + return any( + fnmatch.fnmatch(head_ref, pattern) for pattern in run_on_branches + ) + return True + + return [label for label, task in full_task_graph.tasks.items() if should_run(task)] + + +def decision_parameters(graph_config, parameters): + head_ref = parameters["head_ref"] + if head_ref.startswith("refs/heads/"): + head_ref = head_ref[len("refs/heads/") :] + + if head_ref == "testing": + parameters["channel"] = "testing" + parameters["backend_url"] = "https://api.code-review.testing.moz.tools" + elif head_ref == "production": + parameters["channel"] = "production" + parameters["backend_url"] = "https://api.code-review.moz.tools" + else: + parameters["channel"] = "dev" + parameters["backend_url"] = "https://api.code-review.testing.moz.tools" diff --git a/taskcluster/code_review_taskgraph/transforms/__init__.py b/taskcluster/code_review_taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/config.yml b/taskcluster/config.yml new file mode 100644 index 000000000..a0516f1e2 --- /dev/null +++ b/taskcluster/config.yml @@ -0,0 +1,21 @@ +--- +trust-domain: code-analysis +task-priority: low +taskgraph: + register: code_review_taskgraph:register + decision-parameters: code_review_taskgraph.parameters:decision_parameters + repositories: + code_review: + name: code-review +workers: + aliases: + b-linux: + provisioner: "{trust-domain}-{level}" + implementation: docker-worker + os: linux + worker-type: linux-gw-gcp + images: + provisioner: "{trust-domain}-{level}" + implementation: docker-worker + os: linux + worker-type: linux-gcp diff --git a/taskcluster/docker/node/Dockerfile b/taskcluster/docker/node/Dockerfile new file mode 100644 index 000000000..7aef7a18d --- /dev/null +++ b/taskcluster/docker/node/Dockerfile @@ -0,0 +1,22 @@ +FROM node:16-alpine + +# Add worker user +RUN mkdir -p /builds/worker/artifacts && \ + delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \ + deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \ + addgroup -g 1000 worker && \ + adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \ + chown -R worker:worker /builds/worker + +RUN apk add --no-cache git bash lcms2-dev libpng-dev autoconf build-base coreutils shadow python3 + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile new file mode 100644 index 000000000..5d721c592 --- /dev/null +++ b/taskcluster/docker/python/Dockerfile @@ -0,0 +1,42 @@ +FROM python:3.12-slim + +# Add worker user +RUN mkdir -p /builds/worker/artifacts && \ + groupadd --gid 1000 worker && \ + useradd --uid 1000 --gid worker --home-dir /builds/worker --shell /bin/bash worker && \ + chown -R worker:worker /builds/worker + +# Install build deps (needed for mercurial setup) +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + gcc \ + python3-dev \ + libatomic1 && \ + rm -rf /var/lib/apt/lists/* + +# Install mercurial +RUN pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==7.2 + +# Clone version-control-tools +RUN hg clone -r 5b6e8298d035 https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/ && \ + rm -rf /src/version-control-tools/.hg \ + /src/version-control-tools/ansible \ + /src/version-control-tools/docs \ + /src/version-control-tools/testing + +# %include-run-task + +# Cleanup build-only deps +RUN apt-get purge -y gcc python3-dev && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/docker/taskboot/Dockerfile b/taskcluster/docker/taskboot/Dockerfile new file mode 100644 index 000000000..d640fa2f5 --- /dev/null +++ b/taskcluster/docker/taskboot/Dockerfile @@ -0,0 +1,21 @@ +FROM mozilla/taskboot:0.4.5 + +RUN apk add --no-cache bash coreutils shadow + +RUN mkdir -p /builds/worker/artifacts && \ + delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \ + deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \ + addgroup -g 1000 worker && \ + adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \ + chown -R worker:worker /builds/worker + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + +CMD ["/bin/bash"] diff --git a/taskcluster/kinds/docker-image/kind.yml b/taskcluster/kinds/docker-image/kind.yml new file mode 100644 index 000000000..510186fbd --- /dev/null +++ b/taskcluster/kinds/docker-image/kind.yml @@ -0,0 +1,9 @@ +loader: taskgraph.loader.transform:loader +transforms: + - taskgraph.transforms.docker_image + - taskgraph.transforms.cached_tasks + - taskgraph.transforms.task +tasks: + python: {} + taskboot: {} + node: {} diff --git a/taskcluster/test/params/github-pull-request.yml b/taskcluster/test/params/github-pull-request.yml new file mode 100644 index 000000000..13131f243 --- /dev/null +++ b/taskcluster/test/params/github-pull-request.yml @@ -0,0 +1,29 @@ +base_ref: master +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: some-feature-branch +head_repository: https://github.com/user/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "1" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-pull-request +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-master.yml b/taskcluster/test/params/github-push-master.yml new file mode 100644 index 000000000..543bcb5ad --- /dev/null +++ b/taskcluster/test/params/github-push-master.yml @@ -0,0 +1,29 @@ +base_ref: master +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/master +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-production.yml b/taskcluster/test/params/github-push-production.yml new file mode 100644 index 000000000..0cd875c95 --- /dev/null +++ b/taskcluster/test/params/github-push-production.yml @@ -0,0 +1,29 @@ +base_ref: production +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: production +backend_url: "https://api.code-review.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/production +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-tag.yml b/taskcluster/test/params/github-push-tag.yml new file mode 100644 index 000000000..5db8e37a6 --- /dev/null +++ b/taskcluster/test/params/github-push-tag.yml @@ -0,0 +1,29 @@ +base_ref: "" +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: dev +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/tags/v1.0.0 +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null diff --git a/taskcluster/test/params/github-push-testing.yml b/taskcluster/test/params/github-push-testing.yml new file mode 100644 index 000000000..5708642ad --- /dev/null +++ b/taskcluster/test/params/github-push-testing.yml @@ -0,0 +1,29 @@ +base_ref: testing +base_repository: https://github.com/mozilla/code-review +base_rev: "0000000000000000000000000000000000000000" +build_date: 1710000000 +build_number: 1 +channel: testing +backend_url: "https://api.code-review.testing.moz.tools" +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +files_changed: [] +filters: [target_tasks_method] +head_ref: refs/heads/testing +head_repository: https://github.com/mozilla/code-review +head_rev: abc123def456abc123def456abc123def456abc1 +head_tag: "" +level: "3" +moz_build_date: "20250310120000" +optimize_strategies: null +optimize_target_tasks: true +owner: test@example.com +project: code-review +pushdate: 1710000000 +pushlog_id: "0" +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null +next_version: null From f32e4602a90106ec44adabee1829c3f7dd14d968 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 13:17:46 +0100 Subject: [PATCH 02/11] Add lint kind --- taskcluster/kinds/lint/kind.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 taskcluster/kinds/lint/kind.yml diff --git a/taskcluster/kinds/lint/kind.yml b/taskcluster/kinds/lint/kind.yml new file mode 100644 index 000000000..f80ea68b5 --- /dev/null +++ b/taskcluster/kinds/lint/kind.yml @@ -0,0 +1,23 @@ +loader: taskgraph.loader.transform:loader +transforms: + - taskgraph.transforms.run + - taskgraph.transforms.task +kind-dependencies: + - docker-image +task-defaults: + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + run: + using: run-task + cwd: "{checkout}" +tasks: + pre-commit: + description: Check code style with pre-commit hooks + run: + command: >- + cd bot && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + cd .. && + pre-commit run -a --show-diff-on-failure From d0bacac4ce42de7ddd25618000390a377fb369bd Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 14:22:34 +0100 Subject: [PATCH 03/11] Fix pre-existing ruff violations --- .../issues/management/commands/load_issues.py | 2 +- bot/code_review_bot/__init__.py | 2 +- bot/code_review_bot/workflow.py | 2 +- bot/tests/test_tools.py | 2 +- bot/tools/fix_missing.py | 2 +- integration/run.py | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/code_review_backend/issues/management/commands/load_issues.py b/backend/code_review_backend/issues/management/commands/load_issues.py index 76e2f7eaa..79c1ada5c 100644 --- a/backend/code_review_backend/issues/management/commands/load_issues.py +++ b/backend/code_review_backend/issues/management/commands/load_issues.py @@ -8,12 +8,12 @@ import tempfile from urllib.parse import urlparse -import taskcluster from django.conf import settings from django.core.management.base import BaseCommand from django.db import transaction from requests.exceptions import HTTPError +import taskcluster from code_review_backend.issues.compare import detect_new_for_revision from code_review_backend.issues.models import Issue, IssueLink, Repository diff --git a/bot/code_review_bot/__init__.py b/bot/code_review_bot/__init__.py index 1bfe0c0f8..df447926d 100644 --- a/bot/code_review_bot/__init__.py +++ b/bot/code_review_bot/__init__.py @@ -12,11 +12,11 @@ import requests import structlog from libmozdata.phabricator import LintResult, UnitResult, UnitResultState -from taskcluster.helper import TaskclusterConfig from code_review_bot.config import settings from code_review_bot.stats import InfluxDb from code_review_bot.tasks.base import AnalysisTask +from taskcluster.helper import TaskclusterConfig logger = structlog.get_logger(__name__) diff --git a/bot/code_review_bot/workflow.py b/bot/code_review_bot/workflow.py index 260a638e3..408d2a5b2 100644 --- a/bot/code_review_bot/workflow.py +++ b/bot/code_review_bot/workflow.py @@ -8,7 +8,6 @@ import structlog from libmozdata.phabricator import BuildState, PhabricatorAPI -from taskcluster.utils import stringDate from code_review_bot import Level, stats from code_review_bot.analysis import ( @@ -34,6 +33,7 @@ from code_review_bot.tasks.docupload import DocUploadTask from code_review_bot.tasks.lint import MozLintTask from code_review_bot.tasks.tgdiff import TaskGraphDiffTask +from taskcluster.utils import stringDate logger = structlog.get_logger(__name__) diff --git a/bot/tests/test_tools.py b/bot/tests/test_tools.py index 9ed62ca72..a0d6b1e29 100644 --- a/bot/tests/test_tools.py +++ b/bot/tests/test_tools.py @@ -3,9 +3,9 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import pytest -from taskcluster.helper import TaskclusterConfig from code_review_bot.tools.log import remove_color_codes +from taskcluster.helper import TaskclusterConfig def test_taskcluster_service(): diff --git a/bot/tools/fix_missing.py b/bot/tools/fix_missing.py index 3e42f86c9..55538ce2d 100644 --- a/bot/tools/fix_missing.py +++ b/bot/tools/fix_missing.py @@ -8,9 +8,9 @@ from datetime import datetime, timedelta import requests -from taskcluster.helper import TaskclusterConfig from code_review_bot.config import GetAppUserAgent +from taskcluster.helper import TaskclusterConfig TREEHERDER_PUSH_URL = "https://treeherder.mozilla.org/api/project/try/push/" TREEHERDER_JOBS_URL = "https://treeherder.mozilla.org/api/jobs/" diff --git a/integration/run.py b/integration/run.py index 7a25270ac..4c32ace7d 100755 --- a/integration/run.py +++ b/integration/run.py @@ -12,6 +12,7 @@ import structlog import yaml + from taskcluster.helper import TaskclusterConfig taskcluster = TaskclusterConfig("https://community-tc.services.mozilla.com") From f9c1bab713cbd9671fb1861f35c441a0d034b577 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 14:57:51 +0100 Subject: [PATCH 04/11] Add test kind --- taskcluster/kinds/test/kind.yml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 taskcluster/kinds/test/kind.yml diff --git a/taskcluster/kinds/test/kind.yml b/taskcluster/kinds/test/kind.yml new file mode 100644 index 000000000..7f1f9f603 --- /dev/null +++ b/taskcluster/kinds/test/kind.yml @@ -0,0 +1,54 @@ +loader: taskgraph.loader.transform:loader +transforms: + - taskgraph.transforms.run + - taskgraph.transforms.task +kind-dependencies: + - docker-image +tasks: + bot: + description: Check bot code with pytest + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + run: + using: run-task + cwd: "{checkout}" + command: >- + ln -sf /builds/worker/checkouts/vcs/tools/docker/hgrc ~/.hgrc && + cd bot && + pip install --disable-pip-version-check --no-cache-dir --quiet . && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + pytest -v + backend: + description: Check backend with Django tests + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + env: + DATABASE_URL: postgres://tester@127.0.0.1/code-review + run: + using: run-task + cwd: "{checkout}" + run-as-root: true + command: >- + cd backend && + pip install --disable-pip-version-check --no-cache-dir --quiet . && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements-dev.txt && + ./ci/setup_postgres.sh && + ./manage.py test && + ./manage.py makemigrations --check --noinput --dry-run -v 3 + integration: + description: Check integration code with pytest + worker-type: b-linux + worker: + docker-image: { in-tree: python } + max-run-time: 3600 + run: + using: run-task + cwd: "{checkout}" + command: >- + cd integration && + pip install --disable-pip-version-check --no-cache-dir --quiet -r requirements.txt -r requirements-dev.txt && + pytest -v From 78dd28a177d75a1ca77e9cc49ac71187c36f194f Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 15:20:15 +0100 Subject: [PATCH 05/11] Add frontend kind --- taskcluster/kinds/frontend/kind.yml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 taskcluster/kinds/frontend/kind.yml diff --git a/taskcluster/kinds/frontend/kind.yml b/taskcluster/kinds/frontend/kind.yml new file mode 100644 index 000000000..d4db05363 --- /dev/null +++ b/taskcluster/kinds/frontend/kind.yml @@ -0,0 +1,32 @@ +loader: taskgraph.loader.transform:loader +transforms: + - taskgraph.transforms.task_context + - taskgraph.transforms.run + - taskgraph.transforms.task +kind-dependencies: + - docker-image +tasks: + build: + description: Build frontend SPA + worker-type: b-linux + worker: + docker-image: { in-tree: node } + max-run-time: 3600 + env: + BACKEND_URL: "{backend_url}" + artifacts: + - type: directory + name: public/frontend + path: /builds/worker/artifacts/frontend + run: + using: run-task + cwd: "{checkout}/frontend" + command: >- + npm install && + npm run build && + cp -r build /builds/worker/artifacts/frontend + task-context: + from-parameters: + backend_url: backend_url + substitution-fields: + - worker.env From af8f2386bfc0ffa9748de91c8b44dc6283196b1a Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 15:39:56 +0100 Subject: [PATCH 06/11] Add build kind --- .../code_review_taskgraph/transforms/build.py | 72 +++++++++++++++++++ taskcluster/kinds/build/kind.yml | 60 ++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 taskcluster/code_review_taskgraph/transforms/build.py create mode 100644 taskcluster/kinds/build/kind.yml diff --git a/taskcluster/code_review_taskgraph/transforms/build.py b/taskcluster/code_review_taskgraph/transforms/build.py new file mode 100644 index 000000000..a9a3ca17b --- /dev/null +++ b/taskcluster/code_review_taskgraph/transforms/build.py @@ -0,0 +1,72 @@ +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + +_CHECKOUT = "/builds/worker/checkouts/vcs" + + +@transforms.add +def add_build_config(config, tasks): + for task in tasks: + params = config.params + head_rev = params["head_rev"] + channel = params.get("channel", "dev") + head_ref = params["head_ref"] + for prefix in ("refs/heads/", "refs/tags/"): + if head_ref.startswith(prefix): + head_ref = head_ref[len(prefix) :] + break + + task_name = task["name"] + + if task_name == "bot": + command = ( + f"taskboot --target {_CHECKOUT}" + f" build --image mozilla/code-review" + f" --tag {channel} --tag {head_rev}" + " --write /bot.tar bot/docker/Dockerfile" + ) + elif task_name == "backend": + command = ( + f'python3 -c "' + f"import json, os; " + f"open('{_CHECKOUT}/backend/code_review_backend/version.json', 'w')" + f".write(json.dumps({{'commit': '{head_rev}', 'version': '{head_ref}', " + f"'source': os.environ['CODE_REVIEW_HEAD_REPOSITORY'], " + f"'build': os.environ.get('TASKCLUSTER_ROOT_URL', '') + '/tasks/' + os.environ.get('TASK_ID', '')}}))" + f'" &&' + f" taskboot --target {_CHECKOUT}" + f" build --image mozilla/code-review" + f" --tag {channel} --tag {head_rev}" + " --write /backend.tar backend/Dockerfile" + ) + elif task_name == "integration": + command = ( + f"taskboot --target {_CHECKOUT}" + f" build --image mozilla/code-review" + f" --tag integration-{channel} --tag integration-{head_rev}" + " --write /integration.tar integration/docker/Dockerfile" + ) + else: + raise ValueError(f"Unknown build task: {task_name}") + + task["run"] = { + "using": "run-task", + "cwd": "{checkout}", + "run-as-root": True, + "command": command, + } + + if params["tasks_for"] == "github-pull-request": + index_prefix = "code-review-pr" + else: + index_prefix = "code-review" + + task.setdefault("routes", []).extend( + [ + f"index.code-analysis.v2.{index_prefix}.{task_name}.revision.{head_rev}", + f"index.code-analysis.v2.{index_prefix}.{task_name}.branch.{head_ref}", + ] + ) + + yield task diff --git a/taskcluster/kinds/build/kind.yml b/taskcluster/kinds/build/kind.yml new file mode 100644 index 000000000..874f65d89 --- /dev/null +++ b/taskcluster/kinds/build/kind.yml @@ -0,0 +1,60 @@ +loader: taskgraph.loader.transform:loader +transforms: + - code_review_taskgraph.transforms.build + - taskgraph.transforms.run + - taskgraph.transforms.task +kind-dependencies: + - lint + - test +tasks: + bot: + description: Build bot Docker image + attributes: + run-on-tasks-for: + - github-pull-request + - github-push + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + artifacts: + - type: file + name: public/code-review-bot.tar.zst + path: /bot.tar.zst + dependencies: + lint: lint-pre-commit + test: test-bot + backend: + description: Build backend Docker image + attributes: + run-on-tasks-for: + - github-pull-request + - github-push + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + artifacts: + - type: file + name: public/code-review-backend.tar.zst + path: /backend.tar.zst + dependencies: + lint: lint-pre-commit + test: test-backend + integration: + description: Build integration Docker image + attributes: + run-on-tasks-for: + - github-pull-request + - github-push + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + artifacts: + - type: file + name: public/code-review-integration.tar.zst + path: /integration.tar.zst + dependencies: + lint: lint-pre-commit + test: test-integration From 1a673069ec159e0fea45b06a8a9337ceec47a566 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 17:33:13 +0100 Subject: [PATCH 07/11] Retrigger CI after fxci-config staging grants update From 0b5705e0d931b8ed929c1cdf290fac9b0ec374ee Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 17:41:36 +0100 Subject: [PATCH 08/11] Retrigger CI (2) From e9c6fb014db6c7a2cef2f09a326a676fdc6ebdef Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 18 Mar 2026 17:57:11 +0100 Subject: [PATCH 09/11] Retrigger CI (lint 504 timeout) From 271f145ab892cd49ffcbb540e892c992d7ca0263 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 19 Mar 2026 06:32:52 +0100 Subject: [PATCH 10/11] Fix build kind: write artifacts to /builds/worker/artifacts/ --- taskcluster/code_review_taskgraph/transforms/build.py | 7 ++++--- taskcluster/kinds/build/kind.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/taskcluster/code_review_taskgraph/transforms/build.py b/taskcluster/code_review_taskgraph/transforms/build.py index a9a3ca17b..607a22709 100644 --- a/taskcluster/code_review_taskgraph/transforms/build.py +++ b/taskcluster/code_review_taskgraph/transforms/build.py @@ -3,6 +3,7 @@ transforms = TransformSequence() _CHECKOUT = "/builds/worker/checkouts/vcs" +_ARTIFACTS = "/builds/worker/artifacts" @transforms.add @@ -24,7 +25,7 @@ def add_build_config(config, tasks): f"taskboot --target {_CHECKOUT}" f" build --image mozilla/code-review" f" --tag {channel} --tag {head_rev}" - " --write /bot.tar bot/docker/Dockerfile" + f" --write {_ARTIFACTS}/bot.tar bot/docker/Dockerfile" ) elif task_name == "backend": command = ( @@ -38,14 +39,14 @@ def add_build_config(config, tasks): f" taskboot --target {_CHECKOUT}" f" build --image mozilla/code-review" f" --tag {channel} --tag {head_rev}" - " --write /backend.tar backend/Dockerfile" + f" --write {_ARTIFACTS}/backend.tar backend/Dockerfile" ) elif task_name == "integration": command = ( f"taskboot --target {_CHECKOUT}" f" build --image mozilla/code-review" f" --tag integration-{channel} --tag integration-{head_rev}" - " --write /integration.tar integration/docker/Dockerfile" + f" --write {_ARTIFACTS}/integration.tar integration/docker/Dockerfile" ) else: raise ValueError(f"Unknown build task: {task_name}") diff --git a/taskcluster/kinds/build/kind.yml b/taskcluster/kinds/build/kind.yml index 874f65d89..9291388cd 100644 --- a/taskcluster/kinds/build/kind.yml +++ b/taskcluster/kinds/build/kind.yml @@ -20,7 +20,7 @@ tasks: artifacts: - type: file name: public/code-review-bot.tar.zst - path: /bot.tar.zst + path: /builds/worker/artifacts/bot.tar.zst dependencies: lint: lint-pre-commit test: test-bot @@ -37,7 +37,7 @@ tasks: artifacts: - type: file name: public/code-review-backend.tar.zst - path: /backend.tar.zst + path: /builds/worker/artifacts/backend.tar.zst dependencies: lint: lint-pre-commit test: test-backend @@ -54,7 +54,7 @@ tasks: artifacts: - type: file name: public/code-review-integration.tar.zst - path: /integration.tar.zst + path: /builds/worker/artifacts/integration.tar.zst dependencies: lint: lint-pre-commit test: test-integration From 5241c46b36ecd3259f871d98b01431aae77ed770 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 19 Mar 2026 09:40:59 +0100 Subject: [PATCH 11/11] Add deploy kind --- .../transforms/deploy.py | 40 ++++++++++++++ taskcluster/kinds/deploy/kind.yml | 54 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 taskcluster/code_review_taskgraph/transforms/deploy.py create mode 100644 taskcluster/kinds/deploy/kind.yml diff --git a/taskcluster/code_review_taskgraph/transforms/deploy.py b/taskcluster/code_review_taskgraph/transforms/deploy.py new file mode 100644 index 000000000..bb9d2c566 --- /dev/null +++ b/taskcluster/code_review_taskgraph/transforms/deploy.py @@ -0,0 +1,40 @@ +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def add_deploy_config(config, tasks): + for task in tasks: + params = config.params + channel = params.get("channel", "dev") + task_name = task["name"] + + secret = f"project/relman/code-review/deploy-{channel}" + + if task_name == "frontend": + command = ( + f"taskboot deploy-s3" + f" --artifact-folder public/frontend" + f" --bucket relman-{channel}-code-review-{channel}-static-website" + ) + elif task_name == "backend": + command = ( + f"taskboot deploy-heroku" + f" --heroku-app code-review-backend-{channel}" + f" web:public/code-review-backend.tar.zst" + ) + elif task_name == "integration": + command = "taskboot push-artifact" + else: + raise ValueError(f"Unknown deploy task: {task_name}") + + task["run"] = { + "using": "run-task", + "checkout": False, + "command": command, + } + task["worker"].setdefault("env", {})["TASKCLUSTER_SECRET"] = secret + task.setdefault("scopes", []).append(f"secrets:get:{secret}") + + yield task diff --git a/taskcluster/kinds/deploy/kind.yml b/taskcluster/kinds/deploy/kind.yml new file mode 100644 index 000000000..0fc22fa52 --- /dev/null +++ b/taskcluster/kinds/deploy/kind.yml @@ -0,0 +1,54 @@ +loader: taskgraph.loader.transform:loader +transforms: + - code_review_taskgraph.transforms.deploy + - taskgraph.transforms.run + - taskgraph.transforms.task +kind-dependencies: + - build + - frontend +tasks: + frontend: + description: Deploy frontend SPA to S3 + attributes: + run-on-tasks-for: + - github-push + run-on-git-branches: + - testing + - production + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + dependencies: + frontend: frontend-build + backend: + description: Deploy backend to Heroku + attributes: + run-on-tasks-for: + - github-push + run-on-git-branches: + - testing + - production + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + dependencies: + build: build-backend + integration: + description: Push integration Docker image + attributes: + run-on-tasks-for: + - github-push + run-on-git-branches: + - testing + - production + worker-type: b-linux + worker: + docker-image: { in-tree: taskboot } + max-run-time: 3600 + taskcluster-proxy: true + dependencies: + build: build-integration