Migrate CI from hand-crafted .taskcluster.yml to taskgraph

backend/code_review_backend/issues/management/commands/load_issues.py

@@ -8,12 +8,12 @@
 import tempfile
 from urllib.parse import urlparse
 
-import taskcluster
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from requests.exceptions import HTTPError
 
+import taskcluster
 from code_review_backend.issues.compare import detect_new_for_revision
 from code_review_backend.issues.models import Issue, IssueLink, Repository
 

bot/code_review_bot/__init__.py

@@ -12,11 +12,11 @@
 import requests
 import structlog
 from libmozdata.phabricator import LintResult, UnitResult, UnitResultState
-from taskcluster.helper import TaskclusterConfig
 
 from code_review_bot.config import settings
 from code_review_bot.stats import InfluxDb
 from code_review_bot.tasks.base import AnalysisTask
+from taskcluster.helper import TaskclusterConfig
 
 logger = structlog.get_logger(__name__)
 

bot/code_review_bot/workflow.py

@@ -8,7 +8,6 @@
 
 import structlog
 from libmozdata.phabricator import BuildState, PhabricatorAPI
-from taskcluster.utils import stringDate
 
 from code_review_bot import Level, stats
 from code_review_bot.analysis import (
@@ -34,6 +33,7 @@
 from code_review_bot.tasks.docupload import DocUploadTask
 from code_review_bot.tasks.lint import MozLintTask
 from code_review_bot.tasks.tgdiff import TaskGraphDiffTask
+from taskcluster.utils import stringDate
 
 logger = structlog.get_logger(__name__)
 

bot/tests/test_tools.py

@@ -3,9 +3,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import pytest
-from taskcluster.helper import TaskclusterConfig
 
 from code_review_bot.tools.log import remove_color_codes
+from taskcluster.helper import TaskclusterConfig
 
 
 def test_taskcluster_service():

bot/tools/fix_missing.py

@@ -8,9 +8,9 @@
 from datetime import datetime, timedelta
 
 import requests
-from taskcluster.helper import TaskclusterConfig
 
 from code_review_bot.config import GetAppUserAgent
+from taskcluster.helper import TaskclusterConfig
 
 TREEHERDER_PUSH_URL = "https://treeherder.mozilla.org/api/project/try/push/"
 TREEHERDER_JOBS_URL = "https://treeherder.mozilla.org/api/jobs/"

integration/run.py

@@ -12,6 +12,7 @@
 
 import structlog
 import yaml
+
 from taskcluster.helper import TaskclusterConfig
 
 taskcluster = TaskclusterConfig("https://community-tc.services.mozilla.com")

taskcluster/code_review_taskgraph/__init__.py

@@ -0,0 +1,10 @@
+from importlib import import_module
+
+
+def register(graph_config):
+    _import_modules(["parameters"])
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(f".{module}", package=__name__)

taskcluster/code_review_taskgraph/parameters.py

@@ -0,0 +1,51 @@
+import fnmatch
+
+from taskgraph.parameters import extend_parameters_schema
+from taskgraph.target_tasks import register_target_task
+from voluptuous import Optional
+
+extend_parameters_schema(
+    {
+        Optional("channel"): str,
+        Optional("backend_url"): str,
+    },
+)
+
+
+@register_target_task("default")
+def target_tasks_default(full_task_graph, parameters, graph_config):
+    head_ref = parameters["head_ref"]
+    for prefix in ("refs/heads/", "refs/tags/"):
+        if head_ref.startswith(prefix):
+            head_ref = head_ref[len(prefix) :]
+            break
+
+    def should_run(task):
+        if parameters["tasks_for"] not in task.attributes.get(
+            "run-on-tasks-for", [parameters["tasks_for"]]
+        ):
+            return False
+        run_on_branches = task.attributes.get("run-on-git-branches")
+        if run_on_branches is not None:
+            return any(
+                fnmatch.fnmatch(head_ref, pattern) for pattern in run_on_branches
+            )
+        return True
+
+    return [label for label, task in full_task_graph.tasks.items() if should_run(task)]
+
+
+def decision_parameters(graph_config, parameters):
+    head_ref = parameters["head_ref"]
+    if head_ref.startswith("refs/heads/"):
+        head_ref = head_ref[len("refs/heads/") :]
+
+    if head_ref == "testing":
+        parameters["channel"] = "testing"
+        parameters["backend_url"] = "https://api.code-review.testing.moz.tools"
+    elif head_ref == "production":
+        parameters["channel"] = "production"
+        parameters["backend_url"] = "https://api.code-review.moz.tools"
+    else:
+        parameters["channel"] = "dev"
+        parameters["backend_url"] = "https://api.code-review.testing.moz.tools"

taskcluster/code_review_taskgraph/transforms/build.py

@@ -0,0 +1,71 @@
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+_CHECKOUT = "/builds/worker/checkouts/vcs"
+
+
+@transforms.add
+def add_build_config(config, tasks):
+    for task in tasks:
+        params = config.params
+        head_rev = params["head_rev"]
+        channel = params.get("channel", "dev")
+        head_ref = params["head_ref"]
+        for prefix in ("refs/heads/", "refs/tags/"):
+            if head_ref.startswith(prefix):
+                head_ref = head_ref[len(prefix) :]
+                break
+
+        task_name = task["name"]
+
+        if task_name == "bot":
+            command = (
+                f"taskboot --target {_CHECKOUT}"
+                f" build --image mozilla/code-review"
+                f" --tag {channel} --tag {head_rev}"
+                " --write /bot.tar bot/docker/Dockerfile"
+            )
+        elif task_name == "backend":
+            command = (
+                f'python3 -c "'
+                f"import json, os; "
+                f"open('{_CHECKOUT}/backend/code_review_backend/version.json', 'w')"
+                f".write(json.dumps({{'commit': '{head_rev}', 'version': '{head_ref}', "
+                f"'source': os.environ['CODE_REVIEW_HEAD_REPOSITORY'], "
+                f"'build': os.environ.get('TASKCLUSTER_ROOT_URL', '') + '/tasks/' + os.environ.get('TASK_ID', '')}}))"
+                f'" &&'
+                f" taskboot --target {_CHECKOUT}"
+                f" build --image mozilla/code-review"
+                f" --tag {channel} --tag {head_rev}"
+                " --write /backend.tar backend/Dockerfile"
+            )
+        elif task_name == "integration":
+            command = (
+                f"taskboot --target {_CHECKOUT}"
+                f" build --image mozilla/code-review"
+                f" --tag integration-{channel} --tag integration-{head_rev}"
+                " --write /integration.tar integration/docker/Dockerfile"
+            )
+        else:
+            raise ValueError(f"Unknown build task: {task_name}")
+
+        task["run"] = {
+            "using": "run-task",
+            "cwd": "{checkout}",
+            "command": command,
+        }
+
+        if params["tasks_for"] == "github-pull-request":
+            index_prefix = "code-review-pr"
+        else:
+            index_prefix = "code-review"
+
+        task.setdefault("routes", []).extend(
+            [
+                f"index.code-analysis.v2.{index_prefix}.{task_name}.revision.{head_rev}",
+                f"index.code-analysis.v2.{index_prefix}.{task_name}.branch.{head_ref}",
+            ]
+        )
+
+        yield task

taskcluster/config.yml

@@ -0,0 +1,21 @@
+---
+trust-domain: code-analysis
+task-priority: low
+taskgraph:
+  register: code_review_taskgraph:register
+  decision-parameters: code_review_taskgraph.parameters:decision_parameters
+  repositories:
+    code_review:
+      name: code-review
+workers:
+  aliases:
+    b-linux:
+      provisioner: "{trust-domain}-{level}"
+      implementation: docker-worker
+      os: linux
+      worker-type: linux-gw-gcp
+    images:
+      provisioner: "{trust-domain}-{level}"
+      implementation: docker-worker
+      os: linux
+      worker-type: linux-gcp

taskcluster/docker/node/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:16-alpine
+
+# Add worker user
+RUN mkdir -p /builds/worker/artifacts && \
+    delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \
+    deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \
+    addgroup -g 1000 worker && \
+    adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \
+    chown -R worker:worker /builds/worker
+
+RUN apk add --no-cache git bash lcms2-dev libpng-dev autoconf build-base coreutils shadow python3
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/docker/python/Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.12-slim
+
+# Add worker user
+RUN mkdir -p /builds/worker/artifacts && \
+    groupadd --gid 1000 worker && \
+    useradd --uid 1000 --gid worker --home-dir /builds/worker --shell /bin/bash worker && \
+    chown -R worker:worker /builds/worker
+
+# Install build deps (needed for mercurial setup)
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+        git \
+        gcc \
+        python3-dev \
+        libatomic1 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install mercurial
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==7.2
+
+# Clone version-control-tools
+RUN hg clone -r 5b6e8298d035 https://hg.mozilla.org/hgcustom/version-control-tools /src/version-control-tools/ && \
+    rm -rf /src/version-control-tools/.hg \
+           /src/version-control-tools/ansible \
+           /src/version-control-tools/docs \
+           /src/version-control-tools/testing
+
+# %include-run-task
+
+# Cleanup build-only deps
+RUN apt-get purge -y gcc python3-dev && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/docker/taskboot/Dockerfile

@@ -0,0 +1,21 @@
+FROM mozilla/taskboot:0.4.5
+
+RUN apk add --no-cache bash coreutils shadow
+
+RUN mkdir -p /builds/worker/artifacts && \
+    delgroup $(grep ":1000:" /etc/group | cut -d: -f1) 2>/dev/null || true && \
+    deluser $(grep ":1000:" /etc/passwd | cut -d: -f1) 2>/dev/null || true && \
+    addgroup -g 1000 worker && \
+    adduser -u 1000 -G worker -h /builds/worker -s /bin/bash -D worker && \
+    chown -R worker:worker /builds/worker
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+CMD ["/bin/bash"]

taskcluster/kinds/build/kind.yml

@@ -0,0 +1,60 @@
+loader: taskgraph.loader.transform:loader
+transforms:
+  - code_review_taskgraph.transforms.build
+  - taskgraph.transforms.run
+  - taskgraph.transforms.task
+kind-dependencies:
+  - lint
+  - test
+tasks:
+  bot:
+    description: Build bot Docker image
+    attributes:
+      run-on-tasks-for:
+        - github-pull-request
+        - github-push
+    worker-type: b-linux
+    worker:
+      docker-image: { in-tree: taskboot }
+      max-run-time: 3600
+      artifacts:
+        - type: file
+          name: public/code-review-bot.tar.zst
+          path: /bot.tar.zst
+    dependencies:
+      lint: lint-pre-commit
+      test: test-bot
+  backend:
+    description: Build backend Docker image
+    attributes:
+      run-on-tasks-for:
+        - github-pull-request
+        - github-push
+    worker-type: b-linux
+    worker:
+      docker-image: { in-tree: taskboot }
+      max-run-time: 3600
+      artifacts:
+        - type: file
+          name: public/code-review-backend.tar.zst
+          path: /backend.tar.zst
+    dependencies:
+      lint: lint-pre-commit
+      test: test-backend
+  integration:
+    description: Build integration Docker image
+    attributes:
+      run-on-tasks-for:
+        - github-pull-request
+        - github-push
+    worker-type: b-linux
+    worker:
+      docker-image: { in-tree: taskboot }
+      max-run-time: 3600
+      artifacts:
+        - type: file
+          name: public/code-review-integration.tar.zst
+          path: /integration.tar.zst
+    dependencies:
+      lint: lint-pre-commit
+      test: test-integration

taskcluster/kinds/docker-image/kind.yml

@@ -0,0 +1,9 @@
+loader: taskgraph.loader.transform:loader
+transforms:
+  - taskgraph.transforms.docker_image
+  - taskgraph.transforms.cached_tasks
+  - taskgraph.transforms.task
+tasks:
+  python: {}
+  taskboot: {}
+  node: {}