Merge pull request #82 from mozilla-releng/hneiva/taskgraph-ci

feat(ci): Use taskgraph for generating CI tasks

taskcluster/config.yml

@@ -0,0 +1,22 @@
+---
+trust-domain: "mozilla"
+task-priority: low
+
+taskgraph:
+  cached-task-prefix: "mozilla.v2.redo"
+  repositories:
+    redo:
+      name: "redo"
+
+workers:
+  aliases:
+    images:
+      provisioner: '{trust-domain}-{level}'
+      implementation: docker-worker
+      os: linux
+      worker-type: '{alias}-gcp'
+    linux:
+      provisioner: '{trust-domain}-t'
+      implementation: docker-worker
+      os: linux
+      worker-type: 't-{alias}-large-gcp'

taskcluster/docker/python/Dockerfile

@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#%ARG PYTHON_VERSION
+FROM python:$PYTHON_VERSION-slim
+LABEL maintainer="Release Engineering <release@mozilla.com>"
+
+# Add worker user
+RUN mkdir /builds && \
+    useradd -d /builds/worker -s /bin/bash -m worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+# %include-run-task
+
+RUN apt-get update \
+ && apt-get install -y --reinstall ca-certificates \
+ && apt-get install -y --force-yes --no-install-recommends \
+    build-essential \
+    mercurial \
+    git
+
+RUN pip install tox
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

taskcluster/kinds/docker-image/kind.yml

@@ -0,0 +1,29 @@
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+tasks:
+    py38:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.8"
+    py39:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.9"
+    py310:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.10"
+    py311:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.11"
+    py312:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.12"

taskcluster/kinds/pr/kind.yml

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - test
+    - docker-image
+
+transforms:
+    - taskgraph.transforms.code_review:transforms
+    - taskgraph.transforms.task:transforms
+
+tasks:
+    complete:
+        description: PR Summary Task
+        run-on-tasks-for: [github-pull-request]
+        worker-type: succeed

taskcluster/kinds/test/kind.yml

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - docker-image
+
+transforms:
+    - redo_taskgraph.transforms.python_version:transforms
+    - taskgraph.transforms.run:transforms
+    - taskgraph.transforms.task:transforms
+
+task-defaults:
+    description: "{name} py{python_version}"
+    run-on-tasks-for:
+        - "github-pull-request"
+        - "github-push"
+    attributes:
+        code-review: true
+    worker-type: linux
+    worker:
+        docker-image: {in-tree: 'py{python_version}'}
+        max-run-time: 1800
+    run:
+        using: run-task
+        cache-dotcache: false
+        cwd: '{checkout}'
+        command:
+            - sh
+            - -lxce
+            - >-
+              tox -e py{python_version}
+
+tasks:
+    tox:
+        python-versions: [38, 39, 310, 311, 312]

taskcluster/redo_taskgraph/transforms/python_version.py

@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Create a task per python-version
+"""
+
+from copy import deepcopy
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+def _replace_string(obj, subs):
+    if isinstance(obj, dict):
+        return {k: v.format(**subs) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        for c in range(0, len(obj)):
+            obj[c] = obj[c].format(**subs)
+    else:
+        obj = obj.format(**subs)
+    return obj
+
+
+def _resolve_replace_string(item, field, subs):
+    # largely from resolve_keyed_by
+    container, subfield = item, field
+    while "." in subfield:
+        f, subfield = subfield.split(".", 1)
+        if f not in container:
+            return item
+        container = container[f]
+        if not isinstance(container, dict):
+            return item
+
+    if subfield not in container:
+        return item
+
+    container[subfield] = _replace_string(container[subfield], subs)
+    return item
+
+
+@transforms.add
+def set_script_name(config, tasks):
+    for task in tasks:
+        task.setdefault("attributes", {}).update(
+            {
+                "script-name": task["name"],
+            }
+        )
+        yield task
+
+
+@transforms.add
+def tasks_per_python_version(config, tasks):
+    fields = [
+        "description",
+        "docker-repo",
+        "run.command",
+        "worker.command",
+        "worker.docker-image",
+    ]
+    for task_raw in tasks:
+        for python_version in task_raw.pop("python-versions"):
+            task = deepcopy(task_raw)
+            subs = {"name": task["name"], "python_version": python_version}
+            for field in fields:
+                _resolve_replace_string(task, field, subs)
+            task["attributes"]["python-version"] = python_version
+            yield task
+
+
+@transforms.add
+def update_name_with_python_version(config, tasks):
+    for task in tasks:
+        task["name"] = "{}-python{}".format(task["name"], task["attributes"]["python-version"])
+        yield task

taskcluster/requirements.in

@@ -0,0 +1 @@
+taskcluster-taskgraph>=7.0.0