Bug 2052985 - feat: add build_signing transform and artifact signing specs

src/mozilla_taskgraph/transforms/build_signing.py

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.dependencies import get_primary_dependency
+
+from mozilla_taskgraph.util.attributes import copy_attributes_from_dependent_job
+from mozilla_taskgraph.util.signed_artifacts import (
+    generate_specifications_of_artifacts_to_sign,
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_signed_routes(config, jobs):
+    """Add routes corresponding to the routes of the build task
+    this corresponds to, with .signed inserted, for all gecko.v2 routes"""
+
+    for job in jobs:
+        dep_job = get_primary_dependency(config, job)
+        enable_signing_routes = job.pop("enable-signing-routes", True)
+
+        job["routes"] = []
+        if dep_job.attributes.get("shippable") and enable_signing_routes:
+            for dep_route in dep_job.task.get("routes", []):
+                if not dep_route.startswith("index.gecko.v2"):
+                    continue
+                branch = dep_route.split(".")[3]
+                rest = ".".join(dep_route.split(".")[4:])
+                job["routes"].append(f"index.gecko.v2.{branch}.signed.{rest}")
+
+        yield job
+
+
+@transforms.add
+def define_upstream_artifacts(config, jobs):
+    for job in jobs:
+        dep_job = get_primary_dependency(config, job)
+
+        job.setdefault("attributes", {}).update(
+            copy_attributes_from_dependent_job(dep_job)
+        )
+
+        artifacts_specifications = generate_specifications_of_artifacts_to_sign(
+            config,
+            job,
+            keep_locale_template=False,
+            kind=config.kind,
+            dep_kind=dep_job.kind,
+        )
+
+        task_ref = f"<{dep_job.kind}>"
+        task_type = "build"
+        if "notarization" in dep_job.kind:
+            task_type = "scriptworker"
+
+        job["upstream-artifacts"] = [
+            {
+                "taskId": {"task-reference": task_ref},
+                "taskType": task_type,
+                "paths": spec["artifacts"],
+                "formats": spec["formats"],
+            }
+            for spec in artifacts_specifications
+        ]
+
+        yield job

src/mozilla_taskgraph/util/signed_artifacts.py

@@ -2,6 +2,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from taskgraph.util.taskcluster import get_artifact_path
+
+LANGPACK_SIGN_PLATFORMS = {  # set
+    "linux64-shippable",
+    "linux64-devedition",
+    "macosx64-shippable",
+    "macosx64-devedition",
+}
+
 
 def get_signed_artifacts(input, formats, behavior=None):
     """
@@ -19,3 +28,150 @@ def get_signed_artifacts(input, formats, behavior=None):
         artifacts.add(f"{input}.asc")
 
     return artifacts
+
+
+def is_partner_kind(kind):
+    if kind and kind.startswith(("release-partner", "release-eme-free")):
+        return True
+
+
+def is_notarization_kind(kind):
+    if kind and "notarization" in kind:
+        return True
+
+
+def generate_specifications_of_artifacts_to_sign(
+    config, job, keep_locale_template=True, kind=None, dep_kind=None
+):
+    """Return the list of artifact/format specifications to sign for ``job``.
+
+    Handles the desktop and source build platforms shared across Gecko
+    applications. Consumers with additional platforms (e.g. GeckoView on
+    Android) should handle those before delegating here.
+    """
+    build_platform = job["attributes"].get("build_platform")
+    use_stub = job["attributes"].get("stub-installer")
+    # Get locales to know if we want to sign ja-JP-mac langpack
+    locales = job["attributes"].get("chunk_locales", [])
+    if kind == "release-source-signing":
+        artifacts_specifications = [
+            {
+                "artifacts": [get_artifact_path(job, "source.tar.xz")],
+                "formats": ["gcp_prod_autograph_gpg"],
+            }
+        ]
+    # XXX: Mars aren't signed here (on any platform) because internals will be
+    # signed at after this stage of the release
+    elif "macosx" in build_platform:
+        langpack_formats = []
+        if is_notarization_kind(config.kind):
+            formats = ["apple_notarization_stacked"]
+            artifacts_specifications = [
+                {
+                    "artifacts": [
+                        get_artifact_path(job, "{locale}/target.tar.gz"),
+                        get_artifact_path(job, "{locale}/target.pkg"),
+                    ],
+                    "formats": formats,
+                }
+            ]
+        else:
+            # This task is mac-signing
+            if is_partner_kind(kind):
+                extension = "tar.gz"
+            else:
+                extension = "dmg"
+            artifacts_specifications = [
+                {
+                    "artifacts": [
+                        get_artifact_path(job, f"{{locale}}/target.{extension}")
+                    ],
+                    "formats": ["macapp", "gcp_prod_autograph_widevine"],
+                }
+            ]
+            langpack_formats = ["gcp_prod_autograph_langpack"]
+
+        if "ja-JP-mac" in locales and build_platform in LANGPACK_SIGN_PLATFORMS:
+            artifacts_specifications += [
+                {
+                    "artifacts": [
+                        get_artifact_path(job, "ja-JP-mac/target.langpack.xpi")
+                    ],
+                    "formats": langpack_formats,
+                }
+            ]
+    elif "win" in build_platform:
+        artifacts_specifications = [
+            {
+                "artifacts": [
+                    get_artifact_path(job, "{locale}/setup.exe"),
+                ],
+                "formats": ["gcp_prod_autograph_authenticode_202412"],
+            },
+            {
+                "artifacts": [
+                    get_artifact_path(job, "{locale}/target.zip"),
+                ],
+                "formats": [
+                    "gcp_prod_autograph_authenticode_202412",
+                    "gcp_prod_autograph_widevine",
+                ],
+            },
+        ]
+
+        if use_stub:
+            artifacts_specifications[0]["artifacts"] += [
+                get_artifact_path(job, "{locale}/setup-stub.exe")
+            ]
+    elif "linux" in build_platform:
+        artifacts_specifications = [
+            {
+                "artifacts": [get_artifact_path(job, "{locale}/target.tar.xz")],
+                "formats": ["gcp_prod_autograph_gpg", "gcp_prod_autograph_widevine"],
+            }
+        ]
+        dep_job = config.kind_dependencies_tasks[job["dependencies"][dep_kind]]
+        if build_platform in LANGPACK_SIGN_PLATFORMS and not dep_job.attributes.get(
+            "artifact-build"
+        ):
+            artifacts_specifications += [
+                {
+                    "artifacts": [
+                        get_artifact_path(job, "{locale}/target.langpack.xpi")
+                    ],
+                    "formats": ["gcp_prod_autograph_langpack"],
+                }
+            ]
+    else:
+        raise Exception("Platform not implemented for signing")
+
+    if not keep_locale_template:
+        artifacts_specifications = _strip_locale_template(artifacts_specifications)
+
+    if is_partner_kind(kind):
+        artifacts_specifications = _strip_widevine_for_partners(
+            artifacts_specifications
+        )
+
+    return artifacts_specifications
+
+
+def _strip_locale_template(artifacts_without_locales):
+    for spec in artifacts_without_locales:
+        for index, artifact in enumerate(spec["artifacts"]):
+            stripped_artifact = artifact.format(locale="")
+            stripped_artifact = stripped_artifact.replace("//", "/")
+            spec["artifacts"][index] = stripped_artifact
+
+    return artifacts_without_locales
+
+
+def _strip_widevine_for_partners(artifacts_specifications):
+    """Partner repacks should not resign that's previously signed for fear of breaking partial
+    updates
+    """
+    for spec in artifacts_specifications:
+        if "gcp_prod_autograph_widevine" in spec["formats"]:
+            spec["formats"].remove("gcp_prod_autograph_widevine")
+
+    return artifacts_specifications

test/transforms/test_build_signing.py

@@ -0,0 +1,80 @@
+from pprint import pprint
+
+from mozilla_taskgraph.transforms.build_signing import transforms
+
+from ..conftest import make_task
+
+
+def _make_dep(
+    routes=None, shippable=True, build_platform="win64-shippable", kind="build"
+):
+    return make_task(
+        f"{kind}-win",
+        kind=kind,
+        attributes={"build_platform": build_platform, "shippable": shippable},
+        task_def={"routes": routes or []},
+    )
+
+
+def _make_job(dep):
+    return {
+        "attributes": {"primary-dependency-label": dep.label},
+        "dependencies": {dep.kind: dep.label},
+    }
+
+
+def _run(run_transform, make_transform_config, dep, job):
+    config = make_transform_config(kind_dependencies_tasks={dep.label: dep})
+    result = run_transform(transforms, job, config=config)
+    assert len(result) == 1
+    pprint(result[0], indent=2)
+    return result[0]
+
+
+def test_signed_routes_and_upstream_artifacts(run_transform, make_transform_config):
+    dep = _make_dep(
+        routes=[
+            "index.gecko.v2.mozilla-central.latest.firefox.win64",
+            "index.some.other.route",
+        ]
+    )
+    task = _run(run_transform, make_transform_config, dep, _make_job(dep))
+
+    # gecko.v2 route gets ``.signed`` inserted; non-gecko routes are dropped
+    assert task["routes"] == [
+        "index.gecko.v2.mozilla-central.signed.latest.firefox.win64"
+    ]
+
+    ua = task["upstream-artifacts"]
+    assert ua[0]["taskId"] == {"task-reference": "<build>"}
+    assert ua[0]["taskType"] == "build"
+    assert ua[0]["paths"] == ["public/build/setup.exe"]
+    assert ua[1]["paths"] == ["public/build/target.zip"]
+
+
+def test_enable_signing_routes_false(run_transform, make_transform_config):
+    dep = _make_dep(routes=["index.gecko.v2.mozilla-central.latest.firefox.win64"])
+    job = _make_job(dep)
+    job["enable-signing-routes"] = False
+    task = _run(run_transform, make_transform_config, dep, job)
+    assert task["routes"] == []
+
+
+def test_non_shippable_dep_has_no_routes(run_transform, make_transform_config):
+    dep = _make_dep(
+        routes=["index.gecko.v2.mozilla-central.latest.firefox.win64"],
+        shippable=False,
+    )
+    task = _run(run_transform, make_transform_config, dep, _make_job(dep))
+    assert task["routes"] == []
+
+
+def test_notarization_dep_sets_scriptworker_task_type(
+    run_transform, make_transform_config
+):
+    dep = _make_dep(build_platform="macosx64-shippable", kind="build-mac-notarization")
+    task = _run(run_transform, make_transform_config, dep, _make_job(dep))
+    assert task["upstream-artifacts"][0]["taskType"] == "scriptworker"
+    assert task["upstream-artifacts"][0]["taskId"] == {
+        "task-reference": "<build-mac-notarization>"
+    }