Julien Cristau
7e409f0229
Rename the shippable-l10n-signing kind (and its transform module) to l10n-signing, and make it depend on both the shippable-l10n and l10n kinds so that the non-shippable l10n repacks (built on cheaper opt builds with the on-change locale set) get gpg- and widevine-signed too. This gives a fast iteration path for signing changes without waiting on a shippable build. The shippable l10n kind is chunked and carries chunk_locales; the non-shippable l10n kind is unchunked, so the signing transform falls back to all_locales. The only-for-attributes filter is dropped: it was already a no-op for the shippable-l10n set. The shippable/nightly release graph is unchanged; the only delta is the four new opt l10n-signing tasks (linux64, linux64-aarch64, win32, win64). Differential Revision: https://phabricator.services.mozilla.com/D323512
410 lines
16 KiB
Python
410 lines
16 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import shlex
|
|
import urllib.parse
|
|
|
|
import requests
|
|
from mozilla_version.gecko import GeckoVersion
|
|
from mozrelease.paths import getNightlyDir, getReleaseInstallerPath, getReleasesDir
|
|
from mozrelease.platforms import buildPlatform2ftp, updatePlatform2ftp
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from taskgraph.util.schema import resolve_keyed_by
|
|
|
|
transforms = TransformSequence()
|
|
|
|
|
|
@transforms.add
|
|
def skip_for_non_nightly(config, jobs):
|
|
"""Don't generate any jobs unless running as a nightly. Other code in this transform depends on nightly-specific parameters being set."""
|
|
if not config.params["release_history"]:
|
|
return
|
|
|
|
yield from jobs
|
|
|
|
|
|
@transforms.add
|
|
def add_build_target(config, jobs):
|
|
for job in jobs:
|
|
# checked before `linux64` to avoid `linux64-aarch64` ending up with
|
|
# `linux64` information
|
|
if job["attributes"]["build_platform"].startswith("linux64-aarch64"):
|
|
build_target = "Linux_aarch64-gcc3"
|
|
elif job["attributes"]["build_platform"].startswith("linux64"):
|
|
build_target = "Linux_x86_64-gcc3"
|
|
elif job["attributes"]["build_platform"].startswith("mac"):
|
|
build_target = "Darwin_x86_64-gcc3-u-i386-x86_64"
|
|
elif job["attributes"]["build_platform"].startswith("win32"):
|
|
build_target = "WINNT_x86-msvc"
|
|
# checked before `win64` to avoid `win64-aarch64` ending up with
|
|
# `win64` information
|
|
elif job["attributes"]["build_platform"].startswith("win64-aarch64"):
|
|
build_target = "WINNT_aarch64-msvc-aarch64"
|
|
elif job["attributes"]["build_platform"].startswith("win64"):
|
|
build_target = "WINNT_x86_64-msvc"
|
|
else:
|
|
raise Exception("couldn't detect build target")
|
|
|
|
job["attributes"]["build_target"] = build_target
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def skip_for_new_locales_and_platforms(config, jobs):
|
|
"""Don't generate any jobs for newly added locales or platforms that don't have `from` releases to test."""
|
|
for job in jobs:
|
|
locale = job["attributes"].get("locale", "en-US")
|
|
build_target = job["attributes"]["build_target"]
|
|
|
|
if locale not in config.params["release_history"].get(build_target, {}):
|
|
continue
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def resolve_keys(config, jobs):
|
|
for job in jobs:
|
|
for key in (
|
|
"cert-overrides",
|
|
"fetches.toolchain",
|
|
"archive-prefix",
|
|
"last-watershed",
|
|
):
|
|
resolve_keyed_by(
|
|
job,
|
|
key,
|
|
job["name"],
|
|
**{
|
|
"build-platform": job["attributes"]["build_platform"],
|
|
"project": config.params["project"],
|
|
"release-type": config.params["release_type"],
|
|
"locale": job["attributes"].get("locale", "en-US"),
|
|
"shipping-product": job["attributes"]["shipping_product"],
|
|
},
|
|
)
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def set_treeherder(config, jobs):
|
|
for job in jobs:
|
|
th = job.setdefault("treeherder", {})
|
|
attrs = job["attributes"]
|
|
attrs["locale"] = attrs.get("locale", "en-US")
|
|
|
|
th["platform"] = f"{attrs['build_platform']}/{attrs['build_type']}"
|
|
th["symbol"] = th["symbol"].format(**attrs)
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def adjust_locale_watershed(config, jobs):
|
|
"""Adjusts the `last-watershed` for locales that are newer than the last
|
|
general watershed. eg: if `last-watershed` is 72.0 but `sco` didn't ship
|
|
91.0, it will be adjusted to 91.0."""
|
|
|
|
# cache results to make sure no file is fetched more than once
|
|
locale_history_cache = {}
|
|
|
|
for job in jobs:
|
|
history_file = job.pop("locale-history-file")
|
|
if history_file in locale_history_cache:
|
|
locale_history = locale_history_cache[history_file]
|
|
else:
|
|
req = requests.get(history_file)
|
|
req.raise_for_status()
|
|
locale_history = req.json()
|
|
locale_history_cache[history_file] = locale_history
|
|
|
|
last_watershed = job["last-watershed"]
|
|
locale = job["attributes"].get("locale", "en-US")
|
|
channel = job["attributes"]["update-channel"]
|
|
# because we're always using the production version of this file, we need to
|
|
# rewrite the nightly channel on try. (release builds use the real channel
|
|
# names on try; no need to rewrite them.)
|
|
if channel == "nightly-try":
|
|
channel = "nightly"
|
|
|
|
if locale == "en-US":
|
|
# en-US never has different availability than the default
|
|
# `last-watershed`
|
|
yield job
|
|
continue
|
|
|
|
# locale does not exist in history; can't do anything
|
|
# most likely it is a brand new locale that hasn't had its first release yet
|
|
if not locale_history.get(locale, {}).get("first_release", {}).get(channel):
|
|
continue
|
|
|
|
first_release = locale_history[locale]["first_release"][channel]
|
|
watershed_version = GeckoVersion.parse(last_watershed["version"])
|
|
first_version = GeckoVersion.parse(first_release["version"])
|
|
# we must also check buildid for nightly; locales may be added with the same
|
|
# version but a newer buildid
|
|
if channel == "nightly":
|
|
if (
|
|
watershed_version <= first_version
|
|
and last_watershed["buildid"] < first_release["buildid"]
|
|
):
|
|
job["last-watershed"] = first_release
|
|
elif watershed_version < first_version:
|
|
job["last-watershed"] = first_release
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def add_to_installer(config, jobs):
|
|
"""Adds fetch entries for the "to" installer to fetches."""
|
|
for job in jobs:
|
|
locale = job["attributes"].get("locale", "en-US")
|
|
# en-US and l10n tasks have different upstream tasks, and different
|
|
# artifact names.
|
|
if locale == "en-US":
|
|
if "linux" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["build-signing"] = [
|
|
{"artifact": "target.tar.xz", "extract": False}
|
|
]
|
|
elif "mac" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["repackage"] = [{"artifact": "target.dmg"}]
|
|
elif "win" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["repackage"] = [{"artifact": "target.installer.exe"}]
|
|
else:
|
|
raise Exception(
|
|
"unsupported platform: {job['attributes']['build_platform']}!"
|
|
)
|
|
else: # noqa: PLR5501 -- this is more readable with a separate `else` block for l10n
|
|
if "linux" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["l10n-signing"] = [
|
|
{"artifact": f"{locale}/target.tar.xz", "extract": False}
|
|
]
|
|
elif "mac" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["repackage-l10n"] = [
|
|
{"artifact": f"{locale}/target.dmg"}
|
|
]
|
|
elif "win" in job["attributes"]["build_platform"]:
|
|
job["fetches"]["repackage-l10n"] = [
|
|
{"artifact": f"{locale}/target.installer.exe"}
|
|
]
|
|
else:
|
|
raise Exception(
|
|
"unsupported platform: {job['attributes']['build_platform']}!"
|
|
)
|
|
|
|
yield job
|
|
|
|
|
|
@transforms.add
|
|
def add_additional_fetches_and_command(config, jobs):
|
|
"""Adds fetch entries for the "from" installers and partial MARs."""
|
|
for job in jobs:
|
|
build_platform = job["attributes"]["build_platform"]
|
|
if build_platform.startswith("linux"):
|
|
platform = "linux"
|
|
installer_suffix = "tar.xz"
|
|
elif build_platform.startswith("mac"):
|
|
platform = "mac"
|
|
installer_suffix = "dmg"
|
|
elif build_platform.startswith("win"):
|
|
platform = "win"
|
|
installer_suffix = "installer.exe"
|
|
else:
|
|
raise Exception("couldn't detect platform specific variables")
|
|
|
|
# ideally, this attribute would be set on en-US jobs as well...but it's not, so we have to assume
|
|
locale = job["attributes"].get("locale", "en-US")
|
|
# the locale identifier is different for japanese depending on the
|
|
# platform...make sure we translate it for the updater download
|
|
linux_locale = "ja" if locale == "ja-JP-mac" else locale
|
|
build_target = job["attributes"]["build_target"]
|
|
product = job.pop("product")
|
|
brand = job["attributes"]["shipping_product"]
|
|
|
|
cmd = [
|
|
# add dmg tool location to the $PATH. this is not strictly necessary
|
|
# for non-mac tests, but it's harmless
|
|
"export PATH=$MOZ_FETCHES_DIR/dmg:$PATH &&",
|
|
# test runner
|
|
"/builds/worker/fetches/marannon/marannon",
|
|
# script that actually runs the tests - eventually to be replaced
|
|
# with native code
|
|
"tools/update-verify/release/common/check_updates.sh",
|
|
# platform - used to determine how to unpack builds
|
|
platform,
|
|
# "to" installer
|
|
f"/builds/worker/fetches/target.{installer_suffix}",
|
|
# "to" complete mar
|
|
"/builds/worker/fetches/target.complete.mar",
|
|
# directory containing partial mars
|
|
"/builds/worker/fetches",
|
|
# locale
|
|
locale,
|
|
# channel
|
|
job["attributes"]["update-channel"],
|
|
# product name
|
|
product,
|
|
# artifact dir
|
|
"/builds/worker/artifacts",
|
|
]
|
|
|
|
cert_overrides = job.pop("cert-overrides")
|
|
if cert_overrides:
|
|
cmd.extend([
|
|
# directory containing mar certificates
|
|
# note we use versions from tools/update-verify, not the ones
|
|
# in toolkit/mozapps/update/updater, which are not precisely
|
|
# the same size, and injecting them would corrupt the binary
|
|
"--cert-dir",
|
|
"tools/update-verify/release/mar_certs",
|
|
])
|
|
for override in cert_overrides:
|
|
cmd.extend(["--cert-override", shlex.quote(override)])
|
|
|
|
archive_prefix = job.pop("archive-prefix")
|
|
|
|
tested_identifiers = set()
|
|
fetches = []
|
|
for mar, info in config.params["release_history"][build_target][locale].items():
|
|
if locale == "en-US":
|
|
mar_prefix = ""
|
|
else:
|
|
mar_prefix = f"{locale}/"
|
|
|
|
fetches.append({"artifact": f"{mar_prefix}{mar}"})
|
|
|
|
# URLs for nightlies and releases are significantly different; they
|
|
# can't be constructed in the same manner
|
|
if "nightly" in info["mar_url"]:
|
|
# parameters give us the complete MAR url. installers are found right
|
|
# beside them
|
|
base_url = info["mar_url"].split(".complete.mar")[0]
|
|
identifier = info["buildid"]
|
|
|
|
# regardless of what platform is under test, we perform the tests
|
|
# with the 64-bit linux updater
|
|
linux64_info = config.params["release_history"]["Linux_x86_64-gcc3"][
|
|
linux_locale
|
|
][mar]
|
|
|
|
from_installer_url = f"{base_url}.{installer_suffix}"
|
|
linux64_installer_url = linux64_info["mar_url"].replace(
|
|
".complete.mar", ".tar.xz"
|
|
)
|
|
else:
|
|
identifier = info["previousVersion"]
|
|
# `info['product']` actually contains the brand name (that is:
|
|
# Firefox for nightly/beta/release/esr, and Devedition
|
|
# for devedition; this is a necessary distinction for
|
|
# URL generation (devedition is in a `devedition` directory)
|
|
# but uses `firefox`/`Firefox` in filenames
|
|
from_installer_url = _get_release_installer_url(
|
|
brand,
|
|
product,
|
|
build_target,
|
|
locale,
|
|
info["previousVersion"],
|
|
archive_prefix,
|
|
)
|
|
linux64_installer_url = _get_release_installer_url(
|
|
brand,
|
|
product,
|
|
"Linux_x86_64-gcc3",
|
|
linux_locale,
|
|
info["previousVersion"],
|
|
archive_prefix,
|
|
)
|
|
|
|
# installers and updaters are fetched from URLs (not upstream tasks); we simply
|
|
# inject these into the task for the payload to deal with
|
|
cmd.append("--from")
|
|
cmd.append(
|
|
shlex.quote(
|
|
f"{identifier}|{from_installer_url}|{linux64_installer_url}|{mar}"
|
|
)
|
|
)
|
|
tested_identifiers.add(identifier)
|
|
|
|
last_watershed = job.pop("last-watershed")
|
|
if config.params["release_type"] == "nightly":
|
|
watershed_identifier = last_watershed["buildid"]
|
|
# don't add a last watershed test if the build has already been added for
|
|
# testing earlier; this case comes up for newly added locales
|
|
if watershed_identifier not in tested_identifiers:
|
|
nightly_dir = getNightlyDir(
|
|
product,
|
|
last_watershed["buildid"],
|
|
locale,
|
|
config.params["project"],
|
|
protocol="https",
|
|
server=archive_prefix,
|
|
)
|
|
version = last_watershed["version"]
|
|
platform = buildPlatform2ftp(build_platform)
|
|
linux_suffix = "tar.xz"
|
|
if GeckoVersion.parse(version) < GeckoVersion.parse("135.0a1"):
|
|
installer_suffix = installer_suffix.replace("xz", "bz2")
|
|
linux_suffix = "tar.bz2"
|
|
from_installer_url = f"{nightly_dir}/{product}-{version}.{locale}.{platform}.{installer_suffix}"
|
|
linux64_installer_url = f"{nightly_dir}/{product}-{version}.{linux_locale}.linux-x86_64.{linux_suffix}"
|
|
cmd.append("--from")
|
|
cmd.append(
|
|
shlex.quote(
|
|
f"{watershed_identifier}|{from_installer_url}|{linux64_installer_url}"
|
|
)
|
|
)
|
|
else:
|
|
watershed_identifier = last_watershed["version"]
|
|
# don't add a last watershed test if the build has already been added for
|
|
# testing earlier; this case comes up for newly added locales
|
|
if watershed_identifier not in tested_identifiers:
|
|
from_installer_url = _get_release_installer_url(
|
|
brand,
|
|
product,
|
|
build_target,
|
|
locale,
|
|
last_watershed["version"],
|
|
archive_prefix,
|
|
)
|
|
linux64_installer_url = _get_release_installer_url(
|
|
brand,
|
|
product,
|
|
"Linux_x86_64-gcc3",
|
|
linux_locale,
|
|
last_watershed["version"],
|
|
archive_prefix,
|
|
)
|
|
cmd.append("--from")
|
|
cmd.append(
|
|
shlex.quote(
|
|
f"{watershed_identifier}|{from_installer_url}|{linux64_installer_url}"
|
|
)
|
|
)
|
|
|
|
job["fetches"]["partials-signing"] = fetches
|
|
job["run"]["command"] = " ".join(cmd)
|
|
|
|
yield job
|
|
|
|
|
|
def _get_release_installer_url(
|
|
brand, product, build_target, locale, from_version, archive_prefix
|
|
):
|
|
ftp_platform = updatePlatform2ftp(build_target)
|
|
releases_dir = getReleasesDir(
|
|
brand, from_version, protocol="https", server=archive_prefix
|
|
)
|
|
path = urllib.parse.quote(
|
|
# although this function calls the second argument `brandName`, it
|
|
# is actually the product name as used in filenames, eg: Firefox
|
|
# using the `brand` given to us would cause errors for Devedition, which
|
|
# uses Firefox in its filenames 🙃
|
|
getReleaseInstallerPath(
|
|
product, product.capitalize(), from_version, ftp_platform, locale
|
|
)
|
|
)
|
|
return f"{releases_dir}/{path}"
|