From d436c321ed7bdb4a34b85a512326d3225c474d0f Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Fri, 4 Sep 2026 05:28:06 +0000 Subject: [PATCH] Bug 2035575 - Add desktop architecture support (Linux, macOS, Windows) to the fat AAR pipeline for `libsForTests` r=nalexander,firefox-build-system-reviewers,geckoview-reviewers The `libsForTests` JAR consumers need a complete set of native libraries for whichever host platform their JVM tests run on, and the fat AAR handled Android architectures only. It now also stages target.libmegazord_so_artifacts.zip from the desktop builds. Only a build with an in-tree app-services produces that archive, so the desktop paths are named explicitly rather than downloaded, and the Android fallback takes its AAR from the app-services build when one is in tree. Differential Revision: https://phabricator.services.mozilla.com/D292612 --- Makefile.in | 12 +- mobile/android/moz.configure | 19 +++ python/mozbuild/mozbuild/action/fat_aar.py | 110 +++++++++++++++--- .../mozbuild/test/action/test_fat_aar.py | 46 ++++++++ python/mozbuild/mozbuild/test/python.toml | 2 + 5 files changed, 171 insertions(+), 18 deletions(-) create mode 100644 python/mozbuild/mozbuild/test/action/test_fat_aar.py diff --git a/Makefile.in b/Makefile.in index 705b653d6b08..3e2a35caac08 100644 --- a/Makefile.in +++ b/Makefile.in @@ -69,11 +69,19 @@ recurse_artifact: ifdef MOZ_ANDROID_FAT_AAR_ARCHITECTURES recurse_android-fat-aar-artifact: - $(call py_action,fat_aar,$(MOZ_ANDROID_FAT_AAR_ARCHITECTURES) \ + $(call py_action,fat_aar,--android-architectures $(MOZ_ANDROID_FAT_AAR_ARCHITECTURES) \ $(if $(MOZ_ANDROID_FAT_AAR_ARMEABI_V7A),--armeabi-v7a $(MOZ_ANDROID_FAT_AAR_ARMEABI_V7A)) \ $(if $(MOZ_ANDROID_FAT_AAR_ARM64_V8A),--arm64-v8a $(MOZ_ANDROID_FAT_AAR_ARM64_V8A)) \ $(if $(MOZ_ANDROID_FAT_AAR_X86_64),--x86-64 $(MOZ_ANDROID_FAT_AAR_X86_64)) \ - --distdir $(abspath $(DIST)/fat-aar)) + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES),--desktop-architectures $(MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_LINUX_X86_64),--linux-x86-64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_LINUX_X86_64)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_LINUX_AARCH64),--linux-aarch64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_LINUX_AARCH64)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_DARWIN_X86_64),--darwin-x86-64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_DARWIN_X86_64)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_DARWIN_AARCH64),--darwin-aarch64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_DARWIN_AARCH64)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_WIN32_X86_64),--win32-x86-64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_WIN32_X86_64)) \ + $(if $(MOZ_ANDROID_FAT_AAR_DESKTOP_WIN32_AARCH64),--win32-aarch64 $(MOZ_ANDROID_FAT_AAR_DESKTOP_WIN32_AARCH64)) \ + --distdir $(abspath $(DIST)/fat-aar) \ + $(verbose_flag)) endif ifeq ($(MOZ_BUILD_APP),mobile/android) diff --git a/mobile/android/moz.configure b/mobile/android/moz.configure index 4832b4cf8710..cc5fee19690d 100644 --- a/mobile/android/moz.configure +++ b/mobile/android/moz.configure @@ -168,6 +168,25 @@ set_config( depends("MOZ_ANDROID_FAT_AAR_ARCHITECTURES")(lambda x: x), ) +option( + env="MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES", + nargs="*", + choices=( + "linux-x86-64", + "linux-aarch64", + "darwin-x86-64", + "darwin-aarch64", + "win32-x86-64", + "win32-aarch64", + ), + help='Comma-separated list of Desktop CPU architectures to include in fat AARS, like "linux-x86-64"', +) + +set_config( + "MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES", + depends("MOZ_ANDROID_FAT_AAR_DESKTOP_ARCHITECTURES")(lambda x: x), +) + # Template to check for a build-time host/bin program that is either produced as # part of the build or is fetched from a toolchain artifact. Also allows diff --git a/python/mozbuild/mozbuild/action/fat_aar.py b/python/mozbuild/mozbuild/action/fat_aar.py index ebdb63e955e9..77073361fc57 100644 --- a/python/mozbuild/mozbuild/action/fat_aar.py +++ b/python/mozbuild/mozbuild/action/fat_aar.py @@ -34,16 +34,37 @@ def print_copy_result(elapsed, destdir, result): ) -def _download_zip(distdir, arch): - # The mapping from Android CPU architecture to TC job is defined here, and the TC index - # lookup is mediated by python/mozbuild/mozbuild/artifacts.py and +def _artifact_job(arch, appservices_in_tree): + # The mapping from Android CPU architecture (as required for the + # Android-Gradle plugin) to TC job is defined here, and the TC index lookup + # is mediated by python/mozbuild/mozbuild/artifacts.py and # python/mozbuild/mozbuild/artifact_builds.py. + # + # Ditto for the mapping from Desktop CPU architecture (as required for JNA) + # to TC job. jobs = { "arm64-v8a": "android-aarch64-opt", "armeabi-v7a": "android-arm-opt", "x86_64": "android-x86_64-opt", + "darwin-aarch64": "macosx64-aarch64-opt", + "darwin-x86-64": "macosx64-opt", + "linux-x86-64": "linux64-opt", + "linux-aarch64": "linux64-aarch64-opt", + "win32-x86-64": "win64-opt", + "win32-aarch64": "win64-aarch64-opt", } + job = jobs[arch] + if appservices_in_tree and job.startswith("android-"): + # An in-tree app-services changes what the megazord contains, so the + # duplicated build is the one to take the AAR from. + job = job.replace("-opt", "-appservices-opt") + return job + + +def _download_zip(distdir, arch, artifact_filter): + job = _artifact_job(arch, buildconfig.substs.get("MOZ_APPSERVICES_IN_TREE")) + dest = mozpath.join(distdir, "input", arch) subprocess.check_call([ sys.executable, @@ -51,20 +72,26 @@ def _download_zip(distdir, arch): "artifact", "install", "--job", - jobs[arch], + job, "--distdir", dest, "--no-tests", "--no-process", "--artifact-filter", - "public/build/target.maven.zip", + artifact_filter, ]) - return mozpath.join(dest, "target.maven.zip") + return mozpath.join(dest, mozpath.basename(artifact_filter)) def fat_aar( - distdir, zip_paths, no_process=False, no_compatibility_check=False, verbose=False + distdir, + android_zip_paths, + desktop_zip_paths=None, + no_process=False, + no_compatibility_check=False, + verbose=False, ): + desktop_zip_paths = desktop_zip_paths or {} if no_process: print("Not processing architecture-specific artifact Maven AARs.") return 0 @@ -77,8 +104,10 @@ def fat_aar( # Collect multi-architecture inputs to the fat AAR. copier = FileCopier() - for arch, zip_path_opt in zip_paths.items(): - zip_path = zip_path_opt or _download_zip(distdir, arch) + for arch, android_zip_path in android_zip_paths.items(): + zip_path = android_zip_path or _download_zip( + distdir, arch, "public/build/target.maven.zip" + ) if verbose: print(f"Processing '{zip_path}' for architecture {arch}") # Map old non-architecture-specific path to new architecture-specific path. @@ -127,7 +156,9 @@ def fat_aar( for key, aar_path, aar_file, aar_prefix in aars: if verbose: - print(f"Processing '{key}' AAR '{aar_path}' for architecture {arch}") + print( + f"Processing '{key}' AAR '{aar_path}' for Android architecture {arch}" + ) jar_finder = JarFinder( aar_file.file.filename, JarReader(fileobj=aar_file.open()) @@ -218,6 +249,23 @@ def fat_aar( if not no_compatibility_check and (missing_arch_prefs or not_allowed): return 1 + # Process Desktop artifacts, if requested. + for arch, desktop_zip_path in desktop_zip_paths.items(): + if not desktop_zip_path: + print( + f"Disallowed: no archive was provided for Desktop architecture " + f"{arch}; Desktop app-services archives must be supplied " + "explicitly." + ) + return 1 + + zip_path = desktop_zip_path + if verbose: + print(f"Processing '{zip_path}' for Desktop architecture {arch}") + + for path, file in JarFinder(zip_path, JarReader(zip_path)): + copier.add(mozpath.join("desktop", "resources", arch, path), file) + output_dir = mozpath.join(distdir, "output") result = copier.copy(output_dir) @@ -234,12 +282,35 @@ def fat_aar( _ALL_ARCHS = ("armeabi-v7a", "arm64-v8a", "x86_64") +_ALL_DESKTOP_ARCHS = ( + "darwin-aarch64", + "darwin-x86-64", + "linux-x86-64", + "linux-aarch64", + "win32-x86-64", + "win32-aarch64", +) + + def main(argv): description = """Fetch and unpack architecture-specific Maven zips, verify cross-architecture compatibility, and ready inputs to an Android multi-architecture fat AAR build.""" parser = argparse.ArgumentParser(description=description) - parser.add_argument("architectures", metavar="arch", nargs="+", choices=_ALL_ARCHS) + parser.add_argument( + "--android-architectures", + metavar="android-arch", + nargs="+", + choices=_ALL_ARCHS, + required=True, + ) + parser.add_argument( + "--desktop-architectures", + metavar="desktop-arch", + nargs="*", + choices=_ALL_DESKTOP_ARCHS, + default=[], + ) parser.add_argument( "--no-process", action="store_true", help="Do not process Maven AARs." ) @@ -251,21 +322,28 @@ compatibility, and ready inputs to an Android multi-architecture fat AAR build." parser.add_argument("--distdir", required=True) parser.add_argument("--verbose", "-v", action="store_true", default=False) - for arch in _ALL_ARCHS: + for arch in (*_ALL_ARCHS, *_ALL_DESKTOP_ARCHS): command_line_flag = arch.replace("_", "-") parser.add_argument(f"--{command_line_flag}", dest=arch) args = parser.parse_args(argv) args_dict = vars(args) - zip_paths = {arch: args_dict.get(arch) for arch in args.architectures} + android_zip_paths = { + arch: args_dict.get(arch) for arch in args.android_architectures + } - if not zip_paths: - raise ValueError("You must provide at least one Maven zip!") + if not android_zip_paths: + raise ValueError("You must provide at least one Android Maven zip!") + + desktop_zip_paths = { + arch: args_dict.get(arch) for arch in args.desktop_architectures + } return fat_aar( args.distdir, - zip_paths, + android_zip_paths, + desktop_zip_paths, no_process=args.no_process, no_compatibility_check=args.no_compatibility_check, verbose=args.verbose, diff --git a/python/mozbuild/mozbuild/test/action/test_fat_aar.py b/python/mozbuild/mozbuild/test/action/test_fat_aar.py new file mode 100644 index 000000000000..2f6747886ff0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/test_fat_aar.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from unittest.mock import patch + +import mozunit + +from mozbuild.action.fat_aar import _artifact_job, fat_aar + + +class TestArtifactJob(unittest.TestCase): + def test_android_without_appservices(self): + self.assertEqual(_artifact_job("arm64-v8a", False), "android-aarch64-opt") + self.assertEqual(_artifact_job("armeabi-v7a", False), "android-arm-opt") + self.assertEqual(_artifact_job("x86_64", False), "android-x86_64-opt") + + def test_android_with_appservices(self): + """An in-tree app-services changes what the megazord contains.""" + self.assertEqual( + _artifact_job("arm64-v8a", True), "android-aarch64-appservices-opt" + ) + self.assertEqual( + _artifact_job("armeabi-v7a", True), "android-arm-appservices-opt" + ) + self.assertEqual( + _artifact_job("x86_64", True), "android-x86_64-appservices-opt" + ) + + def test_desktop_is_unaffected(self): + for arch in ("linux-x86-64", "darwin-x86-64", "win32-x86-64"): + self.assertEqual(_artifact_job(arch, False), _artifact_job(arch, True)) + + +class TestDesktopArchives(unittest.TestCase): + def test_missing_desktop_archive_is_rejected(self): + with patch("mozbuild.action.fat_aar._download_zip") as download: + self.assertEqual( + fat_aar("distdir", {}, desktop_zip_paths={"linux-x86-64": None}), 1 + ) + download.assert_not_called() + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/python.toml b/python/mozbuild/mozbuild/test/python.toml index 8692f4ce5c4e..dd0144068288 100644 --- a/python/mozbuild/mozbuild/test/python.toml +++ b/python/mozbuild/mozbuild/test/python.toml @@ -5,6 +5,8 @@ subsuite = "mozbuild" ["action/test_buildlist.py"] +["action/test_fat_aar.py"] + ["action/test_generate_archive_symbols.py"] ["action/test_html_fragment_preprocessor.py"]