From 9546d398346184f543e4066bca055ee41c2050c2 Mon Sep 17 00:00:00 2001 From: Alex Hochheiden Date: Mon, 1 Jun 2026 18:50:03 +0000 Subject: [PATCH] Bug 2038789 - Add `extra_deps` to `GeneratedFile` for hidden runtime prereqs, and use it for `application.ini` r=firefox-build-system-reviewers,sergesanspaille Some `GeneratedFile` scripts open additional files at runtime (e.g. #include @TOPOBJDIR@/...), so backends need a way to declare those as prerequisites without passing them as positional args. Also normalizes the `_target_per_file` key to an absolute path so `GENERATED_FILES` dependencies that cross directory boundaries actually resolve. Differential Revision: https://phabricator.services.mozilla.com/D302161 --- Makefile.in | 2 -- build/moz.build | 5 +++ build/templates.mozbuild | 2 ++ config/faster/rules.mk | 6 ---- python/mozbuild/mozbuild/backend/make.py | 11 ++++++- .../mozbuild/backend/recursivemake.py | 8 ++--- python/mozbuild/mozbuild/frontend/context.py | 20 ++++++++--- python/mozbuild/mozbuild/frontend/data.py | 3 ++ python/mozbuild/mozbuild/frontend/emitter.py | 11 +++++++ .../consumer/generate-consumer.py | 0 .../consumer/input-data | 0 .../consumer/moz.build | 9 +++++ .../consumer/source-extra | 0 .../data/generated-files-extra-deps/moz.build | 7 ++++ .../producer/generate-producer.py | 0 .../producer/moz.build | 7 ++++ .../test/backend/test_recursivemake.py | 33 +++++++++++++++++++ .../moz.build | 9 +++++ .../script.py | 0 .../data/generated-files-extra-deps/moz.build | 9 +++++ .../data/generated-files-extra-deps/script.py | 0 .../mozbuild/test/frontend/test_emitter.py | 23 +++++++++++++ 22 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/generate-consumer.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/input-data create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/source-extra create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/generate-producer.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/script.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/script.py diff --git a/Makefile.in b/Makefile.in index 684a55e1f866..911b1098c538 100644 --- a/Makefile.in +++ b/Makefile.in @@ -24,8 +24,6 @@ endif endif source-repo.h: $(MDDEPDIR)/source-repo.h.stub buildid.h: $(MDDEPDIR)/buildid.h.stub -# Add explicit dependencies that moz.build can't declare yet. -build/$(MDDEPDIR)/application.ini.stub: source-repo.h buildid.h BUILD_BACKEND_FILES := $(addprefix backend.,$(addsuffix Backend,$(BUILD_BACKENDS))) diff --git a/build/moz.build b/build/moz.build index 60cf9a852efa..0182121940f1 100644 --- a/build/moz.build +++ b/build/moz.build @@ -101,6 +101,11 @@ if CONFIG["MOZ_APP_BASENAME"]: script="../python/mozbuild/mozbuild/action/preprocessor.py", entry_point="generate", inputs=["application.ini.in"], + # `application.ini.in` does `#include @TOPOBJDIR@/source-repo.h` + # and `#include @TOPOBJDIR@/buildid.h`. The preprocessor opens + # those files at runtime. Declare them as extra_deps so backends + # wire the order without passing them as preprocessor inputs. + extra_deps=["!/source-repo.h", "!/buildid.h"], flags=[ "-D%s=%s" % (k, "1" if v is True else v) for k, v in sorted(appini_defines.items(), key=lambda t: t[0]) diff --git a/build/templates.mozbuild b/build/templates.mozbuild index da3bd62a2285..34e66ff2c658 100644 --- a/build/templates.mozbuild +++ b/build/templates.mozbuild @@ -248,6 +248,7 @@ def GeneratedFile(name, *names, **kwargs): script = kwargs.pop("script", None) entry_point = kwargs.pop("entry_point", None) inputs = kwargs.pop("inputs", []) + extra_deps = kwargs.pop("extra_deps", []) flags = kwargs.pop("flags", []) force = kwargs.pop("force", False) if kwargs: @@ -269,6 +270,7 @@ def GeneratedFile(name, *names, **kwargs): if script and entry_point: generated_file.script = script + ":" + entry_point generated_file.inputs = inputs + generated_file.extra_deps = extra_deps generated_file.flags = flags generated_file.force = force diff --git a/config/faster/rules.mk b/config/faster/rules.mk index 79833a10348e..0429edc9f417 100644 --- a/config/faster/rules.mk +++ b/config/faster/rules.mk @@ -87,9 +87,3 @@ $(addprefix install-,$(INSTALL_MANIFESTS)): install-%: $(addprefix $(TOPOBJDIR)/ -DAB_CD=en-US \ $(ACDEFINES) \ install_$(subst /,_,$*) - -# ============================================================================ -# Below is a set of additional dependencies and variables used to build things -# that are not supported by data in moz.build. - -$(TOPOBJDIR)/build/.deps/application.ini.stub: $(TOPOBJDIR)/buildid.h $(TOPOBJDIR)/source-repo.h diff --git a/python/mozbuild/mozbuild/backend/make.py b/python/mozbuild/mozbuild/backend/make.py index 75fc7689252c..d0b79056db3b 100644 --- a/python/mozbuild/mozbuild/backend/make.py +++ b/python/mozbuild/mozbuild/backend/make.py @@ -69,6 +69,14 @@ class MakeBackend(CommonBackend): else: inputs = [] + # extra_deps are make prerequisites only (not passed to the script as + # positional args). Use this for dependencies the recipe needs to + # exist on disk but that the script discovers itself at run time + # (e.g. preprocessor `#include @TOPOBJDIR@/foo.h`). + extra_deps = [ + self._format_generated_file_input_name(d, obj) for d in obj.extra_deps + ] + force = "" if obj.force: force = " FORCE" @@ -104,7 +112,7 @@ class MakeBackend(CommonBackend): ret.append( ( - """{stub}: {script}{inputs}{backend}{force} + """{stub}: {script}{inputs}{extra_deps}{backend}{force} \t$(REPORT_BUILD) \t$(call py_action,file_generate {output},{locale}{script} """ # wrap for E501 """{method} {output} {dep_file} {stub}{inputs}{flags}) @@ -115,6 +123,7 @@ class MakeBackend(CommonBackend): output=first_output, dep_file=dep_file, inputs=" " + " ".join(inputs) if inputs else "", + extra_deps=" " + " ".join(extra_deps) if extra_deps else "", flags=( " " + " ".join(shell_quote(f) for f in obj.flags) if obj.flags diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index cd29bbb446f7..28f39c28a225 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -550,15 +550,15 @@ class RecursiveMakeBackend(MakeBackend): reldir = mozpath.relpath(obj.objdir, backend_file.objdir) if not reldir: for out in obj.outputs: - target = mozpath.join(obj.relobjdir, out) + target = mozpath.join(obj.objdir, out) assert target not in self._target_per_file self._target_per_file[target] = (obj.relobjdir, tier) - for input in obj.inputs: - if isinstance(input, ObjDirPath): + for dep in chain(obj.inputs, obj.extra_deps): + if isinstance(dep, ObjDirPath): self._post_process_dependencies.append(( obj.relobjdir, tier, - input, + dep, )) # For generated files that we handle in the top-level backend file, # we want to have a `directory/tier` target depending on the file. diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index b91ca6ce902e..533a87c7a3a7 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -1197,6 +1197,7 @@ SchedulingComponents = ContextDerivedTypedRecord( GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({ "script": str, "inputs": list, + "extra_deps": list, "force": bool, "flags": list, }) @@ -1521,13 +1522,14 @@ VARIABLES = { Unless you have a reason not to, use the GeneratedFile template rather than referencing GENERATED_FILES directly. The GeneratedFile template has all the same arguments as the attributes listed below (``script``, - ``inputs``, ``flags``, ``force``), plus an additional ``entry_point`` - argument to specify a particular function to run in the given script. + ``inputs``, ``extra_deps``, ``flags``, ``force``), plus an additional + ``entry_point`` argument to specify a particular function to run in + the given script. This variable contains a list of files for the build system to generate at export time. The generation method may be declared - with optional ``script``, ``inputs``, ``flags``, and ``force`` - attributes on individual entries. + with optional ``script``, ``inputs``, ``extra_deps``, ``flags``, + and ``force`` attributes on individual entries. If the optional ``script`` attribute is not present on an entry, it is assumed that rules for generating the file are present in the associated Makefile.in. @@ -1566,6 +1568,16 @@ VARIABLES = { When the ``flags`` attribute is present, the given list of flags is passed as extra arguments following the inputs. + When the ``extra_deps`` attribute is present, the listed paths are + added as build-graph prerequisites for the generation step but are + not passed to ``script`` as positional arguments. Use this when the + script opens additional files itself at runtime (e.g. via the + preprocessor's #include @TOPOBJDIR@/...) and those files must + therefore exist on disk before the step runs. An objdir-relative + path like ``"!/source-repo.h"`` resolves against ``$topobjdir``, + and a plain path resolves relative to the directory containing the + moz.build file. + When the ``force`` attribute is present, the file is generated every build, regardless of whether it is stale. This is special to the RecursiveMake backend and intended for special situations only (e.g., diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 342bf739e6a9..578a6fc5bb5d 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -1355,6 +1355,7 @@ class GeneratedFile(ContextDerived): "method", "outputs", "inputs", + "extra_deps", "flags", "required_before_export", "required_before_compile", @@ -1374,6 +1375,7 @@ class GeneratedFile(ContextDerived): localized=False, force=False, required_during_compile=None, + extra_deps=(), ): ContextDerived.__init__(self, context) self.script = script @@ -1381,6 +1383,7 @@ class GeneratedFile(ContextDerived): self.outputs = outputs if isinstance(outputs, tuple) else (outputs,) self.inputs = inputs self.flags = flags + self.extra_deps = extra_deps self.localized = localized self.force = force diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index d5a10712df3e..cec02b7b271b 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -1759,6 +1759,16 @@ class TreeMetadataEmitter(LoggingMixin): ) inputs.append(p) + extra_deps = [] + for d in flags.extra_deps: + p = Path(context, d) + if isinstance(p, SourcePath) and not os.path.exists(p.full_path): + raise SandboxValidationError( + f"extra_dep for generating {f} does not exist: {p.full_path}", + context, + ) + extra_deps.append(p) + yield GeneratedFile( context, script, @@ -1768,6 +1778,7 @@ class TreeMetadataEmitter(LoggingMixin): flags.flags, localized=localized, force=flags.force, + extra_deps=extra_deps, ) def _process_test_manifests(self, context): diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/generate-consumer.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/generate-consumer.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/input-data b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/input-data new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/moz.build new file mode 100644 index 000000000000..937c7531af39 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ["consumer.c"] + +consumer = GENERATED_FILES["consumer.c"] +consumer.script = "generate-consumer.py" +consumer.inputs = ["input-data"] +consumer.extra_deps = ["source-extra", "!/producer/producer.c"] diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/source-extra b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/consumer/source-extra new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/moz.build new file mode 100644 index 000000000000..0b940911a90b --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/moz.build @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += [ + "producer", + "consumer", +] diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/generate-producer.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/generate-producer.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/moz.build new file mode 100644 index 000000000000..3f46aa497976 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files-extra-deps/producer/moz.build @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ["producer.c"] + +producer = GENERATED_FILES["producer.c"] +producer.script = "generate-producer.py" diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 323b5b653404..ec76048cc914 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -449,6 +449,39 @@ class TestRecursiveMakeBackend(BackendTester): self.maxDiff = None self.assertEqual(lines, expected) + def test_generated_files_extra_deps(self): + """Ensure GENERATED_FILES extra_deps are handled properly.""" + env = self._consume("generated-files-extra-deps", RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, "consumer", "backend.mk") + lines = [l.strip() for l in open(backend_path).readlines()[2:]] + + expected = [ + "include $(topsrcdir)/config/AB_rCD.mk", + "PRE_COMPILE_TARGETS += $(MDDEPDIR)/consumer.c.stub", + "consumer.c: $(MDDEPDIR)/consumer.c.stub ;", + "EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/consumer.c.pp", + "$(MDDEPDIR)/consumer.c.stub: " + f"{env.topsrcdir}/consumer/generate-consumer.py " + "$(srcdir)/input-data " + "$(srcdir)/source-extra " + "$(DEPTH)/producer/producer.c", + "$(REPORT_BUILD)", + "$(call py_action,file_generate consumer.c," + f"{env.topsrcdir}/consumer/generate-consumer.py main " + "consumer.c $(MDDEPDIR)/consumer.c.pp " + "$(MDDEPDIR)/consumer.c.stub $(srcdir)/input-data)", + "@$(TOUCH) $@", + "", + ] + + self.maxDiff = None + self.assertEqual(lines, expected) + + root_deps_path = mozpath.join(env.topobjdir, "root-deps.mk") + lines = [l.strip() for l in open(root_deps_path).readlines()] + self.assertIn("consumer/pre-compile: producer/pre-compile", lines) + def test_generated_files_force(self): """Ensure GENERATED_FILES with .force is handled properly.""" env = self._consume("generated-files-force", RecursiveMakeBackend) diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/moz.build new file mode 100644 index 000000000000..eb2827afb7da --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ["foo.c"] + +foo = GENERATED_FILES["foo.c"] +foo.script = "script.py" +foo.inputs = [] +foo.extra_deps = ["does-not-exist.h"] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps-missing/script.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/moz.build new file mode 100644 index 000000000000..581d26865ac7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ["foo.c"] + +foo = GENERATED_FILES["foo.c"] +foo.script = "script.py" +foo.inputs = [] +foo.extra_deps = ["!/generated-header.h"] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-extra-deps/script.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 8f0a4858fe4c..2041289acfe4 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -656,6 +656,29 @@ class TestEmitterBasic(unittest.TestCase): ): self.read_topsrcdir(reader) + def test_generated_files_extra_deps(self): + """Test that GENERATED_FILES extra_deps is properly parsed.""" + reader = self.reader("generated-files-extra-deps") + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + o = objs[0] + self.assertIsInstance(o, GeneratedFile) + self.assertEqual(o.outputs, ("foo.c",)) + self.assertEqual(o.inputs, []) + self.assertEqual(len(o.extra_deps), 1) + self.assertIsInstance(o.extra_deps[0], ObjDirPath) + self.assertEqual(o.extra_deps[0], "!/generated-header.h") + + def test_generated_files_extra_deps_missing(self): + """Test that a missing extra_deps source is an error.""" + reader = self.reader("generated-files-extra-deps-missing") + with self.assertRaisesRegex( + SandboxValidationError, + "extra_dep for generating foo.c does not exist", + ): + self.read_topsrcdir(reader) + def test_exports(self): reader = self.reader("exports") objs = self.read_topsrcdir(reader)