XLinfini/sousa-gecko
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 2065363 - Build the newtab bundles from the vendored node packages r=mconley,ai4dev-reviewers,padenot

Browse Source 
Nothing is installed during the build now, and it can run without network access.
Webpack's inputs are recorded and hashed rather than returned as dependencies,
since the FasterMake backend never includes the dependency files it writes.

Differential Revision: https://phabricator.services.mozilla.com/D320323
This commit is contained in:
Alex Hochheiden
2026-09-10 03:12:34 +00:00
committed by ahochheiden@mozilla.com
parent f174ceb3cf
commit 95aa7c1db3
12 changed files with 165 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser/extensions/newtab/AGENTS.md
-2
View File
@@ -28,8 +28,6 @@ Most of the code should be located in `browser/extensions/newtab`
- After making code changes: format with `./mach format path/to/file`, lint with `./mach lint --fix path/to/file`, then build
## Build Output
- Webpack bundles to: `data/content/activity-stream.bundle.js`
- Sass compiles to: `css/activity-stream.css`
- Pre-rendered HTML: `prerendered/activity-stream.html`
- Final package copied to `browser/components/newtab/` by mach
browser/extensions/newtab/build-newtab-bundles.py
+96 -121
View File
@@ -4,14 +4,15 @@
import hashlib
import os
import shutil
import subprocess
import tempfile
import buildconfig
from mozbuild.nodeutil import find_node_executable, package_setup
from mozbuild.nodeutil import find_node_executable
from mozbuild.util import FileAvoidWrite
HASHED_EXTENSIONS = (".jsx", ".js", ".mjs", ".scss")
def _newtab_dir():
return os.path.join(buildconfig.topsrcdir, "browser", "extensions", "newtab")
@@ -27,136 +28,115 @@ def _node():
return node
def _hash_sources(newtab_dir):
hasher = hashlib.sha256()
watch_dirs = [
def _node_modules():
node_modules = os.path.join(
buildconfig.topsrcdir, "third_party", "node", "node_modules"
)
if not os.path.isdir(node_modules):
raise Exception(
"The vendored node packages under third_party/node are missing. "
"Run `./mach vendor node`, or check out third_party/node."
)
return node_modules
def _deps_record(objdir):
return os.path.join(objdir, "webpack-deps.txt")
def _reported_inputs(objdir):
"""Files webpack read during the previous build. The FasterMake backend
ignores the dependency files an action returns, so hashing these is what
makes a change to one of them rebuild the bundles."""
record = _deps_record(objdir)
if not os.path.exists(record):
return set()
with open(record, encoding="utf-8") as f:
return {line.strip() for line in f if line.strip()}
def _declared_inputs(newtab_dir):
inputs = {
os.path.join(newtab_dir, "package.json"),
os.path.join(newtab_dir, "webpack.system-addon.config.js"),
os.path.join(buildconfig.topsrcdir, "browser", "tools", "mozsrcUriPlugin.js"),
os.path.join(buildconfig.topsrcdir, "browser", "tools", "resourceUriPlugin.js"),
os.path.join(buildconfig.topsrcdir, "browser", "modules", "Dedupe.sys.mjs"),
os.path.join(
buildconfig.topsrcdir, "third_party", "node", "vendor-inputs.hash"
),
}
for directory in (
os.path.join(newtab_dir, "content-src"),
os.path.join(newtab_dir, "common"),
]
watch_files = [
os.path.join(newtab_dir, "package.json"),
os.path.join(newtab_dir, "package-lock.json"),
os.path.join(newtab_dir, "webpack.system-addon.config.js"),
]
for path in watch_files:
if os.path.exists(path):
with open(path, "rb") as f:
hasher.update(f.read())
for watch_dir in watch_dirs:
for root, dirs, files in os.walk(watch_dir):
os.path.join(buildconfig.topsrcdir, "browser", "components", "topsites"),
):
for root, dirs, files in os.walk(directory):
dirs.sort()
for fname in sorted(files):
if fname.endswith((".jsx", ".js", ".mjs", ".scss")):
with open(os.path.join(root, fname), "rb") as f:
hasher.update(f.read())
return hasher.digest()
inputs.update(
os.path.join(root, f) for f in files if f.endswith(HASHED_EXTENSIONS)
)
return inputs
def _lockfile_hash(newtab_dir):
lock_path = os.path.join(newtab_dir, "package-lock.json")
if not os.path.exists(lock_path):
return b""
def hash_sources(output):
objdir = os.path.dirname(output.name)
inputs = _declared_inputs(_newtab_dir()) | _reported_inputs(objdir)
hasher = hashlib.sha256()
with open(lock_path, "rb") as f:
hasher.update(f.read())
return hasher.digest()
for path in sorted(inputs):
if not os.path.isfile(path):
continue
hasher.update(path.encode("utf-8"))
with open(path, "rb") as f:
hasher.update(f.read())
output.write(hasher.digest())
def _fetched_node_modules():
"""In CI the deps are provided by the newtab-node-modules toolchain task and
unpacked to $MOZ_FETCHES_DIR/newtab/node_modules, so the build stays hermetic
(no network access at build time). Returns that path if present."""
fetches_dir = os.environ.get("MOZ_FETCHES_DIR")
if not fetches_dir:
return None
fetched = os.path.join(fetches_dir, "newtab", "node_modules")
return fetched if os.path.isdir(fetched) else None
def _record_reported_inputs(objdir, deps_file):
if not os.path.exists(deps_file):
return
with open(deps_file, encoding="utf-8") as f:
reported = sorted({line.strip() for line in f if line.strip()})
kept = [path for path in reported if os.path.isfile(path)]
with FileAvoidWrite(_deps_record(objdir)) as record:
for path in kept:
record.write(path + "\n")
def ensure_node_modules(output):
"""Stamp target; runs before generate_js/generate_css so the install
completes once before they run in parallel.
def _run(argv, newtab_dir, node_modules, extra_env=None):
env = dict(os.environ)
env["MOZ_NODE_MODULES"] = node_modules
env.update(extra_env or {})
result = subprocess.run(argv, check=False, cwd=newtab_dir, env=env)
if result.returncode != 0:
raise Exception(f"{argv[1]} failed with exit code {result.returncode}")
Writes a hash of all source files so downstream GeneratedFile targets
only rebuild when sources actually change."""
def generate_js(output, sources_stamp):
newtab_dir = _newtab_dir()
webpack = os.path.join(newtab_dir, "node_modules", "webpack", "bin", "webpack.js")
# Reinstall when node_modules is missing/incomplete or when package-lock.json
# has changed since the last install, so dependency bumps take effect without
# a manual `./mach newtab install`.
install_stamp = os.path.join(newtab_dir, "node_modules", ".newtab-install-stamp")
lock_hash = _lockfile_hash(newtab_dir)
installed_hash = b""
if os.path.exists(install_stamp):
with open(install_stamp, "rb") as f:
installed_hash = f.read()
if not os.path.exists(webpack) or installed_hash != lock_hash:
# Prefer node_modules staged by the newtab-node-modules toolchain in CI,
# which avoids a network `npm ci` during the build.
fetched = _fetched_node_modules()
if fetched:
dest = os.path.join(newtab_dir, "node_modules")
if os.path.exists(dest):
shutil.rmtree(dest)
# Skip .bin: its POSIX symlinks (archive built on Linux) break
# copytree on Windows, and the build never invokes tools via .bin.
shutil.copytree(fetched, dest, ignore=shutil.ignore_patterns(".bin"))
with open(install_stamp, "wb") as f:
f.write(lock_hash)
output.write(_hash_sources(newtab_dir))
return
if os.environ.get("MOZ_FETCHES_DIR"):
raise Exception(
"newtab-node-modules must be fetched in CI: add it to this "
"task's toolchain fetches."
)
# Local dev builds: install via mozbuild's shared helper, which locates
# node/npm and runs `npm ci` (handles the node-on-PATH and Windows npm
# launcher details for us).
if package_setup(newtab_dir, "newtab"):
raise Exception(
"Failed to install newtab node dependencies. "
"Run ./mach newtab install manually."
)
with open(install_stamp, "wb") as f:
f.write(lock_hash)
output.write(_hash_sources(newtab_dir))
def generate_js(output, node_modules_stamp):
newtab_dir = _newtab_dir()
node = _node()
webpack = os.path.join(newtab_dir, "node_modules", "webpack", "bin", "webpack.js")
if not os.path.exists(webpack):
raise Exception(
"webpack not found. Run ./mach newtab install to install dependencies."
)
node_modules = _node_modules()
webpack = os.path.join(node_modules, "webpack", "bin", "webpack.js")
content_dir = os.path.dirname(output.name)
as_path = os.path.join(content_dir, "activity-stream.bundle.js")
objdir = os.path.dirname(os.path.dirname(content_dir))
with tempfile.TemporaryDirectory() as tmpdir:
result = subprocess.run(
deps_file = os.path.join(tmpdir, "webpack-deps.txt")
_run(
[
node,
_node(),
webpack,
"--config",
os.path.join(newtab_dir, "webpack.system-addon.config.js"),
"--env",
"outputPath=" + tmpdir,
],
check=False,
cwd=newtab_dir,
newtab_dir,
node_modules,
extra_env={"MOZ_WEBPACK_DEPS": deps_file},
)
if result.returncode != 0:
raise Exception(f"webpack failed with exit code {result.returncode}")
with open(os.path.join(tmpdir, "vendor.bundle.js"), "rb") as f:
output.write(f.read())
@@ -165,16 +145,13 @@ def generate_js(output, node_modules_stamp):
with open(os.path.join(tmpdir, "activity-stream.bundle.js"), "rb") as f:
as_output.write(f.read())
_record_reported_inputs(objdir, deps_file)
def generate_css(output, node_modules_stamp):
def generate_css(output, sources_stamp):
newtab_dir = _newtab_dir()
node = _node()
sass = os.path.join(newtab_dir, "node_modules", "sass", "sass.js")
if not os.path.exists(sass):
raise Exception(
"sass not found. Run ./mach newtab install to install dependencies."
)
node_modules = _node_modules()
sass = os.path.join(node_modules, "sass", "sass.js")
css_dir = os.path.dirname(output.name)
nova_css_path = os.path.join(css_dir, "nova", "activity-stream.css")
@@ -182,18 +159,16 @@ def generate_css(output, node_modules_stamp):
src_styles_dir = os.path.join(newtab_dir, "content-src", "styles")
with tempfile.TemporaryDirectory() as tmpdir:
result = subprocess.run(
_run(
[
node,
_node(),
sass,
src_styles_dir + ":" + tmpdir,
"--no-source-map",
],
check=False,
cwd=newtab_dir,
newtab_dir,
node_modules,
)
if result.returncode != 0:
raise Exception(f"sass failed with exit code {result.returncode}")
with open(os.path.join(tmpdir, "activity-stream.css"), "rb") as f:
output.write(f.read())
browser/extensions/newtab/moz.build
+4 -4
View File
@@ -21,9 +21,9 @@ JAR_MANIFESTS += ["addon-jar.mn"]
if CONFIG["NODEJS"]:
GeneratedFile(
"newtab-node-modules.stamp",
"newtab-sources.stamp",
script="build-newtab-bundles.py",
entry_point="ensure_node_modules",
entry_point="hash_sources",
force=True,
)
@@ -32,7 +32,7 @@ if CONFIG["NODEJS"]:
"data/content/activity-stream.bundle.js",
script="build-newtab-bundles.py",
entry_point="generate_js",
inputs=["!newtab-node-modules.stamp"],
inputs=["!newtab-sources.stamp"],
)
GeneratedFile(
@@ -40,7 +40,7 @@ if CONFIG["NODEJS"]:
"css/nova/activity-stream.css",
script="build-newtab-bundles.py",
entry_point="generate_css",
inputs=["!newtab-node-modules.stamp"],
inputs=["!newtab-sources.stamp"],
)
DIRS += ["webext-glue"]
browser/extensions/newtab/webpack.system-addon.config.js
+37 -8
View File
@@ -3,13 +3,19 @@
* You can obtain one at http://mozilla.org/MPL/2.0/. */
const path = require("path");
const webpack = require("webpack");
const MinimizerPlugin = require("minimizer-webpack-plugin");
const { ResourceUriPlugin } = require("../../tools/resourceUriPlugin");
const { MozSrcUriPlugin } = require("../../tools/mozsrcUriPlugin");
const absolute = relPath => path.join(__dirname, relPath);
const nodeModules = process.env.MOZ_NODE_MODULES
? path.resolve(process.env.MOZ_NODE_MODULES)
: absolute("node_modules");
const vendored = name => path.join(nodeModules, ...name.split("/"));
const webpack = require(vendored("webpack"));
const MinimizerPlugin = require(vendored("minimizer-webpack-plugin"));
const { ResourceUriPlugin } = require("../../tools/resourceUriPlugin");
const { MozSrcUriPlugin } = require("../../tools/mozsrcUriPlugin");
const baseConfig = env => ({
mode: "none",
devtool: env.development ? "inline-source-map" : false,
@@ -17,10 +23,10 @@ const baseConfig = env => ({
rules: [
{
test: /\.jsx?$/,
exclude: /node_modules\/(?!@fluent\/).*/,
loader: "babel-loader",
exclude: /node_modules[\\/](?!@fluent[\\/]).*/,
loader: vendored("babel-loader"),
options: {
presets: ["@babel/preset-react"],
presets: [vendored("@babel/preset-react")],
},
},
{
@@ -33,10 +39,31 @@ const baseConfig = env => ({
},
resolve: {
extensions: [".js", ".jsx", ".mjs"],
modules: ["node_modules", "."],
modules: [nodeModules, "."],
},
resolveLoader: {
modules: [nodeModules],
},
});
class DependencyListPlugin {
constructor(outputPath) {
this.outputPath = outputPath;
}
apply(compiler) {
compiler.hooks.done.tap("DependencyListPlugin", stats => {
const deps = [...stats.compilation.fileDependencies].sort();
require("fs").appendFileSync(this.outputPath, `${deps.join("\n")}\n`);
});
}
}
const dependencyListPlugins = () =>
process.env.MOZ_WEBPACK_DEPS
? [new DependencyListPlugin(process.env.MOZ_WEBPACK_DEPS)]
: [];
const vendorOptimization = {
minimize: true,
minimizer: [
@@ -78,6 +105,7 @@ module.exports = (env = {}) => {
new webpack.BannerPlugin(
`THIS FILE IS AUTO-GENERATED: ${path.basename(__filename)}`
),
...dependencyListPlugins(),
],
}),
// Activity stream bundle (uses vendor as externals)
@@ -137,6 +165,7 @@ module.exports = (env = {}) => {
`THIS FILE IS AUTO-GENERATED: ${path.basename(__filename)}`
),
new webpack.optimize.ModuleConcatenationPlugin(),
...dependencyListPlugins(),
],
}),
];
taskcluster/kinds/artifact-build/kind.yml
-1
View File
@@ -46,7 +46,6 @@ task-defaults:
fetches:
toolchain:
- terser
- newtab-node-modules
tasks:
linux64-artifact/opt:
taskcluster/kinds/build-extensions/kind.yml
-1
View File
@@ -18,7 +18,6 @@ task-defaults:
fetches:
toolchain:
- linux64-node
- newtab-node-modules
optimization:
skip-unless-expanded: null
taskcluster/kinds/build/kind.yml
-1
View File
@@ -40,7 +40,6 @@ task-defaults:
fetches:
toolchain:
- terser
- newtab-node-modules
run:
clone-with: hg
mozconfig-variant:
taskcluster/kinds/hazard/kind.yml
-1
View File
@@ -83,6 +83,5 @@ tasks:
- linux64-pkgconf
- sysroot-x86_64-linux-gnu
- sysroot-wasm32-wasi
- newtab-node-modules
optimization:
skip-unless-backstop: null
taskcluster/kinds/instrumented-build/kind.yml
-1
View File
@@ -45,7 +45,6 @@ task-defaults:
toolchain:
- linux64-sccache
- terser
- newtab-node-modules
tasks:
linux64-shippable/opt:
taskcluster/kinds/searchfox/kind.yml
-3
View File
@@ -40,9 +40,6 @@ task-defaults:
env:
MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system
PERFHERDER_EXTRA_OPTIONS: searchfox
fetches:
toolchain:
- newtab-node-modules
tasks:
linux64-searchfox/opt:
taskcluster/kinds/source-test/vendor.yml
+28
View File
@@ -45,3 +45,31 @@ rust:
- 'python/mozbuild/mozbuild/vendor/vendor_rust.py'
- 'supply-chain/**'
- 'third_party/rust/**'
node:
description: Node vendoring validation
attributes:
code-review: false
treeherder:
symbol: node(vendor)
worker:
artifacts: []
env:
MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system
run:
clone-with: hg
cwd: '{checkout}'
command: >-
./mach vendor node --ignore-modified &&
test -z "$(hg status third_party/node)"
fetches:
toolchain:
- linux64-node
fetch:
- pnpm
when:
files-changed:
- 'third_party/node/**'
- 'browser/extensions/newtab/package.json'
- 'python/mozbuild/mozbuild/vendor/vendor_node.py'
- 'taskcluster/kinds/fetch/pnpm.yml'
taskcluster/kinds/valgrind/kind.yml
-1
View File
@@ -70,6 +70,5 @@ tasks:
- sysroot-x86_64-linux-gnu
- sysroot-wasm32-wasi
- terser
- newtab-node-modules
optimization:
skip-unless-expanded: null