Bug 2013417 - Avoid calling ./mach environment every Gradle configuration r=nalexander,firefox-build-system-reviewers,geckoview-reviewers,glandium
Differential Revision: https://phabricator.services.mozilla.com/D281080
This commit is contained in:
Alex Hochheiden
committed by
ahochheiden@mozilla.com
ahochheiden@mozilla.com
parent
2929816119
commit
f401cc2f3f
@@ -6,6 +6,7 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import textwrap
|
||||
from collections import defaultdict
|
||||
@@ -22,6 +23,93 @@ from mozprocess import ProcessHandler
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_mach_with_config(mozconfig, argv, cwd=None, pop_moz_automation=False):
|
||||
"""Run mach with a specific mozconfig."""
|
||||
env = os.environ.copy()
|
||||
env["MOZCONFIG"] = str(mozconfig)
|
||||
env["MACH_NO_TERMINAL_FOOTER"] = "1"
|
||||
env["MACH_NO_WRITE_TIMES"] = "1"
|
||||
if pop_moz_automation:
|
||||
env.pop("MOZ_AUTOMATION", None)
|
||||
if env.get("MOZ_AUTOMATION"):
|
||||
env["MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE"] = "system"
|
||||
|
||||
output_lines = []
|
||||
|
||||
def pol(line):
|
||||
logger.debug(line)
|
||||
output_lines.append(line)
|
||||
|
||||
proc = ProcessHandler(
|
||||
[sys.executable, "mach"] + argv,
|
||||
env=env,
|
||||
cwd=cwd or topsrcdir,
|
||||
processOutputLine=pol,
|
||||
universal_newlines=True,
|
||||
)
|
||||
proc.run()
|
||||
proc.wait()
|
||||
return proc.poll(), output_lines
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def run_mach(mozconfig):
|
||||
"""Fixture providing run_mach bound to the default mozconfig."""
|
||||
|
||||
def inner(argv, cwd=None):
|
||||
return run_mach_with_config(mozconfig, argv, cwd=cwd)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
def run_gradle(mozconfig, args, use_config_cache=True, pop_moz_automation=False):
|
||||
"""Run mach Gradle with --debug flag.
|
||||
|
||||
Args:
|
||||
mozconfig: Path to mozconfig file
|
||||
args: List of Gradle arguments
|
||||
use_config_cache: If False, passes --no-configuration-cache to disable
|
||||
Gradle's configuration cache (useful for testing the local cache layer)
|
||||
pop_moz_automation: If True, removes MOZ_AUTOMATION from the environment
|
||||
(needed for testing local cache since it's disabled in automation)
|
||||
"""
|
||||
extra_args = ["--debug"]
|
||||
if not use_config_cache:
|
||||
extra_args.append("--no-configuration-cache")
|
||||
return run_mach_with_config(
|
||||
mozconfig, ["gradle"] + args + extra_args, pop_moz_automation=pop_moz_automation
|
||||
)
|
||||
|
||||
|
||||
def clear_local_cache():
|
||||
"""Clear the local topobjdir cache files."""
|
||||
cache_dir = Path(topsrcdir) / ".gradle" / "mach-environment-cache"
|
||||
if cache_dir.exists():
|
||||
shutil.rmtree(cache_dir)
|
||||
|
||||
|
||||
def create_mozconfig(test_dir, name):
|
||||
"""Create a mozconfig and objdir pair for testing.
|
||||
|
||||
Returns (mozconfig_path, objdir_path) tuple.
|
||||
"""
|
||||
objdir = test_dir / f"objdir-{name}"
|
||||
mozconfig_path = test_dir / f"mozconfig-{name}"
|
||||
mozconfig_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
mozconfig_path.write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
ac_add_options --enable-application=mobile/android
|
||||
ac_add_options --enable-artifact-builds
|
||||
ac_add_options --target=aarch64-linux-android
|
||||
mk_add_options MOZ_OBJDIR="{objdir.as_posix()}"
|
||||
export GRADLE_FLAGS="-PbuildMetrics -PbuildMetricsOutputDir={objdir.as_posix()}/gradle/build/metrics -PbuildMetricsFileSuffix=test"
|
||||
"""
|
||||
)
|
||||
)
|
||||
return mozconfig_path, objdir
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def test_dir():
|
||||
return (
|
||||
@@ -31,57 +119,27 @@ def test_dir():
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def objdir(test_dir):
|
||||
return test_dir / "objdir"
|
||||
def primary_config(test_dir):
|
||||
return create_mozconfig(test_dir, "primary")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mozconfig(test_dir, objdir):
|
||||
mozconfig_path = test_dir / "mozconfig"
|
||||
mozconfig_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
mozconfig_path.write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
ac_add_options --enable-application=mobile/android
|
||||
ac_add_options --enable-artifact-builds
|
||||
ac_add_options --target=arm
|
||||
mk_add_options MOZ_OBJDIR="{objdir}"
|
||||
export GRADLE_FLAGS="-PbuildMetrics -PbuildMetricsOutputDir={objdir}/gradle/build/metrics -PbuildMetricsFileSuffix=test"
|
||||
"""
|
||||
)
|
||||
)
|
||||
def secondary_config(test_dir):
|
||||
return create_mozconfig(test_dir, "secondary")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def objdir(primary_config):
|
||||
_, objdir = primary_config
|
||||
return objdir
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mozconfig(primary_config):
|
||||
mozconfig_path, _ = primary_config
|
||||
return mozconfig_path
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def run_mach(mozconfig):
|
||||
def inner(argv, cwd=None):
|
||||
env = os.environ.copy()
|
||||
env["MOZCONFIG"] = str(mozconfig)
|
||||
env["MACH_NO_TERMINAL_FOOTER"] = "1"
|
||||
env["MACH_NO_WRITE_TIMES"] = "1"
|
||||
|
||||
if os.environ.get("MOZ_AUTOMATION"):
|
||||
env["MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE"] = "system"
|
||||
|
||||
def pol(line):
|
||||
logger.debug(line)
|
||||
|
||||
proc = ProcessHandler(
|
||||
[sys.executable, "mach"] + argv,
|
||||
env=env,
|
||||
cwd=cwd or topsrcdir,
|
||||
processOutputLine=pol,
|
||||
universal_newlines=True,
|
||||
)
|
||||
proc.run()
|
||||
proc.wait()
|
||||
|
||||
return proc.poll(), proc.output
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
AARS = {
|
||||
"geckoview.aar": "gradle/build/mobile/android/geckoview/outputs/aar/geckoview-debug.aar",
|
||||
}
|
||||
@@ -333,5 +391,193 @@ def test_android_export(objdir, mozconfig, run_mach):
|
||||
assert_ordered_task_outcomes(objdir, [(":verifyGleanVersion", "UP-TO-DATE")])
|
||||
|
||||
|
||||
def test_mach_environment_configuration_cache(primary_config, secondary_config):
|
||||
"""Test that Gradle's configuration cache invalidates when objdir-determining inputs change."""
|
||||
|
||||
def get_config_cache_status(output):
|
||||
for line in output:
|
||||
if "Reusing configuration cache" in line:
|
||||
return "reused"
|
||||
return None
|
||||
|
||||
primary_mozconfig, primary_objdir = primary_config
|
||||
secondary_mozconfig, secondary_objdir = secondary_config
|
||||
|
||||
assert_success(*run_mach_with_config(primary_mozconfig, ["build"]))
|
||||
assert_success(*run_mach_with_config(secondary_mozconfig, ["build"]))
|
||||
|
||||
assert (primary_objdir / "config.status.json").exists(), (
|
||||
f"{primary_objdir} should have config.status.json"
|
||||
)
|
||||
assert (secondary_objdir / "config.status.json").exists(), (
|
||||
f"{secondary_objdir} should have config.status.json"
|
||||
)
|
||||
|
||||
returncode, output = run_gradle(secondary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
|
||||
gradle_cache_dir = Path(topsrcdir) / ".gradle" / "configuration-cache"
|
||||
if gradle_cache_dir.exists():
|
||||
shutil.rmtree(gradle_cache_dir)
|
||||
|
||||
# First run, config cache miss
|
||||
returncode, output = run_gradle(primary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
assert get_config_cache_status(output) is None, (
|
||||
"Config cache should not be reused on first run"
|
||||
)
|
||||
|
||||
# Second run, same config, expect config cache reused
|
||||
returncode, output = run_gradle(primary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
config_status = get_config_cache_status(output)
|
||||
assert config_status == "reused", (
|
||||
f"Expected Gradle config cache 'reused' on second run, got '{config_status}'"
|
||||
)
|
||||
|
||||
# Third run, switch to secondary mozconfig, expect config cache miss
|
||||
returncode, output = run_gradle(secondary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
assert get_config_cache_status(output) is None, (
|
||||
"Config cache should be invalidated when MOZCONFIG changes"
|
||||
)
|
||||
|
||||
# Fourth run, still secondary mozconfig, expect config cache reused
|
||||
returncode, output = run_gradle(secondary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
config_status = get_config_cache_status(output)
|
||||
assert config_status == "reused", (
|
||||
f"Expected config cache 'reused' on repeat run, got '{config_status}'"
|
||||
)
|
||||
|
||||
original_content = secondary_mozconfig.read_text()
|
||||
try:
|
||||
# Modify mozconfig content to invalidate config cache
|
||||
secondary_mozconfig.write_text(
|
||||
original_content + "\n# config cache invalidation test\n"
|
||||
)
|
||||
|
||||
# Fifth run, config cache miss due to content change
|
||||
returncode, output = run_gradle(secondary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
assert get_config_cache_status(output) is None, (
|
||||
"Config cache should be invalidated when mozconfig content changes"
|
||||
)
|
||||
|
||||
# Sixth run, no change, config cache reused
|
||||
returncode, output = run_gradle(secondary_mozconfig, ["help"])
|
||||
assert_success(returncode, output)
|
||||
config_status = get_config_cache_status(output)
|
||||
assert config_status == "reused", (
|
||||
f"Expected config cache 'reused' after no changes, got '{config_status}'"
|
||||
)
|
||||
finally:
|
||||
secondary_mozconfig.write_text(original_content)
|
||||
|
||||
|
||||
def test_mach_environment_local_topobjdir_cache(primary_config, secondary_config):
|
||||
"""Test that local topobjdir caching avoids running `./mach environment` unnecessarily."""
|
||||
|
||||
def get_local_cache_status(output):
|
||||
for line in output:
|
||||
if "topobjdir cache hit!" in line:
|
||||
return "hit"
|
||||
if "topobjdir cache miss!" in line:
|
||||
return "miss"
|
||||
return None
|
||||
|
||||
primary_mozconfig, primary_objdir = primary_config
|
||||
secondary_mozconfig, secondary_objdir = secondary_config
|
||||
|
||||
assert_success(*run_mach_with_config(primary_mozconfig, ["build"]))
|
||||
assert_success(*run_mach_with_config(secondary_mozconfig, ["build"]))
|
||||
assert (primary_objdir / "config.status.json").exists()
|
||||
assert (secondary_objdir / "config.status.json").exists()
|
||||
|
||||
local_cache_dir = Path(topsrcdir) / ".gradle" / "mach-environment-cache"
|
||||
|
||||
clear_local_cache()
|
||||
|
||||
# First run, local cache miss
|
||||
returncode, output = run_gradle(
|
||||
primary_mozconfig, ["help"], use_config_cache=False, pop_moz_automation=True
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "miss", (
|
||||
f"Expected local cache 'miss' on first run, got '{local_status}'"
|
||||
)
|
||||
assert local_cache_dir.exists(), "Local cache directory should be created"
|
||||
assert (local_cache_dir / "inputs.sha256").exists(), (
|
||||
"Cache hash file should be created"
|
||||
)
|
||||
assert (local_cache_dir / "topobjdir.txt").exists(), (
|
||||
"topobjdir cache file should be created"
|
||||
)
|
||||
|
||||
# Second run, same config, expect local cache hit
|
||||
returncode, output = run_gradle(
|
||||
primary_mozconfig, ["help"], use_config_cache=False, pop_moz_automation=True
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "hit", (
|
||||
f"Expected local cache 'hit' on second run, got '{local_status}'"
|
||||
)
|
||||
|
||||
# Third run, switch to secondary mozconfig, expect local cache miss
|
||||
returncode, output = run_gradle(
|
||||
secondary_mozconfig, ["help"], use_config_cache=False, pop_moz_automation=True
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "miss", (
|
||||
f"Expected local cache 'miss' when switching mozconfig, got '{local_status}'"
|
||||
)
|
||||
|
||||
# Fourth run, still secondary mozconfig, expect local cache hit
|
||||
returncode, output = run_gradle(
|
||||
secondary_mozconfig, ["help"], use_config_cache=False, pop_moz_automation=True
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "hit", (
|
||||
f"Expected local cache 'hit' on repeat with secondary, got '{local_status}'"
|
||||
)
|
||||
|
||||
original_content = secondary_mozconfig.read_text()
|
||||
try:
|
||||
# Modify mozconfig content to invalidate local cache
|
||||
secondary_mozconfig.write_text(original_content + "\n# local cache test\n")
|
||||
|
||||
# Fifth run, local cache miss due to content change
|
||||
returncode, output = run_gradle(
|
||||
secondary_mozconfig,
|
||||
["help"],
|
||||
use_config_cache=False,
|
||||
pop_moz_automation=True,
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "miss", (
|
||||
f"Expected local cache 'miss' after mozconfig change, got '{local_status}'"
|
||||
)
|
||||
|
||||
# Sixth run, no change, local cache hit
|
||||
returncode, output = run_gradle(
|
||||
secondary_mozconfig,
|
||||
["help"],
|
||||
use_config_cache=False,
|
||||
pop_moz_automation=True,
|
||||
)
|
||||
assert_success(returncode, output)
|
||||
local_status = get_local_cache_status(output)
|
||||
assert local_status == "hit", (
|
||||
f"Expected local cache 'hit' after no changes, got '{local_status}'"
|
||||
)
|
||||
finally:
|
||||
secondary_mozconfig.write_text(original_content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
@@ -258,6 +259,16 @@ def config_status(config, execute=True):
|
||||
partial_config = PartialConfigEnvironment(config["TOPOBJDIR"])
|
||||
partial_config.write_vars(sanitized_config)
|
||||
|
||||
mach_env = {
|
||||
"topobjdir": sanitized_config["topobjdir"],
|
||||
"topsrcdir": sanitized_config["topsrcdir"],
|
||||
"defines": dict(sanitized_config["defines"]),
|
||||
"substs": dict(sanitized_config["substs"]),
|
||||
}
|
||||
# Write config.status.json for fast Gradle configuration.
|
||||
with FileAvoidWrite("config.status.json") as fh:
|
||||
fh.write(json.dumps(mach_env, indent=2, sort_keys=True))
|
||||
|
||||
# Write out a file so the build backend knows to re-run configure when
|
||||
# relevant Python changes. Use FileAvoidWrite to only write if the
|
||||
# deps_content has changed to avoid invalidating Gradle's configuration cache
|
||||
|
||||
@@ -4,62 +4,231 @@
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.security.MessageDigest
|
||||
import javax.inject.Inject
|
||||
import org.gradle.api.provider.ValueSource
|
||||
import org.gradle.api.provider.ValueSourceParameters
|
||||
import org.gradle.process.ExecOperations
|
||||
|
||||
// Loads the mach environment configurations into a gradle extension property.
|
||||
//
|
||||
// This script runs during Gradle configuration. Originally it called `./mach
|
||||
// environment` to get the full build configuration, but that's expensive (often 2-3
|
||||
// seconds). Now, `./mach configure` dumps the configuration to
|
||||
// `config.status.json`, so we just need to know the topobjdir to read it.
|
||||
// Determining the topobjdir without mach would require duplicating topobjdir
|
||||
// resolution logic, so we run `./mach environment` to get just the topobjdir.
|
||||
//
|
||||
// We use two layers of caching to avoid running `./mach environment` unnecessarily:
|
||||
//
|
||||
// 1. Gradle's configuration cache (via ValueSource API): We declare all inputs that
|
||||
// affect topobjdir resolution as ValueSource parameters. When these inputs are
|
||||
// unchanged, Gradle reuses the entire cached configuration without calling obtain().
|
||||
//
|
||||
// 2. Local file cache (inside obtain()): When Gradle's config cache is invalidated
|
||||
// for other reasons (e.g., build script changes), we still check our local cache
|
||||
// before running `./mach environment`. This cache is stored in
|
||||
// .gradle/mach-environment-cache/ and keyed by a hash of the same inputs.
|
||||
//
|
||||
// This means `./mach environment` only runs when inputs actually change.
|
||||
|
||||
def startTime = System.currentTimeMillis()
|
||||
logger.debug("mozconfig.gradle> Loading mach environment")
|
||||
|
||||
apply from: file('./mach_env.gradle')
|
||||
|
||||
logger.lifecycle("mozconfig.gradle> Loading mach environment into a gradle extension property")
|
||||
|
||||
if (!ext.hasProperty("topsrcdir")) {
|
||||
ext.topsrcdir = file(buildscript.getSourceFile()).getParentFile().getParentFile().getParentFile().getParentFile().absolutePath
|
||||
}
|
||||
|
||||
def command = ["${topsrcdir}/mach", "environment", "--format", "json", "--verbose"]
|
||||
if (System.env.GRADLE_MACH_PYTHON) {
|
||||
command.addAll(0, [System.env.GRADLE_MACH_PYTHON])
|
||||
} else if (System.properties["os.name"].contains("Windows")) {
|
||||
command.addAll(0, ["python"])
|
||||
abstract class TopobjdirValueSource implements ValueSource<String, TopobjdirValueSource.Params> {
|
||||
interface Params extends ValueSourceParameters {
|
||||
Property<String> getTopsrcdir()
|
||||
Property<String> getMozconfigEnv()
|
||||
Property<String> getMozObjdirEnv()
|
||||
Property<String> getMozAutomation()
|
||||
Property<String> getMozconfigContents()
|
||||
Property<String> getDotMozconfigContents()
|
||||
Property<String> getDefaultMozconfigContents()
|
||||
Property<String> getLocalPropertiesContents()
|
||||
Property<String> getLocalMozconfigContents()
|
||||
}
|
||||
|
||||
@Inject
|
||||
abstract ExecOperations getExecOperations()
|
||||
|
||||
// Called when Gradle's configuration cache is invalidated. We maintain our own
|
||||
// local file cache here so that `./mach environment` only runs when inputs
|
||||
// actually change, not just when Gradle's config cache is wiped for other reasons.
|
||||
@Override
|
||||
String obtain() {
|
||||
def logger = org.gradle.api.logging.Logging.getLogger(TopobjdirValueSource.class)
|
||||
def topsrcdir = parameters.topsrcdir.get()
|
||||
|
||||
// Compute hash of all inputs for local caching
|
||||
def inputs = new StringBuilder()
|
||||
inputs.append("MOZCONFIG=${parameters.mozconfigEnv.get()}\n")
|
||||
inputs.append("MOZ_OBJDIR=${parameters.mozObjdirEnv.get()}\n")
|
||||
inputs.append("mozconfigContents=${parameters.mozconfigContents.get().hashCode()}\n")
|
||||
inputs.append("dotMozconfigContents=${parameters.dotMozconfigContents.get().hashCode()}\n")
|
||||
inputs.append("defaultMozconfigContents=${parameters.defaultMozconfigContents.get().hashCode()}\n")
|
||||
inputs.append("localPropertiesContents=${parameters.localPropertiesContents.get().hashCode()}\n")
|
||||
inputs.append("localMozconfigContents=${parameters.localMozconfigContents.get().hashCode()}\n")
|
||||
|
||||
logger.debug("mozconfig.gradle> Cache inputs:\n${inputs}")
|
||||
|
||||
def sha256 = MessageDigest.getInstance("SHA-256")
|
||||
sha256.update(inputs.toString().getBytes(StandardCharsets.UTF_8))
|
||||
def currentHash = sha256.digest().encodeHex().toString()
|
||||
|
||||
// Check local cache (this doesn't affect Gradle's config cache since we're inside obtain())
|
||||
// Disable local caching in automation to ensure consistent behavior
|
||||
def useCache = !parameters.mozAutomation.get()
|
||||
|
||||
def cacheDir = new File(topsrcdir, ".gradle/mach-environment-cache")
|
||||
cacheDir.mkdirs()
|
||||
|
||||
def cacheHashFile = new File(cacheDir, "inputs.sha256")
|
||||
def topobjdirCacheFile = new File(cacheDir, "topobjdir.txt")
|
||||
|
||||
if (useCache && cacheHashFile.exists() && topobjdirCacheFile.exists()) {
|
||||
def cachedHash = cacheHashFile.text.trim()
|
||||
if (cachedHash == currentHash) {
|
||||
logger.debug("mozconfig.gradle> topobjdir cache hit!")
|
||||
return topobjdirCacheFile.text.trim()
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("mozconfig.gradle> topobjdir cache miss! Running `./mach environment`")
|
||||
def command = ["${topsrcdir}/mach", "environment", "--format", "json"]
|
||||
if (System.properties["os.name"].contains("Windows")) {
|
||||
command.addAll(0, ["python"])
|
||||
}
|
||||
|
||||
def stdout = new ByteArrayOutputStream()
|
||||
def stderr = new ByteArrayOutputStream()
|
||||
|
||||
def result = execOperations.exec { spec ->
|
||||
spec.workingDir = new File(topsrcdir)
|
||||
spec.commandLine = command
|
||||
spec.standardOutput = stdout
|
||||
spec.errorOutput = stderr
|
||||
spec.ignoreExitValue = true
|
||||
}
|
||||
|
||||
if (result.exitValue != 0) {
|
||||
throw new GradleException(
|
||||
"mozconfig.gradle> Error running ./mach environment:\n" +
|
||||
"Process '${command}' finished with non-zero exit value ${result.exitValue}:\n\n" +
|
||||
"stdout:\n${stdout}\n\n" +
|
||||
"stderr:\n${stderr}"
|
||||
)
|
||||
}
|
||||
|
||||
def outputString = stdout.toString().normalize().trim()
|
||||
// Ignore possible lines of output from pip installing packages,
|
||||
// so only start at what looks like the beginning of a JSON object
|
||||
if (outputString.lastIndexOf("\n") != -1) {
|
||||
outputString = outputString.substring(outputString.lastIndexOf("\n") + 1).trim()
|
||||
}
|
||||
|
||||
def slurper = new JsonSlurper()
|
||||
def jsonMachEnv
|
||||
try {
|
||||
jsonMachEnv = slurper.parseText(outputString)
|
||||
} catch (Exception e) {
|
||||
throw new GradleException("Failed to parse JSON from ./mach environment: ${e.message}\nOutput: ${outputString}")
|
||||
}
|
||||
|
||||
def topobjdir = jsonMachEnv.topobjdir
|
||||
|
||||
if (useCache) {
|
||||
try {
|
||||
cacheHashFile.text = currentHash
|
||||
topobjdirCacheFile.text = topobjdir
|
||||
} catch (Exception e) {
|
||||
logger.debug("mozconfig.gradle> Warning: Failed to write cache: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
return topobjdir
|
||||
}
|
||||
}
|
||||
|
||||
def proc = providers.exec {
|
||||
workingDir = new File(topsrcdir)
|
||||
environment = machEnv(topsrcdir)
|
||||
commandLine = command
|
||||
ignoreExitValue = true
|
||||
def readFileContents = { File f ->
|
||||
if (!f.exists() || f.isDirectory()) {
|
||||
return ""
|
||||
}
|
||||
return providers.fileContents(layout.rootDirectory.file(f.absolutePath)).asText.getOrElse("")
|
||||
}
|
||||
|
||||
def result = proc.result.get().exitValue
|
||||
def standardOutput = proc.standardOutput.asText.get()
|
||||
// Only show the output if something went wrong.
|
||||
if (result != 0) {
|
||||
logger.info("mozconfig.gradle> Error running ./mach environment: \n\n"
|
||||
+ "Process '${command}' finished with non-zero exit value ${result}:\n\n"
|
||||
+ "stdout:\n"
|
||||
+ "${standardOutput}\n\n"
|
||||
+ "stderr:\n"
|
||||
+ "${proc.standardError.asText.get()}")
|
||||
throw new StopExecutionException(
|
||||
"Could not run ./mach environment. Try running ./mach build first.")
|
||||
def mozconfigEnv = providers.environmentVariable('MOZCONFIG').getOrElse("")
|
||||
def mozObjdirEnv = providers.environmentVariable('MOZ_OBJDIR').getOrElse("")
|
||||
def mozAutomation = providers.environmentVariable('MOZ_AUTOMATION').getOrElse("")
|
||||
|
||||
def mozconfigFile = mozconfigEnv ? (new File(mozconfigEnv).isAbsolute() ? new File(mozconfigEnv) : new File(topsrcdir, mozconfigEnv)) : null
|
||||
def mozconfigContents = mozconfigFile ? readFileContents(mozconfigFile) : ""
|
||||
|
||||
def dotMozconfigContents = readFileContents(new File(topsrcdir, '.mozconfig'))
|
||||
def defaultMozconfigContents = readFileContents(new File(topsrcdir, 'mozconfig'))
|
||||
def localPropertiesContents = readFileContents(new File(topsrcdir, 'local.properties'))
|
||||
|
||||
def localMozconfigContents = ""
|
||||
if (localPropertiesContents) {
|
||||
def localProperties = new Properties()
|
||||
localProperties.load(new StringReader(localPropertiesContents))
|
||||
def localMozconfigPath = localProperties.getProperty("mozilla-central.mozconfig")
|
||||
if (localMozconfigPath) {
|
||||
def localMozconfigFile = new File(localMozconfigPath).isAbsolute() ? new File(localMozconfigPath) : new File(topsrcdir, localMozconfigPath)
|
||||
localMozconfigContents = readFileContents(localMozconfigFile)
|
||||
}
|
||||
}
|
||||
|
||||
def outputString = standardOutput.toString().normalize()
|
||||
// Ignore possible lines of output from pip installing packages,
|
||||
// so only start at what looks like the beginning of a JSON object
|
||||
if (outputString.lastIndexOf("\n") != -1) {
|
||||
outputString = outputString.substring(outputString.lastIndexOf("\n") + 1)
|
||||
def topsrcdirValue = topsrcdir
|
||||
def mozconfigEnvValue = mozconfigEnv
|
||||
def mozObjdirEnvValue = mozObjdirEnv
|
||||
def mozAutomationValue = mozAutomation
|
||||
def mozconfigContentsValue = mozconfigContents
|
||||
def dotMozconfigContentsValue = dotMozconfigContents
|
||||
def defaultMozconfigContentsValue = defaultMozconfigContents
|
||||
def localPropertiesContentsValue = localPropertiesContents
|
||||
def localMozconfigContentsValue = localMozconfigContents
|
||||
|
||||
def topobjdirProvider = providers.of(TopobjdirValueSource) {
|
||||
parameters {
|
||||
it.topsrcdir.set(topsrcdirValue)
|
||||
it.mozconfigEnv.set(mozconfigEnvValue)
|
||||
it.mozObjdirEnv.set(mozObjdirEnvValue)
|
||||
it.mozAutomation.set(mozAutomationValue)
|
||||
it.mozconfigContents.set(mozconfigContentsValue)
|
||||
it.dotMozconfigContents.set(dotMozconfigContentsValue)
|
||||
it.defaultMozconfigContents.set(defaultMozconfigContentsValue)
|
||||
it.localPropertiesContents.set(localPropertiesContentsValue)
|
||||
it.localMozconfigContents.set(localMozconfigContentsValue)
|
||||
}
|
||||
}
|
||||
|
||||
def topobjdir = topobjdirProvider.get()
|
||||
logger.debug("mozconfig.gradle> topobjdir=${topobjdir}")
|
||||
|
||||
def machEnvFile = new File(topobjdir, 'config.status.json')
|
||||
if (!machEnvFile.exists()) {
|
||||
throw new GradleException("config.status.json not found at ${machEnvFile.absolutePath}. Run ./mach configure first.")
|
||||
}
|
||||
|
||||
def slurper = new JsonSlurper()
|
||||
def json;
|
||||
def json
|
||||
try {
|
||||
json = slurper.parseText(outputString)
|
||||
} catch (ignored) {
|
||||
logger.info("mozconfig.gradle> Failed to parse JSON output from ./mach environment: \n\n" +
|
||||
outputString);
|
||||
throw new StopExecutionException(
|
||||
"Failed to parse JSON output from ./mach environment.\n\n" + outputString);
|
||||
json = slurper.parse(machEnvFile)
|
||||
} catch (Exception e) {
|
||||
logger.error("mozconfig.gradle> Failed to parse: ${machEnvFile.text}")
|
||||
throw new GradleException("Failed to parse ${machEnvFile.absolutePath}: ${e.message}")
|
||||
}
|
||||
|
||||
def elapsed = System.currentTimeMillis() - startTime
|
||||
logger.info("mozconfig.gradle> Loaded mach environment for ${topobjdir} in ${elapsed} ms")
|
||||
|
||||
if (json.substs.MOZ_BUILD_APP != 'mobile/android') {
|
||||
throw new GradleException("Building with Gradle is only supported for Firefox for Android, i.e., MOZ_BUILD_APP == 'mobile/android'.")
|
||||
}
|
||||
|
||||
@@ -169,11 +169,11 @@ configurations.all { config ->
|
||||
}
|
||||
|
||||
if (geckoviewModules.contains(module) && !isLite) {
|
||||
warn("Substituting a geckoview omni build into a lite dependency. Add ac_add_options --enable-geckoview-lite to ${mozconfig.mozconfig.path} to fix this.")
|
||||
warn("Substituting a geckoview omni build into a lite dependency. Add ac_add_options --enable-geckoview-lite to your mozconfig to fix this.")
|
||||
} else if (geckoviewOmniModules.contains(module) && isLite) {
|
||||
// Substituting lite into omni is unlikely to work at
|
||||
// all so we just error out here.
|
||||
throw new GradleException("Substituting a geckoview lite build into an omni dependency. Remove ac_add_options --enable-geckoview-lite in ${mozconfig.mozconfig.path} to fix this.")
|
||||
throw new GradleException("Substituting a geckoview lite build into an omni dependency. Remove ac_add_options --enable-geckoview-lite from your mozconfig to fix this.")
|
||||
}
|
||||
|
||||
log("Substituting ${group}:${dependency.requested.module} with local GeckoView ${group}:${name} in ${config}")
|
||||
|
||||
@@ -23,7 +23,7 @@ android-gradle-build:
|
||||
MOZ_OBJDIR: obj-firefox
|
||||
PERFHERDER_EXTRA_OPTIONS: android-gradle-build
|
||||
TINDERBOX_OUTPUT: '1'
|
||||
max-run-time: 3600
|
||||
max-run-time: 7200
|
||||
run-on-repo-type: [hg]
|
||||
run:
|
||||
using: run-task
|
||||
|
||||
Reference in New Issue
Block a user