Files
sousa-gecko/gfx/angle/update-angle.py
T
Erich Gubler 6a2def449e Bug 1908744, part 5 - chore(angle): update base to 9eadd12e33d65b8c9101ddef1d77d1c9a07e07bf r=gfx-reviewers,nical
It's been years in the making! 😅 I return to revendoring ANGLE with _tons_ of support from @jnicol, who has done much (most?) of the work associated with this patch. The most important call-outs I can give here are:

- This patch has are a _huge_ number of changes, but don't feel like you need to scrutinize everything exhaustively. This patch only imports changes from upstream ANGLE. In general, we don't expect reviewers to exhaustively review vendored source for code in, say, `third_party/` (which we eventually hope to move our checkout to).
- This revendoring required nontrivial changes to the update script, which is in a previous patch in this stack.
- Some Mochitests required adjustments in a previous patch in this stack.

Differential Revision: https://phabricator.services.mozilla.com/D306414
2026-07-10 16:30:03 +00:00

622 lines
18 KiB
Python
Executable File

#! /usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
assert __name__ == "__main__"
r"""
To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools and python3.
Upstream: https://chromium.googlesource.com/angle/angle
Our repo: https://github.com/mozilla/angle
It has branches like 'firefox-60' which is the branch we use for pulling into
Gecko with this script.
This script leaves a record of the merge-base and cherry-picks that we pull into
Gecko. (gfx/angle/cherries.log)
You can find out the current git hash of the ANGLE shipping in Chrome Beta using this command:
curl https://chromiumdash.appspot.com/fetch_releases?platform=Win32&channel=beta' | jq -r '.[0].hashes.angle
(or just go to https://chromiumdash.appspot.com/fetch_releases?platform=Win32&channel=beta and look for yourself)
## Usage
Prepare your env:
* If in `cmd`:
`export PATH="$PATH:/path/to/depot_tools"`
* If in `powershell`:
`$env:Path += ";C:\path\to\depot_tools"`
If this is a new repo, don't forget:
~~~
# In the angle repo:
./scripts/bootstrap.py
gclient sync
~~~
Update: (in the angle repo)
~~~
# In the angle repo:
/path/to/gecko/gfx/angle/update-angle.py origin
git push moz # Push the firefox-XX branch to github.com/mozilla/angle
~~~~
"""
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
from typing import * # mypy annotations
REPO_DIR = pathlib.Path.cwd()
GECKO_ANGLE_DIR = pathlib.Path(__file__).parent
OUT_DIR = pathlib.Path("out")
COMMON_HEADER = [
"# Generated by update-angle.py",
"",
'include("../../moz.build.common")',
]
ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"]
CHECK_ONLY = False
args = sys.argv[1:]
while True:
arg = args.pop(0)
if arg == "--check":
CHECK_ONLY = True
continue
args.insert(0, arg)
break
GN_ENV = dict(os.environ)
GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"
(GIT_REMOTE,) = args # Not always 'origin'!
# ------------------------------------------------------------------------------
def run_checked(*args, **kwargs):
print(" ", args)
sys.stdout.flush()
return subprocess.run(args, check=True, **kwargs)
def sorted_items(x):
for k in sorted(x.keys()):
yield (k, x[k])
def collapse_dotdots(path):
split = path.split("/")
ret = []
for x in split:
if x == ".." and ret:
ret.pop()
continue
ret.append(x)
continue
return "/".join(ret)
def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
visited_keys: Set[str] = set()
def recurse(key):
if key in visited_keys:
return
visited_keys.add(key)
t = pre_recurse_func(key)
try:
(next_keys, post_recurse_func) = t
except ValueError:
(next_keys,) = t
post_recurse_func = None
for x in next_keys:
recurse(x)
if post_recurse_func:
post_recurse_func(key)
return
for x in root_keys:
recurse(x)
return
# ------------------------------------------------------------------------------
print("Importing graph")
# shutil.rmtree(str(OUT_DIR), True)
OUT_DIR.mkdir(exist_ok=True)
GN_ARGS = b"""
# Build arguments go here.
# See "gn args <out_dir> --list" for available build arguments.
is_clang = true
is_debug = false
angle_build_all = false
angle_enable_abseil = false
angle_enable_essl = true
angle_enable_gl = false
angle_enable_glsl = true
angle_enable_null = false
angle_enable_share_context_lock = true
angle_enable_vulkan = false
angle_enable_wgpu = false
angle_has_astc_encoder = false
use_custom_libcxx = false
"""[
1:
]
args_gn_path = OUT_DIR / "args.gn"
args_gn_path.write_bytes(GN_ARGS)
try:
run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV)
except subprocess.CalledProcessError:
sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n")
exit(1)
p = run_checked(
"python3",
"scripts/export_targets.py",
str(OUT_DIR),
*ROOTS,
stdout=subprocess.PIPE,
shell=True,
env=GN_ENV,
)
# -
print("\nProcessing graph")
libraries = json.loads(p.stdout.decode())
# -
# HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
# platforms.
# descs["//:angle_common"]["sources"] +=
EXTRA_ANGLE_COMMON_SOURCES = [
"//src/common/system_utils_apple.cpp",
"//src/common/system_utils_linux.cpp",
"//src/common/system_utils_mac.cpp",
"//src/common/system_utils_posix.cpp",
]
angle_common = libraries["//:angle_common"]
angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES
angle_common["sources"] = sorted(angle_common["sources"])
# Similar hack as above, to ensure apple-specific translator sources are
# included. This used to be achieved with the gn arg
# "angle_enable_apple_translator_workarounds", but it has been removed upstream.
EXTRA_TRANSLATOR_APPLE_SOURCES = [
"//src/compiler/translator/tree_ops/glsl/apple/AddAndTrueToLoopCondition.cpp",
"//src/compiler/translator/tree_ops/glsl/apple/RewriteRowMajorMatrices.cpp",
"//src/compiler/translator/tree_ops/glsl/apple/UnfoldShortCircuitAST.cpp",
]
translator = libraries["//:translator"]
translator["sources"] += EXTRA_TRANSLATOR_APPLE_SOURCES
translator["sources"] = sorted(translator["sources"])
# xxhash.c is pulled into both angle_common and angle_common_shader_state
# causing linker errors. Since these libraries are both always linked into the
# same final library, we can simply remove it from one or the other.
angle_common_shader_state = libraries["//:angle_common_shader_state"]
angle_common_shader_state["sources"].remove("//src/common/third_party/xxhash/xxhash.c")
# angle_version_info.cpp is pulled into both translator and libGLESv2 causing
# linker errors. libGLESv2 depends on translator, so remove the duplicate from
# libGLESv2.
libglesv2 = libraries["//:libGLESv2"]
libglesv2["sources"].remove("//src/common/angle_version_info.cpp")
# -
# Reuse our own zlib
del libraries["//third_party/zlib:zlib"]
# -
if CHECK_ONLY:
print("\n--check complete.")
exit(0)
# ------------------------------------------------------------------------------
# Output to moz.builds
import vendor_from_git
print("")
vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)
# --
def sortedi(x):
return sorted(x, key=str.lower)
def append_arr(dest, name, vals, indent=0):
if not vals:
return
dest.append("{}{} += [".format(" " * 4 * indent, name))
for x in sortedi(vals):
dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x))
dest.append("{}]".format(" " * 4 * indent))
dest.append("")
return
REGISTERED_DEFINES = {
"ADLER32_SIMD_SSSE3": False,
"ANGLE_CAPTURE_ENABLED": True,
"ANGLE_DISABLE_POOL_ALLOC": True,
"ANGLE_DISPATCH_LIBRARY": True,
"ANGLE_EGL_LIBRARY_NAME": False,
"ANGLE_ENABLE_CONTEXT_MUTEX": True,
"ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
"ANGLE_ENABLE_CONTEXT_MUTEX": True,
"ANGLE_ENABLE_D3D11": True,
"ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
"ANGLE_ENABLE_D3D9": True,
"ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
"ANGLE_ENABLE_NULL": False,
"ANGLE_ENABLE_OPENGL": False,
"ANGLE_ENABLE_OPENGL_NULL": False,
"ANGLE_ENABLE_ESSL": True,
"ANGLE_ENABLE_GLSL": True,
"ANGLE_ENABLE_HLSL": True,
"ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
"ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
"ANGLE_GLESV2_LIBRARY_NAME": True,
"ANGLE_HAS_ASTCENC": True,
"ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
"ANGLE_IS_64_BIT_CPU": False,
"ANGLE_IS_WIN": False,
"ANGLE_MESA_EGL_LIBRARY_NAME": True,
"ANGLE_MESA_GLESV2_LIBRARY_NAME": True,
"ANGLE_OUTSIDE_WEBKIT": True,
"ANGLE_PLATFORM_EXPORT": True,
"ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
"ANGLE_SHARED_LIBVULKAN": True,
"ANGLE_USE_CUSTOM_LIBVULKAN": True,
"ANGLE_USE_EGL_LOADER": True,
"ANGLE_VK_LAYERS_DIR": True,
"ANGLE_VK_MOCK_ICD_JSON": True,
"ANGLE_VMA_VERSION": True,
"ANGLE_VULKAN_SECONDARIES_EGL_LIBRARY_NAME": True,
"ANGLE_VULKAN_SECONDARIES_GLESV2_LIBRARY_NAME": True,
"ASTCENC_DECOMPRESS_ONLY": True,
"CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
"CHROMIUM_BUILD": False,
"COMPONENT_BUILD": False,
"CRC32_SIMD_SSE42_PCLMUL": False,
"DEFLATE_FILL_WINDOW_SSE2": False,
"DYNAMIC_ANNOTATIONS_ENABLED": True,
"EGL_EGL_PROTOTYPES": True,
"EGL_EGLEXT_PROTOTYPES": True,
"EGLAPI": True,
"FIELDTRIAL_TESTING_ENABLED": False,
"FULL_SAFE_BROWSING": False,
"GL_API": True,
"GL_APICALL": True,
"GL_GLES_PROTOTYPES": True,
"GL_GLEXT_PROTOTYPES": True,
"GPU_INFO_USE_SETUPAPI": True,
"INFLATE_CHUNK_READ_64LE": False,
"INFLATE_CHUNK_SIMD_SSE2": False,
"LIBANGLE_IMPLEMENTATION": True,
"LIBEGL_IMPLEMENTATION": True,
"LIBGLESV2_IMPLEMENTATION": True,
"NOMINMAX": True,
"NO_TCMALLOC": False,
# Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
"NTDDI_VERSION": False,
"PSAPI_VERSION": False,
"SAFE_BROWSING_CSD": False,
"SAFE_BROWSING_DB_LOCAL": False,
"UNICODE": True,
"USE_AURA": False,
"V8_DEPRECATION_WARNINGS": False,
"VK_USE_PLATFORM_WIN32_KHR": False,
"WIN32": False,
"WIN32_LEAN_AND_MEAN": False,
"WINAPI_FAMILY": False,
"WINCRYPT_USE_SYMBOL_PREFIX": True,
"WINVER": True,
# Otherwise:
# gfx/angle/targets/libANGLE
# In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
# In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
# C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
# _In_ const POINTER_INFO& pointerInfo
# ^
"WTF_USE_DYNAMIC_ANNOTATIONS": False,
"X86_WINDOWS": False,
"ZLIB_IMPLEMENTATION": True,
"_ATL_NO_OPENGL": True,
"_CRT_NONSTDC_NO_DEPRECATE": True,
"_CRT_NONSTDC_NO_WARNINGS": True,
"_CRT_RAND_S": True,
"_CRT_SECURE_NO_DEPRECATE": True,
"_DEBUG": False,
"_HAS_EXCEPTIONS": True,
"_HAS_ITERATOR_DEBUGGING": False,
"_LIBCPP_HARDENING_MODE": True,
"_SCL_SECURE_NO_DEPRECATE": True,
"_SECURE_ATL": True,
"_UNICODE": True,
"_USING_V110_SDK71_": False,
"_WIN32_WINNT": False,
"_WINDOWS": False,
"_WINSOCK_DEPRECATED_NO_WARNINGS": True,
"__STD_C": False,
# clang specific
"CR_CLANG_REVISION": True,
"NDEBUG": False,
"NVALGRIND": False,
"_HAS_NODISCARD": False,
}
# -
print("\nRun actions")
required_files: Set[str] = set()
required_files.add("//LICENSE")
run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True)
required_files.add("//out/gen/angle/angle_commit.h")
run_checked("ninja", "-C", str(OUT_DIR), ":angle_program_version_id", shell=True)
required_files.add("//out/gen/angle/ANGLEShaderProgramVersion.h")
# -
# Export our targets
print("\nExport targets")
# Clear our dest directories
targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets")
checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout")
shutil.rmtree(targets_dir, True)
shutil.rmtree(checkout_dir, True)
targets_dir.mkdir(exist_ok=True)
checkout_dir.mkdir(exist_ok=True)
# -
RE_TARGET_NAME = re.compile("//(.*):(.+)")
def export_target(target_full_name) -> Set[str]:
# print(' ', target_full_name)
descs = libraries
desc = descs[target_name]
flat = desc
m = RE_TARGET_NAME.match(target_name)
assert m, target_name
name = m.group(2)
required_files: Set[str] = set(flat["sources"])
# Create our manifest lines
target_dir = targets_dir / name
target_dir.mkdir(exist_ok=True)
lines = list(COMMON_HEADER)
lines.append("")
for x in sorted(set(desc["defines"])):
try:
(k, v) = x.split("=", 1)
if v.startswith('"'):
v = f"'{v}'"
else:
v = f'"{v}"'
except ValueError:
(k, v) = (x, "True")
line = f'DEFINES["{k}"] = {v}'
try:
if REGISTERED_DEFINES[k] == False:
line = "# " + line
except KeyError:
print(f"[{name}] Unrecognized define: {k} = {v}")
line = "# Unrecognized: " + line
lines.append(line)
lines.append("")
cxxflags = set(desc["cflags"] + desc["cflags_cc"])
def fixup_paths(listt):
for x in set(listt):
assert x.startswith("//"), x
yield "../../checkout/" + x[2:]
sources_by_config: Dict[str, List[str]] = {}
extras: Dict[str, str] = dict()
for x in fixup_paths(flat["sources"]):
# print(' '*5, x)
(b, e) = x.rsplit(".", 1)
if e in ["h", "hpp", "y", "l", "inc", "inl"]:
continue
elif e in ["cpp", "cc", "c"]:
if b.endswith("_win") or b.endswith("_win32"):
config = 'CONFIG["OS_ARCH"] == "WINNT"'
elif b.endswith("_linux"):
# Include these on BSDs too.
config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
elif (
b.endswith("_apple")
or b.endswith("_mac")
or x in fixup_paths(EXTRA_TRANSLATOR_APPLE_SOURCES)
):
config = 'CONFIG["OS_ARCH"] == "Darwin"'
elif b.endswith("_posix"):
config = 'CONFIG["OS_ARCH"] != "WINNT"'
else:
config = "" # None can't compare against str.
sources_by_config.setdefault(config, []).append(x)
continue
elif e == "rc":
assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x)
extras["RCFILE"] = f'"{x}"'
continue
elif e == "def":
assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x)
extras["DEFFILE"] = f'"{x}"'
continue
else:
assert False, ("Unhandled ext:", x)
ldflags = set(desc["ldflags"])
DEF_PREFIX = "/DEF:"
for x in set(ldflags):
if x.startswith(DEF_PREFIX):
def_path = x[len(DEF_PREFIX) :]
required_files.add(def_path)
assert "DEFFILE" not in extras
ldflags.remove(x)
def_path = str(OUT_DIR) + "/" + def_path
def_path = "//" + collapse_dotdots(def_path)
def_rel_path = list(fixup_paths([def_path]))[0]
extras["DEFFILE"] = '"{}"'.format(def_rel_path)
elif x.startswith("/PDBSourcePath:"):
ldflags.remove(x)
os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", []))))
# Prevent //third_party/llvm-build/Release+Asserts/lib/clang/23/lib/windows/clang_rt.builtins-x86_64.lib
# being brought in as a dependency.
os_libs = [x for x in os_libs if not x.startswith("//")]
def append_arr_commented(dest, name, src):
lines = []
append_arr(lines, name, src)
def comment(x):
if x:
x = "# " + x
return x
lines = map(comment, lines)
dest += lines
append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"]))
append_arr_commented(lines, "CXXFLAGS", cxxflags)
for config, v in sorted_items(sources_by_config):
indent = 0
if config:
lines.append("if {}:".format(config))
indent = 1
append_arr(lines, "SOURCES", v, indent=indent)
dep_libs: Set[str] = set()
for dep_full_name in set(flat["dep_libs"]):
assert dep_full_name.startswith("//"), dep_name
(_, dep_name) = dep_full_name.split(":")
dep_libs.add(dep_name)
dep_dirs = set(dep_libs)
dep_dirs.discard("zlib")
# Those directories are added by gfx/angle/moz.build.
already_added_dirs = [
"angle_common",
"angle_common_shader_state",
"translator",
"libEGL",
"libGLESv2",
]
append_arr(lines, "USE_LIBS", dep_libs)
append_arr(
lines, "DIRS", ["../" + x for x in dep_dirs if x not in already_added_dirs]
)
# Only add to OS_LIBS on windows. This avoids attempting to link to
# "synchronization" on other platforms where it doesn't exist.
# This only works because we don't need any OS_LIBS on other platforms.
# Switching to gn_processor.py in bug 2035532 will fix this properly.
if os_libs:
lines.append('if CONFIG["OS_ARCH"] == "WINNT":')
append_arr(lines, "OS_LIBS", os_libs, indent=1)
append_arr_commented(lines, "LDFLAGS", ldflags)
for k, v in sorted(extras.items()):
lines.append("{} = {}".format(k, v))
lib_type = desc["type"]
if lib_type == "shared_library":
lines.append(f'GeckoSharedLibrary("{name}")')
elif lib_type == "static_library":
lines.append(f'Library("{name}")')
else:
assert False, lib_type
lines.append("")
# EOF newline
# Write it out
mozbuild = target_dir / "moz.build"
print(" ", " ", f"Writing {mozbuild}")
data = b"\n".join((x.encode() for x in lines))
mozbuild.write_bytes(data)
return required_files
# -
for target_name in libraries:
reqs = export_target(target_name)
required_files |= reqs
# Copy all the files
print("\nMigrate required files")
i = 0
for x in required_files:
i += 1
sys.stdout.write(f"\r Copying {i}/{len(required_files)}")
sys.stdout.flush()
assert x.startswith("//"), x
x = x[2:]
src = REPO_DIR / x
dest = checkout_dir / x
dest.parent.mkdir(parents=True, exist_ok=True)
data = src.read_bytes()
data = data.replace(b"\r\n", b"\n")
dest.write_bytes(data)
print("\n\nDone")