135 lines
4.2 KiB
Python
135 lines
4.2 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
"""
|
|
Support for optimizing tasks based on the set of files that have changed.
|
|
"""
|
|
|
|
import functools
|
|
import logging
|
|
import os
|
|
from subprocess import CalledProcessError
|
|
|
|
from mozpack.path import join as join_path
|
|
from mozpack.path import match as mozpackmatch
|
|
from mozversioncontrol import InvalidRepoPath, get_repository_object
|
|
|
|
from gecko_taskgraph import GECKO
|
|
from gecko_taskgraph.util.hg import get_json_pushchangedfiles
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@functools.cache
|
|
def get_changed_files(repository, revision):
|
|
"""
|
|
Get the set of files changed in the push headed by the given revision.
|
|
Responses are cached, so multiple calls with the same arguments are OK.
|
|
"""
|
|
try:
|
|
return get_json_pushchangedfiles(repository, revision)["files"]
|
|
except KeyError:
|
|
# We shouldn't hit this error in CI.
|
|
if os.environ.get("MOZ_AUTOMATION"):
|
|
raise
|
|
|
|
# We're likely on an unpublished commit, grab changed files from
|
|
# version control.
|
|
return get_locally_changed_files(GECKO)
|
|
|
|
|
|
def check(params, file_patterns):
|
|
"""Determine whether any of the files changed in the indicated push to
|
|
https://hg.mozilla.org match any of the given file patterns."""
|
|
repository = params.get("head_repository")
|
|
revision = params.get("head_rev")
|
|
if not repository or not revision:
|
|
logger.warning(
|
|
"Missing `head_repository` or `head_rev` parameters; "
|
|
"assuming all files have changed"
|
|
)
|
|
return True
|
|
|
|
changed_files = get_changed_files(repository, revision)
|
|
|
|
if "comm_head_repository" in params:
|
|
repository = params.get("comm_head_repository")
|
|
revision = params.get("comm_head_rev")
|
|
if not revision:
|
|
logger.warning(
|
|
"Missing `comm_head_rev` parameters; assuming all files have changed"
|
|
)
|
|
return True
|
|
|
|
changed_files |= {
|
|
join_path("comm", file) for file in get_changed_files(repository, revision)
|
|
}
|
|
|
|
for pattern in file_patterns:
|
|
for path in changed_files:
|
|
if mozpackmatch(path, pattern):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def _get_locally_changed_files(repo):
|
|
try:
|
|
vcs = get_repository_object(repo)
|
|
s = set(vcs.get_outgoing_files("AM"))
|
|
return s
|
|
except (InvalidRepoPath, CalledProcessError):
|
|
return set()
|
|
|
|
|
|
class PreloadedGetLocallyChangedFiles:
|
|
"""
|
|
Function-like class that performs eager computation of _get_locally_changed_files
|
|
for what looks the default repo.
|
|
|
|
The rationale is the following:
|
|
- computing _get_locally_changed_files is relatively slow (~600ms)
|
|
- it's already done through an external command
|
|
|
|
So we do that in a background thread as soon as possible, so that at the
|
|
point when we need the result, it's already "prefetched".
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.preloaded_repo = None
|
|
self.preloading_thread = None
|
|
self.preloaded_answer = None
|
|
|
|
def preload(self, repo):
|
|
"""
|
|
Fire off preloading of get_locally_changed_files(repo).
|
|
|
|
For the sake of simplicity, there can be only one preloaded repo.
|
|
"""
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
if self.preloaded_repo is not None:
|
|
raise ValueError("Can only preload one repo")
|
|
|
|
self.preloaded_repo = Path(repo)
|
|
|
|
def preloading():
|
|
self.preloaded_answer = _get_locally_changed_files(self.preloaded_repo)
|
|
|
|
self.preloading_thread = threading.Thread(target=preloading, daemon=True)
|
|
self.preloading_thread.start()
|
|
|
|
@functools.cache
|
|
def __call__(self, repo):
|
|
if repo == self.preloaded_repo:
|
|
# A thread can be joined many times, but it's going to happen only
|
|
# once, thanks to @memoize.
|
|
self.preloading_thread.join()
|
|
return self.preloaded_answer
|
|
return _get_locally_changed_files(repo)
|
|
|
|
|
|
get_locally_changed_files = PreloadedGetLocallyChangedFiles()
|