Maxim Evtush
cf2fcffffe
Automatic update from web-platform-tests Fix Typo in Comments for gitignore and serve Tools (#53160) * Update serve.py * Update gitignore.py -- wpt-commits: 93054de2bd8a773cab6da0d82e463c76ed3fb12b wpt-pr: 53160
269 lines
9.9 KiB
Python
269 lines
9.9 KiB
Python
import re
|
|
import os
|
|
import itertools
|
|
from collections import defaultdict
|
|
from typing import (Any, Dict, Iterable, List, MutableMapping, Optional, Pattern, Tuple, TypeVar,
|
|
Union, cast)
|
|
|
|
|
|
T = TypeVar('T')
|
|
|
|
end_space = re.compile(r"([^\\]\s)*$")
|
|
|
|
|
|
def fnmatch_translate(pat: bytes) -> Tuple[bool, Pattern[bytes]]:
|
|
parts = []
|
|
seq: Optional[int] = None
|
|
i = 0
|
|
any_char = b"[^/]"
|
|
if pat[0:1] == b"/":
|
|
parts.append(b"^")
|
|
pat = pat[1:]
|
|
else:
|
|
# By default match the entire path up to a /
|
|
# but if / doesn't appear in the pattern we will mark is as
|
|
# a name pattern and just produce a pattern that matches against
|
|
# the filename
|
|
parts.append(b"^(?:.*/)?")
|
|
|
|
name_pattern = True
|
|
if pat[-1:] == b"/":
|
|
# If the last character is / match this directory or any subdirectory
|
|
pat = pat[:-1]
|
|
suffix = b"(?:/|$)"
|
|
else:
|
|
suffix = b"$"
|
|
while i < len(pat):
|
|
c = pat[i:i+1]
|
|
if c == b"\\":
|
|
if i < len(pat) - 1:
|
|
i += 1
|
|
c = pat[i:i+1]
|
|
parts.append(re.escape(c))
|
|
else:
|
|
raise ValueError
|
|
elif seq is not None:
|
|
# TODO: this doesn't really handle invalid sequences in the right way
|
|
if c == b"]":
|
|
seq = None
|
|
if parts[-1] == b"[":
|
|
parts = parts[:-1]
|
|
elif parts[-1] == b"^" and parts[-2] == b"[":
|
|
raise ValueError
|
|
else:
|
|
parts.append(c)
|
|
elif c == b"-":
|
|
parts.append(c)
|
|
elif c == b"[":
|
|
raise ValueError
|
|
else:
|
|
parts.append(re.escape(c))
|
|
elif c == b"[":
|
|
parts.append(b"[")
|
|
if i < len(pat) - 1 and pat[i+1:i+2] in (b"!", b"^"):
|
|
parts.append(b"^")
|
|
i += 1
|
|
seq = i
|
|
elif c == b"*":
|
|
if i < len(pat) - 1 and pat[i+1:i+2] == b"*":
|
|
if i > 0 and pat[i-1:i] != b"/":
|
|
raise ValueError
|
|
parts.append(b".*")
|
|
i += 1
|
|
if i < len(pat) - 1 and pat[i+1:i+2] != b"/":
|
|
raise ValueError
|
|
else:
|
|
parts.append(any_char + b"*")
|
|
elif c == b"?":
|
|
parts.append(any_char)
|
|
elif c == b"/" and not seq:
|
|
name_pattern = False
|
|
parts.append(c)
|
|
else:
|
|
parts.append(re.escape(c))
|
|
i += 1
|
|
|
|
if name_pattern:
|
|
parts[0] = b"^"
|
|
|
|
if seq is not None:
|
|
raise ValueError
|
|
parts.append(suffix)
|
|
try:
|
|
return name_pattern, re.compile(b"".join(parts))
|
|
except Exception:
|
|
raise ValueError
|
|
|
|
# Regexp matching rules that have to be converted to patterns
|
|
pattern_re = re.compile(br".*[\*\[\?]")
|
|
|
|
|
|
def parse_line(line: bytes) -> Optional[Tuple[bool, bool, bool, Union[Tuple[bytes, ...], Tuple[bool, Pattern[bytes]]]]]:
|
|
line = line.rstrip()
|
|
if not line or line[0:1] == b"#":
|
|
return None
|
|
|
|
invert = line[0:1] == b"!"
|
|
if invert:
|
|
line = line[1:]
|
|
|
|
dir_only = line[-1:] == b"/"
|
|
|
|
if dir_only:
|
|
line = line[:-1]
|
|
|
|
# Could make a special case for **/foo, but we don't have any patterns like that
|
|
if not invert and not pattern_re.match(line):
|
|
literal = True
|
|
pattern: Union[Tuple[bytes, ...], Tuple[bool, Pattern[bytes]]] = tuple(line.rsplit(b"/", 1))
|
|
else:
|
|
pattern = fnmatch_translate(line)
|
|
literal = False
|
|
|
|
return invert, dir_only, literal, pattern
|
|
|
|
|
|
class PathFilter:
|
|
def __init__(self, root: bytes, extras: Optional[List[bytes]] = None, cache: Optional[MutableMapping[bytes, bool]] = None) -> None:
|
|
if root:
|
|
ignore_path: Optional[bytes] = os.path.join(root, b".gitignore")
|
|
else:
|
|
ignore_path = None
|
|
if not ignore_path and not extras:
|
|
self.trivial = True
|
|
return
|
|
self.trivial = False
|
|
|
|
self.literals_file: Dict[Optional[bytes], Dict[bytes, List[Tuple[bool, Pattern[bytes]]]]] = defaultdict(dict)
|
|
self.literals_dir: Dict[Optional[bytes], Dict[bytes, List[Tuple[bool, Pattern[bytes]]]]] = defaultdict(dict)
|
|
self.patterns_file: List[Tuple[Tuple[bool, Pattern[bytes]], List[Tuple[bool, Pattern[bytes]]]]] = []
|
|
self.patterns_dir: List[Tuple[Tuple[bool, Pattern[bytes]], List[Tuple[bool, Pattern[bytes]]]]] = []
|
|
|
|
if cache is None:
|
|
cache = {}
|
|
self.cache: MutableMapping[bytes, bool] = cache
|
|
|
|
if extras is None:
|
|
extras = []
|
|
|
|
if ignore_path and os.path.exists(ignore_path):
|
|
args: Tuple[Optional[bytes], List[bytes]] = (ignore_path, extras)
|
|
else:
|
|
args = None, extras
|
|
self._read_ignore(*args)
|
|
|
|
def _read_ignore(self, ignore_path: Optional[bytes], extras: List[bytes]) -> None:
|
|
if ignore_path is not None:
|
|
with open(ignore_path, "rb") as f:
|
|
for line in f:
|
|
self._read_line(line)
|
|
for line in extras:
|
|
self._read_line(line)
|
|
|
|
def _read_line(self, line: bytes) -> None:
|
|
parsed = parse_line(line)
|
|
if not parsed:
|
|
return
|
|
invert, dir_only, literal, rule = parsed
|
|
|
|
if invert:
|
|
# For exclude rules, we attach the rules to all preceeding patterns, so
|
|
# that we can match patterns out of order and check if they were later
|
|
# overridden by an exclude rule
|
|
assert not literal
|
|
rule = cast(Tuple[bool, Pattern[bytes]], rule)
|
|
if not dir_only:
|
|
rules_iter: Iterable[Tuple[Any, List[Tuple[bool, Pattern[bytes]]]]] = itertools.chain(
|
|
itertools.chain(*(item.items() for item in self.literals_dir.values())),
|
|
itertools.chain(*(item.items() for item in self.literals_file.values())),
|
|
self.patterns_dir,
|
|
self.patterns_file)
|
|
else:
|
|
rules_iter = itertools.chain(
|
|
itertools.chain(*(item.items() for item in self.literals_dir.values())),
|
|
self.patterns_dir)
|
|
|
|
for rules in rules_iter:
|
|
rules[1].append(rule)
|
|
else:
|
|
if literal:
|
|
rule = cast(Tuple[bytes, ...], rule)
|
|
if len(rule) == 1:
|
|
dir_name, pattern = None, rule[0] # type: Tuple[Optional[bytes], bytes]
|
|
else:
|
|
dir_name, pattern = rule
|
|
self.literals_dir[dir_name][pattern] = []
|
|
if not dir_only:
|
|
self.literals_file[dir_name][pattern] = []
|
|
else:
|
|
rule = cast(Tuple[bool, Pattern[bytes]], rule)
|
|
self.patterns_dir.append((rule, []))
|
|
if not dir_only:
|
|
self.patterns_file.append((rule, []))
|
|
|
|
def filter(self,
|
|
iterator: Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]
|
|
) -> Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]:
|
|
empty: Dict[Any, Any] = {}
|
|
for dirpath, dirnames, filenames in iterator:
|
|
orig_dirpath = dirpath
|
|
path_sep = os.path.sep.encode()
|
|
if path_sep != b"/":
|
|
dirpath = dirpath.replace(path_sep, b"/")
|
|
|
|
keep_dirs: List[Tuple[bytes, T]] = []
|
|
keep_files: List[Tuple[bytes, T]] = []
|
|
|
|
for iter_items, literals, patterns, target, suffix in [
|
|
(dirnames, self.literals_dir, self.patterns_dir, keep_dirs, b"/"),
|
|
(filenames, self.literals_file, self.patterns_file, keep_files, b"")]:
|
|
for item in iter_items:
|
|
name = item[0]
|
|
if dirpath:
|
|
path = b"%s/%s" % (dirpath, name) + suffix
|
|
else:
|
|
path = name + suffix
|
|
if path in self.cache:
|
|
if not self.cache[path]:
|
|
target.append(item)
|
|
continue
|
|
for rule_dir in [None, dirpath if dirpath != b"." else b""]:
|
|
if name in literals.get(rule_dir, empty):
|
|
exclude = literals[rule_dir][name]
|
|
if not any(rule.match(name if name_only else path)
|
|
for name_only, rule in exclude):
|
|
# Skip this item
|
|
self.cache[path] = True
|
|
break
|
|
else:
|
|
for (component_only, pattern), exclude in patterns:
|
|
if component_only:
|
|
match = pattern.match(name)
|
|
else:
|
|
match = pattern.match(path)
|
|
if match:
|
|
if not any(rule.match(name if name_only else path)
|
|
for name_only, rule in exclude):
|
|
# Skip this item
|
|
self.cache[path] = True
|
|
break
|
|
else:
|
|
self.cache[path] = False
|
|
target.append(item)
|
|
|
|
dirnames[:] = keep_dirs
|
|
assert not any(b".git" == name for name, _ in dirnames)
|
|
yield orig_dirpath, dirnames, keep_files
|
|
|
|
def __call__(self,
|
|
iterator: Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]
|
|
) -> Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]:
|
|
if self.trivial:
|
|
return iterator
|
|
|
|
return self.filter(iterator)
|
|
|
|
|
|
def has_ignore(dirpath: bytes) -> bool:
|
|
return os.path.exists(os.path.join(dirpath, b".gitignore"))
|