Alex Hochheiden
c49fedbda1
Updated aiohttp v3.12.12 -> v3.12.13 Updated attrs v23.1.0 -> v23.2.0 Updated blessed v1.19.1 -> v1.21.0 Updated cbor2 v4.0.1 -> v4.1.2 Updated compare-locales v9.0.1 -> v9.0.4 Updated distro v1.8.0 -> v1.9.0 Updated fluent-migrate v0.13.2 -> v0.13.3 Updated importlib-metadata v6.0.0 -> v6.11.0 Updated jsmin v3.0.0 -> v3.0.1 Updated json-e v4.5.3 -> v4.8.0 Updated looseversion v1.0.1 -> v1.3.0 Updated mozilla-taskgraph v3.3.1 -> v3.3.2 Updated multidict v6.4.4 -> v6.5.0 Updated pathspec v0.9.0 -> v0.12.1 Updated pyasn1 v0.4.8 -> v0.6.1 Updated pyasn1-modules v0.2.8 -> v0.4.2 Updated pylru v1.0.9 -> v1.2.1 Updated python-hglib v2.4 -> v2.6.2 Updated redo v2.0.3 -> v2.0.4 Updated requests-unixsocket v0.2.0 -> v0.4.1 Updated responses v0.10.6 -> v0.25.7 Updated rsa v4.9 -> v4.9.1 Updated sentry-sdk v1.14.0 -> v1.45.1 Updated six v1.16.0 -> v1.17.0 Updated tomlkit v0.12.3 -> v0.13.3 Updated tqdm v4.66.3 -> v4.67.1 Updated typing-extensions v4.12.2 -> v4.14.0 Updated voluptuous v0.12.1 -> v0.15.2 Updated yamllint v1.23.0 -> v1.37.1 Differential Revision: https://phabricator.services.mozilla.com/D254096
290 lines
9.5 KiB
Python
290 lines
9.5 KiB
Python
from .AST import (
|
|
Primitive,
|
|
UnaryOp,
|
|
ContextValue,
|
|
BinOp,
|
|
FunctionCall,
|
|
ValueAccess,
|
|
Object,
|
|
List,
|
|
)
|
|
from collections import namedtuple
|
|
import re
|
|
from .shared import TemplateError
|
|
|
|
Token = namedtuple("Token", ["kind", "value", "start", "end"])
|
|
|
|
expectedTokens = [
|
|
"!",
|
|
"(",
|
|
"+",
|
|
"-",
|
|
"[",
|
|
"false",
|
|
"identifier",
|
|
"null",
|
|
"number",
|
|
"string",
|
|
"true",
|
|
"{",
|
|
]
|
|
|
|
|
|
class SyntaxError(TemplateError):
|
|
@classmethod
|
|
def unexpected(cls, got, exp):
|
|
exp = ", ".join(sorted(exp))
|
|
return cls("Found: {} token, expected one of: {}".format(got.value, exp))
|
|
|
|
|
|
class Parser(object):
|
|
def __init__(self, source, tokenizer):
|
|
self.tokens = tokenizer.generate_tokens(source)
|
|
self.source = source
|
|
self.current_token = next(self.tokens)
|
|
self.unaryOpTokens = ["-", "+", "!"]
|
|
self.primitivesTokens = ["number", "null", "true", "false", "string"]
|
|
self.operatorsByPriority = [
|
|
["||"],
|
|
["&&"],
|
|
["in"],
|
|
["==", "!="],
|
|
["<", ">", "<=", ">="],
|
|
["+", "-"],
|
|
["*", "/"],
|
|
["**"],
|
|
]
|
|
|
|
def take_token(self, *kinds):
|
|
if not self.current_token:
|
|
raise SyntaxError("Unexpected end of input")
|
|
if kinds and self.current_token.kind not in kinds:
|
|
raise SyntaxError.unexpected(self.current_token, kinds)
|
|
try:
|
|
self.current_token = next(self.tokens)
|
|
except StopIteration:
|
|
self.current_token = None
|
|
except SyntaxError as exc:
|
|
raise exc
|
|
|
|
def parse(self, level=0):
|
|
"""expr : logicalAnd (OR logicalAnd)*"""
|
|
""" logicalAnd : inStatement (AND inStatement)* """
|
|
""" inStatement : equality (IN equality)* """
|
|
""" equality : comparison (EQUALITY | INEQUALITY comparison)* """
|
|
""" comparison : addition (LESS | GREATER | LESSEQUAL | GREATEREQUAL addition)* """
|
|
""" addition : multiplication (PLUS | MINUS multiplication)* """
|
|
""" multiplication : exponentiation (MUL | DIV exponentiation)* """
|
|
""" exponentiation : propertyAccessOrFunc (EXP exponentiation)* """
|
|
if level == len(self.operatorsByPriority) - 1:
|
|
node = self.parse_property_access_or_func()
|
|
token = self.current_token
|
|
|
|
while token is not None and token.kind in self.operatorsByPriority[level]:
|
|
self.take_token(token.kind)
|
|
node = BinOp(token, self.parse(level), node)
|
|
token = self.current_token
|
|
else:
|
|
node = self.parse(level + 1)
|
|
token = self.current_token
|
|
|
|
while token is not None and token.kind in self.operatorsByPriority[level]:
|
|
self.take_token(token.kind)
|
|
node = BinOp(token, node, self.parse(level + 1))
|
|
token = self.current_token
|
|
|
|
return node
|
|
|
|
def parse_property_access_or_func(self):
|
|
"""propertyAccessOrFunc : unit (accessWithBrackets | DOT id | functionCall)*"""
|
|
node = self.parse_unit()
|
|
token = self.current_token
|
|
operators = ["[", "(", "."]
|
|
while token is not None and token.kind in operators:
|
|
if token.kind == "[":
|
|
node = self.parse_access_with_brackets(node)
|
|
elif token.kind == ".":
|
|
token = self.current_token
|
|
self.take_token(".")
|
|
right_part = Primitive(self.current_token)
|
|
self.take_token("identifier")
|
|
node = BinOp(token, node, right_part)
|
|
elif token.kind == "(":
|
|
node = self.parse_function_call(node)
|
|
token = self.current_token
|
|
return node
|
|
|
|
def parse_unit(self):
|
|
# unit : unaryOp unit | primitives | contextValue | LPAREN expr RPAREN | list | object
|
|
token = self.current_token
|
|
if self.current_token is None:
|
|
raise SyntaxError("Unexpected end of input")
|
|
node = None
|
|
|
|
if token.kind in self.unaryOpTokens:
|
|
self.take_token(token.kind)
|
|
node = UnaryOp(token, self.parse_unit())
|
|
elif token.kind in self.primitivesTokens:
|
|
self.take_token(token.kind)
|
|
node = Primitive(token)
|
|
elif token.kind == "identifier":
|
|
self.take_token(token.kind)
|
|
node = ContextValue(token)
|
|
elif token.kind == "(":
|
|
self.take_token("(")
|
|
node = self.parse()
|
|
if node is None:
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
self.take_token(")")
|
|
elif token.kind == "[":
|
|
node = self.parse_list()
|
|
elif token.kind == "{":
|
|
node = self.parse_object()
|
|
|
|
return node
|
|
|
|
def parse_function_call(self, name):
|
|
"""functionCall: LPAREN (expr ( COMMA expr)*)? RPAREN"""
|
|
args = []
|
|
token = self.current_token
|
|
self.take_token("(")
|
|
|
|
if self.current_token.kind != ")":
|
|
node = self.parse()
|
|
args.append(node)
|
|
|
|
while self.current_token is not None and self.current_token.kind == ",":
|
|
if args[-1] is None:
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
self.take_token(",")
|
|
node = self.parse()
|
|
args.append(node)
|
|
|
|
self.take_token(")")
|
|
node = FunctionCall(token, name, args)
|
|
|
|
return node
|
|
|
|
def parse_list(self):
|
|
"""list: LSQAREBRAKET (expr (COMMA expr)*)? RSQAREBRAKET"""
|
|
arr = []
|
|
token = self.current_token
|
|
self.take_token("[")
|
|
|
|
if self.current_token != "]":
|
|
node = self.parse()
|
|
arr.append(node)
|
|
|
|
while self.current_token and self.current_token.kind == ",":
|
|
if arr[-1] is None:
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
self.take_token(",")
|
|
node = self.parse()
|
|
if node is None:
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
arr.append(node)
|
|
|
|
self.take_token("]")
|
|
node = List(token, arr)
|
|
|
|
return node
|
|
|
|
def parse_access_with_brackets(self, node):
|
|
"""valueAccess : LSQAREBRAKET expr |(expr? COLON expr?) RSQAREBRAKET)"""
|
|
left = None
|
|
right = None
|
|
is_interval = False
|
|
token = self.current_token
|
|
self.take_token("[")
|
|
if self.current_token.kind == "]":
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
if self.current_token.kind != ":":
|
|
left = self.parse()
|
|
if self.current_token.kind == ":":
|
|
is_interval = True
|
|
self.take_token(":")
|
|
if self.current_token.kind != "]":
|
|
right = self.parse()
|
|
|
|
if is_interval and right is None and self.current_token.kind != "]":
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
|
|
self.take_token("]")
|
|
node = ValueAccess(token, node, is_interval, left, right)
|
|
|
|
return node
|
|
|
|
def parse_object(self):
|
|
# """ object : LCURLYBRACE ( STR | ID COLON expr (COMMA STR | ID COLON expr)*)?
|
|
# RCURLYBRACE """
|
|
obj = {}
|
|
objToken = self.current_token
|
|
self.take_token("{")
|
|
token = self.current_token
|
|
|
|
while token is not None and (
|
|
token.kind == "string" or token.kind == "identifier"
|
|
):
|
|
key = token.value
|
|
if token.kind == "string":
|
|
key = parse_string(key)
|
|
self.take_token(token.kind)
|
|
self.take_token(":")
|
|
value = self.parse()
|
|
if value is None:
|
|
raise SyntaxError.unexpected(self.current_token, expectedTokens)
|
|
obj[key] = value
|
|
if self.current_token and self.current_token.kind == "}":
|
|
break
|
|
else:
|
|
self.take_token(",")
|
|
token = self.current_token
|
|
|
|
self.take_token("}")
|
|
node = Object(objToken, obj)
|
|
|
|
return node
|
|
|
|
|
|
def parse_string(string):
|
|
return string[1:-1]
|
|
|
|
|
|
class Tokenizer(object):
|
|
def __init__(self, ignore, patterns, tokens):
|
|
self.ignore = ignore
|
|
self.patterns = patterns
|
|
self.tokens = tokens
|
|
# build a regular expression to generate a sequence of tokens
|
|
token_patterns = [
|
|
"({})".format(self.patterns.get(t, re.escape(t))) for t in self.tokens
|
|
]
|
|
if self.ignore:
|
|
token_patterns.append("(?:{})".format(self.ignore))
|
|
self.token_re = re.compile("^(?:" + "|".join(token_patterns) + ")")
|
|
|
|
def generate_tokens(self, source):
|
|
offset = 0
|
|
while True:
|
|
start = offset
|
|
remainder = source[offset:]
|
|
mo = self.token_re.match(remainder)
|
|
if not mo:
|
|
if remainder:
|
|
raise SyntaxError(
|
|
"Unexpected input for '{}' at '{}'".format(source, remainder)
|
|
)
|
|
break
|
|
offset += mo.end()
|
|
|
|
# figure out which token matched (note that idx is 0-based)
|
|
indexes = [idx for idx, grp in enumerate(mo.groups()) if grp is not None]
|
|
if indexes:
|
|
idx = indexes[0]
|
|
yield Token(
|
|
kind=self.tokens[idx],
|
|
value=mo.group(idx + 1), # (mo.group is 1-based)
|
|
start=start,
|
|
end=offset,
|
|
)
|