Valentin Pollet
72284c6525
Updated moz.yaml with two extra include globs as upstream refactored llama-model.cpp into multiple smaller files in src/models. Updated cmakeparser.py to handle globs in CMakeLists.txt (introduced upstream to compile models/*.cpp). Fixed run_parser.py needed to discard files excluded by moz.yaml. Updated moz.build to provide src/ as an include directory during build. This was needed to compile models/*.cpp which include llama-model.h without a relative path. Updated moz.build to export ggml/include/gguf.h as upstream now includes it in llama.h. Differential Revision: https://phabricator.services.mozilla.com/D307355
376 lines
14 KiB
Python
376 lines
14 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
from pyparsing import (CharsNotIn, Group, Forward, Literal, Suppress, Word,
|
|
QuotedString, ZeroOrMore, alphas, alphanums)
|
|
from string import Template
|
|
import glob
|
|
import re
|
|
import logging
|
|
from colorama import Fore, Style, init
|
|
import os
|
|
|
|
# Initialize colorama
|
|
init(autoreset=True)
|
|
|
|
# Define a custom formatter with colors and indentation
|
|
class ColoredFormatter(logging.Formatter):
|
|
COLORS = {
|
|
'DEBUG': Fore.CYAN,
|
|
'INFO': Fore.GREEN,
|
|
'WARNING': Fore.YELLOW,
|
|
'ERROR': Fore.RED,
|
|
'CRITICAL': Fore.RED + Style.BRIGHT,
|
|
}
|
|
|
|
def format(self, record):
|
|
log_color = self.COLORS.get(record.levelname, '')
|
|
record.msg = f"{log_color}{record.msg}{Style.RESET_ALL}"
|
|
return super().format(record)
|
|
|
|
logger = logging.getLogger('cmakeparser')
|
|
logger.setLevel(logging.DEBUG)
|
|
console_handler = logging.StreamHandler()
|
|
# Change this to enable more logging
|
|
console_handler.setLevel(logging.ERROR)
|
|
formatter = ColoredFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
console_handler.setFormatter(formatter)
|
|
logger.addHandler(console_handler)
|
|
|
|
def log(level, message, indent=0):
|
|
indent_str = ' ' * indent
|
|
logger.log(level, f"{indent_str}{message}")
|
|
|
|
indentlevel = 0
|
|
|
|
def debug(msg):
|
|
log(logging.DEBUG, msg, indentlevel)
|
|
|
|
def info(msg):
|
|
log(logging.INFO, msg, indentlevel)
|
|
|
|
def error(msg):
|
|
log(logging.ERROR, msg, indentlevel)
|
|
|
|
# Grammar for CMake
|
|
comment = Literal('#') + ZeroOrMore(CharsNotIn('\n'))
|
|
quoted_argument = QuotedString('\"', '\\', multiline=True)
|
|
unquoted_argument = CharsNotIn('\n ()#\"\\')
|
|
argument = quoted_argument | unquoted_argument | Suppress(comment)
|
|
arguments = Forward()
|
|
arguments << (argument | (Literal('(') + ZeroOrMore(arguments) + Literal(')')))
|
|
identifier = Word(alphas, alphanums+'_')
|
|
command = Group(identifier + Literal('(') + ZeroOrMore(arguments) + Literal(')'))
|
|
file_elements = command | Suppress(comment)
|
|
cmake = ZeroOrMore(file_elements)
|
|
|
|
def extract_arguments(parsed):
|
|
"""Extract the command arguments skipping the parentheses"""
|
|
return parsed[2:len(parsed) - 1]
|
|
|
|
|
|
def match_block(command, parsed, start):
|
|
"""Find the end of block starting with the command"""
|
|
depth = 0
|
|
end = start + 1
|
|
endcommand = 'end' + command
|
|
while parsed[end][0] != endcommand or depth > 0:
|
|
if parsed[end][0] == command:
|
|
depth += 1
|
|
elif parsed[end][0] == endcommand:
|
|
depth -= 1
|
|
end = end + 1
|
|
if end == len(parsed):
|
|
error('error: eof when trying to match block statement: %s'
|
|
% parsed[start])
|
|
return end
|
|
|
|
|
|
def parse_if(parsed, start):
|
|
"""Parse if/elseif/else/endif into a list of conditions and commands"""
|
|
depth = 0
|
|
conditions = []
|
|
condition = [extract_arguments(parsed[start])]
|
|
start = start + 1
|
|
end = start
|
|
|
|
while parsed[end][0] != 'endif' or depth > 0:
|
|
command = parsed[end][0]
|
|
if command == 'if':
|
|
depth += 1
|
|
elif command == 'else' and depth == 0:
|
|
condition.append(parsed[start:end])
|
|
conditions.append(condition)
|
|
start = end + 1
|
|
condition = [['TRUE']]
|
|
elif command == 'elseif' and depth == 0:
|
|
condition.append(parsed[start:end])
|
|
conditions.append(condition)
|
|
condition = [extract_arguments(parsed[end])]
|
|
start = end + 1
|
|
elif command == 'endif':
|
|
depth -= 1
|
|
end = end + 1
|
|
if end == len(parsed):
|
|
print('error: eof when trying to match if statement: %s'
|
|
% parsed[start])
|
|
condition.append(parsed[start:end])
|
|
conditions.append(condition)
|
|
return end, conditions
|
|
|
|
|
|
def substs(variables, values):
|
|
"""Substitute variables into values"""
|
|
new_values = []
|
|
for value in values:
|
|
t = Template(value)
|
|
new_value = t.safe_substitute(variables)
|
|
|
|
# Safe substitute leaves unrecognized variables in place.
|
|
# We replace them with the empty string.
|
|
new_values.append(re.sub(r'\$\{\w+\}', '', new_value))
|
|
return new_values
|
|
|
|
|
|
def evaluate(variables, cache_variables, parsed, pwd):
|
|
global indentlevel
|
|
"""Evaluate a list of parsed commands, returning sources to build"""
|
|
i = 0
|
|
sources = []
|
|
while i < len(parsed):
|
|
command = parsed[i][0]
|
|
arguments = substs(variables, extract_arguments(parsed[i]))
|
|
|
|
if command == 'foreach':
|
|
end = match_block(command, parsed, i)
|
|
for argument in arguments[1:]:
|
|
# ; is also a valid divider, why have one when you can have two?
|
|
argument = argument.replace(';', ' ')
|
|
for value in argument.split():
|
|
variables[arguments[0]] = value
|
|
cont_eval, new_sources = evaluate(variables, cache_variables,
|
|
parsed[i+1:end], pwd)
|
|
sources.extend(new_sources)
|
|
if not cont_eval:
|
|
return cont_eval, sources
|
|
elif command == 'function':
|
|
# for now we just execute functions inline at point of declaration
|
|
# as this is sufficient to build libaom
|
|
pass
|
|
elif command == 'if':
|
|
i, conditions = parse_if(parsed, i)
|
|
for condition in conditions:
|
|
if evaluate_boolean(variables, condition[0]):
|
|
cont_eval, new_sources = evaluate(variables,
|
|
cache_variables,
|
|
condition[1], pwd)
|
|
sources.extend(new_sources)
|
|
if not cont_eval:
|
|
return cont_eval, sources
|
|
break
|
|
|
|
|
|
elif command == 'include':
|
|
if arguments:
|
|
try:
|
|
sources.extend(parse(variables, cache_variables, pwd, arguments[0]))
|
|
except IOError:
|
|
info('warning: could not include: %s' % arguments[0])
|
|
elif command == 'list':
|
|
try:
|
|
action = arguments[0]
|
|
variable = arguments[1]
|
|
values = arguments[2:]
|
|
if action == 'APPEND':
|
|
if variable not in variables:
|
|
variables[variable] = ' '.join(values)
|
|
else:
|
|
variables[variable] += ' ' + ' '.join(values)
|
|
except (IndexError, KeyError):
|
|
pass
|
|
elif command == 'option':
|
|
variable = arguments[0]
|
|
value = arguments[2]
|
|
# Allow options to be override without changing CMake files
|
|
if variable not in variables:
|
|
variables[variable] = value
|
|
elif command == 'return':
|
|
return False, sources
|
|
elif command == 'set':
|
|
variable = arguments[0]
|
|
values = arguments[1:]
|
|
# CACHE variables are not set if already present
|
|
try:
|
|
cache = values.index('CACHE')
|
|
values = values[0:cache]
|
|
if variable not in variables:
|
|
variables[variable] = ' '.join(values)
|
|
cache_variables.append(variable)
|
|
except ValueError:
|
|
variables[variable] = ' '.join(values)
|
|
elif command == 'file':
|
|
# Expand file(GLOB <var> [GLOB options] <globs>...) by globbing on
|
|
# disk, relative to the current directory. The matched files are
|
|
# stored relative to pwd so that add_library re-joins them the same
|
|
# way it does for explicitly-listed sources.
|
|
if arguments and arguments[0] in ('GLOB', 'GLOB_RECURSE'):
|
|
recursive = arguments[0] == 'GLOB_RECURSE'
|
|
variable = arguments[1]
|
|
base = os.path.join(*pwd)
|
|
matches = []
|
|
for pattern in arguments[2:]:
|
|
# Skip CMake keywords that may precede the globbing
|
|
# expressions; we don't need to honor them here.
|
|
if pattern in ('LIST_DIRECTORIES', 'RELATIVE',
|
|
'CONFIGURE_DEPENDS', 'true', 'false'):
|
|
continue
|
|
for match in glob.glob(os.path.join(base, pattern),
|
|
recursive=recursive):
|
|
matches.append(os.path.relpath(match, base))
|
|
variables[variable] = ' '.join(sorted(matches))
|
|
elif command in ['ggml_add_backend']:
|
|
backend = arguments[0].lower()
|
|
if backend in variables["MOZ_GGML_BACKENDS"]:
|
|
subdir = "ggml-" + arguments[0].lower()
|
|
parse_target = os.path.join(*pwd, subdir + '/CMakeLists.txt')
|
|
sources.extend(parse(variables, cache_variables, pwd, parse_target))
|
|
elif command in ['ggml_add_cpu_backend_variant_impl']:
|
|
# execute ggml/src/ggml-cpu/CMakeLists.txt in current dir
|
|
sources.extend(parse(variables, cache_variables, pwd, os.path.join(*pwd, "ggml-cpu/CMakeLists.txt")))
|
|
elif command == 'set_aom_option_var':
|
|
# option vars cannot go into cache_variables
|
|
variable = arguments[0]
|
|
value = arguments[2]
|
|
if variable not in variables:
|
|
variables[variable] = value
|
|
elif command == 'add_asm_library':
|
|
try:
|
|
sources.extend(variables[arguments[1]].split(' '))
|
|
except (IndexError, KeyError):
|
|
pass
|
|
elif command == 'add_intrinsics_object_library':
|
|
try:
|
|
source_files = variables[arguments[3]]
|
|
for source_file in source_files.split(' '):
|
|
sources.append(os.path.join(*pwd, source_file))
|
|
except (IndexError, KeyError):
|
|
pass
|
|
elif command == 'add_library':
|
|
if len(arguments) != 3 or arguments[2] != "ggml":
|
|
for source in arguments[1:]:
|
|
for source_file in source.split(' '):
|
|
if source_file != "ALIAS" and source_file != "OBJECT":
|
|
sources.append(os.path.join(*pwd, source_file))
|
|
elif command == 'target_sources':
|
|
for source in arguments[1:]:
|
|
for source_file in source.split(' '):
|
|
if source_file != "PRIVATE":
|
|
sources.append(os.path.join(*pwd, source_file))
|
|
elif command == 'add_subdirectory':
|
|
pwd.append(arguments[0])
|
|
parse_target = os.path.join(*pwd, 'CMakeLists.txt')
|
|
sources.extend(parse(variables, cache_variables, pwd, parse_target))
|
|
pwd.pop()
|
|
elif command == 'MOZDEBUG':
|
|
info('>>>> MOZDEBUG: %s' % ' '.join(arguments))
|
|
i += 1
|
|
return True, sources
|
|
|
|
|
|
def evaluate_boolean(variables, arguments):
|
|
"""Evaluate a boolean expression"""
|
|
if not arguments:
|
|
return False
|
|
|
|
argument = arguments[0]
|
|
|
|
if argument == "TARGET" and arguments[1] == "ggml":
|
|
return False # just enough for it to work for our purpose
|
|
|
|
if argument == 'NOT':
|
|
return not evaluate_boolean(variables, arguments[1:])
|
|
|
|
if argument == '(':
|
|
i = 0
|
|
depth = 1
|
|
while depth > 0 and i < len(arguments):
|
|
i += 1
|
|
if arguments[i] == '(':
|
|
depth += 1
|
|
if arguments[i] == ')':
|
|
depth -= 1
|
|
return evaluate_boolean(variables, arguments[1:i])
|
|
|
|
def evaluate_constant(argument):
|
|
try:
|
|
as_int = int(argument)
|
|
if as_int != 0:
|
|
return True
|
|
else:
|
|
return False
|
|
except ValueError:
|
|
upper = argument.upper()
|
|
if upper in ['ON', 'YES', 'TRUE', 'Y']:
|
|
return True
|
|
elif upper in ['OFF', 'NO', 'FALSE', 'N', 'IGNORE', '', 'NOTFOUND']:
|
|
return False
|
|
elif upper.endswith('-NOTFOUND'):
|
|
return False
|
|
return None
|
|
|
|
def lookup_variable(argument):
|
|
# If statements can have old-style variables which are not demarcated
|
|
# like ${VARIABLE}. Attempt to look up the variable both ways.
|
|
try:
|
|
if re.search(r'\$\{\w+\}', argument):
|
|
try:
|
|
t = Template(argument)
|
|
value = t.substitute(variables)
|
|
try:
|
|
# Attempt an old-style variable lookup with the
|
|
# substituted value.
|
|
return variables[value]
|
|
except KeyError:
|
|
return value
|
|
except ValueError:
|
|
# TODO: CMake supports nesting, e.g. ${${foo}}
|
|
return None
|
|
else:
|
|
return variables[argument]
|
|
except KeyError:
|
|
return None
|
|
|
|
lhs = lookup_variable(argument)
|
|
if lhs is None:
|
|
# variable resolution failed, treat as string
|
|
lhs = argument
|
|
|
|
if len(arguments) > 1:
|
|
op = arguments[1]
|
|
if op == 'AND':
|
|
return evaluate_constant(lhs) and evaluate_boolean(variables, arguments[2:])
|
|
elif op == 'MATCHES':
|
|
rhs = lookup_variable(arguments[2])
|
|
if not rhs:
|
|
rhs = arguments[2]
|
|
return not re.match(rhs, lhs) is None
|
|
elif op == 'OR':
|
|
return evaluate_constant(lhs) or evaluate_boolean(variables, arguments[2:])
|
|
elif op == 'STREQUAL':
|
|
rhs = lookup_variable(arguments[2])
|
|
if not rhs:
|
|
rhs = arguments[2]
|
|
return lhs == rhs
|
|
else:
|
|
lhs = evaluate_constant(lhs)
|
|
if lhs is None:
|
|
lhs = lookup_variable(argument)
|
|
|
|
return lhs
|
|
|
|
|
|
def parse(variables, cache_variables, pwd, filename):
|
|
parsed = cmake.parseFile(filename)
|
|
cont_eval, sources = evaluate(variables, cache_variables, parsed, pwd)
|
|
return sources
|