# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from pyparsing import (CharsNotIn, Group, Forward, Literal, Suppress, Word, QuotedString, ZeroOrMore, alphas, alphanums) from string import Template import glob import re import logging from colorama import Fore, Style, init import os # Initialize colorama init(autoreset=True) # Define a custom formatter with colors and indentation class ColoredFormatter(logging.Formatter): COLORS = { 'DEBUG': Fore.CYAN, 'INFO': Fore.GREEN, 'WARNING': Fore.YELLOW, 'ERROR': Fore.RED, 'CRITICAL': Fore.RED + Style.BRIGHT, } def format(self, record): log_color = self.COLORS.get(record.levelname, '') record.msg = f"{log_color}{record.msg}{Style.RESET_ALL}" return super().format(record) logger = logging.getLogger('cmakeparser') logger.setLevel(logging.DEBUG) console_handler = logging.StreamHandler() # Change this to enable more logging console_handler.setLevel(logging.ERROR) formatter = ColoredFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') console_handler.setFormatter(formatter) logger.addHandler(console_handler) def log(level, message, indent=0): indent_str = ' ' * indent logger.log(level, f"{indent_str}{message}") indentlevel = 0 def debug(msg): log(logging.DEBUG, msg, indentlevel) def info(msg): log(logging.INFO, msg, indentlevel) def error(msg): log(logging.ERROR, msg, indentlevel) # Grammar for CMake comment = Literal('#') + ZeroOrMore(CharsNotIn('\n')) quoted_argument = QuotedString('\"', '\\', multiline=True) unquoted_argument = CharsNotIn('\n ()#\"\\') argument = quoted_argument | unquoted_argument | Suppress(comment) arguments = Forward() arguments << (argument | (Literal('(') + ZeroOrMore(arguments) + Literal(')'))) identifier = Word(alphas, alphanums+'_') command = Group(identifier + Literal('(') + ZeroOrMore(arguments) + Literal(')')) file_elements = command | Suppress(comment) cmake = ZeroOrMore(file_elements) def extract_arguments(parsed): """Extract the command arguments skipping the parentheses""" return parsed[2:len(parsed) - 1] def match_block(command, parsed, start): """Find the end of block starting with the command""" depth = 0 end = start + 1 endcommand = 'end' + command while parsed[end][0] != endcommand or depth > 0: if parsed[end][0] == command: depth += 1 elif parsed[end][0] == endcommand: depth -= 1 end = end + 1 if end == len(parsed): error('error: eof when trying to match block statement: %s' % parsed[start]) return end def parse_if(parsed, start): """Parse if/elseif/else/endif into a list of conditions and commands""" depth = 0 conditions = [] condition = [extract_arguments(parsed[start])] start = start + 1 end = start while parsed[end][0] != 'endif' or depth > 0: command = parsed[end][0] if command == 'if': depth += 1 elif command == 'else' and depth == 0: condition.append(parsed[start:end]) conditions.append(condition) start = end + 1 condition = [['TRUE']] elif command == 'elseif' and depth == 0: condition.append(parsed[start:end]) conditions.append(condition) condition = [extract_arguments(parsed[end])] start = end + 1 elif command == 'endif': depth -= 1 end = end + 1 if end == len(parsed): print('error: eof when trying to match if statement: %s' % parsed[start]) condition.append(parsed[start:end]) conditions.append(condition) return end, conditions def substs(variables, values): """Substitute variables into values""" new_values = [] for value in values: t = Template(value) new_value = t.safe_substitute(variables) # Safe substitute leaves unrecognized variables in place. # We replace them with the empty string. new_values.append(re.sub(r'\$\{\w+\}', '', new_value)) return new_values def evaluate(variables, cache_variables, parsed, pwd): global indentlevel """Evaluate a list of parsed commands, returning sources to build""" i = 0 sources = [] while i < len(parsed): command = parsed[i][0] arguments = substs(variables, extract_arguments(parsed[i])) if command == 'foreach': end = match_block(command, parsed, i) for argument in arguments[1:]: # ; is also a valid divider, why have one when you can have two? argument = argument.replace(';', ' ') for value in argument.split(): variables[arguments[0]] = value cont_eval, new_sources = evaluate(variables, cache_variables, parsed[i+1:end], pwd) sources.extend(new_sources) if not cont_eval: return cont_eval, sources elif command == 'function': # for now we just execute functions inline at point of declaration # as this is sufficient to build libaom pass elif command == 'if': i, conditions = parse_if(parsed, i) for condition in conditions: if evaluate_boolean(variables, condition[0]): cont_eval, new_sources = evaluate(variables, cache_variables, condition[1], pwd) sources.extend(new_sources) if not cont_eval: return cont_eval, sources break elif command == 'include': if arguments: try: sources.extend(parse(variables, cache_variables, pwd, arguments[0])) except IOError: info('warning: could not include: %s' % arguments[0]) elif command == 'list': try: action = arguments[0] variable = arguments[1] values = arguments[2:] if action == 'APPEND': if variable not in variables: variables[variable] = ' '.join(values) else: variables[variable] += ' ' + ' '.join(values) except (IndexError, KeyError): pass elif command == 'option': variable = arguments[0] value = arguments[2] # Allow options to be override without changing CMake files if variable not in variables: variables[variable] = value elif command == 'return': return False, sources elif command == 'set': variable = arguments[0] values = arguments[1:] # CACHE variables are not set if already present try: cache = values.index('CACHE') values = values[0:cache] if variable not in variables: variables[variable] = ' '.join(values) cache_variables.append(variable) except ValueError: variables[variable] = ' '.join(values) elif command == 'file': # Expand file(GLOB [GLOB options] ...) by globbing on # disk, relative to the current directory. The matched files are # stored relative to pwd so that add_library re-joins them the same # way it does for explicitly-listed sources. if arguments and arguments[0] in ('GLOB', 'GLOB_RECURSE'): recursive = arguments[0] == 'GLOB_RECURSE' variable = arguments[1] base = os.path.join(*pwd) matches = [] for pattern in arguments[2:]: # Skip CMake keywords that may precede the globbing # expressions; we don't need to honor them here. if pattern in ('LIST_DIRECTORIES', 'RELATIVE', 'CONFIGURE_DEPENDS', 'true', 'false'): continue for match in glob.glob(os.path.join(base, pattern), recursive=recursive): matches.append(os.path.relpath(match, base)) variables[variable] = ' '.join(sorted(matches)) elif command in ['ggml_add_backend']: backend = arguments[0].lower() if backend in variables["MOZ_GGML_BACKENDS"]: subdir = "ggml-" + arguments[0].lower() parse_target = os.path.join(*pwd, subdir + '/CMakeLists.txt') sources.extend(parse(variables, cache_variables, pwd, parse_target)) elif command in ['ggml_add_cpu_backend_variant_impl']: # execute ggml/src/ggml-cpu/CMakeLists.txt in current dir sources.extend(parse(variables, cache_variables, pwd, os.path.join(*pwd, "ggml-cpu/CMakeLists.txt"))) elif command == 'set_aom_option_var': # option vars cannot go into cache_variables variable = arguments[0] value = arguments[2] if variable not in variables: variables[variable] = value elif command == 'add_asm_library': try: sources.extend(variables[arguments[1]].split(' ')) except (IndexError, KeyError): pass elif command == 'add_intrinsics_object_library': try: source_files = variables[arguments[3]] for source_file in source_files.split(' '): sources.append(os.path.join(*pwd, source_file)) except (IndexError, KeyError): pass elif command == 'add_library': if len(arguments) != 3 or arguments[2] != "ggml": for source in arguments[1:]: for source_file in source.split(' '): if source_file != "ALIAS" and source_file != "OBJECT": sources.append(os.path.join(*pwd, source_file)) elif command == 'target_sources': for source in arguments[1:]: for source_file in source.split(' '): if source_file != "PRIVATE": sources.append(os.path.join(*pwd, source_file)) elif command == 'add_subdirectory': pwd.append(arguments[0]) parse_target = os.path.join(*pwd, 'CMakeLists.txt') sources.extend(parse(variables, cache_variables, pwd, parse_target)) pwd.pop() elif command == 'MOZDEBUG': info('>>>> MOZDEBUG: %s' % ' '.join(arguments)) i += 1 return True, sources def evaluate_boolean(variables, arguments): """Evaluate a boolean expression""" if not arguments: return False argument = arguments[0] if argument == "TARGET" and arguments[1] == "ggml": return False # just enough for it to work for our purpose if argument == 'NOT': return not evaluate_boolean(variables, arguments[1:]) if argument == '(': i = 0 depth = 1 while depth > 0 and i < len(arguments): i += 1 if arguments[i] == '(': depth += 1 if arguments[i] == ')': depth -= 1 return evaluate_boolean(variables, arguments[1:i]) def evaluate_constant(argument): try: as_int = int(argument) if as_int != 0: return True else: return False except ValueError: upper = argument.upper() if upper in ['ON', 'YES', 'TRUE', 'Y']: return True elif upper in ['OFF', 'NO', 'FALSE', 'N', 'IGNORE', '', 'NOTFOUND']: return False elif upper.endswith('-NOTFOUND'): return False return None def lookup_variable(argument): # If statements can have old-style variables which are not demarcated # like ${VARIABLE}. Attempt to look up the variable both ways. try: if re.search(r'\$\{\w+\}', argument): try: t = Template(argument) value = t.substitute(variables) try: # Attempt an old-style variable lookup with the # substituted value. return variables[value] except KeyError: return value except ValueError: # TODO: CMake supports nesting, e.g. ${${foo}} return None else: return variables[argument] except KeyError: return None lhs = lookup_variable(argument) if lhs is None: # variable resolution failed, treat as string lhs = argument if len(arguments) > 1: op = arguments[1] if op == 'AND': return evaluate_constant(lhs) and evaluate_boolean(variables, arguments[2:]) elif op == 'MATCHES': rhs = lookup_variable(arguments[2]) if not rhs: rhs = arguments[2] return not re.match(rhs, lhs) is None elif op == 'OR': return evaluate_constant(lhs) or evaluate_boolean(variables, arguments[2:]) elif op == 'STREQUAL': rhs = lookup_variable(arguments[2]) if not rhs: rhs = arguments[2] return lhs == rhs else: lhs = evaluate_constant(lhs) if lhs is None: lhs = lookup_variable(argument) return lhs def parse(variables, cache_variables, pwd, filename): parsed = cmake.parseFile(filename) cont_eval, sources = evaluate(variables, cache_variables, parsed, pwd) return sources