Thomas Wisniewski
2eb69a6063
https://phabricator.services.mozilla.com/D300897 Differential Revision: https://phabricator.services.mozilla.com/D301494
140 lines
4.0 KiB
Python
140 lines
4.0 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
import traceback
|
|
|
|
|
|
class TestHarness:
|
|
def __init__(self, test, verbose):
|
|
self.test = test
|
|
self.verbose = verbose
|
|
self.printed_intro = False
|
|
self.passed = 0
|
|
self.failures = []
|
|
|
|
def start(self):
|
|
if self.verbose:
|
|
self.maybe_print_intro()
|
|
|
|
def finish(self):
|
|
if self.verbose or self.printed_intro:
|
|
print(f"Finished test {self.test}")
|
|
|
|
def maybe_print_intro(self):
|
|
if not self.printed_intro:
|
|
print(f"Starting test {self.test}")
|
|
self.printed_intro = True
|
|
|
|
def test_pass(self, msg):
|
|
self.passed += 1
|
|
if self.verbose:
|
|
print(f"TEST-PASS | {msg}")
|
|
|
|
def test_fail(self, msg):
|
|
self.maybe_print_intro()
|
|
self.failures.append(msg)
|
|
print(f"TEST-UNEXPECTED-FAIL | {msg}")
|
|
|
|
def ok(self, condition, msg):
|
|
if condition:
|
|
self.test_pass(msg)
|
|
else:
|
|
self.test_fail(msg)
|
|
|
|
def equals(self, a, b, msg):
|
|
if json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True):
|
|
self.test_pass(msg)
|
|
else:
|
|
self.test_fail(f"{msg} | Got {a} expected {b}")
|
|
|
|
def should_throw(self, description, exception, code):
|
|
threw = "(no exception)"
|
|
try:
|
|
code()
|
|
except Exception as e:
|
|
threw = f"{type(e).__name__}: {e} {traceback.format_exc()}"
|
|
if threw.startswith(exception):
|
|
self.test_pass(f"{description} - should throw `{exception}`")
|
|
else:
|
|
self.test_fail(
|
|
f"{description} - should throw `{exception}` | Got `{threw}`"
|
|
)
|
|
|
|
|
|
def run_tests(tests, verbose):
|
|
testdir = os.path.join(os.path.dirname(__file__), "tests/build")
|
|
print("testdir", testdir)
|
|
if not tests:
|
|
tests = glob.iglob(os.path.join(testdir, "*.py"))
|
|
sys.path.append(testdir)
|
|
|
|
all_passed = 0
|
|
failed_tests = []
|
|
|
|
for test in tests:
|
|
(testpath, ext) = os.path.splitext(os.path.basename(test))
|
|
_test = __import__(testpath, globals(), locals(), ["WebCompatBuildTest"])
|
|
|
|
harness = TestHarness(test, verbose)
|
|
harness.start()
|
|
try:
|
|
_test.WebCompatBuildTest.__call__(harness)
|
|
except Exception as ex:
|
|
harness.test_fail(f"Unhandled exception in test {testpath}: {ex}")
|
|
print(traceback.format_exc())
|
|
finally:
|
|
harness.finish()
|
|
all_passed += harness.passed
|
|
if harness.failures:
|
|
failed_tests.append((test, harness.failures))
|
|
|
|
if verbose or failed_tests:
|
|
print()
|
|
print("Result summary:")
|
|
print(f"Successful: {all_passed}")
|
|
print(f"Unexpected: {sum(len(failures) for _, failures in failed_tests)}")
|
|
for test, failures in failed_tests:
|
|
print(f"{test}:")
|
|
for failure in failures:
|
|
print(f"TEST-UNEXPECTED-FAIL | {failure}")
|
|
return 1 if failed_tests else 0
|
|
|
|
|
|
def get_parser():
|
|
usage = """%(prog)s [OPTIONS] [TESTS]
|
|
Where TESTS are relative to the tests directory."""
|
|
parser = argparse.ArgumentParser(usage=usage)
|
|
parser.add_argument(
|
|
"-q",
|
|
"--quiet",
|
|
action="store_false",
|
|
dest="verbose",
|
|
help="Don't print passing tests.",
|
|
default=None,
|
|
)
|
|
parser.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
action="store_true",
|
|
dest="verbose",
|
|
help="Run tests in verbose mode.",
|
|
)
|
|
parser.add_argument("tests", nargs="*", help="Tests to run")
|
|
return parser
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = get_parser()
|
|
args = parser.parse_args()
|
|
if args.verbose is None:
|
|
args.verbose = True
|
|
|
|
sys.exit(run_tests(args.tests, verbose=args.verbose))
|