diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py index b76984e06b6c..42aab16a0534 100644 --- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -3,11 +3,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import json import os import shutil import sys import tarfile import tempfile +import zipfile +import uuid import mozinfo from mozharness.base.script import ( @@ -58,6 +61,7 @@ class CodeCoverageMixin(SingleTestMixin): gcov_dir = None jsvm_dir = None prefix = None + per_test_reports = {} def __init__(self): super(CodeCoverageMixin, self).__init__() @@ -190,7 +194,7 @@ class CodeCoverageMixin(SingleTestMixin): self.gcov_dir, self.jsvm_dir = self.set_coverage_env(os.environ) - def parse_coverage_artifacts(self, gcov_dir, jsvm_dir): + def parse_coverage_artifacts(self, gcov_dir, jsvm_dir, merge=False, output_format='lcov'): jsvm_output_file = 'jsvm_lcov_output.info' grcov_output_file = 'grcov_lcov_output.info' @@ -211,13 +215,19 @@ class CodeCoverageMixin(SingleTestMixin): # Run grcov on the zipped .gcno and .gcda files. grcov_command = [ os.path.join(self.grcov_dir, 'grcov'), - '-t', 'lcov', + '-t', output_format, '-p', self.prefix, '--ignore-dir', 'gcc*', '--ignore-dir', 'vs2017_*', os.path.join(self.grcov_dir, 'target.code-coverage-gcno.zip'), file_path_gcda ] + if 'coveralls' in output_format: + grcov_command += ['--token', 'UNUSED', '--commit-sha', 'UNUSED'] + + if merge: + grcov_command += [jsvm_output_file] + if mozinfo.os == 'win': grcov_command += ['--llvm'] @@ -232,13 +242,34 @@ class CodeCoverageMixin(SingleTestMixin): ) shutil.move(tmp_output_file, grcov_output_file) - return grcov_output_file, jsvm_output_file + shutil.rmtree(gcov_dir) + shutil.rmtree(jsvm_dir) + + if merge: + os.remove(jsvm_output_file) + return grcov_output_file + else: + return grcov_output_file, jsvm_output_file + + def add_per_test_coverage_report(self, gcov_dir, jsvm_dir, suite, test): + grcov_file = self.parse_coverage_artifacts( + gcov_dir, jsvm_dir, merge=True, output_format='coveralls' + ) + + report_file = str(uuid.uuid4()) + '.json' + shutil.move(grcov_file, report_file) + + if suite not in self.per_test_reports: + self.per_test_reports[suite] = {} + assert test not in self.per_test_reports[suite] + self.per_test_reports[suite][test] = report_file @PostScriptAction('run-tests') def _package_coverage_data(self, action, success=None): + dirs = self.query_abs_dirs() + if self.jsd_code_coverage_enabled: # Setup the command for compression - dirs = self.query_abs_dirs() jsdcov_dir = dirs['abs_blob_upload_dir'] zipFile = os.path.join(jsdcov_dir, "jsdcov_artifacts.zip") command = ["zip", "-r", "-q", zipFile, ".", "-i", "jscov*.json"] @@ -258,6 +289,23 @@ class CodeCoverageMixin(SingleTestMixin): return if self.per_test_coverage: + dest = os.path.join(dirs['abs_blob_upload_dir'], 'per-test-coverage-reports.zip') + with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED) as z: + for suite, data in self.per_test_reports.items(): + for test, grcov_file in data.items(): + with open(grcov_file, 'r') as f: + report = json.load(f) + + # TODO: Diff this coverage report with the baseline one. + + with open(grcov_file, 'w') as f: + json.dump({ + 'test': test, + 'suite': suite, + 'report': report, + }, f) + + z.write(grcov_file) return del os.environ['GCOV_PREFIX_STRIP'] @@ -267,8 +315,6 @@ class CodeCoverageMixin(SingleTestMixin): if not self.ccov_upload_disabled: grcov_output_file, jsvm_output_file = self.parse_coverage_artifacts(self.gcov_dir, self.jsvm_dir) - dirs = self.query_abs_dirs() - # Zip the grcov output and upload it. self.run_command( ['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), grcov_output_file] @@ -279,6 +325,4 @@ class CodeCoverageMixin(SingleTestMixin): ['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip'), jsvm_output_file] ) - shutil.rmtree(self.gcov_dir) - shutil.rmtree(self.jsvm_dir) shutil.rmtree(self.grcov_dir) diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index f4707408b87e..079fc1ae260b 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -854,18 +854,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix env = self.query_env(partial_env=env, log_level=INFO) cmd_timeout = self.get_timeout_for_category(suite_category) - # Run basic startup/shutdown test to collect baseline coverage. - # This way, after we run a test, we can generate a diff between the - # full coverage of the test and the baseline coverage and only get - # the coverage data specific to the test. - if self.per_test_coverage: - gcov_dir, jsvm_dir = self.set_coverage_env(env) - # TODO: Run basic startup/shutdown test to collect baseline coverage. - # grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) - # shutil.rmtree(gcov_dir) - # shutil.rmtree(jsvm_dir) - # TODO: Parse coverage report - for per_test_args in self.query_args(suite): if (datetime.now() - self.start_time) > max_per_test_time: # Running tests has run out of time. That is okay! Stop running @@ -898,11 +886,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix env=env) if self.per_test_coverage: - grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) - shutil.rmtree(gcov_dir) - shutil.rmtree(jsvm_dir) - # TODO: Parse coverage report - # TODO: Diff this coverage report with the baseline one + self.add_per_test_coverage_report( + gcov_dir, jsvm_dir, suite, per_test_args[-1] + ) # mochitest, reftest, and xpcshell suites do not return # appropriate return codes. Therefore, we must parse the output diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index a6a6c1080924..844678e9826a 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -6,7 +6,6 @@ # ***** END LICENSE BLOCK ***** import copy import os -import shutil import sys from datetime import datetime, timedelta @@ -338,18 +337,6 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera if suite: test_types = [suite] - # Run basic startup/shutdown test to collect baseline coverage. - # This way, after we run a test, we can generate a diff between the - # full coverage of the test and the baseline coverage and only get - # the coverage data specific to the test. - if self.per_test_coverage: - gcov_dir, jsvm_dir = self.set_coverage_env(env) - # TODO: Run basic startup/shutdown test to collect baseline coverage. - # grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) - # shutil.rmtree(gcov_dir) - # shutil.rmtree(jsvm_dir) - # TODO: Parse coverage report - for per_test_args in self.query_args(suite): if (datetime.now() - start_time) > max_per_test_time: # Running tests has run out of time. That is okay! Stop running @@ -381,11 +368,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera env=env) if self.per_test_coverage: - grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir) - shutil.rmtree(gcov_dir) - shutil.rmtree(jsvm_dir) - # TODO: Parse coverage report - # TODO: Diff this coverage report with the baseline one + self.add_per_test_coverage_report(gcov_dir, jsvm_dir, suite, per_test_args[-1]) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level)