XLinfini/sousa-gecko
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1431753 - Upload per-test coverage reports. r=sparky

Browse Source
This commit is contained in:
Marco Castelluccio
2018-04-19 12:42:38 +02:00
parent 0b0f3e453d
commit c4049ce5c2
3 changed files with 56 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+52 -8
View File
@@ -3,11 +3,14 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import shutil
import sys
import tarfile
import tempfile
import zipfile
import uuid
import mozinfo
from mozharness.base.script import (
@@ -58,6 +61,7 @@ class CodeCoverageMixin(SingleTestMixin):
gcov_dir = None
jsvm_dir = None
prefix = None
per_test_reports = {}
def __init__(self):
super(CodeCoverageMixin, self).__init__()
@@ -190,7 +194,7 @@ class CodeCoverageMixin(SingleTestMixin):
self.gcov_dir, self.jsvm_dir = self.set_coverage_env(os.environ)
def parse_coverage_artifacts(self, gcov_dir, jsvm_dir):
def parse_coverage_artifacts(self, gcov_dir, jsvm_dir, merge=False, output_format='lcov'):
jsvm_output_file = 'jsvm_lcov_output.info'
grcov_output_file = 'grcov_lcov_output.info'
@@ -211,13 +215,19 @@ class CodeCoverageMixin(SingleTestMixin):
# Run grcov on the zipped .gcno and .gcda files.
grcov_command = [
os.path.join(self.grcov_dir, 'grcov'),
'-t', 'lcov',
'-t', output_format,
'-p', self.prefix,
'--ignore-dir', 'gcc*',
'--ignore-dir', 'vs2017_*',
os.path.join(self.grcov_dir, 'target.code-coverage-gcno.zip'), file_path_gcda
]
if 'coveralls' in output_format:
grcov_command += ['--token', 'UNUSED', '--commit-sha', 'UNUSED']
if merge:
grcov_command += [jsvm_output_file]
if mozinfo.os == 'win':
grcov_command += ['--llvm']
@@ -232,13 +242,34 @@ class CodeCoverageMixin(SingleTestMixin):
)
shutil.move(tmp_output_file, grcov_output_file)
return grcov_output_file, jsvm_output_file
shutil.rmtree(gcov_dir)
shutil.rmtree(jsvm_dir)
if merge:
os.remove(jsvm_output_file)
return grcov_output_file
else:
return grcov_output_file, jsvm_output_file
def add_per_test_coverage_report(self, gcov_dir, jsvm_dir, suite, test):
grcov_file = self.parse_coverage_artifacts(
gcov_dir, jsvm_dir, merge=True, output_format='coveralls'
)
report_file = str(uuid.uuid4()) + '.json'
shutil.move(grcov_file, report_file)
if suite not in self.per_test_reports:
self.per_test_reports[suite] = {}
assert test not in self.per_test_reports[suite]
self.per_test_reports[suite][test] = report_file
@PostScriptAction('run-tests')
def _package_coverage_data(self, action, success=None):
dirs = self.query_abs_dirs()
if self.jsd_code_coverage_enabled:
# Setup the command for compression
dirs = self.query_abs_dirs()
jsdcov_dir = dirs['abs_blob_upload_dir']
zipFile = os.path.join(jsdcov_dir, "jsdcov_artifacts.zip")
command = ["zip", "-r", "-q", zipFile, ".", "-i", "jscov*.json"]
@@ -258,6 +289,23 @@ class CodeCoverageMixin(SingleTestMixin):
return
if self.per_test_coverage:
dest = os.path.join(dirs['abs_blob_upload_dir'], 'per-test-coverage-reports.zip')
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_DEFLATED) as z:
for suite, data in self.per_test_reports.items():
for test, grcov_file in data.items():
with open(grcov_file, 'r') as f:
report = json.load(f)
# TODO: Diff this coverage report with the baseline one.
with open(grcov_file, 'w') as f:
json.dump({
'test': test,
'suite': suite,
'report': report,
}, f)
z.write(grcov_file)
return
del os.environ['GCOV_PREFIX_STRIP']
@@ -267,8 +315,6 @@ class CodeCoverageMixin(SingleTestMixin):
if not self.ccov_upload_disabled:
grcov_output_file, jsvm_output_file = self.parse_coverage_artifacts(self.gcov_dir, self.jsvm_dir)
dirs = self.query_abs_dirs()
# Zip the grcov output and upload it.
self.run_command(
['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), grcov_output_file]
@@ -279,6 +325,4 @@ class CodeCoverageMixin(SingleTestMixin):
['zip', '-q', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip'), jsvm_output_file]
)
shutil.rmtree(self.gcov_dir)
shutil.rmtree(self.jsvm_dir)
shutil.rmtree(self.grcov_dir)
testing/mozharness/scripts/desktop_unittest.py
+3 -17
View File
@@ -854,18 +854,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
env = self.query_env(partial_env=env, log_level=INFO)
cmd_timeout = self.get_timeout_for_category(suite_category)
# Run basic startup/shutdown test to collect baseline coverage.
# This way, after we run a test, we can generate a diff between the
# full coverage of the test and the baseline coverage and only get
# the coverage data specific to the test.
if self.per_test_coverage:
gcov_dir, jsvm_dir = self.set_coverage_env(env)
# TODO: Run basic startup/shutdown test to collect baseline coverage.
# grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir)
# shutil.rmtree(gcov_dir)
# shutil.rmtree(jsvm_dir)
# TODO: Parse coverage report
for per_test_args in self.query_args(suite):
if (datetime.now() - self.start_time) > max_per_test_time:
# Running tests has run out of time. That is okay! Stop running
@@ -898,11 +886,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
env=env)
if self.per_test_coverage:
grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir)
shutil.rmtree(gcov_dir)
shutil.rmtree(jsvm_dir)
# TODO: Parse coverage report
# TODO: Diff this coverage report with the baseline one
self.add_per_test_coverage_report(
gcov_dir, jsvm_dir, suite, per_test_args[-1]
)
# mochitest, reftest, and xpcshell suites do not return
# appropriate return codes. Therefore, we must parse the output
testing/mozharness/scripts/web_platform_tests.py
+1 -18
View File
@@ -6,7 +6,6 @@
# ***** END LICENSE BLOCK *****
import copy
import os
import shutil
import sys
from datetime import datetime, timedelta
@@ -338,18 +337,6 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera
if suite:
test_types = [suite]
# Run basic startup/shutdown test to collect baseline coverage.
# This way, after we run a test, we can generate a diff between the
# full coverage of the test and the baseline coverage and only get
# the coverage data specific to the test.
if self.per_test_coverage:
gcov_dir, jsvm_dir = self.set_coverage_env(env)
# TODO: Run basic startup/shutdown test to collect baseline coverage.
# grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir)
# shutil.rmtree(gcov_dir)
# shutil.rmtree(jsvm_dir)
# TODO: Parse coverage report
for per_test_args in self.query_args(suite):
if (datetime.now() - start_time) > max_per_test_time:
# Running tests has run out of time. That is okay! Stop running
@@ -381,11 +368,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera
env=env)
if self.per_test_coverage:
grcov_file, jsvm_file = self.parse_coverage_artifacts(gcov_dir, jsvm_dir)
shutil.rmtree(gcov_dir)
shutil.rmtree(jsvm_dir)
# TODO: Parse coverage report
# TODO: Diff this coverage report with the baseline one
self.add_per_test_coverage_report(gcov_dir, jsvm_dir, suite, per_test_args[-1])
tbpl_status, log_level = parser.evaluate_parser(return_code)
self.buildbot_status(tbpl_status, level=log_level)