Florian Quèze
9e6930edf3
add_try_task_duplicates only set task_duplicates for chunked jobs with a numeric suffix or for explicitly listed labels. A dynamically-chunked job that fits in a single chunk has no numeric suffix yet is selected via a -* glob, so it was scheduled but never duplicated. Match the unchunked label against the selected glob too, mirroring _try_task_config. Differential Revision: https://phabricator.services.mozilla.com/D308887
266 lines
9.0 KiB
Python
266 lines
9.0 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
|
import pytest
|
|
from mozunit import main
|
|
from taskgraph.graph import Graph
|
|
from taskgraph.parameters import Parameters
|
|
from taskgraph.task import Task
|
|
from taskgraph.taskgraph import TaskGraph
|
|
|
|
from gecko_taskgraph import morph
|
|
|
|
|
|
@pytest.fixture
|
|
def make_taskgraph():
|
|
def inner(tasks):
|
|
label_to_taskid = {k: k + "-tid" for k in tasks}
|
|
for label, task_id in label_to_taskid.items():
|
|
tasks[label].task_id = task_id
|
|
graph = Graph(nodes=set(tasks), edges=set())
|
|
taskgraph = TaskGraph(tasks, graph)
|
|
return taskgraph, label_to_taskid
|
|
|
|
return inner
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"params,expected",
|
|
(
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 10,
|
|
"tasks": ["b"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"b": 10},
|
|
id="duplicates no chunks",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 10,
|
|
"tasks": ["a-*"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"a-1": 10, "a-2": 10},
|
|
id="duplicates with chunks",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 10,
|
|
"tasks": ["a-*", "b"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"a-1": 10, "a-2": 10, "b": 10},
|
|
id="duplicates with and without chunks",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"tasks": ["a-*"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"a-1": -1, "a-2": -1},
|
|
id="no rebuild, no duplicates",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 0,
|
|
"tasks": ["a-*"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"a-1": -1, "a-2": -1},
|
|
id="rebuild of zero",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 100,
|
|
"tasks": ["a-*"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"a-1": 100, "a-2": 100},
|
|
id="rebuild 100",
|
|
),
|
|
pytest.param(
|
|
{
|
|
"try_mode": "try_task_config",
|
|
"try_task_config": {
|
|
"rebuild": 10,
|
|
"tasks": ["c-*"],
|
|
},
|
|
"project": "try",
|
|
},
|
|
{"c": 10},
|
|
id="duplicates single dynamic chunk",
|
|
),
|
|
),
|
|
)
|
|
def test_try_task_duplicates(make_taskgraph, graph_config, params, expected):
|
|
taskb = Task(kind="test", label="b", attributes={}, task={})
|
|
task1 = Task(kind="test", label="a-1", attributes={}, task={})
|
|
task2 = Task(kind="test", label="a-2", attributes={}, task={})
|
|
# Dynamic chunking that fits in a single chunk yields no numeric suffix.
|
|
taskc = Task(kind="test", label="c", attributes={}, task={})
|
|
taskgraph, label_to_taskid = make_taskgraph({
|
|
taskb.label: taskb,
|
|
task1.label: task1,
|
|
task2.label: task2,
|
|
taskc.label: taskc,
|
|
})
|
|
|
|
taskgraph, label_to_taskid = morph.add_try_task_duplicates(
|
|
taskgraph, label_to_taskid, params, graph_config
|
|
)
|
|
for label in expected:
|
|
task = taskgraph.tasks[label]
|
|
assert task.attributes.get("task_duplicates", -1) == expected[label]
|
|
|
|
|
|
def test_make_index_tasks(make_taskgraph, graph_config):
|
|
task_def = {
|
|
"routes": [
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk",
|
|
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk",
|
|
"index.gecko.v2.mozilla-central.pushdate."
|
|
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
|
|
"index.gecko.v2.mozilla-central.revision."
|
|
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN",
|
|
],
|
|
"deadline": "soon",
|
|
"metadata": {
|
|
"description": "desc",
|
|
"owner": "owner@foo.com",
|
|
"source": "https://source",
|
|
},
|
|
"extra": {
|
|
"index": {"rank": 1540722354},
|
|
},
|
|
}
|
|
task = Task(kind="test", label="a", attributes={}, task=task_def)
|
|
docker_task = Task(
|
|
kind="docker-image", label="docker-image-index-task", attributes={}, task={}
|
|
)
|
|
taskgraph, label_to_taskid = make_taskgraph({
|
|
task.label: task,
|
|
docker_task.label: docker_task,
|
|
})
|
|
|
|
index_paths = [
|
|
r.split(".", 1)[1] for r in task_def["routes"] if r.startswith("index.")
|
|
]
|
|
index_task = morph.make_index_task(
|
|
task,
|
|
taskgraph,
|
|
label_to_taskid,
|
|
Parameters(strict=False),
|
|
graph_config,
|
|
index_paths=index_paths,
|
|
index_rank=1540722354,
|
|
purpose="index-task",
|
|
dependencies={},
|
|
)
|
|
|
|
assert index_task.task["payload"]["command"][0] == "insert-indexes.js"
|
|
assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid"
|
|
assert index_task.task["payload"]["env"]["INDEX_RANK"] == "1540722354"
|
|
|
|
# check the scope summary
|
|
assert index_task.task["scopes"] == ["index:insert-task:gecko.v2.mozilla-central.*"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"has_ccov,expected_task_added",
|
|
(
|
|
pytest.param(True, True, id="with ccov tasks"),
|
|
pytest.param(False, False, id="without ccov tasks"),
|
|
),
|
|
)
|
|
def test_add_code_coverage_task(
|
|
make_taskgraph, graph_config, has_ccov, expected_task_added
|
|
):
|
|
tasks = {}
|
|
if has_ccov:
|
|
tasks["ccov-test-1"] = Task(
|
|
kind="test",
|
|
label="ccov-test-1",
|
|
attributes={"ccov": True},
|
|
task={},
|
|
)
|
|
tasks["ccov-test-2"] = Task(
|
|
kind="test",
|
|
label="ccov-test-2",
|
|
attributes={"ccov": True},
|
|
task={},
|
|
)
|
|
tasks["non-ccov"] = Task(
|
|
kind="test",
|
|
label="non-ccov",
|
|
attributes={},
|
|
task={},
|
|
)
|
|
|
|
taskgraph, label_to_taskid = make_taskgraph(tasks)
|
|
params = Parameters(
|
|
strict=False,
|
|
owner="test@example.com",
|
|
head_repository="https://hg.mozilla.org/mozilla-central",
|
|
)
|
|
|
|
taskgraph, label_to_taskid = morph.add_code_coverage_task(
|
|
taskgraph, label_to_taskid, params, graph_config
|
|
)
|
|
|
|
assert ("code-coverage-artifacts" in label_to_taskid) == expected_task_added
|
|
if expected_task_added:
|
|
task = taskgraph.tasks[label_to_taskid["code-coverage-artifacts"]]
|
|
assert task.task["routes"] == ["project.codecoverage.v1.tasks_done"]
|
|
assert task.task["requires"] == "all-resolved"
|
|
assert set(task.dependencies.keys()) == {"ccov-test-1", "ccov-test-2"}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|