Bug 2034933 - Add an alert severity field, and set it for sp3. r=perftest-reviewers,fbilt

This patch adds a new alert severity field to the performance data and sets it up for speedometer3 critical and subcritical tests.

Differential Revision: https://phabricator.services.mozilla.com/D296486

testing/mozharness/external_tools/performance-artifact-schema.json

@@ -99,6 +99,12 @@
           "minimum": 0.0,
           "maximum": 1000.0
         },
+        "alertSeverity": {
+          "description": "Severity level for alerts produced by this subtest",
+          "title": "Alert severity",
+          "enum": ["critical", "subcritical", "normal"],
+          "type": "string"
+        },
         "minBackWindow": {
           "description": "Minimum back window to use for alerting",
           "title": "Minimum back window",
@@ -203,6 +209,12 @@
           "minimum": 0.0,
           "maximum": 1000.0
         },
+        "alertSeverity": {
+          "description": "Severity level for alerts produced by this suite",
+          "title": "Alert severity",
+          "enum": ["critical", "subcritical", "normal"],
+          "type": "string"
+        },
         "minBackWindow": {
           "description": "Minimum back window to use for alerting",
           "title": "Minimum back window",

testing/raptor/browsertime/support-scripts/speedometer3.py

@@ -73,9 +73,14 @@ class Speedometer3Support(BasePythonSupport):
             lower_is_better = False
             unit = "score"
 
+        alert_severity = "subcritical"
+        if measurement_name == "score" and self.platform == "Windows":
+            alert_severity = "critical"
+
         subtest = {
             "unit": unit,
             "alertThreshold": float(test.get("alert_threshold", 2.0)),
+            "alertSeverity": alert_severity,
             "lowerIsBetter": lower_is_better,
             "minBackWindow": 24,
             "maxBackWindow": 48,
@@ -128,6 +133,9 @@ class Speedometer3Support(BasePythonSupport):
                 break
         suite["value"] = score
         suite["replicates"] = replicates
+        suite["alertSeverity"] = "subcritical"
+        if self.platform == "Windows":
+            suite["alertSeverity"] = "critical"
 
     def modify_command(self, cmd, test):
         """Modify the browsertime command for speedometer 3.

testing/raptor/raptor/base_python_support.py

@@ -1,6 +1,8 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import platform
+
 import filters
 from cmdline import FIREFOX_APPS
 from utils import flatten
@@ -20,6 +22,7 @@ class BasePythonSupport:
         self.app = None
         self.raw_result = []
         self.bt_result = []
+        self.platform = platform.system()
 
     def save_data(self, raw_result, bt_result):
         """