Bug 2067228: Add form fill outcomes test for SFF telemetry r=omarg

Differential Revision: https://phabricator.services.mozilla.com/D323739
2026-09-10 02:19:05 +00:00
parent a918492dd0
commit 0accc4119a
1 changed files with 340 additions and 28 deletions
@@ -3,6 +3,12 @@

 "use strict";

+/* import-globals-from head_smartformfill_form_review.js */
+Services.scriptloader.loadSubScript(
+  getRootDirectory(gTestPath) + "head_smartformfill_form_review.js",
+  this
+);
+
 const { SmartFormFillModel } = ChromeUtils.importESModule(
  "moz-src:///browser/components/aiwindow/models/SmartFormFillModel.sys.mjs"
 );
@@ -12,9 +18,15 @@ const TEST_PAGE = `${getRootDirectory(gTestPath).replace(
  "https://example.com"
 )}test_smartformfill_telemetry.html`;

+// Somewhere to navigate to, to make the form's page go away.
+const NAVIGATION_PAGE = "https://example.com/";
+
 // The page has two forms: one with email + tel, one with a textarea.
 const EXPECTED_FORMS = 2;

+// Fields of the contact form, which is the one every round below runs on.
+const CONTACT_FIELDS = 2;
+
 // What the fakes report as the model and prompt a request was built with.
 const TEST_MODEL_INFO = { model: "test-model", promptVersion: "42" };

@@ -201,6 +213,73 @@ function focusField(browser, selector) {
  );
 }

+/**
+ * Blurs a field, which is what ends the fill of that one field.
+ *
+ * The report a blur triggers is sent before the reply to this call, both being
+ * messages of the same window global, so an outcome it recorded has landed by
+ * the time this resolves.
+ *
+ * @param {MozBrowser} browser
+ * @param {string} selector Field to blur
+ * @returns {Promise<void>}
+ */
+function blurField(browser, selector) {
+  return SpecialPowers.spawn(browser, [selector], fieldSelector =>
+    content.document.querySelector(fieldSelector).blur()
+  );
+}
+
+/**
+ * Replaces what a field holds, selecting the filled value first so an empty
+ * string leaves the field empty rather than appending nothing.
+ *
+ * @param {MozBrowser} browser
+ * @param {string} selector Field to replace the value of
+ * @param {string} value Text to leave the field with
+ * @returns {Promise<void>}
+ */
+function replaceFieldValue(browser, selector, value) {
+  return SpecialPowers.spawn(
+    browser,
+    [selector, value],
+    async (fieldSelector, text) => {
+      const field = content.document.querySelector(fieldSelector);
+      field.focus();
+      EventUtils.synthesizeKey("a", { accelKey: true }, content);
+      EventUtils.synthesizeKey("KEY_Backspace", {}, content);
+
+      if (text) {
+        await EventUtils.sendString(text, content);
+      }
+    }
+  );
+}
+
+/**
+ * Submits a form without letting it navigate, which ends the fill of every
+ * field of that form at once.
+ *
+ * The listener that reports the outcomes is in the system group, so cancelling
+ * the submission from the page stops the navigation without stopping the
+ * report. Validation is turned off because a generated value is not a valid
+ * email.
+ *
+ * @param {MozBrowser} browser
+ * @param {string} selector Form to submit
+ * @returns {Promise<void>}
+ */
+function submitForm(browser, selector) {
+  return SpecialPowers.spawn(browser, [selector], formSelector => {
+    const form = content.document.querySelector(formSelector);
+    form.noValidate = true;
+    form.addEventListener("submit", event => event.preventDefault(), {
+      once: true,
+    });
+    form.requestSubmit();
+  });
+}
+
 /**
 * Runs a whole round on the form a field belongs to: relevant tabs,
 * classification and generation. Nothing is requested until a form is asked
@@ -242,37 +321,33 @@ async function fillFormReview(win, browser) {
  const reviewBrowser = dialog._frame.contentWindow.document.querySelector(
    "#form-review-browser"
  );
-  const { x, y } = await SpecialPowers.spawn(reviewBrowser, [], async () => {
-    let review;
-    await ContentTaskUtils.waitForCondition(() => {
-      review = Cu.waiveXrays(
-        content.document.querySelector("ai-sff-form-review")
-      );
-      return review?.state === "review";
-    }, "Waiting for generated values to be ready for review");
-    await review.updateComplete;

-    const fields = review.renderRoot.querySelector(".form-review-fields");
-    fields.scrollTop = fields.scrollHeight;
-    fields.dispatchEvent(new content.Event("scroll"));
-    await review.updateComplete;
+  await waitForFormReviewState(reviewBrowser, FORM_REVIEW_STATES.REVIEW);
+  // Fill form only enables once the generated values have been scrolled
+  // through.
+  await scrollFormReviewFieldsToBottom(reviewBrowser);
+  await activateFormReviewButton(reviewBrowser, "ai-smart-form-fill-fill-form");
+}

-    const button = review.renderRoot.querySelector(
-      'moz-button[data-l10n-id="ai-smart-form-fill-fill-form"]'
-    );
-    await ContentTaskUtils.waitForCondition(
-      () => !button.disabled,
-      "Waiting for Fill form to be enabled"
-    );
+/**
+ * Runs a round on the contact form and approves the reviewed values, which is
+ * the state every outcome assertion starts from: the page has written the
+ * values and is waiting for something to end their fill.
+ *
+ * @param {Window} win
+ * @param {MozBrowser} browser
+ * @param {object} actor The SmartFormFill parent actor
+ * @returns {Promise<void>}
+ */
+async function fillContactForm(win, browser, actor) {
+  await runRoundOnForm(browser, actor);
+  await fillFormReview(win, browser);

-    const rect = button.buttonEl.getBoundingClientRect();
-    return {
-      x: rect.left + rect.width / 2,
-      y: rect.top + rect.height / 2,
-    };
-  });
-
-  await BrowserTestUtils.synthesizeMouseAtPoint(x, y, {}, reviewBrowser);
+  // The decision events land when the page reports what it filled, which is
+  // also what starts the tracking the outcomes come from.
+  await waitForEvents("formFillField", CONTACT_FIELDS);
+  await closeFormReview(win, browser);
+  await SimpleTest.promiseFocus(browser);
 }

 add_setup(async function () {
@@ -839,3 +914,240 @@ add_task(async function test_a_failure_before_dispatch_records_no_events() {
    }
  );
 });
+
+add_task(async function test_an_edited_value_is_reported_as_edited() {
+  await withFormPage({}, async ({ win, browser, actor }) => {
+    await fillContactForm(win, browser, actor);
+
+    await typeInFormField(browser, "#email", "typed");
+    await blurField(browser, "#email");
+
+    const [outcome] = await waitForEvents("formFillFieldOutcome", 1);
+
+    Assert.equal(
+      recordedExtras("formFillFieldOutcome").length,
+      1,
+      "A blur ends the fill of the blurred field only"
+    );
+    Assert.equal(
+      Number(outcome.field_seq),
+      0,
+      "The outcome is the email field's"
+    );
+    Assert.equal(
+      outcome.outcome,
+      "edited",
+      "A field the user typed into is reported as edited"
+    );
+  });
+});
+
+add_task(async function test_a_cleared_value_is_reported_as_cleared() {
+  await withFormPage({}, async ({ win, browser, actor }) => {
+    await fillContactForm(win, browser, actor);
+
+    await replaceFieldValue(browser, "#email", "");
+    await blurField(browser, "#email");
+
+    const [email] = await waitForEvents("formFillFieldOutcome", 1);
+    Assert.equal(
+      email.outcome,
+      "cleared",
+      "A field the user emptied is reported as cleared"
+    );
+
+    // Whitespace is not a value the user kept either.
+    await replaceFieldValue(browser, "#phone", "  ");
+    await blurField(browser, "#phone");
+
+    const [, phone] = await waitForEvents("formFillFieldOutcome", 2);
+    Assert.equal(
+      Number(phone.field_seq),
+      1,
+      "The second outcome is the phone field's"
+    );
+    Assert.equal(
+      phone.outcome,
+      "cleared",
+      "A field left with only whitespace is reported as cleared"
+    );
+  });
+});
+
+add_task(async function test_submitting_reports_every_filled_field() {
+  await withFormPage(
+    {
+      // Classified apart so the field kind tells the two decisions apart: with
+      // both fields classified the same, an outcome carrying the other field's
+      // decision would still look right.
+      classifyFields: async (request, { onDispatch }) => {
+        onDispatch?.(TEST_MODEL_INFO);
+
+        const types = ["email", "phone"];
+
+        return {
+          fields: request.fields.map(({ id }, index) => ({
+            id,
+            type: types[index],
+            confidence: "high",
+          })),
+        };
+      },
+    },
+    async ({ win, browser, actor }) => {
+      await fillContactForm(win, browser, actor);
+
+      // A blur only ends the fill of the field it happened on, so the field the
+      // user never focused is only reported once the form is submitted.
+      await submitForm(browser, "#contact");
+
+      const outcomes = await waitForEvents(
+        "formFillFieldOutcome",
+        CONTACT_FIELDS
+      );
+      const decisions = recordedExtras("formFillField");
+
+      Assert.equal(
+        outcomes.length,
+        CONTACT_FIELDS,
+        "Submitting reports every field the round filled"
+      );
+      Assert.notEqual(
+        decisions[0].field_kind,
+        decisions[1].field_kind,
+        "The two decisions differ, so the join below can tell them apart"
+      );
+
+      for (const decision of decisions) {
+        const outcome = outcomes.find(
+          candidate => candidate.field_seq === decision.field_seq
+        );
+
+        Assert.ok(outcome, `Field ${decision.field_seq} reported an outcome`);
+        Assert.equal(
+          outcome.outcome,
+          "kept",
+          "A value the user left alone is reported as kept"
+        );
+
+        // The outcome carries no field id, so what it is worth is what it can
+        // be joined with: each of these has to be the decision of the field it
+        // is reported for rather than of whichever field was resolved first.
+        Assert.equal(
+          outcome.flow_id,
+          decision.flow_id,
+          "The outcome shares the flow of the round that filled the field"
+        );
+        Assert.equal(
+          outcome.field_kind,
+          decision.field_kind,
+          "The outcome reports the field kind its decision was made against"
+        );
+        Assert.equal(
+          outcome.source,
+          decision.source,
+          "The outcome reports what the model decided filled the field"
+        );
+        Assert.equal(
+          outcome.confidence,
+          decision.confidence,
+          "The outcome reports the confidence the value was filled at"
+        );
+      }
+    }
+  );
+});
+
+add_task(async function test_a_filled_field_reports_its_outcome_once() {
+  await withFormPage(
+    {
+      // Only the email clears the threshold, so the phone is left for the user
+      // to fill and the page never tracks it.
+      generateFormValues: async (request, { onDispatch }) => {
+        onDispatch?.(TEST_MODEL_INFO);
+
+        const confidences = ["high", "low"];
+
+        return {
+          memories_used: [],
+          tabs_used: [],
+          fields: request.fields.map(({ id }, index) => ({
+            id,
+            action: "generate",
+            value: "generated value",
+            confidence: confidences[index],
+          })),
+          batches: { total: 1, failed: 0 },
+        };
+      },
+    },
+    async ({ win, browser, actor }) => {
+      await fillContactForm(win, browser, actor);
+
+      await focusField(browser, "#phone");
+      await blurField(browser, "#phone");
+
+      await focusField(browser, "#email");
+      await blurField(browser, "#email");
+
+      const [outcome] = await waitForEvents("formFillFieldOutcome", 1);
+      Assert.equal(
+        recordedExtras("formFillFieldOutcome").length,
+        1,
+        "A field that was never filled has no outcome to report"
+      );
+      Assert.equal(
+        Number(outcome.field_seq),
+        0,
+        "The outcome is the filled field's"
+      );
+
+      await focusField(browser, "#email");
+      await blurField(browser, "#email");
+
+      Assert.equal(
+        recordedExtras("formFillFieldOutcome").length,
+        1,
+        "A field reports its outcome once, however often its fill ends again"
+      );
+    }
+  );
+});
+
+add_task(async function test_navigating_reports_the_outcomes_left() {
+  await withFormPage({}, async ({ win, browser, actor }) => {
+    await fillContactForm(win, browser, actor);
+
+    const loaded = BrowserTestUtils.browserLoaded(
+      browser,
+      false,
+      NAVIGATION_PAGE
+    );
+    BrowserTestUtils.startLoadingURIString(browser, NAVIGATION_PAGE);
+    await loaded;
+
+    const outcomes = await waitForEvents(
+      "formFillFieldOutcome",
+      CONTACT_FIELDS
+    );
+
+    Assert.equal(
+      outcomes.length,
+      CONTACT_FIELDS,
+      "The page going away reports every field whose fill had not ended"
+    );
+    Assert.deepEqual(
+      outcomes.map(outcome => Number(outcome.field_seq)).sort(),
+      [0, 1],
+      "Each filled field is reported once"
+    );
+
+    for (const outcome of outcomes) {
+      Assert.equal(
+        outcome.outcome,
+        "kept",
+        "A value the page navigated away from untouched is reported as kept"
+      );
+    }
+  });
+});