diff --git a/browser/components/aiwindow/ui/test/browser/browser_smartwindow_smartformfill_telemetry.js b/browser/components/aiwindow/ui/test/browser/browser_smartwindow_smartformfill_telemetry.js index 66ff8126483d..37bf36b35d3a 100644 --- a/browser/components/aiwindow/ui/test/browser/browser_smartwindow_smartformfill_telemetry.js +++ b/browser/components/aiwindow/ui/test/browser/browser_smartwindow_smartformfill_telemetry.js @@ -3,6 +3,12 @@ "use strict"; +/* import-globals-from head_smartformfill_form_review.js */ +Services.scriptloader.loadSubScript( + getRootDirectory(gTestPath) + "head_smartformfill_form_review.js", + this +); + const { SmartFormFillModel } = ChromeUtils.importESModule( "moz-src:///browser/components/aiwindow/models/SmartFormFillModel.sys.mjs" ); @@ -12,9 +18,15 @@ const TEST_PAGE = `${getRootDirectory(gTestPath).replace( "https://example.com" )}test_smartformfill_telemetry.html`; +// Somewhere to navigate to, to make the form's page go away. +const NAVIGATION_PAGE = "https://example.com/"; + // The page has two forms: one with email + tel, one with a textarea. const EXPECTED_FORMS = 2; +// Fields of the contact form, which is the one every round below runs on. +const CONTACT_FIELDS = 2; + // What the fakes report as the model and prompt a request was built with. const TEST_MODEL_INFO = { model: "test-model", promptVersion: "42" }; @@ -201,6 +213,73 @@ function focusField(browser, selector) { ); } +/** + * Blurs a field, which is what ends the fill of that one field. + * + * The report a blur triggers is sent before the reply to this call, both being + * messages of the same window global, so an outcome it recorded has landed by + * the time this resolves. + * + * @param {MozBrowser} browser + * @param {string} selector Field to blur + * @returns {Promise} + */ +function blurField(browser, selector) { + return SpecialPowers.spawn(browser, [selector], fieldSelector => + content.document.querySelector(fieldSelector).blur() + ); +} + +/** + * Replaces what a field holds, selecting the filled value first so an empty + * string leaves the field empty rather than appending nothing. + * + * @param {MozBrowser} browser + * @param {string} selector Field to replace the value of + * @param {string} value Text to leave the field with + * @returns {Promise} + */ +function replaceFieldValue(browser, selector, value) { + return SpecialPowers.spawn( + browser, + [selector, value], + async (fieldSelector, text) => { + const field = content.document.querySelector(fieldSelector); + field.focus(); + EventUtils.synthesizeKey("a", { accelKey: true }, content); + EventUtils.synthesizeKey("KEY_Backspace", {}, content); + + if (text) { + await EventUtils.sendString(text, content); + } + } + ); +} + +/** + * Submits a form without letting it navigate, which ends the fill of every + * field of that form at once. + * + * The listener that reports the outcomes is in the system group, so cancelling + * the submission from the page stops the navigation without stopping the + * report. Validation is turned off because a generated value is not a valid + * email. + * + * @param {MozBrowser} browser + * @param {string} selector Form to submit + * @returns {Promise} + */ +function submitForm(browser, selector) { + return SpecialPowers.spawn(browser, [selector], formSelector => { + const form = content.document.querySelector(formSelector); + form.noValidate = true; + form.addEventListener("submit", event => event.preventDefault(), { + once: true, + }); + form.requestSubmit(); + }); +} + /** * Runs a whole round on the form a field belongs to: relevant tabs, * classification and generation. Nothing is requested until a form is asked @@ -242,37 +321,33 @@ async function fillFormReview(win, browser) { const reviewBrowser = dialog._frame.contentWindow.document.querySelector( "#form-review-browser" ); - const { x, y } = await SpecialPowers.spawn(reviewBrowser, [], async () => { - let review; - await ContentTaskUtils.waitForCondition(() => { - review = Cu.waiveXrays( - content.document.querySelector("ai-sff-form-review") - ); - return review?.state === "review"; - }, "Waiting for generated values to be ready for review"); - await review.updateComplete; - const fields = review.renderRoot.querySelector(".form-review-fields"); - fields.scrollTop = fields.scrollHeight; - fields.dispatchEvent(new content.Event("scroll")); - await review.updateComplete; + await waitForFormReviewState(reviewBrowser, FORM_REVIEW_STATES.REVIEW); + // Fill form only enables once the generated values have been scrolled + // through. + await scrollFormReviewFieldsToBottom(reviewBrowser); + await activateFormReviewButton(reviewBrowser, "ai-smart-form-fill-fill-form"); +} - const button = review.renderRoot.querySelector( - 'moz-button[data-l10n-id="ai-smart-form-fill-fill-form"]' - ); - await ContentTaskUtils.waitForCondition( - () => !button.disabled, - "Waiting for Fill form to be enabled" - ); +/** + * Runs a round on the contact form and approves the reviewed values, which is + * the state every outcome assertion starts from: the page has written the + * values and is waiting for something to end their fill. + * + * @param {Window} win + * @param {MozBrowser} browser + * @param {object} actor The SmartFormFill parent actor + * @returns {Promise} + */ +async function fillContactForm(win, browser, actor) { + await runRoundOnForm(browser, actor); + await fillFormReview(win, browser); - const rect = button.buttonEl.getBoundingClientRect(); - return { - x: rect.left + rect.width / 2, - y: rect.top + rect.height / 2, - }; - }); - - await BrowserTestUtils.synthesizeMouseAtPoint(x, y, {}, reviewBrowser); + // The decision events land when the page reports what it filled, which is + // also what starts the tracking the outcomes come from. + await waitForEvents("formFillField", CONTACT_FIELDS); + await closeFormReview(win, browser); + await SimpleTest.promiseFocus(browser); } add_setup(async function () { @@ -839,3 +914,240 @@ add_task(async function test_a_failure_before_dispatch_records_no_events() { } ); }); + +add_task(async function test_an_edited_value_is_reported_as_edited() { + await withFormPage({}, async ({ win, browser, actor }) => { + await fillContactForm(win, browser, actor); + + await typeInFormField(browser, "#email", "typed"); + await blurField(browser, "#email"); + + const [outcome] = await waitForEvents("formFillFieldOutcome", 1); + + Assert.equal( + recordedExtras("formFillFieldOutcome").length, + 1, + "A blur ends the fill of the blurred field only" + ); + Assert.equal( + Number(outcome.field_seq), + 0, + "The outcome is the email field's" + ); + Assert.equal( + outcome.outcome, + "edited", + "A field the user typed into is reported as edited" + ); + }); +}); + +add_task(async function test_a_cleared_value_is_reported_as_cleared() { + await withFormPage({}, async ({ win, browser, actor }) => { + await fillContactForm(win, browser, actor); + + await replaceFieldValue(browser, "#email", ""); + await blurField(browser, "#email"); + + const [email] = await waitForEvents("formFillFieldOutcome", 1); + Assert.equal( + email.outcome, + "cleared", + "A field the user emptied is reported as cleared" + ); + + // Whitespace is not a value the user kept either. + await replaceFieldValue(browser, "#phone", " "); + await blurField(browser, "#phone"); + + const [, phone] = await waitForEvents("formFillFieldOutcome", 2); + Assert.equal( + Number(phone.field_seq), + 1, + "The second outcome is the phone field's" + ); + Assert.equal( + phone.outcome, + "cleared", + "A field left with only whitespace is reported as cleared" + ); + }); +}); + +add_task(async function test_submitting_reports_every_filled_field() { + await withFormPage( + { + // Classified apart so the field kind tells the two decisions apart: with + // both fields classified the same, an outcome carrying the other field's + // decision would still look right. + classifyFields: async (request, { onDispatch }) => { + onDispatch?.(TEST_MODEL_INFO); + + const types = ["email", "phone"]; + + return { + fields: request.fields.map(({ id }, index) => ({ + id, + type: types[index], + confidence: "high", + })), + }; + }, + }, + async ({ win, browser, actor }) => { + await fillContactForm(win, browser, actor); + + // A blur only ends the fill of the field it happened on, so the field the + // user never focused is only reported once the form is submitted. + await submitForm(browser, "#contact"); + + const outcomes = await waitForEvents( + "formFillFieldOutcome", + CONTACT_FIELDS + ); + const decisions = recordedExtras("formFillField"); + + Assert.equal( + outcomes.length, + CONTACT_FIELDS, + "Submitting reports every field the round filled" + ); + Assert.notEqual( + decisions[0].field_kind, + decisions[1].field_kind, + "The two decisions differ, so the join below can tell them apart" + ); + + for (const decision of decisions) { + const outcome = outcomes.find( + candidate => candidate.field_seq === decision.field_seq + ); + + Assert.ok(outcome, `Field ${decision.field_seq} reported an outcome`); + Assert.equal( + outcome.outcome, + "kept", + "A value the user left alone is reported as kept" + ); + + // The outcome carries no field id, so what it is worth is what it can + // be joined with: each of these has to be the decision of the field it + // is reported for rather than of whichever field was resolved first. + Assert.equal( + outcome.flow_id, + decision.flow_id, + "The outcome shares the flow of the round that filled the field" + ); + Assert.equal( + outcome.field_kind, + decision.field_kind, + "The outcome reports the field kind its decision was made against" + ); + Assert.equal( + outcome.source, + decision.source, + "The outcome reports what the model decided filled the field" + ); + Assert.equal( + outcome.confidence, + decision.confidence, + "The outcome reports the confidence the value was filled at" + ); + } + } + ); +}); + +add_task(async function test_a_filled_field_reports_its_outcome_once() { + await withFormPage( + { + // Only the email clears the threshold, so the phone is left for the user + // to fill and the page never tracks it. + generateFormValues: async (request, { onDispatch }) => { + onDispatch?.(TEST_MODEL_INFO); + + const confidences = ["high", "low"]; + + return { + memories_used: [], + tabs_used: [], + fields: request.fields.map(({ id }, index) => ({ + id, + action: "generate", + value: "generated value", + confidence: confidences[index], + })), + batches: { total: 1, failed: 0 }, + }; + }, + }, + async ({ win, browser, actor }) => { + await fillContactForm(win, browser, actor); + + await focusField(browser, "#phone"); + await blurField(browser, "#phone"); + + await focusField(browser, "#email"); + await blurField(browser, "#email"); + + const [outcome] = await waitForEvents("formFillFieldOutcome", 1); + Assert.equal( + recordedExtras("formFillFieldOutcome").length, + 1, + "A field that was never filled has no outcome to report" + ); + Assert.equal( + Number(outcome.field_seq), + 0, + "The outcome is the filled field's" + ); + + await focusField(browser, "#email"); + await blurField(browser, "#email"); + + Assert.equal( + recordedExtras("formFillFieldOutcome").length, + 1, + "A field reports its outcome once, however often its fill ends again" + ); + } + ); +}); + +add_task(async function test_navigating_reports_the_outcomes_left() { + await withFormPage({}, async ({ win, browser, actor }) => { + await fillContactForm(win, browser, actor); + + const loaded = BrowserTestUtils.browserLoaded( + browser, + false, + NAVIGATION_PAGE + ); + BrowserTestUtils.startLoadingURIString(browser, NAVIGATION_PAGE); + await loaded; + + const outcomes = await waitForEvents( + "formFillFieldOutcome", + CONTACT_FIELDS + ); + + Assert.equal( + outcomes.length, + CONTACT_FIELDS, + "The page going away reports every field whose fill had not ended" + ); + Assert.deepEqual( + outcomes.map(outcome => Number(outcome.field_seq)).sort(), + [0, 1], + "Each filled field is reported once" + ); + + for (const outcome of outcomes) { + Assert.equal( + outcome.outcome, + "kept", + "A value the page navigated away from untouched is reported as kept" + ); + } + }); +});