diff --git a/browser/components/storybook/docs/README.other-widgets.stories.md b/browser/components/storybook/docs/README.other-widgets.stories.md index 49191b726a0c..6756e75cdf50 100644 --- a/browser/components/storybook/docs/README.other-widgets.stories.md +++ b/browser/components/storybook/docs/README.other-widgets.stories.md @@ -23,7 +23,6 @@ User agent (UA) widgets are like custom elements but run in per-origin UA widget There are a much smaller number of these widgets compared to the Mozilla Custom Elements: - [datetimebox.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/datetimebox.js) - [marquee.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/marquee.js) -- [textrecognition.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/textrecognition.js) - [videocontrols.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/videocontrols.js) Please refer to the existing [UA widgets documentation](https://firefox-source-docs.mozilla.org/toolkit/content/toolkit_widgets/ua_widget.html) for more details. diff --git a/dom/base/nsImageLoadingContent.cpp b/dom/base/nsImageLoadingContent.cpp index 07e88ceb734b..84e92d17553e 100644 --- a/dom/base/nsImageLoadingContent.cpp +++ b/dom/base/nsImageLoadingContent.cpp @@ -1349,17 +1349,6 @@ already_AddRefed nsImageLoadingContent::RecognizeCurrentImageText( auto& textRecognitionResult = aValue.ResolveValue(); Element* el = ilc->AsContent()->AsElement(); - // When enabled, this feature will place the recognized text as - // spans inside of the shadow dom of the img element. These are then - // positioned so that the user can select the text. - if (Preferences::GetBool("dom.text-recognition.shadow-dom-enabled", - false)) { - el->AttachAndSetUAShadowRoot(Element::NotifyUAWidget::Yes); - TextRecognition::FillShadow(*el->GetShadowRoot(), - textRecognitionResult); - el->NotifyUAWidgetSetupOrChange(); - } - nsTArray imageTexts( textRecognitionResult.quads().Length()); // XXX shouldn't this be GetRelevantGlobal? But it's privileged code diff --git a/toolkit/actors/UAWidgetsChild.sys.mjs b/toolkit/actors/UAWidgetsChild.sys.mjs index e4e56dd7e590..963bdeb7a986 100644 --- a/toolkit/actors/UAWidgetsChild.sys.mjs +++ b/toolkit/actors/UAWidgetsChild.sys.mjs @@ -104,9 +104,6 @@ export class UAWidgetsChild extends JSWindowActorChild { uri = "chrome://global/content/elements/marquee.js"; widgetName = "MarqueeWidget"; break; - case "img": - uri = "chrome://global/content/elements/textrecognition.js"; - widgetName = "TextRecognitionWidget"; } if (!uri || !widgetName) { diff --git a/toolkit/content/jar.mn b/toolkit/content/jar.mn index 79b6ee4c8eec..f8b471c22e3d 100644 --- a/toolkit/content/jar.mn +++ b/toolkit/content/jar.mn @@ -198,7 +198,6 @@ toolkit.jar: content/global/elements/stringbundle.js (widgets/stringbundle.js) content/global/elements/tabbox.js (widgets/tabbox.js) content/global/elements/text.js (widgets/text.js) - content/global/elements/textrecognition.js (widgets/textrecognition.js) content/global/elements/theme-picker.css (widgets/theme-picker/theme-picker.css) content/global/elements/theme-picker.mjs (widgets/theme-picker/theme-picker.mjs) content/global/elements/theme-picker-storybook-controller.mjs (widgets/theme-picker/theme-picker-storybook-controller.mjs) diff --git a/toolkit/content/tests/widgets/test_image_recognition.html b/toolkit/content/tests/widgets/test_image_recognition.html index b2a9b93180bf..5f426e33b9dd 100644 --- a/toolkit/content/tests/widgets/test_image_recognition.html +++ b/toolkit/content/tests/widgets/test_image_recognition.html @@ -19,19 +19,11 @@ "resource://testing-common/TestUtils.sys.mjs" ); - function pushPref(preferenceName, value) { - return new Promise(resolve => { - const options = { set: [[preferenceName, value]] }; - SpecialPowers.pushPrefEnv(options, resolve); - }); - } - add_task(async () => { // Performing text recognition in CI can take some time, and test verify runs have // timed out. SimpleTest.requestLongerTimeout(2); - await pushPref("dom.text-recognition.shadow-dom-enabled", true); const img = document.querySelector("#content img"); info("Recognizing the image text"); @@ -52,26 +44,6 @@ mozilla.quad.p1.y > firefox.quad.p2.y, "The Mozilla text is above Firefox" ); - - const spans = await TestUtils.waitForCondition( - () => shadowRootQuerySelectorAll(img, "span"), - "Attempting to get image recognition spans." - ); - - const mozillaSpan = [...spans].find(s => s.innerText === "Mozilla"); - const firefoxSpan = [...spans].find(s => s.innerText === "Firefox"); - - ok(mozillaSpan, "The word Mozilla span was found."); - ok(firefoxSpan, "The word Firefox span was found."); - - ok( - mozillaSpan.style.transform.startsWith("matrix3d("), - "A matrix transform was applied" - ); - ok( - firefoxSpan.style.transform.startsWith("matrix3d("), - "A matrix transform was applied" - ); }); diff --git a/toolkit/content/widgets/textrecognition.js b/toolkit/content/widgets/textrecognition.js deleted file mode 100644 index e0d55eb7db56..000000000000 --- a/toolkit/content/widgets/textrecognition.js +++ /dev/null @@ -1,366 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -"use strict"; - -// This is a UA widget. It runs in per-origin UA widget scope, -// to be loaded by UAWidgetsChild.sys.mjs. - -this.TextRecognitionWidget = class { - /** - * @param {ShadowRoot} shadowRoot - * @param {Record} _prefs - */ - constructor(shadowRoot, _prefs) { - /** @type {ShadowRoot} */ - this.shadowRoot = shadowRoot; - /** @type {HTMLElement} */ - this.element = shadowRoot.host; - /** @type {Document} */ - this.document = this.element.ownerDocument; - /** @type {Window} */ - this.window = this.document.defaultView; - /** @type {ResizeObserver} */ - this.resizeObserver = null; - /** @type {Map { - this.positionSpans(); - }); - this.resizeObserver.observe(this.element); - } - - positionSpans() { - if (!this.shadowRoot.firstChild) { - return; - } - this.lazilyInitialize(); - - /** @type {HTMLDivElement} */ - const div = this.shadowRoot.firstChild; - const canvas = div.querySelector("canvas"); - const spans = div.querySelectorAll("span"); - - // TODO Bug 1770438 - The element does not currently let child elements be - // sized relative to the size of the containing element. It would be better - // to teach the element how to do this. For the prototype, do the more expensive - // operation of getting the bounding client rect, and handle the positioning manually. - const imgRect = this.element.getBoundingClientRect(); - div.style.width = imgRect.width + "px"; - div.style.height = imgRect.height + "px"; - canvas.style.width = imgRect.width + "px"; - canvas.style.height = imgRect.height + "px"; - - // The ctx is only available when redrawing the canvas. This is operation is only - // done when necessary, as it can be expensive. - /** @type {null | CanvasRenderingContext2D} */ - let ctx = null; - - if ( - // The canvas hasn't been drawn to yet. - this.lastCanvasStyleWidth === null || - // Only redraw when the image has grown 25% larger. This percentage was chosen - // as it visually seemed to work well, with the canvas never appearing blurry - // when manually testing it. - imgRect.width > this.lastCanvasStyleWidth * 1.25 - ) { - const dpr = this.window.devicePixelRatio; - canvas.width = imgRect.width * dpr; - canvas.height = imgRect.height * dpr; - this.lastCanvasStyleWidth = imgRect.width; - - ctx = canvas.getContext("2d"); - ctx.scale(dpr, dpr); - ctx.fillStyle = "#00000088"; - ctx.fillRect(0, 0, imgRect.width, imgRect.height); - - ctx.beginPath(); - } - - for (const span of spans) { - let spanRect = this.spanRects.get(span); - if (!spanRect) { - // This only needs to happen once. - spanRect = span.getBoundingClientRect(); - this.spanRects.set(span, spanRect); - } - - const points = span.dataset.points.split(",").map(p => Number(p)); - // Use the points in the string, e.g. - // "0.0275349,0.14537,0.0275349,0.244662,0.176966,0.244565,0.176966,0.145273" - // 0 1 2 3 4 5 6 7 - // ^ bottomleft ^ topleft ^ topright ^ bottomright - let [ - bottomLeftX, - bottomLeftY, - topLeftX, - topLeftY, - topRightX, - topRightY, - bottomRightX, - bottomRightY, - ] = points; - - // Invert the Y. - topLeftY = 1 - topLeftY; - topRightY = 1 - topRightY; - bottomLeftY = 1 - bottomLeftY; - bottomRightY = 1 - bottomRightY; - - // Create a projection matrix to position the relative to the bounds. - // prettier-ignore - const mat4 = projectPoints( - spanRect.width, spanRect.height, - imgRect.width * topLeftX, imgRect.height * topLeftY, - imgRect.width * topRightX, imgRect.height * topRightY, - imgRect.width * bottomLeftX, imgRect.height * bottomLeftY, - imgRect.width * bottomRightX, imgRect.height * bottomRightY - ); - - span.style.transform = "matrix3d(" + mat4.join(", ") + ")"; - - if (ctx) { - const inset = 3; - ctx.moveTo( - imgRect.width * bottomLeftX + inset, - imgRect.height * bottomLeftY - inset - ); - ctx.lineTo( - imgRect.width * topLeftX + inset, - imgRect.height * topLeftY + inset - ); - ctx.lineTo( - imgRect.width * topRightX - inset, - imgRect.height * topRightY + inset - ); - ctx.lineTo( - imgRect.width * bottomRightX - inset, - imgRect.height * bottomRightY - inset - ); - ctx.closePath(); - } - } - - if (ctx) { - // This composite operation will cut out the quads. The color is arbitrary. - ctx.globalCompositeOperation = "destination-out"; - ctx.fillStyle = "#ffffff"; - ctx.fill(); - - // Creating a round line will grow the selection slightly, and round the corners. - ctx.lineWidth = 10; - ctx.lineJoin = "round"; - ctx.strokeStyle = "#ffffff"; - ctx.stroke(); - } - } - - teardown() { - this.shadowRoot.firstChild.remove(); - this.resizeObserver.disconnect(); - this.spanRects.clear(); - } - - lazilyInitialize() { - if (this.isInitialized) { - return; - } - this.isInitialized = true; - - const parser = new this.window.DOMParser(); - let parserDoc = parser.parseFromSafeString( - `
- - - -
`, - "application/xml" - ); - if ( - this.shadowRoot.children.length !== 1 || - this.shadowRoot.firstChild.tagName !== "DIV" - ) { - throw new Error( - "Expected the shadowRoot to have a single div as the root element." - ); - } - - const spansDiv = this.shadowRoot.firstChild; - // Example layout of spansDiv: - //
- // - // Text that has been recognized - // - // ... - //
- spansDiv.remove(); - - this.shadowRoot.importNodeAndAppendChildAt( - this.shadowRoot, - parserDoc.documentElement, - true /* deep */ - ); - - this.shadowRoot.importNodeAndAppendChildAt( - this.shadowRoot.firstChild, - spansDiv, - true /* deep */ - ); - } -}; - -/** - * A three dimensional vector. - * - * @typedef {[number, number, number]} Vec3 - */ - -/** - * A 3x3 matrix. - * - * @typedef {[number, number, number, - * number, number, number, - * number, number, number]} Matrix3 - */ - -/** - * A 4x4 matrix. - * - * @typedef {[number, number, number, number, - * number, number, number, number, - * number, number, number, number, - * number, number, number, number]} Matrix4 - */ - -/** - * Compute the adjugate matrix. - * https://en.wikipedia.org/wiki/Adjugate_matrix - * - * @param {Matrix3} m - * @returns {Matrix3} - */ -function computeAdjugate(m) { - // prettier-ignore - return [ - m[4] * m[8] - m[5] * m[7], - m[2] * m[7] - m[1] * m[8], - m[1] * m[5] - m[2] * m[4], - m[5] * m[6] - m[3] * m[8], - m[0] * m[8] - m[2] * m[6], - m[2] * m[3] - m[0] * m[5], - m[3] * m[7] - m[4] * m[6], - m[1] * m[6] - m[0] * m[7], - m[0] * m[4] - m[1] * m[3], - ]; -} - -/** - * @param {Matrix3} a - * @param {Matrix3} b - * @returns {Matrix3} - */ -function multiplyMat3(a, b) { - let out = []; - for (let i = 0; i < 3; i++) { - for (let j = 0; j < 3; j++) { - let sum = 0; - for (let k = 0; k < 3; k++) { - sum += a[3 * i + k] * b[3 * k + j]; - } - out[3 * i + j] = sum; - } - } - return out; -} - -/** - * @param {Matrix3} m - * @param {Vec3} v - * @returns {Vec3} - */ -function multiplyMat3Vec3(m, v) { - // prettier-ignore - return [ - m[0] * v[0] + m[1] * v[1] + m[2] * v[2], - m[3] * v[0] + m[4] * v[1] + m[5] * v[2], - m[6] * v[0] + m[7] * v[1] + m[8] * v[2], - ]; -} - -/** - * @returns {Matrix3} - */ -function basisToPoints(x1, y1, x2, y2, x3, y3, x4, y4) { - /** @type {Matrix3} */ - let mat3 = [x1, x2, x3, y1, y2, y3, 1, 1, 1]; - let vec3 = multiplyMat3Vec3(computeAdjugate(mat3), [x4, y4, 1]); - // prettier-ignore - return multiplyMat3( - mat3, - [ - vec3[0], 0, 0, - 0, vec3[1], 0, - 0, 0, vec3[2] - ] - ); -} - -/** - * @type {(...Matrix4) => Matrix3} - */ -// prettier-ignore -function general2DProjection( - x1s, y1s, x1d, y1d, - x2s, y2s, x2d, y2d, - x3s, y3s, x3d, y3d, - x4s, y4s, x4d, y4d -) { - let s = basisToPoints(x1s, y1s, x2s, y2s, x3s, y3s, x4s, y4s); - let d = basisToPoints(x1d, y1d, x2d, y2d, x3d, y3d, x4d, y4d); - return multiplyMat3(d, computeAdjugate(s)); -} - -/** - * Given a width and height, compute a projection matrix to points 1-4. - * - * The points (x1,y1) through (x4, y4) use the following ordering: - * - * w - * ┌─────┐ project 1 ─────── 2 - * h │ │ --> │ / - * └─────┘ │ / - * 3 ──── 4 - * - * @returns {Matrix4} - */ -function projectPoints(w, h, x1, y1, x2, y2, x3, y3, x4, y4) { - // prettier-ignore - const mat3 = general2DProjection( - 0, 0, x1, y1, - w, 0, x2, y2, - 0, h, x3, y3, - w, h, x4, y4 - ); - - for (let i = 0; i < 9; i++) { - mat3[i] = mat3[i] / mat3[8]; - } - - // prettier-ignore - return [ - mat3[0], mat3[3], 0, mat3[6], - mat3[1], mat3[4], 0, mat3[7], - 0, 0, 1, 0, - mat3[2], mat3[5], 0, mat3[8], - ]; -} diff --git a/widget/TextRecognition.cpp b/widget/TextRecognition.cpp index 650be5b1b7d1..88e296b620c5 100644 --- a/widget/TextRecognition.cpp +++ b/widget/TextRecognition.cpp @@ -77,39 +77,6 @@ auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface, return DoFindText(aSurface, aLanguages); } -void TextRecognition::FillShadow(ShadowRoot& aShadow, - const TextRecognitionResult& aResult) { - auto& doc = *aShadow.OwnerDoc(); - RefPtr div = doc.CreateHTMLElement(nsGkAtoms::div); - for (const auto& quad : aResult.quads()) { - RefPtr span = doc.CreateHTMLElement(nsGkAtoms::span); - // TODO: We probably want to position them here and so on. For now, expose - // the data as attributes so that it's easy to play with the returned values - // in JS. - { - nsAutoString points; - for (const auto& point : quad.points()) { - points.AppendFloat(point.x); - points.Append(u','); - points.AppendFloat(point.y); - points.Append(u','); - } - points.Trim(","); - span->SetAttribute(u"data-points"_ns, points, IgnoreErrors()); - nsAutoString confidence; - confidence.AppendFloat(quad.confidence()); - span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors()); - } - - { - RefPtr text = doc.CreateTextNode(quad.string()); - span->AppendChildTo(text, true, IgnoreErrors()); - } - div->AppendChildTo(span, true, IgnoreErrors()); - } - aShadow.AppendChildTo(div, true, IgnoreErrors()); -} - #ifndef XP_MACOSX auto TextRecognition::DoFindText(gfx::DataSourceSurface&, const nsTArray&)