XLinfini/sousa-gecko
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 2056207 - Remove image text recognition shadow DOM code. r=nordzilla

Browse Source 
This never shipped, and is only testable on macOS. Nobody seems actively
working on this so we should probably delete it.

We can always rescue from git history if needed.

Differential Revision: https://phabricator.services.mozilla.com/D312977
This commit is contained in:
Emilio Cobos Álvarez
2026-07-20 13:39:50 +00:00
committed by ealvarez@mozilla.com
parent ecc252f7ff
commit ea0e13eecb
7 changed files with 0 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser/components/storybook/docs/README.other-widgets.stories.md
-1
View File
@@ -23,7 +23,6 @@ User agent (UA) widgets are like custom elements but run in per-origin UA widget
There are a much smaller number of these widgets compared to the Mozilla Custom Elements:
- [datetimebox.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/datetimebox.js)
- [marquee.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/marquee.js)
- [textrecognition.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/textrecognition.js)
- [videocontrols.js](https://searchfox.org/firefox-main/source/toolkit/content/widgets/videocontrols.js)
Please refer to the existing [UA widgets documentation](https://firefox-source-docs.mozilla.org/toolkit/content/toolkit_widgets/ua_widget.html) for more details.
dom/base/nsImageLoadingContent.cpp
-11
View File
@@ -1349,17 +1349,6 @@ already_AddRefed<Promise> nsImageLoadingContent::RecognizeCurrentImageText(
auto& textRecognitionResult = aValue.ResolveValue();
Element* el = ilc->AsContent()->AsElement();
// When enabled, this feature will place the recognized text as
// spans inside of the shadow dom of the img element. These are then
// positioned so that the user can select the text.
if (Preferences::GetBool("dom.text-recognition.shadow-dom-enabled",
false)) {
el->AttachAndSetUAShadowRoot(Element::NotifyUAWidget::Yes);
TextRecognition::FillShadow(*el->GetShadowRoot(),
textRecognitionResult);
el->NotifyUAWidgetSetupOrChange();
}
nsTArray<ImageText> imageTexts(
textRecognitionResult.quads().Length());
// XXX shouldn't this be GetRelevantGlobal? But it's privileged code
toolkit/actors/UAWidgetsChild.sys.mjs
-3
View File
@@ -104,9 +104,6 @@ export class UAWidgetsChild extends JSWindowActorChild {
uri = "chrome://global/content/elements/marquee.js";
widgetName = "MarqueeWidget";
break;
case "img":
uri = "chrome://global/content/elements/textrecognition.js";
widgetName = "TextRecognitionWidget";
}
if (!uri || !widgetName) {
toolkit/content/jar.mn
-1
View File
@@ -198,7 +198,6 @@ toolkit.jar:
content/global/elements/stringbundle.js (widgets/stringbundle.js)
content/global/elements/tabbox.js (widgets/tabbox.js)
content/global/elements/text.js (widgets/text.js)
content/global/elements/textrecognition.js (widgets/textrecognition.js)
content/global/elements/theme-picker.css (widgets/theme-picker/theme-picker.css)
content/global/elements/theme-picker.mjs (widgets/theme-picker/theme-picker.mjs)
content/global/elements/theme-picker-storybook-controller.mjs (widgets/theme-picker/theme-picker-storybook-controller.mjs)
toolkit/content/tests/widgets/test_image_recognition.html
-28
View File
@@ -19,19 +19,11 @@
"resource://testing-common/TestUtils.sys.mjs"
);
function pushPref(preferenceName, value) {
return new Promise(resolve => {
const options = { set: [[preferenceName, value]] };
SpecialPowers.pushPrefEnv(options, resolve);
});
}
add_task(async () => {
// Performing text recognition in CI can take some time, and test verify runs have
// timed out.
SimpleTest.requestLongerTimeout(2);
await pushPref("dom.text-recognition.shadow-dom-enabled", true);
const img = document.querySelector("#content img");
info("Recognizing the image text");
@@ -52,26 +44,6 @@
mozilla.quad.p1.y > firefox.quad.p2.y,
"The Mozilla text is above Firefox"
);
const spans = await TestUtils.waitForCondition(
() => shadowRootQuerySelectorAll(img, "span"),
"Attempting to get image recognition spans."
);
const mozillaSpan = [...spans].find(s => s.innerText === "Mozilla");
const firefoxSpan = [...spans].find(s => s.innerText === "Firefox");
ok(mozillaSpan, "The word Mozilla span was found.");
ok(firefoxSpan, "The word Firefox span was found.");
ok(
mozillaSpan.style.transform.startsWith("matrix3d("),
"A matrix transform was applied"
);
ok(
firefoxSpan.style.transform.startsWith("matrix3d("),
"A matrix transform was applied"
);
});
</script>
</body>
toolkit/content/widgets/textrecognition.js
-366
View File
@@ -1,366 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
// This is a UA widget. It runs in per-origin UA widget scope,
// to be loaded by UAWidgetsChild.sys.mjs.
this.TextRecognitionWidget = class {
/**
* @param {ShadowRoot} shadowRoot
* @param {Record<string, string | boolean | number>} _prefs
*/
constructor(shadowRoot, _prefs) {
/** @type {ShadowRoot} */
this.shadowRoot = shadowRoot;
/** @type {HTMLElement} */
this.element = shadowRoot.host;
/** @type {Document} */
this.document = this.element.ownerDocument;
/** @type {Window} */
this.window = this.document.defaultView;
/** @type {ResizeObserver} */
this.resizeObserver = null;
/** @type {Map<HTMLSpanElement, DOMRect} */
this.spanRects = new Map();
/** @type {boolean} */
this.isInitialized = false;
/** @type {null | number} */
this.lastCanvasStyleWidth = null;
}
/*
* Callback called by UAWidgets right after constructor.
*/
onsetup() {
this.resizeObserver = new this.window.ResizeObserver(() => {
this.positionSpans();
});
this.resizeObserver.observe(this.element);
}
positionSpans() {
if (!this.shadowRoot.firstChild) {
return;
}
this.lazilyInitialize();
/** @type {HTMLDivElement} */
const div = this.shadowRoot.firstChild;
const canvas = div.querySelector("canvas");
const spans = div.querySelectorAll("span");
// TODO Bug 1770438 - The <img> element does not currently let child elements be
// sized relative to the size of the containing <img> element. It would be better
// to teach the <img> element how to do this. For the prototype, do the more expensive
// operation of getting the bounding client rect, and handle the positioning manually.
const imgRect = this.element.getBoundingClientRect();
div.style.width = imgRect.width + "px";
div.style.height = imgRect.height + "px";
canvas.style.width = imgRect.width + "px";
canvas.style.height = imgRect.height + "px";
// The ctx is only available when redrawing the canvas. This is operation is only
// done when necessary, as it can be expensive.
/** @type {null | CanvasRenderingContext2D} */
let ctx = null;
if (
// The canvas hasn't been drawn to yet.
this.lastCanvasStyleWidth === null ||
// Only redraw when the image has grown 25% larger. This percentage was chosen
// as it visually seemed to work well, with the canvas never appearing blurry
// when manually testing it.
imgRect.width > this.lastCanvasStyleWidth * 1.25
) {
const dpr = this.window.devicePixelRatio;
canvas.width = imgRect.width * dpr;
canvas.height = imgRect.height * dpr;
this.lastCanvasStyleWidth = imgRect.width;
ctx = canvas.getContext("2d");
ctx.scale(dpr, dpr);
ctx.fillStyle = "#00000088";
ctx.fillRect(0, 0, imgRect.width, imgRect.height);
ctx.beginPath();
}
for (const span of spans) {
let spanRect = this.spanRects.get(span);
if (!spanRect) {
// This only needs to happen once.
spanRect = span.getBoundingClientRect();
this.spanRects.set(span, spanRect);
}
const points = span.dataset.points.split(",").map(p => Number(p));
// Use the points in the string, e.g.
// "0.0275349,0.14537,0.0275349,0.244662,0.176966,0.244565,0.176966,0.145273"
// 0 1 2 3 4 5 6 7
// ^ bottomleft ^ topleft ^ topright ^ bottomright
let [
bottomLeftX,
bottomLeftY,
topLeftX,
topLeftY,
topRightX,
topRightY,
bottomRightX,
bottomRightY,
] = points;
// Invert the Y.
topLeftY = 1 - topLeftY;
topRightY = 1 - topRightY;
bottomLeftY = 1 - bottomLeftY;
bottomRightY = 1 - bottomRightY;
// Create a projection matrix to position the <span> relative to the bounds.
// prettier-ignore
const mat4 = projectPoints(
spanRect.width, spanRect.height,
imgRect.width * topLeftX, imgRect.height * topLeftY,
imgRect.width * topRightX, imgRect.height * topRightY,
imgRect.width * bottomLeftX, imgRect.height * bottomLeftY,
imgRect.width * bottomRightX, imgRect.height * bottomRightY
);
span.style.transform = "matrix3d(" + mat4.join(", ") + ")";
if (ctx) {
const inset = 3;
ctx.moveTo(
imgRect.width * bottomLeftX + inset,
imgRect.height * bottomLeftY - inset
);
ctx.lineTo(
imgRect.width * topLeftX + inset,
imgRect.height * topLeftY + inset
);
ctx.lineTo(
imgRect.width * topRightX - inset,
imgRect.height * topRightY + inset
);
ctx.lineTo(
imgRect.width * bottomRightX - inset,
imgRect.height * bottomRightY - inset
);
ctx.closePath();
}
}
if (ctx) {
// This composite operation will cut out the quads. The color is arbitrary.
ctx.globalCompositeOperation = "destination-out";
ctx.fillStyle = "#ffffff";
ctx.fill();
// Creating a round line will grow the selection slightly, and round the corners.
ctx.lineWidth = 10;
ctx.lineJoin = "round";
ctx.strokeStyle = "#ffffff";
ctx.stroke();
}
}
teardown() {
this.shadowRoot.firstChild.remove();
this.resizeObserver.disconnect();
this.spanRects.clear();
}
lazilyInitialize() {
if (this.isInitialized) {
return;
}
this.isInitialized = true;
const parser = new this.window.DOMParser();
let parserDoc = parser.parseFromSafeString(
`<div class="textrecognition" xmlns="http://www.w3.org/1999/xhtml" role="none">
<link rel="stylesheet" href="chrome://global/skin/media/textrecognition.css" />
<canvas />
<!-- The spans will be reattached here -->
</div>`,
"application/xml"
);
if (
this.shadowRoot.children.length !== 1 ||
this.shadowRoot.firstChild.tagName !== "DIV"
) {
throw new Error(
"Expected the shadowRoot to have a single div as the root element."
);
}
const spansDiv = this.shadowRoot.firstChild;
// Example layout of spansDiv:
// <div>
// <span data-points="0.0275349,0.14537,0.0275349,0.244662,0.176966,0.244565,0.176966,0.145273">
// Text that has been recognized
// </span>
// ...
// </div>
spansDiv.remove();
this.shadowRoot.importNodeAndAppendChildAt(
this.shadowRoot,
parserDoc.documentElement,
true /* deep */
);
this.shadowRoot.importNodeAndAppendChildAt(
this.shadowRoot.firstChild,
spansDiv,
true /* deep */
);
}
};
/**
* A three dimensional vector.
*
* @typedef {[number, number, number]} Vec3
*/
/**
* A 3x3 matrix.
*
* @typedef {[number, number, number,
* number, number, number,
* number, number, number]} Matrix3
*/
/**
* A 4x4 matrix.
*
* @typedef {[number, number, number, number,
* number, number, number, number,
* number, number, number, number,
* number, number, number, number]} Matrix4
*/
/**
* Compute the adjugate matrix.
* https://en.wikipedia.org/wiki/Adjugate_matrix
*
* @param {Matrix3} m
* @returns {Matrix3}
*/
function computeAdjugate(m) {
// prettier-ignore
return [
m[4] * m[8] - m[5] * m[7],
m[2] * m[7] - m[1] * m[8],
m[1] * m[5] - m[2] * m[4],
m[5] * m[6] - m[3] * m[8],
m[0] * m[8] - m[2] * m[6],
m[2] * m[3] - m[0] * m[5],
m[3] * m[7] - m[4] * m[6],
m[1] * m[6] - m[0] * m[7],
m[0] * m[4] - m[1] * m[3],
];
}
/**
* @param {Matrix3} a
* @param {Matrix3} b
* @returns {Matrix3}
*/
function multiplyMat3(a, b) {
let out = [];
for (let i = 0; i < 3; i++) {
for (let j = 0; j < 3; j++) {
let sum = 0;
for (let k = 0; k < 3; k++) {
sum += a[3 * i + k] * b[3 * k + j];
}
out[3 * i + j] = sum;
}
}
return out;
}
/**
* @param {Matrix3} m
* @param {Vec3} v
* @returns {Vec3}
*/
function multiplyMat3Vec3(m, v) {
// prettier-ignore
return [
m[0] * v[0] + m[1] * v[1] + m[2] * v[2],
m[3] * v[0] + m[4] * v[1] + m[5] * v[2],
m[6] * v[0] + m[7] * v[1] + m[8] * v[2],
];
}
/**
* @returns {Matrix3}
*/
function basisToPoints(x1, y1, x2, y2, x3, y3, x4, y4) {
/** @type {Matrix3} */
let mat3 = [x1, x2, x3, y1, y2, y3, 1, 1, 1];
let vec3 = multiplyMat3Vec3(computeAdjugate(mat3), [x4, y4, 1]);
// prettier-ignore
return multiplyMat3(
mat3,
[
vec3[0], 0, 0,
0, vec3[1], 0,
0, 0, vec3[2]
]
);
}
/**
* @type {(...Matrix4) => Matrix3}
*/
// prettier-ignore
function general2DProjection(
x1s, y1s, x1d, y1d,
x2s, y2s, x2d, y2d,
x3s, y3s, x3d, y3d,
x4s, y4s, x4d, y4d
) {
let s = basisToPoints(x1s, y1s, x2s, y2s, x3s, y3s, x4s, y4s);
let d = basisToPoints(x1d, y1d, x2d, y2d, x3d, y3d, x4d, y4d);
return multiplyMat3(d, computeAdjugate(s));
}
/**
* Given a width and height, compute a projection matrix to points 1-4.
*
* The points (x1,y1) through (x4, y4) use the following ordering:
*
* w
* ┌─────┐ project 1 ─────── 2
* h │ │ --> │ /
* └─────┘ │ /
* 3 ──── 4
*
* @returns {Matrix4}
*/
function projectPoints(w, h, x1, y1, x2, y2, x3, y3, x4, y4) {
// prettier-ignore
const mat3 = general2DProjection(
0, 0, x1, y1,
w, 0, x2, y2,
0, h, x3, y3,
w, h, x4, y4
);
for (let i = 0; i < 9; i++) {
mat3[i] = mat3[i] / mat3[8];
}
// prettier-ignore
return [
mat3[0], mat3[3], 0, mat3[6],
mat3[1], mat3[4], 0, mat3[7],
0, 0, 1, 0,
mat3[2], mat3[5], 0, mat3[8],
];
}
widget/TextRecognition.cpp
-33
View File
@@ -77,39 +77,6 @@ auto TextRecognition::FindText(gfx::DataSourceSurface& aSurface,
return DoFindText(aSurface, aLanguages);
}
void TextRecognition::FillShadow(ShadowRoot& aShadow,
const TextRecognitionResult& aResult) {
auto& doc = *aShadow.OwnerDoc();
RefPtr<Element> div = doc.CreateHTMLElement(nsGkAtoms::div);
for (const auto& quad : aResult.quads()) {
RefPtr<Element> span = doc.CreateHTMLElement(nsGkAtoms::span);
// TODO: We probably want to position them here and so on. For now, expose
// the data as attributes so that it's easy to play with the returned values
// in JS.
{
nsAutoString points;
for (const auto& point : quad.points()) {
points.AppendFloat(point.x);
points.Append(u',');
points.AppendFloat(point.y);
points.Append(u',');
}
points.Trim(",");
span->SetAttribute(u"data-points"_ns, points, IgnoreErrors());
nsAutoString confidence;
confidence.AppendFloat(quad.confidence());
span->SetAttribute(u"data-confidence"_ns, confidence, IgnoreErrors());
}
{
RefPtr<nsTextNode> text = doc.CreateTextNode(quad.string());
span->AppendChildTo(text, true, IgnoreErrors());
}
div->AppendChildTo(span, true, IgnoreErrors());
}
aShadow.AppendChildTo(div, true, IgnoreErrors());
}
#ifndef XP_MACOSX
auto TextRecognition::DoFindText(gfx::DataSourceSurface&,
const nsTArray<nsCString>&)