Google Docs accessibility update (#2235)

* Update Google Docs injection script

* Create GoogleDocsUtil

* Update Frontend.js to register GoogleDocsUtil's getRangeFromPoint handler

* Update setting name and description

* Add comment

* Fix Firefox support

ext/js/accessibility/google-docs-util.js

@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2022  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/* global
+ * DocumentUtil
+ * TextSourceRange
+ */
+
+/**
+ * This class is a helper for handling Google Docs content in content scripts.
+ */
+class GoogleDocsUtil {
+    /**
+     * Scans the document for text or elements with text information at the given coordinate.
+     * Coordinates are provided in [client space](https://developer.mozilla.org/en-US/docs/Web/CSS/CSSOM_View/Coordinate_systems).
+     * @param {number} x The x coordinate to search at.
+     * @param {number} y The y coordinate to search at.
+     * @param {GetRangeFromPointOptions} options Options to configure how element detection is performed.
+     * @returns {?TextSourceRange|TextSourceElement} A range for the hovered text or element, or `null` if no applicable content was found.
+     */
+    static getRangeFromPoint(x, y, {normalizeCssZoom}) {
+        const selector = '.kix-canvas-tile-content svg>g>rect';
+        const styleNode = this._getStyleNode(selector);
+        styleNode.disabled = false;
+        const elements = document.elementsFromPoint(x, y);
+        styleNode.disabled = true;
+        for (const element of elements) {
+            if (!element.matches(selector)) { continue; }
+            const ariaLabel = element.getAttribute('aria-label');
+            if (typeof ariaLabel !== 'string' || ariaLabel.length === 0) { continue; }
+            return this._createRange(element, ariaLabel, x, y, normalizeCssZoom);
+        }
+        return null;
+    }
+
+    static _getStyleNode(selector) {
+        // This <style> node is necessary to force the SVG <rect> elements to have a fill,
+        // which allows them to be included in document.elementsFromPoint's return value.
+        if (this._styleNode === null) {
+            const style = document.createElement('style');
+            style.textContent = `${selector}{fill:#0000 !important;}`;
+            const parent = document.head || document.documentElement;
+            if (parent !== null) {
+                parent.appendChild(style);
+            }
+            this._styleNode = style;
+        }
+        return this._styleNode;
+    }
+
+    static _createRange(element, text, x, y, normalizeCssZoom) {
+        // Create imposter
+        const content = document.createTextNode(text);
+        const svgText = document.createElementNS('http://www.w3.org/2000/svg', 'text');
+        const transform = element.getAttribute('transform') || '';
+        svgText.setAttribute('x', element.getAttribute('x'));
+        svgText.setAttribute('y', element.getAttribute('y'));
+        svgText.appendChild(content);
+        const textStyle = svgText.style;
+        this._setImportantStyle(textStyle, 'all', 'initial');
+        this._setImportantStyle(textStyle, 'transform', transform);
+        this._setImportantStyle(textStyle, 'font', element.dataset.fontCss);
+        this._setImportantStyle(textStyle, 'text-anchor', 'start');
+        element.parentNode.appendChild(svgText);
+
+        // Adjust offset
+        const elementRect = element.getBoundingClientRect();
+        const textRect = svgText.getBoundingClientRect();
+        const yOffset = ((elementRect.top - textRect.top) + (elementRect.bottom - textRect.bottom)) * 0.5;
+        this._setImportantStyle(textStyle, 'transform', `translate(0px,${yOffset}px) ${transform}`);
+
+        // Create range
+        const range = this._getRangeWithPoint(content, x, y, normalizeCssZoom);
+        this._setImportantStyle(textStyle, 'pointer-events', 'none');
+        this._setImportantStyle(textStyle, 'opacity', '0');
+        return new TextSourceRange(range, '', svgText, element);
+    }
+
+    static _getRangeWithPoint(textNode, x, y, normalizeCssZoom) {
+        if (normalizeCssZoom) {
+            const scale = DocumentUtil.computeZoomScale(textNode);
+            x /= scale;
+            y /= scale;
+        }
+        const range = document.createRange();
+        let start = 0;
+        let end = textNode.nodeValue.length;
+        while (end - start > 1) {
+            const mid = Math.floor((start + end) / 2);
+            range.setStart(textNode, mid);
+            range.setEnd(textNode, end);
+            if (DocumentUtil.isPointInAnyRect(x, y, range.getClientRects())) {
+                start = mid;
+            } else {
+                end = mid;
+            }
+        }
+        range.setStart(textNode, start);
+        range.setEnd(textNode, end);
+        return range;
+    }
+
+    static _setImportantStyle(style, propertyName, value) {
+        style.setProperty(propertyName, value, 'important');
+    }
+}
+// eslint-disable-next-line no-underscore-dangle
+GoogleDocsUtil._styleNode = null;

ext/js/accessibility/google-docs.js

@@ -45,35 +45,10 @@
 
     if (!options.accessibility.forceGoogleDocsHtmlRendering) { return; }
 
+    // The extension ID below is on an allow-list that is used on the Google Docs webpage.
     /* eslint-disable */
     const inject = () => {
-        const start = Date.now();
+        window._docs_annotate_canvas_by_ext = 'ogmnaimimemjmbakcfefmnahgdfhfami';
-        const maxDuration = 10000;
-        const updateDocData = () => {
-            const target = window._docs_flag_initialData;
-            if (typeof target === 'object' && target !== null) {
-                try {
-                    target['kix-awcp'] = true;
-                } catch (e) {
-                    // NOP
-                }
-            } else if (Date.now() - start < maxDuration) {
-                setTimeout(updateDocData, 0);
-            }
-        };
-        const params = new URLSearchParams(location.search);
-        if (params.get('mode') !== 'html') {
-            const url = new URL(location.href);
-            params.set('mode', 'html');
-            url.search = params.toString();
-            try {
-                history.replaceState(history.state, '', url.toString());
-            } catch (e) {
-                // Ignore
-            }
-        }
-        window._docs_force_html_by_ext = true;
-        updateDocData();
     };
     /* eslint-enable */
 

ext/js/app/frontend.js

@@ -17,6 +17,7 @@
 
 /* global
  * DocumentUtil
+ * GoogleDocsUtil
  * TextScanner
  * TextSourceRange
  */
@@ -164,6 +165,7 @@ class Frontend {
             ['Frontend.getPageInfo',      {async: false, handler: this._onApiGetPageInfo.bind(this)}]
         ]);
 
+        this._prepareSiteSpecific();
         this._updateContentScale();
         this._signalFrontendReady();
     }
@@ -770,4 +772,21 @@ class Frontend {
         }
         return null;
     }
+
+    _prepareSiteSpecific() {
+        switch (location.hostname.toLowerCase()) {
+            case 'docs.google.com':
+                this._prepareGoogleDocs();
+                break;
+        }
+    }
+
+    async _prepareGoogleDocs() {
+        if (typeof GoogleDocsUtil !== 'undefined') { return; }
+        await yomichan.api.loadExtensionScripts([
+            '/js/accessibility/google-docs-util.js'
+        ]);
+        if (typeof GoogleDocsUtil === 'undefined') { return; }
+        DocumentUtil.registerGetRangeFromPointHandler(GoogleDocsUtil.getRangeFromPoint.bind(GoogleDocsUtil));
+    }
 }

ext/settings.html

@@ -2067,7 +2067,7 @@
             <div class="settings-item-inner">
                 <div class="settings-item-left">
                     <div class="settings-item-label">
-                        Force HTML-based rendering for Google Docs
+                        Enable Google Docs compatibility mode
                         <a tabindex="0" class="more-toggle more-only danger-text" data-parent-distance="4">(?)</a>
                     </div>
                 </div>
@@ -2077,14 +2077,16 @@
             </div>
             <div class="settings-item-children more" hidden>
                 <p>
-                    Google Docs is moving from HTML-based rendering to
+                    Google Docs now uses
                     <a href="https://developer.mozilla.org/en-US/docs/Web/HTML/Element/canvas" target="_blank" rel="noopener noreferrer">canvas-based</a>
-                    rendering to display content<sup><a href="https://workspaceupdates.googleblog.com/2021/05/Google-Docs-Canvas-Based-Rendering-Update.html" target="_blank" rel="noopener noreferrer">[2]</a></sup>,
+                    rendering to display content<sup><a href="https://workspaceupdates.googleblog.com/2021/05/Google-Docs-Canvas-Based-Rendering-Update.html" target="_blank" rel="noopener noreferrer">[2]</a></sup>
-                    which prevents Yomichan from being able to scan text.
+                    which prevents Yomichan from being able to scan text using the standard methods.
-                    Enabling this option will force HTML-based rendering to be used.
+                    Enabling this option will force Google Docs webpages to expose some additional text
+                    information which should allow Yomichan to still work.
                 </p>
                 <p class="danger-text">
-                    This is a workaround and it is likely that Google will unfortunately remove support for this workaround in the future.
+                    Google has changed this compatibility implementation several times, and the changes do not seem to be announced or documented.
+                    Therefore, it is possible that this feature could stop working at any time the future without warning.
                 </p>
                 <p>
                     <a tabindex="0" class="more-toggle" data-parent-distance="3">Less&hellip;</a>