Strip request origin (#710)

* Add web request permissions

* Create fetch wrapper that anonymizes the request

* Fix Firefox not supporting 'extraHeaders' option

ext/bg/background.html

@@ -40,6 +40,7 @@
         <script src="/bg/js/media-utility.js"></script>
         <script src="/bg/js/options.js"></script>
         <script src="/bg/js/profile-conditions.js"></script>
+        <script src="/bg/js/request-builder.js"></script>
         <script src="/bg/js/template-renderer.js"></script>
         <script src="/bg/js/text-source-map.js"></script>
         <script src="/bg/js/translator.js"></script>

ext/bg/js/audio-uri-builder.js

@@ -20,7 +20,8 @@
  */
 
 class AudioUriBuilder {
-    constructor() {
+    constructor({requestBuilder}) {
+        this._requestBuilder = requestBuilder;
         this._getUrlHandlers = new Map([
             ['jpod101', this._getUriJpod101.bind(this)],
             ['jpod101-alternate', this._getUriJpod101Alternate.bind(this)],
@@ -82,14 +83,21 @@ class AudioUriBuilder {
     }
 
     async _getUriJpod101Alternate(definition) {
-        const responseText = await new Promise((resolve, reject) => {
+        const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post';
-            const xhr = new XMLHttpRequest();
+        const data = `post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`;
-            xhr.open('POST', 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post');
+        const response = await this._requestBuilder.fetchAnonymous(fetchUrl, {
-            xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
+            method: 'POST',
-            xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
+            mode: 'cors',
-            xhr.addEventListener('load', () => resolve(xhr.responseText));
+            cache: 'default',
-            xhr.send(`post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`);
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded'
+            },
+            body: data
         });
+        const responseText = await response.text();
 
         const dom = new DOMParser().parseFromString(responseText, 'text/html');
         for (const row of dom.getElementsByClassName('dc-result-row')) {
@@ -108,13 +116,16 @@ class AudioUriBuilder {
     }
 
     async _getUriJisho(definition) {
-        const responseText = await new Promise((resolve, reject) => {
+        const fetchUrl = `https://jisho.org/search/${definition.expression}`;
-            const xhr = new XMLHttpRequest();
+        const response = await this._requestBuilder.fetchAnonymous(fetchUrl, {
-            xhr.open('GET', `https://jisho.org/search/${definition.expression}`);
+            method: 'GET',
-            xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
+            mode: 'cors',
-            xhr.addEventListener('load', () => resolve(xhr.responseText));
+            cache: 'default',
-            xhr.send();
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer'
         });
+        const responseText = await response.text();
 
         const dom = new DOMParser().parseFromString(responseText, 'text/html');
         try {

ext/bg/js/backend.js

@@ -28,6 +28,7 @@
  * Mecab
  * ObjectPropertyAccessor
  * OptionsUtil
+ * RequestBuilder
  * TemplateRenderer
  * Translator
  * conditionsTestValue
@@ -49,9 +50,13 @@ class Backend {
         this._options = null;
         this._optionsSchema = null;
         this._defaultAnkiFieldTemplates = null;
-        this._audioUriBuilder = new AudioUriBuilder();
+        this._requestBuilder = new RequestBuilder();
+        this._audioUriBuilder = new AudioUriBuilder({
+            requestBuilder: this._requestBuilder
+        });
         this._audioSystem = new AudioSystem({
             audioUriBuilder: this._audioUriBuilder,
+            requestBuilder: this._requestBuilder,
             useCache: false
         });
         this._ankiNoteBuilder = new AnkiNoteBuilder({

ext/bg/js/request-builder.js

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+class RequestBuilder {
+    constructor() {
+        this._extraHeadersSupported = null;
+        this._onBeforeSendHeadersExtraInfoSpec = ['blocking', 'requestHeaders', 'extraHeaders'];
+    }
+
+    async fetchAnonymous(url, init) {
+        const originURL = this._getOriginURL(url);
+        const modifications = [
+            ['cookie', null],
+            ['origin', {name: 'Origin', value: originURL}]
+        ];
+        return this.fetchModifyHeaders(url, init, modifications);
+    }
+
+    async fetchModifyHeaders(url, init, modifications) {
+        const matchURL = this._getMatchURL(url);
+
+        let done = false;
+        const callback = (details) => {
+            if (done || details.url !== url) { return {}; }
+            done = true;
+
+            const requestHeaders = details.requestHeaders;
+            this._modifyHeaders(requestHeaders, modifications);
+            return {requestHeaders};
+        };
+        const filter = {
+            urls: [matchURL],
+            types: ['xmlhttprequest']
+        };
+
+        let needsCleanup = false;
+        try {
+            this._onBeforeSendHeadersAddListener(callback, filter);
+            needsCleanup = true;
+        } catch (e) {
+            // NOP
+        }
+
+        try {
+            return await fetch(url, init);
+        } finally {
+            if (needsCleanup) {
+                try {
+                    chrome.webRequest.onBeforeSendHeaders.removeListener(callback);
+                } catch (e) {
+                    // NOP
+                }
+            }
+        }
+    }
+
+    // Private
+
+    _onBeforeSendHeadersAddListener(callback, filter) {
+        const extraInfoSpec = this._onBeforeSendHeadersExtraInfoSpec;
+        for (let i = 0; i < 2; ++i) {
+            try {
+                chrome.webRequest.onBeforeSendHeaders.addListener(callback, filter, extraInfoSpec);
+                if (this._extraHeadersSupported === null) {
+                    this._extraHeadersSupported = true;
+                }
+                break;
+            } catch (e) {
+                // Firefox doesn't support the 'extraHeaders' option and will throw the following error:
+                // Type error for parameter extraInfoSpec (Error processing 2: Invalid enumeration value "extraHeaders") for webRequest.onBeforeSendHeaders.
+                if (this._extraHeadersSupported !== null || !`${e.message}`.includes('extraHeaders')) {
+                    throw e;
+                }
+            }
+
+            // addListener failed; remove 'extraHeaders' from extraInfoSpec.
+            this._extraHeadersSupported = false;
+            const index = extraInfoSpec.indexOf('extraHeaders');
+            if (index >= 0) { extraInfoSpec.splice(index, 1); }
+        }
+    }
+
+    _getMatchURL(url) {
+        const url2 = new URL(url);
+        return `${url2.protocol}//${url2.host}${url2.pathname}`;
+    }
+
+    _getOriginURL(url) {
+        const url2 = new URL(url);
+        return `${url2.protocol}//${url2.host}`;
+    }
+
+    _modifyHeaders(headers, modifications) {
+        modifications = new Map(modifications);
+
+        for (let i = 0, ii = headers.length; i < ii; ++i) {
+            const header = headers[i];
+            const name = header.name.toLowerCase();
+            const modification = modifications.get(name);
+            if (typeof modification === 'undefined') { continue; }
+
+            modifications.delete(name);
+
+            if (modification === null) {
+                headers.splice(i, 1);
+                --i;
+                --ii;
+            } else {
+                headers[i] = modification;
+            }
+        }
+
+        for (const header of modifications.values()) {
+            if (header !== null) {
+                headers.push(header);
+            }
+        }
+    }
+}

ext/manifest.json

@@ -67,7 +67,9 @@
         "storage",
         "clipboardWrite",
         "unlimitedStorage",
-        "nativeMessaging"
+        "nativeMessaging",
+        "webRequest",
+        "webRequestBlocking"
     ],
     "optional_permissions": [
         "clipboardRead"

ext/mixed/js/audio-system.js

@@ -66,10 +66,11 @@ class TextToSpeechAudio {
 }
 
 class AudioSystem {
-    constructor({audioUriBuilder, useCache}) {
+    constructor({audioUriBuilder, requestBuilder=null, useCache}) {
         this._cache = useCache ? new Map() : null;
         this._cacheSizeMaximum = 32;
         this._audioUriBuilder = audioUriBuilder;
+        this._requestBuilder = requestBuilder;
 
         if (typeof speechSynthesis !== 'undefined') {
             // speechSynthesis.getVoices() will not be populated unless some API call is made.
@@ -169,22 +170,22 @@ class AudioSystem {
         });
     }
 
-    _createAudioBinaryFromUrl(url) {
+    async _createAudioBinaryFromUrl(url) {
-        return new Promise((resolve, reject) => {
+        const response = await this._requestBuilder.fetchAnonymous(url, {
-            const xhr = new XMLHttpRequest();
+            method: 'GET',
-            xhr.responseType = 'arraybuffer';
+            mode: 'cors',
-            xhr.addEventListener('load', async () => {
+            cache: 'default',
-                const arrayBuffer = xhr.response;
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer'
+        });
+        const arrayBuffer = await response.arrayBuffer();
+
         if (!await this._isAudioBinaryValid(arrayBuffer)) {
-                    reject(new Error('Could not retrieve audio'));
+            throw new Error('Could not retrieve audio');
-                } else {
-                    resolve(arrayBuffer);
         }
-            });
+
-            xhr.addEventListener('error', () => reject(new Error('Failed to connect')));
+        return arrayBuffer;
-            xhr.open('GET', url);
-            xhr.send();
-        });
     }
 
     _isAudioValid(audio) {