Fix fetch requests (#708)

* Revert audio fetching functionality to use XMLHttpRequest

* Replace requestJson

* Replace requestJson

* Replace requestJson

* Replace requestJson and requestText

* Fix tests

* Include support for vulgar word searches

* Remove request.js

ext/bg/background.html

@@ -40,7 +40,6 @@
         <script src="/bg/js/media-utility.js"></script>
         <script src="/bg/js/options.js"></script>
         <script src="/bg/js/profile-conditions.js"></script>
-        <script src="/bg/js/request.js"></script>
         <script src="/bg/js/template-renderer.js"></script>
         <script src="/bg/js/text-source-map.js"></script>
         <script src="/bg/js/translator.js"></script>

ext/bg/js/anki.js

@@ -15,10 +15,6 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * requestJson
- */
-
 class AnkiConnect {
     constructor(server) {
         this._enabled = false;
@@ -110,7 +106,16 @@ class AnkiConnect {
     }
 
     async _invoke(action, params) {
-        const result = await requestJson(this._server, 'POST', {action, params, version: this._localVersion}, true);
+        const response = await fetch(this._server, {
+            method: 'POST',
+            mode: 'cors',
+            cache: 'default',
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer',
+            body: JSON.stringify({action, params, version: this._localVersion})
+        });
+        const result = await response.json();
         if (isObject(result)) {
             const error = result.error;
             if (typeof error !== 'undefined') {

ext/bg/js/audio-uri-builder.js

@@ -82,21 +82,14 @@ class AudioUriBuilder {
     }
 
     async _getUriJpod101Alternate(definition) {
-        const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post';
+        const responseText = await new Promise((resolve, reject) => {
-        const data = `post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}`;
+            const xhr = new XMLHttpRequest();
-        const response = await fetch(fetchUrl, {
+            xhr.open('POST', 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post');
-            method: 'POST',
+            xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
-            mode: 'no-cors',
+            xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
-            cache: 'default',
+            xhr.addEventListener('load', () => resolve(xhr.responseText));
-            credentials: 'omit',
+            xhr.send(`post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`);
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
-            headers: {
-                'Content-Type': 'application/x-www-form-urlencoded'
-            },
-            body: data
         });
-        const responseText = await response.text();
 
         const dom = new DOMParser().parseFromString(responseText, 'text/html');
         for (const row of dom.getElementsByClassName('dc-result-row')) {
@@ -115,16 +108,13 @@ class AudioUriBuilder {
     }
 
     async _getUriJisho(definition) {
-        const fetchUrl = `https://jisho.org/search/${definition.expression}`;
+        const responseText = await new Promise((resolve, reject) => {
-        const response = await fetch(fetchUrl, {
+            const xhr = new XMLHttpRequest();
-            method: 'GET',
+            xhr.open('GET', `https://jisho.org/search/${definition.expression}`);
-            mode: 'no-cors',
+            xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
-            cache: 'default',
+            xhr.addEventListener('load', () => resolve(xhr.responseText));
-            credentials: 'omit',
+            xhr.send();
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer'
         });
-        const responseText = await response.text();
 
         const dom = new DOMParser().parseFromString(responseText, 'text/html');
         try {

ext/bg/js/backend.js

@@ -35,8 +35,6 @@
  * jp
  * profileConditionsDescriptor
  * profileConditionsDescriptorPromise
- * requestJson
- * requestText
  */
 
 class Backend {
@@ -199,8 +197,8 @@ class Backend {
 
             await profileConditionsDescriptorPromise;
 
-            this._optionsSchema = await requestJson(chrome.runtime.getURL('/bg/data/options-schema.json'), 'GET');
+            this._optionsSchema = await this._fetchAsset('/bg/data/options-schema.json', true);
-            this._defaultAnkiFieldTemplates = (await requestText(chrome.runtime.getURL('/bg/data/default-anki-field-templates.handlebars'), 'GET')).trim();
+            this._defaultAnkiFieldTemplates = (await this._fetchAsset('/bg/data/default-anki-field-templates.handlebars')).trim();
             this._options = await OptionsUtil.load();
             this._options = JsonSchema.getValidValueOrDefault(this._optionsSchema, this._options);
 
@@ -615,7 +613,7 @@ class Backend {
         if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) {
             throw new Error('Invalid URL');
         }
-        return await requestText(url, 'GET');
+        return await this._fetchAsset(url);
     }
 
     _onApiGetEnvironmentInfo() {
@@ -653,13 +651,11 @@ class Backend {
     }
 
     async _onApiGetDisplayTemplatesHtml() {
-        const url = chrome.runtime.getURL('/mixed/display-templates.html');
+        return await this._fetchAsset('/mixed/display-templates.html');
-        return await requestText(url, 'GET');
     }
 
     async _onApiGetQueryParserTemplatesHtml() {
-        const url = chrome.runtime.getURL('/bg/query-parser-templates.html');
+        return await this._fetchAsset('/bg/query-parser-templates.html');
-        return await requestText(url, 'GET');
     }
 
     _onApiGetZoom(params, sender) {
@@ -1522,4 +1518,19 @@ class Backend {
             }
         });
     }
+
+    async _fetchAsset(url, json=false) {
+        const response = await fetch(chrome.runtime.getURL(url), {
+            method: 'GET',
+            mode: 'no-cors',
+            cache: 'default',
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer'
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status}`);
+        }
+        return await (json ? response.json() : response.text());
+    }
 }

ext/bg/js/dictionary-importer.js

@@ -19,7 +19,6 @@
  * JSZip
  * JsonSchema
  * mediaUtility
- * requestJson
  */
 
 class DictionaryImporter {
@@ -235,7 +234,7 @@ class DictionaryImporter {
             return schemaPromise;
         }
 
-        schemaPromise = requestJson(chrome.runtime.getURL(fileName), 'GET');
+        schemaPromise = this._fetchJsonAsset(fileName);
         this._schemas.set(fileName, schemaPromise);
         return schemaPromise;
     }
@@ -365,4 +364,19 @@ class DictionaryImporter {
 
         return newData;
     }
+
+    async _fetchJsonAsset(url) {
+        const response = await fetch(chrome.runtime.getURL(url), {
+            method: 'GET',
+            mode: 'no-cors',
+            cache: 'default',
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer'
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status}`);
+        }
+        return await response.json();
+    }
 }

ext/bg/js/request.js

@@ -1,43 +0,0 @@
-/*
- * Copyright (C) 2017-2020  Yomichan Authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-
-async function requestText(url, method, data, cors=false) {
-    const response = await fetch(url, {
-        method,
-        mode: (cors ? 'cors' : 'no-cors'),
-        cache: 'default',
-        credentials: 'omit',
-        redirect: 'follow',
-        referrerPolicy: 'no-referrer',
-        body: (data ? JSON.stringify(data) : void 0)
-    });
-    return await response.text();
-}
-
-async function requestJson(url, method, data, cors=false) {
-    const response = await fetch(url, {
-        method,
-        mode: (cors ? 'cors' : 'no-cors'),
-        cache: 'default',
-        credentials: 'omit',
-        redirect: 'follow',
-        referrerPolicy: 'no-referrer',
-        body: (data ? JSON.stringify(data) : void 0)
-    });
-    return await response.json();
-}

ext/bg/js/translator.js

@@ -29,7 +29,6 @@
  * dictTermsSort
  * dictTermsUndupe
  * jp
- * requestJson
  */
 
 class Translator {
@@ -40,8 +39,7 @@ class Translator {
     }
 
     async prepare() {
-        const url = chrome.runtime.getURL('/bg/lang/deinflect.json');
+        const reasons = await this._fetchJsonAsset('/bg/lang/deinflect.json');
-        const reasons = await requestJson(url, 'GET');
         this.deinflector = new Deinflector(reasons);
     }
 
@@ -657,4 +655,19 @@ class Translator {
 
         return text;
     }
+
+    async _fetchJsonAsset(url) {
+        const response = await fetch(chrome.runtime.getURL(url), {
+            method: 'GET',
+            mode: 'no-cors',
+            cache: 'default',
+            credentials: 'omit',
+            redirect: 'follow',
+            referrerPolicy: 'no-referrer'
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${url}: ${response.status}`);
+        }
+        return await response.json();
+    }
 }

ext/mixed/js/audio-system.js

@@ -169,22 +169,22 @@ class AudioSystem {
         });
     }
 
-    async _createAudioBinaryFromUrl(url) {
+    _createAudioBinaryFromUrl(url) {
-        const response = await fetch(url, {
+        return new Promise((resolve, reject) => {
-            method: 'GET',
+            const xhr = new XMLHttpRequest();
-            mode: 'no-cors',
+            xhr.responseType = 'arraybuffer';
-            cache: 'default',
+            xhr.addEventListener('load', async () => {
-            credentials: 'omit',
+                const arrayBuffer = xhr.response;
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer'
-        });
-        const arrayBuffer = await response.arrayBuffer();
-
                 if (!await this._isAudioBinaryValid(arrayBuffer)) {
-            throw new Error('Could not retrieve audio');
+                    reject(new Error('Could not retrieve audio'));
+                } else {
+                    resolve(arrayBuffer);
                 }
-
+            });
-        return arrayBuffer;
+            xhr.addEventListener('error', () => reject(new Error('Failed to connect')));
+            xhr.open('GET', url);
+            xhr.send();
+        });
     }
 
     _isAudioValid(audio) {

test/test-database.js

@@ -30,7 +30,7 @@ const chrome = {
             removeListener() { /* NOP */ }
         },
         getURL(path2) {
-            return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, '')));
+            return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href;
         },
         sendMessage() {
             // NOP
@@ -89,6 +89,9 @@ async function fetch(url2) {
     await Promise.resolve();
     const content = fs.readFileSync(filePath, {encoding: null});
     return {
+        ok: true,
+        status: 200,
+        statusText: 'OK',
         text: async () => Promise.resolve(content.toString('utf8')),
         json: async () => Promise.resolve(JSON.parse(content.toString('utf8')))
     };
@@ -113,7 +116,6 @@ vm.execute([
     'bg/js/json-schema.js',
     'bg/js/dictionary.js',
     'bg/js/media-utility.js',
-    'bg/js/request.js',
     'bg/js/dictionary-importer.js',
     'bg/js/database.js',
     'bg/js/dictionary-database.js'