Fix fetch requests (#708)

* Revert audio fetching functionality to use XMLHttpRequest

* Replace requestJson

* Replace requestJson

* Replace requestJson

* Replace requestJson and requestText

* Fix tests

* Include support for vulgar word searches

* Remove request.js
This commit is contained in:
toasted-nutbread 2020-08-02 13:30:55 -04:00 committed by GitHub
parent a562a11498
commit b1b33f8beb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 94 additions and 103 deletions

View File

@ -40,7 +40,6 @@
<script src="/bg/js/media-utility.js"></script> <script src="/bg/js/media-utility.js"></script>
<script src="/bg/js/options.js"></script> <script src="/bg/js/options.js"></script>
<script src="/bg/js/profile-conditions.js"></script> <script src="/bg/js/profile-conditions.js"></script>
<script src="/bg/js/request.js"></script>
<script src="/bg/js/template-renderer.js"></script> <script src="/bg/js/template-renderer.js"></script>
<script src="/bg/js/text-source-map.js"></script> <script src="/bg/js/text-source-map.js"></script>
<script src="/bg/js/translator.js"></script> <script src="/bg/js/translator.js"></script>

View File

@ -15,10 +15,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
/* global
* requestJson
*/
class AnkiConnect { class AnkiConnect {
constructor(server) { constructor(server) {
this._enabled = false; this._enabled = false;
@ -110,7 +106,16 @@ class AnkiConnect {
} }
async _invoke(action, params) { async _invoke(action, params) {
const result = await requestJson(this._server, 'POST', {action, params, version: this._localVersion}, true); const response = await fetch(this._server, {
method: 'POST',
mode: 'cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: JSON.stringify({action, params, version: this._localVersion})
});
const result = await response.json();
if (isObject(result)) { if (isObject(result)) {
const error = result.error; const error = result.error;
if (typeof error !== 'undefined') { if (typeof error !== 'undefined') {

View File

@ -82,21 +82,14 @@ class AudioUriBuilder {
} }
async _getUriJpod101Alternate(definition) { async _getUriJpod101Alternate(definition) {
const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post'; const responseText = await new Promise((resolve, reject) => {
const data = `post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}`; const xhr = new XMLHttpRequest();
const response = await fetch(fetchUrl, { xhr.open('POST', 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post');
method: 'POST', xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
mode: 'no-cors', xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
cache: 'default', xhr.addEventListener('load', () => resolve(xhr.responseText));
credentials: 'omit', xhr.send(`post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`);
redirect: 'follow',
referrerPolicy: 'no-referrer',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
},
body: data
}); });
const responseText = await response.text();
const dom = new DOMParser().parseFromString(responseText, 'text/html'); const dom = new DOMParser().parseFromString(responseText, 'text/html');
for (const row of dom.getElementsByClassName('dc-result-row')) { for (const row of dom.getElementsByClassName('dc-result-row')) {
@ -115,16 +108,13 @@ class AudioUriBuilder {
} }
async _getUriJisho(definition) { async _getUriJisho(definition) {
const fetchUrl = `https://jisho.org/search/${definition.expression}`; const responseText = await new Promise((resolve, reject) => {
const response = await fetch(fetchUrl, { const xhr = new XMLHttpRequest();
method: 'GET', xhr.open('GET', `https://jisho.org/search/${definition.expression}`);
mode: 'no-cors', xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
cache: 'default', xhr.addEventListener('load', () => resolve(xhr.responseText));
credentials: 'omit', xhr.send();
redirect: 'follow',
referrerPolicy: 'no-referrer'
}); });
const responseText = await response.text();
const dom = new DOMParser().parseFromString(responseText, 'text/html'); const dom = new DOMParser().parseFromString(responseText, 'text/html');
try { try {

View File

@ -35,8 +35,6 @@
* jp * jp
* profileConditionsDescriptor * profileConditionsDescriptor
* profileConditionsDescriptorPromise * profileConditionsDescriptorPromise
* requestJson
* requestText
*/ */
class Backend { class Backend {
@ -199,8 +197,8 @@ class Backend {
await profileConditionsDescriptorPromise; await profileConditionsDescriptorPromise;
this._optionsSchema = await requestJson(chrome.runtime.getURL('/bg/data/options-schema.json'), 'GET'); this._optionsSchema = await this._fetchAsset('/bg/data/options-schema.json', true);
this._defaultAnkiFieldTemplates = (await requestText(chrome.runtime.getURL('/bg/data/default-anki-field-templates.handlebars'), 'GET')).trim(); this._defaultAnkiFieldTemplates = (await this._fetchAsset('/bg/data/default-anki-field-templates.handlebars')).trim();
this._options = await OptionsUtil.load(); this._options = await OptionsUtil.load();
this._options = JsonSchema.getValidValueOrDefault(this._optionsSchema, this._options); this._options = JsonSchema.getValidValueOrDefault(this._optionsSchema, this._options);
@ -615,7 +613,7 @@ class Backend {
if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) { if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) {
throw new Error('Invalid URL'); throw new Error('Invalid URL');
} }
return await requestText(url, 'GET'); return await this._fetchAsset(url);
} }
_onApiGetEnvironmentInfo() { _onApiGetEnvironmentInfo() {
@ -653,13 +651,11 @@ class Backend {
} }
async _onApiGetDisplayTemplatesHtml() { async _onApiGetDisplayTemplatesHtml() {
const url = chrome.runtime.getURL('/mixed/display-templates.html'); return await this._fetchAsset('/mixed/display-templates.html');
return await requestText(url, 'GET');
} }
async _onApiGetQueryParserTemplatesHtml() { async _onApiGetQueryParserTemplatesHtml() {
const url = chrome.runtime.getURL('/bg/query-parser-templates.html'); return await this._fetchAsset('/bg/query-parser-templates.html');
return await requestText(url, 'GET');
} }
_onApiGetZoom(params, sender) { _onApiGetZoom(params, sender) {
@ -1522,4 +1518,19 @@ class Backend {
} }
}); });
} }
async _fetchAsset(url, json=false) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await (json ? response.json() : response.text());
}
} }

View File

@ -19,7 +19,6 @@
* JSZip * JSZip
* JsonSchema * JsonSchema
* mediaUtility * mediaUtility
* requestJson
*/ */
class DictionaryImporter { class DictionaryImporter {
@ -235,7 +234,7 @@ class DictionaryImporter {
return schemaPromise; return schemaPromise;
} }
schemaPromise = requestJson(chrome.runtime.getURL(fileName), 'GET'); schemaPromise = this._fetchJsonAsset(fileName);
this._schemas.set(fileName, schemaPromise); this._schemas.set(fileName, schemaPromise);
return schemaPromise; return schemaPromise;
} }
@ -365,4 +364,19 @@ class DictionaryImporter {
return newData; return newData;
} }
async _fetchJsonAsset(url) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await response.json();
}
} }

View File

@ -1,43 +0,0 @@
/*
* Copyright (C) 2017-2020 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
async function requestText(url, method, data, cors=false) {
const response = await fetch(url, {
method,
mode: (cors ? 'cors' : 'no-cors'),
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: (data ? JSON.stringify(data) : void 0)
});
return await response.text();
}
async function requestJson(url, method, data, cors=false) {
const response = await fetch(url, {
method,
mode: (cors ? 'cors' : 'no-cors'),
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: (data ? JSON.stringify(data) : void 0)
});
return await response.json();
}

View File

@ -29,7 +29,6 @@
* dictTermsSort * dictTermsSort
* dictTermsUndupe * dictTermsUndupe
* jp * jp
* requestJson
*/ */
class Translator { class Translator {
@ -40,8 +39,7 @@ class Translator {
} }
async prepare() { async prepare() {
const url = chrome.runtime.getURL('/bg/lang/deinflect.json'); const reasons = await this._fetchJsonAsset('/bg/lang/deinflect.json');
const reasons = await requestJson(url, 'GET');
this.deinflector = new Deinflector(reasons); this.deinflector = new Deinflector(reasons);
} }
@ -657,4 +655,19 @@ class Translator {
return text; return text;
} }
async _fetchJsonAsset(url) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await response.json();
}
} }

View File

@ -169,22 +169,22 @@ class AudioSystem {
}); });
} }
async _createAudioBinaryFromUrl(url) { _createAudioBinaryFromUrl(url) {
const response = await fetch(url, { return new Promise((resolve, reject) => {
method: 'GET', const xhr = new XMLHttpRequest();
mode: 'no-cors', xhr.responseType = 'arraybuffer';
cache: 'default', xhr.addEventListener('load', async () => {
credentials: 'omit', const arrayBuffer = xhr.response;
redirect: 'follow', if (!await this._isAudioBinaryValid(arrayBuffer)) {
referrerPolicy: 'no-referrer' reject(new Error('Could not retrieve audio'));
} else {
resolve(arrayBuffer);
}
});
xhr.addEventListener('error', () => reject(new Error('Failed to connect')));
xhr.open('GET', url);
xhr.send();
}); });
const arrayBuffer = await response.arrayBuffer();
if (!await this._isAudioBinaryValid(arrayBuffer)) {
throw new Error('Could not retrieve audio');
}
return arrayBuffer;
} }
_isAudioValid(audio) { _isAudioValid(audio) {

View File

@ -30,7 +30,7 @@ const chrome = {
removeListener() { /* NOP */ } removeListener() { /* NOP */ }
}, },
getURL(path2) { getURL(path2) {
return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))); return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href;
}, },
sendMessage() { sendMessage() {
// NOP // NOP
@ -89,6 +89,9 @@ async function fetch(url2) {
await Promise.resolve(); await Promise.resolve();
const content = fs.readFileSync(filePath, {encoding: null}); const content = fs.readFileSync(filePath, {encoding: null});
return { return {
ok: true,
status: 200,
statusText: 'OK',
text: async () => Promise.resolve(content.toString('utf8')), text: async () => Promise.resolve(content.toString('utf8')),
json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) json: async () => Promise.resolve(JSON.parse(content.toString('utf8')))
}; };
@ -113,7 +116,6 @@ vm.execute([
'bg/js/json-schema.js', 'bg/js/json-schema.js',
'bg/js/dictionary.js', 'bg/js/dictionary.js',
'bg/js/media-utility.js', 'bg/js/media-utility.js',
'bg/js/request.js',
'bg/js/dictionary-importer.js', 'bg/js/dictionary-importer.js',
'bg/js/database.js', 'bg/js/database.js',
'bg/js/dictionary-database.js' 'bg/js/dictionary-database.js'