alex/yomichan
1
Fork 0
Code Releases Activity

Fix fetch requests (#708)

Browse Source 
* Revert audio fetching functionality to use XMLHttpRequest

* Replace requestJson

* Replace requestJson

* Replace requestJson

* Replace requestJson and requestText

* Fix tests

* Include support for vulgar word searches

* Remove request.js
This commit is contained in:
toasted-nutbread 2020-08-02 13:30:55 -04:00 committed by GitHub
parent a562a11498
commit b1b33f8beb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 94 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ext/bg/background.html
View File

@ -40,7 +40,6 @@
<script src="/bg/js/media-utility.js"></script>
<script src="/bg/js/options.js"></script>
<script src="/bg/js/profile-conditions.js"></script>
<script src="/bg/js/request.js"></script>
<script src="/bg/js/template-renderer.js"></script>
<script src="/bg/js/text-source-map.js"></script>
<script src="/bg/js/translator.js"></script>

15
ext/bg/js/anki.js
View File

@ -15,10 +15,6 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/* global
* requestJson
*/
class AnkiConnect {
constructor(server) {
this._enabled = false;
@ -110,7 +106,16 @@ class AnkiConnect {
}
async _invoke(action, params) {
const result = await requestJson(this._server, 'POST', {action, params, version: this._localVersion}, true);
const response = await fetch(this._server, {
method: 'POST',
mode: 'cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: JSON.stringify({action, params, version: this._localVersion})
});
const result = await response.json();
if (isObject(result)) {
const error = result.error;
if (typeof error !== 'undefined') {

36
ext/bg/js/audio-uri-builder.js
View File

@ -82,21 +82,14 @@ class AudioUriBuilder {
}
async _getUriJpod101Alternate(definition) {
const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post';
const data = `post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}`;
const response = await fetch(fetchUrl, {
method: 'POST',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
},
body: data
const responseText = await new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('POST', 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post');
xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
xhr.addEventListener('load', () => resolve(xhr.responseText));
xhr.send(`post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`);
});
const responseText = await response.text();
const dom = new DOMParser().parseFromString(responseText, 'text/html');
for (const row of dom.getElementsByClassName('dc-result-row')) {
@ -115,16 +108,13 @@ class AudioUriBuilder {
}
async _getUriJisho(definition) {
const fetchUrl = `https://jisho.org/search/${definition.expression}`;
const response = await fetch(fetchUrl, {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
const responseText = await new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('GET', `https://jisho.org/search/${definition.expression}`);
xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data')));
xhr.addEventListener('load', () => resolve(xhr.responseText));
xhr.send();
});
const responseText = await response.text();
const dom = new DOMParser().parseFromString(responseText, 'text/html');
try {

29
ext/bg/js/backend.js
View File

@ -35,8 +35,6 @@
* jp
* profileConditionsDescriptor
* profileConditionsDescriptorPromise
* requestJson
* requestText
*/
class Backend {
@ -199,8 +197,8 @@ class Backend {
await profileConditionsDescriptorPromise;
this._optionsSchema = await requestJson(chrome.runtime.getURL('/bg/data/options-schema.json'), 'GET');
this._defaultAnkiFieldTemplates = (await requestText(chrome.runtime.getURL('/bg/data/default-anki-field-templates.handlebars'), 'GET')).trim();
this._optionsSchema = await this._fetchAsset('/bg/data/options-schema.json', true);
this._defaultAnkiFieldTemplates = (await this._fetchAsset('/bg/data/default-anki-field-templates.handlebars')).trim();
this._options = await OptionsUtil.load();
this._options = JsonSchema.getValidValueOrDefault(this._optionsSchema, this._options);
@ -615,7 +613,7 @@ class Backend {
if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) {
throw new Error('Invalid URL');
}
return await requestText(url, 'GET');
return await this._fetchAsset(url);
}
_onApiGetEnvironmentInfo() {
@ -653,13 +651,11 @@ class Backend {
}
async _onApiGetDisplayTemplatesHtml() {
const url = chrome.runtime.getURL('/mixed/display-templates.html');
return await requestText(url, 'GET');
return await this._fetchAsset('/mixed/display-templates.html');
}
async _onApiGetQueryParserTemplatesHtml() {
const url = chrome.runtime.getURL('/bg/query-parser-templates.html');
return await requestText(url, 'GET');
return await this._fetchAsset('/bg/query-parser-templates.html');
}
_onApiGetZoom(params, sender) {
@ -1522,4 +1518,19 @@ class Backend {
}
});
}
async _fetchAsset(url, json=false) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await (json ? response.json() : response.text());
}
}

18
ext/bg/js/dictionary-importer.js
View File

@ -19,7 +19,6 @@
* JSZip
* JsonSchema
* mediaUtility
* requestJson
*/
class DictionaryImporter {
@ -235,7 +234,7 @@ class DictionaryImporter {
return schemaPromise;
}
schemaPromise = requestJson(chrome.runtime.getURL(fileName), 'GET');
schemaPromise = this._fetchJsonAsset(fileName);
this._schemas.set(fileName, schemaPromise);
return schemaPromise;
}
@ -365,4 +364,19 @@ class DictionaryImporter {
return newData;
}
async _fetchJsonAsset(url) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await response.json();
}
}

43
ext/bg/js/request.js
View File

@ -1,43 +0,0 @@
/*
* Copyright (C) 2017-2020 Yomichan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
async function requestText(url, method, data, cors=false) {
const response = await fetch(url, {
method,
mode: (cors ? 'cors' : 'no-cors'),
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: (data ? JSON.stringify(data) : void 0)
});
return await response.text();
}
async function requestJson(url, method, data, cors=false) {
const response = await fetch(url, {
method,
mode: (cors ? 'cors' : 'no-cors'),
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer',
body: (data ? JSON.stringify(data) : void 0)
});
return await response.json();
}

19
ext/bg/js/translator.js
View File

@ -29,7 +29,6 @@
* dictTermsSort
* dictTermsUndupe
* jp
* requestJson
*/
class Translator {
@ -40,8 +39,7 @@ class Translator {
}
async prepare() {
const url = chrome.runtime.getURL('/bg/lang/deinflect.json');
const reasons = await requestJson(url, 'GET');
const reasons = await this._fetchJsonAsset('/bg/lang/deinflect.json');
this.deinflector = new Deinflector(reasons);
}
@ -657,4 +655,19 @@ class Translator {
return text;
}
async _fetchJsonAsset(url) {
const response = await fetch(chrome.runtime.getURL(url), {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return await response.json();
}
}

28
ext/mixed/js/audio-system.js
View File

@ -169,22 +169,22 @@ class AudioSystem {
});
}
async _createAudioBinaryFromUrl(url) {
const response = await fetch(url, {
method: 'GET',
mode: 'no-cors',
cache: 'default',
credentials: 'omit',
redirect: 'follow',
referrerPolicy: 'no-referrer'
});
const arrayBuffer = await response.arrayBuffer();
_createAudioBinaryFromUrl(url) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.responseType = 'arraybuffer';
xhr.addEventListener('load', async () => {
const arrayBuffer = xhr.response;
if (!await this._isAudioBinaryValid(arrayBuffer)) {
throw new Error('Could not retrieve audio');
reject(new Error('Could not retrieve audio'));
} else {
resolve(arrayBuffer);
}
return arrayBuffer;
});
xhr.addEventListener('error', () => reject(new Error('Failed to connect')));
xhr.open('GET', url);
xhr.send();
});
}
_isAudioValid(audio) {

6
test/test-database.js
View File

@ -30,7 +30,7 @@ const chrome = {
removeListener() { /* NOP */ }
},
getURL(path2) {
return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, '')));
return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href;
},
sendMessage() {
// NOP
@ -89,6 +89,9 @@ async function fetch(url2) {
await Promise.resolve();
const content = fs.readFileSync(filePath, {encoding: null});
return {
ok: true,
status: 200,
statusText: 'OK',
text: async () => Promise.resolve(content.toString('utf8')),
json: async () => Promise.resolve(JSON.parse(content.toString('utf8')))
};
@ -113,7 +116,6 @@ vm.execute([
'bg/js/json-schema.js',
'bg/js/dictionary.js',
'bg/js/media-utility.js',
'bg/js/request.js',
'bg/js/dictionary-importer.js',
'bg/js/database.js',
'bg/js/dictionary-database.js'