From f63e8e4be0e7bdb1a2e45e349bf667ea7ca4adab Mon Sep 17 00:00:00 2001 From: siikamiika Date: Tue, 29 Oct 2019 23:49:36 +0200 Subject: [PATCH 01/36] add simple query parser --- ext/bg/js/search-query-parser.js | 44 ++++++++++++++++++++++++++++++++ ext/bg/js/search.js | 24 +++++++++++------ ext/bg/search.html | 3 +++ ext/mixed/css/display.css | 9 +++++++ ext/mixed/js/display.js | 2 ++ 5 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 ext/bg/js/search-query-parser.js diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js new file mode 100644 index 00000000..debe53b4 --- /dev/null +++ b/ext/bg/js/search-query-parser.js @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2019 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +class QueryParser { + constructor(search) { + this.search = search; + + this.queryParser = document.querySelector('#query-parser'); + + // TODO also enable for mouseover scanning + this.queryParser.addEventListener('click', (e) => this.onTermLookup(e)); + } + + onError(error) { + logError(error, false); + } + + async onTermLookup(e) { + const {textSource} = await this.search.onTermLookup(e, {isQueryParser: true}); + if (textSource) { + textSource.select(); + } + } + + setText(text) { + this.queryParser.innerText = text; + } +} diff --git a/ext/bg/js/search.js b/ext/bg/js/search.js index 56316217..20d0c58c 100644 --- a/ext/bg/js/search.js +++ b/ext/bg/js/search.js @@ -32,6 +32,8 @@ class DisplaySearch extends Display { url: window.location.href }; + this.queryParser = new QueryParser(this); + this.search = document.querySelector('#search'); this.query = document.querySelector('#query'); this.intro = document.querySelector('#intro'); @@ -72,11 +74,11 @@ class DisplaySearch extends Display { const query = DisplaySearch.getSearchQueryFromLocation(window.location.href) || ''; if (e.target.checked) { window.wanakana.bind(this.query); - this.query.value = window.wanakana.toKana(query); + this.setQuery(window.wanakana.toKana(query)); apiOptionsSet({general: {enableWanakana: true}}, this.getOptionsContext()); } else { window.wanakana.unbind(this.query); - this.query.value = query; + this.setQuery(query); apiOptionsSet({general: {enableWanakana: false}}, this.getOptionsContext()); } this.onSearchQueryUpdated(this.query.value, false); @@ -86,9 +88,9 @@ class DisplaySearch extends Display { const query = DisplaySearch.getSearchQueryFromLocation(window.location.href); if (query !== null) { if (this.isWanakanaEnabled()) { - this.query.value = window.wanakana.toKana(query); + this.setQuery(window.wanakana.toKana(query)); } else { - this.query.value = query; + this.setQuery(query); } this.onSearchQueryUpdated(this.query.value, false); } @@ -159,6 +161,7 @@ class DisplaySearch extends Display { e.preventDefault(); const query = this.query.value; + this.queryParser.setText(query); const queryString = query.length > 0 ? `?query=${encodeURIComponent(query)}` : ''; window.history.pushState(null, '', `${window.location.pathname}${queryString}`); this.onSearchQueryUpdated(query, true); @@ -168,9 +171,9 @@ class DisplaySearch extends Display { const query = DisplaySearch.getSearchQueryFromLocation(window.location.href) || ''; if (this.query !== null) { if (this.isWanakanaEnabled()) { - this.query.value = window.wanakana.toKana(query); + this.setQuery(window.wanakana.toKana(query)); } else { - this.query.value = query; + this.setQuery(query); } } @@ -258,9 +261,9 @@ class DisplaySearch extends Display { } if (curText && (curText !== this.clipboardPrevText) && jpIsJapaneseText(curText)) { if (this.isWanakanaEnabled()) { - this.query.value = window.wanakana.toKana(curText); + this.setQuery(window.wanakana.toKana(curText)); } else { - this.query.value = curText; + this.setQuery(curText); } const queryString = curText.length > 0 ? `?query=${encodeURIComponent(curText)}` : ''; @@ -287,6 +290,11 @@ class DisplaySearch extends Display { return this.optionsContext; } + setQuery(query) { + this.query.value = query; + this.queryParser.setText(query); + } + setIntroVisible(visible, animate) { if (this.introVisible === visible) { return; diff --git a/ext/bg/search.html b/ext/bg/search.html index 54c5fb6c..48e7dbf5 100644 --- a/ext/bg/search.html +++ b/ext/bg/search.html @@ -47,6 +47,8 @@ +
+
@@ -67,6 +69,7 @@ + diff --git a/ext/mixed/css/display.css b/ext/mixed/css/display.css index 7ee6f5ac..5e5213ff 100644 --- a/ext/mixed/css/display.css +++ b/ext/mixed/css/display.css @@ -88,6 +88,15 @@ ol, ul { user-select: none; } +#query-parser { + margin-top: 10px; + font-size: 24px; +} + +.highlight { + background-color: lightblue; +} + /* * Entries diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js index 8ad3ee1b..07a851f5 100644 --- a/ext/mixed/js/display.js +++ b/ext/mixed/js/display.js @@ -140,6 +140,8 @@ class Display { } this.setContentTerms(definitions, context); + + return {textSource}; } catch (error) { this.onError(error); } From c35a05cd62d43ff435c022a353de55510b020277 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 03:58:24 +0200 Subject: [PATCH 02/36] add kana to text --- ext/bg/js/api.js | 37 ++++++++++++ ext/bg/js/backend.js | 1 + ext/bg/js/search-query-parser.js | 97 +++++++++++++++++++++++++++++--- ext/fg/js/api.js | 4 ++ ext/mixed/css/display.css | 4 ++ ext/mixed/js/display.js | 78 +++++++++++++++++-------- 6 files changed, 188 insertions(+), 33 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index df73aa2a..064903ca 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -79,6 +79,43 @@ async function apiTermsFind(text, details, optionsContext) { return {length, definitions}; } +async function apiTextParse(text, optionsContext) { + const options = await apiOptionsGet(optionsContext); + const translator = utilBackend().translator; + + const results = []; + while (text) { + let [definitions, length] = await translator.findTerms(text, {}, options); + if (definitions.length > 0) { + definitions = dictTermsSort(definitions); + const {expression, source, reading} = definitions[0]; + + let stemLength = source.length; + while (source[stemLength - 1] !== expression[stemLength - 1]) { + --stemLength; + } + const offset = source.length - stemLength; + + for (const result of jpDistributeFurigana( + source.slice(0, offset === 0 ? source.length : source.length - offset), + reading.slice(0, offset === 0 ? reading.length : source.length + (reading.length - expression.length) - offset) + )) { + results.push(result); + } + + if (stemLength !== source.length) { + results.push({text: source.slice(stemLength)}); + } + + text = text.slice(source.length); + } else { + results.push({text: text[0]}); + text = text.slice(1); + } + } + return results; +} + async function apiKanjiFind(text, optionsContext) { const options = await apiOptionsGet(optionsContext); const definitions = await utilBackend().translator.findKanji(text, options); diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index efad153a..d0e404f2 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -180,6 +180,7 @@ Backend.messageHandlers = { optionsSet: ({changedOptions, optionsContext, source}) => apiOptionsSet(changedOptions, optionsContext, source), kanjiFind: ({text, optionsContext}) => apiKanjiFind(text, optionsContext), termsFind: ({text, details, optionsContext}) => apiTermsFind(text, details, optionsContext), + textParse: ({text, optionsContext}) => apiTextParse(text, optionsContext), definitionAdd: ({definition, mode, context, optionsContext}) => apiDefinitionAdd(definition, mode, context, optionsContext), definitionsAddable: ({definitions, modes, optionsContext}) => apiDefinitionsAddable(definitions, modes, optionsContext), noteView: ({noteId}) => apiNoteView(noteId), diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index debe53b4..c3a3900b 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -23,22 +23,103 @@ class QueryParser { this.queryParser = document.querySelector('#query-parser'); - // TODO also enable for mouseover scanning - this.queryParser.addEventListener('click', (e) => this.onTermLookup(e)); + this.queryParser.addEventListener('click', (e) => this.onClick(e)); + this.queryParser.addEventListener('mousemove', (e) => this.onMouseMove(e)); } onError(error) { logError(error, false); } - async onTermLookup(e) { - const {textSource} = await this.search.onTermLookup(e, {isQueryParser: true}); - if (textSource) { - textSource.select(); + onClick(e) { + this.onTermLookup(e, {disableScroll: true, selectText: true}); + } + + async onMouseMove(e) { + if ( + (e.buttons & 0x1) !== 0x0 // Left mouse button + ) { + return; + } + + const scanningOptions = this.search.options.scanning; + const scanningModifier = scanningOptions.modifier; + if (!( + QueryParser.isScanningModifierPressed(scanningModifier, e) || + (scanningOptions.middleMouse && (e.buttons & 0x4) !== 0x0) // Middle mouse button + )) { + return; + } + + await this.onTermLookup(e, {disableScroll: true, selectText: true, disableHistory: true}) + } + + onTermLookup(e, params) { + this.search.onTermLookup(e, params); + } + + async setText(text) { + this.search.setSpinnerVisible(true); + + const results = await apiTextParse(text, this.search.getOptionsContext()); + + const tempContainer = document.createElement('div'); + for (const {text, furigana} of results) { + if (furigana) { + const rubyElement = document.createElement('ruby'); + const furiganaElement = document.createElement('rt'); + furiganaElement.innerText = furigana; + rubyElement.appendChild(document.createTextNode(text)); + rubyElement.appendChild(furiganaElement); + tempContainer.appendChild(rubyElement); + } else { + tempContainer.appendChild(document.createTextNode(text)); + } + } + this.queryParser.innerHTML = ''; + this.queryParser.appendChild(tempContainer); + + this.search.setSpinnerVisible(false); + } + + async parseText(text) { + const results = []; + while (text) { + const {definitions, length} = await apiTermsFind(text, {}, this.search.getOptionsContext()); + if (length) { + results.push(definitions); + text = text.slice(length); + } else { + results.push(text[0]); + text = text.slice(1); + } + } + return results; + } + + popupTimerSet(callback) { + const delay = this.options.scanning.delay; + if (delay > 0) { + this.popupTimer = window.setTimeout(callback, delay); + } else { + Promise.resolve().then(callback); } } - setText(text) { - this.queryParser.innerText = text; + popupTimerClear() { + if (this.popupTimer !== null) { + window.clearTimeout(this.popupTimer); + this.popupTimer = null; + } + } + + static isScanningModifierPressed(scanningModifier, mouseEvent) { + switch (scanningModifier) { + case 'alt': return mouseEvent.altKey; + case 'ctrl': return mouseEvent.ctrlKey; + case 'shift': return mouseEvent.shiftKey; + case 'none': return true; + default: return false; + } } } diff --git a/ext/fg/js/api.js b/ext/fg/js/api.js index 945ba076..cc1e0e90 100644 --- a/ext/fg/js/api.js +++ b/ext/fg/js/api.js @@ -29,6 +29,10 @@ function apiTermsFind(text, details, optionsContext) { return utilInvoke('termsFind', {text, details, optionsContext}); } +function apiTextParse(text, optionsContext) { + return utilInvoke('textParse', {text, optionsContext}); +} + function apiKanjiFind(text, optionsContext) { return utilInvoke('kanjiFind', {text, optionsContext}); } diff --git a/ext/mixed/css/display.css b/ext/mixed/css/display.css index 5e5213ff..81109fc6 100644 --- a/ext/mixed/css/display.css +++ b/ext/mixed/css/display.css @@ -73,6 +73,10 @@ ol, ul { } +html:root[data-yomichan-page=search] body { + min-height: 101vh; /* always show scroll bar to avoid scanning problems */ +} + /* * Search page */ diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js index 07a851f5..82385bf9 100644 --- a/ext/mixed/js/display.js +++ b/ext/mixed/js/display.js @@ -98,17 +98,62 @@ class Display { } } - async onTermLookup(e) { + async onTermLookup(e, {disableScroll, selectText, disableHistory}) { + const termLookupResults = await this.termLookup(e); + if (!termLookupResults) { + return false; + } + + try { + const {textSource, definitions} = termLookupResults; + + const scannedElement = e.target; + const sentence = docSentenceExtract(textSource, this.options.anki.sentenceExt); + + if (!disableScroll) { + this.windowScroll.toY(0); + } + let context; + if (disableHistory) { + const {url, source} = this.context || {}; + context = {sentence, url, source, disableScroll}; + } else { + context = { + disableScroll, + source: { + definitions: this.definitions, + index: this.entryIndexFind(scannedElement), + scroll: this.windowScroll.y + } + }; + + if (this.context) { + context.sentence = sentence; + context.url = this.context.url; + context.source.source = this.context.source; + } + } + + this.setContentTerms(definitions, context); + + if (selectText) { + textSource.select(); + } + } catch (error) { + this.onError(error); + } + } + + async termLookup(e) { try { e.preventDefault(); - const clickedElement = e.target; const textSource = docRangeFromPoint(e.clientX, e.clientY, this.options); if (textSource === null) { return false; } - let definitions, length, sentence; + let definitions, length; try { textSource.setEndOffset(this.options.scanning.length); @@ -118,30 +163,11 @@ class Display { } textSource.setEndOffset(length); - - sentence = docSentenceExtract(textSource, this.options.anki.sentenceExt); } finally { textSource.cleanup(); } - this.windowScroll.toY(0); - const context = { - source: { - definitions: this.definitions, - index: this.entryIndexFind(clickedElement), - scroll: this.windowScroll.y - } - }; - - if (this.context) { - context.sentence = sentence; - context.url = this.context.url; - context.source.source = this.context.source; - } - - this.setContentTerms(definitions, context); - - return {textSource}; + return {textSource, definitions}; } catch (error) { this.onError(error); } @@ -338,8 +364,10 @@ class Display { const content = await apiTemplateRender('terms.html', params); this.container.innerHTML = content; - const {index, scroll} = context || {}; - this.entryScrollIntoView(index || 0, scroll); + const {index, scroll, disableScroll} = context || {}; + if (!disableScroll) { + this.entryScrollIntoView(index || 0, scroll); + } if (options.audio.enabled && options.audio.autoPlay) { this.autoPlayAudio(); From d19f447b80e286610a83114e2294a976a27adca5 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 12:04:49 +0200 Subject: [PATCH 03/36] fix stem length checking Starting from the end and stopping at first match doesn't guarantee correctness. Starting from the beginning does. --- ext/bg/js/api.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 064903ca..174a439e 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -90,9 +90,10 @@ async function apiTextParse(text, optionsContext) { definitions = dictTermsSort(definitions); const {expression, source, reading} = definitions[0]; - let stemLength = source.length; - while (source[stemLength - 1] !== expression[stemLength - 1]) { - --stemLength; + let stemLength = 0; + const shortest = Math.min(source.length, expression.length); + while (stemLength < shortest && source[stemLength] === expression[stemLength]) { + ++stemLength; } const offset = source.length - stemLength; From 530b95895bb8a807cbaea6a324323c49152c16ab Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 12:06:25 +0200 Subject: [PATCH 04/36] remove unused css --- ext/mixed/css/display.css | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/mixed/css/display.css b/ext/mixed/css/display.css index 81109fc6..65b8b466 100644 --- a/ext/mixed/css/display.css +++ b/ext/mixed/css/display.css @@ -73,10 +73,6 @@ ol, ul { } -html:root[data-yomichan-page=search] body { - min-height: 101vh; /* always show scroll bar to avoid scanning problems */ -} - /* * Search page */ @@ -97,8 +93,13 @@ html:root[data-yomichan-page=search] body { font-size: 24px; } -.highlight { - background-color: lightblue; +html:root[data-yomichan-page=search] body { + min-height: 101vh; /* always show scroll bar to avoid scanning problems */ +} + +#query-parser { + margin-top: 10px; + font-size: 24px; } From 627e16d44b19bdae1bc51fa6f76bc766939e0786 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 13:01:06 +0200 Subject: [PATCH 05/36] improve text preview --- ext/bg/js/search-query-parser.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index c3a3900b..9bea6508 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -59,25 +59,39 @@ class QueryParser { } async setText(text) { + this.queryParser.innerHTML = ''; this.search.setSpinnerVisible(true); + let previewText = text; + while (previewText) { + const tempText = previewText.slice(0, 2); + previewText = previewText.slice(2); + + const tempRuby = document.createElement('ruby'); + const tempFurigana = document.createElement('rt'); + tempRuby.appendChild(document.createTextNode(tempText)); + tempRuby.appendChild(tempFurigana); + this.queryParser.appendChild(tempRuby); + } + const results = await apiTextParse(text, this.search.getOptionsContext()); - const tempContainer = document.createElement('div'); + const textContainer = document.createElement('div'); for (const {text, furigana} of results) { + const rubyElement = document.createElement('ruby'); + const furiganaElement = document.createElement('rt'); if (furigana) { - const rubyElement = document.createElement('ruby'); - const furiganaElement = document.createElement('rt'); furiganaElement.innerText = furigana; rubyElement.appendChild(document.createTextNode(text)); rubyElement.appendChild(furiganaElement); - tempContainer.appendChild(rubyElement); } else { - tempContainer.appendChild(document.createTextNode(text)); + rubyElement.appendChild(document.createTextNode(text)); + rubyElement.appendChild(furiganaElement); } + textContainer.appendChild(rubyElement); } this.queryParser.innerHTML = ''; - this.queryParser.appendChild(tempContainer); + this.queryParser.appendChild(textContainer); this.search.setSpinnerVisible(false); } From 408aa73cced09e1d4fd00cfd193d7f3bcb18b689 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 13:09:11 +0200 Subject: [PATCH 06/36] fix default params for term clicking --- ext/mixed/js/display.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js index 82385bf9..4c698ecf 100644 --- a/ext/mixed/js/display.js +++ b/ext/mixed/js/display.js @@ -98,7 +98,7 @@ class Display { } } - async onTermLookup(e, {disableScroll, selectText, disableHistory}) { + async onTermLookup(e, {disableScroll, selectText, disableHistory}={}) { const termLookupResults = await this.termLookup(e); if (!termLookupResults) { return false; From e6a1b781648b8ab965a4508ea29ab85f0e070b35 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 30 Oct 2019 18:13:45 +0200 Subject: [PATCH 07/36] use correct source text --- ext/bg/js/api.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 174a439e..dbbe7368 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -85,10 +85,11 @@ async function apiTextParse(text, optionsContext) { const results = []; while (text) { - let [definitions, length] = await translator.findTerms(text, {}, options); + let [definitions, sourceLength] = await translator.findTerms(text, {}, options); if (definitions.length > 0) { definitions = dictTermsSort(definitions); - const {expression, source, reading} = definitions[0]; + const {expression, reading} = definitions[0]; + const source = text.slice(0, sourceLength); let stemLength = 0; const shortest = Math.min(source.length, expression.length); From 3881457e4ed3f9c7833ac21a5e7fc44c2ba00b0f Mon Sep 17 00:00:00 2001 From: siikamiika Date: Thu, 31 Oct 2019 23:56:44 +0200 Subject: [PATCH 08/36] use handlebars templates for query parser --- ext/bg/js/api.js | 12 ++++---- ext/bg/js/search-query-parser.js | 40 +++++++++++--------------- ext/bg/js/templates.js | 48 ++++++++++++++++++++++++++++++++ ext/mixed/css/display.css | 9 +++--- tmpl/query-parser.html | 23 +++++++++++++++ 5 files changed, 99 insertions(+), 33 deletions(-) create mode 100644 tmpl/query-parser.html diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index dbbe7368..7c9a72a7 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -85,6 +85,7 @@ async function apiTextParse(text, optionsContext) { const results = []; while (text) { + const term = []; let [definitions, sourceLength] = await translator.findTerms(text, {}, options); if (definitions.length > 0) { definitions = dictTermsSort(definitions); @@ -98,22 +99,23 @@ async function apiTextParse(text, optionsContext) { } const offset = source.length - stemLength; - for (const result of jpDistributeFurigana( + for (const {text, furigana} of jpDistributeFurigana( source.slice(0, offset === 0 ? source.length : source.length - offset), - reading.slice(0, offset === 0 ? reading.length : source.length + (reading.length - expression.length) - offset) + reading.slice(0, offset === 0 ? reading.length : reading.length - expression.length + stemLength) )) { - results.push(result); + term.push({text, reading: furigana || ''}); } if (stemLength !== source.length) { - results.push({text: source.slice(stemLength)}); + term.push({text: source.slice(stemLength)}); } text = text.slice(source.length); } else { - results.push({text: text[0]}); + term.push({text: text[0]}); text = text.slice(1); } + results.push(term); } return results; } diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 9bea6508..8a7db69a 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -59,39 +59,33 @@ class QueryParser { } async setText(text) { - this.queryParser.innerHTML = ''; this.search.setSpinnerVisible(true); + const previewTerms = []; let previewText = text; while (previewText) { const tempText = previewText.slice(0, 2); + previewTerms.push([{text: tempText}]); previewText = previewText.slice(2); - - const tempRuby = document.createElement('ruby'); - const tempFurigana = document.createElement('rt'); - tempRuby.appendChild(document.createTextNode(tempText)); - tempRuby.appendChild(tempFurigana); - this.queryParser.appendChild(tempRuby); } + this.queryParser.innerHTML = await apiTemplateRender('query-parser.html', { + terms: previewTerms, + preview: true + }); + const results = await apiTextParse(text, this.search.getOptionsContext()); - const textContainer = document.createElement('div'); - for (const {text, furigana} of results) { - const rubyElement = document.createElement('ruby'); - const furiganaElement = document.createElement('rt'); - if (furigana) { - furiganaElement.innerText = furigana; - rubyElement.appendChild(document.createTextNode(text)); - rubyElement.appendChild(furiganaElement); - } else { - rubyElement.appendChild(document.createTextNode(text)); - rubyElement.appendChild(furiganaElement); - } - textContainer.appendChild(rubyElement); - } - this.queryParser.innerHTML = ''; - this.queryParser.appendChild(textContainer); + const content = await apiTemplateRender('query-parser.html', { + terms: results.map((term) => { + return term.map((part) => { + part.raw = !part.text.trim() && (!part.reading || !part.reading.trim()); + return part; + }); + }) + }); + + this.queryParser.innerHTML = content; this.search.setSpinnerVisible(false); } diff --git a/ext/bg/js/templates.js b/ext/bg/js/templates.js index 823b9e6f..cc233d49 100644 --- a/ext/bg/js/templates.js +++ b/ext/bg/js/templates.js @@ -162,6 +162,54 @@ templates['kanji.html'] = template({"1":function(container,depth0,helpers,partia return fn; } +,"useDecorators":true,"usePartial":true,"useData":true,"useDepths":true}); +templates['query-parser.html'] = template({"1":function(container,depth0,helpers,partials,data) { + var stack1, alias1=depth0 != null ? depth0 : (container.nullContext || {}); + + return ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.preview : depth0),{"name":"if","hash":{},"fn":container.program(2, data, 0),"inverse":container.program(4, data, 0),"data":data})) != null ? stack1 : "") + + ((stack1 = helpers.each.call(alias1,depth0,{"name":"each","hash":{},"fn":container.program(6, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "") + + ""; +},"2":function(container,depth0,helpers,partials,data) { + return ""; +},"4":function(container,depth0,helpers,partials,data) { + return ""; +},"6":function(container,depth0,helpers,partials,data) { + var stack1; + + return ((stack1 = container.invokePartial(partials.part,depth0,{"name":"part","data":data,"helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : ""); +},"8":function(container,depth0,helpers,partials,data) { + var stack1; + + return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.raw : depth0),{"name":"if","hash":{},"fn":container.program(9, data, 0),"inverse":container.program(11, data, 0),"data":data})) != null ? stack1 : ""); +},"9":function(container,depth0,helpers,partials,data) { + var helper; + + return container.escapeExpression(((helper = (helper = helpers.text || (depth0 != null ? depth0.text : depth0)) != null ? helper : helpers.helperMissing),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),{"name":"text","hash":{},"data":data}) : helper))); +},"11":function(container,depth0,helpers,partials,data) { + var helper, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", alias4=container.escapeExpression; + + return "" + + alias4(((helper = (helper = helpers.text || (depth0 != null ? depth0.text : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"text","hash":{},"data":data}) : helper))) + + "" + + alias4(((helper = (helper = helpers.reading || (depth0 != null ? depth0.reading : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"reading","hash":{},"data":data}) : helper))) + + ""; +},"13":function(container,depth0,helpers,partials,data,blockParams,depths) { + var stack1; + + return ((stack1 = container.invokePartial(partials.term,depth0,{"name":"term","hash":{"preview":(depths[1] != null ? depths[1].preview : depths[1])},"data":data,"helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : ""); +},"compiler":[7,">= 4.0.0"],"main":function(container,depth0,helpers,partials,data,blockParams,depths) { + var stack1; + + return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.terms : depth0),{"name":"each","hash":{},"fn":container.program(13, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : ""); +},"main_d": function(fn, props, container, depth0, data, blockParams, depths) { + + var decorators = container.decorators; + + fn = decorators.inline(fn,props,container,{"name":"inline","hash":{},"fn":container.program(1, data, 0, blockParams, depths),"inverse":container.noop,"args":["term"],"data":data}) || fn; + fn = decorators.inline(fn,props,container,{"name":"inline","hash":{},"fn":container.program(8, data, 0, blockParams, depths),"inverse":container.noop,"args":["part"],"data":data}) || fn; + return fn; + } + ,"useDecorators":true,"usePartial":true,"useData":true,"useDepths":true}); templates['terms.html'] = template({"1":function(container,depth0,helpers,partials,data) { var stack1, helper, options, alias1=depth0 != null ? depth0 : (container.nullContext || {}), buffer = diff --git a/ext/mixed/css/display.css b/ext/mixed/css/display.css index 65b8b466..d24aa58c 100644 --- a/ext/mixed/css/display.css +++ b/ext/mixed/css/display.css @@ -93,13 +93,12 @@ ol, ul { font-size: 24px; } -html:root[data-yomichan-page=search] body { - min-height: 101vh; /* always show scroll bar to avoid scanning problems */ +.query-parser-term { + margin-right: 5px; } -#query-parser { - margin-top: 10px; - font-size: 24px; +html:root[data-yomichan-page=search] body { + min-height: 101vh; /* always show scroll bar to avoid scanning problems */ } diff --git a/tmpl/query-parser.html b/tmpl/query-parser.html new file mode 100644 index 00000000..818650e6 --- /dev/null +++ b/tmpl/query-parser.html @@ -0,0 +1,23 @@ +{{~#*inline "term"~}} +{{~#if preview~}} + +{{~else~}} + +{{~/if~}} +{{~#each this~}} +{{> part }} +{{~/each~}} + +{{~/inline~}} + +{{~#*inline "part"~}} +{{~#if raw~}} +{{text}} +{{~else~}} +{{text}}{{reading}} +{{~/if~}} +{{~/inline~}} + +{{~#each terms~}} +{{> term preview=../preview }} +{{~/each~}} From 41020289ab68ef22a0691a9f268a79d6a706df6b Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 05:08:57 +0200 Subject: [PATCH 09/36] add mecab support --- ext/bg/background.html | 1 + ext/bg/js/api.js | 48 +++++++++++++++--------- ext/bg/js/backend.js | 2 + ext/bg/js/mecab.js | 63 ++++++++++++++++++++++++++++++++ ext/bg/js/search-query-parser.js | 3 +- ext/fg/js/api.js | 4 ++ ext/manifest.json | 3 +- ext/mixed/js/japanese.js | 35 ++++++++++++++++-- 8 files changed, 136 insertions(+), 23 deletions(-) create mode 100644 ext/bg/js/mecab.js diff --git a/ext/bg/background.html b/ext/bg/background.html index bbfbd1e1..6e6e7c26 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -21,6 +21,7 @@ + diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 7c9a72a7..2ab01af3 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -91,25 +91,10 @@ async function apiTextParse(text, optionsContext) { definitions = dictTermsSort(definitions); const {expression, reading} = definitions[0]; const source = text.slice(0, sourceLength); - - let stemLength = 0; - const shortest = Math.min(source.length, expression.length); - while (stemLength < shortest && source[stemLength] === expression[stemLength]) { - ++stemLength; + for (const {text, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) { + // can't use 'furigana' in templates + term.push({text, reading: furigana}); } - const offset = source.length - stemLength; - - for (const {text, furigana} of jpDistributeFurigana( - source.slice(0, offset === 0 ? source.length : source.length - offset), - reading.slice(0, offset === 0 ? reading.length : reading.length - expression.length + stemLength) - )) { - term.push({text, reading: furigana || ''}); - } - - if (stemLength !== source.length) { - term.push({text: source.slice(stemLength)}); - } - text = text.slice(source.length); } else { term.push({text: text[0]}); @@ -120,6 +105,33 @@ async function apiTextParse(text, optionsContext) { return results; } +async function apiTextParseMecab(text, optionsContext) { + const options = await apiOptionsGet(optionsContext); + const mecab = utilBackend().mecab; + + const results = []; + for (const parsedLine of await mecab.parseText(text)) { + for (const {expression, reading, source} of parsedLine) { + const term = []; + if (expression && reading) { + for (const {text, furigana} of jpDistributeFuriganaInflected( + expression, + jpKatakanaToHiragana(reading), + source + )) { + // can't use 'furigana' in templates + term.push({text, reading: furigana}); + } + } else { + term.push({text: source}); + } + results.push(term); + } + results.push([{text: '\n'}]); + } + return results; +} + async function apiKanjiFind(text, optionsContext) { const options = await apiOptionsGet(optionsContext); const definitions = await utilBackend().translator.findKanji(text, options); diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index d0e404f2..e97f32b5 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -21,6 +21,7 @@ class Backend { constructor() { this.translator = new Translator(); this.anki = new AnkiNull(); + this.mecab = new Mecab(); this.options = null; this.optionsContext = { depth: 0, @@ -181,6 +182,7 @@ Backend.messageHandlers = { kanjiFind: ({text, optionsContext}) => apiKanjiFind(text, optionsContext), termsFind: ({text, details, optionsContext}) => apiTermsFind(text, details, optionsContext), textParse: ({text, optionsContext}) => apiTextParse(text, optionsContext), + textParseMecab: ({text, optionsContext}) => apiTextParseMecab(text, optionsContext), definitionAdd: ({definition, mode, context, optionsContext}) => apiDefinitionAdd(definition, mode, context, optionsContext), definitionsAddable: ({definitions, modes, optionsContext}) => apiDefinitionsAddable(definitions, modes, optionsContext), noteView: ({noteId}) => apiNoteView(noteId), diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js new file mode 100644 index 00000000..dc46ded2 --- /dev/null +++ b/ext/bg/js/mecab.js @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +class Mecab { + constructor() { + this.listeners = {}; + this.sequence = 0; + this.startListener(); + } + + async parseText(text) { + return await this.invoke('parse_text', {text}); + } + + startListener() { + this.port = chrome.runtime.connectNative('mecab'); + this.port.onMessage.addListener((message) => { + const {sequence, data} = message; + const {callback, timer} = this.listeners[sequence] || {}; + if (timer) { + clearTimeout(timer); + delete this.listeners[sequence]; + callback(data); + } + }); + } + + invoke(action, params) { + return new Promise((resolve, reject) => { + const sequence = this.sequence++; + + this.listeners[sequence] = { + callback: (data) => { + resolve(data); + }, + timer: setTimeout(() => { + delete this.listeners[sequence]; + reject(`Mecab invoke timed out in ${Mecab.timeout} ms`); + }, 1000) + } + + this.port.postMessage({action, params, sequence}); + }); + } +} + +Mecab.timeout = 1000; diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 8a7db69a..0c74e550 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -74,7 +74,8 @@ class QueryParser { preview: true }); - const results = await apiTextParse(text, this.search.getOptionsContext()); + // const results = await apiTextParse(text, this.search.getOptionsContext()); + const results = await apiTextParseMecab(text, this.search.getOptionsContext()); const content = await apiTemplateRender('query-parser.html', { terms: results.map((term) => { diff --git a/ext/fg/js/api.js b/ext/fg/js/api.js index cc1e0e90..92330d9c 100644 --- a/ext/fg/js/api.js +++ b/ext/fg/js/api.js @@ -33,6 +33,10 @@ function apiTextParse(text, optionsContext) { return utilInvoke('textParse', {text, optionsContext}); } +function apiTextParseMecab(text, optionsContext) { + return utilInvoke('textParseMecab', {text, optionsContext}); +} + function apiKanjiFind(text, optionsContext) { return utilInvoke('kanjiFind', {text, optionsContext}); } diff --git a/ext/manifest.json b/ext/manifest.json index fabceafd..4d75cd54 100644 --- a/ext/manifest.json +++ b/ext/manifest.json @@ -42,7 +42,8 @@ "", "storage", "clipboardWrite", - "unlimitedStorage" + "unlimitedStorage", + "nativeMessaging" ], "optional_permissions": [ "clipboardRead" diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index d24f56a6..78c419b2 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -61,12 +61,11 @@ function jpDistributeFurigana(expression, reading) { const group = groups[0]; if (group.mode === 'kana') { - if (reading.startsWith(group.text)) { - const readingUsed = reading.substring(0, group.text.length); + if (jpKatakanaToHiragana(reading).startsWith(jpKatakanaToHiragana(group.text))) { const readingLeft = reading.substring(group.text.length); const segs = segmentize(readingLeft, groups.splice(1)); if (segs) { - return [{text: readingUsed}].concat(segs); + return [{text: group.text}].concat(segs); } } } else { @@ -95,3 +94,33 @@ function jpDistributeFurigana(expression, reading) { return segmentize(reading, groups) || fallback; } + +function jpDistributeFuriganaInflected(expression, reading, source) { + const output = []; + + let stemLength = 0; + const shortest = Math.min(source.length, expression.length); + const sourceHiragana = jpKatakanaToHiragana(source); + const expressionHiragana = jpKatakanaToHiragana(expression); + while ( + stemLength < shortest && + // sometimes an expression can use a kanji that's different from the source + (!jpIsKana(source[stemLength]) || (sourceHiragana[stemLength] === expressionHiragana[stemLength])) + ) { + ++stemLength; + } + const offset = source.length - stemLength; + + for (const segment of jpDistributeFurigana( + source.slice(0, offset === 0 ? source.length : source.length - offset), + reading.slice(0, offset === 0 ? reading.length : reading.length - expression.length + stemLength) + )) { + output.push(segment); + } + + if (stemLength !== source.length) { + output.push({text: source.slice(stemLength)}); + } + + return output; +} From 5a3e8c819c94be628101311e53c164b0cdd234b4 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 15:19:42 +0200 Subject: [PATCH 10/36] optimize mouseover scanning in query parser --- ext/bg/js/search-query-parser.js | 91 +++++++++++++++++++------------- ext/bg/js/templates.js | 22 ++++---- tmpl/query-parser.html | 8 ++- 3 files changed, 72 insertions(+), 49 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 0c74e550..8c3159cd 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -20,11 +20,11 @@ class QueryParser { constructor(search) { this.search = search; + this.pendingLookup = false; this.queryParser = document.querySelector('#query-parser'); this.queryParser.addEventListener('click', (e) => this.onClick(e)); - this.queryParser.addEventListener('mousemove', (e) => this.onMouseMove(e)); } onError(error) { @@ -35,8 +35,9 @@ class QueryParser { this.onTermLookup(e, {disableScroll: true, selectText: true}); } - async onMouseMove(e) { + onMouseEnter(e) { if ( + this.pendingLookup || (e.buttons & 0x1) !== 0x0 // Left mouse button ) { return; @@ -51,22 +52,51 @@ class QueryParser { return; } - await this.onTermLookup(e, {disableScroll: true, selectText: true, disableHistory: true}) + this.onTermLookup(e, {disableScroll: true, selectText: true, disableHistory: true}); } onTermLookup(e, params) { - this.search.onTermLookup(e, params); + this.pendingLookup = true; + (async () => { + await this.search.onTermLookup(e, params); + this.pendingLookup = false; + })(); } async setText(text) { this.search.setSpinnerVisible(true); + await this.setPreview(text); + // const results = await apiTextParse(text, this.search.getOptionsContext()); + const results = await apiTextParseMecab(text, this.search.getOptionsContext()); + + const content = await apiTemplateRender('query-parser.html', { + terms: results.map((term) => { + return term.filter(part => part.text.trim()).map((part) => { + return { + text: Array.from(part.text), + reading: part.reading, + raw: !part.reading || !part.reading.trim(), + }; + }); + }) + }); + + this.queryParser.innerHTML = content; + + this.queryParser.querySelectorAll('.query-parser-char').forEach((charElement) => { + this.activateScanning(charElement); + }); + + this.search.setSpinnerVisible(false); + } + + async setPreview(text) { const previewTerms = []; - let previewText = text; - while (previewText) { - const tempText = previewText.slice(0, 2); - previewTerms.push([{text: tempText}]); - previewText = previewText.slice(2); + while (text) { + const tempText = text.slice(0, 2); + previewTerms.push([{text: Array.from(tempText)}]); + text = text.slice(2); } this.queryParser.innerHTML = await apiTemplateRender('query-parser.html', { @@ -74,21 +104,22 @@ class QueryParser { preview: true }); - // const results = await apiTextParse(text, this.search.getOptionsContext()); - const results = await apiTextParseMecab(text, this.search.getOptionsContext()); - - const content = await apiTemplateRender('query-parser.html', { - terms: results.map((term) => { - return term.map((part) => { - part.raw = !part.text.trim() && (!part.reading || !part.reading.trim()); - return part; - }); - }) + this.queryParser.querySelectorAll('.query-parser-char').forEach((charElement) => { + this.activateScanning(charElement); }); + } - this.queryParser.innerHTML = content; - - this.search.setSpinnerVisible(false); + activateScanning(element) { + element.addEventListener('mouseenter', (e) => { + e.target.dataset.timer = setTimeout(() => { + this.onMouseEnter(e); + delete e.target.dataset.timer; + }, this.search.options.scanning.delay); + }); + element.addEventListener('mouseleave', (e) => { + clearTimeout(e.target.dataset.timer); + delete e.target.dataset.timer; + }); } async parseText(text) { @@ -106,22 +137,6 @@ class QueryParser { return results; } - popupTimerSet(callback) { - const delay = this.options.scanning.delay; - if (delay > 0) { - this.popupTimer = window.setTimeout(callback, delay); - } else { - Promise.resolve().then(callback); - } - } - - popupTimerClear() { - if (this.popupTimer !== null) { - window.clearTimeout(this.popupTimer); - this.popupTimer = null; - } - } - static isScanningModifierPressed(scanningModifier, mouseEvent) { switch (scanningModifier) { case 'alt': return mouseEvent.altKey; diff --git a/ext/bg/js/templates.js b/ext/bg/js/templates.js index cc233d49..6e377957 100644 --- a/ext/bg/js/templates.js +++ b/ext/bg/js/templates.js @@ -180,27 +180,31 @@ templates['query-parser.html'] = template({"1":function(container,depth0,helpers },"8":function(container,depth0,helpers,partials,data) { var stack1; - return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.raw : depth0),{"name":"if","hash":{},"fn":container.program(9, data, 0),"inverse":container.program(11, data, 0),"data":data})) != null ? stack1 : ""); + return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.raw : depth0),{"name":"if","hash":{},"fn":container.program(9, data, 0),"inverse":container.program(12, data, 0),"data":data})) != null ? stack1 : ""); },"9":function(container,depth0,helpers,partials,data) { - var helper; + var stack1; - return container.escapeExpression(((helper = (helper = helpers.text || (depth0 != null ? depth0.text : depth0)) != null ? helper : helpers.helperMissing),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),{"name":"text","hash":{},"data":data}) : helper))); -},"11":function(container,depth0,helpers,partials,data) { - var helper, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", alias4=container.escapeExpression; + return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.text : depth0),{"name":"each","hash":{},"fn":container.program(10, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : ""); +},"10":function(container,depth0,helpers,partials,data) { + return "" + + container.escapeExpression(container.lambda(depth0, depth0)) + + ""; +},"12":function(container,depth0,helpers,partials,data) { + var stack1, helper, alias1=depth0 != null ? depth0 : (container.nullContext || {}); return "" - + alias4(((helper = (helper = helpers.text || (depth0 != null ? depth0.text : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"text","hash":{},"data":data}) : helper))) + + ((stack1 = helpers.each.call(alias1,(depth0 != null ? depth0.text : depth0),{"name":"each","hash":{},"fn":container.program(10, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "") + "" - + alias4(((helper = (helper = helpers.reading || (depth0 != null ? depth0.reading : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"reading","hash":{},"data":data}) : helper))) + + container.escapeExpression(((helper = (helper = helpers.reading || (depth0 != null ? depth0.reading : depth0)) != null ? helper : helpers.helperMissing),(typeof helper === "function" ? helper.call(alias1,{"name":"reading","hash":{},"data":data}) : helper))) + ""; -},"13":function(container,depth0,helpers,partials,data,blockParams,depths) { +},"14":function(container,depth0,helpers,partials,data,blockParams,depths) { var stack1; return ((stack1 = container.invokePartial(partials.term,depth0,{"name":"term","hash":{"preview":(depths[1] != null ? depths[1].preview : depths[1])},"data":data,"helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : ""); },"compiler":[7,">= 4.0.0"],"main":function(container,depth0,helpers,partials,data,blockParams,depths) { var stack1; - return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.terms : depth0),{"name":"each","hash":{},"fn":container.program(13, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : ""); + return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.terms : depth0),{"name":"each","hash":{},"fn":container.program(14, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : ""); },"main_d": function(fn, props, container, depth0, data, blockParams, depths) { var decorators = container.decorators; diff --git a/tmpl/query-parser.html b/tmpl/query-parser.html index 818650e6..db98b5ff 100644 --- a/tmpl/query-parser.html +++ b/tmpl/query-parser.html @@ -12,9 +12,13 @@ {{~#*inline "part"~}} {{~#if raw~}} -{{text}} +{{~#each text~}} +{{this}} +{{~/each~}} {{~else~}} -{{text}}{{reading}} +{{~#each text~}} +{{this}} +{{~/each~}}{{reading}} {{~/if~}} {{~/inline~}} From 8825c481b5139ce41ad227da4d7e8725666aa072 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 15:43:27 +0200 Subject: [PATCH 11/36] respect text selection option in query parser --- ext/bg/js/search-query-parser.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 8c3159cd..dfab0d9a 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -32,7 +32,8 @@ class QueryParser { } onClick(e) { - this.onTermLookup(e, {disableScroll: true, selectText: true}); + const selectText = this.search.options.scanning.selectText; + this.onTermLookup(e, {disableScroll: true, selectText}); } onMouseEnter(e) { @@ -52,7 +53,8 @@ class QueryParser { return; } - this.onTermLookup(e, {disableScroll: true, selectText: true, disableHistory: true}); + const selectText = this.search.options.scanning.selectText; + this.onTermLookup(e, {disableScroll: true, disableHistory: true, selectText}); } onTermLookup(e, params) { From c78ca36f3dca8a3ca32923bd65e84a59d0f51613 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 15:45:07 +0200 Subject: [PATCH 12/36] switch to mousemove events in query parser --- ext/bg/js/search-query-parser.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index dfab0d9a..cff48c46 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -36,7 +36,7 @@ class QueryParser { this.onTermLookup(e, {disableScroll: true, selectText}); } - onMouseEnter(e) { + onMouseMove(e) { if ( this.pendingLookup || (e.buttons & 0x1) !== 0x0 // Left mouse button @@ -112,11 +112,16 @@ class QueryParser { } activateScanning(element) { - element.addEventListener('mouseenter', (e) => { - e.target.dataset.timer = setTimeout(() => { - this.onMouseEnter(e); - delete e.target.dataset.timer; - }, this.search.options.scanning.delay); + element.addEventListener('mousemove', (e) => { + clearTimeout(e.target.dataset.timer); + if (this.search.options.scanning.modifier === 'none') { + e.target.dataset.timer = setTimeout(() => { + this.onMouseMove(e); + delete e.target.dataset.timer; + }, this.search.options.scanning.delay); + } else { + this.onMouseMove(e); + } }); element.addEventListener('mouseleave', (e) => { clearTimeout(e.target.dataset.timer); From bc66f254ea719bb254cafabe5eb127ec7c340b1e Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 17:04:32 +0200 Subject: [PATCH 13/36] click & text selection improvements on search page --- ext/bg/js/search-query-parser.js | 52 +++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index cff48c46..60b94ca8 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -21,25 +21,39 @@ class QueryParser { constructor(search) { this.search = search; this.pendingLookup = false; + this.clickScanPrevent = false; this.queryParser = document.querySelector('#query-parser'); - this.queryParser.addEventListener('click', (e) => this.onClick(e)); + this.queryParser.addEventListener('mousedown', (e) => this.onMouseDown(e)); + this.queryParser.addEventListener('mouseup', (e) => this.onMouseUp(e)); } onError(error) { logError(error, false); } - onClick(e) { - const selectText = this.search.options.scanning.selectText; - this.onTermLookup(e, {disableScroll: true, selectText}); + onMouseDown(e) { + if (QueryParser.isMouseButton('primary', e)) { + this.clickScanPrevent = false; + } + } + + onMouseUp(e) { + if ( + this.search.options.scanning.clickGlossary && + !this.clickScanPrevent && + QueryParser.isMouseButton('primary', e) + ) { + const selectText = this.search.options.scanning.selectText; + this.onTermLookup(e, {disableScroll: true, selectText}); + } } onMouseMove(e) { if ( this.pendingLookup || - (e.buttons & 0x1) !== 0x0 // Left mouse button + QueryParser.isMouseButton('primary', e) ) { return; } @@ -48,7 +62,7 @@ class QueryParser { const scanningModifier = scanningOptions.modifier; if (!( QueryParser.isScanningModifierPressed(scanningModifier, e) || - (scanningOptions.middleMouse && (e.buttons & 0x4) !== 0x0) // Middle mouse button + (scanningOptions.middleMouse && QueryParser.isMouseButton('auxiliary', e)) )) { return; } @@ -57,6 +71,12 @@ class QueryParser { this.onTermLookup(e, {disableScroll: true, disableHistory: true, selectText}); } + onMouseLeave(e) { + this.clickScanPrevent = true; + clearTimeout(e.target.dataset.timer); + delete e.target.dataset.timer; + } + onTermLookup(e, params) { this.pendingLookup = true; (async () => { @@ -124,8 +144,7 @@ class QueryParser { } }); element.addEventListener('mouseleave', (e) => { - clearTimeout(e.target.dataset.timer); - delete e.target.dataset.timer; + this.onMouseLeave(e); }); } @@ -153,4 +172,21 @@ class QueryParser { default: return false; } } + + static isMouseButton(button, mouseEvent) { + if (['mouseup', 'mousedown'].includes(mouseEvent.type)) { + switch (button) { + case 'primary': return mouseEvent.button === 0; + case 'secondary': return mouseEvent.button === 2; + case 'auxiliary': return mouseEvent.button === 1; + default: return false; + } + } + switch (button) { + case 'primary': return (mouseEvent.buttons & 0x1) !== 0x0; + case 'secondary': return (mouseEvent.buttons & 0x2) !== 0x0; + case 'auxiliary': return (mouseEvent.buttons & 0x4) !== 0x0; + default: return false; + } + } } From b0c924d4bdaa0a606a6b559701047283ec0ff1db Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 17:06:02 +0200 Subject: [PATCH 14/36] fix mecab variable --- ext/bg/js/mecab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index dc46ded2..d28d80c0 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -52,7 +52,7 @@ class Mecab { timer: setTimeout(() => { delete this.listeners[sequence]; reject(`Mecab invoke timed out in ${Mecab.timeout} ms`); - }, 1000) + }, Mecab.timeout) } this.port.postMessage({action, params, sequence}); From 515345ba0a9844ff55518779f799e4cf2a003d62 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 3 Nov 2019 18:03:35 +0200 Subject: [PATCH 15/36] remove code duplication --- ext/bg/js/search-query-parser.js | 40 ++++---------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 60b94ca8..1cf00425 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -34,7 +34,7 @@ class QueryParser { } onMouseDown(e) { - if (QueryParser.isMouseButton('primary', e)) { + if (Frontend.isMouseButton('primary', e)) { this.clickScanPrevent = false; } } @@ -43,7 +43,7 @@ class QueryParser { if ( this.search.options.scanning.clickGlossary && !this.clickScanPrevent && - QueryParser.isMouseButton('primary', e) + Frontend.isMouseButton('primary', e) ) { const selectText = this.search.options.scanning.selectText; this.onTermLookup(e, {disableScroll: true, selectText}); @@ -51,18 +51,15 @@ class QueryParser { } onMouseMove(e) { - if ( - this.pendingLookup || - QueryParser.isMouseButton('primary', e) - ) { + if (this.pendingLookup || Frontend.isMouseButton('primary', e)) { return; } const scanningOptions = this.search.options.scanning; const scanningModifier = scanningOptions.modifier; if (!( - QueryParser.isScanningModifierPressed(scanningModifier, e) || - (scanningOptions.middleMouse && QueryParser.isMouseButton('auxiliary', e)) + Frontend.isScanningModifierPressed(scanningModifier, e) || + (scanningOptions.middleMouse && Frontend.isMouseButton('auxiliary', e)) )) { return; } @@ -162,31 +159,4 @@ class QueryParser { } return results; } - - static isScanningModifierPressed(scanningModifier, mouseEvent) { - switch (scanningModifier) { - case 'alt': return mouseEvent.altKey; - case 'ctrl': return mouseEvent.ctrlKey; - case 'shift': return mouseEvent.shiftKey; - case 'none': return true; - default: return false; - } - } - - static isMouseButton(button, mouseEvent) { - if (['mouseup', 'mousedown'].includes(mouseEvent.type)) { - switch (button) { - case 'primary': return mouseEvent.button === 0; - case 'secondary': return mouseEvent.button === 2; - case 'auxiliary': return mouseEvent.button === 1; - default: return false; - } - } - switch (button) { - case 'primary': return (mouseEvent.buttons & 0x1) !== 0x0; - case 'secondary': return (mouseEvent.buttons & 0x2) !== 0x0; - case 'auxiliary': return (mouseEvent.buttons & 0x4) !== 0x0; - default: return false; - } - } } From 1bf48d24ef4a0a1ed09d8ec6acc74ff5f78dd6c2 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 4 Nov 2019 01:25:32 +0200 Subject: [PATCH 16/36] change mecab path https://github.com/siikamiika/yomichan-mecab-installer --- ext/bg/js/mecab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index d28d80c0..14f68393 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -29,7 +29,7 @@ class Mecab { } startListener() { - this.port = chrome.runtime.connectNative('mecab'); + this.port = chrome.runtime.connectNative('yomichan_mecab'); this.port.onMessage.addListener((message) => { const {sequence, data} = message; const {callback, timer} = this.listeners[sequence] || {}; From 17003189888694da51d840dcf0464355bb342a4c Mon Sep 17 00:00:00 2001 From: siikamiika Date: Tue, 5 Nov 2019 02:43:04 +0200 Subject: [PATCH 17/36] remove unneeded feature Unidic actually has a field for the base form of the input --- ext/mixed/js/japanese.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index 78c419b2..1201e524 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -102,11 +102,7 @@ function jpDistributeFuriganaInflected(expression, reading, source) { const shortest = Math.min(source.length, expression.length); const sourceHiragana = jpKatakanaToHiragana(source); const expressionHiragana = jpKatakanaToHiragana(expression); - while ( - stemLength < shortest && - // sometimes an expression can use a kanji that's different from the source - (!jpIsKana(source[stemLength]) || (sourceHiragana[stemLength] === expressionHiragana[stemLength])) - ) { + while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { ++stemLength; } const offset = source.length - stemLength; From 955e131f9673e006556bc2c5e0b3551a614ccc48 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Tue, 5 Nov 2019 15:56:45 +0200 Subject: [PATCH 18/36] add parser selection options --- ext/bg/js/api.js | 37 +++++++++++++++++-------------- ext/bg/js/mecab.js | 2 +- ext/bg/js/options.js | 5 +++++ ext/bg/js/search-query-parser.js | 38 ++++++++++++++++++++------------ ext/bg/js/settings.js | 6 +++++ ext/bg/settings.html | 29 ++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 31 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 2ab01af3..967bded7 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -109,25 +109,30 @@ async function apiTextParseMecab(text, optionsContext) { const options = await apiOptionsGet(optionsContext); const mecab = utilBackend().mecab; - const results = []; - for (const parsedLine of await mecab.parseText(text)) { - for (const {expression, reading, source} of parsedLine) { - const term = []; - if (expression && reading) { - for (const {text, furigana} of jpDistributeFuriganaInflected( - expression, - jpKatakanaToHiragana(reading), - source - )) { - // can't use 'furigana' in templates - term.push({text, reading: furigana}); + const results = {}; + const rawResults = await mecab.parseText(text); + for (const mecabName in rawResults) { + const result = []; + for (const parsedLine of rawResults[mecabName]) { + for (const {expression, reading, source} of parsedLine) { + const term = []; + if (expression && reading) { + for (const {text, furigana} of jpDistributeFuriganaInflected( + expression, + jpKatakanaToHiragana(reading), + source + )) { + // can't use 'furigana' in templates + term.push({text, reading: furigana}); + } + } else { + term.push({text: source}); } - } else { - term.push({text: source}); + result.push(term); } - results.push(term); + result.push([{text: '\n'}]); } - results.push([{text: '\n'}]); + results[mecabName] = result; } return results; } diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index 14f68393..fba9b2eb 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -60,4 +60,4 @@ class Mecab { } } -Mecab.timeout = 1000; +Mecab.timeout = 5000; diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index be1ccfbb..f1bafaf9 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -311,6 +311,11 @@ function profileOptionsCreateDefaults() { dictionaries: {}, + parsing: { + enableScanningParser: true, + enableMecabParser: false + }, + anki: { enable: false, server: 'http://127.0.0.1:8765', diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 1cf00425..81eb18c3 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -86,22 +86,32 @@ class QueryParser { this.search.setSpinnerVisible(true); await this.setPreview(text); - // const results = await apiTextParse(text, this.search.getOptionsContext()); - const results = await apiTextParseMecab(text, this.search.getOptionsContext()); + const results = {}; + if (this.search.options.parsing.enableScanningParser) { + results['scan'] = await apiTextParse(text, this.search.getOptionsContext()); + } + if (this.search.options.parsing.enableMecabParser) { + let mecabResults = await apiTextParseMecab(text, this.search.getOptionsContext()); + for (const mecabDictName in mecabResults) { + results[`mecab-${mecabDictName}`] = mecabResults[mecabDictName]; + } + } - const content = await apiTemplateRender('query-parser.html', { - terms: results.map((term) => { - return term.filter(part => part.text.trim()).map((part) => { - return { - text: Array.from(part.text), - reading: part.reading, - raw: !part.reading || !part.reading.trim(), - }; - }); - }) - }); + const contents = await Promise.all(Object.values(results).map(async result => { + return await apiTemplateRender('query-parser.html', { + terms: result.map((term) => { + return term.filter(part => part.text.trim()).map((part) => { + return { + text: Array.from(part.text), + reading: part.reading, + raw: !part.reading || !part.reading.trim(), + }; + }); + }) + }); + })); - this.queryParser.innerHTML = content; + this.queryParser.innerHTML = contents.join('
'); this.queryParser.querySelectorAll('.query-parser-char').forEach((charElement) => { this.activateScanning(charElement); diff --git a/ext/bg/js/settings.js b/ext/bg/js/settings.js index e562c54e..f4fe032a 100644 --- a/ext/bg/js/settings.js +++ b/ext/bg/js/settings.js @@ -64,6 +64,9 @@ async function formRead(options) { options.scanning.modifier = $('#scan-modifier-key').val(); options.scanning.popupNestingMaxDepth = parseInt($('#popup-nesting-max-depth').val(), 10); + options.parsing.enableScanningParser = $('#parsing-scan-enable').prop('checked'); + options.parsing.enableMecabParser = $('#parsing-mecab-enable').prop('checked'); + const optionsAnkiEnableOld = options.anki.enable; options.anki.enable = $('#anki-enable').prop('checked'); options.anki.tags = utilBackgroundIsolate($('#card-tags').val().split(/[,; ]+/)); @@ -126,6 +129,9 @@ async function formWrite(options) { $('#scan-modifier-key').val(options.scanning.modifier); $('#popup-nesting-max-depth').val(options.scanning.popupNestingMaxDepth); + $('#parsing-scan-enable').prop('checked', options.parsing.enableScanningParser); + $('#parsing-mecab-enable').prop('checked', options.parsing.enableMecabParser); + $('#anki-enable').prop('checked', options.anki.enable); $('#card-tags').val(options.anki.tags.join(' ')); $('#sentence-detection-extent').val(options.anki.sentenceExt); diff --git a/ext/bg/settings.html b/ext/bg/settings.html index bdcc11d3..8505567b 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -587,6 +587,35 @@ +
+

Text Parsing Options

+ +

+ Yomichan can attempt to parse entire sentences or longer text blocks on the search page, + adding furigana above words and a small space between words. +

+ +

+ Two types of parsers are supported. The first one, enabled by default, works using the built-in + scanning functionality by automatically advancing in the sentence after a matching word. +

+ +

+ The second type is an external program called MeCab + that uses its own dictionaries and a special parsing algorithm. To get it working, you must first + install it and a native messaging component + that acts as a bridge between the program and Yomichan. +

+ +
+ +
+ +
+ +
+
+
From c1d24208d383e687f443c4e7dfc7bfda81d191bd Mon Sep 17 00:00:00 2001 From: siikamiika Date: Fri, 8 Nov 2019 00:49:20 +0200 Subject: [PATCH 19/36] start mecab only after enabling the setting --- ext/bg/js/backend.js | 4 +++ ext/bg/js/mecab.js | 14 ++++++++++- ext/bg/js/settings.js | 15 +++++++++++ ext/bg/settings.html | 58 +++++++++++++++++++++---------------------- 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index e97f32b5..027cc250 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -98,6 +98,10 @@ class Backend { } this.anki = options.anki.enable ? new AnkiConnect(options.anki.server) : new AnkiNull(); + + if (options.parsing.enableMecabParser) { + this.mecab.startListener(); + } } async getFullOptions() { diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index fba9b2eb..4c62c2b0 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -19,16 +19,20 @@ class Mecab { constructor() { + this.port = null; this.listeners = {}; this.sequence = 0; - this.startListener(); } async parseText(text) { + if (this.port === null) { + return {}; + } return await this.invoke('parse_text', {text}); } startListener() { + if (this.port !== null) { return; } this.port = chrome.runtime.connectNative('yomichan_mecab'); this.port.onMessage.addListener((message) => { const {sequence, data} = message; @@ -41,6 +45,14 @@ class Mecab { }); } + stopListener() { + if (this.port === null) { return; } + this.port.disconnect(); + this.port = null; + this.listeners = {}; + this.sequence = 0; + } + invoke(action, params) { return new Promise((resolve, reject) => { const sequence = this.sequence++; diff --git a/ext/bg/js/settings.js b/ext/bg/js/settings.js index f4fe032a..0013291a 100644 --- a/ext/bg/js/settings.js +++ b/ext/bg/js/settings.js @@ -154,6 +154,7 @@ async function formWrite(options) { function formSetupEventListeners() { $('input, select, textarea').not('.anki-model').not('.ignore-form-changes *').change(utilAsync(onFormOptionsChanged)); + $('#parsing-mecab-enable').change(onParseMecabChanged); $('.anki-model').change(utilAsync(onAnkiModelChanged)); } @@ -440,6 +441,20 @@ function onMessage({action, params}, sender, callback) { } +/* + * Text parsing + */ + +function onParseMecabChanged(e) { + const mecab = utilBackend().mecab; + if (e.target.checked) { + mecab.startListener(); + } else { + mecab.stopListener(); + } +} + + /* * Anki */ diff --git a/ext/bg/settings.html b/ext/bg/settings.html index 8505567b..08b9b6c1 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -410,6 +410,35 @@
+
+

Text Parsing Options

+ +

+ Yomichan can attempt to parse entire sentences or longer text blocks on the search page, + adding furigana above words and a small space between words. +

+ +

+ Two types of parsers are supported. The first one, enabled by default, works using the built-in + scanning functionality by automatically advancing in the sentence after a matching word. +

+ +

+ The second type is an external program called MeCab + that uses its own dictionaries and a special parsing algorithm. To get it working, you must first + install it and a native messaging component + that acts as a bridge between the program and Yomichan. +

+ +
+ +
+ +
+ +
+
+
@@ -587,35 +616,6 @@
-
-

Text Parsing Options

- -

- Yomichan can attempt to parse entire sentences or longer text blocks on the search page, - adding furigana above words and a small space between words. -

- -

- Two types of parsers are supported. The first one, enabled by default, works using the built-in - scanning functionality by automatically advancing in the sentence after a matching word. -

- -

- The second type is an external program called MeCab - that uses its own dictionaries and a special parsing algorithm. To get it working, you must first - install it and a native messaging component - that acts as a bridge between the program and Yomichan. -

- -
- -
- -
- -
-
-
From 8d9a635d5c01e9a954284c82d1cddfab89f8dd5b Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 01:03:35 +0200 Subject: [PATCH 20/36] remove dead code --- ext/bg/js/search-query-parser.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 81eb18c3..2e17688e 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -154,19 +154,4 @@ class QueryParser { this.onMouseLeave(e); }); } - - async parseText(text) { - const results = []; - while (text) { - const {definitions, length} = await apiTermsFind(text, {}, this.search.getOptionsContext()); - if (length) { - results.push(definitions); - text = text.slice(length); - } else { - results.push(text[0]); - text = text.slice(1); - } - } - return results; - } } From b336ab3a9a0a2ac9a569ae217b506ceeeab6fda0 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 01:43:35 +0200 Subject: [PATCH 21/36] use const --- ext/bg/js/api.js | 4 ++-- ext/bg/js/search-query-parser.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 967bded7..40e9b6d2 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -86,9 +86,9 @@ async function apiTextParse(text, optionsContext) { const results = []; while (text) { const term = []; - let [definitions, sourceLength] = await translator.findTerms(text, {}, options); + const [definitions, sourceLength] = await translator.findTerms(text, {}, options); if (definitions.length > 0) { - definitions = dictTermsSort(definitions); + dictTermsSort(definitions); const {expression, reading} = definitions[0]; const source = text.slice(0, sourceLength); for (const {text, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) { diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 2e17688e..f8e53963 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -113,9 +113,9 @@ class QueryParser { this.queryParser.innerHTML = contents.join('
'); - this.queryParser.querySelectorAll('.query-parser-char').forEach((charElement) => { + for (const charElement of this.queryParser.querySelectorAll('.query-parser-char')) { this.activateScanning(charElement); - }); + } this.search.setSpinnerVisible(false); } @@ -133,9 +133,9 @@ class QueryParser { preview: true }); - this.queryParser.querySelectorAll('.query-parser-char').forEach((charElement) => { + for (const charElement of this.queryParser.querySelectorAll('.query-parser-char')) { this.activateScanning(charElement); - }); + } } activateScanning(element) { From f97877a2097c8b27b75099e489b93db255cb68be Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 02:28:00 +0200 Subject: [PATCH 22/36] promise improvements --- ext/bg/js/mecab.js | 6 ++---- ext/bg/js/search-query-parser.js | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index 4c62c2b0..b9f2d0b3 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -58,12 +58,10 @@ class Mecab { const sequence = this.sequence++; this.listeners[sequence] = { - callback: (data) => { - resolve(data); - }, + callback: resolve, timer: setTimeout(() => { delete this.listeners[sequence]; - reject(`Mecab invoke timed out in ${Mecab.timeout} ms`); + reject(new Error(`Mecab invoke timed out in ${Mecab.timeout} ms`)); }, Mecab.timeout) } diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index f8e53963..2aee45dd 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -97,8 +97,8 @@ class QueryParser { } } - const contents = await Promise.all(Object.values(results).map(async result => { - return await apiTemplateRender('query-parser.html', { + const contents = await Promise.all(Object.values(results).map(result => { + return apiTemplateRender('query-parser.html', { terms: result.map((term) => { return term.filter(part => part.text.trim()).map((part) => { return { From 1f2eee449e2c1e0baf20dba038da7eaf3424aefe Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 20:54:23 +0200 Subject: [PATCH 23/36] mecab refactoring and bugfix --- ext/bg/js/backend.js | 2 ++ ext/bg/js/mecab.js | 19 ++++++++++--------- ext/bg/js/settings.js | 15 --------------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index 027cc250..45db9660 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -101,6 +101,8 @@ class Backend { if (options.parsing.enableMecabParser) { this.mecab.startListener(); + } else { + this.mecab.stopListener(); } } diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index b9f2d0b3..ada96945 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -34,15 +34,7 @@ class Mecab { startListener() { if (this.port !== null) { return; } this.port = chrome.runtime.connectNative('yomichan_mecab'); - this.port.onMessage.addListener((message) => { - const {sequence, data} = message; - const {callback, timer} = this.listeners[sequence] || {}; - if (timer) { - clearTimeout(timer); - delete this.listeners[sequence]; - callback(data); - } - }); + this.port.onMessage.addListener(this.onNativeMessage.bind(this)); } stopListener() { @@ -53,6 +45,15 @@ class Mecab { this.sequence = 0; } + onNativeMessage({sequence, data}) { + if (this.listeners.hasOwnProperty(sequence)) { + const {callback, timer} = this.listeners[sequence]; + clearTimeout(timer); + callback(data); + delete this.listeners[sequence]; + } + } + invoke(action, params) { return new Promise((resolve, reject) => { const sequence = this.sequence++; diff --git a/ext/bg/js/settings.js b/ext/bg/js/settings.js index 0013291a..f4fe032a 100644 --- a/ext/bg/js/settings.js +++ b/ext/bg/js/settings.js @@ -154,7 +154,6 @@ async function formWrite(options) { function formSetupEventListeners() { $('input, select, textarea').not('.anki-model').not('.ignore-form-changes *').change(utilAsync(onFormOptionsChanged)); - $('#parsing-mecab-enable').change(onParseMecabChanged); $('.anki-model').change(utilAsync(onAnkiModelChanged)); } @@ -441,20 +440,6 @@ function onMessage({action, params}, sender, callback) { } -/* - * Text parsing - */ - -function onParseMecabChanged(e) { - const mecab = utilBackend().mecab; - if (e.target.checked) { - mecab.startListener(); - } else { - mecab.stopListener(); - } -} - - /* * Anki */ From b02a30a2fddb3b660a62f66541e90dba519cb270 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 21:43:35 +0200 Subject: [PATCH 24/36] explicit checks in while and if --- ext/bg/js/api.js | 4 ++-- ext/bg/js/search-query-parser.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 40e9b6d2..bc9dfba1 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -84,7 +84,7 @@ async function apiTextParse(text, optionsContext) { const translator = utilBackend().translator; const results = []; - while (text) { + while (text.length > 0) { const term = []; const [definitions, sourceLength] = await translator.findTerms(text, {}, options); if (definitions.length > 0) { @@ -116,7 +116,7 @@ async function apiTextParseMecab(text, optionsContext) { for (const parsedLine of rawResults[mecabName]) { for (const {expression, reading, source} of parsedLine) { const term = []; - if (expression && reading) { + if (expression !== null && reading !== null) { for (const {text, furigana} of jpDistributeFuriganaInflected( expression, jpKatakanaToHiragana(reading), diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 2aee45dd..14b78105 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -122,7 +122,7 @@ class QueryParser { async setPreview(text) { const previewTerms = []; - while (text) { + while (text.length > 0) { const tempText = text.slice(0, 2); previewTerms.push([{text: Array.from(tempText)}]); text = text.slice(2); From 84f30113e4b8b750525b5e67a2a6bfa68da666ff Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 11 Nov 2019 21:58:04 +0200 Subject: [PATCH 25/36] give names to complex slices --- ext/mixed/js/japanese.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index 1201e524..e2d7a090 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -107,10 +107,11 @@ function jpDistributeFuriganaInflected(expression, reading, source) { } const offset = source.length - stemLength; - for (const segment of jpDistributeFurigana( - source.slice(0, offset === 0 ? source.length : source.length - offset), - reading.slice(0, offset === 0 ? reading.length : reading.length - expression.length + stemLength) - )) { + const stemExpression = source.slice(0, source.length - offset); + const stemReading = reading.slice( + 0, offset === 0 ? reading.length : reading.length - expression.length + stemLength + ); + for (const segment of jpDistributeFurigana(stemExpression, stemReading)) { output.push(segment); } From 9dff658640d864fbabe063161b68e752a6bd3b3b Mon Sep 17 00:00:00 2001 From: siikamiika Date: Tue, 12 Nov 2019 23:57:21 +0200 Subject: [PATCH 26/36] add parser selection --- ext/bg/js/options.js | 3 +- ext/bg/js/search-query-parser.js | 113 ++++++++++++++++++++++++------- ext/bg/search.html | 7 +- 3 files changed, 95 insertions(+), 28 deletions(-) diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index f1bafaf9..abfe5fc6 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -313,7 +313,8 @@ function profileOptionsCreateDefaults() { parsing: { enableScanningParser: true, - enableMecabParser: false + enableMecabParser: false, + selectedParser = null }, anki: { diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 14b78105..15c394fe 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -23,7 +23,10 @@ class QueryParser { this.pendingLookup = false; this.clickScanPrevent = false; + this.parseResults = []; + this.queryParser = document.querySelector('#query-parser'); + this.queryParserSelect = document.querySelector('#query-parser-select'); this.queryParser.addEventListener('mousedown', (e) => this.onMouseDown(e)); this.queryParser.addEventListener('mouseup', (e) => this.onMouseUp(e)); @@ -82,42 +85,55 @@ class QueryParser { })(); } + onParserChange(e) { + const selectedParser = e.target.value; + apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); + this.renderParseResult(this.getParseResult()); + } + + getParseResult() { + return this.parseResults.find(r => r.id === this.search.options.parsing.selectedParser); + } + async setText(text) { this.search.setSpinnerVisible(true); + await this.setPreview(text); - const results = {}; + this.parseResults = await this.parseText(text); + if (this.parseResults.length > 0) { + if (this.search.options.parsing.selectedParser === null || !this.getParseResult()) { + const selectedParser = this.parseResults[0].id; + apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); + } + } + + this.renderParserSelect(); + await this.renderParseResult(); + + this.search.setSpinnerVisible(false); + } + + async parseText(text) { + const results = []; if (this.search.options.parsing.enableScanningParser) { - results['scan'] = await apiTextParse(text, this.search.getOptionsContext()); + results.push({ + name: 'Scanning parser', + id: 'scan', + parsedText: await apiTextParse(text, this.search.getOptionsContext()) + }); } if (this.search.options.parsing.enableMecabParser) { let mecabResults = await apiTextParseMecab(text, this.search.getOptionsContext()); for (const mecabDictName in mecabResults) { - results[`mecab-${mecabDictName}`] = mecabResults[mecabDictName]; + results.push({ + name: `MeCab: ${mecabDictName}`, + id: `mecab-${mecabDictName}`, + parsedText: mecabResults[mecabDictName] + }); } } - - const contents = await Promise.all(Object.values(results).map(result => { - return apiTemplateRender('query-parser.html', { - terms: result.map((term) => { - return term.filter(part => part.text.trim()).map((part) => { - return { - text: Array.from(part.text), - reading: part.reading, - raw: !part.reading || !part.reading.trim(), - }; - }); - }) - }); - })); - - this.queryParser.innerHTML = contents.join('
'); - - for (const charElement of this.queryParser.querySelectorAll('.query-parser-char')) { - this.activateScanning(charElement); - } - - this.search.setSpinnerVisible(false); + return results; } async setPreview(text) { @@ -127,7 +143,6 @@ class QueryParser { previewTerms.push([{text: Array.from(tempText)}]); text = text.slice(2); } - this.queryParser.innerHTML = await apiTemplateRender('query-parser.html', { terms: previewTerms, preview: true @@ -138,6 +153,40 @@ class QueryParser { } } + renderParserSelect() { + this.queryParserSelect.innerHTML = ''; + if (this.parseResults.length > 1) { + const select = document.createElement('select'); + select.classList.add('form-control'); + for (const parseResult of this.parseResults) { + const option = document.createElement('option'); + option.value = parseResult.id; + option.innerText = parseResult.name; + option.defaultSelected = this.search.options.parsing.selectedParser === parseResult.id; + select.appendChild(option); + } + select.addEventListener('change', this.onParserChange.bind(this)); + this.queryParserSelect.appendChild(select); + } + } + + async renderParseResult() { + const parseResult = this.getParseResult(); + if (!parseResult) { + this.queryParser.innerHTML = ''; + return; + } + + this.queryParser.innerHTML = await apiTemplateRender( + 'query-parser.html', + {terms: QueryParser.processParseResultForDisplay(parseResult.parsedText)} + ); + + for (const charElement of this.queryParser.querySelectorAll('.query-parser-char')) { + this.activateScanning(charElement); + } + } + activateScanning(element) { element.addEventListener('mousemove', (e) => { clearTimeout(e.target.dataset.timer); @@ -154,4 +203,16 @@ class QueryParser { this.onMouseLeave(e); }); } + + static processParseResultForDisplay(result) { + return result.map((term) => { + return term.filter(part => part.text.trim()).map((part) => { + return { + text: Array.from(part.text), + reading: part.reading, + raw: !part.reading || !part.reading.trim(), + }; + }); + }); + } } diff --git a/ext/bg/search.html b/ext/bg/search.html index 48e7dbf5..e819ebe6 100644 --- a/ext/bg/search.html +++ b/ext/bg/search.html @@ -47,7 +47,12 @@
-
+
+
+
+
+ +
From f6f19dc9deeaf6ae88d36b776d2f1f4d0ef40bd3 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 00:37:09 +0200 Subject: [PATCH 27/36] typo --- ext/bg/js/options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index abfe5fc6..053fb13d 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -314,7 +314,7 @@ function profileOptionsCreateDefaults() { parsing: { enableScanningParser: true, enableMecabParser: false, - selectedParser = null + selectedParser: null }, anki: { From 0d6e0edc31d4ceac58b886e238b77f0143a5c3ec Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 12:05:39 +0200 Subject: [PATCH 28/36] remove height hack and use overflow-y: scroll --- ext/mixed/css/display.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/mixed/css/display.css b/ext/mixed/css/display.css index d24aa58c..ba2fadb7 100644 --- a/ext/mixed/css/display.css +++ b/ext/mixed/css/display.css @@ -98,7 +98,7 @@ ol, ul { } html:root[data-yomichan-page=search] body { - min-height: 101vh; /* always show scroll bar to avoid scanning problems */ + overflow-y: scroll; /* always show scroll bar to avoid scanning problems */ } From 707b039927f27443f9e32cffe342e7f778dff955 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 13:09:23 +0200 Subject: [PATCH 29/36] store local copy of selected parser Options don't update early enough even after awaiting --- ext/bg/js/search-query-parser.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 15c394fe..71ebb389 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -24,6 +24,7 @@ class QueryParser { this.clickScanPrevent = false; this.parseResults = []; + this.selectedParser = null; this.queryParser = document.querySelector('#query-parser'); this.queryParserSelect = document.querySelector('#query-parser-select'); @@ -85,14 +86,15 @@ class QueryParser { })(); } - onParserChange(e) { + async onParserChange(e) { const selectedParser = e.target.value; + this.selectedParser = selectedParser; apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); this.renderParseResult(this.getParseResult()); } getParseResult() { - return this.parseResults.find(r => r.id === this.search.options.parsing.selectedParser); + return this.parseResults.find(r => r.id === this.selectedParser); } async setText(text) { @@ -102,8 +104,12 @@ class QueryParser { this.parseResults = await this.parseText(text); if (this.parseResults.length > 0) { - if (this.search.options.parsing.selectedParser === null || !this.getParseResult()) { + if (this.selectedParser === null) { + this.selectedParser = this.search.options.parsing.selectedParser; + } + if (this.selectedParser === null || !this.getParseResult()) { const selectedParser = this.parseResults[0].id; + this.selectedParser = selectedParser; apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); } } @@ -162,7 +168,7 @@ class QueryParser { const option = document.createElement('option'); option.value = parseResult.id; option.innerText = parseResult.name; - option.defaultSelected = this.search.options.parsing.selectedParser === parseResult.id; + option.defaultSelected = this.selectedParser === parseResult.id; select.appendChild(option); } select.addEventListener('change', this.onParserChange.bind(this)); From 933fd77957cac4027c389b43cddaeb3ea65d9faa Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 13:11:41 +0200 Subject: [PATCH 30/36] remove async --- ext/bg/js/search-query-parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 71ebb389..c0cad773 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -86,7 +86,7 @@ class QueryParser { })(); } - async onParserChange(e) { + onParserChange(e) { const selectedParser = e.target.value; this.selectedParser = selectedParser; apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); From f6253216505737d588ccff1f07cb5ce0332297a5 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 15:29:53 +0200 Subject: [PATCH 31/36] refactor selected parser refreshing --- ext/bg/js/search-query-parser.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index c0cad773..42e53989 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -93,16 +93,7 @@ class QueryParser { this.renderParseResult(this.getParseResult()); } - getParseResult() { - return this.parseResults.find(r => r.id === this.selectedParser); - } - - async setText(text) { - this.search.setSpinnerVisible(true); - - await this.setPreview(text); - - this.parseResults = await this.parseText(text); + refreshSelectedParser() { if (this.parseResults.length > 0) { if (this.selectedParser === null) { this.selectedParser = this.search.options.parsing.selectedParser; @@ -113,6 +104,19 @@ class QueryParser { apiOptionsSet({parsing: {selectedParser}}, this.search.getOptionsContext()); } } + } + + getParseResult() { + return this.parseResults.find(r => r.id === this.selectedParser); + } + + async setText(text) { + this.search.setSpinnerVisible(true); + + await this.setPreview(text); + + this.parseResults = await this.parseText(text); + this.refreshSelectedParser(); this.renderParserSelect(); await this.renderParseResult(); From cc8221c6ea686521261e2ac562d3d5a6d0b9913a Mon Sep 17 00:00:00 2001 From: siikamiika Date: Wed, 13 Nov 2019 20:24:11 +0200 Subject: [PATCH 32/36] add reading modes --- ext/bg/js/api.js | 14 ++++++++------ ext/bg/js/options.js | 3 ++- ext/bg/js/settings.js | 2 ++ ext/bg/settings.html | 9 +++++++++ ext/mixed/js/japanese.js | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 7 deletions(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index bc9dfba1..228447c3 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -92,12 +92,13 @@ async function apiTextParse(text, optionsContext) { const {expression, reading} = definitions[0]; const source = text.slice(0, sourceLength); for (const {text, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) { - // can't use 'furigana' in templates - term.push({text, reading: furigana}); + const reading = jpConvertReading(text, furigana, options.parsing.readingMode); + term.push({text, reading}); } text = text.slice(source.length); } else { - term.push({text: text[0]}); + const reading = jpConvertReading(text[0], null, options.parsing.readingMode); + term.push({text: text[0], reading}); text = text.slice(1); } results.push(term); @@ -122,11 +123,12 @@ async function apiTextParseMecab(text, optionsContext) { jpKatakanaToHiragana(reading), source )) { - // can't use 'furigana' in templates - term.push({text, reading: furigana}); + const reading = jpConvertReading(text, furigana, options.parsing.readingMode); + term.push({text, reading}); } } else { - term.push({text: source}); + const reading = jpConvertReading(source, null, options.parsing.readingMode); + term.push({text: source, reading}); } result.push(term); } diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index 053fb13d..b9bf85f3 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -314,7 +314,8 @@ function profileOptionsCreateDefaults() { parsing: { enableScanningParser: true, enableMecabParser: false, - selectedParser: null + selectedParser: null, + readingMode: 'hiragana' }, anki: { diff --git a/ext/bg/js/settings.js b/ext/bg/js/settings.js index f4fe032a..ab267c32 100644 --- a/ext/bg/js/settings.js +++ b/ext/bg/js/settings.js @@ -66,6 +66,7 @@ async function formRead(options) { options.parsing.enableScanningParser = $('#parsing-scan-enable').prop('checked'); options.parsing.enableMecabParser = $('#parsing-mecab-enable').prop('checked'); + options.parsing.readingMode = $('#parsing-reading-mode').val(); const optionsAnkiEnableOld = options.anki.enable; options.anki.enable = $('#anki-enable').prop('checked'); @@ -131,6 +132,7 @@ async function formWrite(options) { $('#parsing-scan-enable').prop('checked', options.parsing.enableScanningParser); $('#parsing-mecab-enable').prop('checked', options.parsing.enableMecabParser); + $('#parsing-reading-mode').val(options.parsing.readingMode); $('#anki-enable').prop('checked', options.anki.enable); $('#card-tags').val(options.anki.tags.join(' ')); diff --git a/ext/bg/settings.html b/ext/bg/settings.html index 08b9b6c1..0badb817 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -437,6 +437,15 @@
+ +
+ + +
diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index e2d7a090..a7cd0452 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -48,6 +48,43 @@ function jpKatakanaToHiragana(text) { return result; } +function jpHiraganaToKatakana(text) { + let result = ''; + for (const c of text) { + if (wanakana.isHiragana(c)) { + result += wanakana.toKatakana(c); + } else { + result += c; + } + } + + return result; +} + +function jpToRomaji(text) { + return wanakana.toRomaji(text); +} + +function jpConvertReading(expressionFragment, readingFragment, readingMode) { + switch (readingMode) { + case 'hiragana': + return jpKatakanaToHiragana(readingFragment || ''); + case 'katakana': + return jpHiraganaToKatakana(readingFragment || ''); + case 'romaji': + if (readingFragment) { + return jpToRomaji(readingFragment); + } else { + if (jpIsKana(expressionFragment)) { + return jpToRomaji(expressionFragment); + } + } + return readingFragment; + default: + return readingFragment; + } +} + function jpDistributeFurigana(expression, reading) { const fallback = [{furigana: reading, text: expression}]; if (!reading) { From 2577d4054e344d2e6dc08d2cf95e83478c05bee6 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sat, 23 Nov 2019 18:08:22 +0200 Subject: [PATCH 33/36] fix scanning parser --- ext/bg/js/api.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ext/bg/js/api.js b/ext/bg/js/api.js index 228447c3..766fb0ed 100644 --- a/ext/bg/js/api.js +++ b/ext/bg/js/api.js @@ -86,7 +86,12 @@ async function apiTextParse(text, optionsContext) { const results = []; while (text.length > 0) { const term = []; - const [definitions, sourceLength] = await translator.findTerms(text, {}, options); + const [definitions, sourceLength] = await translator.findTermsInternal( + text.slice(0, options.scanning.length), + dictEnabledSet(options), + options.scanning.alphanumeric, + {} + ); if (definitions.length > 0) { dictTermsSort(definitions); const {expression, reading} = definitions[0]; From 7bf2c8048d82c62dafdfa6a6866ec63e7e95a56b Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sat, 23 Nov 2019 19:18:29 +0200 Subject: [PATCH 34/36] add mecab version check --- ext/bg/js/mecab.js | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index ada96945..a26b7f39 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -24,10 +24,23 @@ class Mecab { this.sequence = 0; } - async parseText(text) { - if (this.port === null) { - return {}; + onError(error) { + logError(error, true); + } + + async checkVersion() { + try { + const {version} = await this.invoke('get_version', {}); + if (version !== Mecab.version) { + this.stopListener(); + throw new Error(`Unsupported MeCab native messenger version ${version}. Yomichan supports version ${Mecab.version}.`); + } + } catch (error) { + this.onError(error); } + } + + async parseText(text) { return await this.invoke('parse_text', {text}); } @@ -35,6 +48,7 @@ class Mecab { if (this.port !== null) { return; } this.port = chrome.runtime.connectNative('yomichan_mecab'); this.port.onMessage.addListener(this.onNativeMessage.bind(this)); + this.checkVersion(); } stopListener() { @@ -55,6 +69,9 @@ class Mecab { } invoke(action, params) { + if (this.port === null) { + return {}; + } return new Promise((resolve, reject) => { const sequence = this.sequence++; @@ -72,3 +89,4 @@ class Mecab { } Mecab.timeout = 5000; +Mecab.version = 1; From 43fad608fb5833f95c215dd7596bac5a821dbe60 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sat, 23 Nov 2019 19:25:11 +0200 Subject: [PATCH 35/36] remove popup from background page Not supported on all browsers --- ext/bg/js/mecab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index a26b7f39..4983f656 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -25,7 +25,7 @@ class Mecab { } onError(error) { - logError(error, true); + logError(error, false); } async checkVersion() { From 89c6ef54b0a44685cde530dd4a94405a578ce132 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 24 Nov 2019 02:34:16 +0200 Subject: [PATCH 36/36] always return a promise from Mecab.invoke --- ext/bg/js/mecab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index 4983f656..246f8bba 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -70,7 +70,7 @@ class Mecab { invoke(action, params) { if (this.port === null) { - return {}; + return Promise.resolve({}); } return new Promise((resolve, reject) => { const sequence = this.sequence++;