From d31bff5700cd50a44077984b0413a2ebd5eb378c Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Sat, 16 Nov 2013 11:21:35 -0800 Subject: [PATCH] Adding imporved support for searching using wildcards Former-commit-id: 95d392046cc4d703fee433afabbd4ba3d04fad36 --- ui/reader.ui | 4 +-- yomi_base/gen/about_ui.py | 2 +- yomi_base/gen/preferences_ui.py | 2 +- yomi_base/gen/reader_ui.py | 6 ++-- yomi_base/gen/resources_rc.py | 2 +- yomi_base/japanese/dictionary.py | 4 +-- yomi_base/japanese/translate.py | 14 ++++---- yomi_base/japanese/util.py | 61 ++++++++++++++++++++++++++++++++ yomi_base/reader_util.py | 7 ++-- 9 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 yomi_base/japanese/util.py diff --git a/ui/reader.ui b/ui/reader.ui index 9ab5f9e..97380a4 100644 --- a/ui/reader.ui +++ b/ui/reader.ui @@ -159,7 +159,7 @@ - Search + Expression @@ -235,7 +235,7 @@ - Search + Character diff --git a/yomi_base/gen/about_ui.py b/yomi_base/gen/about_ui.py index 02f4ac0..d3b643e 100644 --- a/yomi_base/gen/about_ui.py +++ b/yomi_base/gen/about_ui.py @@ -2,7 +2,7 @@ # Form implementation generated from reading ui file 'ui/about.ui' # -# Created: Thu Nov 14 09:02:10 2013 +# Created: Fri Nov 15 20:29:12 2013 # by: PyQt4 UI code generator 4.10 # # WARNING! All changes made in this file will be lost! diff --git a/yomi_base/gen/preferences_ui.py b/yomi_base/gen/preferences_ui.py index 064db8e..b1f1866 100644 --- a/yomi_base/gen/preferences_ui.py +++ b/yomi_base/gen/preferences_ui.py @@ -2,7 +2,7 @@ # Form implementation generated from reading ui file 'ui/preferences.ui' # -# Created: Thu Nov 14 09:02:10 2013 +# Created: Fri Nov 15 20:29:12 2013 # by: PyQt4 UI code generator 4.10 # # WARNING! All changes made in this file will be lost! diff --git a/yomi_base/gen/reader_ui.py b/yomi_base/gen/reader_ui.py index 0b8610a..4d968e0 100644 --- a/yomi_base/gen/reader_ui.py +++ b/yomi_base/gen/reader_ui.py @@ -2,7 +2,7 @@ # Form implementation generated from reading ui file 'ui/reader.ui' # -# Created: Thu Nov 14 09:02:10 2013 +# Created: Fri Nov 15 20:29:13 2013 # by: PyQt4 UI code generator 4.10 # # WARNING! All changes made in this file will be lost! @@ -266,11 +266,11 @@ class Ui_MainWindowReader(object): self.menuTextSize.setTitle(_translate("MainWindowReader", "&Zoom", None)) self.toolBar.setWindowTitle(_translate("MainWindowReader", "toolBar", None)) self.dockVocab.setWindowTitle(_translate("MainWindowReader", "Vocabulary", None)) - self.label.setText(_translate("MainWindowReader", "Search", None)) + self.label.setText(_translate("MainWindowReader", "Expression", None)) self.dockAnki.setWindowTitle(_translate("MainWindowReader", "Anki", None)) self.label_3.setText(_translate("MainWindowReader", "Active tag(s)", None)) self.dockKanji.setWindowTitle(_translate("MainWindowReader", "Kanji", None)) - self.label_2.setText(_translate("MainWindowReader", "Search", None)) + self.label_2.setText(_translate("MainWindowReader", "Character", None)) self.actionOpen.setText(_translate("MainWindowReader", "&Open...", None)) self.actionOpen.setToolTip(_translate("MainWindowReader", "Open file", None)) self.actionOpen.setShortcut(_translate("MainWindowReader", "Ctrl+O", None)) diff --git a/yomi_base/gen/resources_rc.py b/yomi_base/gen/resources_rc.py index 02db60f..5692c47 100644 --- a/yomi_base/gen/resources_rc.py +++ b/yomi_base/gen/resources_rc.py @@ -2,7 +2,7 @@ # Resource object code # -# Created: Thu Nov 14 09:02:10 2013 +# Created: Fri Nov 15 20:29:13 2013 # by: The Resource Compiler for PyQt (Qt v4.8.4) # # WARNING! All changes made in this file will be lost! diff --git a/yomi_base/japanese/dictionary.py b/yomi_base/japanese/dictionary.py index 08a5f1e..d7a9966 100644 --- a/yomi_base/japanese/dictionary.py +++ b/yomi_base/japanese/dictionary.py @@ -26,12 +26,12 @@ class Dictionary: self.indices = set() - def findTerm(self, word, partial=False): + def findTerm(self, word, wildcards=False): self.requireIndex('Terms', 'expression') self.requireIndex('Terms', 'reading') cursor = self.db.cursor() - cursor.execute('SELECT * FROM Terms WHERE expression {0} ? OR reading=? LIMIT 100'.format('LIKE' if partial else '='), (word, word)) + cursor.execute('SELECT * FROM Terms WHERE expression {0} ? OR reading=? LIMIT 100'.format('LIKE' if wildcards else '='), (word, word)) results = list() for expression, reading, glossary, tags in cursor.fetchall(): diff --git a/yomi_base/japanese/translate.py b/yomi_base/japanese/translate.py index 0bbdc50..202706d 100644 --- a/yomi_base/japanese/translate.py +++ b/yomi_base/japanese/translate.py @@ -18,6 +18,7 @@ import operator +import util class Translator: @@ -26,15 +27,15 @@ class Translator: self.dictionary = dictionary - def findTerm(self, text, partial=False): - groups = dict() + def findTerm(self, text, wildcards=False): + text = util.sanitize(text, wildcards=wildcards) + groups = dict() for i in xrange(len(text), 0, -1): term = text[:i] - deinflections = self.deinflector.deinflect(term, self.validator) if deinflections is None: - self.processTerm(groups, term, partial=partial) + self.processTerm(groups, term, wildcards=wildcards) else: for deinflection in deinflections: self.processTerm(groups, **deinflection) @@ -51,6 +52,7 @@ class Translator: def findCharacters(self, text): + text = util.sanitize(text, kana=False) results = list() processed = dict() @@ -64,10 +66,10 @@ class Translator: return results - def processTerm(self, groups, source, rules=list(), root=str(), partial=False): + def processTerm(self, groups, source, rules=list(), root=str(), wildcards=False): root = root or source - for entry in self.dictionary.findTerm(root, partial): + for entry in self.dictionary.findTerm(root, wildcards): key = entry['expression'], entry['reading'], entry['glossary'] if key not in groups: groups[key] = entry['tags'], source, rules diff --git a/yomi_base/japanese/util.py b/yomi_base/japanese/util.py new file mode 100644 index 0000000..6285f86 --- /dev/null +++ b/yomi_base/japanese/util.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2013 Alex Yatskov +# This module is based on Rikaichan code written by Jonathan Zarate +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import re + + +def isHiragana(c): + return 0x3040 <= ord(c) < 0x30a0 + + +def isKatakana(c): + return 0x30a0 <= ord(c) < 0x3100 + + +def isKana(c): + return isHiragana(c) or isKatakana(c) + + +def isKanji(c): + return 0x4e00 <= ord(c) < 0x9fb0 or 0x3400 <= ord(c) < 0x4dc0 + + +def isJapanese(c): + return isKana(c) or isKanji(c) + + +def sanitize(text, kana=True, wildcards=False): + if kana: + checker = isJapanese + else: + checker = isKanji + + if wildcards: + text = re.sub(u'[\*＊]', u'%', text) + text = re.sub(u'[\?？]', u'_', text) + overrides = [u'%', u'_'] + else: + overrides = list() + + result = unicode() + for c in text: + if checker(c) or c in overrides: + result += c + + return result diff --git a/yomi_base/reader_util.py b/yomi_base/reader_util.py index a92bff5..5cbfcb0 100644 --- a/yomi_base/reader_util.py +++ b/yomi_base/reader_util.py @@ -188,7 +188,7 @@ def buildVocabDefs(definitions, query): html += """

No definitions to display.

Mouse over text with the middle mouse button or shift key pressed to search.

-

You can also also input terms in the search box below.

""" +

You can also also input terms in the search box below, using the "*" and "?" wildcards where needed.

""" return html + buildDefFooter() @@ -216,6 +216,9 @@ def buildKanjiDefs(definitions, query): for i, definition in enumerate(definitions): html += buildKanjiDef(definition, i, query) else: - html += '

No definitions to display.

' + html += """ +

No definitions to display.

+

Mouse over text with the middle mouse button or shift key pressed to search.

+

You can also also input terms in the search box below.""" return html + buildDefFooter()