/* * Copyright (c) 2016 Alex Yatskov * Author: Alex Yatskov * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package main import ( "encoding/json" "io" "strconv" "strings" "github.com/FooSoft/jmdict" ) // // Edict and Enamdict processing // type vocabDictJson struct { Indices map[string]string `json:"i"` Entities map[string]string `json:"e"` Defs [][]string `json:"d"` } type vocabDictSource struct { Expression string Reading string Tags []string Glossary []string } func (d *vocabDictSource) addTags(tags []string) { for _, tag := range tags { if findString(tag, d.Tags) == -1 { d.Tags = append(d.Tags, tag) } } } func appendIndex(indices map[string]string, key string, value int) { def, _ := indices[key] if len(def) > 0 { def += " " } def += strconv.Itoa(value) indices[key] = def } func buildVocabDictJson(entries []vocabDictSource, entities map[string]string) vocabDictJson { dict := vocabDictJson{ Indices: make(map[string]string), Entities: entities, } for i, e := range entries { entry := []string{e.Expression, e.Reading, strings.Join(e.Tags, " ")} entry = append(entry, e.Glossary...) dict.Defs = append(dict.Defs, entry) appendIndex(dict.Indices, e.Expression, i) if len(e.Reading) > 0 { appendIndex(dict.Indices, e.Reading, i) } } return dict } func outputVocabDictJson(writer io.Writer, entries []vocabDictSource, entities map[string]string, pretty bool) error { dict := buildVocabDictJson(entries, entities) var ( bytes []byte err error ) if pretty { bytes, err = json.MarshalIndent(dict, "", " ") } else { bytes, err = json.Marshal(dict) } if err != nil { return err } _, err = writer.Write(bytes) return err } func findString(needle string, haystack []string) int { for index, value := range haystack { if needle == value { return index } } return -1 } func convertEnamdictEntry(enamdictEntry jmdict.EnamdictEntry) []vocabDictSource { var entries []vocabDictSource convert := func(reading jmdict.EnamdictReading, kanji *jmdict.EnamdictKanji) { if kanji != nil && findString(kanji.Expression, reading.Restrictions) != -1 { return } var entry vocabDictSource if kanji == nil { entry.Expression = reading.Reading } else { entry.Expression = kanji.Expression entry.Reading = reading.Reading entry.addTags(kanji.Information) entry.addTags(kanji.Priorities) } entry.addTags(reading.Information) entry.addTags(reading.Priorities) for _, trans := range enamdictEntry.Translations { entry.Glossary = append(entry.Glossary, trans.Translations...) entry.addTags(trans.NameTypes) } entries = append(entries, entry) } if len(enamdictEntry.Kanji) > 0 { for _, kanji := range enamdictEntry.Kanji { for _, reading := range enamdictEntry.Readings { convert(reading, &kanji) } } } else { for _, reading := range enamdictEntry.Readings { convert(reading, nil) } } return entries } func convertEdictEntry(edictEntry jmdict.EdictEntry) []vocabDictSource { var entries []vocabDictSource convert := func(reading jmdict.EdictReading, kanji *jmdict.EdictKanji) { if kanji != nil && findString(kanji.Expression, reading.Restrictions) != -1 { return } var entry vocabDictSource if kanji == nil { entry.Expression = reading.Reading } else { entry.Expression = kanji.Expression entry.Reading = reading.Reading entry.addTags(kanji.Information) entry.addTags(kanji.Priorities) } entry.addTags(reading.Information) entry.addTags(reading.Priorities) for _, sense := range edictEntry.Sense { if findString(reading.Reading, sense.RestrictedReadings) != -1 { continue } if kanji != nil && findString(kanji.Expression, sense.RestrictedKanji) != -1 { continue } for _, glossary := range sense.Glossary { entry.Glossary = append(entry.Glossary, glossary.Content) } entry.addTags(sense.PartsOfSpeech) entry.addTags(sense.Fields) entry.addTags(sense.Misc) entry.addTags(sense.Dialects) } entries = append(entries, entry) } if len(edictEntry.Kanji) > 0 { for _, kanji := range edictEntry.Kanji { for _, reading := range edictEntry.Readings { convert(reading, &kanji) } } } else { for _, reading := range edictEntry.Readings { convert(reading, nil) } } return entries } func processEnamdict(writer io.Writer, reader io.Reader, flags int) error { enamdictEntries, entities, err := jmdict.LoadEnamdict(reader, false) if err != nil { return err } var entries []vocabDictSource for _, enamdictEntry := range enamdictEntries { entries = append(entries, convertEnamdictEntry(enamdictEntry)...) } return outputVocabDictJson(writer, entries, entities, flags&flagPrettyJson == flagPrettyJson) } func processEdict(writer io.Writer, reader io.Reader, flags int) error { edictEntries, entities, err := jmdict.LoadEdict(reader, false) if err != nil { return err } var entries []vocabDictSource for _, edictEntry := range edictEntries { entries = append(entries, convertEdictEntry(edictEntry)...) } return outputVocabDictJson(writer, entries, entities, flags&flagPrettyJson == flagPrettyJson) } // // Kanjidic processing // type characterDictJson struct { Characters map[string][]string `json:"c"` } type characterDictSource struct { Character string Kunyomi []string Onyomi []string Meanings []string } func buildCharacterDictJson(characters []characterDictSource) characterDictJson { dict := characterDictJson{make(map[string][]string)} for _, c := range characters { var params []string params = append(params, strings.Join(c.Onyomi, " ")) params = append(params, strings.Join(c.Kunyomi, " ")) params = append(params, c.Meanings...) dict.Characters[c.Character] = params } return dict } func outputCharacterDictJson(writer io.Writer, characters []characterDictSource, pretty bool) error { dict := buildCharacterDictJson(characters) var ( bytes []byte err error ) if pretty { bytes, err = json.MarshalIndent(dict, "", " ") } else { bytes, err = json.Marshal(dict) } if err != nil { return err } _, err = writer.Write(bytes) return err } func convertKanjidicCharacter(kanjidicCharacter jmdict.KanjidicCharacter) characterDictSource { character := characterDictSource{Character: kanjidicCharacter.Literal} if kanjidicCharacter.ReadingMeaning != nil { for _, m := range kanjidicCharacter.ReadingMeaning.Meanings { if m.Language == "en" || m.Language == "" { character.Meanings = append(character.Meanings, m.Meaning) } } for _, r := range kanjidicCharacter.ReadingMeaning.Readings { switch r.Type { case "ja_on": character.Onyomi = append(character.Onyomi, r.Value) break case "ja_kun": character.Kunyomi = append(character.Kunyomi, r.Value) break } } } return character } func processKanjidic(writer io.Writer, reader io.Reader, flags int) error { kanjidic, err := jmdict.LoadKanjidic(reader) if err != nil { return err } var characters []characterDictSource for _, kanjidicCharacter := range kanjidic.Characters { characters = append(characters, convertKanjidicCharacter(kanjidicCharacter)) } return outputCharacterDictJson(writer, characters, flags&flagPrettyJson == flagPrettyJson) }