/* * Copyright (c) 2016 Alex Yatskov * Author: Alex Yatskov * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package main import ( "encoding/json" "fmt" "os" "path" "strconv" "strings" ) type dbTagMeta struct { Notes string `json:"notes"` Class string `json:"class"` Order int `json:"order"` } type dbTerm struct { Expression string Reading string Tags []string Rules []string Score int Glossary []string } type dbTermList []dbTerm func (term *dbTerm) addTags(tags ...string) { term.Tags = appendStringUnique(term.Tags, tags...) } func (term *dbTerm) addRules(rules ...string) { term.Rules = appendStringUnique(term.Rules, rules...) } func (terms dbTermList) crush() [][]string { var results [][]string for _, t := range terms { result := []string{ t.Expression, t.Reading, strings.Join(t.Tags, " "), strings.Join(t.Rules, " "), strconv.Itoa(t.Score), } result = append(result, t.Glossary...) results = append(results, result) } return results } type dbKanji struct { Character string Onyomi []string Kunyomi []string Tags []string Meanings []string } type dbKanjiList []dbKanji func (kanji *dbKanji) addTags(tags ...string) { for _, tag := range tags { if !hasString(tag, kanji.Tags) { kanji.Tags = append(kanji.Tags, tag) } } } func (kanji dbKanjiList) crush() [][]string { var results [][]string for _, k := range kanji { result := []string{ k.Character, strings.Join(k.Onyomi, " "), strings.Join(k.Kunyomi, " "), strings.Join(k.Tags, " "), } result = append(result, k.Meanings...) results = append(results, result) } return results } func writeDb(outputDir, title string, termRecords [][]string, kanjiRecords [][]string, tagMeta map[string]dbTagMeta, pretty bool) error { const DB_VERSION = 1 const BANK_STRIDE = 50000 marshalJson := func(obj interface{}, pretty bool) ([]byte, error) { if pretty { return json.MarshalIndent(obj, "", " ") } return json.Marshal(obj) } writeDbRecords := func(prefix string, records [][]string) (int, error) { recordCount := len(records) bankCount := 0 for i := 0; i < recordCount; i += BANK_STRIDE { indexSrc := i indexDst := i + BANK_STRIDE if indexDst > recordCount { indexDst = recordCount } bytes, err := marshalJson(records[indexSrc:indexDst], pretty) if err != nil { return 0, err } fp, err := os.Create(path.Join(outputDir, fmt.Sprintf("%s_bank_%d.json", prefix, i/BANK_STRIDE+1))) if err != nil { return 0, err } defer fp.Close() if _, err = fp.Write(bytes); err != nil { return 0, err } bankCount += 1 } return bankCount, nil } if err := os.MkdirAll(outputDir, 0755); err != nil { return err } var err error var db struct { Title string `json:"title"` Version int `json:"version"` TagMeta map[string]dbTagMeta `json:"tagMeta"` TermBanks int `json:"termBanks"` KanjiBanks int `json:"kanjiBanks"` } db.Title = title db.Version = DB_VERSION db.TagMeta = tagMeta if db.TermBanks, err = writeDbRecords("term", termRecords); err != nil { return err } if db.KanjiBanks, err = writeDbRecords("kanji", kanjiRecords); err != nil { return err } bytes, err := marshalJson(db, pretty) if err != nil { return err } fp, err := os.Create(path.Join(outputDir, "index.json")) if err != nil { return err } defer fp.Close() if _, err := fp.Write(bytes); err != nil { return err } return nil } func appendStringUnique(target []string, source ...string) []string { for _, str := range source { if !hasString(str, target) { target = append(target, str) } } return target } func hasString(needle string, haystack []string) bool { for _, value := range haystack { if needle == value { return true } } return false }