Cleanup

common.go

@@ -30,129 +30,157 @@ import (
 	"strings"
 )
 
-const (
+type dbTerm struct {
-	BANK_STRIDE = 50000
-	DB_VERSION  = 0
-)
-
-type termIndex struct {
-	Version  int        `json:"version"`
-	Banks    int        `json:"banks"`
-	Entities [][]string `json:"ents"`
-	defs     [][]string
-}
-
-type termSource struct {
 	Expression string
 	Reading    string
 	Tags       []string
 	Glossary   []string
 }
 
-func (s *termSource) addTags(tags ...string) {
+type dbTermList []dbTerm
+
+func (term *dbTerm) addTags(tags ...string) {
 	for _, tag := range tags {
-		if !hasString(tag, s.Tags) {
+		if !hasString(tag, term.Tags) {
-			s.Tags = append(s.Tags, tag)
+			term.Tags = append(term.Tags, tag)
 		}
 	}
 }
 
-func (s *termSource) addTagsPri(tags ...string) {
+func (term *dbTerm) addTagsPri(tags ...string) {
 	for _, tag := range tags {
 		switch tag {
 		case "news1", "ichi1", "spec1", "gai1":
-			s.addTags("P")
+			term.addTags("P")
 			fallthrough
 		case "news2", "ichi2", "spec2", "gai2":
-			s.addTags(tag[:len(tag)-1])
+			term.addTags(tag[:len(tag)-1])
 			break
 		}
 	}
 }
 
-func buildTermIndex(entries []termSource, entities map[string]string) termIndex {
+func (terms dbTermList) crush() [][]string {
-	dict := termIndex{
+	var results [][]string
-		Version: DB_VERSION,
+	for _, t := range terms {
-		Banks:   bankCount(len(entries)),
+		result := []string{
+			t.Expression,
+			t.Reading,
+			strings.Join(t.Tags, " "),
 		}
 
-	for _, e := range entries {
+		result = append(result, t.Glossary...)
-		def := []string{e.Expression, e.Reading, strings.Join(e.Tags, " ")}
+		results = append(results, result)
-		def = append(def, e.Glossary...)
-		dict.defs = append(dict.defs, def)
 	}
 
-	for name, value := range entities {
+	return results
-		ent := []string{name, value}
-		dict.Entities = append(dict.Entities, ent)
-	}
-
-	return dict
 }
 
-func outputTermIndex(outputDir string, entries []termSource, entities map[string]string, pretty bool) error {
+type dbKanji struct {
-	if err := os.MkdirAll(outputDir, 0755); err != nil {
+	Character string
-		return err
+	Onyomi    []string
-	}
+	Kunyomi   []string
-
+	Tags      []string
-	outputIndex, err := os.Create(path.Join(outputDir, "index.json"))
+	Meanings  []string
-	if err != nil {
-		return err
-	}
-	defer outputIndex.Close()
-
-	dict := buildTermIndex(entries, entities)
-	indexBytes, err := marshalJson(dict, pretty)
-	if err != nil {
-		return err
-	}
-
-	if _, err = outputIndex.Write(indexBytes); err != nil {
-		return err
-	}
-
-	defCnt := len(dict.defs)
-	for i := 0; i < defCnt; i += BANK_STRIDE {
-		outputRef, err := os.Create(path.Join(outputDir, fmt.Sprintf("bank_%d.json", i/BANK_STRIDE+1)))
-		if err != nil {
-			return err
-		}
-		defer outputRef.Close()
-
-		indexSrc := i
-		indexDst := i + BANK_STRIDE
-		if indexDst > defCnt {
-			indexDst = defCnt
-		}
-
-		refBytes, err := marshalJson(dict.defs[indexSrc:indexDst], pretty)
-		if err != nil {
-			return err
-		}
-
-		if _, err = outputRef.Write(refBytes); err != nil {
-			return err
-		}
-	}
-
-	return nil
 }
 
-func marshalJson(obj interface{}, pretty bool) ([]byte, error) {
+type dbKanjiList []dbKanji
+
+func (kanji *dbKanji) addTags(tags ...string) {
+	for _, tag := range tags {
+		if !hasString(tag, kanji.Tags) {
+			kanji.Tags = append(kanji.Tags, tag)
+		}
+	}
+}
+
+func (kanji dbKanjiList) crush() [][]string {
+	var results [][]string
+	for _, k := range kanji {
+		result := []string{
+			k.Character,
+			strings.Join(k.Onyomi, " "),
+			strings.Join(k.Kunyomi, " "),
+			strings.Join(k.Tags, " "),
+		}
+
+		result = append(result, k.Meanings...)
+		results = append(results, result)
+	}
+
+	return results
+}
+
+func writeDb(outputDir string, records [][]string, entities map[string]string, pretty bool) error {
+	const DB_VERSION = 1
+	const BANK_STRIDE = 50000
+
+	marshalJson := func(obj interface{}, pretty bool) ([]byte, error) {
 		if pretty {
 			return json.MarshalIndent(obj, "", "    ")
 		}
 
 		return json.Marshal(obj)
-}
-
-func bankCount(defCount int) int {
-	count := defCount / BANK_STRIDE
-	if defCount%BANK_STRIDE > 0 {
-		count += 1
 	}
 
-	return count
+	var db struct {
+		Version  int               `json:"version"`
+		Banks    int               `json:"banks"`
+		Entities map[string]string `json:"entities"`
+	}
+
+	recordCount := len(records)
+
+	db.Version = 0
+	db.Entities = entities
+	db.Banks = recordCount / BANK_STRIDE
+	if recordCount%BANK_STRIDE > 0 {
+		db.Banks += 1
+	}
+
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		return err
+	}
+
+	bytes, err := marshalJson(db, pretty)
+	if err != nil {
+		return err
+	}
+
+	fp, err := os.Create(path.Join(outputDir, "index.json"))
+	if err != nil {
+		return err
+	}
+	defer fp.Close()
+
+	if _, err = fp.Write(bytes); err != nil {
+		return err
+	}
+
+	for i := 0; i < recordCount; i += BANK_STRIDE {
+		indexSrc := i
+		indexDst := i + BANK_STRIDE
+		if indexDst > recordCount {
+			indexDst = recordCount
+		}
+
+		bytes, err := marshalJson(records[indexSrc:indexDst], pretty)
+		if err != nil {
+			return err
+		}
+
+		fp, err := os.Create(path.Join(outputDir, fmt.Sprintf("bank_%d.json", i/BANK_STRIDE+1)))
+		if err != nil {
+			return err
+		}
+		defer fp.Close()
+
+		if _, err = fp.Write(bytes); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func hasString(needle string, haystack []string) bool {

edict.go

@@ -28,25 +28,30 @@ import (
 	"github.com/FooSoft/jmdict"
 )
 
-func convertEdictEntry(edictEntry jmdict.JmdictEntry) []termSource {
+func extractJmdictTerms(edictEntry jmdict.JmdictEntry) []dbTerm {
-	var entries []termSource
+	var terms []dbTerm
 
 	convert := func(reading jmdict.JmdictReading, kanji *jmdict.JmdictKanji) {
 		if kanji != nil && reading.Restrictions != nil && !hasString(kanji.Expression, reading.Restrictions) {
 			return
 		}
 
-		var entryBase termSource
+		var termBase dbTerm
-		entryBase.addTags(reading.Information...)
+		termBase.addTags(reading.Information...)
-		entryBase.addTagsPri(reading.Priorities...)
 
 		if kanji == nil {
-			entryBase.Expression = reading.Reading
+			termBase.Expression = reading.Reading
+			termBase.addTagsPri(reading.Priorities...)
 		} else {
-			entryBase.Expression = kanji.Expression
+			termBase.Expression = kanji.Expression
-			entryBase.Reading = reading.Reading
+			termBase.Reading = reading.Reading
-			entryBase.addTags(kanji.Information...)
+			termBase.addTags(kanji.Information...)
-			entryBase.addTagsPri(kanji.Priorities...)
+
+			for _, priority := range kanji.Priorities {
+				if hasString(priority, reading.Priorities) {
+					termBase.addTagsPri(priority)
+				}
+			}
 		}
 
 		for _, sense := range edictEntry.Sense {
@@ -58,23 +63,18 @@ func convertEdictEntry(edictEntry jmdict.JmdictEntry) []termSource {
 				continue
 			}
 
-			entry := termSource{
+			term := dbTerm{Reading: termBase.Reading, Expression: termBase.Expression}
-				Reading:    entryBase.Reading,
+			term.addTags(termBase.Tags...)
-				Expression: entryBase.Expression,
+			term.addTags(sense.PartsOfSpeech...)
-			}
+			term.addTags(sense.Fields...)
-
+			term.addTags(sense.Misc...)
-			entry.addTags(entryBase.Tags...)
+			term.addTags(sense.Dialects...)
 
 			for _, glossary := range sense.Glossary {
-				entry.Glossary = append(entry.Glossary, glossary.Content)
+				term.Glossary = append(term.Glossary, glossary.Content)
 			}
 
-			entry.addTags(sense.PartsOfSpeech...)
+			terms = append(terms, term)
-			entry.addTags(sense.Fields...)
-			entry.addTags(sense.Misc...)
-			entry.addTags(sense.Dialects...)
-
-			entries = append(entries, entry)
 		}
 	}
 
@@ -90,19 +90,24 @@ func convertEdictEntry(edictEntry jmdict.JmdictEntry) []termSource {
 		}
 	}
 
-	return entries
+	return terms
 }
 
-func outputEdictJson(outputDir string, reader io.Reader, flags int) error {
+func exportJmdictDb(outputDir string, reader io.Reader, flags int) error {
 	dict, entities, err := jmdict.LoadJmdictNoTransform(reader)
 	if err != nil {
 		return err
 	}
 
-	var entries []termSource
+	var terms dbTermList
-	for _, e := range dict.Entries {
+	for _, entry := range dict.Entries {
-		entries = append(entries, convertEdictEntry(e)...)
+		terms = append(terms, extractJmdictTerms(entry)...)
 	}
 
-	return outputTermIndex(outputDir, entries, entities, flags&flagPrettyJson == flagPrettyJson)
+	return writeDb(
+		outputDir,
+		terms.crush(),
+		entities,
+		flags&flagPrettyJson == flagPrettyJson,
+	)
 }

enamdict.go

@@ -28,34 +28,38 @@ import (
 	"github.com/FooSoft/jmdict"
 )
 
-func convertJmnedictEntry(enamdictEntry jmdict.JmnedictEntry) []termSource {
+func extractJmnedictTerms(enamdictEntry jmdict.JmnedictEntry) []dbTerm {
-	var entries []termSource
+	var terms []dbTerm
 
 	convert := func(reading jmdict.JmnedictReading, kanji *jmdict.JmnedictKanji) {
 		if kanji != nil && hasString(kanji.Expression, reading.Restrictions) {
 			return
 		}
 
-		var entry termSource
+		var term dbTerm
+		term.addTags(reading.Information...)
+
 		if kanji == nil {
-			entry.Expression = reading.Reading
+			term.Expression = reading.Reading
+			term.addTags(reading.Information...)
 		} else {
-			entry.Expression = kanji.Expression
+			term.Expression = kanji.Expression
-			entry.Reading = reading.Reading
+			term.Reading = reading.Reading
+			term.addTags(kanji.Information...)
 
-			entry.addTags(kanji.Information...)
+			for _, priority := range kanji.Priorities {
-			entry.addTagsPri(kanji.Priorities...)
+				if hasString(priority, reading.Priorities) {
+					term.addTagsPri(priority)
+				}
+			}
 		}
-
-		entry.addTags(reading.Information...)
-		entry.addTagsPri(reading.Priorities...)
 
 		for _, trans := range enamdictEntry.Translations {
-			entry.Glossary = append(entry.Glossary, trans.Translations...)
+			term.Glossary = append(term.Glossary, trans.Translations...)
-			entry.addTags(trans.NameTypes...)
+			term.addTags(trans.NameTypes...)
 		}
 
-		entries = append(entries, entry)
+		terms = append(terms, term)
 	}
 
 	if len(enamdictEntry.Kanji) > 0 {
@@ -70,19 +74,24 @@ func convertJmnedictEntry(enamdictEntry jmdict.JmnedictEntry) []termSource {
 		}
 	}
 
-	return entries
+	return terms
 }
 
-func outputJmnedictJson(outputDir string, reader io.Reader, flags int) error {
+func exportJmnedictDb(outputDir string, reader io.Reader, flags int) error {
 	dict, entities, err := jmdict.LoadJmnedictNoTransform(reader)
 	if err != nil {
 		return err
 	}
 
-	var entries []termSource
+	var terms dbTermList
 	for _, e := range dict.Entries {
-		entries = append(entries, convertJmnedictEntry(e)...)
+		terms = append(terms, extractJmnedictTerms(e)...)
 	}
 
-	return outputTermIndex(outputDir, entries, entities, flags&flagPrettyJson == flagPrettyJson)
+	return writeDb(
+		outputDir,
+		terms.crush(),
+		entities,
+		flags&flagPrettyJson == flagPrettyJson,
+	)
 }

kanjidic.go

@@ -25,159 +25,76 @@ package main
 import (
 	"fmt"
 	"io"
-	"os"
-	"path"
 	"strconv"
-	"strings"
 
 	"github.com/FooSoft/jmdict"
 )
 
-type kanjiIndex struct {
+func extractKanjidicKanji(entry jmdict.KanjidicCharacter) dbKanji {
-	Version int `json:"version"`
+	kanji := dbKanji{Character: entry.Literal}
-	Banks   int `json:"banks"`
-	defs    [][]string
-}
 
-type kanjiSource struct {
+	if level := entry.Misc.JlptLevel; level != nil {
-	Character string
+		kanji.addTags(fmt.Sprintf("jlpt:%s", *level))
-	Onyomi    []string
-	Kunyomi   []string
-	Tags      []string
-	Meanings  []string
-}
-
-func (s *kanjiSource) addTags(tags ...string) {
-	for _, tag := range tags {
-		if !hasString(tag, s.Tags) {
-			s.Tags = append(s.Tags, tag)
-		}
-	}
-}
-
-func buildKanjiIndex(entries []kanjiSource) kanjiIndex {
-	dict := kanjiIndex{
-		Version: DB_VERSION,
-		Banks:   bankCount(len(entries)),
 	}
 
-	for _, e := range entries {
+	if grade := entry.Misc.Grade; grade != nil {
-		def := []string{e.Character, strings.Join(e.Onyomi, " "), strings.Join(e.Kunyomi, " "), strings.Join(e.Tags, " ")}
+		kanji.addTags(fmt.Sprintf("grade:%s", *grade))
-		def = append(def, e.Meanings...)
-		dict.defs = append(dict.defs, def)
-	}
-
-	return dict
-}
-
-func outputKanjiIndex(outputDir string, entries []kanjiSource, pretty bool) error {
-	if err := os.MkdirAll(outputDir, 0755); err != nil {
-		return err
-	}
-
-	outputIndex, err := os.Create(path.Join(outputDir, "index.json"))
-	if err != nil {
-		return err
-	}
-	defer outputIndex.Close()
-
-	dict := buildKanjiIndex(entries)
-	indexBytes, err := marshalJson(dict, pretty)
-	if err != nil {
-		return err
-	}
-
-	if _, err = outputIndex.Write(indexBytes); err != nil {
-		return err
-	}
-
-	defCnt := len(dict.defs)
-	for i := 0; i < defCnt; i += BANK_STRIDE {
-		outputRef, err := os.Create(path.Join(outputDir, fmt.Sprintf("bank_%d.json", i/BANK_STRIDE+1)))
-		if err != nil {
-			return err
-		}
-		defer outputRef.Close()
-
-		indexSrc := i
-		indexDst := i + BANK_STRIDE
-		if indexDst > defCnt {
-			indexDst = defCnt
-		}
-
-		refBytes, err := marshalJson(dict.defs[indexSrc:indexDst], pretty)
-		if err != nil {
-			return err
-		}
-
-		if _, err = outputRef.Write(refBytes); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func convertKanjidicCharacter(kanjidicCharacter jmdict.KanjidicCharacter) kanjiSource {
-	character := kanjiSource{Character: kanjidicCharacter.Literal}
-
-	if level := kanjidicCharacter.Misc.JlptLevel; level != nil {
-		character.addTags(fmt.Sprintf("jlpt:%s", *level))
-	}
-
-	if grade := kanjidicCharacter.Misc.Grade; grade != nil {
-		character.addTags(fmt.Sprintf("grade:%s", *grade))
 		if gradeInt, err := strconv.Atoi(*grade); err == nil {
 			if gradeInt >= 1 && gradeInt <= 8 {
-				character.addTags("jouyou")
+				kanji.addTags("jouyou")
 			} else if gradeInt >= 9 && gradeInt <= 10 {
-				character.addTags("jinmeiyou")
+				kanji.addTags("jinmeiyou")
 			}
 		}
 	}
 
-	for _, number := range kanjidicCharacter.DictionaryNumbers {
+	for _, number := range entry.DictionaryNumbers {
 		if number.Type == "heisig" {
-			character.addTags(fmt.Sprintf("heisig:%s", number.Value))
+			kanji.addTags(fmt.Sprintf("heisig:%s", number.Value))
 		}
 	}
 
-	if counts := kanjidicCharacter.Misc.StrokeCounts; len(counts) > 0 {
+	if counts := entry.Misc.StrokeCounts; len(counts) > 0 {
-		character.addTags(fmt.Sprintf("strokes:%s", counts[0]))
+		kanji.addTags(fmt.Sprintf("strokes:%s", counts[0]))
 	}
 
-	if kanjidicCharacter.ReadingMeaning != nil {
+	if entry.ReadingMeaning != nil {
-		for _, m := range kanjidicCharacter.ReadingMeaning.Meanings {
+		for _, m := range entry.ReadingMeaning.Meanings {
 			if m.Language == nil || *m.Language == "en" {
-				character.Meanings = append(character.Meanings, m.Meaning)
+				kanji.Meanings = append(kanji.Meanings, m.Meaning)
 			}
 		}
 
-		for _, r := range kanjidicCharacter.ReadingMeaning.Readings {
+		for _, r := range entry.ReadingMeaning.Readings {
 			switch r.Type {
 			case "ja_on":
-				character.Onyomi = append(character.Onyomi, r.Value)
+				kanji.Onyomi = append(kanji.Onyomi, r.Value)
 				break
 			case "ja_kun":
-				character.Kunyomi = append(character.Kunyomi, r.Value)
+				kanji.Kunyomi = append(kanji.Kunyomi, r.Value)
 				break
 			}
 		}
 	}
 
-	return character
+	return kanji
 }
 
-func outputKanjidicJson(outputDir string, reader io.Reader, flags int) error {
+func exportKanjidicDb(outputDir string, reader io.Reader, flags int) error {
 	dict, err := jmdict.LoadKanjidic(reader)
 	if err != nil {
 		return err
 	}
 
-	var kanji []kanjiSource
+	var kanji dbKanjiList
-	for _, kanjidicCharacter := range dict.Characters {
+	for _, entry := range dict.Characters {
-		kanji = append(kanji, convertKanjidicCharacter(kanjidicCharacter))
+		kanji = append(kanji, extractKanjidicKanji(entry))
 	}
 
-	return outputKanjiIndex(outputDir, kanji, flags&flagPrettyJson == flagPrettyJson)
+	return writeDb(
+		outputDir,
+		kanji.crush(),
+		nil,
+		flags&flagPrettyJson == flagPrettyJson,
+	)
 }

main.go

@@ -37,17 +37,16 @@ const (
 )
 
 func usage() {
-	fmt.Fprintf(os.Stderr, "Usage: %s format input output\n\n", path.Base(os.Args[0]))
+	fmt.Fprintf(os.Stderr, "Usage: %s [edict|enamdict|kanjidic] input output\n\n", path.Base(os.Args[0]))
 	fmt.Fprintf(os.Stderr, "Parameters:\n")
 	flag.PrintDefaults()
 }
 
-func outputJson(fileFormat, inputPath, outputDir string, flags int) error {
+func exportDb(fileFormat, inputPath, outputDir string, flags int) error {
 	handlers := map[string]func(string, io.Reader, int) error{
-		"tmeta":    outputTermMetaJson,
+		"edict":    exportJmdictDb,
-		"edict":    outputEdictJson,
+		"enamdict": exportJmnedictDb,
-		"enamdict": outputJmnedictJson,
+		"kanjidic": exportKanjidicDb,
-		"kanjidic": outputKanjidicJson,
 	}
 
 	handler, ok := handlers[fileFormat]
@@ -76,7 +75,7 @@ func main() {
 	}
 
 	if flag.NArg() == 3 {
-		if err := outputJson(flag.Arg(0), flag.Arg(1), flag.Arg(2), flags); err != nil {
+		if err := exportDb(flag.Arg(0), flag.Arg(1), flag.Arg(2), flags); err != nil {
 			log.Fatal(err)
 		}
 	} else {

term_meta.go

@@ -1,217 +0,0 @@
-/*
- * Copyright (c) 2016 Alex Yatskov <alex@foosoft.net>
- * Author: Alex Yatskov <alex@foosoft.net>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
- * the Software, and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
- * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
- * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
- * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/FooSoft/jmdict"
-)
-
-type termMetaEntry struct {
-	Expression string   `json:"exp"`
-	Reading    string   `json:"read"`
-	Tags       []string `json:"tags"`
-}
-
-func (meta *termMetaEntry) addTags(tags ...string) {
-	for _, tag := range tags {
-		if !hasString(tag, meta.Tags) {
-			meta.Tags = append(meta.Tags, tag)
-		}
-	}
-}
-
-func (meta *termMetaEntry) addTagsPri(tags ...string) {
-	for _, tag := range tags {
-		switch tag {
-		case "news1", "ichi1", "spec1", "gai1":
-			meta.addTags("P")
-			fallthrough
-		case "news2", "ichi2", "spec2", "gai2":
-			meta.addTags(tag[:len(tag)-1])
-			break
-		}
-	}
-}
-
-type termMetaDb struct {
-	Version  int               `json:"version"`
-	Banks    int               `json:"banks"`
-	Entities map[string]string `json:"entities"`
-	entries  []termMetaEntry
-}
-
-func newTermMetaIndex(entries []termMetaEntry, entities map[string]string) termMetaDb {
-	return termMetaDb{
-		Version:  DB_VERSION,
-		Banks:    bankCount(len(entries)),
-		Entities: entities,
-		entries:  entries,
-	}
-}
-
-func (db *termMetaDb) output(dir string, pretty bool) error {
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return err
-	}
-
-	bytes, err := marshalJson(db, pretty)
-	if err != nil {
-		return err
-	}
-
-	fp, err := os.Create(path.Join(dir, "index.json"))
-	if err != nil {
-		return err
-	}
-	defer fp.Close()
-
-	if _, err = fp.Write(bytes); err != nil {
-		return err
-	}
-
-	var entries [][]string
-	for _, e := range db.entries {
-		entries = append(
-			entries,
-			[]string{
-				e.Expression,
-				e.Reading,
-				strings.Join(e.Tags, " "),
-			},
-		)
-	}
-
-	entryCount := len(entries)
-
-	for i := 0; i < entryCount; i += BANK_STRIDE {
-		indexSrc := i
-		indexDst := i + BANK_STRIDE
-		if indexDst > entryCount {
-			indexDst = entryCount
-		}
-
-		bytes, err := marshalJson(entries[indexSrc:indexDst], pretty)
-		if err != nil {
-			return err
-		}
-
-		fp, err := os.Create(path.Join(dir, fmt.Sprintf("bank_%d.json", i/BANK_STRIDE+1)))
-		if err != nil {
-			return err
-		}
-		defer fp.Close()
-
-		if _, err = fp.Write(bytes); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func extractEdictTermMeta(edictEntry jmdict.JmdictEntry) []termMetaEntry {
-	var entries []termMetaEntry
-
-	convert := func(reading jmdict.JmdictReading, kanji *jmdict.JmdictKanji) {
-		if kanji != nil && reading.Restrictions != nil && !hasString(kanji.Expression, reading.Restrictions) {
-			return
-		}
-
-		var entryBase termMetaEntry
-		entryBase.addTags(reading.Information...)
-		entryBase.addTagsPri(reading.Priorities...)
-
-		if kanji == nil {
-			entryBase.Expression = reading.Reading
-			entryBase.addTagsPri(reading.Priorities...)
-		} else {
-			entryBase.Expression = kanji.Expression
-			entryBase.Reading = reading.Reading
-			entryBase.addTags(kanji.Information...)
-
-			for _, priority := range kanji.Priorities {
-				if hasString(priority, reading.Priorities) {
-					entryBase.addTagsPri(priority)
-				}
-			}
-		}
-
-		for _, sense := range edictEntry.Sense {
-			if sense.RestrictedReadings != nil && !hasString(reading.Reading, sense.RestrictedReadings) {
-				continue
-			}
-
-			if kanji != nil && sense.RestrictedKanji != nil && !hasString(kanji.Expression, sense.RestrictedKanji) {
-				continue
-			}
-
-			entry := termMetaEntry{
-				Reading:    entryBase.Reading,
-				Expression: entryBase.Expression,
-			}
-
-			entry.addTags(entryBase.Tags...)
-			entry.addTags(sense.PartsOfSpeech...)
-			entry.addTags(sense.Fields...)
-			entry.addTags(sense.Misc...)
-			entry.addTags(sense.Dialects...)
-
-			entries = append(entries, entry)
-		}
-	}
-
-	if len(edictEntry.Kanji) > 0 {
-		for _, kanji := range edictEntry.Kanji {
-			for _, reading := range edictEntry.Readings {
-				convert(reading, &kanji)
-			}
-		}
-	} else {
-		for _, reading := range edictEntry.Readings {
-			convert(reading, nil)
-		}
-	}
-
-	return entries
-}
-
-func outputTermMetaJson(dir string, reader io.Reader, flags int) error {
-	dict, entities, err := jmdict.LoadJmdictNoTransform(reader)
-	if err != nil {
-		return err
-	}
-
-	var entries []termMetaEntry
-	for _, jmdictEntry := range dict.Entries {
-		entries = append(entries, extractEdictTermMeta(jmdictEntry)...)
-	}
-
-	db := newTermMetaIndex(entries, entities)
-	return db.output(dir, flags&flagPrettyJson == flagPrettyJson)
-}