alex/yomichan-import
1
Fork 0
Code Releases Activity

zip support

Browse Source
This commit is contained in:
Alex Yatskov 2017-06-25 17:22:17 -07:00
parent e53d9d70eb
commit 4c3e292470
6 changed files with 42 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
common.go
View File

@ -23,10 +23,11 @@
package main
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"strings"
)
@ -115,9 +116,12 @@ func (kanji dbKanjiList) crush() [][]interface{} {
return results
}
func writeDb(outputDir, title, revision string, termRecords [][]interface{}, kanjiRecords [][]interface{}, tagMeta map[string]dbTagMeta, stride int, pretty bool) error {
func writeDb(outputPath, title, revision string, termRecords [][]interface{}, kanjiRecords [][]interface{}, tagMeta map[string]dbTagMeta, stride int, pretty bool) error {
const DB_VERSION = 1
var zbuff bytes.Buffer
zip := zip.NewWriter(&zbuff)
marshalJson := func(obj interface{}, pretty bool) ([]byte, error) {
if pretty {
return json.MarshalIndent(obj, "", " ")
@ -142,13 +146,12 @@ func writeDb(outputDir, title, revision string, termRecords [][]interface{}, kan
return 0, err
}
fp, err := os.Create(path.Join(outputDir, fmt.Sprintf("%s_bank_%d.json", prefix, i/stride+1)))
zw, err := zip.Create(fmt.Sprintf("%s_bank_%d.json", prefix, i/stride+1))
if err != nil {
return 0, err
}
defer fp.Close()
if _, err = fp.Write(bytes); err != nil {
if _, err := zw.Write(bytes); err != nil {
return 0, err
}
@ -158,10 +161,6 @@ func writeDb(outputDir, title, revision string, termRecords [][]interface{}, kan
return bankCount, nil
}
if err := os.MkdirAll(outputDir, 0755); err != nil {
return err
}
var err error
var db struct {
Title string `json:"title"`
@ -190,17 +189,27 @@ func writeDb(outputDir, title, revision string, termRecords [][]interface{}, kan
return err
}
fp, err := os.Create(path.Join(outputDir, "index.json"))
zw, err := zip.Create("index.json")
if err != nil {
return err
}
defer fp.Close()
if _, err := fp.Write(bytes); err != nil {
if _, err := zw.Write(bytes); err != nil {
return err
}
return nil
zip.Close()
fp, err := os.Create(outputPath)
if err != nil {
return err
}
if _, err := fp.Write(zbuff.Bytes()); err != nil {
return err
}
return fp.Close()
}
func appendStringUnique(target []string, source ...string) []string {

4
edict.go
View File

@ -182,7 +182,7 @@ func jmdictExtractTerms(edictEntry jmdict.JmdictEntry, language string) []dbTerm
return terms
}
func jmdictExportDb(inputPath, outputDir, language, title string, stride int, pretty bool) error {
func jmdictExportDb(inputPath, outputPath, language, title string, stride int, pretty bool) error {
reader, err := os.Open(inputPath)
if err != nil {
return err
@ -226,7 +226,7 @@ func jmdictExportDb(inputPath, outputDir, language, title string, stride int, pr
}
return writeDb(
outputDir,
outputPath,
title,
JMDICT_REVISION,
terms.crush(),

4
enamdict.go
View File

@ -97,7 +97,7 @@ func jmnedictExtractTerms(enamdictEntry jmdict.JmnedictEntry) []dbTerm {
return terms
}
func jmnedictExportDb(inputPath, outputDir, language, title string, stride int, pretty bool) error {
func jmnedictExportDb(inputPath, outputPath, language, title string, stride int, pretty bool) error {
reader, err := os.Open(inputPath)
if err != nil {
return err
@ -119,7 +119,7 @@ func jmnedictExportDb(inputPath, outputDir, language, title string, stride int,
}
return writeDb(
outputDir,
outputPath,
title,
JMNEDICT_REVISION,
terms.crush(),

4
epwing.go
View File

@ -62,7 +62,7 @@ type epwingExtractor interface {
getRevision() string
}
func epwingExportDb(inputPath, outputDir, language, title string, stride int, pretty bool) error {
func epwingExportDb(inputPath, outputPath, language, title string, stride int, pretty bool) error {
stat, err := os.Stat(inputPath)
if err != nil {
return err
@ -193,7 +193,7 @@ func epwingExportDb(inputPath, outputDir, language, title string, stride int, pr
}
return writeDb(
outputDir,
outputPath,
title,
strings.Join(revisions, ";"),
terms.crush(),

4
kanjidic.go
View File

@ -86,7 +86,7 @@ func kanjidicExtractKanji(entry jmdict.KanjidicCharacter, language string) *dbKa
return &kanji
}
func kanjidicExportDb(inputPath, outputDir, language, title string, stride int, pretty bool) error {
func kanjidicExportDb(inputPath, outputPath, language, title string, stride int, pretty bool) error {
reader, err := os.Open(inputPath)
if err != nil {
return err
@ -130,7 +130,7 @@ func kanjidicExportDb(inputPath, outputDir, language, title string, stride int,
}
return writeDb(
outputDir,
outputPath,
title,
KANJIDIC_REVISION,
nil,

46
main.go
View File

@ -40,13 +40,13 @@ const (
)
func usage() {
fmt.Fprintf(os.Stderr, "Usage: %s [options] input-path [output-dir]\n", path.Base(os.Args[0]))
fmt.Fprintf(os.Stderr, "Usage: %s [options] input-path output-path\n", path.Base(os.Args[0]))
fmt.Fprint(os.Stderr, "https://foosoft.net/projects/yomichan-import/\n\n")
fmt.Fprint(os.Stderr, "Parameters:\n")
flag.PrintDefaults()
}
func exportDb(inputPath, outputDir, format, language, title string, stride int, pretty bool) error {
func exportDb(inputPath, outputPath, format, language, title string, stride int, pretty bool) error {
handlers := map[string]func(string, string, string, string, int, bool) error{
"edict": jmdictExportDb,
"enamdict": jmnedictExportDb,
@ -59,8 +59,8 @@ func exportDb(inputPath, outputDir, format, language, title string, stride int,
return errors.New("unrecognized dictionray format")
}
log.Printf("converting '%s' to '%s' in '%s' format...", inputPath, outputDir, format)
return handler(inputPath, outputDir, language, title, stride, pretty)
log.Printf("converting '%s' to '%s' in '%s' format...", inputPath, outputPath, format)
return handler(inputPath, outputPath, language, title, stride, pretty)
}
func serveDb(serveDir string, port int) error {
@ -75,36 +75,29 @@ func makeTmpDir() (string, error) {
func main() {
var (
format = flag.String("format", "", "dictionary format [edict|enamdict|kanjidic|epwing]")
language = flag.String("language", DEFAULT_LANGUAGE, "dictionary language (if applicable)")
language = flag.String("language", DEFAULT_LANGUAGE, "dictionary language (if supported)")
title = flag.String("title", "", "dictionary title")
port = flag.Int("port", DEFAULT_PORT, "port to serve dictionary JSON on")
stride = flag.Int("stride", DEFAULT_STRIDE, "dictionary bank stride")
pretty = flag.Bool("pretty", false, "output prettified dictionary JSON")
serve = flag.Bool("serve", false, "serve dictionary JSON for extension")
)
flag.Usage = usage
flag.Parse()
var (
inputPath string
outputDir string
)
if flag.NArg() == 0 {
if flag.NArg() != 2 {
if err := gui(); err == nil {
return
} else {
usage()
os.Exit(2)
}
} else {
inputPath = flag.Arg(0)
if flag.NArg() > 1 {
outputDir = flag.Arg(1)
}
}
var (
inputPath = flag.Arg(0)
outputPath = flag.Arg(1)
)
if _, err := os.Stat(inputPath); err != nil {
log.Fatalf("dictionary path '%s' does not exist", inputPath)
}
@ -115,22 +108,7 @@ func main() {
}
}
if outputDir == "" {
var err error
if outputDir, err = makeTmpDir(); err != nil {
log.Fatal(err)
}
*serve = true
}
if err := exportDb(inputPath, outputDir, *format, *language, *title, *stride, *pretty); err != nil {
if err := exportDb(inputPath, outputPath, *format, *language, *title, *stride, *pretty); err != nil {
log.Fatal(err)
}
if *serve {
if err := serveDb(outputDir, *port); err != nil {
log.Fatal(err)
}
}
}