yomichan-import/jmdict_headword.go

package yomichan

import (
	"fmt"
	"hash/fnv"
	"regexp"
	"strconv"

	"foosoft.net/projects/jmdict"
	"golang.org/x/exp/slices"
)

type headword struct {
	Expression   string
	Reading      string
	TermTags     []string
	Index        int
	IsPriority   bool
	IsFrequent   bool
	IsIrregular  bool
	IsOutdated   bool
	IsRareKanji  bool
	IsSearchOnly bool
	IsAteji      bool
	IsGikun      bool
}

type hash uint64

func (h *headword) Hash() hash {
	return hashText(h.Expression + "␞" + h.Reading)
}

func (h *headword) ExpHash() hash {
	return hashText(h.Expression + "␞" + h.Expression)
}

func (h *headword) ReadingHash() hash {
	return hashText(h.Reading + "␞" + h.Reading)
}

func hashText(s string) hash {
	h := fnv.New64a()
	h.Write([]byte(s))
	return hash(h.Sum64())
}

func (h *headword) IsKanaOnly() bool {
	if h.Expression != h.Reading {
		return false
	}
	for _, char := range h.Expression {
		if char >= 'ぁ' && char <= 'ヿ' {
			// hiragana and katakana range
			continue
		} else if char >= '･' && char <= 'ﾟ' {
			// halfwidth katakana range
			continue
		} else if char == '〜' {
			continue
		} else {
			return false
		}
	}
	return true
}

func (h *headword) Score() int {
	score := 0
	if h.IsPriority {
		score += 1
	}
	if h.IsFrequent {
		score += 1
	}
	if h.IsIrregular {
		score -= 5
	}
	if h.IsOutdated {
		score -= 5
	}
	if h.IsRareKanji {
		score -= 5
	}
	if h.IsSearchOnly {
		score -= 5
	}
	return score
}

func (h *headword) ToInternalLink(includeReading bool) any {
	if !includeReading || h.Expression == h.Reading {
		return contentInternalLink(
			contentAttr{lang: ISOtoHTML["jpn"]},
			h.Expression,
		)
	} else {
		return contentSpan(
			contentAttr{lang: ISOtoHTML["jpn"]},
			contentInternalLink(contentAttr{}, h.Expression),
			"（",
			contentInternalLink(contentAttr{}, h.Reading),
			"）",
		)
	}
}

func (h *headword) SetFlags(infoTags, freqTags []string) {
	priorityTags := []string{"ichi1", "news1", "gai1", "spec1", "spec2"}
	for _, priorityTag := range priorityTags {
		if slices.Contains(freqTags, priorityTag) {
			h.IsPriority = true
			break
		}
	}
	if len(freqTags) > 1 {
		h.IsFrequent = true
	}
	for _, infoTag := range infoTags {
		switch infoTag {
		case "iK", "ik", "io":
			h.IsIrregular = true
		case "oK", "ok":
			h.IsOutdated = true
		case "sK", "sk":
			h.IsSearchOnly = true
		case "rK":
			h.IsRareKanji = true
		case "ateji":
			h.IsAteji = true
		case "gikun":
			h.IsGikun = true
		default:
			fmt.Println("Unknown information tag type: " + infoTag)
			h.TermTags = append(h.TermTags, infoTag)
		}
	}
	if h.IsOutdated && h.IsRareKanji {
		h.IsRareKanji = false
	}
}

func (h *headword) SetTermTags(freqTags []string) {
	if h.IsPriority {
		h.TermTags = append(h.TermTags, priorityTagName)
	}
	knownFreqTags := []string{"ichi1", "ichi2", "gai1", "gai2", "spec1", "spec2"}
	for _, tag := range freqTags {
		isNewsFreqTag, _ := regexp.MatchString(`nf\d\d`, tag)
		if isNewsFreqTag {
			// nf tags are divided into ranks of 500
			// (nf01 to nf48). Let's combine them into
			// ranks of 1k (news1k, news2k, ..., news24k).
			var i int
			if _, err := fmt.Sscanf(tag, "nf%2d", &i); err == nil {
				i = (i + (i % 2)) / 2
				newsTag := "news" + strconv.Itoa(i) + "k"
				h.TermTags = append(h.TermTags, newsTag)
			}
		} else if tag == "news1" || tag == "news2" {
			// News tags are derived from the nf
			// rankings, so these are not needed.
			continue
		} else if slices.Contains(knownFreqTags, tag) {
			tagWithoutTheNumber := tag[:len(tag)-1]
			h.TermTags = append(h.TermTags, tagWithoutTheNumber)
		} else {
			fmt.Println("Unknown frequency tag type: " + tag)
			h.TermTags = append(h.TermTags, tag)
		}
	}
	if h.IsIrregular {
		h.TermTags = append(h.TermTags, irregularTagName)
	}
	if h.IsOutdated {
		h.TermTags = append(h.TermTags, outdatedTagName)
	}
	if h.IsRareKanji {
		h.TermTags = append(h.TermTags, rareKanjiTagName)
	}
	if h.IsAteji {
		h.TermTags = append(h.TermTags, atejiTagName)
	}
	if h.IsGikun {
		h.TermTags = append(h.TermTags, gikunTagName)
	}
}

func newHeadword(kanji *jmdict.JmdictKanji, reading *jmdict.JmdictReading) headword {
	h := headword{}
	infoTags := []string{}
	freqTags := []string{}
	if kanji == nil {
		h.Expression = reading.Reading
		h.Reading = reading.Reading
		infoTags = reading.Information
		freqTags = reading.Priorities
	} else if reading == nil {
		// should only apply to search-only kanji terms
		h.Expression = kanji.Expression
		h.Reading = ""
		infoTags = kanji.Information
		freqTags = kanji.Priorities
	} else {
		h.Expression = kanji.Expression
		h.Reading = reading.Reading
		infoTags = union(kanji.Information, reading.Information)
		freqTags = intersection(kanji.Priorities, reading.Priorities)
	}
	h.SetFlags(infoTags, freqTags)
	h.SetTermTags(freqTags)
	return h
}

func areAllKanjiIrregular(allKanji []jmdict.JmdictKanji) bool {
	// If every kanji form is rare or irregular, then we'll make
	// kana-only headwords for each kana form.
	if len(allKanji) == 0 {
		return false
	}
	for _, kanji := range allKanji {
		h := newHeadword(&kanji, nil)
		kanjiIsIrregular := h.IsRareKanji || h.IsIrregular || h.IsOutdated || h.IsSearchOnly
		if !kanjiIsIrregular {
			return false
		}
	}
	return true
}

func extractHeadwords(entry jmdict.JmdictEntry) []headword {
	headwords := []headword{}
	allKanjiAreIrregular := areAllKanjiIrregular(entry.Kanji)

	if allKanjiAreIrregular {
		// Adding the reading-only terms before kanji+reading
		// terms here for the sake of the Index property,
		// which affects the yomichan term ranking.
		for _, reading := range entry.Readings {
			h := newHeadword(nil, &reading)
			h.Index = len(headwords)
			headwords = append(headwords, h)
		}
	}

	for _, kanji := range entry.Kanji {
		if slices.Contains(kanji.Information, "sK") {
			// Search-only kanji forms do not have associated readings.
			h := newHeadword(&kanji, nil)
			h.Index = len(headwords)
			headwords = append(headwords, h)
			continue
		}
		for _, reading := range entry.Readings {
			if reading.NoKanji != nil {
				continue
			} else if slices.Contains(reading.Information, "sk") {
				// Search-only kana forms do not have associated kanji forms.
				continue
			} else if reading.Restrictions != nil && !slices.Contains(reading.Restrictions, kanji.Expression) {
				continue
			} else {
				h := newHeadword(&kanji, &reading)
				h.Index = len(headwords)
				headwords = append(headwords, h)
			}
		}
	}

	if !allKanjiAreIrregular {
		noKanjiInEntry := (len(entry.Kanji) == 0)
		for _, reading := range entry.Readings {
			if reading.NoKanji != nil || noKanjiInEntry || slices.Contains(reading.Information, "sk") {
				h := newHeadword(nil, &reading)
				h.Index = len(headwords)
				headwords = append(headwords, h)
			}
		}
	}

	return headwords
}