#!/usr/bin/env python3
"""Build the embedded English->Chinese gloss dataset from ECDICT.

Petal shows an instant Chinese gloss when the writer (an English-as-a-second-
language user whose first language is Mandarin) hovers or right-clicks a word.
The gloss is served offline from a small map compiled into the Go binary, the
same way the English definitions/synonyms are (see internal/lexicon).

Source: ECDICT (https://github.com/skywind3000/ECDICT), MIT-licensed. We keep
only common single words that carry a Chinese translation, trim each gloss to a
couple of senses, and drop the noisy "[网络]" (internet-slang) lines — the result
gzips to a few MB, in line with the other lexicon assets.

Usage:
    curl -sL https://raw.githubusercontent.com/skywind3000/ECDICT/master/ecdict.csv -o ecdict.csv
    python3 scripts/build_gloss.py ecdict.csv internal/lexicon/data/gloss.json.gz
"""
import csv
import gzip
import json
import re
import sys

# Frequency gate: keep a word only if ECDICT ranks it in the BNC or COCA
# frequency lists (frq/bnc > 0). That bounds the asset to the words an ESL
# writer actually meets, dropping the long tail of archaic/technical headwords.
# Words below this rank but present nowhere in the frequency lists are skipped.
MAX_RANK = 50000

# A gloss is at most this many sense-lines and characters, so the hover bubble
# stays a glance, not a wall of text.
MAX_SENSES = 3
MAX_CHARS = 80

# Single English word: letters, with internal apostrophe/hyphen (so "don't",
# "well-being" survive but multi-word phrases and codes are dropped).
WORD_RE = re.compile(r"^[a-z][a-z'\-]*[a-z]$|^[a-z]$")

# Tag lines we drop from a translation: "[网络]" is crowd-sourced internet slang,
# "[俚]" slang, "[古]" archaic — none help an ESL writer pick everyday meaning.
DROP_TAG_RE = re.compile(r"^\[(网络|俚|古|罕|废)\]")


def clean_gloss(translation: str) -> str:
    """Trim an ECDICT translation to a compact, glanceable Chinese gloss."""
    # ECDICT separates senses with a literal backslash-n; normalize to real
    # newlines (and tolerate genuine newlines) before splitting.
    translation = translation.replace("\\n", "\n")
    senses = []
    for line in translation.split("\n"):
        line = line.strip()
        if not line or DROP_TAG_RE.match(line):
            continue
        senses.append(line)
        if len(senses) >= MAX_SENSES:
            break
    gloss = "；".join(senses)
    if len(gloss) > MAX_CHARS:
        gloss = gloss[:MAX_CHARS].rstrip("；，;, ") + "…"
    return gloss


def rank(row: dict) -> int:
    """Best (lowest, non-zero) frequency rank across COCA and BNC."""
    ranks = []
    for key in ("frq", "bnc"):
        try:
            v = int(row.get(key) or 0)
        except ValueError:
            v = 0
        if v > 0:
            ranks.append(v)
    return min(ranks) if ranks else 0


def main(src: str, dst: str) -> None:
    out: dict[str, str] = {}
    kept_rank: dict[str, int] = {}
    with open(src, newline="", encoding="utf-8") as f:
        for row in csv.DictReader(f):
            word = (row.get("word") or "").strip().lower()
            translation = (row.get("translation") or "").strip()
            if not word or not translation or not WORD_RE.match(word):
                continue
            r = rank(row)
            if r == 0 or r > MAX_RANK:
                continue
            gloss = clean_gloss(translation)
            if not gloss:
                continue
            # On a duplicate headword keep the more frequent (lower-rank) entry.
            if word in kept_rank and kept_rank[word] <= r:
                continue
            out[word] = gloss
            kept_rank[word] = r

    payload = json.dumps(out, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
    with gzip.open(dst, "wb", compresslevel=9) as gz:
        gz.write(payload)
    print(f"{len(out)} words -> {dst} ({len(payload)} bytes json)")


if __name__ == "__main__":
    if len(sys.argv) != 3:
        sys.exit("usage: build_gloss.py <ecdict.csv> <out.json.gz>")
    main(sys.argv[1], sys.argv[2])