60eba25fee
Inline Chinese gloss (offline) and a "say it more naturally" / tone-rewrite,
the two ESL features for the Mandarin-speaking writer.
Gloss: embedded English→Chinese dictionary (gloss.json.gz, 57k common words
built from ECDICT via scripts/build_gloss.py). lexicon gains Gloss()/Result.Gloss
and a lightweight GET /api/gloss/{word}; the right-click WordCard leads with the
中文; GlossTip shows it on a 350ms hover (reuses wordAt, so CJK is never glossed).
Offline + instant, works with the LLM down.
Rewrite: selecting text pops a SelectionBubble (✨更自然 + the tone vocabulary);
picking a style calls POST /api/docs/:id/rewrite (llm.RunRewrite, stateless,
owner-scoped) and shows a RewritePreview (original→rewrite, accept/cancel/retry).
Accept applies it in-editor.
Tests added in lexicon and suggestions. go build/vet/test, tsc, vite all clean;
live smoke vs a fake vLLM verified gloss + rewrite + 400/404/502 paths.
Claude-Session: https://claude.ai/code/session_016Yr6jELuRc7hyzYLccQKZd
107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the embedded English->Chinese gloss dataset from ECDICT.
|
||
|
||
Petal shows an instant Chinese gloss when the writer (an English-as-a-second-
|
||
language user whose first language is Mandarin) hovers or right-clicks a word.
|
||
The gloss is served offline from a small map compiled into the Go binary, the
|
||
same way the English definitions/synonyms are (see internal/lexicon).
|
||
|
||
Source: ECDICT (https://github.com/skywind3000/ECDICT), MIT-licensed. We keep
|
||
only common single words that carry a Chinese translation, trim each gloss to a
|
||
couple of senses, and drop the noisy "[网络]" (internet-slang) lines — the result
|
||
gzips to a few MB, in line with the other lexicon assets.
|
||
|
||
Usage:
|
||
curl -sL https://raw.githubusercontent.com/skywind3000/ECDICT/master/ecdict.csv -o ecdict.csv
|
||
python3 scripts/build_gloss.py ecdict.csv internal/lexicon/data/gloss.json.gz
|
||
"""
|
||
import csv
|
||
import gzip
|
||
import json
|
||
import re
|
||
import sys
|
||
|
||
# Frequency gate: keep a word only if ECDICT ranks it in the BNC or COCA
|
||
# frequency lists (frq/bnc > 0). That bounds the asset to the words an ESL
|
||
# writer actually meets, dropping the long tail of archaic/technical headwords.
|
||
# Words below this rank but present nowhere in the frequency lists are skipped.
|
||
MAX_RANK = 50000
|
||
|
||
# A gloss is at most this many sense-lines and characters, so the hover bubble
|
||
# stays a glance, not a wall of text.
|
||
MAX_SENSES = 3
|
||
MAX_CHARS = 80
|
||
|
||
# Single English word: letters, with internal apostrophe/hyphen (so "don't",
|
||
# "well-being" survive but multi-word phrases and codes are dropped).
|
||
WORD_RE = re.compile(r"^[a-z][a-z'\-]*[a-z]$|^[a-z]$")
|
||
|
||
# Tag lines we drop from a translation: "[网络]" is crowd-sourced internet slang,
|
||
# "[俚]" slang, "[古]" archaic — none help an ESL writer pick everyday meaning.
|
||
DROP_TAG_RE = re.compile(r"^\[(网络|俚|古|罕|废)\]")
|
||
|
||
|
||
def clean_gloss(translation: str) -> str:
|
||
"""Trim an ECDICT translation to a compact, glanceable Chinese gloss."""
|
||
# ECDICT separates senses with a literal backslash-n; normalize to real
|
||
# newlines (and tolerate genuine newlines) before splitting.
|
||
translation = translation.replace("\\n", "\n")
|
||
senses = []
|
||
for line in translation.split("\n"):
|
||
line = line.strip()
|
||
if not line or DROP_TAG_RE.match(line):
|
||
continue
|
||
senses.append(line)
|
||
if len(senses) >= MAX_SENSES:
|
||
break
|
||
gloss = "；".join(senses)
|
||
if len(gloss) > MAX_CHARS:
|
||
gloss = gloss[:MAX_CHARS].rstrip("；，;, ") + "…"
|
||
return gloss
|
||
|
||
|
||
def rank(row: dict) -> int:
|
||
"""Best (lowest, non-zero) frequency rank across COCA and BNC."""
|
||
ranks = []
|
||
for key in ("frq", "bnc"):
|
||
try:
|
||
v = int(row.get(key) or 0)
|
||
except ValueError:
|
||
v = 0
|
||
if v > 0:
|
||
ranks.append(v)
|
||
return min(ranks) if ranks else 0
|
||
|
||
|
||
def main(src: str, dst: str) -> None:
|
||
out: dict[str, str] = {}
|
||
kept_rank: dict[str, int] = {}
|
||
with open(src, newline="", encoding="utf-8") as f:
|
||
for row in csv.DictReader(f):
|
||
word = (row.get("word") or "").strip().lower()
|
||
translation = (row.get("translation") or "").strip()
|
||
if not word or not translation or not WORD_RE.match(word):
|
||
continue
|
||
r = rank(row)
|
||
if r == 0 or r > MAX_RANK:
|
||
continue
|
||
gloss = clean_gloss(translation)
|
||
if not gloss:
|
||
continue
|
||
# On a duplicate headword keep the more frequent (lower-rank) entry.
|
||
if word in kept_rank and kept_rank[word] <= r:
|
||
continue
|
||
out[word] = gloss
|
||
kept_rank[word] = r
|
||
|
||
payload = json.dumps(out, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||
with gzip.open(dst, "wb", compresslevel=9) as gz:
|
||
gz.write(payload)
|
||
print(f"{len(out)} words -> {dst} ({len(payload)} bytes json)")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) != 3:
|
||
sys.exit("usage: build_gloss.py <ecdict.csv> <out.json.gz>")
|
||
main(sys.argv[1], sys.argv[2])
|