207086da08
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
525 lines
21 KiB
Bash
Executable File
525 lines
21 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
DATA_DIR="${1:-./data}"
|
|
FORCE="${FORCE:-false}"
|
|
|
|
if [[ "${1:-}" == "--force" ]] || [[ "${2:-}" == "--force" ]]; then
|
|
FORCE=true
|
|
if [[ "${1:-}" == "--force" ]]; then
|
|
DATA_DIR="./data"
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$DATA_DIR"
|
|
|
|
# ---- Integrity helpers ----
|
|
|
|
# verify_archive <file> — test that a compressed archive is intact.
|
|
# Supports .tar.gz, .gz, .bz2. Returns 0 on success.
|
|
verify_archive() {
|
|
local file="$1"
|
|
case "$file" in
|
|
*.tar.gz|*.tgz) tar -tzf "$file" >/dev/null 2>&1 ;;
|
|
*.gz) gunzip -t "$file" 2>/dev/null ;;
|
|
*.bz2) bunzip2 -t "$file" 2>/dev/null ;;
|
|
*) return 0 ;; # unknown format, skip check
|
|
esac
|
|
}
|
|
|
|
# verify_checksum <file> <expected_sha256>
|
|
# Returns 0 if the file's SHA-256 matches.
|
|
verify_checksum() {
|
|
local file="$1" expected="$2"
|
|
local actual
|
|
actual=$(sha256sum "$file" | awk '{print $1}')
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
echo " [FAIL] checksum mismatch for $file"
|
|
echo " expected: $expected"
|
|
echo " got: $actual"
|
|
return 1
|
|
fi
|
|
echo " [ok] checksum verified: $file"
|
|
return 0
|
|
}
|
|
|
|
# check_min_size <file> <min_bytes>
|
|
# Returns 0 if file exists and is at least min_bytes.
|
|
check_min_size() {
|
|
local file="$1" min="$2"
|
|
if [[ ! -f "$file" ]]; then return 1; fi
|
|
local size
|
|
size=$(stat --printf='%s' "$file" 2>/dev/null || stat -f '%z' "$file" 2>/dev/null || echo 0)
|
|
(( size >= min ))
|
|
}
|
|
|
|
# Known SHA-256 checksums for pinned-version downloads.
|
|
# These are verified once and recorded; update when bumping versions.
|
|
CHECKSUM_FILE="$DATA_DIR/.checksums"
|
|
touch "$CHECKSUM_FILE"
|
|
|
|
# save_checksum <label> <file> — record sha256 after a successful verified download
|
|
save_checksum() {
|
|
local label="$1" file="$2"
|
|
local hash
|
|
hash=$(sha256sum "$file" | awk '{print $1}')
|
|
# Remove old entry, append new
|
|
grep -v "^${label} " "$CHECKSUM_FILE" > "$CHECKSUM_FILE.tmp" 2>/dev/null || true
|
|
echo "${label} ${hash}" >> "$CHECKSUM_FILE.tmp"
|
|
mv "$CHECKSUM_FILE.tmp" "$CHECKSUM_FILE"
|
|
}
|
|
|
|
# check_saved_checksum <label> <file> — verify file matches its recorded checksum
|
|
check_saved_checksum() {
|
|
local label="$1" file="$2"
|
|
local expected
|
|
expected=$(grep "^${label} " "$CHECKSUM_FILE" 2>/dev/null | awk '{print $2}')
|
|
if [[ -z "$expected" ]]; then
|
|
echo " [warn] no saved checksum for $label — will re-download"
|
|
return 1
|
|
fi
|
|
verify_checksum "$file" "$expected"
|
|
}
|
|
|
|
download() {
|
|
local url="$1"
|
|
local dest="$2"
|
|
if [[ -f "$dest" && "$FORCE" != "true" ]]; then
|
|
echo " [skip] $dest already exists"
|
|
return
|
|
fi
|
|
echo " [download] $url -> $dest"
|
|
mkdir -p "$(dirname "$dest")"
|
|
curl -fSL --progress-bar -o "$dest" "$url"
|
|
# Verify archive integrity immediately after download
|
|
if verify_archive "$dest"; then
|
|
echo " [ok] archive integrity verified: $dest"
|
|
else
|
|
echo " [FAIL] corrupt download, removing: $dest"
|
|
rm -f "$dest"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
clone_if_missing() {
|
|
local repo="$1"
|
|
local dest="$2"
|
|
if [[ -d "$dest" && "$FORCE" != "true" ]]; then
|
|
echo " [skip] $dest already exists"
|
|
return
|
|
fi
|
|
echo " [clone] $repo -> $dest"
|
|
rm -rf "$dest"
|
|
git clone --depth 1 "$repo" "$dest"
|
|
}
|
|
|
|
# ============================================================
|
|
# English — SCOWL (latest word list release: 2020.12.07)
|
|
# ============================================================
|
|
echo "=== SCOWL (English word lists) ==="
|
|
SCOWL_DIR="$DATA_DIR/scowl"
|
|
SCOWL_OK=false
|
|
if [[ -d "$SCOWL_DIR/final" && "$FORCE" != "true" ]]; then
|
|
# Verify extracted data has reasonable content
|
|
scowl_count=$(find "$SCOWL_DIR/final" -type f 2>/dev/null | wc -l)
|
|
if (( scowl_count > 10 )); then
|
|
echo " [skip] $SCOWL_DIR/final already exists ($scowl_count files)"
|
|
SCOWL_OK=true
|
|
else
|
|
echo " [warn] $SCOWL_DIR/final looks incomplete ($scowl_count files), re-downloading"
|
|
rm -rf "$SCOWL_DIR"
|
|
fi
|
|
fi
|
|
if [[ "$SCOWL_OK" != "true" ]]; then
|
|
SCOWL_TAR="$DATA_DIR/scowl.tar.gz"
|
|
download "https://downloads.sourceforge.net/wordlist/scowl-2020.12.07.tar.gz" "$SCOWL_TAR"
|
|
echo " [extract] $SCOWL_TAR"
|
|
mkdir -p "$SCOWL_DIR"
|
|
tar -xzf "$SCOWL_TAR" -C "$SCOWL_DIR" --strip-components=1
|
|
save_checksum "scowl" "$SCOWL_TAR"
|
|
rm -f "$SCOWL_TAR"
|
|
fi
|
|
|
|
# ============================================================
|
|
# English — WordNet
|
|
# ============================================================
|
|
echo "=== WordNet 3.0 (English definitions + synonyms) ==="
|
|
# IMPORTANT: We use WordNet 3.0, NOT 3.1, because WOLF's synset IDs
|
|
# use WN 3.0 byte offsets (eng-30-XXXXXXXX). WN 3.1 has different
|
|
# offsets, so cross-language synset backing would silently fail.
|
|
WN_DIR="$DATA_DIR/wordnet"
|
|
if [[ -d "$WN_DIR/dict" && "$FORCE" != "true" ]]; then
|
|
# Verify the dict directory has the expected data files
|
|
if [[ -f "$WN_DIR/dict/data.noun" && -f "$WN_DIR/dict/data.verb" ]]; then
|
|
echo " [skip] $WN_DIR/dict already exists (data files present)"
|
|
else
|
|
echo " [warn] $WN_DIR/dict looks incomplete, re-downloading"
|
|
rm -rf "$WN_DIR"
|
|
fi
|
|
fi
|
|
if [[ ! -d "$WN_DIR/dict" ]]; then
|
|
WN_TAR="$DATA_DIR/wordnet.tar.gz"
|
|
# Use WordNet 3.0 to match WOLF synset offsets
|
|
download "https://wordnetcode.princeton.edu/3.0/WNdb-3.0.tar.gz" "$WN_TAR" 2>/dev/null || {
|
|
echo " [FAIL] Could not download WordNet 3.0"
|
|
exit 1
|
|
}
|
|
echo " [extract] $WN_TAR"
|
|
mkdir -p "$WN_DIR"
|
|
tar -xzf "$WN_TAR" -C "$WN_DIR"
|
|
save_checksum "wordnet" "$WN_TAR"
|
|
rm -f "$WN_TAR"
|
|
# Normalize extraction directory — may be WNdb-3.0/ or dict/
|
|
for d in "WNdb-3.0" "WNdb-3.1"; do
|
|
if [[ -d "$WN_DIR/$d/dict" ]]; then
|
|
mv "$WN_DIR/$d/dict" "$WN_DIR/dict"
|
|
rm -rf "$WN_DIR/$d"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ============================================================
|
|
# English — SUBTLEX-US (word frequency)
|
|
# ============================================================
|
|
echo "=== SUBTLEX-US (English word frequency) ==="
|
|
SUBTLEX_FILE="$DATA_DIR/SUBTLEX-US.tsv"
|
|
if [[ -f "$SUBTLEX_FILE" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$SUBTLEX_FILE" 1000000; then
|
|
echo " [skip] SUBTLEX-US.tsv already exists (size ok)"
|
|
else
|
|
echo " [warn] SUBTLEX-US.tsv looks truncated, re-downloading"
|
|
rm -f "$SUBTLEX_FILE"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$SUBTLEX_FILE" ]]; then
|
|
# Try the GitHub-hosted copy first (TSV, direct download), then Ghent University.
|
|
download "https://raw.githubusercontent.com/Wikipedia2Vec/Wikipedia2Vec/master/resources/SUBTLEX-US.tsv" "$SUBTLEX_FILE" 2>/dev/null || \
|
|
download "https://raw.githubusercontent.com/Wikipedia2Vec/Wikipedia2Vec/refs/heads/master/resources/SUBTLEX-US.tsv" "$SUBTLEX_FILE" 2>/dev/null || true
|
|
|
|
if [[ ! -f "$SUBTLEX_FILE" ]]; then
|
|
# Fallback: Ghent University zip (may require browser)
|
|
SUBTLEX_ZIP="$DATA_DIR/subtlex-us.zip"
|
|
download "https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus/subtlexus2.zip/at_download/file" "$SUBTLEX_ZIP" 2>/dev/null || \
|
|
download "https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus/subtlexus2.zip" "$SUBTLEX_ZIP" 2>/dev/null || true
|
|
|
|
if [[ -f "$SUBTLEX_ZIP" ]]; then
|
|
echo " [extract] $SUBTLEX_ZIP"
|
|
unzip -qo "$SUBTLEX_ZIP" -d "$DATA_DIR/subtlex-tmp" 2>/dev/null || true
|
|
FOUND_SUBTLEX=$(find "$DATA_DIR/subtlex-tmp" -type f \( -name "*.tsv" -o -name "*.txt" -o -name "*.csv" \) | head -1)
|
|
if [[ -n "$FOUND_SUBTLEX" ]]; then
|
|
mv "$FOUND_SUBTLEX" "$SUBTLEX_FILE"
|
|
echo " [ok] extracted to SUBTLEX-US.tsv"
|
|
else
|
|
echo " [warn] could not find TSV in archive — SUBTLEX may need manual conversion from XLSX"
|
|
fi
|
|
rm -rf "$DATA_DIR/subtlex-tmp" "$SUBTLEX_ZIP"
|
|
else
|
|
echo " [warn] SUBTLEX-US download failed."
|
|
echo " Download manually from: https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus"
|
|
echo " Export the XLSX to TSV and place as: $SUBTLEX_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ============================================================
|
|
# English — CMU Pronouncing Dictionary
|
|
# ============================================================
|
|
echo "=== CMUdict (English pronunciation) ==="
|
|
CMUDICT_FILE="$DATA_DIR/cmudict-0.7b"
|
|
if [[ -f "$CMUDICT_FILE" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$CMUDICT_FILE" 2000000; then
|
|
echo " [skip] cmudict-0.7b already exists (size ok)"
|
|
else
|
|
echo " [warn] cmudict-0.7b looks truncated, re-downloading"
|
|
rm -f "$CMUDICT_FILE"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$CMUDICT_FILE" ]]; then
|
|
download "https://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || \
|
|
download "https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || \
|
|
download "https://raw.githubusercontent.com/Alexir/CMUdict/master/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || {
|
|
echo " [warn] CMUdict download failed. Download manually and place at: $CMUDICT_FILE"
|
|
}
|
|
fi
|
|
|
|
# ============================================================
|
|
# English — Wiktionary (kaikki)
|
|
# ============================================================
|
|
echo "=== Wiktionary English (compressed: ~436 MB) ==="
|
|
if [[ -f "$DATA_DIR/kaikki-en.jsonl" && "$FORCE" != "true" ]]; then
|
|
# Verify the decompressed file isn't truncated (expect at least 500 MB)
|
|
if check_min_size "$DATA_DIR/kaikki-en.jsonl" 500000000; then
|
|
echo " [skip] kaikki-en.jsonl already exists (size ok)"
|
|
else
|
|
echo " [warn] kaikki-en.jsonl looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/kaikki-en.jsonl"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/kaikki-en.jsonl" ]]; then
|
|
KAIKKI_EN_GZ="$DATA_DIR/kaikki-en.jsonl.gz"
|
|
download "https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz" "$KAIKKI_EN_GZ"
|
|
echo " [decompress] kaikki-en.jsonl.gz"
|
|
gunzip -kf "$KAIKKI_EN_GZ"
|
|
save_checksum "kaikki-en" "$KAIKKI_EN_GZ"
|
|
fi
|
|
|
|
# ============================================================
|
|
# French + pt-PT — Hunspell (LibreOffice dictionaries)
|
|
# ============================================================
|
|
echo "=== Hunspell (French + Portuguese) ==="
|
|
HUNSPELL_DIR="$DATA_DIR/hunspell-dicts"
|
|
clone_if_missing "https://github.com/LibreOffice/dictionaries.git" "$HUNSPELL_DIR"
|
|
# Copy the files we need and verify they exist
|
|
mkdir -p "$DATA_DIR/fr_FR" "$DATA_DIR/pt_PT"
|
|
for dic in "$HUNSPELL_DIR/fr_FR/fr.dic" "$HUNSPELL_DIR/pt_PT/pt_PT.dic"; do
|
|
if [[ ! -f "$dic" ]]; then
|
|
echo " [FAIL] expected file missing from clone: $dic"
|
|
exit 1
|
|
fi
|
|
done
|
|
cp -f "$HUNSPELL_DIR/fr_FR/fr.dic" "$DATA_DIR/fr_FR/fr.dic"
|
|
if [[ -f "$HUNSPELL_DIR/fr_FR/fr.aff" ]]; then
|
|
cp -f "$HUNSPELL_DIR/fr_FR/fr.aff" "$DATA_DIR/fr_FR/fr.aff"
|
|
else
|
|
FR_AFF=$(find "$HUNSPELL_DIR/fr_FR" -name "*.aff" -type f 2>/dev/null | head -1)
|
|
if [[ -n "$FR_AFF" ]]; then
|
|
cp -f "$FR_AFF" "$DATA_DIR/fr_FR/fr.aff"
|
|
echo " [ok] found fr .aff file at: $FR_AFF"
|
|
else
|
|
echo " [warn] no .aff file found for fr — affix expansion will be skipped"
|
|
fi
|
|
fi
|
|
cp -f "$HUNSPELL_DIR/pt_PT/pt_PT.dic" "$DATA_DIR/pt_PT/pt_PT.dic"
|
|
# .aff file may be in pt_PT/ or at a different path in the repo
|
|
if [[ -f "$HUNSPELL_DIR/pt_PT/pt_PT.aff" ]]; then
|
|
cp -f "$HUNSPELL_DIR/pt_PT/pt_PT.aff" "$DATA_DIR/pt_PT/pt_PT.aff"
|
|
else
|
|
PT_AFF=$(find "$HUNSPELL_DIR/pt_PT" -name "*.aff" -type f 2>/dev/null | head -1)
|
|
if [[ -n "$PT_AFF" ]]; then
|
|
cp -f "$PT_AFF" "$DATA_DIR/pt_PT/pt_PT.aff"
|
|
echo " [ok] found pt_PT .aff file at: $PT_AFF"
|
|
else
|
|
echo " [warn] no .aff file found for pt_PT — affix expansion will be skipped"
|
|
fi
|
|
fi
|
|
|
|
# ============================================================
|
|
# French — Lexique
|
|
# ============================================================
|
|
echo "=== Lexique (French POS + frequency) ==="
|
|
if [[ -f "$DATA_DIR/Lexique383.tsv" && "$FORCE" != "true" ]]; then
|
|
# TSV should be at least 10 MB
|
|
if check_min_size "$DATA_DIR/Lexique383.tsv" 10000000; then
|
|
echo " [skip] Lexique383.tsv already exists (size ok)"
|
|
else
|
|
echo " [warn] Lexique383.tsv looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/Lexique383.tsv"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/Lexique383.tsv" ]]; then
|
|
# lexique.org is intermittently unreachable; if this fails, obtain
|
|
# Lexique383.tsv manually and place it in the data directory.
|
|
download "http://www.lexique.org/databases/Lexique383/Lexique383.tsv" "$DATA_DIR/Lexique383.tsv" || {
|
|
echo " [warn] Lexique download failed — lexique.org may be down."
|
|
echo " Place Lexique383.tsv in $DATA_DIR manually, or use: pip install pylexique"
|
|
}
|
|
fi
|
|
|
|
# ============================================================
|
|
# French — WOLF
|
|
# ============================================================
|
|
echo "=== WOLF (French definitions + synonyms) ==="
|
|
WOLF_BZ2="$DATA_DIR/wolf-1.0b4.xml.bz2"
|
|
if [[ -f "$DATA_DIR/wolf.xml" && "$FORCE" != "true" ]]; then
|
|
# WOLF XML should be at least 20 MB
|
|
if check_min_size "$DATA_DIR/wolf.xml" 20000000; then
|
|
echo " [skip] wolf.xml already exists (size ok)"
|
|
else
|
|
echo " [warn] wolf.xml looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/wolf.xml" "$WOLF_BZ2"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/wolf.xml" ]]; then
|
|
download "https://almanach.inria.fr/software_and_resources/downloads/wolf-1.0b4.xml.bz2" "$WOLF_BZ2"
|
|
echo " [decompress] $WOLF_BZ2"
|
|
bunzip2 -kf "$WOLF_BZ2"
|
|
mv -f "$DATA_DIR/wolf-1.0b4.xml" "$DATA_DIR/wolf.xml" 2>/dev/null || true
|
|
save_checksum "wolf" "$WOLF_BZ2"
|
|
fi
|
|
|
|
# ============================================================
|
|
# French — Wiktionary (kaikki)
|
|
# ============================================================
|
|
echo "=== Wiktionary French (compressed: ~47 MB) ==="
|
|
if [[ -f "$DATA_DIR/kaikki-fr.jsonl" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$DATA_DIR/kaikki-fr.jsonl" 50000000; then
|
|
echo " [skip] kaikki-fr.jsonl already exists (size ok)"
|
|
else
|
|
echo " [warn] kaikki-fr.jsonl looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/kaikki-fr.jsonl"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/kaikki-fr.jsonl" ]]; then
|
|
KAIKKI_FR_GZ="$DATA_DIR/kaikki-fr.jsonl.gz"
|
|
download "https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz" "$KAIKKI_FR_GZ"
|
|
echo " [decompress] kaikki-fr.jsonl.gz"
|
|
gunzip -kf "$KAIKKI_FR_GZ"
|
|
save_checksum "kaikki-fr" "$KAIKKI_FR_GZ"
|
|
fi
|
|
|
|
# ============================================================
|
|
# pt-PT — Dicionário Aberto
|
|
# ============================================================
|
|
echo "=== Dicionário Aberto (Portuguese definitions) ==="
|
|
DICIO_DIR="$DATA_DIR/dicionario-aberto-repo"
|
|
if [[ -f "$DATA_DIR/dicionario-aberto.xml" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$DATA_DIR/dicionario-aberto.xml" 1000000; then
|
|
echo " [skip] dicionario-aberto.xml already exists (size ok)"
|
|
else
|
|
echo " [warn] dicionario-aberto.xml looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/dicionario-aberto.xml"
|
|
rm -rf "$DICIO_DIR"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/dicionario-aberto.xml" ]]; then
|
|
clone_if_missing "https://github.com/ambs/Dicionario-Aberto.git" "$DICIO_DIR"
|
|
# Data lives in Resources/xmls.zip as per-letter XML files; extract and merge
|
|
DICIO_ZIP="$DICIO_DIR/Resources/xmls.zip"
|
|
if [[ ! -f "$DICIO_ZIP" ]]; then
|
|
echo " [FAIL] expected $DICIO_ZIP not found in clone"
|
|
exit 1
|
|
fi
|
|
DICIO_TMP="$DATA_DIR/dicionario-aberto-xmls"
|
|
echo " [extract] $DICIO_ZIP"
|
|
unzip -qo "$DICIO_ZIP" -d "$DICIO_TMP"
|
|
echo " [merge] combining per-letter XML files into dicionario-aberto.xml"
|
|
echo '<?xml version="1.0"?>' > "$DATA_DIR/dicionario-aberto.xml"
|
|
echo '<dic>' >> "$DATA_DIR/dicionario-aberto.xml"
|
|
for xmlfile in "$DICIO_TMP"/xmls/*.xml; do
|
|
# Extract entry elements, skipping the <?xml?>, <dic>, <head> wrappers
|
|
sed -n '/<entry /,/<\/entry>/p' "$xmlfile" >> "$DATA_DIR/dicionario-aberto.xml"
|
|
done
|
|
echo '</dic>' >> "$DATA_DIR/dicionario-aberto.xml"
|
|
rm -rf "$DICIO_TMP" "$DICIO_DIR"
|
|
if ! check_min_size "$DATA_DIR/dicionario-aberto.xml" 1000000; then
|
|
echo " [FAIL] dicionario-aberto.xml seems too small after merge"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ============================================================
|
|
# pt-PT — CETEMPúblico frequency (or fallback frequency list)
|
|
# ============================================================
|
|
echo "=== Portuguese frequency data ==="
|
|
PT_FREQ_FILE="$DATA_DIR/cetempublico-freq.tsv"
|
|
if [[ -f "$PT_FREQ_FILE" && "$FORCE" != "true" ]]; then
|
|
echo " [skip] cetempublico-freq.tsv already exists"
|
|
elif [[ -f "$DATA_DIR/pt-freq.tsv" && "$FORCE" != "true" ]]; then
|
|
echo " [skip] pt-freq.tsv already exists"
|
|
else
|
|
echo " [info] CETEMPúblico frequency data requires manual download from Linguateca."
|
|
echo " Visit: https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO"
|
|
echo " Generate a word frequency list and save as: $PT_FREQ_FILE"
|
|
echo " Format: word<TAB>frequency (one per line)"
|
|
echo " The import will proceed without frequency data if this file is absent."
|
|
fi
|
|
|
|
# ============================================================
|
|
# pt-PT — Open Multilingual Wordnet
|
|
# ============================================================
|
|
echo "=== Open Multilingual Wordnet (Portuguese synset mappings) ==="
|
|
OMW_DIR="$DATA_DIR/omw"
|
|
OMW_FILE="$OMW_DIR/wn-data-por.tab"
|
|
if [[ -f "$OMW_FILE" && "$FORCE" != "true" ]]; then
|
|
echo " [skip] wn-data-por.tab already exists"
|
|
fi
|
|
if [[ ! -f "$OMW_FILE" ]]; then
|
|
OMW_REPO="$DATA_DIR/omw-data-repo"
|
|
clone_if_missing "https://github.com/omwn/omw-data.git" "$OMW_REPO"
|
|
mkdir -p "$OMW_DIR"
|
|
# Find the Portuguese tab file in the repo
|
|
FOUND_OMW=$(find "$OMW_REPO" -type f -name "wn-data-por.tab" 2>/dev/null | head -1)
|
|
if [[ -n "$FOUND_OMW" ]]; then
|
|
cp "$FOUND_OMW" "$OMW_FILE"
|
|
echo " [ok] extracted wn-data-por.tab"
|
|
else
|
|
# Try alternative naming
|
|
FOUND_OMW=$(find "$OMW_REPO" -type f -name "*por*" -name "*.tab" 2>/dev/null | head -1)
|
|
if [[ -n "$FOUND_OMW" ]]; then
|
|
cp "$FOUND_OMW" "$OMW_FILE"
|
|
echo " [ok] extracted Portuguese OMW data"
|
|
else
|
|
echo " [warn] could not find Portuguese data in OMW repo"
|
|
echo " The import will proceed without synset mappings for pt-PT."
|
|
fi
|
|
fi
|
|
rm -rf "$OMW_REPO"
|
|
fi
|
|
|
|
# ============================================================
|
|
# pt-PT — Wiktionary (kaikki)
|
|
# ============================================================
|
|
echo "=== Wiktionary Portuguese (compressed: ~45 MB) ==="
|
|
if [[ -f "$DATA_DIR/kaikki-pt.jsonl" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$DATA_DIR/kaikki-pt.jsonl" 50000000; then
|
|
echo " [skip] kaikki-pt.jsonl already exists (size ok)"
|
|
else
|
|
echo " [warn] kaikki-pt.jsonl looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/kaikki-pt.jsonl"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/kaikki-pt.jsonl" ]]; then
|
|
KAIKKI_PT_GZ="$DATA_DIR/kaikki-pt.jsonl.gz"
|
|
download "https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.jsonl.gz" "$KAIKKI_PT_GZ"
|
|
echo " [decompress] kaikki-pt.jsonl.gz"
|
|
gunzip -kf "$KAIKKI_PT_GZ"
|
|
save_checksum "kaikki-pt" "$KAIKKI_PT_GZ"
|
|
fi
|
|
|
|
# ============================================================
|
|
# Mandarin — CC-CEDICT
|
|
# ============================================================
|
|
echo "=== CC-CEDICT (Mandarin word list + definitions) ==="
|
|
if [[ -f "$DATA_DIR/cedict_ts.u8" && "$FORCE" != "true" ]]; then
|
|
# Expect at least 3 MB
|
|
if check_min_size "$DATA_DIR/cedict_ts.u8" 3000000; then
|
|
echo " [skip] cedict_ts.u8 already exists (size ok)"
|
|
else
|
|
echo " [warn] cedict_ts.u8 looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/cedict_ts.u8"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/cedict_ts.u8" ]]; then
|
|
CEDICT_GZ="$DATA_DIR/cedict_ts.u8.gz"
|
|
download "https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz" "$CEDICT_GZ"
|
|
echo " [decompress] cedict_ts.u8.gz"
|
|
gunzip -kf "$CEDICT_GZ"
|
|
save_checksum "cedict" "$CEDICT_GZ"
|
|
fi
|
|
|
|
# ============================================================
|
|
# Mandarin — Wiktionary (kaikki)
|
|
# ============================================================
|
|
echo "=== Wiktionary Chinese (compressed) ==="
|
|
if [[ -f "$DATA_DIR/kaikki-zh.jsonl" && "$FORCE" != "true" ]]; then
|
|
if check_min_size "$DATA_DIR/kaikki-zh.jsonl" 5000000; then
|
|
echo " [skip] kaikki-zh.jsonl already exists (size ok)"
|
|
else
|
|
echo " [warn] kaikki-zh.jsonl looks truncated, re-downloading"
|
|
rm -f "$DATA_DIR/kaikki-zh.jsonl"
|
|
fi
|
|
fi
|
|
if [[ ! -f "$DATA_DIR/kaikki-zh.jsonl" ]]; then
|
|
KAIKKI_ZH_GZ="$DATA_DIR/kaikki-zh.jsonl.gz"
|
|
download "https://kaikki.org/dictionary/Chinese/kaikki.org-dictionary-Chinese.jsonl.gz" "$KAIKKI_ZH_GZ"
|
|
echo " [decompress] kaikki-zh.jsonl.gz"
|
|
gunzip -kf "$KAIKKI_ZH_GZ"
|
|
save_checksum "kaikki-zh" "$KAIKKI_ZH_GZ"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== All downloads complete ==="
|
|
echo "Data directory: $DATA_DIR"
|
|
du -sh "$DATA_DIR" 2>/dev/null || true
|