Add loader tests and fix header detection and punctuation filtering

Tests: 23 new tests across cetempublico, affix, and wiktionary loaders
covering column order detection, ISO-8859-1 encoding, junk filtering,
frequency scaling, affix expansion, and sense-level synonym extraction.

Fixes: header detection now checks either field for numeric value (was
only checking last field, breaking count-first format). Added hasLetter
filter to reject pure punctuation entries like "." and ",".

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-03 09:15:13 -07:00
parent a0cca2cb19
commit 2045cc2ada
4 changed files with 405 additions and 4 deletions

View File

@@ -9,6 +9,7 @@ import (
"path/filepath"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
@@ -74,14 +75,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
// Peek first line to detect header vs data
// Peek first line to detect header vs data.
// A data line has at least one numeric field (either "word count" or "count word").
var firstLine string
if scanner.Scan() {
firstLine = scanner.Text()
fields := strings.Fields(firstLine)
if len(fields) >= 2 {
if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
firstLine = "" // It's a header, skip it
_, errFirst := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64)
_, errLast := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64)
if errFirst != nil && errLast != nil {
firstLine = "" // Neither field is numeric — it's a header, skip it
}
}
}
@@ -109,7 +113,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
freqStr = strings.TrimSpace(fields[1])
}
if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
return nil
}
@@ -175,6 +179,15 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
return nil
}
func hasLetter(s string) bool {
for _, r := range s {
if unicode.IsLetter(r) {
return true
}
}
return false
}
// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
// Each byte in ISO-8859-1 maps directly to the same Unicode code point.
func latin1ToUTF8(b []byte) string {