Add loader tests and fix header detection and punctuation filtering
Tests: 23 new tests across cetempublico, affix, and wiktionary loaders covering column order detection, ISO-8859-1 encoding, junk filtering, frequency scaling, affix expansion, and sense-level synonym extraction. Fixes: header detection now checks either field for numeric value (was only checking last field, breaking count-first format). Added hasLetter filter to reject pure punctuation entries like "." and ",". Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
@@ -74,14 +75,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
|
||||
|
||||
// Peek first line to detect header vs data
|
||||
// Peek first line to detect header vs data.
|
||||
// A data line has at least one numeric field (either "word count" or "count word").
|
||||
var firstLine string
|
||||
if scanner.Scan() {
|
||||
firstLine = scanner.Text()
|
||||
fields := strings.Fields(firstLine)
|
||||
if len(fields) >= 2 {
|
||||
if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
|
||||
firstLine = "" // It's a header, skip it
|
||||
_, errFirst := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64)
|
||||
_, errLast := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64)
|
||||
if errFirst != nil && errLast != nil {
|
||||
firstLine = "" // Neither field is numeric — it's a header, skip it
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,7 +113,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
|
||||
freqStr = strings.TrimSpace(fields[1])
|
||||
}
|
||||
|
||||
if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
|
||||
if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -175,6 +179,15 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasLetter(s string) bool {
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
|
||||
// Each byte in ISO-8859-1 maps directly to the same Unicode code point.
|
||||
func latin1ToUTF8(b []byte) string {
|
||||
|
||||
Reference in New Issue
Block a user