dreamdict/internal/loader/scowl.go

package loader

import (
	"bufio"
	"database/sql"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"unicode"
)

type SCOWLLoader struct{}

func (SCOWLLoader) Name() string { return "scowl" }

// scowlFrequency maps SCOWL tier to a frequency score.
// Higher = more common, matching the Lexique convention.
// SCOWL 10 = most common words, 70 = rare/specialized.
var scowlFrequency = map[int]int{
	10: 1000,
	20: 800,
	35: 600,
	40: 500,
	50: 400,
	55: 300,
	60: 200,
	65: 100,
	70: 50,
}

type scowlFile struct {
	path    string
	size    int
	variant string // "us", "gb", or "" for common English
}

func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
	type prefixVariant struct {
		prefix  string
		variant string
	}
	prefixes := []prefixVariant{
		{"english-words", ""},
		{"american-words", "us"},
		{"british-words", "gb"},
	}

	var allFiles []string
	for _, pv := range prefixes {
		pattern := filepath.Join(dataDir, "scowl", "final", pv.prefix+".*")
		matches, err := filepath.Glob(pattern)
		if err != nil {
			return fmt.Errorf("scowl: glob: %w", err)
		}
		allFiles = append(allFiles, matches...)
	}
	if len(allFiles) == 0 {
		return fmt.Errorf("scowl: no word files found in %s", filepath.Join(dataDir, "scowl", "final"))
	}

	// Build a map from prefix to variant for quick lookup
	prefixToVariant := make(map[string]string)
	for _, pv := range prefixes {
		prefixToVariant[pv.prefix] = pv.variant
	}

	// Filter to sizes <= 70 and track the tier + variant
	var selected []scowlFile
	for _, f := range allFiles {
		base := filepath.Base(f)
		// Files are like english-words.10, american-words.35, etc.
		parts := strings.SplitN(base, ".", 2)
		if len(parts) != 2 {
			continue
		}
		prefix := parts[0]
		variant, ok := prefixToVariant[prefix]
		if !ok {
			continue
		}
		var size int
		if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil {
			continue
		}
		if size <= 70 {
			selected = append(selected, scowlFile{path: f, size: size, variant: variant})
		}
	}

	tx, err := db.Begin()
	if err != nil {
		return fmt.Errorf("scowl: begin tx: %w", err)
	}
	defer tx.Rollback()

	stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency, variant) VALUES (?, 'en', ?, ?)
		ON CONFLICT(word, lang) DO UPDATE SET
			frequency = MAX(frequency, excluded.frequency),
			variant = CASE
				WHEN words.variant IS NULL THEN excluded.variant
				WHEN excluded.variant IS NULL THEN words.variant
				WHEN words.variant = excluded.variant THEN words.variant
				ELSE NULL
			END`)
	if err != nil {
		return fmt.Errorf("scowl: prepare: %w", err)
	}
	defer stmt.Close()

	var count int
	for _, sf := range selected {
		freq := scowlFrequency[sf.size]
		n, err := loadSCOWLFile(stmt, sf.path, freq, sf.variant)
		if err != nil {
			return fmt.Errorf("scowl: load %s: %w", sf.path, err)
		}
		count += n
	}

	if err := tx.Commit(); err != nil {
		return fmt.Errorf("scowl: commit: %w", err)
	}
	slog.Info("scowl loaded", "words", count, "files", len(selected))
	return nil
}

func loadSCOWLFile(stmt *sql.Stmt, path string, freq int, variant string) (int, error) {
	f, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	// Convert empty variant to SQL NULL
	var variantParam any
	if variant != "" {
		variantParam = variant
	}

	var count int
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		word := strings.ToLower(line)
		if containsAny(word, " ", "'", "-") || containsDigit(word) {
			continue
		}
		if _, err := stmt.Exec(word, freq, variantParam); err != nil {
			return 0, err
		}
		count++
	}
	return count, scanner.Err()
}

func containsAny(s string, subs ...string) bool {
	for _, sub := range subs {
		if strings.Contains(s, sub) {
			return true
		}
	}
	return false
}

func containsDigit(s string) bool {
	for _, r := range s {
		if r >= '0' && r <= '9' {
			return true
		}
	}
	return false
}

func containsSpace(s string) bool {
	return strings.ContainsRune(s, ' ')
}

func containsNonLatin(s string) bool {
	for _, r := range s {
		if r > 0x024F && r != '\'' && r != '-' {
			return true
		}
	}
	return false
}

func hasLetter(s string) bool {
	for _, r := range s {
		if unicode.IsLetter(r) {
			return true
		}
	}
	return false
}

// isJunkWord returns true if the word should be rejected from the dictionary.
func isJunkWord(word string) bool {
	return word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word)
}