From eb471b6f6de6487423e3ad42bab4ef47282bb076 Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Mon, 30 Mar 2026 21:27:45 -0700
Subject: [PATCH] Add word frequency endpoint and SCOWL frequency data

SCOWL loader now sets frequency based on tier (10=1000 most common,
70=50 rare). New GET /frequency?word=X&lang=Y endpoint returns frequency
score for a word. Enables downstream consumers to identify rare/
sophisticated vocabulary from dictionary data rather than LLM guessing.

Also fix .gitignore matching cmd/server/ directory instead of just the
server binary.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitignore                   |   4 +-
 cmd/server/main.go           | 367 +++++++++++++++++++++++++++++++++++
 internal/dictionary/query.go |  18 ++
 internal/loader/scowl.go     |  39 +++-
 4 files changed, 417 insertions(+), 11 deletions(-)
 create mode 100644 cmd/server/main.go

diff --git a/.gitignore b/.gitignore
index 064178d..4c37a1b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
 dict.db
 data/
 !data/.gitkeep
-dictimport
-server
+/dictimport
+/server
diff --git a/cmd/server/main.go b/cmd/server/main.go
new file mode 100644
index 0000000..6728171
--- /dev/null
+++ b/cmd/server/main.go
@@ -0,0 +1,367 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"dreamdict/internal/dictionary"
+)
+
+func main() {
+	port := flag.Int("port", 0, "Listen port (default 7777)")
+	host := flag.String("host", "", "Listen host (default 127.0.0.1)")
+	dbPath := flag.String("db", "", "Path to dict.db")
+	logLevel := flag.String("log-level", "info", "Log level: debug, info, warn, error")
+	flag.Parse()
+
+	// Resolve config: flag > env > default
+	if *port == 0 {
+		if v := os.Getenv("DREAMDICT_PORT"); v != "" {
+			if p, err := strconv.Atoi(v); err == nil {
+				*port = p
+			}
+		}
+	}
+	if *port == 0 {
+		*port = 7777
+	}
+	if *host == "" {
+		if v := os.Getenv("DREAMDICT_HOST"); v != "" {
+			*host = v
+		} else {
+			*host = "127.0.0.1"
+		}
+	}
+	if *dbPath == "" {
+		if v := os.Getenv("DREAMDICT_DB"); v != "" {
+			*dbPath = v
+		} else {
+			*dbPath = "./dict.db"
+		}
+	}
+
+	var level slog.Level
+	switch *logLevel {
+	case "debug":
+		level = slog.LevelDebug
+	case "warn":
+		level = slog.LevelWarn
+	case "error":
+		level = slog.LevelError
+	default:
+		level = slog.LevelInfo
+	}
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})))
+
+	dict, err := dictionary.NewReadOnly(*dbPath)
+	if err != nil {
+		slog.Error("failed to open dictionary", "error", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := dict.Close(); err != nil {
+			slog.Error("failed to close dictionary", "error", err)
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /valid", handleValid(dict))
+	mux.HandleFunc("GET /random", handleRandom(dict))
+	mux.HandleFunc("GET /define", handleDefine(dict))
+	mux.HandleFunc("GET /synonyms", handleSynonyms(dict))
+	mux.HandleFunc("GET /translate", handleTranslate(dict))
+	mux.HandleFunc("GET /frequency", handleFrequency(dict))
+	mux.HandleFunc("GET /health", handleHealth(dict, *dbPath))
+
+	addr := fmt.Sprintf("%s:%d", *host, *port)
+	srv := &http.Server{
+		Addr:         addr,
+		Handler:      loggingMiddleware(mux),
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+
+	go func() {
+		slog.Info("dreamdict listening", "addr", addr)
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			slog.Error("server error", "error", err)
+			os.Exit(1)
+		}
+	}()
+
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
+	<-quit
+
+	slog.Info("shutting down")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	srv.Shutdown(ctx)
+}
+
+// statusRecorder wraps ResponseWriter to capture the status code.
+type statusRecorder struct {
+	http.ResponseWriter
+	status int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	r.status = code
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func loggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		rec := &statusRecorder{ResponseWriter: w, status: 200}
+		next.ServeHTTP(rec, r)
+		slog.Debug("request",
+			"method", r.Method,
+			"path", r.URL.Path,
+			"query", r.URL.RawQuery,
+			"status", rec.status,
+			"duration", time.Since(start).Round(time.Microsecond),
+			"remote", r.RemoteAddr,
+		)
+	})
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(v)
+}
+
+func writeError(w http.ResponseWriter, status int, errCode, message string) {
+	writeJSON(w, status, map[string]string{"error": errCode, "message": message})
+}
+
+func handleValid(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		word := r.URL.Query().Get("word")
+		lang := r.URL.Query().Get("lang")
+		if word == "" || lang == "" {
+			writeError(w, 400, "bad_request", "word and lang are required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		valid, err := dict.IsValidWord(word, lang)
+		if err != nil {
+			slog.Error("valid", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]bool{"valid": valid})
+	}
+}
+
+func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		lang := r.URL.Query().Get("lang")
+		if lang == "" {
+			writeError(w, 400, "bad_request", "lang is required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		opts := dictionary.Options{
+			POS: r.URL.Query().Get("pos"),
+		}
+		if v := r.URL.Query().Get("min"); v != "" {
+			if n, err := strconv.Atoi(v); err == nil {
+				opts.MinLength = n
+			}
+		}
+		if v := r.URL.Query().Get("max"); v != "" {
+			if n, err := strconv.Atoi(v); err == nil {
+				opts.MaxLength = n
+			}
+		}
+		if v := r.URL.Query().Get("min_freq"); v != "" {
+			if n, err := strconv.Atoi(v); err == nil {
+				opts.MinFrequency = n
+			}
+		}
+
+		word, err := dict.RandomWord(lang, opts)
+		if err == dictionary.ErrNoMatch {
+			writeError(w, 404, "no_match", "no words match the given filters")
+			return
+		}
+		if err != nil {
+			slog.Error("random", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]string{"word": word})
+	}
+}
+
+func handleDefine(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		word := r.URL.Query().Get("word")
+		lang := r.URL.Query().Get("lang")
+		if word == "" || lang == "" {
+			writeError(w, 400, "bad_request", "word and lang are required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		defs, err := dict.Define(word, lang)
+		if err != nil {
+			slog.Error("define", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]any{
+			"word":        word,
+			"lang":        lang,
+			"definitions": defs,
+		})
+	}
+}
+
+func handleSynonyms(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		word := r.URL.Query().Get("word")
+		lang := r.URL.Query().Get("lang")
+		if word == "" || lang == "" {
+			writeError(w, 400, "bad_request", "word and lang are required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		syns, err := dict.Synonyms(word, lang)
+		if err != nil {
+			slog.Error("synonyms", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]any{
+			"word":     word,
+			"lang":     lang,
+			"synonyms": syns,
+		})
+	}
+}
+
+func handleTranslate(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		word := r.URL.Query().Get("word")
+		from := r.URL.Query().Get("from")
+		to := r.URL.Query().Get("to")
+		if word == "" || from == "" || to == "" {
+			writeError(w, 400, "bad_request", "word, from, and to are required")
+			return
+		}
+		if !dictionary.ValidLang(from) {
+			writeError(w, 400, "bad_request", "unsupported language: "+from)
+			return
+		}
+		if !dictionary.ValidLang(to) {
+			writeError(w, 400, "bad_request", "unsupported language: "+to)
+			return
+		}
+
+		trans, err := dict.Translate(word, from, to)
+		if err != nil {
+			slog.Error("translate", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]any{
+			"word":         word,
+			"from":         from,
+			"to":           to,
+			"translations": trans,
+		})
+	}
+}
+
+func handleFrequency(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		word := r.URL.Query().Get("word")
+		lang := r.URL.Query().Get("lang")
+		if word == "" || lang == "" {
+			writeError(w, 400, "bad_request", "word and lang are required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		freq, err := dict.Frequency(word, lang)
+		if err != nil {
+			slog.Error("frequency", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]any{
+			"word":      word,
+			"lang":      lang,
+			"frequency": freq,
+		})
+	}
+}
+
+func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		wordCounts, err := dict.WordCount()
+		if err != nil {
+			slog.Error("health: word count", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		defCounts, err := dict.DefCount()
+		if err != nil {
+			slog.Error("health: def count", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+
+		importedAt, err := dict.Meta("imported_at")
+		if err != nil {
+			slog.Error("health: imported_at", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		schemaVersion, err := dict.Meta("schema_version")
+		if err != nil {
+			slog.Error("health: schema_version", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+
+		writeJSON(w, 200, map[string]any{
+			"status":         "ok",
+			"db_path":        dbPath,
+			"word_counts":    wordCounts,
+			"def_counts":     defCounts,
+			"imported_at":    importedAt,
+			"schema_version": schemaVersion,
+		})
+	}
+}
diff --git a/internal/dictionary/query.go b/internal/dictionary/query.go
index 9aae969..62d536b 100644
--- a/internal/dictionary/query.go
+++ b/internal/dictionary/query.go
@@ -157,6 +157,24 @@ func (d *Dictionary) WordCount() (map[string]int, error) {
 	return counts, rows.Err()
 }
 
+// Frequency returns the frequency score for a word in a language.
+// Returns 0 if the word is not found or has no frequency data.
+// Higher values indicate more common words.
+func (d *Dictionary) Frequency(word, lang string) (int, error) {
+	var freq int
+	err := d.db.QueryRow(
+		"SELECT COALESCE(frequency, 0) FROM words WHERE word = ? AND lang = ?",
+		strings.ToLower(word), lang,
+	).Scan(&freq)
+	if err == sql.ErrNoRows {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, fmt.Errorf("dictionary: frequency: %w", err)
+	}
+	return freq, nil
+}
+
 func (d *Dictionary) Meta(key string) (string, error) {
 	var val string
 	err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val)
diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go
index a0d0f1c..73ce252 100644
--- a/internal/loader/scowl.go
+++ b/internal/loader/scowl.go
@@ -14,6 +14,26 @@ type SCOWLLoader struct{}
 
 func (SCOWLLoader) Name() string { return "scowl" }
 
+// scowlFrequency maps SCOWL tier to a frequency score.
+// Higher = more common, matching the Lexique convention.
+// SCOWL 10 = most common words, 70 = rare/specialized.
+var scowlFrequency = map[int]int{
+	10: 1000,
+	20: 800,
+	35: 600,
+	40: 500,
+	50: 400,
+	55: 300,
+	60: 200,
+	65: 100,
+	70: 50,
+}
+
+type scowlFile struct {
+	path string
+	size int
+}
+
 func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*")
 	files, err := filepath.Glob(pattern)
@@ -24,8 +44,8 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 		return fmt.Errorf("scowl: no files matching %s", pattern)
 	}
 
-	// Filter to sizes <= 70
-	var selected []string
+	// Filter to sizes <= 70 and track the tier
+	var selected []scowlFile
 	for _, f := range files {
 		base := filepath.Base(f)
 		// Files are like english-words.10, english-words.20, etc.
@@ -38,7 +58,7 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 			continue
 		}
 		if size <= 70 {
-			selected = append(selected, f)
+			selected = append(selected, scowlFile{path: f, size: size})
 		}
 	}
 
@@ -48,17 +68,18 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer tx.Rollback()
 
-	stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'en')")
+	stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'en', ?)")
 	if err != nil {
 		return fmt.Errorf("scowl: prepare: %w", err)
 	}
 	defer stmt.Close()
 
 	var count int
-	for _, f := range selected {
-		n, err := loadSCOWLFile(stmt, f)
+	for _, sf := range selected {
+		freq := scowlFrequency[sf.size]
+		n, err := loadSCOWLFile(stmt, sf.path, freq)
 		if err != nil {
-			return fmt.Errorf("scowl: load %s: %w", f, err)
+			return fmt.Errorf("scowl: load %s: %w", sf.path, err)
 		}
 		count += n
 	}
@@ -70,7 +91,7 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	return nil
 }
 
-func loadSCOWLFile(stmt *sql.Stmt, path string) (int, error) {
+func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return 0, err
@@ -88,7 +109,7 @@ func loadSCOWLFile(stmt *sql.Stmt, path string) (int, error) {
 		if containsAny(word, " ", "'", "-") || containsDigit(word) {
 			continue
 		}
-		if _, err := stmt.Exec(word); err != nil {
+		if _, err := stmt.Exec(word, freq); err != nil {
 			return 0, err
 		}
 		count++