package llm import ( "context" "encoding/json" "fmt" "strings" "sync" "time" ) // CheckpointInterval is the minimum time between grammar checkpoints for a // single document. The frontend debounces at 4s; this is the server-side floor // that protects the inference endpoint from rapid repeat checks. const CheckpointInterval = 30 * time.Second // RawSuggestion is one item as the model emits it. Positions are resolved // server-side from Original; the frontend re-anchors by string at render time. type RawSuggestion struct { Original string `json:"original"` Replacement string `json:"replacement"` Explanation string `json:"explanation"` Type string `json:"type"` } // checkpointResponse is the top-level JSON shape the checkpoint prompt requests. type checkpointResponse struct { Suggestions []RawSuggestion `json:"suggestions"` } // RunCheckpoint sends the grammar checkpoint and parses the JSON result. It // applies the latency-guard truncation and the checkpoint sampling parameters // from the spec. func RunCheckpoint(ctx context.Context, client LLMClient, contentText string) ([]RawSuggestion, error) { raw, err := client.Complete(ctx, CompletionRequest{ Messages: CheckpointMessages(TruncateDoc(contentText)), MaxTokens: 1024, Temperature: 0.3, RepetitionPenalty: 1.15, TopP: 0.9, Stop: []string{"```", "\n\n\n\n"}, }) if err != nil { return nil, err } return ParseCheckpoint(raw) } // ParseCheckpoint extracts the suggestions array from a model response. Smaller // models sometimes wrap JSON in prose or markdown fences despite instructions, // so we salvage the outermost {...} object before decoding. func ParseCheckpoint(raw string) ([]RawSuggestion, error) { jsonText := extractJSONObject(raw) if jsonText == "" { return nil, fmt.Errorf("checkpoint: no JSON object in model output: %q", truncateForError(raw)) } var parsed checkpointResponse if err := json.Unmarshal([]byte(jsonText), &parsed); err != nil { return nil, fmt.Errorf("checkpoint: parse JSON: %w (got %q)", err, truncateForError(jsonText)) } // Drop items the model returned with an empty original — they can't be // anchored — and normalize whitespace the model may have echoed. out := parsed.Suggestions[:0] for _, s := range parsed.Suggestions { s.Original = strings.TrimSpace(s.Original) s.Replacement = strings.TrimSpace(s.Replacement) if s.Original == "" { continue } out = append(out, s) } return out, nil } // extractJSONObject returns the substring from the first '{' to its matching // closing '}', or "" if none. Tolerates fences/preamble around the object. func extractJSONObject(s string) string { start := strings.IndexByte(s, '{') if start < 0 { return "" } depth := 0 inString := false escaped := false for i := start; i < len(s); i++ { c := s[i] switch { case escaped: escaped = false case c == '\\' && inString: escaped = true case c == '"': inString = !inString case inString: // ignore braces inside strings case c == '{': depth++ case c == '}': depth-- if depth == 0 { return s[start : i+1] } } } return "" } func truncateForError(s string) string { const max = 200 if len(s) > max { return s[:max] + "…" } return s } // RateLimiter enforces a minimum interval between checkpoints per document. // Concurrency-safe; one instance is shared across requests. type RateLimiter struct { mu sync.Mutex interval time.Duration last map[string]time.Time } // NewRateLimiter constructs a limiter with the given per-document minimum gap. func NewRateLimiter(interval time.Duration) *RateLimiter { return &RateLimiter{interval: interval, last: make(map[string]time.Time)} } // Allow reports whether a checkpoint may run for docID now. When allowed it // records the time and returns (true, 0); when throttled it returns // (false, retryAfter) where retryAfter is the wait until the next allowed run. func (rl *RateLimiter) Allow(docID string) (bool, time.Duration) { rl.mu.Lock() defer rl.mu.Unlock() now := time.Now() if last, ok := rl.last[docID]; ok { if elapsed := now.Sub(last); elapsed < rl.interval { return false, rl.interval - elapsed } } rl.last[docID] = now return true, 0 }