package classifier import ( "encoding/json" "fmt" "log/slog" "sync/atomic" "pete/internal/ingestion" "pete/internal/matrix" "pete/internal/storage" ) const recentHeadlineLimit = 50 // ClassifyResult is the unified output of the classification pipeline. type ClassifyResult struct { Channel string Platforms []string DuplicateOf *string ShouldPost bool Rationale string } // Classifier runs the classification pipeline for incoming stories. type Classifier struct { ollama *OllamaClient matrixClient *matrix.Client consecutiveFailures atomic.Int32 } // NewClassifier creates a new classifier. func NewClassifier(ollama *OllamaClient, matrixClient *matrix.Client) *Classifier { return &Classifier{ ollama: ollama, matrixClient: matrixClient, } } // Classify runs the appropriate classification pipeline for a feed item. func (c *Classifier) Classify(item *ingestion.FeedItem) (*ClassifyResult, error) { // Direct routing — no LLM call needed if item.DirectRoute != nil { slog.Debug("direct route", "guid", item.GUID, "channel", *item.DirectRoute) // Still check GUID dedup via recent headlines for semantic dedup // but direct-routed stories always post return &ClassifyResult{ Channel: *item.DirectRoute, ShouldPost: true, Rationale: "direct route from source config", }, nil } // Get recent headlines for dedup context recent, err := storage.GetRecentHeadlines(recentHeadlineLimit) if err != nil { slog.Error("failed to get recent headlines", "err", err) recent = nil // continue without dedup context } if item.Tier == 2 { return c.classifyTier2(item, recent) } return c.classifyTier1(item, recent) } func (c *Classifier) classifyTier1(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) { prompt := BuildTier1Prompt(item, recent) raw, err := c.ollama.Generate(prompt) if err != nil { c.handleOllamaFailure(err) return nil, fmt.Errorf("ollama generate: %w", err) } c.handleOllamaSuccess() result, err := ParseTier1Response(raw) if err != nil { slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err) return nil, fmt.Errorf("parse tier1: %w", err) } // Log classification logResult(item.GUID, raw) return &ClassifyResult{ Channel: result.Channel, Platforms: result.Platforms, DuplicateOf: result.DuplicateOf, ShouldPost: true, // Tier 1 always posts Rationale: result.Rationale, }, nil } func (c *Classifier) classifyTier2(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) { // Run keyword gating first gatingResult, reason := RunGating(item.Headline, item.Lede) switch gatingResult { case GatingDiscard: slog.Info("tier2 gating: discard", "guid", item.GUID, "reason", reason) logResult(item.GUID, fmt.Sprintf(`{"gating":"discard","reason":%q}`, reason)) return &ClassifyResult{ ShouldPost: false, Rationale: reason, }, nil case GatingPost: slog.Info("tier2 gating: definite post, routing via LLM", "guid", item.GUID, "reason", reason) // Fall through to LLM for routing (but post=true is guaranteed) } // Stage 3 (ambiguous) or Stage 1 (definite post needing routing) — send to LLM prompt := BuildTier2Prompt(item, recent) raw, err := c.ollama.Generate(prompt) if err != nil { c.handleOllamaFailure(err) return nil, fmt.Errorf("ollama generate: %w", err) } c.handleOllamaSuccess() result, err := ParseTier2Response(raw) if err != nil { slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err) return nil, fmt.Errorf("parse tier2: %w", err) } logResult(item.GUID, raw) // If gating said definite post, override LLM's post decision shouldPost := result.Post if gatingResult == GatingPost { shouldPost = true } return &ClassifyResult{ Channel: result.Channel, Platforms: result.Platforms, DuplicateOf: result.DuplicateOf, ShouldPost: shouldPost, Rationale: result.Rationale, }, nil } func (c *Classifier) handleOllamaFailure(err error) { n := c.consecutiveFailures.Add(1) slog.Error("ollama failure", "err", err, "consecutive", n) if n == 3 && c.matrixClient != nil { c.matrixClient.SendAdminWarning( fmt.Sprintf("Ollama unavailable for 3 consecutive attempts: %v", err)) } } func (c *Classifier) handleOllamaSuccess() { c.consecutiveFailures.Store(0) } func logResult(guid, raw string) { // Try to compact the JSON for cleaner logging var compact json.RawMessage if err := json.Unmarshal([]byte(raw), &compact); err == nil { storage.InsertClassificationLog(guid, string(compact)) } else { storage.InsertClassificationLog(guid, raw) } }