Initial commit

internal/classifier/classifier.go

@@ -0,0 +1,169 @@
+package classifier
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"sync/atomic"
+
+	"pete/internal/ingestion"
+	"pete/internal/matrix"
+	"pete/internal/storage"
+)
+
+const recentHeadlineLimit = 50
+
+// ClassifyResult is the unified output of the classification pipeline.
+type ClassifyResult struct {
+	Channel     string
+	Platforms   []string
+	DuplicateOf *string
+	ShouldPost  bool
+	Rationale   string
+}
+
+// Classifier runs the classification pipeline for incoming stories.
+type Classifier struct {
+	ollama              *OllamaClient
+	matrixClient        *matrix.Client
+	consecutiveFailures atomic.Int32
+}
+
+// NewClassifier creates a new classifier.
+func NewClassifier(ollama *OllamaClient, matrixClient *matrix.Client) *Classifier {
+	return &Classifier{
+		ollama:       ollama,
+		matrixClient: matrixClient,
+	}
+}
+
+// Classify runs the appropriate classification pipeline for a feed item.
+func (c *Classifier) Classify(item *ingestion.FeedItem) (*ClassifyResult, error) {
+	// Direct routing — no LLM call needed
+	if item.DirectRoute != nil {
+		slog.Debug("direct route", "guid", item.GUID, "channel", *item.DirectRoute)
+
+		// Still check GUID dedup via recent headlines for semantic dedup
+		// but direct-routed stories always post
+		return &ClassifyResult{
+			Channel:    *item.DirectRoute,
+			ShouldPost: true,
+			Rationale:  "direct route from source config",
+		}, nil
+	}
+
+	// Get recent headlines for dedup context
+	recent, err := storage.GetRecentHeadlines(recentHeadlineLimit)
+	if err != nil {
+		slog.Error("failed to get recent headlines", "err", err)
+		recent = nil // continue without dedup context
+	}
+
+	if item.Tier == 2 {
+		return c.classifyTier2(item, recent)
+	}
+
+	return c.classifyTier1(item, recent)
+}
+
+func (c *Classifier) classifyTier1(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
+	prompt := BuildTier1Prompt(item, recent)
+
+	raw, err := c.ollama.Generate(prompt)
+	if err != nil {
+		c.handleOllamaFailure(err)
+		return nil, fmt.Errorf("ollama generate: %w", err)
+	}
+	c.handleOllamaSuccess()
+
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
+		return nil, fmt.Errorf("parse tier1: %w", err)
+	}
+
+	// Log classification
+	logResult(item.GUID, raw)
+
+	return &ClassifyResult{
+		Channel:     result.Channel,
+		Platforms:   result.Platforms,
+		DuplicateOf: result.DuplicateOf,
+		ShouldPost:  true, // Tier 1 always posts
+		Rationale:   result.Rationale,
+	}, nil
+}
+
+func (c *Classifier) classifyTier2(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
+	// Run keyword gating first
+	gatingResult, reason := RunGating(item.Headline, item.Lede)
+
+	switch gatingResult {
+	case GatingDiscard:
+		slog.Info("tier2 gating: discard", "guid", item.GUID, "reason", reason)
+		logResult(item.GUID, fmt.Sprintf(`{"gating":"discard","reason":%q}`, reason))
+		return &ClassifyResult{
+			ShouldPost: false,
+			Rationale:  reason,
+		}, nil
+
+	case GatingPost:
+		slog.Info("tier2 gating: definite post, routing via LLM", "guid", item.GUID, "reason", reason)
+		// Fall through to LLM for routing (but post=true is guaranteed)
+	}
+
+	// Stage 3 (ambiguous) or Stage 1 (definite post needing routing) — send to LLM
+	prompt := BuildTier2Prompt(item, recent)
+
+	raw, err := c.ollama.Generate(prompt)
+	if err != nil {
+		c.handleOllamaFailure(err)
+		return nil, fmt.Errorf("ollama generate: %w", err)
+	}
+	c.handleOllamaSuccess()
+
+	result, err := ParseTier2Response(raw)
+	if err != nil {
+		slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
+		return nil, fmt.Errorf("parse tier2: %w", err)
+	}
+
+	logResult(item.GUID, raw)
+
+	// If gating said definite post, override LLM's post decision
+	shouldPost := result.Post
+	if gatingResult == GatingPost {
+		shouldPost = true
+	}
+
+	return &ClassifyResult{
+		Channel:     result.Channel,
+		Platforms:   result.Platforms,
+		DuplicateOf: result.DuplicateOf,
+		ShouldPost:  shouldPost,
+		Rationale:   result.Rationale,
+	}, nil
+}
+
+func (c *Classifier) handleOllamaFailure(err error) {
+	n := c.consecutiveFailures.Add(1)
+	slog.Error("ollama failure", "err", err, "consecutive", n)
+	if n == 3 && c.matrixClient != nil {
+		c.matrixClient.SendAdminWarning(
+			fmt.Sprintf("Ollama unavailable for 3 consecutive attempts: %v", err))
+	}
+}
+
+func (c *Classifier) handleOllamaSuccess() {
+	c.consecutiveFailures.Store(0)
+}
+
+func logResult(guid, raw string) {
+	// Try to compact the JSON for cleaner logging
+	var compact json.RawMessage
+	if err := json.Unmarshal([]byte(raw), &compact); err == nil {
+		storage.InsertClassificationLog(guid, string(compact))
+	} else {
+		storage.InsertClassificationLog(guid, raw)
+	}
+}