Initial commit

internal/classifier/classifier.go

@@ -0,0 +1,169 @@
+package classifier
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"sync/atomic"
+
+	"pete/internal/ingestion"
+	"pete/internal/matrix"
+	"pete/internal/storage"
+)
+
+const recentHeadlineLimit = 50
+
+// ClassifyResult is the unified output of the classification pipeline.
+type ClassifyResult struct {
+	Channel     string
+	Platforms   []string
+	DuplicateOf *string
+	ShouldPost  bool
+	Rationale   string
+}
+
+// Classifier runs the classification pipeline for incoming stories.
+type Classifier struct {
+	ollama              *OllamaClient
+	matrixClient        *matrix.Client
+	consecutiveFailures atomic.Int32
+}
+
+// NewClassifier creates a new classifier.
+func NewClassifier(ollama *OllamaClient, matrixClient *matrix.Client) *Classifier {
+	return &Classifier{
+		ollama:       ollama,
+		matrixClient: matrixClient,
+	}
+}
+
+// Classify runs the appropriate classification pipeline for a feed item.
+func (c *Classifier) Classify(item *ingestion.FeedItem) (*ClassifyResult, error) {
+	// Direct routing — no LLM call needed
+	if item.DirectRoute != nil {
+		slog.Debug("direct route", "guid", item.GUID, "channel", *item.DirectRoute)
+
+		// Still check GUID dedup via recent headlines for semantic dedup
+		// but direct-routed stories always post
+		return &ClassifyResult{
+			Channel:    *item.DirectRoute,
+			ShouldPost: true,
+			Rationale:  "direct route from source config",
+		}, nil
+	}
+
+	// Get recent headlines for dedup context
+	recent, err := storage.GetRecentHeadlines(recentHeadlineLimit)
+	if err != nil {
+		slog.Error("failed to get recent headlines", "err", err)
+		recent = nil // continue without dedup context
+	}
+
+	if item.Tier == 2 {
+		return c.classifyTier2(item, recent)
+	}
+
+	return c.classifyTier1(item, recent)
+}
+
+func (c *Classifier) classifyTier1(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
+	prompt := BuildTier1Prompt(item, recent)
+
+	raw, err := c.ollama.Generate(prompt)
+	if err != nil {
+		c.handleOllamaFailure(err)
+		return nil, fmt.Errorf("ollama generate: %w", err)
+	}
+	c.handleOllamaSuccess()
+
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
+		return nil, fmt.Errorf("parse tier1: %w", err)
+	}
+
+	// Log classification
+	logResult(item.GUID, raw)
+
+	return &ClassifyResult{
+		Channel:     result.Channel,
+		Platforms:   result.Platforms,
+		DuplicateOf: result.DuplicateOf,
+		ShouldPost:  true, // Tier 1 always posts
+		Rationale:   result.Rationale,
+	}, nil
+}
+
+func (c *Classifier) classifyTier2(item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
+	// Run keyword gating first
+	gatingResult, reason := RunGating(item.Headline, item.Lede)
+
+	switch gatingResult {
+	case GatingDiscard:
+		slog.Info("tier2 gating: discard", "guid", item.GUID, "reason", reason)
+		logResult(item.GUID, fmt.Sprintf(`{"gating":"discard","reason":%q}`, reason))
+		return &ClassifyResult{
+			ShouldPost: false,
+			Rationale:  reason,
+		}, nil
+
+	case GatingPost:
+		slog.Info("tier2 gating: definite post, routing via LLM", "guid", item.GUID, "reason", reason)
+		// Fall through to LLM for routing (but post=true is guaranteed)
+	}
+
+	// Stage 3 (ambiguous) or Stage 1 (definite post needing routing) — send to LLM
+	prompt := BuildTier2Prompt(item, recent)
+
+	raw, err := c.ollama.Generate(prompt)
+	if err != nil {
+		c.handleOllamaFailure(err)
+		return nil, fmt.Errorf("ollama generate: %w", err)
+	}
+	c.handleOllamaSuccess()
+
+	result, err := ParseTier2Response(raw)
+	if err != nil {
+		slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
+		return nil, fmt.Errorf("parse tier2: %w", err)
+	}
+
+	logResult(item.GUID, raw)
+
+	// If gating said definite post, override LLM's post decision
+	shouldPost := result.Post
+	if gatingResult == GatingPost {
+		shouldPost = true
+	}
+
+	return &ClassifyResult{
+		Channel:     result.Channel,
+		Platforms:   result.Platforms,
+		DuplicateOf: result.DuplicateOf,
+		ShouldPost:  shouldPost,
+		Rationale:   result.Rationale,
+	}, nil
+}
+
+func (c *Classifier) handleOllamaFailure(err error) {
+	n := c.consecutiveFailures.Add(1)
+	slog.Error("ollama failure", "err", err, "consecutive", n)
+	if n == 3 && c.matrixClient != nil {
+		c.matrixClient.SendAdminWarning(
+			fmt.Sprintf("Ollama unavailable for 3 consecutive attempts: %v", err))
+	}
+}
+
+func (c *Classifier) handleOllamaSuccess() {
+	c.consecutiveFailures.Store(0)
+}
+
+func logResult(guid, raw string) {
+	// Try to compact the JSON for cleaner logging
+	var compact json.RawMessage
+	if err := json.Unmarshal([]byte(raw), &compact); err == nil {
+		storage.InsertClassificationLog(guid, string(compact))
+	} else {
+		storage.InsertClassificationLog(guid, raw)
+	}
+}

internal/classifier/gating.go

@@ -0,0 +1,77 @@
+package classifier
+
+import "regexp"
+
+// GatingResult indicates the outcome of keyword-based gating for Tier 2 stories.
+type GatingResult int
+
+const (
+	GatingPost      GatingResult = iota // Stage 1: definite post
+	GatingDiscard                       // Stage 2: definite discard
+	GatingAmbiguous                     // Stage 3: escalate to LLM
+)
+
+// Stage 1: Definite post patterns — stories with clear public significance.
+// Patterns use multi-word context to reduce false positives (e.g. "Star Wars" won't match).
+var postPatterns = []*regexp.Regexp{
+	// Named heads of state / major international bodies
+	regexp.MustCompile(`(?i)\b(president|prime minister|chancellor|pope)\s+\w+`),
+	regexp.MustCompile(`(?i)\b(UN|United Nations|WHO|NATO|EU|European Union|IMF|World Bank)\b`),
+
+	// Casualty language with numbers
+	regexp.MustCompile(`(?i)\b\d+\s*(dead|killed|injured|casualties|wounded)\b`),
+	regexp.MustCompile(`(?i)\b(dead|killed|injured|casualties|wounded)\s*\d+\b`),
+
+	// Disaster language — require a location-like context word nearby
+	regexp.MustCompile(`(?i)\b(earthquake|hurricane|typhoon|tsunami|wildfire|tornado)\b.*\b(hit|struck|destroyed|devastated|kills|deaths)\b`),
+
+	// Conflict language — require multi-word phrases to avoid "Star Wars", "culture war"
+	regexp.MustCompile(`(?i)\b(military invasion|armed coup|airstrike\s+\w+|assassination\s+\w+|car bombing|suicide bombing)\b`),
+	regexp.MustCompile(`(?i)\bwar\s+(in|on|with|against|between)\b`),
+
+	// Crisis language — require context word nearby
+	regexp.MustCompile(`(?i)\b(state of emergency|economic crisis|economic collapse|disease outbreak|pandemic declared|pandemic phase|famine)\b`),
+
+	// Major financial scale
+	regexp.MustCompile(`(?i)\$\d+\s*(bn|tn|billion|trillion)\b`),
+	regexp.MustCompile(`(?i)\b(trillion|billion)\s*(dollar|euro|pound)\b`),
+}
+
+// Stage 2: Definite discard patterns — stories unlikely to warrant posting.
+var discardPatterns = []*regexp.Regexp{
+	// Local/municipal scope
+	regexp.MustCompile(`(?i)\b(city council|municipal|town hall|zoning|local police)\b`),
+
+	// Sports scores and routine results
+	regexp.MustCompile(`(?i)\b(final score|match result|league standings|defeated|innings)\b`),
+	regexp.MustCompile(`(?i)\b\d+-\d+\s*(win|loss|defeat|victory)\b`),
+
+	// Entertainment/celebrity lifestyle
+	regexp.MustCompile(`(?i)\b(red carpet|award show|celebrity gossip|dating rumor|relationship drama|album release)\b`),
+
+	// Routine corporate earnings
+	regexp.MustCompile(`(?i)\b(quarterly earnings|Q[1-4] results|earnings call|fiscal quarter)\b`),
+}
+
+// RunGating evaluates Tier 2 gating stages on headline + lede.
+// Returns the gating result and a reason string for logging.
+func RunGating(headline, lede string) (GatingResult, string) {
+	combined := headline + " " + lede
+
+	// Stage 1: Definite post
+	for _, re := range postPatterns {
+		if re.MatchString(combined) {
+			return GatingPost, "stage1: " + re.String()
+		}
+	}
+
+	// Stage 2: Definite discard
+	for _, re := range discardPatterns {
+		if re.MatchString(combined) {
+			return GatingDiscard, "stage2: " + re.String()
+		}
+	}
+
+	// Stage 3: Ambiguous — escalate to LLM
+	return GatingAmbiguous, "stage3: no keyword match"
+}

internal/classifier/gating_test.go

@@ -0,0 +1,74 @@
+package classifier
+
+import "testing"
+
+func TestGating_Stage1_DefinitePost(t *testing.T) {
+	tests := []struct {
+		name     string
+		headline string
+		lede     string
+	}{
+		{"head of state", "President signs new climate bill", "The president signed legislation today."},
+		{"international body", "WHO declares new pandemic phase", "The World Health Organization announced a new phase."},
+		{"casualties", "12 killed in factory explosion", "Authorities report 12 dead after an explosion."},
+		{"disaster", "Earthquake strikes northern Japan", "A major earthquake hit the region early Tuesday."},
+		{"conflict", "Airstrike hits hospital in conflict zone", "An airstrike damaged a hospital."},
+		{"crisis", "Economic crisis deepens in Argentina", "The ongoing crisis has intensified."},
+		{"financial scale", "EU approves $50bn climate fund", "The European Union approved funding."},
+		{"assassination", "Assassination attempt on opposition leader", "An armed assailant attacked the leader."},
+		{"NATO", "NATO expands eastern flank deployment", "Alliance members agreed to new deployments."},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, reason := RunGating(tt.headline, tt.lede)
+			if result != GatingPost {
+				t.Errorf("expected GatingPost, got %d (reason: %s)", result, reason)
+			}
+		})
+	}
+}
+
+func TestGating_Stage2_DefiniteDiscard(t *testing.T) {
+	tests := []struct {
+		name     string
+		headline string
+		lede     string
+	}{
+		{"local government", "City council approves new parking meters", "The council voted 5-3 in favor."},
+		{"sports scores", "Lakers defeat Celtics 112-98 in regular season", "Final score 112-98."},
+		{"celebrity gossip", "Celebrity couple spotted on red carpet at Cannes", "The pair arrived together."},
+		{"routine earnings", "Regional bank reports Q2 results slightly above estimates", "Quarterly earnings were released."},
+		{"entertainment", "Pop star announces new album release date", "Fans have been waiting since 2023."},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, reason := RunGating(tt.headline, tt.lede)
+			if result != GatingDiscard {
+				t.Errorf("expected GatingDiscard, got %d (reason: %s)", result, reason)
+			}
+		})
+	}
+}
+
+func TestGating_Stage3_Ambiguous(t *testing.T) {
+	tests := []struct {
+		name     string
+		headline string
+		lede     string
+	}{
+		{"tech product", "Apple unveils new MacBook Pro with M5 chip", "The new laptops feature significant performance gains."},
+		{"policy nuanced", "Government considers new data privacy framework", "Ministers met to discuss potential changes."},
+		{"science", "Researchers discover high-temperature superconductor", "A team at MIT published findings in Nature."},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, reason := RunGating(tt.headline, tt.lede)
+			if result != GatingAmbiguous {
+				t.Errorf("expected GatingAmbiguous, got %d (reason: %s)", result, reason)
+			}
+		})
+	}
+}

internal/classifier/ollama.go

@@ -0,0 +1,104 @@
+package classifier
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"time"
+
+	"pete/internal/config"
+)
+
+// OllamaClient handles communication with the Ollama API.
+type OllamaClient struct {
+	baseURL    string
+	model      string
+	httpClient *http.Client
+}
+
+type chatRequest struct {
+	Model    string        `json:"model"`
+	Messages []chatMessage `json:"messages"`
+	Stream   bool          `json:"stream"`
+	Format   string        `json:"format"`
+}
+
+type chatMessage struct {
+	Role    string `json:"role"`
+	Content string `json:"content"`
+}
+
+type chatResponse struct {
+	Message struct {
+		Content string `json:"content"`
+	} `json:"message"`
+}
+
+// NewOllamaClient creates an Ollama API client from config.
+func NewOllamaClient(cfg config.OllamaConfig) *OllamaClient {
+	return &OllamaClient{
+		baseURL: strings.TrimRight(cfg.BaseURL, "/"),
+		model:   cfg.Model,
+		httpClient: &http.Client{
+			Timeout: time.Duration(cfg.TimeoutSeconds) * time.Second,
+		},
+	}
+}
+
+// Generate sends a prompt to Ollama via the chat API and returns the raw response text.
+func (o *OllamaClient) Generate(prompt string) (string, error) {
+	reqBody := chatRequest{
+		Model: o.model,
+		Messages: []chatMessage{
+			{
+				Role:    "system",
+				Content: "You are a JSON-only news classification API. Respond with valid JSON only. No preamble, no explanation, no markdown.",
+			},
+			{
+				Role:    "user",
+				Content: prompt,
+			},
+		},
+		Stream: false,
+		Format: "json",
+	}
+
+	body, err := json.Marshal(reqBody)
+	if err != nil {
+		return "", fmt.Errorf("marshal request: %w", err)
+	}
+
+	url := o.baseURL + "/api/chat"
+	slog.Debug("ollama: calling", "url", url, "model", o.model)
+
+	resp, err := o.httpClient.Post(url, "application/json", bytes.NewReader(body))
+	if err != nil {
+		return "", fmt.Errorf("ollama request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1*1024*1024)) // 1MB cap
+	if err != nil {
+		return "", fmt.Errorf("read ollama response: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("ollama status %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var chatResp chatResponse
+	if err := json.Unmarshal(respBody, &chatResp); err != nil {
+		return "", fmt.Errorf("decode ollama response: %w (body: %s)", err, truncate(string(respBody), 200))
+	}
+
+	result := strings.TrimSpace(chatResp.Message.Content)
+	if result == "" {
+		return "", fmt.Errorf("ollama returned empty response (body: %s)", truncate(string(respBody), 200))
+	}
+
+	return result, nil
+}

internal/classifier/parse.go

@@ -0,0 +1,123 @@
+package classifier
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// Tier1Result is the LLM response for Tier 1 (routing only) stories.
+type Tier1Result struct {
+	Channel     string   `json:"channel"`
+	Platforms   []string `json:"platforms"`
+	DuplicateOf *string  `json:"duplicate_of"`
+	Rationale   string   `json:"rationale"`
+}
+
+// Tier2Result is the LLM response for Tier 2 (gating + routing) stories.
+type Tier2Result struct {
+	Post        bool     `json:"post"`
+	Channel     string   `json:"channel"`
+	Platforms   []string `json:"platforms"`
+	DuplicateOf *string  `json:"duplicate_of"`
+	Rationale   string   `json:"rationale"`
+}
+
+var validChannels = map[string]bool{
+	"tech":     true,
+	"politics": true,
+	"gaming":   true,
+}
+
+var validPlatforms = map[string]bool{
+	"nintendo-switch": true,
+	"playstation":     true,
+	"xbox":            true,
+	"pc":              true,
+	"retro":           true,
+	"mobile":          true,
+	"multi":           true,
+}
+
+var thinkRe = regexp.MustCompile(`(?s)<think>.*?</think>`)
+var fenceRe = regexp.MustCompile("(?s)```(?:json)?\\s*(.+?)\\s*```")
+var trailingCommaRe = regexp.MustCompile(`,\s*([}\]])`)
+
+// ParseTier1Response parses an LLM response into a Tier1Result.
+func ParseTier1Response(raw string) (*Tier1Result, error) {
+	cleaned := repairJSON(raw)
+
+	var result Tier1Result
+	if err := json.Unmarshal([]byte(cleaned), &result); err != nil {
+		return nil, fmt.Errorf("parse tier1 response: %w (raw: %s)", err, truncate(raw, 200))
+	}
+
+	if !validChannels[result.Channel] {
+		return nil, fmt.Errorf("invalid channel: %q", result.Channel)
+	}
+
+	result.Platforms = filterPlatforms(result.Platforms)
+	return &result, nil
+}
+
+// ParseTier2Response parses an LLM response into a Tier2Result.
+func ParseTier2Response(raw string) (*Tier2Result, error) {
+	cleaned := repairJSON(raw)
+
+	var result Tier2Result
+	if err := json.Unmarshal([]byte(cleaned), &result); err != nil {
+		return nil, fmt.Errorf("parse tier2 response: %w (raw: %s)", err, truncate(raw, 200))
+	}
+
+	// Always validate channel — gating may override post=false to true later
+	if !validChannels[result.Channel] {
+		if result.Post {
+			return nil, fmt.Errorf("invalid channel: %q", result.Channel)
+		}
+		// Default to "politics" for discarded stories with garbage channels
+		result.Channel = "politics"
+	}
+
+	result.Platforms = filterPlatforms(result.Platforms)
+	return &result, nil
+}
+
+// repairJSON attempts to fix common LLM output issues.
+func repairJSON(raw string) string {
+	s := strings.TrimSpace(raw)
+
+	// Strip <think>...</think> blocks (some models include reasoning)
+	s = thinkRe.ReplaceAllString(s, "")
+
+	// Strip markdown code fences
+	if matches := fenceRe.FindStringSubmatch(s); len(matches) > 1 {
+		s = matches[1]
+	}
+
+	s = strings.TrimSpace(s)
+
+	// Remove trailing commas before closing braces/brackets (outside quoted strings)
+	s = trailingCommaRe.ReplaceAllString(s, "$1")
+
+	return s
+}
+
+func filterPlatforms(platforms []string) []string {
+	var valid []string
+	for _, p := range platforms {
+		p = strings.TrimSpace(strings.ToLower(p))
+		if validPlatforms[p] {
+			valid = append(valid, p)
+		}
+	}
+	return valid
+}
+
+func truncate(s string, n int) string {
+	runes := []rune(s)
+	if len(runes) <= n {
+		return s
+	}
+	return string(runes[:n]) + "..."
+}

internal/classifier/parse_test.go

@@ -0,0 +1,156 @@
+package classifier
+
+import (
+	"testing"
+)
+
+func TestParseTier1Response_Clean(t *testing.T) {
+	raw := `{"channel": "tech", "platforms": [], "duplicate_of": null, "rationale": "GPU launch story"}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "tech" {
+		t.Errorf("channel = %q, want tech", result.Channel)
+	}
+	if len(result.Platforms) != 0 {
+		t.Errorf("platforms = %v, want empty", result.Platforms)
+	}
+	if result.DuplicateOf != nil {
+		t.Errorf("duplicate_of = %v, want nil", result.DuplicateOf)
+	}
+}
+
+func TestParseTier1Response_WithPlatforms(t *testing.T) {
+	raw := `{"channel":"gaming","platforms":["nintendo-switch","pc"],"duplicate_of":null,"rationale":"Switch 2 launch"}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "gaming" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+	if len(result.Platforms) != 2 {
+		t.Fatalf("platforms = %v, want 2", result.Platforms)
+	}
+	if result.Platforms[0] != "nintendo-switch" || result.Platforms[1] != "pc" {
+		t.Errorf("platforms = %v", result.Platforms)
+	}
+}
+
+func TestParseTier1Response_WithDuplicate(t *testing.T) {
+	raw := `{"channel":"politics","platforms":[],"duplicate_of":"guid-abc-123","rationale":"same event"}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.DuplicateOf == nil || *result.DuplicateOf != "guid-abc-123" {
+		t.Errorf("duplicate_of = %v", result.DuplicateOf)
+	}
+}
+
+func TestParseTier1Response_InvalidChannel(t *testing.T) {
+	raw := `{"channel":"sports","platforms":[],"duplicate_of":null,"rationale":"test"}`
+	_, err := ParseTier1Response(raw)
+	if err == nil {
+		t.Error("expected error for invalid channel")
+	}
+}
+
+func TestParseTier1Response_FiltersInvalidPlatforms(t *testing.T) {
+	raw := `{"channel":"gaming","platforms":["nintendo-switch","fake-platform","pc","also-fake"],"duplicate_of":null,"rationale":"test"}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Platforms) != 2 {
+		t.Fatalf("platforms = %v, want [nintendo-switch pc]", result.Platforms)
+	}
+}
+
+func TestParseTier2Response_PostTrue(t *testing.T) {
+	raw := `{"post":true,"channel":"politics","platforms":[],"duplicate_of":null,"rationale":"major world event"}`
+	result, err := ParseTier2Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !result.Post {
+		t.Error("post = false, want true")
+	}
+	if result.Channel != "politics" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+}
+
+func TestParseTier2Response_PostFalse(t *testing.T) {
+	raw := `{"post":false,"platforms":[],"duplicate_of":null,"rationale":"local news"}`
+	result, err := ParseTier2Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Post {
+		t.Error("post = true, want false")
+	}
+}
+
+func TestRepairJSON_MarkdownFences(t *testing.T) {
+	raw := "```json\n{\"channel\": \"tech\", \"platforms\": [], \"duplicate_of\": null, \"rationale\": \"test\"}\n```"
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "tech" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+}
+
+func TestRepairJSON_ThinkBlocks(t *testing.T) {
+	raw := "<think>I need to classify this as tech because it's about GPUs</think>\n{\"channel\": \"tech\", \"platforms\": [], \"duplicate_of\": null, \"rationale\": \"GPU story\"}"
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "tech" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+}
+
+func TestRepairJSON_TrailingCommas(t *testing.T) {
+	raw := `{"channel": "politics", "platforms": [], "duplicate_of": null, "rationale": "test",}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "politics" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+}
+
+func TestRepairJSON_TrailingCommaInArray(t *testing.T) {
+	raw := `{"channel": "gaming", "platforms": ["pc", "xbox",], "duplicate_of": null, "rationale": "test"}`
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Platforms) != 2 {
+		t.Errorf("platforms = %v", result.Platforms)
+	}
+}
+
+func TestRepairJSON_CombinedIssues(t *testing.T) {
+	raw := "<think>reasoning here</think>\n```json\n{\"channel\": \"tech\", \"platforms\": [],\"duplicate_of\": null, \"rationale\": \"test\",}\n```"
+	result, err := ParseTier1Response(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Channel != "tech" {
+		t.Errorf("channel = %q", result.Channel)
+	}
+}
+
+func TestRepairJSON_GarbageInput(t *testing.T) {
+	_, err := ParseTier1Response("I think this should go to tech because reasons")
+	if err == nil {
+		t.Error("expected error for garbage input")
+	}
+}

internal/classifier/prompts.go

@@ -0,0 +1,110 @@
+package classifier
+
+import (
+	"fmt"
+	"strings"
+
+	"pete/internal/ingestion"
+	"pete/internal/storage"
+)
+
+// BuildTier1Prompt builds the routing-only prompt for Tier 1 stories.
+func BuildTier1Prompt(item *ingestion.FeedItem, recent []storage.RecentHeadline) string {
+	return fmt.Sprintf(`You are a news router for a community discussion platform with three channels: tech, politics, and gaming.
+
+This story comes from a curated Tier 1 source. It will be posted. Your only job is to decide which channel it belongs in.
+
+Choose exactly one channel:
+- tech: technology products, software, hardware, industry — where technology itself is the primary subject
+- gaming: games, gaming platforms, gaming industry, esports
+- politics: everything else — political, policy, current events, public interest, significant world events
+
+Politics is the default. When in doubt, choose politics.
+
+Routing guidance:
+- GPU launch, software release, hardware review → tech
+- Game release, platform announcement, studio news → gaming
+- Election, legislation, court ruling → politics
+- AI regulation, antitrust action, surveillance policy → politics (policy is the story, not the technology)
+- Death of a public figure → politics
+- Violent crime, assassination attempt, arrest of public figure → politics
+- Natural disaster, humanitarian crisis → politics
+- Celebrity gossip, lifestyle with no broader significance → politics (still gets posted, still needs a home)
+
+For gaming stories only: identify which platforms are relevant from this list:
+nintendo-switch, playstation, xbox, pc, retro, mobile, multi
+Return an empty array if no specific platform applies.
+
+Also check: does this story cover the same underlying event as any story in the recent window? Different outlets covering the same event counts as a duplicate. If yes, set duplicate_of to that story's GUID. If no match, set duplicate_of to null.
+
+Respond ONLY with valid JSON. No preamble, no explanation outside the JSON:
+{
+  "channel": "tech|politics|gaming",
+  "platforms": [string],
+  "duplicate_of": string | null,
+  "rationale": "one sentence max, for logging only"
+}
+
+INCOMING STORY:
+Source: %s
+Tier: %d
+Feed hint: %s
+Headline: %s
+Lede: %s
+
+RECENT STORIES (last 24hr):
+%s`, item.Source, item.Tier, item.FeedHint, item.Headline, item.Lede, formatRecentStories(recent))
+}
+
+// BuildTier2Prompt builds the gating + routing prompt for Tier 2 stories.
+func BuildTier2Prompt(item *ingestion.FeedItem, recent []storage.RecentHeadline) string {
+	return fmt.Sprintf(`You are a news classifier for a community discussion platform with three channels: tech, politics, and gaming.
+
+This story comes from a wire service. Unlike curated sources, wire feeds publish everything — not all stories warrant posting. First decide if this story is worth posting, then decide where.
+
+Step 1 — Should this be posted?
+Post if the story has clear public significance: major world events, policy decisions, significant deaths, natural disasters, major technology developments. Do not post: local news with no broader relevance, routine financial filings, minor sports results, entertainment fluff.
+
+Step 2 — If posting, choose exactly one channel:
+- tech: technology products, software, hardware, industry
+- gaming: games, gaming platforms, gaming industry
+- politics: everything else — the default
+
+Politics is the default when in doubt.
+
+For gaming stories only: identify relevant platforms from: nintendo-switch, playstation, xbox, pc, retro, mobile, multi
+
+Also check for duplicates against the recent window. If this story covers the same underlying event as a recent story, set duplicate_of to that story's GUID.
+
+Respond ONLY with valid JSON:
+{
+  "post": true|false,
+  "channel": "tech|politics|gaming",
+  "platforms": [string],
+  "duplicate_of": string | null,
+  "rationale": "one sentence max, for logging only"
+}
+
+If post is false, channel may be omitted.
+
+INCOMING STORY:
+Source: %s
+Tier: %d
+Feed hint: %s
+Headline: %s
+Lede: %s
+
+RECENT STORIES (last 24hr):
+%s`, item.Source, item.Tier, item.FeedHint, item.Headline, item.Lede, formatRecentStories(recent))
+}
+
+func formatRecentStories(recent []storage.RecentHeadline) string {
+	if len(recent) == 0 {
+		return "(none)"
+	}
+	var sb strings.Builder
+	for _, h := range recent {
+		fmt.Fprintf(&sb, "[%s] %s — %s\n", h.GUID, h.Headline, h.Source)
+	}
+	return sb.String()
+}

internal/config/config.go

@@ -0,0 +1,171 @@
+package config
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"regexp"
+
+	"gopkg.in/yaml.v3"
+)
+
+// envBracketRe matches only ${VAR} style env references, not bare $VAR.
+var envBracketRe = regexp.MustCompile(`\$\{([^}]+)\}`)
+
+type Config struct {
+	Matrix  MatrixConfig   `yaml:"matrix"`
+	Ollama  OllamaConfig   `yaml:"ollama"`
+	Posting PostingConfig  `yaml:"posting"`
+	Storage StorageConfig  `yaml:"storage"`
+	Sources []SourceConfig `yaml:"sources"`
+}
+
+type MatrixConfig struct {
+	Homeserver  string            `yaml:"homeserver"`
+	UserID      string            `yaml:"user_id"`
+	Password    string            `yaml:"password"`
+	PickleKey   string            `yaml:"pickle_key"`
+	DisplayName string            `yaml:"display_name"`
+	DataDir     string            `yaml:"data_dir"`
+	AdminRoom   string            `yaml:"admin_room"`
+	Channels    map[string]string `yaml:"channels"`
+}
+
+type OllamaConfig struct {
+	BaseURL        string `yaml:"base_url"`
+	Model          string `yaml:"model"`
+	TimeoutSeconds int    `yaml:"timeout_seconds"`
+}
+
+type PostingConfig struct {
+	MinIntervalSeconds    int `yaml:"min_interval_seconds"`
+	BurstCapCount         int `yaml:"burst_cap_count"`
+	BurstCapWindowSeconds int `yaml:"burst_cap_window_seconds"`
+	DedupCooldownHours    int `yaml:"dedup_cooldown_hours"`
+}
+
+type StorageConfig struct {
+	DBPath                string `yaml:"db_path"`
+	RecentWindowHours     int    `yaml:"recent_window_hours"`
+	ClassificationLogDays int    `yaml:"classification_log_days"`
+}
+
+type SourceConfig struct {
+	Name                string  `yaml:"name"`
+	FeedURL             string  `yaml:"feed_url"`
+	Tier                int     `yaml:"tier"`
+	PollIntervalMinutes int     `yaml:"poll_interval_minutes"`
+	FeedHint            string  `yaml:"feed_hint"`
+	DirectRoute         *string `yaml:"direct_route"`
+	Enabled             bool    `yaml:"enabled"`
+}
+
+func Load(path string) (*Config, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read config: %w", err)
+	}
+
+	// Expand only ${VAR} style env references, not bare $VAR
+	// (bare $ in passwords like "pa$$word" must not be mangled)
+	expanded := envBracketRe.ReplaceAllStringFunc(string(data), func(match string) string {
+		varName := match[2 : len(match)-1] // strip ${ and }
+		val := os.Getenv(varName)
+		if val == "" {
+			slog.Warn("config: env var referenced but not set", "var", varName)
+		}
+		return val
+	})
+
+	var cfg Config
+	if err := yaml.Unmarshal([]byte(expanded), &cfg); err != nil {
+		return nil, fmt.Errorf("parse config: %w", err)
+	}
+
+	cfg.applyDefaults()
+
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("validate config: %w", err)
+	}
+
+	return &cfg, nil
+}
+
+func (c *Config) validate() error {
+	if c.Matrix.Homeserver == "" {
+		return fmt.Errorf("matrix.homeserver is required")
+	}
+	if c.Matrix.UserID == "" {
+		return fmt.Errorf("matrix.user_id is required")
+	}
+	if c.Matrix.Password == "" {
+		return fmt.Errorf("matrix.password is required")
+	}
+	if len(c.Matrix.Channels) == 0 {
+		return fmt.Errorf("matrix.channels must have at least one entry")
+	}
+	if c.Ollama.BaseURL == "" {
+		return fmt.Errorf("ollama.base_url is required")
+	}
+	if c.Ollama.Model == "" {
+		return fmt.Errorf("ollama.model is required")
+	}
+	if c.Storage.DBPath == "" {
+		return fmt.Errorf("storage.db_path is required")
+	}
+
+	for i, s := range c.Sources {
+		if s.Name == "" {
+			return fmt.Errorf("sources[%d].name is required", i)
+		}
+		if s.FeedURL == "" {
+			return fmt.Errorf("sources[%d].feed_url is required", i)
+		}
+		if s.Tier < 1 || s.Tier > 3 {
+			return fmt.Errorf("sources[%d].tier must be 1-3", i)
+		}
+		if s.PollIntervalMinutes <= 0 {
+			return fmt.Errorf("sources[%d].poll_interval_minutes must be > 0", i)
+		}
+	}
+
+	return nil
+}
+
+func (c *Config) applyDefaults() {
+	if c.Matrix.DataDir == "" {
+		c.Matrix.DataDir = "./data"
+	}
+	if c.Matrix.DisplayName == "" {
+		c.Matrix.DisplayName = "Pete"
+	}
+	if c.Matrix.PickleKey == "" {
+		c.Matrix.PickleKey = "pete_pickle_key"
+	}
+	if c.Ollama.TimeoutSeconds == 0 {
+		c.Ollama.TimeoutSeconds = 30
+	}
+	if c.Posting.MinIntervalSeconds == 0 {
+		c.Posting.MinIntervalSeconds = 300
+	}
+	if c.Posting.BurstCapCount == 0 {
+		c.Posting.BurstCapCount = 3
+	}
+	if c.Posting.BurstCapWindowSeconds == 0 {
+		c.Posting.BurstCapWindowSeconds = 1800
+	}
+	if c.Posting.DedupCooldownHours == 0 {
+		c.Posting.DedupCooldownHours = 48
+	}
+	if c.Storage.RecentWindowHours == 0 {
+		c.Storage.RecentWindowHours = 24
+	}
+	if c.Storage.ClassificationLogDays == 0 {
+		c.Storage.ClassificationLogDays = 7
+	}
+	for i := range c.Sources {
+		if c.Sources[i].PollIntervalMinutes == 0 {
+			c.Sources[i].PollIntervalMinutes = 20
+		}
+	}
+}

internal/config/config_test.go

@@ -0,0 +1,286 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// validMatrix is a reusable matrix config block for tests.
+const validMatrix = `
+matrix:
+  homeserver: "https://matrix.example.org"
+  user_id: "@pete:example.org"
+  password: "testpass"
+  channels:
+    tech: "!tech:example.org"
+    politics: "!politics:example.org"
+`
+
+func TestLoadValid(t *testing.T) {
+	yaml := validMatrix + `
+ollama:
+  base_url: "http://localhost:11434"
+  model: "gemma4:27b"
+  timeout_seconds: 60
+storage:
+  db_path: "/tmp/test.db"
+sources:
+  - name: "Test Source"
+    feed_url: "https://example.com/rss"
+    tier: 1
+    enabled: true
+`
+	cfg := loadFromString(t, yaml)
+
+	if cfg.Matrix.Homeserver != "https://matrix.example.org" {
+		t.Errorf("homeserver = %q", cfg.Matrix.Homeserver)
+	}
+	if cfg.Matrix.UserID != "@pete:example.org" {
+		t.Errorf("user_id = %q", cfg.Matrix.UserID)
+	}
+	if cfg.Ollama.TimeoutSeconds != 60 {
+		t.Errorf("timeout = %d, want 60", cfg.Ollama.TimeoutSeconds)
+	}
+	if len(cfg.Sources) != 1 {
+		t.Fatalf("sources = %d, want 1", len(cfg.Sources))
+	}
+	if cfg.Sources[0].Name != "Test Source" {
+		t.Errorf("source name = %q", cfg.Sources[0].Name)
+	}
+}
+
+func TestEnvVarExpansion(t *testing.T) {
+	t.Setenv("TEST_PETE_PASS", "secret_pass_123")
+
+	yaml := `
+matrix:
+  homeserver: "https://matrix.example.org"
+  user_id: "@pete:example.org"
+  password: "${TEST_PETE_PASS}"
+  channels:
+    tech: "!tech:example.org"
+ollama:
+  base_url: "http://localhost:11434"
+  model: "gemma4:27b"
+storage:
+  db_path: "/tmp/test.db"
+sources: []
+`
+	cfg := loadFromString(t, yaml)
+
+	if cfg.Matrix.Password != "secret_pass_123" {
+		t.Errorf("password = %q, want secret_pass_123", cfg.Matrix.Password)
+	}
+}
+
+func TestDefaults(t *testing.T) {
+	yaml := validMatrix + `
+ollama:
+  base_url: "http://localhost:11434"
+  model: "gemma4:27b"
+storage:
+  db_path: "/tmp/test.db"
+sources:
+  - name: "S"
+    feed_url: "https://example.com/rss"
+    tier: 1
+    enabled: true
+`
+	cfg := loadFromString(t, yaml)
+
+	if cfg.Matrix.DataDir != "./data" {
+		t.Errorf("data_dir default = %q, want ./data", cfg.Matrix.DataDir)
+	}
+	if cfg.Matrix.DisplayName != "Pete" {
+		t.Errorf("display_name default = %q, want Pete", cfg.Matrix.DisplayName)
+	}
+	if cfg.Ollama.TimeoutSeconds != 30 {
+		t.Errorf("timeout default = %d, want 30", cfg.Ollama.TimeoutSeconds)
+	}
+	if cfg.Posting.MinIntervalSeconds != 300 {
+		t.Errorf("min_interval default = %d, want 300", cfg.Posting.MinIntervalSeconds)
+	}
+	if cfg.Posting.BurstCapCount != 3 {
+		t.Errorf("burst_cap default = %d, want 3", cfg.Posting.BurstCapCount)
+	}
+	if cfg.Posting.BurstCapWindowSeconds != 1800 {
+		t.Errorf("burst_window default = %d, want 1800", cfg.Posting.BurstCapWindowSeconds)
+	}
+	if cfg.Storage.RecentWindowHours != 24 {
+		t.Errorf("recent_window default = %d, want 24", cfg.Storage.RecentWindowHours)
+	}
+	if cfg.Storage.ClassificationLogDays != 7 {
+		t.Errorf("classification_log default = %d, want 7", cfg.Storage.ClassificationLogDays)
+	}
+	if cfg.Sources[0].PollIntervalMinutes != 20 {
+		t.Errorf("poll_interval default = %d, want 20", cfg.Sources[0].PollIntervalMinutes)
+	}
+}
+
+// shortMatrix returns a minimal valid matrix block for validation error tests.
+func shortMatrix(overrides string) string {
+	base := `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+`
+	if overrides != "" {
+		return overrides
+	}
+	return base
+}
+
+func TestValidationErrors(t *testing.T) {
+	tests := []struct {
+		name string
+		yaml string
+	}{
+		{"missing homeserver", `
+matrix:
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"missing user_id", `
+matrix:
+  homeserver: "https://h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"missing password", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"no channels", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"missing ollama base_url", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {model: "m"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"missing ollama model", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l"}
+storage: {db_path: "/tmp/t.db"}
+`},
+		{"missing db_path", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {}
+`},
+		{"invalid tier", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+sources:
+  - name: "S"
+    feed_url: "https://e.com/rss"
+    tier: 5
+`},
+		{"missing source name", `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+sources:
+  - feed_url: "https://e.com/rss"
+    tier: 1
+`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "config.yaml")
+			os.WriteFile(path, []byte(tt.yaml), 0o644)
+			_, err := Load(path)
+			if err == nil {
+				t.Error("expected validation error, got nil")
+			}
+		})
+	}
+}
+
+func TestDirectRouteNullable(t *testing.T) {
+	yaml := `
+matrix:
+  homeserver: "https://h"
+  user_id: "@p:h"
+  password: "pw"
+  channels: {tech: "!t:e"}
+ollama: {base_url: "http://l", model: "m"}
+storage: {db_path: "/tmp/t.db"}
+sources:
+  - name: "Direct"
+    feed_url: "https://e.com/rss"
+    tier: 1
+    direct_route: "politics"
+    enabled: true
+  - name: "LLM"
+    feed_url: "https://e.com/rss2"
+    tier: 1
+    direct_route: null
+    enabled: true
+`
+	cfg := loadFromString(t, yaml)
+
+	if cfg.Sources[0].DirectRoute == nil {
+		t.Fatal("direct source should have non-nil DirectRoute")
+	}
+	if *cfg.Sources[0].DirectRoute != "politics" {
+		t.Errorf("direct_route = %q, want politics", *cfg.Sources[0].DirectRoute)
+	}
+	if cfg.Sources[1].DirectRoute != nil {
+		t.Errorf("null source should have nil DirectRoute, got %v", cfg.Sources[1].DirectRoute)
+	}
+}
+
+func loadFromString(t *testing.T, yamlContent string) *Config {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(yamlContent), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg
+}

internal/dedup/dedup.go

@@ -0,0 +1,101 @@
+// Package dedup provides deterministic normalization helpers for detecting
+// duplicate stories across feeds.
+package dedup
+
+import (
+	"net/url"
+	"strings"
+	"unicode"
+)
+
+// trackingParams are query-string keys stripped during URL canonicalization.
+// Anything starting with "utm_" is also stripped.
+var trackingParams = map[string]struct{}{
+	"fbclid":      {},
+	"gclid":       {},
+	"mc_cid":      {},
+	"mc_eid":      {},
+	"ref":         {},
+	"ref_src":     {},
+	"ref_url":     {},
+	"share":       {},
+	"shared":      {},
+	"cmpid":       {},
+	"CMP":         {},
+	"icid":        {},
+	"ito":         {},
+	"yclid":       {},
+	"_ga":         {},
+	"igshid":      {},
+	"feature":     {},
+	"si":          {},
+	"smid":        {},
+	"twclid":      {},
+	"mibextid":    {},
+	"src":         {},
+}
+
+// CanonicalURL returns a stable form of a story URL suitable for dedup.
+// It lowercases the scheme/host, drops fragments and common tracking params,
+// and trims trailing slashes. If parsing fails, the input is returned unchanged.
+func CanonicalURL(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	u, err := url.Parse(raw)
+	if err != nil || u.Host == "" {
+		return raw
+	}
+
+	u.Scheme = strings.ToLower(u.Scheme)
+	u.Host = strings.ToLower(u.Host)
+	u.Host = strings.TrimPrefix(u.Host, "www.")
+	u.Host = strings.TrimPrefix(u.Host, "m.")
+	u.Host = strings.TrimPrefix(u.Host, "amp.")
+	u.Fragment = ""
+
+	if q := u.Query(); len(q) > 0 {
+		for k := range q {
+			lk := strings.ToLower(k)
+			if strings.HasPrefix(lk, "utm_") {
+				q.Del(k)
+				continue
+			}
+			if _, drop := trackingParams[lk]; drop {
+				q.Del(k)
+			}
+		}
+		u.RawQuery = q.Encode()
+	}
+
+	u.Path = strings.TrimRight(u.Path, "/")
+	if u.Path == "" {
+		u.Path = "/"
+	}
+	return u.String()
+}
+
+// NormalizeHeadline returns a comparable form of a headline: lowercase,
+// punctuation stripped, whitespace collapsed. Empty input returns "".
+func NormalizeHeadline(h string) string {
+	if h == "" {
+		return ""
+	}
+	var b strings.Builder
+	b.Grow(len(h))
+	prevSpace := true
+	for _, r := range h {
+		switch {
+		case unicode.IsLetter(r) || unicode.IsDigit(r):
+			b.WriteRune(unicode.ToLower(r))
+			prevSpace = false
+		case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
+			if !prevSpace {
+				b.WriteByte(' ')
+				prevSpace = true
+			}
+		}
+	}
+	return strings.TrimSpace(b.String())
+}

internal/dedup/dedup_test.go

@@ -0,0 +1,43 @@
+package dedup
+
+import "testing"
+
+func TestCanonicalURL(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"https://example.com/foo", "https://example.com/foo"},
+		{"https://www.example.com/foo/", "https://example.com/foo"},
+		{"HTTPS://Example.com/Foo?utm_source=x&utm_medium=y", "https://example.com/Foo"},
+		{"https://example.com/foo?id=1&utm_campaign=z&fbclid=abc", "https://example.com/foo?id=1"},
+		{"https://example.com/foo#section", "https://example.com/foo"},
+		{"https://m.bbc.com/news/123/", "https://bbc.com/news/123"},
+		{"https://amp.cnn.com/article", "https://cnn.com/article"},
+		{"not a url", "not a url"},
+	}
+	for _, c := range cases {
+		got := CanonicalURL(c.in)
+		if got != c.want {
+			t.Errorf("CanonicalURL(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestNormalizeHeadline(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"Hello, World!", "hello world"},
+		{"  Multiple   Spaces  ", "multiple spaces"},
+		{"Nintendo's Switch 2 — Day One!", "nintendo s switch 2 day one"},
+		{"BREAKING: Foo bar", "breaking foo bar"},
+	}
+	for _, c := range cases {
+		got := NormalizeHeadline(c.in)
+		if got != c.want {
+			t.Errorf("NormalizeHeadline(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}

internal/ingestion/image.go

@@ -0,0 +1,63 @@
+package ingestion
+
+import (
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var imageClient = &http.Client{
+	Timeout: 10 * time.Second,
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 3 {
+			return http.ErrUseLastResponse
+		}
+		return nil
+	},
+}
+
+// ValidateImageURL checks that an image URL returns a valid image response.
+// Returns false (with no error) on any failure — story posts without image.
+func ValidateImageURL(url string) bool {
+	if url == "" {
+		return false
+	}
+
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		slog.Debug("image validation: bad url", "url", url, "err", err)
+		return false
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := imageClient.Do(req)
+	if err != nil {
+		slog.Warn("image validation: request failed", "url", url, "err", err)
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode)
+		return false
+	}
+
+	contentType := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(contentType, "image/") {
+		slog.Warn("image validation: not an image", "url", url, "content_type", contentType)
+		return false
+	}
+
+	// Filter tracking pixels: Content-Length must be > 5KB if present
+	if cl := resp.Header.Get("Content-Length"); cl != "" {
+		size, err := strconv.ParseInt(cl, 10, 64)
+		if err == nil && size <= 5120 {
+			slog.Warn("image validation: too small (likely tracking pixel)", "url", url, "size", size)
+			return false
+		}
+	}
+
+	return true
+}

internal/ingestion/og.go

@@ -0,0 +1,98 @@
+package ingestion
+
+import (
+	"context"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Match <meta property="og:image" content="..."> in either attribute order,
+// and the twitter:image variant.
+var metaImageRe = regexp.MustCompile(
+	`(?is)<meta\s+[^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
+		`|<meta\s+[^>]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
+
+var ogClient = &http.Client{Timeout: 8 * time.Second}
+
+// FetchOGImage fetches the article URL and returns the first og:image or
+// twitter:image found in the <head>. Returns "" on any failure — callers
+// should treat this as best-effort.
+func FetchOGImage(articleURL string) string {
+	if articleURL == "" {
+		return ""
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
+	if err != nil {
+		return ""
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := ogClient.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ""
+	}
+
+	// og:image lives in <head>; cap read at 512KB to avoid pulling whole articles.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
+	if err != nil {
+		return ""
+	}
+
+	m := metaImageRe.FindSubmatch(body)
+	if len(m) == 0 {
+		return ""
+	}
+	// One of the two capture groups will be non-empty depending on attr order.
+	for i := 1; i < len(m); i++ {
+		if len(m[i]) > 0 {
+			return resolveURL(articleURL, html.UnescapeString(string(m[i])))
+		}
+	}
+	return ""
+}
+
+// resolveURL turns a possibly-relative image URL into an absolute one using
+// the article URL as the base. Returns the raw input on parse failure.
+func resolveURL(base, ref string) string {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return ""
+	}
+	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		return ref
+	}
+	if strings.HasPrefix(ref, "//") {
+		if i := strings.Index(base, "://"); i > 0 {
+			return base[:i+1] + ref
+		}
+		return "https:" + ref
+	}
+	// relative path — naive join: scheme://host + ref
+	i := strings.Index(base, "://")
+	if i < 0 {
+		return ref
+	}
+	rest := base[i+3:]
+	slash := strings.Index(rest, "/")
+	if slash < 0 {
+		return fmt.Sprintf("%s%s", base, ref)
+	}
+	host := base[:i+3+slash]
+	if strings.HasPrefix(ref, "/") {
+		return host + ref
+	}
+	return host + "/" + ref
+}

internal/ingestion/og_test.go

@@ -0,0 +1,72 @@
+package ingestion
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestFetchOGImage(t *testing.T) {
+	cases := []struct {
+		name string
+		body string
+		want string // suffix match
+	}{
+		{
+			name: "og:image property",
+			body: `<html><head><meta property="og:image" content="https://example.com/lead.jpg"></head></html>`,
+			want: "https://example.com/lead.jpg",
+		},
+		{
+			name: "content before property",
+			body: `<html><head><meta content="https://example.com/swap.jpg" property="og:image"/></head></html>`,
+			want: "https://example.com/swap.jpg",
+		},
+		{
+			name: "twitter:image fallback",
+			body: `<html><head><meta name="twitter:image" content="https://example.com/twit.jpg"></head></html>`,
+			want: "https://example.com/twit.jpg",
+		},
+		{
+			name: "relative URL resolved",
+			body: `<html><head><meta property="og:image" content="/img/lead.jpg"></head></html>`,
+			want: "/img/lead.jpg",
+		},
+		{
+			name: "no og tags",
+			body: `<html><head><title>nothing</title></head></html>`,
+			want: "",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "text/html")
+				w.Write([]byte(c.body))
+			}))
+			defer srv.Close()
+
+			got := FetchOGImage(srv.URL + "/article")
+			if c.want == "" {
+				if got != "" {
+					t.Errorf("expected empty, got %q", got)
+				}
+				return
+			}
+			if !strings.HasSuffix(got, c.want) {
+				t.Errorf("got %q, want suffix %q", got, c.want)
+			}
+		})
+	}
+}
+
+func TestFirstImgSrc(t *testing.T) {
+	in := `<p>Body <img alt="x" src="https://cdn.example.com/lead.png" width="600"/> more</p>`
+	if got := firstImgSrc(in); got != "https://cdn.example.com/lead.png" {
+		t.Errorf("got %q", got)
+	}
+	if got := firstImgSrc(""); got != "" {
+		t.Errorf("empty input should yield empty, got %q", got)
+	}
+}

internal/ingestion/parser.go

@@ -0,0 +1,157 @@
+package ingestion
+
+import (
+	"context"
+	"html"
+	"log/slog"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+)
+
+const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
+
+var feedClient = &http.Client{Timeout: 30 * time.Second}
+
+// FeedItem represents a parsed RSS item ready for classification.
+type FeedItem struct {
+	GUID        string
+	Headline    string
+	Lede        string
+	ImageURL    string
+	ArticleURL  string
+	Source      string
+	FeedHint    string
+	Tier        int
+	DirectRoute *string
+}
+
+var (
+	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
+	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
+)
+
+
+// FetchFeed fetches and parses an RSS/Atom feed, returning items.
+func FetchFeed(feedURL string) ([]FeedItem, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fp := gofeed.NewParser()
+	fp.Client = feedClient
+	fp.UserAgent = userAgent
+
+	feed, err := fp.ParseURLWithContext(feedURL, ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	items := make([]FeedItem, 0, len(feed.Items))
+	for _, item := range feed.Items {
+		fi := FeedItem{
+			GUID:       itemGUID(item),
+			Headline:   strings.TrimSpace(item.Title),
+			Lede:       extractLede(item.Description),
+			ImageURL:   extractImageURL(item),
+			ArticleURL: strings.TrimSpace(item.Link),
+		}
+		if fi.GUID == "" || fi.ArticleURL == "" {
+			continue
+		}
+		items = append(items, fi)
+	}
+
+	slog.Debug("fetched feed", "url", feedURL, "items", len(items))
+	return items, nil
+}
+
+func itemGUID(item *gofeed.Item) string {
+	if item.GUID != "" {
+		return item.GUID
+	}
+	return item.Link
+}
+
+// extractLede returns the feed description verbatim, with HTML stripped.
+func extractLede(desc string) string {
+	if desc == "" {
+		return ""
+	}
+	text := htmlTagRe.ReplaceAllString(desc, "")
+	text = html.UnescapeString(text)
+	return strings.TrimSpace(text)
+}
+
+// extractImageURL tries to find an image URL from feed item metadata.
+// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
+// scraped from content:encoded / description (catches feeds like Ars that
+// embed the lead image in the article body but not as a media:* element).
+func extractImageURL(item *gofeed.Item) string {
+	// Check media content (common in RSS 2.0 with media namespace)
+	if item.Extensions != nil {
+		if media, ok := item.Extensions["media"]; ok {
+			if contents, ok := media["content"]; ok {
+				for _, c := range contents {
+					if url := c.Attrs["url"]; url != "" {
+						return url
+					}
+				}
+			}
+			if thumbs, ok := media["thumbnail"]; ok {
+				for _, t := range thumbs {
+					if url := t.Attrs["url"]; url != "" {
+						return url
+					}
+				}
+			}
+			// media:group wraps nested media:content / media:thumbnail in some feeds
+			if groups, ok := media["group"]; ok {
+				for _, g := range groups {
+					for _, child := range g.Children {
+						for _, n := range child {
+							if url := n.Attrs["url"]; url != "" {
+								return url
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Check enclosures
+	for _, enc := range item.Enclosures {
+		if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
+			return enc.URL
+		}
+	}
+
+	// Check item image
+	if item.Image != nil && item.Image.URL != "" {
+		return item.Image.URL
+	}
+
+	// Scrape first <img src> from content:encoded or description
+	if url := firstImgSrc(item.Content); url != "" {
+		return url
+	}
+	if url := firstImgSrc(item.Description); url != "" {
+		return url
+	}
+
+	return ""
+}
+
+func firstImgSrc(htmlBody string) string {
+	if htmlBody == "" {
+		return ""
+	}
+	m := imgSrcRe.FindStringSubmatch(htmlBody)
+	if len(m) < 2 {
+		return ""
+	}
+	return html.UnescapeString(m[1])
+}

internal/ingestion/parser_test.go

@@ -0,0 +1,66 @@
+package ingestion
+
+import "testing"
+
+func TestExtractLede(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			"plain text verbatim",
+			"The president signed a new bill. More details followed in the afternoon session.",
+			"The president signed a new bill. More details followed in the afternoon session.",
+		},
+		{
+			"single sentence",
+			"Breaking news from the capital.",
+			"Breaking news from the capital.",
+		},
+		{
+			"empty",
+			"",
+			"",
+		},
+		{
+			"html stripped",
+			"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
+			"The company announced a major acquisition. Shares rose 5%.",
+		},
+		{
+			"nested html",
+			"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
+			"Apple revealed the iPhone 16. It features a new chip.",
+		},
+		{
+			"whitespace trimmed",
+			"  \n  Some news happened today. More to come.  \n  ",
+			"Some news happened today. More to come.",
+		},
+		{
+			"html entities decoded",
+			"The U.S. &amp; EU agreed on a &quot;landmark&quot; deal worth &gt;$1bn.",
+			`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
+		},
+		{
+			"numeric entities",
+			"It&#39;s a new era&#8212;one of change.",
+			"It's a new era\u2014one of change.",
+		},
+		{
+			"tags and entities combined",
+			"<p>Oil prices rose &amp; gas fell by &gt;5%.</p>",
+			"Oil prices rose & gas fell by >5%.",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := extractLede(tt.input)
+			if got != tt.want {
+				t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
+			}
+		})
+	}
+}

internal/ingestion/poller.go

@@ -0,0 +1,197 @@
+package ingestion
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"pete/internal/config"
+	"pete/internal/dedup"
+	"pete/internal/storage"
+)
+
+// ProcessFunc is called for each new feed item that needs classification.
+type ProcessFunc func(item *FeedItem)
+
+// Poller manages per-source RSS polling goroutines.
+type Poller struct {
+	sources []config.SourceConfig
+	process ProcessFunc
+	wg      sync.WaitGroup
+}
+
+// NewPoller creates a poller for the given sources.
+func NewPoller(sources []config.SourceConfig, process ProcessFunc) *Poller {
+	return &Poller{
+		sources: sources,
+		process: process,
+	}
+}
+
+// Start launches a goroutine per enabled source. Blocks until ctx is cancelled.
+func (p *Poller) Start(ctx context.Context) {
+	for _, src := range p.sources {
+		if !src.Enabled {
+			slog.Info("source disabled, skipping", "source", src.Name)
+			continue
+		}
+		p.wg.Add(1)
+		go p.pollSource(ctx, src)
+	}
+}
+
+// Wait blocks until all polling goroutines have exited.
+func (p *Poller) Wait() {
+	p.wg.Wait()
+}
+
+func (p *Poller) pollSource(ctx context.Context, src config.SourceConfig) {
+	defer p.wg.Done()
+
+	interval := time.Duration(src.PollIntervalMinutes) * time.Minute
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	slog.Info("starting poller", "source", src.Name, "interval", interval)
+
+	// Poll immediately on start, then on ticker
+	p.pollOnce(src)
+
+	consecutiveFailures := 0
+	const adminWarningThreshold = 5
+
+	for {
+		select {
+		case <-ctx.Done():
+			slog.Info("poller stopping", "source", src.Name)
+			return
+		case <-ticker.C:
+			if err := p.pollOnceWithErr(src); err != nil {
+				consecutiveFailures++
+				slog.Error("feed poll failed",
+					"source", src.Name,
+					"err", err,
+					"consecutive_failures", consecutiveFailures,
+				)
+				if consecutiveFailures == adminWarningThreshold {
+					slog.Warn("persistent feed failure",
+						"source", src.Name,
+						"failures", consecutiveFailures,
+					)
+					// TODO: send admin room warning via matrix client
+				}
+			} else {
+				consecutiveFailures = 0
+			}
+		}
+	}
+}
+
+func (p *Poller) pollOnce(src config.SourceConfig) {
+	if err := p.pollOnceWithErr(src); err != nil {
+		slog.Error("feed poll failed", "source", src.Name, "err", err)
+	}
+}
+
+func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
+	items, err := FetchFeed(src.FeedURL)
+	if err != nil {
+		return err
+	}
+
+	newCount := 0
+	for i := range items {
+		if storage.IsGUIDSeen(items[i].GUID) {
+			continue
+		}
+
+		// Last-resort image fallback: feed gave us nothing, scrape og:image
+		// from the article page. Only runs for genuinely new items.
+		if items[i].ImageURL == "" {
+			if og := FetchOGImage(items[i].ArticleURL); og != "" {
+				items[i].ImageURL = og
+				slog.Debug("og:image fallback used",
+					"guid", items[i].GUID, "url", items[i].ArticleURL, "image", og)
+			}
+		}
+
+		canonical := dedup.CanonicalURL(items[i].ArticleURL)
+		headlineNorm := dedup.NormalizeHeadline(items[i].Headline)
+
+		if storage.IsCanonicalSeen(canonical) {
+			slog.Info("dropping duplicate story (canonical URL match)",
+				"guid", items[i].GUID, "url_canonical", canonical, "source", src.Name)
+			continue
+		}
+		if storage.IsHeadlineSeen(src.Name, headlineNorm) {
+			slog.Info("dropping duplicate story (headline match)",
+				"guid", items[i].GUID, "headline_norm", headlineNorm, "source", src.Name)
+			continue
+		}
+
+		// Stamp source metadata onto the item
+		items[i].Source = src.Name
+		items[i].FeedHint = src.FeedHint
+		items[i].Tier = src.Tier
+		items[i].DirectRoute = src.DirectRoute
+
+		// Store immediately (before classification) to prevent re-ingestion
+		if err := storage.InsertStory(&storage.Story{
+			GUID:         items[i].GUID,
+			Headline:     items[i].Headline,
+			Lede:         items[i].Lede,
+			ImageURL:     items[i].ImageURL,
+			ArticleURL:   items[i].ArticleURL,
+			URLCanonical: canonical,
+			HeadlineNorm: headlineNorm,
+			Source:       items[i].Source,
+			FeedHint:     items[i].FeedHint,
+			SeenAt:       time.Now().Unix(),
+		}); err != nil {
+			slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
+			continue
+		}
+
+		p.process(&items[i])
+		newCount++
+	}
+
+	if newCount > 0 {
+		slog.Info("ingested new stories", "source", src.Name, "count", newCount)
+	}
+
+	// Retry unclassified stories from this source only (cap at 20 to avoid runaway retries)
+	unclassified, err := storage.GetUnclassifiedStories(src.Name)
+	if err != nil {
+		slog.Error("failed to get unclassified stories", "err", err)
+		return nil
+	}
+	for _, s := range unclassified {
+		// Skip stories we just ingested (they're already being processed above)
+		alreadyProcessed := false
+		for _, item := range items {
+			if item.GUID == s.GUID {
+				alreadyProcessed = true
+				break
+			}
+		}
+		if alreadyProcessed {
+			continue
+		}
+
+		p.process(&FeedItem{
+			GUID:        s.GUID,
+			Headline:    s.Headline,
+			Lede:        s.Lede,
+			ImageURL:    s.ImageURL,
+			ArticleURL:  s.ArticleURL,
+			Source:       s.Source,
+			FeedHint:    s.FeedHint,
+			Tier:        src.Tier,
+			DirectRoute: src.DirectRoute,
+		})
+	}
+
+	return nil
+}

internal/matrix/client.go

@@ -0,0 +1,468 @@
+package matrix
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"pete/internal/config"
+
+	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/crypto/cryptohelper"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+)
+
+// ReactionHandler is called when a reaction is received on a post.
+type ReactionHandler func(roomID id.RoomID, eventID id.EventID, targetEventID id.EventID, emoji string, userID id.UserID)
+
+// deviceInfo holds persisted device credentials.
+type deviceInfo struct {
+	AccessToken string `json:"access_token"`
+	DeviceID    string `json:"device_id"`
+	UserID      string `json:"user_id"`
+}
+
+// Client wraps a mautrix client for Pete's needs: posting stories and tracking reactions.
+type Client struct {
+	mx        *mautrix.Client
+	channels  map[string]id.RoomID
+	adminRoom *id.RoomID
+	userID    id.UserID
+	cfg       config.MatrixConfig
+	crypto    *cryptohelper.CryptoHelper
+
+	mu         sync.RWMutex
+	onReact    ReactionHandler
+	cancelSync context.CancelFunc
+}
+
+// New creates a Matrix client with password login, device persistence, and E2EE.
+func New(cfg config.MatrixConfig) (*Client, error) {
+	if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil {
+		return nil, fmt.Errorf("create data dir: %w", err)
+	}
+
+	devicePath := filepath.Join(cfg.DataDir, "device.json")
+
+	// Try to load existing device credentials
+	device, err := loadDevice(devicePath)
+	if err != nil {
+		slog.Info("no existing device found, will login fresh")
+	}
+
+	var mx *mautrix.Client
+
+	if device != nil {
+		valid := isTokenValid(cfg.Homeserver, device.AccessToken)
+		if valid {
+			slog.Info("existing device credentials valid", "device_id", device.DeviceID)
+			userID := id.UserID(device.UserID)
+			mx, err = mautrix.NewClient(cfg.Homeserver, userID, device.AccessToken)
+			if err != nil {
+				return nil, fmt.Errorf("create client with existing token: %w", err)
+			}
+			mx.DeviceID = id.DeviceID(device.DeviceID)
+		} else {
+			slog.Warn("existing device credentials invalid, logging in again")
+			device = nil
+		}
+	}
+
+	if device == nil {
+		mx, err = mautrix.NewClient(cfg.Homeserver, "", "")
+		if err != nil {
+			return nil, fmt.Errorf("create client: %w", err)
+		}
+
+		resp, err := mx.Login(context.Background(), &mautrix.ReqLogin{
+			Type: mautrix.AuthTypePassword,
+			Identifier: mautrix.UserIdentifier{
+				Type: mautrix.IdentifierTypeUser,
+				User: cfg.UserID,
+			},
+			Password:                 cfg.Password,
+			InitialDeviceDisplayName: cfg.DisplayName,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("login: %w", err)
+		}
+
+		mx.AccessToken = resp.AccessToken
+		mx.UserID = resp.UserID
+		mx.DeviceID = resp.DeviceID
+
+		device = &deviceInfo{
+			AccessToken: resp.AccessToken,
+			DeviceID:    string(resp.DeviceID),
+			UserID:      string(resp.UserID),
+		}
+		if err := saveDevice(devicePath, device); err != nil {
+			slog.Warn("failed to save device info", "err", err)
+		}
+
+		slog.Info("logged in successfully",
+			"user_id", resp.UserID,
+			"device_id", resp.DeviceID,
+		)
+	}
+
+	// Set up E2EE via cryptohelper — stores crypto state in its own SQLite DB.
+	// Persists device keys, olm/megolm sessions, cross-signing keys across restarts.
+	cryptoDBPath := filepath.Join(cfg.DataDir, "crypto.db")
+	ch, err := cryptohelper.NewCryptoHelper(mx, []byte(cfg.PickleKey), cryptoDBPath)
+	if err != nil {
+		return nil, fmt.Errorf("init crypto helper: %w", err)
+	}
+
+	// LoginAs enables the cryptohelper to re-login if the token expires
+	ch.LoginAs = &mautrix.ReqLogin{
+		Type: mautrix.AuthTypePassword,
+		Identifier: mautrix.UserIdentifier{
+			Type: mautrix.IdentifierTypeUser,
+			User: cfg.UserID,
+		},
+		Password:                 cfg.Password,
+		InitialDeviceDisplayName: cfg.DisplayName,
+	}
+
+	if err := ch.Init(context.Background()); err != nil {
+		return nil, fmt.Errorf("crypto helper init: %w", err)
+	}
+
+	mx.Crypto = ch
+
+	// Bootstrap cross-signing only if not already set up.
+	mach := ch.Machine()
+	existingKeys, err := mach.GetOwnCrossSigningPublicKeys(context.Background())
+	if err != nil {
+		slog.Warn("cross-signing: failed to fetch existing keys", "err", err)
+	}
+	if existingKeys == nil || existingKeys.MasterKey == "" {
+		_, _, err = mach.GenerateAndUploadCrossSigningKeys(context.Background(), func(ui *mautrix.RespUserInteractive) interface{} {
+			return map[string]interface{}{
+				"type": mautrix.AuthTypePassword,
+				"identifier": map[string]interface{}{
+					"type": mautrix.IdentifierTypeUser,
+					"user": cfg.UserID,
+				},
+				"password": cfg.Password,
+				"session":  ui.Session,
+			}
+		}, "")
+		if err != nil {
+			slog.Warn("cross-signing: key upload failed", "err", err)
+		} else {
+			slog.Info("cross-signing: keys uploaded")
+		}
+
+		if err := mach.SignOwnDevice(context.Background(), mach.OwnIdentity()); err != nil {
+			slog.Warn("cross-signing: sign own device failed", "err", err)
+		} else {
+			slog.Info("cross-signing: own device signed")
+		}
+
+		if err := mach.SignOwnMasterKey(context.Background()); err != nil {
+			slog.Warn("cross-signing: sign master key failed", "err", err)
+		} else {
+			slog.Info("cross-signing: master key signed")
+		}
+	} else {
+		slog.Info("cross-signing: already configured, skipping bootstrap")
+	}
+
+	slog.Info("E2EE initialized",
+		"device_id", mx.DeviceID,
+		"crypto_store", "sqlite-persistent",
+	)
+
+	channels := make(map[string]id.RoomID, len(cfg.Channels))
+	for name, roomID := range cfg.Channels {
+		channels[name] = id.RoomID(roomID)
+	}
+
+	c := &Client{
+		mx:       mx,
+		channels: channels,
+		userID:   mx.UserID,
+		cfg:      cfg,
+		crypto:   ch,
+	}
+
+	if cfg.AdminRoom != "" {
+		adminRoom := id.RoomID(cfg.AdminRoom)
+		c.adminRoom = &adminRoom
+	}
+
+	return c, nil
+}
+
+// SetReactionHandler registers a callback for reaction events.
+func (c *Client) SetReactionHandler(fn ReactionHandler) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.onReact = fn
+}
+
+// Start begins the Matrix sync loop in the background.
+func (c *Client) Start(ctx context.Context) {
+	syncer := c.mx.Syncer.(*mautrix.DefaultSyncer)
+
+	// Listen for reactions
+	syncer.OnEventType(event.EventReaction, func(ctx context.Context, evt *event.Event) {
+		if evt.Sender == c.userID {
+			return
+		}
+
+		content := evt.Content.AsReaction()
+		if content == nil || content.RelatesTo.EventID == "" {
+			return
+		}
+
+		c.mu.RLock()
+		handler := c.onReact
+		c.mu.RUnlock()
+
+		if handler != nil {
+			handler(
+				evt.RoomID,
+				evt.ID,
+				content.RelatesTo.EventID,
+				content.RelatesTo.Key,
+				evt.Sender,
+			)
+		}
+	})
+
+	// Auto-join on invite
+	syncer.OnEventType(event.StateMember, func(ctx context.Context, evt *event.Event) {
+		if evt.GetStateKey() == string(c.userID) && evt.Content.AsMember().Membership == event.MembershipInvite {
+			_, err := c.mx.JoinRoomByID(ctx, evt.RoomID)
+			if err != nil {
+				slog.Error("failed to auto-join room", "room", evt.RoomID, "err", err)
+			} else {
+				slog.Info("auto-joined room", "room", evt.RoomID)
+			}
+		}
+	})
+
+	syncCtx, cancel := context.WithCancel(ctx)
+	c.cancelSync = cancel
+
+	go func() {
+		for {
+			err := c.mx.SyncWithContext(syncCtx)
+			if syncCtx.Err() != nil {
+				return
+			}
+			if err != nil {
+				slog.Error("matrix sync stopped, restarting in 5s", "err", err)
+			} else {
+				slog.Warn("matrix sync returned without error, restarting in 5s")
+			}
+			select {
+			case <-time.After(5 * time.Second):
+			case <-syncCtx.Done():
+				return
+			}
+		}
+	}()
+}
+
+// Stop halts the sync loop and closes the crypto store.
+func (c *Client) Stop() {
+	if c.cancelSync != nil {
+		c.cancelSync()
+	}
+	if c.crypto != nil {
+		if err := c.crypto.Close(); err != nil {
+			slog.Error("failed to close crypto helper", "err", err)
+		}
+	}
+}
+
+// PostStory sends a story to a channel. Returns the text event ID for reaction tracking.
+// If the story has a validated image, it's uploaded and sent as a separate m.image event first.
+func (c *Client) PostStory(channel string, story *PostableStory) (id.EventID, error) {
+	roomID, ok := c.channels[channel]
+	if !ok {
+		return "", fmt.Errorf("unknown channel: %s", channel)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Send image first if present
+	if story.ImageURL != "" {
+		if err := c.sendImage(ctx, roomID, story.ImageURL); err != nil {
+			slog.Warn("failed to send image, posting text only", "url", story.ImageURL, "err", err)
+		}
+	}
+
+	// Send text message
+	plain, html := formatPost(story)
+	content := &event.MessageEventContent{
+		MsgType:       event.MsgText,
+		Body:          plain,
+		Format:        event.FormatHTML,
+		FormattedBody: html,
+	}
+
+	resp, err := c.mx.SendMessageEvent(ctx, roomID, event.EventMessage, content)
+	if err != nil {
+		return "", fmt.Errorf("send message: %w", err)
+	}
+
+	return resp.EventID, nil
+}
+
+// sendImage downloads an image, uploads it to Matrix, and sends it as m.image.
+func (c *Client) sendImage(ctx context.Context, roomID id.RoomID, imageURL string) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", imageURL, nil)
+	if err != nil {
+		return fmt.Errorf("create image request: %w", err)
+	}
+	httpClient := &http.Client{Timeout: 15 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("download image: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("image download status %d", resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
+	if err != nil {
+		return fmt.Errorf("read image: %w", err)
+	}
+
+	contentType := resp.Header.Get("Content-Type")
+	if contentType == "" {
+		contentType = "image/jpeg"
+	}
+
+	// Reject non-image content types
+	if !strings.HasPrefix(contentType, "image/") {
+		return fmt.Errorf("not an image: %s", contentType)
+	}
+
+	// Decode image dimensions so clients render it inline instead of as a download
+	var width, height int
+	if img, _, decErr := image.DecodeConfig(bytes.NewReader(data)); decErr == nil {
+		width = img.Width
+		height = img.Height
+	}
+
+	uploadResp, err := c.mx.UploadBytes(ctx, data, contentType)
+	if err != nil {
+		return fmt.Errorf("upload image: %w", err)
+	}
+
+	// Derive a filename with proper extension — some clients use Body as filename
+	// and won't render inline without a recognized image extension
+	ext := ".jpg"
+	switch contentType {
+	case "image/png":
+		ext = ".png"
+	case "image/gif":
+		ext = ".gif"
+	case "image/webp":
+		ext = ".webp"
+	}
+
+	imgContent := &event.MessageEventContent{
+		MsgType:  event.MsgImage,
+		Body:     "image" + ext,
+		FileName: "image" + ext,
+		URL:      uploadResp.ContentURI.CUString(),
+		Info: &event.FileInfo{
+			MimeType: contentType,
+			Size:     len(data),
+			Width:    width,
+			Height:   height,
+		},
+	}
+
+	_, err = c.mx.SendMessageEvent(ctx, roomID, event.EventMessage, imgContent)
+	return err
+}
+
+// SendAdminWarning sends a warning message to the admin room, if configured.
+func (c *Client) SendAdminWarning(msg string) {
+	if c.adminRoom == nil {
+		slog.Warn("admin warning (no admin room configured)", "msg", msg)
+		return
+	}
+
+	content := &event.MessageEventContent{
+		MsgType: event.MsgText,
+		Body:    msg,
+	}
+
+	if _, err := c.mx.SendMessageEvent(context.Background(), *c.adminRoom, event.EventMessage, content); err != nil {
+		slog.Error("failed to send admin warning", "err", err, "msg", msg)
+	}
+}
+
+// ChannelRoomID returns the room ID for a named channel.
+// Returns false if the channel is not configured or has an empty room ID.
+func (c *Client) ChannelRoomID(channel string) (id.RoomID, bool) {
+	roomID, ok := c.channels[channel]
+	if !ok || roomID == "" {
+		return "", false
+	}
+	return roomID, true
+}
+
+func loadDevice(path string) (*deviceInfo, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var info deviceInfo
+	if err := json.Unmarshal(data, &info); err != nil {
+		return nil, err
+	}
+	return &info, nil
+}
+
+func saveDevice(path string, info *deviceInfo) error {
+	data, err := json.MarshalIndent(info, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, data, 0o600)
+}
+
+func isTokenValid(homeserver, accessToken string) bool {
+	url := homeserver + "/_matrix/client/v3/account/whoami"
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return false
+	}
+	req.Header.Set("Authorization", "Bearer "+accessToken)
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode == http.StatusOK
+}

internal/matrix/post.go

@@ -0,0 +1,62 @@
+package matrix
+
+import (
+	"fmt"
+	"html"
+	"strings"
+)
+
+// PostableStory contains the data needed to format and send a story.
+type PostableStory struct {
+	ImageURL   string
+	Headline   string
+	ArticleURL string
+	Lede       string
+	Source     string
+	Channel    string
+	Platforms  []string
+}
+
+// formatPost returns plain text and HTML bodies for a Matrix message.
+func formatPost(s *PostableStory) (plain, htmlBody string) {
+	// Plain text body
+	var plainParts []string
+	plainParts = append(plainParts, fmt.Sprintf("**%s** \u2192 %s", s.Headline, s.ArticleURL))
+	if s.Lede != "" {
+		plainParts = append(plainParts, s.Lede)
+	}
+	plainParts = append(plainParts, formatSourceTag(s.Source, s.Platforms, false))
+	plain = strings.Join(plainParts, "\n")
+
+	// HTML body
+	var htmlParts []string
+	htmlParts = append(htmlParts, fmt.Sprintf(
+		`<strong><a href="%s">%s</a></strong>`,
+		html.EscapeString(s.ArticleURL),
+		html.EscapeString(s.Headline),
+	))
+	if s.Lede != "" {
+		htmlParts = append(htmlParts, html.EscapeString(s.Lede))
+	}
+	htmlParts = append(htmlParts, formatSourceTag(s.Source, s.Platforms, true))
+	htmlBody = strings.Join(htmlParts, "<br/>")
+
+	return plain, htmlBody
+}
+
+// formatSourceTag builds the source + platform tags line.
+func formatSourceTag(source string, platforms []string, isHTML bool) string {
+	if isHTML {
+		parts := []string{fmt.Sprintf("<code>%s</code>", html.EscapeString(strings.ToLower(source)))}
+		for _, p := range platforms {
+			parts = append(parts, fmt.Sprintf("<code>%s</code>", html.EscapeString(p)))
+		}
+		return strings.Join(parts, " \u00b7 ")
+	}
+
+	parts := []string{fmt.Sprintf("`%s`", strings.ToLower(source))}
+	for _, p := range platforms {
+		parts = append(parts, fmt.Sprintf("`%s`", p))
+	}
+	return strings.Join(parts, " \u00b7 ")
+}

internal/matrix/post_test.go

@@ -0,0 +1,121 @@
+package matrix
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFormatPost_Basic(t *testing.T) {
+	story := &PostableStory{
+		Headline:   "GPU prices drop 30% amid oversupply",
+		ArticleURL: "https://example.com/gpu-prices",
+		Lede:       "Graphics card prices have fallen sharply across all tiers.",
+		Source:     "Ars Technica",
+		Channel:    "tech",
+	}
+
+	plain, html := formatPost(story)
+
+	// Plain text checks
+	if !strings.Contains(plain, "**GPU prices drop 30% amid oversupply**") {
+		t.Errorf("plain missing bold headline: %s", plain)
+	}
+	if !strings.Contains(plain, "→ https://example.com/gpu-prices") {
+		t.Errorf("plain missing article URL: %s", plain)
+	}
+	if !strings.Contains(plain, "Graphics card prices") {
+		t.Errorf("plain missing lede: %s", plain)
+	}
+	if !strings.Contains(plain, "`ars technica`") {
+		t.Errorf("plain missing source tag (lowercase): %s", plain)
+	}
+
+	// HTML checks
+	if !strings.Contains(html, `<strong><a href="https://example.com/gpu-prices">GPU prices drop 30% amid oversupply</a></strong>`) {
+		t.Errorf("html missing linked headline: %s", html)
+	}
+	if !strings.Contains(html, "<code>ars technica</code>") {
+		t.Errorf("html missing source code tag: %s", html)
+	}
+}
+
+func TestFormatPost_WithPlatforms(t *testing.T) {
+	story := &PostableStory{
+		Headline:   "Switch 2 launch lineup revealed",
+		ArticleURL: "https://example.com/switch2",
+		Lede:       "Nintendo confirms 20 titles for day one.",
+		Source:     "Eurogamer",
+		Channel:    "gaming",
+		Platforms:  []string{"nintendo-switch"},
+	}
+
+	plain, html := formatPost(story)
+
+	if !strings.Contains(plain, "`eurogamer` · `nintendo-switch`") {
+		t.Errorf("plain missing platform tag: %s", plain)
+	}
+	if !strings.Contains(html, "<code>eurogamer</code> · <code>nintendo-switch</code>") {
+		t.Errorf("html missing platform tag: %s", html)
+	}
+}
+
+func TestFormatPost_MultiplePlatforms(t *testing.T) {
+	story := &PostableStory{
+		Headline:   "Elden Ring Nightreign launches",
+		ArticleURL: "https://example.com/er",
+		Lede:       "Available on all major platforms.",
+		Source:     "Ars Technica",
+		Channel:    "gaming",
+		Platforms:  []string{"pc", "playstation", "xbox"},
+	}
+
+	plain, _ := formatPost(story)
+
+	if !strings.Contains(plain, "`ars technica` · `pc` · `playstation` · `xbox`") {
+		t.Errorf("plain tags wrong: %s", plain)
+	}
+}
+
+func TestFormatPost_EmptyLede(t *testing.T) {
+	story := &PostableStory{
+		Headline:   "Breaking news",
+		ArticleURL: "https://example.com/breaking",
+		Source:     "Guardian",
+		Channel:    "politics",
+	}
+
+	plain, html := formatPost(story)
+
+	// Should have headline and source but no empty line for lede
+	lines := strings.Split(plain, "\n")
+	if len(lines) != 2 {
+		t.Errorf("expected 2 lines (headline + source), got %d: %v", len(lines), lines)
+	}
+
+	parts := strings.Split(html, "<br/>")
+	if len(parts) != 2 {
+		t.Errorf("expected 2 html parts, got %d: %v", len(parts), parts)
+	}
+}
+
+func TestFormatPost_HTMLEscaping(t *testing.T) {
+	story := &PostableStory{
+		Headline:   `Apple's "M5" chip: <faster> & better`,
+		ArticleURL: "https://example.com/m5",
+		Lede:       `Performance gains of >50% in "real-world" tests.`,
+		Source:     "Test & Source",
+		Channel:    "tech",
+	}
+
+	_, html := formatPost(story)
+
+	if strings.Contains(html, "<faster>") {
+		t.Error("html contains unescaped angle brackets in headline")
+	}
+	if !strings.Contains(html, "&amp;") {
+		t.Error("html missing escaped ampersand")
+	}
+	if !strings.Contains(html, "&gt;50%") {
+		t.Error("html missing escaped > in lede")
+	}
+}

internal/poster/queue.go

@@ -0,0 +1,229 @@
+package poster
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"pete/internal/config"
+	"pete/internal/dedup"
+	"pete/internal/matrix"
+	"pete/internal/storage"
+)
+
+const maxRetries = 3
+
+// QueueItem represents a story ready to be posted.
+type QueueItem struct {
+	GUID       string
+	Headline   string
+	Lede       string
+	ImageURL   string
+	ArticleURL string
+	Source     string
+	Channel    string
+	Platforms  []string
+	retries    int
+}
+
+// Queue manages per-channel metered release of stories.
+type Queue struct {
+	mu       sync.Mutex
+	queues   map[string][]QueueItem // channel -> items
+	config   config.PostingConfig
+	mx       *matrix.Client
+	done     chan struct{}
+}
+
+// NewQueue creates a new metered release queue.
+func NewQueue(cfg config.PostingConfig, mx *matrix.Client) *Queue {
+	return &Queue{
+		queues: make(map[string][]QueueItem),
+		config: cfg,
+		mx:     mx,
+		done:   make(chan struct{}),
+	}
+}
+
+// Enqueue adds a story to the appropriate channel queue.
+// Stories for unconfigured channels are dropped with a warning.
+func (q *Queue) Enqueue(item QueueItem) {
+	if _, ok := q.mx.ChannelRoomID(item.Channel); !ok {
+		slog.Warn("dropping story for unconfigured channel",
+			"guid", item.GUID,
+			"channel", item.Channel,
+		)
+		return
+	}
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	q.queues[item.Channel] = append(q.queues[item.Channel], item)
+	slog.Info("story queued for posting",
+		"guid", item.GUID,
+		"channel", item.Channel,
+		"queue_depth", len(q.queues[item.Channel]),
+	)
+}
+
+// Start runs the queue drain ticker. Blocks until ctx is cancelled, then drains remaining items.
+func (q *Queue) Start(ctx context.Context) {
+	defer close(q.done)
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// Graceful drain: attempt to post remaining queued items
+			q.drainAll()
+			return
+		case <-ticker.C:
+			q.drain()
+		}
+	}
+}
+
+// Wait blocks until the queue goroutine has finished (including graceful drain).
+func (q *Queue) Wait() {
+	<-q.done
+}
+
+func (q *Queue) drain() {
+	q.mu.Lock()
+	channels := make([]string, 0, len(q.queues))
+	for ch := range q.queues {
+		channels = append(channels, ch)
+	}
+	q.mu.Unlock()
+
+	for _, ch := range channels {
+		q.drainChannel(ch)
+	}
+}
+
+// drainAll posts all remaining items ignoring rate limits (shutdown path).
+func (q *Queue) drainAll() {
+	q.mu.Lock()
+	channels := make([]string, 0, len(q.queues))
+	for ch := range q.queues {
+		channels = append(channels, ch)
+	}
+	q.mu.Unlock()
+
+	for _, ch := range channels {
+		for {
+			q.mu.Lock()
+			items := q.queues[ch]
+			if len(items) == 0 {
+				q.mu.Unlock()
+				break
+			}
+			item := items[0]
+			q.queues[ch] = items[1:]
+			q.mu.Unlock()
+
+			q.postItem(item)
+		}
+	}
+}
+
+func (q *Queue) drainChannel(channel string) {
+	q.mu.Lock()
+	items := q.queues[channel]
+	if len(items) == 0 {
+		q.mu.Unlock()
+		return
+	}
+	q.mu.Unlock()
+
+	now := time.Now().Unix()
+	minInterval := int64(q.config.MinIntervalSeconds)
+	burstWindow := int64(q.config.BurstCapWindowSeconds)
+
+	// Check minimum interval since last post
+	lastPost := storage.GetLastPostTime(channel)
+	if now-lastPost < minInterval {
+		return
+	}
+
+	// Check burst cap
+	windowStart := now - burstWindow
+	postsInWindow := storage.CountPostsInWindow(channel, windowStart)
+	if postsInWindow >= q.config.BurstCapCount {
+		return
+	}
+
+	// Dequeue one item
+	q.mu.Lock()
+	items = q.queues[channel]
+	if len(items) == 0 {
+		q.mu.Unlock()
+		return
+	}
+	item := items[0]
+	q.queues[channel] = items[1:]
+	q.mu.Unlock()
+
+	q.postItem(item)
+}
+
+func (q *Queue) postItem(item QueueItem) {
+	// Last-mile dedup: if this canonical URL was already posted to this channel
+	// within the cooldown window, drop silently. Catches "same article, different
+	// GUID across feeds" and any race where two items slipped past ingest dedup.
+	canonical := dedup.CanonicalURL(item.ArticleURL)
+	cooldownSec := int64(q.config.DedupCooldownHours) * 3600
+	if storage.WasCanonicalPostedRecently(canonical, item.Channel, cooldownSec) {
+		slog.Info("dropping duplicate post (canonical URL posted within cooldown)",
+			"guid", item.GUID,
+			"channel", item.Channel,
+			"url_canonical", canonical,
+			"cooldown_hours", q.config.DedupCooldownHours,
+		)
+		return
+	}
+
+	story := &matrix.PostableStory{
+		ImageURL:   item.ImageURL,
+		Headline:   item.Headline,
+		ArticleURL: item.ArticleURL,
+		Lede:       item.Lede,
+		Source:     item.Source,
+		Channel:    item.Channel,
+		Platforms:  item.Platforms,
+	}
+
+	eventID, err := q.mx.PostStory(item.Channel, story)
+	if err != nil {
+		item.retries++
+		if item.retries >= maxRetries {
+			slog.Error("story dead-lettered after max retries",
+				"guid", item.GUID,
+				"channel", item.Channel,
+				"err", err,
+			)
+			return
+		}
+		slog.Error("failed to post story, will retry",
+			"guid", item.GUID,
+			"channel", item.Channel,
+			"attempt", item.retries,
+			"err", err,
+		)
+		// Re-queue at front for retry
+		q.mu.Lock()
+		q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
+		q.mu.Unlock()
+		return
+	}
+
+	// Record in post log (INSERT OR IGNORE via unique index)
+	storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
+
+	slog.Info("story posted",
+		"guid", item.GUID,
+		"channel", item.Channel,
+		"event_id", eventID,
+	)
+}

internal/poster/queue_test.go

@@ -0,0 +1,145 @@
+package poster
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"pete/internal/config"
+	"pete/internal/storage"
+)
+
+// mockMatrix implements the subset of matrix.Client that Queue needs.
+// We test via the public Enqueue/Start interface and check storage state.
+
+func setupTestEnv(t *testing.T) {
+	t.Helper()
+	dbPath := filepath.Join(t.TempDir(), "test.db")
+	if err := storage.Init(dbPath); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { storage.Close() })
+}
+
+func TestQueue_EnqueueAndDrain(t *testing.T) {
+	setupTestEnv(t)
+
+	q := &Queue{
+		queues: make(map[string][]QueueItem),
+		config: config.PostingConfig{
+			MinIntervalSeconds:    0,
+			BurstCapCount:         10,
+			BurstCapWindowSeconds: 1800,
+		},
+		done: make(chan struct{}),
+	}
+
+	// Enqueue directly (bypass ChannelRoomID check)
+	q.mu.Lock()
+	q.queues["tech"] = append(q.queues["tech"], QueueItem{
+		GUID:    "g1",
+		Channel: "tech",
+	})
+	q.mu.Unlock()
+
+	// Verify item is in queue
+	q.mu.Lock()
+	if len(q.queues["tech"]) != 1 {
+		t.Fatalf("expected 1 item in queue, got %d", len(q.queues["tech"]))
+	}
+	q.mu.Unlock()
+}
+
+func TestQueueItem_MaxRetries(t *testing.T) {
+	// Verify the retry counter works
+	item := QueueItem{GUID: "g1", Channel: "tech", retries: 0}
+	item.retries++
+	if item.retries != 1 {
+		t.Errorf("expected retries=1, got %d", item.retries)
+	}
+	item.retries++
+	item.retries++
+	if item.retries < maxRetries {
+		t.Errorf("expected retries >= maxRetries after 3 increments, got %d", item.retries)
+	}
+}
+
+func TestQueue_StartAndWait(t *testing.T) {
+	setupTestEnv(t)
+
+	q := &Queue{
+		queues: make(map[string][]QueueItem),
+		config: config.PostingConfig{
+			MinIntervalSeconds:    0,
+			BurstCapCount:         10,
+			BurstCapWindowSeconds: 1800,
+		},
+		done: make(chan struct{}),
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		q.Start(ctx)
+	}()
+
+	// Cancel immediately — Start should return promptly
+	cancel()
+
+	done := make(chan struct{})
+	go func() {
+		q.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// good
+	case <-time.After(5 * time.Second):
+		t.Fatal("queue.Wait() did not return after cancel")
+	}
+}
+
+func TestQueue_BurstCap(t *testing.T) {
+	setupTestEnv(t)
+
+	cfg := config.PostingConfig{
+		MinIntervalSeconds:    0,
+		BurstCapCount:         2,
+		BurstCapWindowSeconds: 3600,
+	}
+
+	q := &Queue{
+		queues: make(map[string][]QueueItem),
+		config: cfg,
+		done:   make(chan struct{}),
+	}
+
+	// Simulate 2 posts already in window
+	now := time.Now().Unix()
+	storage.Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"prev1", "tech", "$e1", now-100)
+	storage.Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"prev2", "tech", "$e2", now-50)
+
+	// Queue a new item
+	q.mu.Lock()
+	q.queues["tech"] = []QueueItem{{GUID: "g3", Channel: "tech"}}
+	q.mu.Unlock()
+
+	// drainChannel should not post (burst cap reached)
+	q.drainChannel("tech")
+
+	q.mu.Lock()
+	remaining := len(q.queues["tech"])
+	q.mu.Unlock()
+
+	if remaining != 1 {
+		t.Errorf("expected item to remain in queue (burst cap), got %d remaining", remaining)
+	}
+}

internal/poster/tracker.go

@@ -0,0 +1,27 @@
+package poster
+
+import (
+	"log/slog"
+	"time"
+
+	"pete/internal/storage"
+
+	"maunium.net/go/mautrix/id"
+)
+
+// HandleReaction processes a reaction event by mapping it back to the story GUID.
+func HandleReaction(roomID id.RoomID, eventID id.EventID, targetEventID id.EventID, emoji string, userID id.UserID) {
+	guid, channel, found := storage.LookupPostGUID(string(targetEventID))
+	if !found {
+		// Reaction on a message we didn't post — ignore
+		return
+	}
+
+	storage.InsertReaction(guid, channel, string(eventID), emoji, string(userID), time.Now().Unix())
+	slog.Debug("reaction recorded",
+		"guid", guid,
+		"channel", channel,
+		"emoji", emoji,
+		"user", userID,
+	)
+}

internal/poster/tracker_test.go

@@ -0,0 +1,89 @@
+package poster
+
+import (
+	"path/filepath"
+	"testing"
+
+	"pete/internal/storage"
+
+	"maunium.net/go/mautrix/id"
+)
+
+func setupTrackerTestDB(t *testing.T) {
+	t.Helper()
+	dbPath := filepath.Join(t.TempDir(), "test.db")
+	if err := storage.Init(dbPath); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { storage.Close() })
+}
+
+func TestHandleReaction_KnownPost(t *testing.T) {
+	setupTrackerTestDB(t)
+
+	// Insert a post log entry
+	storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
+
+	// Handle a reaction to that post
+	HandleReaction(
+		id.RoomID("!room:example.org"),
+		id.EventID("$react1:example.org"),
+		id.EventID("$post1:example.org"),
+		"👍",
+		id.UserID("@user:example.org"),
+	)
+
+	// Verify reaction was recorded
+	var count int
+	storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE post_guid = ?`, "story-1").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 reaction, got %d", count)
+	}
+}
+
+func TestHandleReaction_UnknownPost(t *testing.T) {
+	setupTrackerTestDB(t)
+
+	// Handle a reaction to an unknown event — should not crash or insert
+	HandleReaction(
+		id.RoomID("!room:example.org"),
+		id.EventID("$react1:example.org"),
+		id.EventID("$unknown:example.org"),
+		"👍",
+		id.UserID("@user:example.org"),
+	)
+
+	var count int
+	storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
+	if count != 0 {
+		t.Errorf("expected 0 reactions for unknown post, got %d", count)
+	}
+}
+
+func TestHandleReaction_DuplicateIgnored(t *testing.T) {
+	setupTrackerTestDB(t)
+
+	storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
+
+	// Send same reaction twice
+	HandleReaction(
+		id.RoomID("!room:example.org"),
+		id.EventID("$react1:example.org"),
+		id.EventID("$post1:example.org"),
+		"👍",
+		id.UserID("@user:example.org"),
+	)
+	HandleReaction(
+		id.RoomID("!room:example.org"),
+		id.EventID("$react1:example.org"),
+		id.EventID("$post1:example.org"),
+		"👍",
+		id.UserID("@user:example.org"),
+	)
+
+	var count int
+	storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 reaction (deduped), got %d", count)
+	}
+}

internal/storage/db.go

@@ -0,0 +1,140 @@
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	_ "modernc.org/sqlite"
+)
+
+var (
+	mu       sync.RWMutex
+	globalDB *sql.DB
+)
+
+// Init opens (or creates) the SQLite database and runs migrations.
+func Init(dbPath string) error {
+	mu.Lock()
+	defer mu.Unlock()
+
+	if globalDB != nil {
+		return nil
+	}
+
+	dir := filepath.Dir(dbPath)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("create data dir: %w", err)
+	}
+
+	d, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)")
+	if err != nil {
+		return fmt.Errorf("open database: %w", err)
+	}
+
+	d.SetMaxOpenConns(1)
+
+	if err := runMigrations(d); err != nil {
+		return fmt.Errorf("run migrations: %w", err)
+	}
+
+	globalDB = d
+	slog.Info("database initialized", "path", dbPath)
+	return nil
+}
+
+// Get returns the global database handle. Panics if Init was not called.
+func Get() *sql.DB {
+	mu.RLock()
+	db := globalDB
+	mu.RUnlock()
+	if db == nil {
+		panic("storage.Get() called before storage.Init()")
+	}
+	return db
+}
+
+// Close closes the global database handle.
+func Close() error {
+	mu.Lock()
+	defer mu.Unlock()
+	if globalDB != nil {
+		err := globalDB.Close()
+		globalDB = nil
+		return err
+	}
+	return nil
+}
+
+func runMigrations(d *sql.DB) error {
+	if _, err := d.Exec(schema); err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	// Idempotent column adds for DBs created before the dedup columns existed.
+	// SQLite errors with "duplicate column name" when the column is already there;
+	// we swallow that specifically.
+	addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
+	addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
+	addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
+
+	// FTS5 virtual tables don't support IF NOT EXISTS reliably.
+	// Check sqlite_master before creating.
+	var ftsExists int
+	d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists)
+	if ftsExists == 0 {
+		if _, err := d.Exec(ftsSchema); err != nil {
+			return fmt.Errorf("create FTS5 table: %w", err)
+		}
+		if _, err := d.Exec(ftsTriggers); err != nil {
+			return fmt.Errorf("create FTS5 triggers: %w", err)
+		}
+		slog.Info("created FTS5 search index")
+	}
+
+	return nil
+}
+
+// RunMaintenance prunes stale data. Called periodically.
+func RunMaintenance(recentWindowHours, classificationLogDays int) {
+	recentCutoff := nowUnix() - int64(recentWindowHours*3600)
+	exec("prune recent_headlines",
+		`DELETE FROM recent_headlines WHERE seen_at < ?`, recentCutoff)
+
+	logCutoff := nowUnix() - int64(classificationLogDays*86400)
+	exec("prune classification_log",
+		`DELETE FROM classification_log WHERE logged_at < ?`, logCutoff)
+
+	// Prune old stories (30 days) and their post logs / reactions
+	storyCutoff := nowUnix() - int64(30*86400)
+	exec("prune old stories",
+		`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
+	exec("prune old post_log",
+		`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
+	exec("prune old reactions",
+		`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
+
+	exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
+	exec("optimize", "PRAGMA optimize")
+}
+
+// exec is a fire-and-forget helper that logs errors.
+func exec(label, query string, args ...any) {
+	if _, err := Get().Exec(query, args...); err != nil {
+		slog.Error("db exec failed", "op", label, "err", err)
+	}
+}
+
+func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
+	q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
+	if _, err := d.Exec(q); err != nil {
+		// SQLite returns "duplicate column name" when the column already exists.
+		if !strings.Contains(err.Error(), "duplicate column name") {
+			slog.Error("alter table failed", "table", table, "column", column, "err", err)
+		}
+	}
+}

internal/storage/queries.go

@@ -0,0 +1,225 @@
+package storage
+
+import (
+	"encoding/json"
+	"log/slog"
+	"time"
+)
+
+func nowUnix() int64 {
+	return time.Now().Unix()
+}
+
+// IsGUIDSeen checks if a GUID has already been ingested.
+func IsGUIDSeen(guid string) bool {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE guid = ?`, guid).Scan(&count); err != nil {
+		slog.Error("IsGUIDSeen query failed", "guid", guid, "err", err)
+		return true // fail closed: assume seen to prevent duplicates
+	}
+	return count > 0
+}
+
+// InsertStory inserts a new story record.
+func InsertStory(s *Story) error {
+	classified := 0
+	if s.Classified {
+		classified = 1
+	}
+	_, err := Get().Exec(
+		`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, feed_hint, platforms, channel, classified, seen_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.FeedHint, s.Platforms, s.Channel, classified, s.SeenAt,
+	)
+	return err
+}
+
+// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
+// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
+func nullIfEmpty(s string) any {
+	if s == "" {
+		return nil
+	}
+	return s
+}
+
+// IsCanonicalSeen reports whether any story with this canonical URL exists.
+// Empty input always returns false (no canonical = no dedup possible).
+func IsCanonicalSeen(canonical string) bool {
+	if canonical == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE url_canonical = ?`, canonical).Scan(&n); err != nil {
+		slog.Error("IsCanonicalSeen query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// IsHeadlineSeen reports whether a same-source story with this normalized
+// headline already exists. Empty inputs return false.
+func IsHeadlineSeen(source, headlineNorm string) bool {
+	if source == "" || headlineNorm == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM stories WHERE source = ? AND headline_norm = ?`,
+		source, headlineNorm).Scan(&n); err != nil {
+		slog.Error("IsHeadlineSeen query failed", "err", err)
+		return true
+	}
+	return n > 0
+}
+
+// WasCanonicalPostedRecently reports whether the canonical URL was posted to
+// the given channel within `cooldownSeconds`. Empty canonical returns false.
+func WasCanonicalPostedRecently(canonical, channel string, cooldownSeconds int64) bool {
+	if canonical == "" || cooldownSeconds <= 0 {
+		return false
+	}
+	cutoff := nowUnix() - cooldownSeconds
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM post_log WHERE url_canonical = ? AND channel = ? AND posted_at >= ?`,
+		canonical, channel, cutoff).Scan(&n); err != nil {
+		slog.Error("WasCanonicalPostedRecently query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// MarkClassified marks a story as successfully classified and sets its channel.
+func MarkClassified(guid, channel, platforms string) {
+	exec("mark classified",
+		`UPDATE stories SET classified = 1, channel = ?, platforms = ? WHERE guid = ?`,
+		channel, platforms, guid)
+}
+
+// GetUnclassifiedStories returns stories from a specific source that were ingested but not yet classified.
+func GetUnclassifiedStories(source string) ([]Story, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, seen_at
+		 FROM stories WHERE classified = 0 AND source = ? ORDER BY seen_at ASC LIMIT 20`, source)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var stories []Story
+	for rows.Next() {
+		var s Story
+		if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.SeenAt); err != nil {
+			return nil, err
+		}
+		stories = append(stories, s)
+	}
+	return stories, rows.Err()
+}
+
+// InsertRecentHeadline adds a headline to the dedup context window.
+func InsertRecentHeadline(guid, headline, source string) {
+	exec("insert recent_headline",
+		`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		guid, headline, source, nowUnix())
+}
+
+// GetRecentHeadlines returns the most recent headlines for LLM dedup context, capped at limit.
+func GetRecentHeadlines(limit int) ([]RecentHeadline, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, source, seen_at FROM recent_headlines ORDER BY seen_at DESC LIMIT ?`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var headlines []RecentHeadline
+	for rows.Next() {
+		var h RecentHeadline
+		if err := rows.Scan(&h.GUID, &h.Headline, &h.Source, &h.SeenAt); err != nil {
+			return nil, err
+		}
+		headlines = append(headlines, h)
+	}
+	return headlines, rows.Err()
+}
+
+// InsertClassificationLog stores a classification result for tuning review.
+func InsertClassificationLog(guid, resultJSON string) {
+	exec("insert classification_log",
+		`INSERT OR REPLACE INTO classification_log (guid, result, logged_at) VALUES (?, ?, ?)`,
+		guid, resultJSON, nowUnix())
+}
+
+// InsertPostLog records that a story was posted to a channel.
+// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
+func InsertPostLog(guid, channel, eventID, urlCanonical string) {
+	exec("insert post_log",
+		`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
+		guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
+}
+
+// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
+func GetLastPostTime(channel string) int64 {
+	var t int64
+	if err := Get().QueryRow(`SELECT COALESCE(MAX(posted_at), 0) FROM post_log WHERE channel = ?`, channel).Scan(&t); err != nil {
+		slog.Error("GetLastPostTime query failed", "channel", channel, "err", err)
+		return nowUnix() // fail closed: pretend we just posted to prevent flooding
+	}
+	return t
+}
+
+// CountPostsInWindow counts posts to a channel within a time window.
+func CountPostsInWindow(channel string, windowStart int64) int {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE channel = ? AND posted_at >= ?`, channel, windowStart).Scan(&count); err != nil {
+		slog.Error("CountPostsInWindow query failed", "channel", channel, "err", err)
+		return 999 // fail closed: pretend burst cap reached to prevent flooding
+	}
+	return count
+}
+
+// LookupPostGUID finds the story GUID for a given Matrix event ID.
+func LookupPostGUID(eventID string) (guid, channel string, found bool) {
+	err := Get().QueryRow(
+		`SELECT guid, channel FROM post_log WHERE event_id = ? LIMIT 1`, eventID).Scan(&guid, &channel)
+	if err != nil {
+		return "", "", false
+	}
+	return guid, channel, true
+}
+
+// InsertReaction records a reaction on a posted story.
+// Uses OR IGNORE to deduplicate if Matrix delivers the same reaction twice.
+func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt int64) {
+	exec("insert reaction",
+		`INSERT OR IGNORE INTO reactions (post_guid, channel, event_id, emoji, user_id, reacted_at) VALUES (?, ?, ?, ?, ?, ?)`,
+		postGUID, channel, eventID, emoji, userID, reactedAt)
+}
+
+// MarshalPlatforms converts a string slice to a JSON array string for storage.
+func MarshalPlatforms(platforms []string) string {
+	if len(platforms) == 0 {
+		return "[]"
+	}
+	data, err := json.Marshal(platforms)
+	if err != nil {
+		slog.Error("marshal platforms failed", "err", err)
+		return "[]"
+	}
+	return string(data)
+}
+
+// UnmarshalPlatforms converts a JSON array string back to a string slice.
+func UnmarshalPlatforms(raw string) []string {
+	if raw == "" || raw == "[]" {
+		return nil
+	}
+	var platforms []string
+	if err := json.Unmarshal([]byte(raw), &platforms); err != nil {
+		slog.Error("unmarshal platforms failed", "err", err, "raw", raw)
+		return nil
+	}
+	return platforms
+}

internal/storage/schema.go

@@ -0,0 +1,93 @@
+package storage
+
+const schema = `
+CREATE TABLE IF NOT EXISTS stories (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    guid          TEXT UNIQUE NOT NULL,
+    headline      TEXT NOT NULL,
+    lede          TEXT,
+    image_url     TEXT,
+    article_url   TEXT NOT NULL,
+    url_canonical TEXT,
+    headline_norm TEXT,
+    source        TEXT NOT NULL,
+    feed_hint     TEXT,
+    platforms     TEXT,
+    channel       TEXT,
+    classified    INTEGER NOT NULL DEFAULT 0,
+    seen_at       INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS recent_headlines (
+    guid       TEXT PRIMARY KEY,
+    headline   TEXT NOT NULL,
+    source     TEXT NOT NULL,
+    seen_at    INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS classification_log (
+    guid       TEXT PRIMARY KEY,
+    result     TEXT NOT NULL,
+    logged_at  INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS post_log (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    guid          TEXT NOT NULL,
+    channel       TEXT NOT NULL,
+    event_id      TEXT,
+    url_canonical TEXT,
+    posted_at     INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS reactions (
+    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+    post_guid  TEXT NOT NULL,
+    channel    TEXT NOT NULL,
+    event_id   TEXT NOT NULL,
+    emoji      TEXT NOT NULL,
+    user_id    TEXT NOT NULL,
+    reacted_at INTEGER NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
+CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
+CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
+CREATE INDEX IF NOT EXISTS idx_post_log_canonical_channel ON post_log(url_canonical, channel, posted_at);
+CREATE INDEX IF NOT EXISTS idx_stories_classified_source ON stories(classified, source);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canonical) WHERE url_canonical IS NOT NULL AND url_canonical <> '';
+CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
+CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
+`
+
+const ftsSchema = `
+CREATE VIRTUAL TABLE stories_fts USING fts5(
+    guid UNINDEXED,
+    headline,
+    lede,
+    source UNINDEXED,
+    platforms UNINDEXED,
+    content='stories',
+    content_rowid='id'
+);
+`
+
+const ftsTriggers = `
+CREATE TRIGGER stories_fts_insert AFTER INSERT ON stories BEGIN
+    INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
+    VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
+END;
+
+CREATE TRIGGER stories_fts_delete AFTER DELETE ON stories BEGIN
+    INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
+    VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
+END;
+
+CREATE TRIGGER stories_fts_update AFTER UPDATE ON stories BEGIN
+    INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
+    VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
+    INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
+    VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
+END;
+`

internal/storage/storage_test.go

@@ -0,0 +1,405 @@
+package storage
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+)
+
+func setupTestDB(t *testing.T) {
+	t.Helper()
+
+	// Reset global state
+	mu.Lock()
+	if globalDB != nil {
+		globalDB.Close()
+		globalDB = nil
+	}
+	mu.Unlock()
+
+	dbPath := filepath.Join(t.TempDir(), "test.db")
+	if err := Init(dbPath); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { Close() })
+}
+
+func TestInitAndGet(t *testing.T) {
+	setupTestDB(t)
+	db := Get()
+	if db == nil {
+		t.Fatal("Get() returned nil after Init()")
+	}
+}
+
+func TestIsGUIDSeen_NotSeen(t *testing.T) {
+	setupTestDB(t)
+	if IsGUIDSeen("never-seen-guid") {
+		t.Error("expected false for unseen GUID")
+	}
+}
+
+func TestInsertStoryAndGUIDSeen(t *testing.T) {
+	setupTestDB(t)
+
+	s := &Story{
+		GUID:       "test-guid-1",
+		Headline:   "Test Headline",
+		Lede:       "Test lede sentence.",
+		ArticleURL: "https://example.com/article",
+		Source:     "Test Source",
+		FeedHint:   "tech",
+		SeenAt:     1700000000,
+	}
+	if err := InsertStory(s); err != nil {
+		t.Fatal(err)
+	}
+
+	if !IsGUIDSeen("test-guid-1") {
+		t.Error("expected true after insert")
+	}
+	if IsGUIDSeen("other-guid") {
+		t.Error("expected false for different GUID")
+	}
+}
+
+func TestInsertStoryDuplicateGUID(t *testing.T) {
+	setupTestDB(t)
+
+	s := &Story{
+		GUID: "dup-guid", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	}
+	if err := InsertStory(s); err != nil {
+		t.Fatal(err)
+	}
+	// Second insert with same GUID should fail
+	err := InsertStory(s)
+	if err == nil {
+		t.Error("expected error on duplicate GUID insert")
+	}
+}
+
+func TestMarkClassifiedAndGetUnclassified(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{
+		GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	})
+	InsertStory(&Story{
+		GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
+	})
+
+	unclassified, err := GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(unclassified) != 2 {
+		t.Fatalf("unclassified = %d, want 2", len(unclassified))
+	}
+
+	MarkClassified("g1", "tech", "[]")
+
+	unclassified, err = GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(unclassified) != 1 {
+		t.Fatalf("unclassified = %d, want 1", len(unclassified))
+	}
+	if unclassified[0].GUID != "g2" {
+		t.Errorf("remaining = %q, want g2", unclassified[0].GUID)
+	}
+}
+
+func TestRecentHeadlines(t *testing.T) {
+	setupTestDB(t)
+
+	// Insert with explicit timestamps to ensure ordering
+	db := Get()
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g1", "Headline One", "Source A", 1000)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g2", "Headline Two", "Source B", 2000)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g3", "Headline Three", "Source A", 3000)
+
+	headlines, err := GetRecentHeadlines(2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(headlines) != 2 {
+		t.Fatalf("headlines = %d, want 2", len(headlines))
+	}
+	// Should be most recent first
+	if headlines[0].GUID != "g3" {
+		t.Errorf("first = %q, want g3", headlines[0].GUID)
+	}
+	if headlines[1].GUID != "g2" {
+		t.Errorf("second = %q, want g2", headlines[1].GUID)
+	}
+}
+
+func TestPostLogAndLookup(t *testing.T) {
+	setupTestDB(t)
+
+	InsertPostLog("story-1", "tech", "$event1:example.org", "")
+	InsertPostLog("story-2", "politics", "$event2:example.org", "")
+
+	guid, channel, found := LookupPostGUID("$event1:example.org")
+	if !found {
+		t.Fatal("expected to find event")
+	}
+	if guid != "story-1" || channel != "tech" {
+		t.Errorf("got guid=%q channel=%q", guid, channel)
+	}
+
+	_, _, found = LookupPostGUID("$nonexistent:example.org")
+	if found {
+		t.Error("expected not found for unknown event")
+	}
+}
+
+func TestLastPostTimeAndCountInWindow(t *testing.T) {
+	setupTestDB(t)
+
+	if got := GetLastPostTime("tech"); got != 0 {
+		t.Errorf("empty channel last post = %d, want 0", got)
+	}
+
+	// Insert posts with specific timestamps
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g1", "tech", "$e1", 1000)
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g2", "tech", "$e2", 2000)
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g3", "politics", "$e3", 1500)
+
+	if got := GetLastPostTime("tech"); got != 2000 {
+		t.Errorf("tech last post = %d, want 2000", got)
+	}
+
+	count := CountPostsInWindow("tech", 500)
+	if count != 2 {
+		t.Errorf("tech posts in window = %d, want 2", count)
+	}
+
+	count = CountPostsInWindow("tech", 1500)
+	if count != 1 {
+		t.Errorf("tech posts after 1500 = %d, want 1", count)
+	}
+
+	count = CountPostsInWindow("politics", 0)
+	if count != 1 {
+		t.Errorf("politics posts = %d, want 1", count)
+	}
+}
+
+func TestReactions(t *testing.T) {
+	setupTestDB(t)
+
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1000)
+	InsertReaction("story-1", "tech", "$react2", "👍", "@user2:example.org", 1001)
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE post_guid = ?`, "story-1").Scan(&count)
+	if count != 2 {
+		t.Errorf("reactions = %d, want 2", count)
+	}
+}
+
+func TestMarshalUnmarshalPlatforms(t *testing.T) {
+	tests := []struct {
+		name  string
+		input []string
+		json  string
+		back  []string
+	}{
+		{"nil", nil, "[]", nil},
+		{"empty", []string{}, "[]", nil},
+		{"single", []string{"pc"}, `["pc"]`, []string{"pc"}},
+		{"multi", []string{"nintendo-switch", "pc"}, `["nintendo-switch","pc"]`, []string{"nintendo-switch", "pc"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := MarshalPlatforms(tt.input)
+			if got != tt.json {
+				t.Errorf("Marshal(%v) = %q, want %q", tt.input, got, tt.json)
+			}
+			back := UnmarshalPlatforms(got)
+			if len(back) != len(tt.back) {
+				t.Errorf("Unmarshal roundtrip: %v, want %v", back, tt.back)
+			}
+		})
+	}
+}
+
+func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
+	setupTestDB(t)
+
+	InsertPostLog("story-1", "tech", "$event1:example.org", "")
+	// Same guid+channel should be silently ignored
+	InsertPostLog("story-1", "tech", "$event1-retry:example.org", "")
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 post_log entry, got %d", count)
+	}
+
+	// Different channel should be allowed
+	InsertPostLog("story-1", "politics", "$event2:example.org", "")
+	Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
+	if count != 2 {
+		t.Errorf("expected 2 post_log entries across channels, got %d", count)
+	}
+}
+
+func TestInsertReaction_DuplicateIgnored(t *testing.T) {
+	setupTestDB(t)
+
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1000)
+	// Same event_id should be silently ignored
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1001)
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE event_id = ?`, "$react1").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 reaction, got %d", count)
+	}
+}
+
+func TestGetUnclassifiedStories_ScopedBySource(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "SourceA", SeenAt: 1})
+	InsertStory(&Story{GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "SourceB", SeenAt: 2})
+	InsertStory(&Story{GUID: "g3", Headline: "H3", ArticleURL: "https://c.com", Source: "SourceA", SeenAt: 3})
+
+	got, err := GetUnclassifiedStories("SourceA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("expected 2 stories for SourceA, got %d", len(got))
+	}
+
+	got, err = GetUnclassifiedStories("SourceB")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 {
+		t.Fatalf("expected 1 story for SourceB, got %d", len(got))
+	}
+
+	got, err = GetUnclassifiedStories("SourceC")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("expected 0 stories for SourceC, got %d", len(got))
+	}
+}
+
+func TestGetUnclassifiedStories_Limit20(t *testing.T) {
+	setupTestDB(t)
+
+	for i := 0; i < 30; i++ {
+		InsertStory(&Story{
+			GUID: fmt.Sprintf("g%d", i), Headline: fmt.Sprintf("H%d", i),
+			ArticleURL: "https://a.com", Source: "S", SeenAt: int64(i),
+		})
+	}
+
+	got, err := GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 20 {
+		t.Fatalf("expected 20 (capped), got %d", len(got))
+	}
+}
+
+func TestRunMaintenance(t *testing.T) {
+	setupTestDB(t)
+	db := Get()
+
+	now := nowUnix()
+	old := now - 100*86400 // 100 days ago
+
+	// Insert old + recent stories
+	InsertStory(&Story{GUID: "old", Headline: "Old", ArticleURL: "https://a.com", Source: "S", Classified: true, SeenAt: old})
+	InsertStory(&Story{GUID: "new", Headline: "New", ArticleURL: "https://b.com", Source: "S", Classified: true, SeenAt: now})
+	MarkClassified("old", "tech", "[]")
+	MarkClassified("new", "tech", "[]")
+
+	// Insert old + recent post logs
+	db.Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`, "old", "tech", "$e1", old)
+	db.Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`, "new", "tech", "$e2", now)
+
+	// Insert old + recent reactions
+	InsertReaction("old", "tech", "$r1", "👍", "@u:x", old)
+	InsertReaction("new", "tech", "$r2", "👍", "@u:x", now)
+
+	// Insert old + recent headlines
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "old", "Old", "S", old)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "new", "New", "S", now)
+
+	RunMaintenance(24, 7)
+
+	// Old stories should be pruned
+	var count int
+	db.QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&count)
+	if count != 1 {
+		t.Errorf("stories: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM post_log`).Scan(&count)
+	if count != 1 {
+		t.Errorf("post_log: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
+	if count != 1 {
+		t.Errorf("reactions: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM recent_headlines`).Scan(&count)
+	if count != 1 {
+		t.Errorf("recent_headlines: expected 1, got %d", count)
+	}
+}
+
+func TestFTS5Search(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{
+		GUID: "g1", Headline: "Elden Ring DLC breaks sales records", Lede: "FromSoftware celebrates milestone.",
+		ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	})
+	InsertStory(&Story{
+		GUID: "g2", Headline: "Apple unveils new MacBook Pro", Lede: "M5 chip delivers impressive gains.",
+		ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
+	})
+
+	// Search for Elden Ring
+	rows, err := Get().Query(`
+		SELECT s.guid, s.headline FROM stories s
+		JOIN stories_fts f ON s.id = f.rowid
+		WHERE stories_fts MATCH 'Elden Ring'
+		ORDER BY rank`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+
+	var results []string
+	for rows.Next() {
+		var guid, headline string
+		rows.Scan(&guid, &headline)
+		results = append(results, guid)
+	}
+	if len(results) != 1 || results[0] != "g1" {
+		t.Errorf("FTS5 search results = %v, want [g1]", results)
+	}
+}

internal/storage/types.go

@@ -0,0 +1,27 @@
+package storage
+
+// Story is the core record for an ingested news item.
+type Story struct {
+	ID         int64
+	GUID       string
+	Headline   string
+	Lede       string
+	ImageURL     string
+	ArticleURL   string
+	URLCanonical string
+	HeadlineNorm string
+	Source       string
+	FeedHint   string
+	Platforms  string // JSON array e.g. '["nintendo-switch","multi"]'
+	Channel    string
+	Classified bool
+	SeenAt     int64
+}
+
+// RecentHeadline is a lightweight record for the LLM dedup context window.
+type RecentHeadline struct {
+	GUID     string
+	Headline string
+	Source   string
+	SeenAt   int64
+}