Rip out Ollama: direct_route only, no LLM layer
Pete moves to a remote host without Ollama access. Every source must declare a direct_route channel; the classifier, explainer, semantic dedup, !explain summaries, feed_hint, and the recent_headlines / classification_log tables are gone. Deterministic dedup (canonical URL, headline_norm, per-channel cooldown) remains.
This commit is contained in:
prosolis
@@ -1,171 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync/atomic"
|
||||
|
||||
"pete/internal/ingestion"
|
||||
"pete/internal/matrix"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
const recentHeadlineLimit = 50
|
||||
|
||||
// ClassifyResult is the unified output of the classification pipeline.
|
||||
type ClassifyResult struct {
|
||||
Channel string
|
||||
Platforms []string
|
||||
DuplicateOf *string
|
||||
ShouldPost bool
|
||||
Rationale string
|
||||
}
|
||||
|
||||
// Classifier runs the classification pipeline for incoming stories.
|
||||
type Classifier struct {
|
||||
ollama *OllamaClient
|
||||
matrixClient *matrix.Client
|
||||
consecutiveFailures atomic.Int32
|
||||
}
|
||||
|
||||
// NewClassifier creates a new classifier.
|
||||
func NewClassifier(ollama *OllamaClient, matrixClient *matrix.Client) *Classifier {
|
||||
return &Classifier{
|
||||
ollama: ollama,
|
||||
matrixClient: matrixClient,
|
||||
}
|
||||
}
|
||||
|
||||
// Classify runs the appropriate classification pipeline for a feed item.
|
||||
// ctx is checked before any LLM call so shutdown aborts in-flight work.
|
||||
func (c *Classifier) Classify(ctx context.Context, item *ingestion.FeedItem) (*ClassifyResult, error) {
|
||||
// Direct routing — no LLM call needed
|
||||
if item.DirectRoute != nil {
|
||||
slog.Debug("direct route", "guid", item.GUID, "channel", *item.DirectRoute)
|
||||
|
||||
// Still check GUID dedup via recent headlines for semantic dedup
|
||||
// but direct-routed stories always post
|
||||
return &ClassifyResult{
|
||||
Channel: *item.DirectRoute,
|
||||
ShouldPost: true,
|
||||
Rationale: "direct route from source config",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get recent headlines for dedup context
|
||||
recent, err := storage.GetRecentHeadlines(recentHeadlineLimit)
|
||||
if err != nil {
|
||||
slog.Error("failed to get recent headlines", "err", err)
|
||||
recent = nil // continue without dedup context
|
||||
}
|
||||
|
||||
if item.Tier == 2 {
|
||||
return c.classifyTier2(ctx, item, recent)
|
||||
}
|
||||
|
||||
return c.classifyTier1(ctx, item, recent)
|
||||
}
|
||||
|
||||
func (c *Classifier) classifyTier1(ctx context.Context, item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
|
||||
prompt := BuildTier1Prompt(item, recent)
|
||||
|
||||
raw, err := c.ollama.Generate(ctx, prompt)
|
||||
if err != nil {
|
||||
c.handleOllamaFailure(err)
|
||||
return nil, fmt.Errorf("ollama generate: %w", err)
|
||||
}
|
||||
c.handleOllamaSuccess()
|
||||
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
|
||||
return nil, fmt.Errorf("parse tier1: %w", err)
|
||||
}
|
||||
|
||||
// Log classification
|
||||
logResult(item.GUID, raw)
|
||||
|
||||
return &ClassifyResult{
|
||||
Channel: result.Channel,
|
||||
Platforms: result.Platforms,
|
||||
DuplicateOf: result.DuplicateOf,
|
||||
ShouldPost: true, // Tier 1 always posts
|
||||
Rationale: result.Rationale,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Classifier) classifyTier2(ctx context.Context, item *ingestion.FeedItem, recent []storage.RecentHeadline) (*ClassifyResult, error) {
|
||||
// Run keyword gating first
|
||||
gatingResult, reason := RunGating(item.Headline, item.Lede)
|
||||
|
||||
switch gatingResult {
|
||||
case GatingDiscard:
|
||||
slog.Info("tier2 gating: discard", "guid", item.GUID, "reason", reason)
|
||||
logResult(item.GUID, fmt.Sprintf(`{"gating":"discard","reason":%q}`, reason))
|
||||
return &ClassifyResult{
|
||||
ShouldPost: false,
|
||||
Rationale: reason,
|
||||
}, nil
|
||||
|
||||
case GatingPost:
|
||||
slog.Info("tier2 gating: definite post, routing via LLM", "guid", item.GUID, "reason", reason)
|
||||
// Fall through to LLM for routing (but post=true is guaranteed)
|
||||
}
|
||||
|
||||
// Stage 3 (ambiguous) or Stage 1 (definite post needing routing) — send to LLM
|
||||
prompt := BuildTier2Prompt(item, recent)
|
||||
|
||||
raw, err := c.ollama.Generate(ctx, prompt)
|
||||
if err != nil {
|
||||
c.handleOllamaFailure(err)
|
||||
return nil, fmt.Errorf("ollama generate: %w", err)
|
||||
}
|
||||
c.handleOllamaSuccess()
|
||||
|
||||
result, err := ParseTier2Response(raw)
|
||||
if err != nil {
|
||||
slog.Error("malformed LLM response", "guid", item.GUID, "raw", raw, "err", err)
|
||||
return nil, fmt.Errorf("parse tier2: %w", err)
|
||||
}
|
||||
|
||||
logResult(item.GUID, raw)
|
||||
|
||||
// If gating said definite post, override LLM's post decision
|
||||
shouldPost := result.Post
|
||||
if gatingResult == GatingPost {
|
||||
shouldPost = true
|
||||
}
|
||||
|
||||
return &ClassifyResult{
|
||||
Channel: result.Channel,
|
||||
Platforms: result.Platforms,
|
||||
DuplicateOf: result.DuplicateOf,
|
||||
ShouldPost: shouldPost,
|
||||
Rationale: result.Rationale,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Classifier) handleOllamaFailure(err error) {
|
||||
n := c.consecutiveFailures.Add(1)
|
||||
slog.Error("ollama failure", "err", err, "consecutive", n)
|
||||
if n == 3 && c.matrixClient != nil {
|
||||
c.matrixClient.SendAdminWarning(
|
||||
fmt.Sprintf("Ollama unavailable for 3 consecutive attempts: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Classifier) handleOllamaSuccess() {
|
||||
c.consecutiveFailures.Store(0)
|
||||
}
|
||||
|
||||
func logResult(guid, raw string) {
|
||||
// Try to compact the JSON for cleaner logging
|
||||
var compact json.RawMessage
|
||||
if err := json.Unmarshal([]byte(raw), &compact); err == nil {
|
||||
storage.InsertClassificationLog(guid, string(compact))
|
||||
} else {
|
||||
storage.InsertClassificationLog(guid, raw)
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import "regexp"
|
||||
|
||||
// GatingResult indicates the outcome of keyword-based gating for Tier 2 stories.
|
||||
type GatingResult int
|
||||
|
||||
const (
|
||||
GatingPost GatingResult = iota // Stage 1: definite post
|
||||
GatingDiscard // Stage 2: definite discard
|
||||
GatingAmbiguous // Stage 3: escalate to LLM
|
||||
)
|
||||
|
||||
// Stage 1: Definite post patterns — stories with clear public significance.
|
||||
// Patterns use multi-word context to reduce false positives (e.g. "Star Wars" won't match).
|
||||
var postPatterns = []*regexp.Regexp{
|
||||
// Named heads of state / major international bodies
|
||||
regexp.MustCompile(`(?i)\b(president|prime minister|chancellor|pope)\s+\w+`),
|
||||
regexp.MustCompile(`(?i)\b(UN|United Nations|WHO|NATO|EU|European Union|IMF|World Bank)\b`),
|
||||
|
||||
// Casualty language with numbers
|
||||
regexp.MustCompile(`(?i)\b\d+\s*(dead|killed|injured|casualties|wounded)\b`),
|
||||
regexp.MustCompile(`(?i)\b(dead|killed|injured|casualties|wounded)\s*\d+\b`),
|
||||
|
||||
// Disaster language — require a location-like context word nearby
|
||||
regexp.MustCompile(`(?i)\b(earthquake|hurricane|typhoon|tsunami|wildfire|tornado)\b.*\b(hit|struck|destroyed|devastated|kills|deaths)\b`),
|
||||
|
||||
// Conflict language — require multi-word phrases to avoid "Star Wars", "culture war"
|
||||
regexp.MustCompile(`(?i)\b(military invasion|armed coup|airstrike\s+\w+|assassination\s+\w+|car bombing|suicide bombing)\b`),
|
||||
regexp.MustCompile(`(?i)\bwar\s+(in|on|with|against|between)\b`),
|
||||
|
||||
// Crisis language — require context word nearby
|
||||
regexp.MustCompile(`(?i)\b(state of emergency|economic crisis|economic collapse|disease outbreak|pandemic declared|pandemic phase|famine)\b`),
|
||||
|
||||
// Major financial scale
|
||||
regexp.MustCompile(`(?i)\$\d+\s*(bn|tn|billion|trillion)\b`),
|
||||
regexp.MustCompile(`(?i)\b(trillion|billion)\s*(dollar|euro|pound)\b`),
|
||||
}
|
||||
|
||||
// Stage 2: Definite discard patterns — stories unlikely to warrant posting.
|
||||
var discardPatterns = []*regexp.Regexp{
|
||||
// Local/municipal scope
|
||||
regexp.MustCompile(`(?i)\b(city council|municipal|town hall|zoning|local police)\b`),
|
||||
|
||||
// Sports scores and routine results
|
||||
regexp.MustCompile(`(?i)\b(final score|match result|league standings|defeated|innings)\b`),
|
||||
regexp.MustCompile(`(?i)\b\d+-\d+\s*(win|loss|defeat|victory)\b`),
|
||||
|
||||
// Entertainment/celebrity lifestyle
|
||||
regexp.MustCompile(`(?i)\b(red carpet|award show|celebrity gossip|dating rumor|relationship drama|album release)\b`),
|
||||
|
||||
// Routine corporate earnings
|
||||
regexp.MustCompile(`(?i)\b(quarterly earnings|Q[1-4] results|earnings call|fiscal quarter)\b`),
|
||||
}
|
||||
|
||||
// RunGating evaluates Tier 2 gating stages on headline + lede.
|
||||
// Returns the gating result and a reason string for logging.
|
||||
func RunGating(headline, lede string) (GatingResult, string) {
|
||||
combined := headline + " " + lede
|
||||
|
||||
// Stage 1: Definite post
|
||||
for _, re := range postPatterns {
|
||||
if re.MatchString(combined) {
|
||||
return GatingPost, "stage1: " + re.String()
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 2: Definite discard
|
||||
for _, re := range discardPatterns {
|
||||
if re.MatchString(combined) {
|
||||
return GatingDiscard, "stage2: " + re.String()
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: Ambiguous — escalate to LLM
|
||||
return GatingAmbiguous, "stage3: no keyword match"
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGating_Stage1_DefinitePost(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headline string
|
||||
lede string
|
||||
}{
|
||||
{"head of state", "President signs new climate bill", "The president signed legislation today."},
|
||||
{"international body", "WHO declares new pandemic phase", "The World Health Organization announced a new phase."},
|
||||
{"casualties", "12 killed in factory explosion", "Authorities report 12 dead after an explosion."},
|
||||
{"disaster", "Earthquake strikes northern Japan", "A major earthquake hit the region early Tuesday."},
|
||||
{"conflict", "Airstrike hits hospital in conflict zone", "An airstrike damaged a hospital."},
|
||||
{"crisis", "Economic crisis deepens in Argentina", "The ongoing crisis has intensified."},
|
||||
{"financial scale", "EU approves $50bn climate fund", "The European Union approved funding."},
|
||||
{"assassination", "Assassination attempt on opposition leader", "An armed assailant attacked the leader."},
|
||||
{"NATO", "NATO expands eastern flank deployment", "Alliance members agreed to new deployments."},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, reason := RunGating(tt.headline, tt.lede)
|
||||
if result != GatingPost {
|
||||
t.Errorf("expected GatingPost, got %d (reason: %s)", result, reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGating_Stage2_DefiniteDiscard(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headline string
|
||||
lede string
|
||||
}{
|
||||
{"local government", "City council approves new parking meters", "The council voted 5-3 in favor."},
|
||||
{"sports scores", "Lakers defeat Celtics 112-98 in regular season", "Final score 112-98."},
|
||||
{"celebrity gossip", "Celebrity couple spotted on red carpet at Cannes", "The pair arrived together."},
|
||||
{"routine earnings", "Regional bank reports Q2 results slightly above estimates", "Quarterly earnings were released."},
|
||||
{"entertainment", "Pop star announces new album release date", "Fans have been waiting since 2023."},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, reason := RunGating(tt.headline, tt.lede)
|
||||
if result != GatingDiscard {
|
||||
t.Errorf("expected GatingDiscard, got %d (reason: %s)", result, reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGating_Stage3_Ambiguous(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
headline string
|
||||
lede string
|
||||
}{
|
||||
{"tech product", "Apple unveils new MacBook Pro with M5 chip", "The new laptops feature significant performance gains."},
|
||||
{"policy nuanced", "Government considers new data privacy framework", "Ministers met to discuss potential changes."},
|
||||
{"science", "Researchers discover high-temperature superconductor", "A team at MIT published findings in Nature."},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, reason := RunGating(tt.headline, tt.lede)
|
||||
if result != GatingAmbiguous {
|
||||
t.Errorf("expected GatingAmbiguous, got %d (reason: %s)", result, reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
)
|
||||
|
||||
// OllamaClient handles communication with the Ollama API.
|
||||
type OllamaClient struct {
|
||||
baseURL string
|
||||
model string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
type chatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []chatMessage `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
Format string `json:"format"`
|
||||
}
|
||||
|
||||
type chatMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type chatResponse struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
}
|
||||
|
||||
// NewOllamaClient creates an Ollama API client from config.
|
||||
func NewOllamaClient(cfg config.OllamaConfig) *OllamaClient {
|
||||
return &OllamaClient{
|
||||
baseURL: strings.TrimRight(cfg.BaseURL, "/"),
|
||||
model: cfg.Model,
|
||||
httpClient: &http.Client{
|
||||
Timeout: time.Duration(cfg.TimeoutSeconds) * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Generate sends a prompt to Ollama via the chat API and returns the raw
|
||||
// response text. Honors ctx cancellation — callers can abort an in-flight
|
||||
// call instead of waiting for the configured timeout.
|
||||
func (o *OllamaClient) Generate(ctx context.Context, prompt string) (string, error) {
|
||||
return o.call(ctx,
|
||||
"You are a JSON-only news classification API. Respond with valid JSON only. No preamble, no explanation, no markdown.",
|
||||
prompt,
|
||||
"json",
|
||||
)
|
||||
}
|
||||
|
||||
// GenerateText sends a system + user prompt to Ollama and returns the raw
|
||||
// text response. Unlike Generate, no JSON mode is forced.
|
||||
func (o *OllamaClient) GenerateText(ctx context.Context, systemPrompt, userPrompt string) (string, error) {
|
||||
return o.call(ctx, systemPrompt, userPrompt, "")
|
||||
}
|
||||
|
||||
func (o *OllamaClient) call(ctx context.Context, systemPrompt, userPrompt, format string) (string, error) {
|
||||
reqBody := chatRequest{
|
||||
Model: o.model,
|
||||
Messages: []chatMessage{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: userPrompt},
|
||||
},
|
||||
Stream: false,
|
||||
Format: format,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := o.baseURL + "/api/chat"
|
||||
slog.Debug("ollama: calling", "url", url, "model", o.model)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ollama request build: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := o.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ollama request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1*1024*1024)) // 1MB cap
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read ollama response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("ollama status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var chatResp chatResponse
|
||||
if err := json.Unmarshal(respBody, &chatResp); err != nil {
|
||||
return "", fmt.Errorf("decode ollama response: %w (body: %s)", err, truncate(string(respBody), 200))
|
||||
}
|
||||
|
||||
result := strings.TrimSpace(chatResp.Message.Content)
|
||||
if result == "" {
|
||||
return "", fmt.Errorf("ollama returned empty response (body: %s)", truncate(string(respBody), 200))
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Tier1Result is the LLM response for Tier 1 (routing only) stories.
|
||||
type Tier1Result struct {
|
||||
Channel string `json:"channel"`
|
||||
Platforms []string `json:"platforms"`
|
||||
DuplicateOf *string `json:"duplicate_of"`
|
||||
Rationale string `json:"rationale"`
|
||||
}
|
||||
|
||||
// Tier2Result is the LLM response for Tier 2 (gating + routing) stories.
|
||||
type Tier2Result struct {
|
||||
Post bool `json:"post"`
|
||||
Channel string `json:"channel"`
|
||||
Platforms []string `json:"platforms"`
|
||||
DuplicateOf *string `json:"duplicate_of"`
|
||||
Rationale string `json:"rationale"`
|
||||
}
|
||||
|
||||
var validChannels = map[string]bool{
|
||||
"tech": true,
|
||||
"politics": true,
|
||||
"gaming": true,
|
||||
}
|
||||
|
||||
var validPlatforms = map[string]bool{
|
||||
"nintendo-switch": true,
|
||||
"playstation": true,
|
||||
"xbox": true,
|
||||
"pc": true,
|
||||
"retro": true,
|
||||
"mobile": true,
|
||||
"multi": true,
|
||||
}
|
||||
|
||||
var thinkRe = regexp.MustCompile(`(?s)<think>.*?</think>`)
|
||||
var fenceRe = regexp.MustCompile("(?s)```(?:json)?\\s*(.+?)\\s*```")
|
||||
var trailingCommaRe = regexp.MustCompile(`,\s*([}\]])`)
|
||||
|
||||
// ParseTier1Response parses an LLM response into a Tier1Result.
|
||||
func ParseTier1Response(raw string) (*Tier1Result, error) {
|
||||
cleaned := repairJSON(raw)
|
||||
|
||||
var result Tier1Result
|
||||
if err := json.Unmarshal([]byte(cleaned), &result); err != nil {
|
||||
return nil, fmt.Errorf("parse tier1 response: %w (raw: %s)", err, truncate(raw, 200))
|
||||
}
|
||||
|
||||
if !validChannels[result.Channel] {
|
||||
return nil, fmt.Errorf("invalid channel: %q", result.Channel)
|
||||
}
|
||||
|
||||
result.Platforms = filterPlatforms(result.Platforms)
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// ParseTier2Response parses an LLM response into a Tier2Result.
|
||||
func ParseTier2Response(raw string) (*Tier2Result, error) {
|
||||
cleaned := repairJSON(raw)
|
||||
|
||||
var result Tier2Result
|
||||
if err := json.Unmarshal([]byte(cleaned), &result); err != nil {
|
||||
return nil, fmt.Errorf("parse tier2 response: %w (raw: %s)", err, truncate(raw, 200))
|
||||
}
|
||||
|
||||
// Always validate channel — gating may override post=false to true later
|
||||
if !validChannels[result.Channel] {
|
||||
if result.Post {
|
||||
return nil, fmt.Errorf("invalid channel: %q", result.Channel)
|
||||
}
|
||||
// Default to "politics" for discarded stories with garbage channels
|
||||
result.Channel = "politics"
|
||||
}
|
||||
|
||||
result.Platforms = filterPlatforms(result.Platforms)
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// repairJSON attempts to fix common LLM output issues.
|
||||
func repairJSON(raw string) string {
|
||||
s := strings.TrimSpace(raw)
|
||||
|
||||
// Strip <think>...</think> blocks (some models include reasoning)
|
||||
s = thinkRe.ReplaceAllString(s, "")
|
||||
|
||||
// Strip markdown code fences
|
||||
if matches := fenceRe.FindStringSubmatch(s); len(matches) > 1 {
|
||||
s = matches[1]
|
||||
}
|
||||
|
||||
s = strings.TrimSpace(s)
|
||||
|
||||
// Remove trailing commas before closing braces/brackets (outside quoted strings)
|
||||
s = trailingCommaRe.ReplaceAllString(s, "$1")
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func filterPlatforms(platforms []string) []string {
|
||||
var valid []string
|
||||
for _, p := range platforms {
|
||||
p = strings.TrimSpace(strings.ToLower(p))
|
||||
if validPlatforms[p] {
|
||||
valid = append(valid, p)
|
||||
}
|
||||
}
|
||||
return valid
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return string(runes[:n]) + "..."
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseTier1Response_Clean(t *testing.T) {
|
||||
raw := `{"channel": "tech", "platforms": [], "duplicate_of": null, "rationale": "GPU launch story"}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "tech" {
|
||||
t.Errorf("channel = %q, want tech", result.Channel)
|
||||
}
|
||||
if len(result.Platforms) != 0 {
|
||||
t.Errorf("platforms = %v, want empty", result.Platforms)
|
||||
}
|
||||
if result.DuplicateOf != nil {
|
||||
t.Errorf("duplicate_of = %v, want nil", result.DuplicateOf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier1Response_WithPlatforms(t *testing.T) {
|
||||
raw := `{"channel":"gaming","platforms":["nintendo-switch","pc"],"duplicate_of":null,"rationale":"Switch 2 launch"}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "gaming" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
if len(result.Platforms) != 2 {
|
||||
t.Fatalf("platforms = %v, want 2", result.Platforms)
|
||||
}
|
||||
if result.Platforms[0] != "nintendo-switch" || result.Platforms[1] != "pc" {
|
||||
t.Errorf("platforms = %v", result.Platforms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier1Response_WithDuplicate(t *testing.T) {
|
||||
raw := `{"channel":"politics","platforms":[],"duplicate_of":"guid-abc-123","rationale":"same event"}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.DuplicateOf == nil || *result.DuplicateOf != "guid-abc-123" {
|
||||
t.Errorf("duplicate_of = %v", result.DuplicateOf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier1Response_InvalidChannel(t *testing.T) {
|
||||
raw := `{"channel":"sports","platforms":[],"duplicate_of":null,"rationale":"test"}`
|
||||
_, err := ParseTier1Response(raw)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid channel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier1Response_FiltersInvalidPlatforms(t *testing.T) {
|
||||
raw := `{"channel":"gaming","platforms":["nintendo-switch","fake-platform","pc","also-fake"],"duplicate_of":null,"rationale":"test"}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Platforms) != 2 {
|
||||
t.Fatalf("platforms = %v, want [nintendo-switch pc]", result.Platforms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier2Response_PostTrue(t *testing.T) {
|
||||
raw := `{"post":true,"channel":"politics","platforms":[],"duplicate_of":null,"rationale":"major world event"}`
|
||||
result, err := ParseTier2Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Post {
|
||||
t.Error("post = false, want true")
|
||||
}
|
||||
if result.Channel != "politics" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTier2Response_PostFalse(t *testing.T) {
|
||||
raw := `{"post":false,"platforms":[],"duplicate_of":null,"rationale":"local news"}`
|
||||
result, err := ParseTier2Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Post {
|
||||
t.Error("post = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_MarkdownFences(t *testing.T) {
|
||||
raw := "```json\n{\"channel\": \"tech\", \"platforms\": [], \"duplicate_of\": null, \"rationale\": \"test\"}\n```"
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "tech" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_ThinkBlocks(t *testing.T) {
|
||||
raw := "<think>I need to classify this as tech because it's about GPUs</think>\n{\"channel\": \"tech\", \"platforms\": [], \"duplicate_of\": null, \"rationale\": \"GPU story\"}"
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "tech" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_TrailingCommas(t *testing.T) {
|
||||
raw := `{"channel": "politics", "platforms": [], "duplicate_of": null, "rationale": "test",}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "politics" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_TrailingCommaInArray(t *testing.T) {
|
||||
raw := `{"channel": "gaming", "platforms": ["pc", "xbox",], "duplicate_of": null, "rationale": "test"}`
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Platforms) != 2 {
|
||||
t.Errorf("platforms = %v", result.Platforms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_CombinedIssues(t *testing.T) {
|
||||
raw := "<think>reasoning here</think>\n```json\n{\"channel\": \"tech\", \"platforms\": [],\"duplicate_of\": null, \"rationale\": \"test\",}\n```"
|
||||
result, err := ParseTier1Response(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Channel != "tech" {
|
||||
t.Errorf("channel = %q", result.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairJSON_GarbageInput(t *testing.T) {
|
||||
_, err := ParseTier1Response("I think this should go to tech because reasons")
|
||||
if err == nil {
|
||||
t.Error("expected error for garbage input")
|
||||
}
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package classifier
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"pete/internal/ingestion"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// BuildTier1Prompt builds the routing-only prompt for Tier 1 stories.
|
||||
func BuildTier1Prompt(item *ingestion.FeedItem, recent []storage.RecentHeadline) string {
|
||||
return fmt.Sprintf(`You are a news router for a community discussion platform with three channels: tech, politics, and gaming.
|
||||
|
||||
This story comes from a curated Tier 1 source. It will be posted. Your only job is to decide which channel it belongs in.
|
||||
|
||||
Choose exactly one channel:
|
||||
- tech: technology products, software, hardware, industry — where technology itself is the primary subject
|
||||
- gaming: games, gaming platforms, gaming industry, esports
|
||||
- politics: everything else — political, policy, current events, public interest, significant world events
|
||||
|
||||
Politics is the default. When in doubt, choose politics.
|
||||
|
||||
Routing guidance:
|
||||
- GPU launch, software release, hardware review → tech
|
||||
- Game release, platform announcement, studio news → gaming
|
||||
- Election, legislation, court ruling → politics
|
||||
- AI regulation, antitrust action, surveillance policy → politics (policy is the story, not the technology)
|
||||
- Death of a public figure → politics
|
||||
- Violent crime, assassination attempt, arrest of public figure → politics
|
||||
- Natural disaster, humanitarian crisis → politics
|
||||
- Celebrity gossip, lifestyle with no broader significance → politics (still gets posted, still needs a home)
|
||||
|
||||
For gaming stories only: identify which platforms are relevant from this list:
|
||||
nintendo-switch, playstation, xbox, pc, retro, mobile, multi
|
||||
Return an empty array if no specific platform applies.
|
||||
|
||||
Also check: does this story cover the same underlying event as any story in the recent window? Different outlets covering the same event counts as a duplicate. If yes, set duplicate_of to that story's GUID. If no match, set duplicate_of to null.
|
||||
|
||||
Respond ONLY with valid JSON. No preamble, no explanation outside the JSON:
|
||||
{
|
||||
"channel": "tech|politics|gaming",
|
||||
"platforms": [string],
|
||||
"duplicate_of": string | null,
|
||||
"rationale": "one sentence max, for logging only"
|
||||
}
|
||||
|
||||
INCOMING STORY:
|
||||
Source: %s
|
||||
Tier: %d
|
||||
Feed hint: %s
|
||||
Headline: %s
|
||||
Lede: %s
|
||||
|
||||
RECENT STORIES (last 24hr):
|
||||
%s`, item.Source, item.Tier, item.FeedHint, item.Headline, item.Lede, formatRecentStories(recent))
|
||||
}
|
||||
|
||||
// BuildTier2Prompt builds the gating + routing prompt for Tier 2 stories.
|
||||
func BuildTier2Prompt(item *ingestion.FeedItem, recent []storage.RecentHeadline) string {
|
||||
return fmt.Sprintf(`You are a news classifier for a community discussion platform with three channels: tech, politics, and gaming.
|
||||
|
||||
This story comes from a wire service. Unlike curated sources, wire feeds publish everything — not all stories warrant posting. First decide if this story is worth posting, then decide where.
|
||||
|
||||
Step 1 — Should this be posted?
|
||||
Post if the story has clear public significance: major world events, policy decisions, significant deaths, natural disasters, major technology developments. Do not post: local news with no broader relevance, routine financial filings, minor sports results, entertainment fluff.
|
||||
|
||||
Step 2 — If posting, choose exactly one channel:
|
||||
- tech: technology products, software, hardware, industry
|
||||
- gaming: games, gaming platforms, gaming industry
|
||||
- politics: everything else — the default
|
||||
|
||||
Politics is the default when in doubt.
|
||||
|
||||
For gaming stories only: identify relevant platforms from: nintendo-switch, playstation, xbox, pc, retro, mobile, multi
|
||||
|
||||
Also check for duplicates against the recent window. If this story covers the same underlying event as a recent story, set duplicate_of to that story's GUID.
|
||||
|
||||
Respond ONLY with valid JSON:
|
||||
{
|
||||
"post": true|false,
|
||||
"channel": "tech|politics|gaming",
|
||||
"platforms": [string],
|
||||
"duplicate_of": string | null,
|
||||
"rationale": "one sentence max, for logging only"
|
||||
}
|
||||
|
||||
If post is false, channel may be omitted.
|
||||
|
||||
INCOMING STORY:
|
||||
Source: %s
|
||||
Tier: %d
|
||||
Feed hint: %s
|
||||
Headline: %s
|
||||
Lede: %s
|
||||
|
||||
RECENT STORIES (last 24hr):
|
||||
%s`, item.Source, item.Tier, item.FeedHint, item.Headline, item.Lede, formatRecentStories(recent))
|
||||
}
|
||||
|
||||
func formatRecentStories(recent []storage.RecentHeadline) string {
|
||||
if len(recent) == 0 {
|
||||
return "(none)"
|
||||
}
|
||||
var sb strings.Builder
|
||||
for _, h := range recent {
|
||||
fmt.Fprintf(&sb, "[%s] %s — %s\n", h.GUID, h.Headline, h.Source)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
@@ -14,7 +14,6 @@ var envBracketRe = regexp.MustCompile(`\$\{([^}]+)\}`)
|
||||
|
||||
type Config struct {
|
||||
Matrix MatrixConfig `yaml:"matrix"`
|
||||
Ollama OllamaConfig `yaml:"ollama"`
|
||||
Posting PostingConfig `yaml:"posting"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
Web WebConfig `yaml:"web"`
|
||||
@@ -40,12 +39,6 @@ type MatrixConfig struct {
|
||||
Channels map[string]string `yaml:"channels"`
|
||||
}
|
||||
|
||||
type OllamaConfig struct {
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Model string `yaml:"model"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
}
|
||||
|
||||
type PostingConfig struct {
|
||||
MinIntervalSeconds int `yaml:"min_interval_seconds"`
|
||||
BurstCapCount int `yaml:"burst_cap_count"`
|
||||
@@ -58,27 +51,25 @@ type PostingConfig struct {
|
||||
}
|
||||
|
||||
// RoundRobinConfig switches Pete from immediate-on-classify posting to a
|
||||
// paced rotation: one story per IntervalHours, picking the next source in
|
||||
// config order that has a postable story (skip-and-advance, newest-first).
|
||||
// paced rotation: one story per IntervalHours, picking the next channel in
|
||||
// rotation order that has a postable story (skip-and-advance, newest-first).
|
||||
type RoundRobinConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
IntervalHours int `yaml:"interval_hours"`
|
||||
}
|
||||
|
||||
type StorageConfig struct {
|
||||
DBPath string `yaml:"db_path"`
|
||||
RecentWindowHours int `yaml:"recent_window_hours"`
|
||||
ClassificationLogDays int `yaml:"classification_log_days"`
|
||||
DBPath string `yaml:"db_path"`
|
||||
RecentWindowHours int `yaml:"recent_window_hours"`
|
||||
}
|
||||
|
||||
type SourceConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
FeedURL string `yaml:"feed_url"`
|
||||
Tier int `yaml:"tier"`
|
||||
PollIntervalMinutes int `yaml:"poll_interval_minutes"`
|
||||
FeedHint string `yaml:"feed_hint"`
|
||||
DirectRoute *string `yaml:"direct_route"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Name string `yaml:"name"`
|
||||
FeedURL string `yaml:"feed_url"`
|
||||
Tier int `yaml:"tier"`
|
||||
PollIntervalMinutes int `yaml:"poll_interval_minutes"`
|
||||
DirectRoute string `yaml:"direct_route"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
@@ -125,12 +116,6 @@ func (c *Config) validate() error {
|
||||
if len(c.Matrix.Channels) == 0 {
|
||||
return fmt.Errorf("matrix.channels must have at least one entry")
|
||||
}
|
||||
if c.Ollama.BaseURL == "" {
|
||||
return fmt.Errorf("ollama.base_url is required")
|
||||
}
|
||||
if c.Ollama.Model == "" {
|
||||
return fmt.Errorf("ollama.model is required")
|
||||
}
|
||||
if c.Storage.DBPath == "" {
|
||||
return fmt.Errorf("storage.db_path is required")
|
||||
}
|
||||
@@ -148,6 +133,15 @@ func (c *Config) validate() error {
|
||||
if s.PollIntervalMinutes <= 0 {
|
||||
return fmt.Errorf("sources[%d].poll_interval_minutes must be > 0", i)
|
||||
}
|
||||
if !s.Enabled {
|
||||
continue
|
||||
}
|
||||
if s.DirectRoute == "" {
|
||||
return fmt.Errorf("sources[%d] (%s): direct_route is required for enabled sources", i, s.Name)
|
||||
}
|
||||
if _, ok := c.Matrix.Channels[s.DirectRoute]; !ok {
|
||||
return fmt.Errorf("sources[%d] (%s): direct_route %q is not a configured matrix.channels key", i, s.Name, s.DirectRoute)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -163,9 +157,6 @@ func (c *Config) applyDefaults() {
|
||||
if c.Matrix.PickleKey == "" {
|
||||
c.Matrix.PickleKey = "pete_pickle_key"
|
||||
}
|
||||
if c.Ollama.TimeoutSeconds == 0 {
|
||||
c.Ollama.TimeoutSeconds = 30
|
||||
}
|
||||
if c.Posting.MinIntervalSeconds == 0 {
|
||||
c.Posting.MinIntervalSeconds = 300
|
||||
}
|
||||
@@ -184,9 +175,6 @@ func (c *Config) applyDefaults() {
|
||||
if c.Storage.RecentWindowHours == 0 {
|
||||
c.Storage.RecentWindowHours = 24
|
||||
}
|
||||
if c.Storage.ClassificationLogDays == 0 {
|
||||
c.Storage.ClassificationLogDays = 7
|
||||
}
|
||||
if c.Web.ListenAddr == "" {
|
||||
c.Web.ListenAddr = ":8080"
|
||||
}
|
||||
|
||||
@@ -19,16 +19,14 @@ matrix:
|
||||
|
||||
func TestLoadValid(t *testing.T) {
|
||||
yaml := validMatrix + `
|
||||
ollama:
|
||||
base_url: "http://localhost:11434"
|
||||
model: "gemma4:27b"
|
||||
timeout_seconds: 60
|
||||
storage:
|
||||
db_path: "/tmp/test.db"
|
||||
sources:
|
||||
- name: "Test Source"
|
||||
feed_url: "https://example.com/rss"
|
||||
tier: 1
|
||||
poll_interval_minutes: 20
|
||||
direct_route: "tech"
|
||||
enabled: true
|
||||
`
|
||||
cfg := loadFromString(t, yaml)
|
||||
@@ -39,15 +37,15 @@ sources:
|
||||
if cfg.Matrix.UserID != "@pete:example.org" {
|
||||
t.Errorf("user_id = %q", cfg.Matrix.UserID)
|
||||
}
|
||||
if cfg.Ollama.TimeoutSeconds != 60 {
|
||||
t.Errorf("timeout = %d, want 60", cfg.Ollama.TimeoutSeconds)
|
||||
}
|
||||
if len(cfg.Sources) != 1 {
|
||||
t.Fatalf("sources = %d, want 1", len(cfg.Sources))
|
||||
}
|
||||
if cfg.Sources[0].Name != "Test Source" {
|
||||
t.Errorf("source name = %q", cfg.Sources[0].Name)
|
||||
}
|
||||
if cfg.Sources[0].DirectRoute != "tech" {
|
||||
t.Errorf("direct_route = %q, want tech", cfg.Sources[0].DirectRoute)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvVarExpansion(t *testing.T) {
|
||||
@@ -60,9 +58,6 @@ matrix:
|
||||
password: "${TEST_PETE_PASS}"
|
||||
channels:
|
||||
tech: "!tech:example.org"
|
||||
ollama:
|
||||
base_url: "http://localhost:11434"
|
||||
model: "gemma4:27b"
|
||||
storage:
|
||||
db_path: "/tmp/test.db"
|
||||
sources: []
|
||||
@@ -76,15 +71,14 @@ sources: []
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
yaml := validMatrix + `
|
||||
ollama:
|
||||
base_url: "http://localhost:11434"
|
||||
model: "gemma4:27b"
|
||||
storage:
|
||||
db_path: "/tmp/test.db"
|
||||
sources:
|
||||
- name: "S"
|
||||
feed_url: "https://example.com/rss"
|
||||
tier: 1
|
||||
poll_interval_minutes: 20
|
||||
direct_route: "tech"
|
||||
enabled: true
|
||||
`
|
||||
cfg := loadFromString(t, yaml)
|
||||
@@ -95,9 +89,6 @@ sources:
|
||||
if cfg.Matrix.DisplayName != "Pete" {
|
||||
t.Errorf("display_name default = %q, want Pete", cfg.Matrix.DisplayName)
|
||||
}
|
||||
if cfg.Ollama.TimeoutSeconds != 30 {
|
||||
t.Errorf("timeout default = %d, want 30", cfg.Ollama.TimeoutSeconds)
|
||||
}
|
||||
if cfg.Posting.MinIntervalSeconds != 300 {
|
||||
t.Errorf("min_interval default = %d, want 300", cfg.Posting.MinIntervalSeconds)
|
||||
}
|
||||
@@ -110,27 +101,6 @@ sources:
|
||||
if cfg.Storage.RecentWindowHours != 24 {
|
||||
t.Errorf("recent_window default = %d, want 24", cfg.Storage.RecentWindowHours)
|
||||
}
|
||||
if cfg.Storage.ClassificationLogDays != 7 {
|
||||
t.Errorf("classification_log default = %d, want 7", cfg.Storage.ClassificationLogDays)
|
||||
}
|
||||
if cfg.Sources[0].PollIntervalMinutes != 20 {
|
||||
t.Errorf("poll_interval default = %d, want 20", cfg.Sources[0].PollIntervalMinutes)
|
||||
}
|
||||
}
|
||||
|
||||
// shortMatrix returns a minimal valid matrix block for validation error tests.
|
||||
func shortMatrix(overrides string) string {
|
||||
base := `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
`
|
||||
if overrides != "" {
|
||||
return overrides
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func TestValidationErrors(t *testing.T) {
|
||||
@@ -143,7 +113,6 @@ matrix:
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"missing user_id", `
|
||||
@@ -151,7 +120,6 @@ matrix:
|
||||
homeserver: "https://h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"missing password", `
|
||||
@@ -159,7 +127,6 @@ matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"no channels", `
|
||||
@@ -167,25 +134,6 @@ matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"missing ollama base_url", `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"missing ollama model", `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
`},
|
||||
{"missing db_path", `
|
||||
@@ -194,7 +142,6 @@ matrix:
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {}
|
||||
`},
|
||||
{"invalid tier", `
|
||||
@@ -203,12 +150,14 @@ matrix:
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
sources:
|
||||
- name: "S"
|
||||
feed_url: "https://e.com/rss"
|
||||
tier: 5
|
||||
poll_interval_minutes: 20
|
||||
direct_route: "tech"
|
||||
enabled: true
|
||||
`},
|
||||
{"missing source name", `
|
||||
matrix:
|
||||
@@ -216,11 +165,42 @@ matrix:
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
sources:
|
||||
- feed_url: "https://e.com/rss"
|
||||
tier: 1
|
||||
poll_interval_minutes: 20
|
||||
direct_route: "tech"
|
||||
enabled: true
|
||||
`},
|
||||
{"enabled source missing direct_route", `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
sources:
|
||||
- name: "S"
|
||||
feed_url: "https://e.com/rss"
|
||||
tier: 1
|
||||
poll_interval_minutes: 20
|
||||
enabled: true
|
||||
`},
|
||||
{"direct_route not in channels", `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
sources:
|
||||
- name: "S"
|
||||
feed_url: "https://e.com/rss"
|
||||
tier: 1
|
||||
poll_interval_minutes: 20
|
||||
direct_route: "gaming"
|
||||
enabled: true
|
||||
`},
|
||||
}
|
||||
|
||||
@@ -237,37 +217,25 @@ sources:
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectRouteNullable(t *testing.T) {
|
||||
func TestDisabledSourceSkipsDirectRouteCheck(t *testing.T) {
|
||||
// A disabled source without direct_route must not fail validation.
|
||||
yaml := `
|
||||
matrix:
|
||||
homeserver: "https://h"
|
||||
user_id: "@p:h"
|
||||
password: "pw"
|
||||
channels: {tech: "!t:e"}
|
||||
ollama: {base_url: "http://l", model: "m"}
|
||||
storage: {db_path: "/tmp/t.db"}
|
||||
sources:
|
||||
- name: "Direct"
|
||||
- name: "Off"
|
||||
feed_url: "https://e.com/rss"
|
||||
tier: 1
|
||||
direct_route: "politics"
|
||||
enabled: true
|
||||
- name: "LLM"
|
||||
feed_url: "https://e.com/rss2"
|
||||
tier: 1
|
||||
direct_route: null
|
||||
enabled: true
|
||||
poll_interval_minutes: 20
|
||||
enabled: false
|
||||
`
|
||||
cfg := loadFromString(t, yaml)
|
||||
|
||||
if cfg.Sources[0].DirectRoute == nil {
|
||||
t.Fatal("direct source should have non-nil DirectRoute")
|
||||
}
|
||||
if *cfg.Sources[0].DirectRoute != "politics" {
|
||||
t.Errorf("direct_route = %q, want politics", *cfg.Sources[0].DirectRoute)
|
||||
}
|
||||
if cfg.Sources[1].DirectRoute != nil {
|
||||
t.Errorf("null source should have nil DirectRoute, got %v", cfg.Sources[1].DirectRoute)
|
||||
if cfg.Sources[0].DirectRoute != "" {
|
||||
t.Errorf("expected empty direct_route, got %q", cfg.Sources[0].DirectRoute)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
// Package explainer summarizes a posted story on demand: when a user reacts ❓
|
||||
// on one of Pete's posts, Pete fetches the article body, asks Ollama for a
|
||||
// 3-bullet TL;DR, and replies in a thread rooted at the original post.
|
||||
package explainer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pete/internal/classifier"
|
||||
"pete/internal/ingestion"
|
||||
"pete/internal/matrix"
|
||||
"pete/internal/storage"
|
||||
|
||||
"maunium.net/go/mautrix/id"
|
||||
)
|
||||
|
||||
// questionReactions is the set of reaction keys that trigger a summary.
|
||||
// Matrix delivers the key verbatim; we accept several question-y glyphs plus
|
||||
// plain "?" so users don't have to hunt for the exact red question mark.
|
||||
var questionReactions = map[string]bool{
|
||||
"❓": true, // U+2753 red question mark
|
||||
"❓️": true, // U+2753 + VS16 (emoji presentation)
|
||||
"❔": true, // U+2754 white question mark
|
||||
"❔️": true, // U+2754 + VS16
|
||||
"⁉": true, // U+2049 exclamation question mark
|
||||
"⁉️": true, // U+2049 + VS16 (emoji presentation)
|
||||
"🤔": true, // U+1F914 thinking face
|
||||
"?": true, // plain ascii
|
||||
"？": true, // U+FF1F fullwidth question mark
|
||||
}
|
||||
|
||||
// IsQuestionReaction reports whether a reaction key should trigger a summary.
|
||||
func IsQuestionReaction(key string) bool {
|
||||
return questionReactions[key]
|
||||
}
|
||||
|
||||
// cooldown for repeat-explain on the same story (per process).
|
||||
const explainCooldown = 5 * time.Minute
|
||||
|
||||
// Explainer holds the dependencies needed to produce a story summary.
|
||||
type Explainer struct {
|
||||
mx *matrix.Client
|
||||
ollama *classifier.OllamaClient
|
||||
|
||||
mu sync.Mutex
|
||||
lastSeen map[string]time.Time // guid -> last explanation time
|
||||
}
|
||||
|
||||
// New builds an Explainer wired to the Matrix client and Ollama backend.
|
||||
func New(mx *matrix.Client, ollama *classifier.OllamaClient) *Explainer {
|
||||
return &Explainer{
|
||||
mx: mx,
|
||||
ollama: ollama,
|
||||
lastSeen: make(map[string]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// Handle is the entry point used as a callback from poster.HandleReaction.
|
||||
// It runs synchronously — callers should invoke it in a goroutine.
|
||||
func (e *Explainer) Handle(roomID id.RoomID, rootEventID id.EventID, guid, channel, emoji string) {
|
||||
if !IsQuestionReaction(emoji) {
|
||||
return
|
||||
}
|
||||
if !e.acquire(guid) {
|
||||
slog.Debug("explain: skipping recent duplicate", "guid", guid)
|
||||
return
|
||||
}
|
||||
|
||||
story, err := storage.GetStoryByGUID(guid)
|
||||
if err != nil || story == nil {
|
||||
slog.Warn("explain: story lookup failed", "guid", guid, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
body := ingestion.FetchArticleBody(story.ArticleURL)
|
||||
if body == "" {
|
||||
// Couldn't get the body — fall back to lede so the user at least gets something.
|
||||
body = story.Lede
|
||||
}
|
||||
if body == "" {
|
||||
slog.Warn("explain: no body or lede available", "guid", guid, "url", story.ArticleURL)
|
||||
_ = e.mx.PostThreadedReply(channel, rootEventID,
|
||||
"Couldn't fetch the article body to summarize.",
|
||||
"<em>Couldn't fetch the article body to summarize.</em>")
|
||||
return
|
||||
}
|
||||
|
||||
summary, err := e.summarize(story.Headline, body)
|
||||
if err != nil {
|
||||
slog.Error("explain: ollama failed", "guid", guid, "err", err)
|
||||
_ = e.mx.PostThreadedReply(channel, rootEventID,
|
||||
"Sorry, I couldn't generate a summary right now.",
|
||||
"<em>Sorry, I couldn't generate a summary right now.</em>")
|
||||
return
|
||||
}
|
||||
|
||||
plain, htmlBody := formatSummary(summary)
|
||||
if err := e.mx.PostThreadedReply(channel, rootEventID, plain, htmlBody); err != nil {
|
||||
slog.Error("explain: send reply failed", "guid", guid, "err", err)
|
||||
return
|
||||
}
|
||||
slog.Info("explain: summary posted", "guid", guid, "channel", channel)
|
||||
}
|
||||
|
||||
// acquire returns true if this guid hasn't been explained within the cooldown.
|
||||
// Updates the timestamp on success.
|
||||
func (e *Explainer) acquire(guid string) bool {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if t, ok := e.lastSeen[guid]; ok && time.Since(t) < explainCooldown {
|
||||
return false
|
||||
}
|
||||
e.lastSeen[guid] = time.Now()
|
||||
return true
|
||||
}
|
||||
|
||||
const summarySystem = `You summarize news articles in exactly 3 short bullet points for a chat reader.
|
||||
Rules:
|
||||
- Output ONLY the 3 bullets, each on its own line, prefixed with "- ".
|
||||
- No preamble, no headings, no closing remarks.
|
||||
- Each bullet is one concise sentence (under 25 words).
|
||||
- Stick strictly to facts in the article. Do not speculate.`
|
||||
|
||||
func (e *Explainer) summarize(headline, body string) (string, error) {
|
||||
// Reaction-driven flow has no parent ctx; bound the LLM call ourselves.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
prompt := fmt.Sprintf("Headline: %s\n\nArticle body:\n%s", headline, body)
|
||||
out, err := e.ollama.GenerateText(ctx, summarySystem, prompt)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(out), nil
|
||||
}
|
||||
|
||||
// formatSummary converts the LLM's "- bullet\n- bullet" output into a Matrix
|
||||
// plain + HTML pair. Any stray prefixes ("Summary:", "•", "*") are normalized
|
||||
// to "-" bullets.
|
||||
func formatSummary(raw string) (plain, htmlBody string) {
|
||||
lines := strings.Split(raw, "\n")
|
||||
var bullets []string
|
||||
sawMarker := false
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
// Skip header-ish lines the model sometimes adds
|
||||
low := strings.ToLower(line)
|
||||
if strings.HasPrefix(low, "summary:") || strings.HasPrefix(low, "tl;dr") {
|
||||
continue
|
||||
}
|
||||
// Normalize common bullet markers; only count this line if it had one.
|
||||
hadMarker := false
|
||||
for _, prefix := range []string{"- ", "* ", "• ", "·"} {
|
||||
if strings.HasPrefix(line, prefix) {
|
||||
line = strings.TrimSpace(line[len(prefix):])
|
||||
hadMarker = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hadMarker {
|
||||
continue
|
||||
}
|
||||
sawMarker = true
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
bullets = append(bullets, line)
|
||||
}
|
||||
if !sawMarker || len(bullets) == 0 {
|
||||
// Model produced something but we couldn't parse bullets — pass through.
|
||||
return raw, "<pre>" + html.EscapeString(raw) + "</pre>"
|
||||
}
|
||||
|
||||
var pb, hb strings.Builder
|
||||
hb.WriteString("<ul>")
|
||||
for i, b := range bullets {
|
||||
if i > 0 {
|
||||
pb.WriteByte('\n')
|
||||
}
|
||||
pb.WriteString("• ")
|
||||
pb.WriteString(b)
|
||||
hb.WriteString("<li>")
|
||||
hb.WriteString(html.EscapeString(b))
|
||||
hb.WriteString("</li>")
|
||||
}
|
||||
hb.WriteString("</ul>")
|
||||
return pb.String(), hb.String()
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package explainer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFormatSummary(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
wantPlain string
|
||||
wantHTML string
|
||||
}{
|
||||
{
|
||||
name: "dash bullets",
|
||||
in: "- First point.\n- Second point.\n- Third point.",
|
||||
wantPlain: "• First point.\n• Second point.\n• Third point.",
|
||||
wantHTML: "<ul><li>First point.</li><li>Second point.</li><li>Third point.</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "asterisk bullets",
|
||||
in: "* alpha\n* beta",
|
||||
wantPlain: "• alpha\n• beta",
|
||||
wantHTML: "<ul><li>alpha</li><li>beta</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "unicode bullets",
|
||||
in: "• one\n• two",
|
||||
wantPlain: "• one\n• two",
|
||||
wantHTML: "<ul><li>one</li><li>two</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "blank lines ignored",
|
||||
in: "- a\n\n- b\n\n\n- c",
|
||||
wantPlain: "• a\n• b\n• c",
|
||||
wantHTML: "<ul><li>a</li><li>b</li><li>c</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "summary header stripped",
|
||||
in: "Summary:\n- first\n- second",
|
||||
wantPlain: "• first\n• second",
|
||||
wantHTML: "<ul><li>first</li><li>second</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "tldr header stripped",
|
||||
in: "TL;DR of the article\n- key point\n- other point",
|
||||
wantPlain: "• key point\n• other point",
|
||||
wantHTML: "<ul><li>key point</li><li>other point</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "single bullet",
|
||||
in: "- just one",
|
||||
wantPlain: "• just one",
|
||||
wantHTML: "<ul><li>just one</li></ul>",
|
||||
},
|
||||
{
|
||||
name: "html escaped in bullets",
|
||||
in: `- A "quote" & <tag>`,
|
||||
wantPlain: `• A "quote" & <tag>`,
|
||||
wantHTML: "<ul><li>A "quote" & <tag></li></ul>",
|
||||
},
|
||||
{
|
||||
name: "leading whitespace trimmed",
|
||||
in: " - first\n - second ",
|
||||
wantPlain: "• first\n• second",
|
||||
wantHTML: "<ul><li>first</li><li>second</li></ul>",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
gotPlain, gotHTML := formatSummary(c.in)
|
||||
if gotPlain != c.wantPlain {
|
||||
t.Errorf("plain:\n got %q\n want %q", gotPlain, c.wantPlain)
|
||||
}
|
||||
if gotHTML != c.wantHTML {
|
||||
t.Errorf("html:\n got %q\n want %q", gotHTML, c.wantHTML)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatSummary_NoBulletsPassthrough(t *testing.T) {
|
||||
in := "Just a single paragraph with no bullets at all."
|
||||
plain, htmlBody := formatSummary(in)
|
||||
if plain != in {
|
||||
t.Errorf("plain should pass through raw, got %q", plain)
|
||||
}
|
||||
if !strings.HasPrefix(htmlBody, "<pre>") || !strings.HasSuffix(htmlBody, "</pre>") {
|
||||
t.Errorf("html should wrap raw in <pre>, got %q", htmlBody)
|
||||
}
|
||||
if !strings.Contains(htmlBody, "single paragraph") {
|
||||
t.Errorf("html should contain the body text, got %q", htmlBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsQuestionReaction(t *testing.T) {
|
||||
want := []string{"❓", "❓️", "❔", "❔️", "⁉", "⁉️", "🤔", "?", "？"}
|
||||
for _, k := range want {
|
||||
if !IsQuestionReaction(k) {
|
||||
t.Errorf("expected %q to trigger explain", k)
|
||||
}
|
||||
}
|
||||
notQuestion := []string{"👍", "👎", "🔥", "", "??", "❤️"}
|
||||
for _, k := range notQuestion {
|
||||
if IsQuestionReaction(k) {
|
||||
t.Errorf("expected %q to NOT trigger explain", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||
|
||||
var feedClient = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// FeedItem represents a parsed RSS item ready for classification.
|
||||
// FeedItem represents a parsed RSS item ready for routing.
|
||||
type FeedItem struct {
|
||||
GUID string
|
||||
Headline string
|
||||
@@ -24,9 +24,7 @@ type FeedItem struct {
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
Source string
|
||||
FeedHint string
|
||||
Tier int
|
||||
DirectRoute *string
|
||||
DirectRoute string
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -11,9 +11,7 @@ import (
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// ProcessFunc is called for each new feed item that needs classification.
|
||||
// ctx is the poller's parent context; callbacks should pass it into any
|
||||
// downstream LLM/HTTP work so shutdown aborts in-flight calls.
|
||||
// ProcessFunc is called for each new feed item that needs routing.
|
||||
type ProcessFunc func(ctx context.Context, item *FeedItem)
|
||||
|
||||
// Poller manages per-source RSS polling goroutines.
|
||||
@@ -81,7 +79,6 @@ func (p *Poller) pollSource(ctx context.Context, src config.SourceConfig) {
|
||||
"source", src.Name,
|
||||
"failures", consecutiveFailures,
|
||||
)
|
||||
// TODO: send admin room warning via matrix client
|
||||
}
|
||||
} else {
|
||||
consecutiveFailures = 0
|
||||
@@ -163,11 +160,9 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
|
||||
// Stamp source metadata onto the item
|
||||
items[i].Source = src.Name
|
||||
items[i].FeedHint = src.FeedHint
|
||||
items[i].Tier = src.Tier
|
||||
items[i].DirectRoute = src.DirectRoute
|
||||
|
||||
// Store immediately (before classification) to prevent re-ingestion
|
||||
// Store immediately (before routing) to prevent re-ingestion
|
||||
if err := storage.InsertStory(&storage.Story{
|
||||
GUID: items[i].GUID,
|
||||
Headline: items[i].Headline,
|
||||
@@ -177,7 +172,6 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
URLCanonical: canonical,
|
||||
HeadlineNorm: headlineNorm,
|
||||
Source: items[i].Source,
|
||||
FeedHint: items[i].FeedHint,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}); err != nil {
|
||||
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
||||
@@ -192,40 +186,5 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
slog.Info("ingested new stories", "source", src.Name, "count", newCount)
|
||||
}
|
||||
|
||||
// Retry unclassified stories from this source only (cap at 20 to avoid runaway retries)
|
||||
unclassified, err := storage.GetUnclassifiedStories(src.Name)
|
||||
if err != nil {
|
||||
slog.Error("failed to get unclassified stories", "err", err)
|
||||
return nil
|
||||
}
|
||||
for _, s := range unclassified {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
// Skip stories we just ingested (they're already being processed above)
|
||||
alreadyProcessed := false
|
||||
for _, item := range items {
|
||||
if item.GUID == s.GUID {
|
||||
alreadyProcessed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if alreadyProcessed {
|
||||
continue
|
||||
}
|
||||
|
||||
p.process(ctx, &FeedItem{
|
||||
GUID: s.GUID,
|
||||
Headline: s.Headline,
|
||||
Lede: s.Lede,
|
||||
ImageURL: s.ImageURL,
|
||||
ArticleURL: s.ArticleURL,
|
||||
Source: s.Source,
|
||||
FeedHint: s.FeedHint,
|
||||
Tier: src.Tier,
|
||||
DirectRoute: src.DirectRoute,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ func insertPostable(t *testing.T, guid, source, channel string, seenAt int64) {
|
||||
Lede: "Lede for " + guid,
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: source,
|
||||
FeedHint: "tech",
|
||||
SeenAt: seenAt,
|
||||
}); err != nil {
|
||||
t.Fatalf("InsertStory(%s): %v", guid, err)
|
||||
|
||||
@@ -101,15 +101,7 @@ func runMigrations(d *sql.DB) error {
|
||||
}
|
||||
|
||||
// RunMaintenance prunes stale data. Called periodically.
|
||||
func RunMaintenance(recentWindowHours, classificationLogDays int) {
|
||||
recentCutoff := nowUnix() - int64(recentWindowHours*3600)
|
||||
exec("prune recent_headlines",
|
||||
`DELETE FROM recent_headlines WHERE seen_at < ?`, recentCutoff)
|
||||
|
||||
logCutoff := nowUnix() - int64(classificationLogDays*86400)
|
||||
exec("prune classification_log",
|
||||
`DELETE FROM classification_log WHERE logged_at < ?`, logCutoff)
|
||||
|
||||
func RunMaintenance() {
|
||||
// Prune old stories (30 days) and their post logs / reactions
|
||||
storyCutoff := nowUnix() - int64(30*86400)
|
||||
exec("prune old stories",
|
||||
|
||||
@@ -29,9 +29,9 @@ func InsertStory(s *Story) error {
|
||||
classified = 1
|
||||
}
|
||||
_, err := Get().Exec(
|
||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, feed_hint, platforms, channel, classified, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.FeedHint, s.Platforms, s.Channel, classified, s.SeenAt,
|
||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -99,73 +99,18 @@ func MarkClassified(guid, channel, platforms string) {
|
||||
channel, platforms, guid)
|
||||
}
|
||||
|
||||
// GetUnclassifiedStories returns stories from a specific source that were ingested but not yet classified.
|
||||
func GetUnclassifiedStories(source string) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, seen_at
|
||||
FROM stories WHERE classified = 0 AND source = ? ORDER BY seen_at ASC LIMIT 20`, source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var stories []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stories = append(stories, s)
|
||||
}
|
||||
return stories, rows.Err()
|
||||
}
|
||||
|
||||
// GetStoryByGUID returns the full story record for a GUID, or nil if not found.
|
||||
func GetStoryByGUID(guid string) (*Story, error) {
|
||||
row := Get().QueryRow(
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
|
||||
FROM stories WHERE guid = ?`, guid)
|
||||
var s Story
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// InsertRecentHeadline adds a headline to the dedup context window.
|
||||
func InsertRecentHeadline(guid, headline, source string) {
|
||||
exec("insert recent_headline",
|
||||
`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
|
||||
guid, headline, source, nowUnix())
|
||||
}
|
||||
|
||||
// GetRecentHeadlines returns the most recent headlines for LLM dedup context, capped at limit.
|
||||
func GetRecentHeadlines(limit int) ([]RecentHeadline, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT guid, headline, source, seen_at FROM recent_headlines ORDER BY seen_at DESC LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var headlines []RecentHeadline
|
||||
for rows.Next() {
|
||||
var h RecentHeadline
|
||||
if err := rows.Scan(&h.GUID, &h.Headline, &h.Source, &h.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
headlines = append(headlines, h)
|
||||
}
|
||||
return headlines, rows.Err()
|
||||
}
|
||||
|
||||
// InsertClassificationLog stores a classification result for tuning review.
|
||||
func InsertClassificationLog(guid, resultJSON string) {
|
||||
exec("insert classification_log",
|
||||
`INSERT OR REPLACE INTO classification_log (guid, result, logged_at) VALUES (?, ?, ?)`,
|
||||
guid, resultJSON, nowUnix())
|
||||
}
|
||||
|
||||
// InsertPostLog records that a story was posted to a channel.
|
||||
// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
|
||||
func InsertPostLog(guid, channel, eventID, urlCanonical string) {
|
||||
@@ -228,7 +173,7 @@ func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt
|
||||
// row in post_log). Returns (nil, nil) when nothing qualifies.
|
||||
func GetNewestPostableStory(source string) (*Story, error) {
|
||||
row := Get().QueryRow(
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
|
||||
FROM stories
|
||||
WHERE classified = 1
|
||||
AND source = ?
|
||||
@@ -238,7 +183,7 @@ func GetNewestPostableStory(source string) (*Story, error) {
|
||||
ORDER BY seen_at DESC
|
||||
LIMIT 1`, source)
|
||||
var s Story
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -253,7 +198,7 @@ func GetNewestPostableStory(source string) (*Story, error) {
|
||||
// between ticks). Returns (nil, nil) when nothing qualifies.
|
||||
func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
row := Get().QueryRow(
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
|
||||
FROM stories
|
||||
WHERE classified = 1
|
||||
AND channel = ?
|
||||
@@ -261,7 +206,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
ORDER BY seen_at DESC
|
||||
LIMIT 1`, channel)
|
||||
var s Story
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -275,7 +220,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
// channels (_discarded, _duplicate) are excluded.
|
||||
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
|
||||
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
|
||||
FROM stories
|
||||
WHERE classified = 1
|
||||
AND channel = ?
|
||||
@@ -288,7 +233,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
|
||||
@@ -11,26 +11,12 @@ CREATE TABLE IF NOT EXISTS stories (
|
||||
url_canonical TEXT,
|
||||
headline_norm TEXT,
|
||||
source TEXT NOT NULL,
|
||||
feed_hint TEXT,
|
||||
platforms TEXT,
|
||||
channel TEXT,
|
||||
classified INTEGER NOT NULL DEFAULT 0,
|
||||
seen_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS recent_headlines (
|
||||
guid TEXT PRIMARY KEY,
|
||||
headline TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
seen_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_log (
|
||||
guid TEXT PRIMARY KEY,
|
||||
result TEXT NOT NULL,
|
||||
logged_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS post_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
guid TEXT NOT NULL,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
@@ -48,7 +47,6 @@ func TestInsertStoryAndGUIDSeen(t *testing.T) {
|
||||
Lede: "Test lede sentence.",
|
||||
ArticleURL: "https://example.com/article",
|
||||
Source: "Test Source",
|
||||
FeedHint: "tech",
|
||||
SeenAt: 1700000000,
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
@@ -79,63 +77,21 @@ func TestInsertStoryDuplicateGUID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkClassifiedAndGetUnclassified(t *testing.T) {
|
||||
func TestMarkClassified(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertStory(&Story{
|
||||
GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
|
||||
})
|
||||
InsertStory(&Story{
|
||||
GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
|
||||
})
|
||||
|
||||
unclassified, err := GetUnclassifiedStories("S")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unclassified) != 2 {
|
||||
t.Fatalf("unclassified = %d, want 2", len(unclassified))
|
||||
}
|
||||
|
||||
MarkClassified("g1", "tech", "[]")
|
||||
|
||||
unclassified, err = GetUnclassifiedStories("S")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unclassified) != 1 {
|
||||
t.Fatalf("unclassified = %d, want 1", len(unclassified))
|
||||
}
|
||||
if unclassified[0].GUID != "g2" {
|
||||
t.Errorf("remaining = %q, want g2", unclassified[0].GUID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecentHeadlines(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Insert with explicit timestamps to ensure ordering
|
||||
db := Get()
|
||||
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
|
||||
"g1", "Headline One", "Source A", 1000)
|
||||
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
|
||||
"g2", "Headline Two", "Source B", 2000)
|
||||
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
|
||||
"g3", "Headline Three", "Source A", 3000)
|
||||
|
||||
headlines, err := GetRecentHeadlines(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(headlines) != 2 {
|
||||
t.Fatalf("headlines = %d, want 2", len(headlines))
|
||||
}
|
||||
// Should be most recent first
|
||||
if headlines[0].GUID != "g3" {
|
||||
t.Errorf("first = %q, want g3", headlines[0].GUID)
|
||||
}
|
||||
if headlines[1].GUID != "g2" {
|
||||
t.Errorf("second = %q, want g2", headlines[1].GUID)
|
||||
var channel string
|
||||
var classified int
|
||||
Get().QueryRow(`SELECT channel, classified FROM stories WHERE guid = ?`, "g1").
|
||||
Scan(&channel, &classified)
|
||||
if channel != "tech" || classified != 1 {
|
||||
t.Errorf("after MarkClassified: channel=%q classified=%d, want tech/1", channel, classified)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,57 +225,6 @@ func TestInsertReaction_DuplicateIgnored(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetUnclassifiedStories_ScopedBySource(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertStory(&Story{GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "SourceA", SeenAt: 1})
|
||||
InsertStory(&Story{GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "SourceB", SeenAt: 2})
|
||||
InsertStory(&Story{GUID: "g3", Headline: "H3", ArticleURL: "https://c.com", Source: "SourceA", SeenAt: 3})
|
||||
|
||||
got, err := GetUnclassifiedStories("SourceA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 stories for SourceA, got %d", len(got))
|
||||
}
|
||||
|
||||
got, err = GetUnclassifiedStories("SourceB")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected 1 story for SourceB, got %d", len(got))
|
||||
}
|
||||
|
||||
got, err = GetUnclassifiedStories("SourceC")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("expected 0 stories for SourceC, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetUnclassifiedStories_Limit20(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
for i := 0; i < 30; i++ {
|
||||
InsertStory(&Story{
|
||||
GUID: fmt.Sprintf("g%d", i), Headline: fmt.Sprintf("H%d", i),
|
||||
ArticleURL: "https://a.com", Source: "S", SeenAt: int64(i),
|
||||
})
|
||||
}
|
||||
|
||||
got, err := GetUnclassifiedStories("S")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 20 {
|
||||
t.Fatalf("expected 20 (capped), got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMaintenance(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
db := Get()
|
||||
@@ -341,11 +246,7 @@ func TestRunMaintenance(t *testing.T) {
|
||||
InsertReaction("old", "tech", "$r1", "👍", "@u:x", old)
|
||||
InsertReaction("new", "tech", "$r2", "👍", "@u:x", now)
|
||||
|
||||
// Insert old + recent headlines
|
||||
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "old", "Old", "S", old)
|
||||
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "new", "New", "S", now)
|
||||
|
||||
RunMaintenance(24, 7)
|
||||
RunMaintenance()
|
||||
|
||||
// Old stories should be pruned
|
||||
var count int
|
||||
@@ -363,11 +264,6 @@ func TestRunMaintenance(t *testing.T) {
|
||||
if count != 1 {
|
||||
t.Errorf("reactions: expected 1, got %d", count)
|
||||
}
|
||||
|
||||
db.QueryRow(`SELECT COUNT(*) FROM recent_headlines`).Scan(&count)
|
||||
if count != 1 {
|
||||
t.Errorf("recent_headlines: expected 1, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFTS5Search(t *testing.T) {
|
||||
|
||||
@@ -2,26 +2,17 @@ package storage
|
||||
|
||||
// Story is the core record for an ingested news item.
|
||||
type Story struct {
|
||||
ID int64
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
ID int64
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
URLCanonical string
|
||||
HeadlineNorm string
|
||||
Source string
|
||||
FeedHint string
|
||||
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
|
||||
Channel string
|
||||
Classified bool
|
||||
SeenAt int64
|
||||
}
|
||||
|
||||
// RecentHeadline is a lightweight record for the LLM dedup context window.
|
||||
type RecentHeadline struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Source string
|
||||
SeenAt int64
|
||||
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
|
||||
Channel string
|
||||
Classified bool
|
||||
SeenAt int64
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user