drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rip out Ollama: direct_route only, no LLM layer

Browse Source 
Pete moves to a remote host without Ollama access. Every source must
declare a direct_route channel; the classifier, explainer, semantic
dedup, !explain summaries, feed_hint, and the recent_headlines /
classification_log tables are gone. Deterministic dedup (canonical URL,
headline_norm, per-channel cooldown) remains.
This commit is contained in:
prosolis
2026-05-24 22:07:13 -07:00
parent fbd4b84eaf
commit e88483526d
23 changed files with 146 additions and 1675 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/storage/db.go
View File

@@ -101,15 +101,7 @@ func runMigrations(d *sql.DB) error {
}
// RunMaintenance prunes stale data. Called periodically.
func RunMaintenance(recentWindowHours, classificationLogDays int) {
recentCutoff := nowUnix() - int64(recentWindowHours*3600)
exec("prune recent_headlines",
`DELETE FROM recent_headlines WHERE seen_at < ?`, recentCutoff)
logCutoff := nowUnix() - int64(classificationLogDays*86400)
exec("prune classification_log",
`DELETE FROM classification_log WHERE logged_at < ?`, logCutoff)
func RunMaintenance() {
// Prune old stories (30 days) and their post logs / reactions
storyCutoff := nowUnix() - int64(30*86400)
exec("prune old stories",

77
internal/storage/queries.go
View File

@@ -29,9 +29,9 @@ func InsertStory(s *Story) error {
classified = 1
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, feed_hint, platforms, channel, classified, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.FeedHint, s.Platforms, s.Channel, classified, s.SeenAt,
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
)
return err
}
@@ -99,73 +99,18 @@ func MarkClassified(guid, channel, platforms string) {
channel, platforms, guid)
}
// GetUnclassifiedStories returns stories from a specific source that were ingested but not yet classified.
func GetUnclassifiedStories(source string) ([]Story, error) {
rows, err := Get().Query(
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, seen_at
FROM stories WHERE classified = 0 AND source = ? ORDER BY seen_at ASC LIMIT 20`, source)
if err != nil {
return nil, err
}
defer rows.Close()
var stories []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.SeenAt); err != nil {
return nil, err
}
stories = append(stories, s)
}
return stories, rows.Err()
}
// GetStoryByGUID returns the full story record for a GUID, or nil if not found.
func GetStoryByGUID(guid string) (*Story, error) {
row := Get().QueryRow(
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
FROM stories WHERE guid = ?`, guid)
var s Story
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
return nil, err
}
return &s, nil
}
// InsertRecentHeadline adds a headline to the dedup context window.
func InsertRecentHeadline(guid, headline, source string) {
exec("insert recent_headline",
`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
guid, headline, source, nowUnix())
}
// GetRecentHeadlines returns the most recent headlines for LLM dedup context, capped at limit.
func GetRecentHeadlines(limit int) ([]RecentHeadline, error) {
rows, err := Get().Query(
`SELECT guid, headline, source, seen_at FROM recent_headlines ORDER BY seen_at DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var headlines []RecentHeadline
for rows.Next() {
var h RecentHeadline
if err := rows.Scan(&h.GUID, &h.Headline, &h.Source, &h.SeenAt); err != nil {
return nil, err
}
headlines = append(headlines, h)
}
return headlines, rows.Err()
}
// InsertClassificationLog stores a classification result for tuning review.
func InsertClassificationLog(guid, resultJSON string) {
exec("insert classification_log",
`INSERT OR REPLACE INTO classification_log (guid, result, logged_at) VALUES (?, ?, ?)`,
guid, resultJSON, nowUnix())
}
// InsertPostLog records that a story was posted to a channel.
// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
func InsertPostLog(guid, channel, eventID, urlCanonical string) {
@@ -228,7 +173,7 @@ func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt
// row in post_log). Returns (nil, nil) when nothing qualifies.
func GetNewestPostableStory(source string) (*Story, error) {
row := Get().QueryRow(
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
FROM stories
WHERE classified = 1
AND source = ?
@@ -238,7 +183,7 @@ func GetNewestPostableStory(source string) (*Story, error) {
ORDER BY seen_at DESC
LIMIT 1`, source)
var s Story
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
@@ -253,7 +198,7 @@ func GetNewestPostableStory(source string) (*Story, error) {
// between ticks). Returns (nil, nil) when nothing qualifies.
func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
row := Get().QueryRow(
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
FROM stories
WHERE classified = 1
AND channel = ?
@@ -261,7 +206,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
ORDER BY seen_at DESC
LIMIT 1`, channel)
var s Story
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
@@ -275,7 +220,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, channel, seen_at
`SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at
FROM stories
WHERE classified = 1
AND channel = ?
@@ -288,7 +233,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
return nil, err
}
out = append(out, s)

14
internal/storage/schema.go
View File

@@ -11,26 +11,12 @@ CREATE TABLE IF NOT EXISTS stories (
url_canonical TEXT,
headline_norm TEXT,
source TEXT NOT NULL,
feed_hint TEXT,
platforms TEXT,
channel TEXT,
classified INTEGER NOT NULL DEFAULT 0,
seen_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS recent_headlines (
guid TEXT PRIMARY KEY,
headline TEXT NOT NULL,
source TEXT NOT NULL,
seen_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS classification_log (
guid TEXT PRIMARY KEY,
result TEXT NOT NULL,
logged_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS post_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
guid TEXT NOT NULL,

120
internal/storage/storage_test.go
View File

@@ -1,7 +1,6 @@
package storage
import (
"fmt"
"path/filepath"
"testing"
)
@@ -48,7 +47,6 @@ func TestInsertStoryAndGUIDSeen(t *testing.T) {
Lede: "Test lede sentence.",
ArticleURL: "https://example.com/article",
Source: "Test Source",
FeedHint: "tech",
SeenAt: 1700000000,
}
if err := InsertStory(s); err != nil {
@@ -79,63 +77,21 @@ func TestInsertStoryDuplicateGUID(t *testing.T) {
}
}
func TestMarkClassifiedAndGetUnclassified(t *testing.T) {
func TestMarkClassified(t *testing.T) {
setupTestDB(t)
InsertStory(&Story{
GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
})
InsertStory(&Story{
GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
})
unclassified, err := GetUnclassifiedStories("S")
if err != nil {
t.Fatal(err)
}
if len(unclassified) != 2 {
t.Fatalf("unclassified = %d, want 2", len(unclassified))
}
MarkClassified("g1", "tech", "[]")
unclassified, err = GetUnclassifiedStories("S")
if err != nil {
t.Fatal(err)
}
if len(unclassified) != 1 {
t.Fatalf("unclassified = %d, want 1", len(unclassified))
}
if unclassified[0].GUID != "g2" {
t.Errorf("remaining = %q, want g2", unclassified[0].GUID)
}
}
func TestRecentHeadlines(t *testing.T) {
setupTestDB(t)
// Insert with explicit timestamps to ensure ordering
db := Get()
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
"g1", "Headline One", "Source A", 1000)
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
"g2", "Headline Two", "Source B", 2000)
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
"g3", "Headline Three", "Source A", 3000)
headlines, err := GetRecentHeadlines(2)
if err != nil {
t.Fatal(err)
}
if len(headlines) != 2 {
t.Fatalf("headlines = %d, want 2", len(headlines))
}
// Should be most recent first
if headlines[0].GUID != "g3" {
t.Errorf("first = %q, want g3", headlines[0].GUID)
}
if headlines[1].GUID != "g2" {
t.Errorf("second = %q, want g2", headlines[1].GUID)
var channel string
var classified int
Get().QueryRow(`SELECT channel, classified FROM stories WHERE guid = ?`, "g1").
Scan(&channel, &classified)
if channel != "tech" || classified != 1 {
t.Errorf("after MarkClassified: channel=%q classified=%d, want tech/1", channel, classified)
}
}
@@ -269,57 +225,6 @@ func TestInsertReaction_DuplicateIgnored(t *testing.T) {
}
}
func TestGetUnclassifiedStories_ScopedBySource(t *testing.T) {
setupTestDB(t)
InsertStory(&Story{GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "SourceA", SeenAt: 1})
InsertStory(&Story{GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "SourceB", SeenAt: 2})
InsertStory(&Story{GUID: "g3", Headline: "H3", ArticleURL: "https://c.com", Source: "SourceA", SeenAt: 3})
got, err := GetUnclassifiedStories("SourceA")
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("expected 2 stories for SourceA, got %d", len(got))
}
got, err = GetUnclassifiedStories("SourceB")
if err != nil {
t.Fatal(err)
}
if len(got) != 1 {
t.Fatalf("expected 1 story for SourceB, got %d", len(got))
}
got, err = GetUnclassifiedStories("SourceC")
if err != nil {
t.Fatal(err)
}
if len(got) != 0 {
t.Fatalf("expected 0 stories for SourceC, got %d", len(got))
}
}
func TestGetUnclassifiedStories_Limit20(t *testing.T) {
setupTestDB(t)
for i := 0; i < 30; i++ {
InsertStory(&Story{
GUID: fmt.Sprintf("g%d", i), Headline: fmt.Sprintf("H%d", i),
ArticleURL: "https://a.com", Source: "S", SeenAt: int64(i),
})
}
got, err := GetUnclassifiedStories("S")
if err != nil {
t.Fatal(err)
}
if len(got) != 20 {
t.Fatalf("expected 20 (capped), got %d", len(got))
}
}
func TestRunMaintenance(t *testing.T) {
setupTestDB(t)
db := Get()
@@ -341,11 +246,7 @@ func TestRunMaintenance(t *testing.T) {
InsertReaction("old", "tech", "$r1", "👍", "@u:x", old)
InsertReaction("new", "tech", "$r2", "👍", "@u:x", now)
// Insert old + recent headlines
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "old", "Old", "S", old)
db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "new", "New", "S", now)
RunMaintenance(24, 7)
RunMaintenance()
// Old stories should be pruned
var count int
@@ -363,11 +264,6 @@ func TestRunMaintenance(t *testing.T) {
if count != 1 {
t.Errorf("reactions: expected 1, got %d", count)
}
db.QueryRow(`SELECT COUNT(*) FROM recent_headlines`).Scan(&count)
if count != 1 {
t.Errorf("recent_headlines: expected 1, got %d", count)
}
}
func TestFTS5Search(t *testing.T) {

25
internal/storage/types.go
View File

@@ -2,26 +2,17 @@ package storage
// Story is the core record for an ingested news item.
type Story struct {
ID int64
GUID string
Headline string
Lede string
ID int64
GUID string
Headline string
Lede string
ImageURL string
ArticleURL string
URLCanonical string
HeadlineNorm string
Source string
FeedHint string
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
Channel string
Classified bool
SeenAt int64
}
// RecentHeadline is a lightweight record for the LLM dedup context window.
type RecentHeadline struct {
GUID string
Headline string
Source string
SeenAt int64
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
Channel string
Classified bool
SeenAt int64
}