- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
169 lines
3.9 KiB
Go
169 lines
3.9 KiB
Go
package main
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"pete/internal/config"
|
|
"pete/internal/safehttp"
|
|
"pete/internal/storage"
|
|
)
|
|
|
|
func init() {
|
|
safehttp.AllowPrivate = true
|
|
}
|
|
|
|
func setupTestDB(t *testing.T) string {
|
|
t.Helper()
|
|
dbPath := filepath.Join(t.TempDir(), "test.db")
|
|
if err := storage.Init(dbPath); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { storage.Close() })
|
|
return dbPath
|
|
}
|
|
|
|
// serveFeed returns a test HTTP server that serves a minimal RSS feed.
|
|
func serveFeed(t *testing.T, items int) *httptest.Server {
|
|
t.Helper()
|
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/rss+xml")
|
|
w.WriteHeader(200)
|
|
w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"><channel><title>Test Feed</title>`))
|
|
for i := 0; i < items; i++ {
|
|
guid := "https://example.com/story-" + string(rune('a'+i))
|
|
w.Write([]byte(`<item><title>Story ` + string(rune('A'+i)) + `</title><link>` + guid + `</link><guid>` + guid + `</guid><description>Lede for story.</description></item>`))
|
|
}
|
|
w.Write([]byte(`</channel></rss>`))
|
|
}))
|
|
}
|
|
|
|
func TestTimeNow(t *testing.T) {
|
|
a := timeNow()
|
|
b := timeNow()
|
|
if b < a {
|
|
t.Errorf("timeNow() went backwards: %d then %d", a, b)
|
|
}
|
|
if b-a > 1 {
|
|
t.Errorf("timeNow() gap too large: %d", b-a)
|
|
}
|
|
}
|
|
|
|
func TestRunSeed_IngestsStories(t *testing.T) {
|
|
setupTestDB(t)
|
|
ts := serveFeed(t, 3)
|
|
defer ts.Close()
|
|
|
|
cfg := &config.Config{
|
|
Sources: []config.SourceConfig{
|
|
{
|
|
Name: "Test Feed",
|
|
FeedURL: ts.URL,
|
|
Tier: 1,
|
|
PollIntervalMinutes: 20,
|
|
DirectRoute: "tech",
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
runSeed(cfg)
|
|
|
|
// All 3 stories should be marked as seen and classified
|
|
for _, c := range []rune{'a', 'b', 'c'} {
|
|
guid := "https://example.com/story-" + string(c)
|
|
if !storage.IsGUIDSeen(guid) {
|
|
t.Errorf("expected GUID %q to be seen after seed", guid)
|
|
}
|
|
}
|
|
|
|
// Seed marks rows classified=1; confirm none remain unclassified.
|
|
var unclassified int
|
|
storage.Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE classified = 0`).Scan(&unclassified)
|
|
if unclassified != 0 {
|
|
t.Errorf("expected 0 unclassified after seed, got %d", unclassified)
|
|
}
|
|
}
|
|
|
|
func TestRunSeed_Idempotent(t *testing.T) {
|
|
setupTestDB(t)
|
|
ts := serveFeed(t, 2)
|
|
defer ts.Close()
|
|
|
|
cfg := &config.Config{
|
|
Sources: []config.SourceConfig{
|
|
{
|
|
Name: "Test Feed",
|
|
FeedURL: ts.URL,
|
|
Tier: 1,
|
|
PollIntervalMinutes: 20,
|
|
DirectRoute: "tech",
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
// Run twice — should not error or duplicate
|
|
runSeed(cfg)
|
|
runSeed(cfg)
|
|
|
|
var count int
|
|
storage.Get().QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&count)
|
|
if count != 2 {
|
|
t.Errorf("expected 2 stories after double seed, got %d", count)
|
|
}
|
|
}
|
|
|
|
func TestRunSeed_SkipsDisabledSources(t *testing.T) {
|
|
setupTestDB(t)
|
|
ts := serveFeed(t, 3)
|
|
defer ts.Close()
|
|
|
|
cfg := &config.Config{
|
|
Sources: []config.SourceConfig{
|
|
{
|
|
Name: "Disabled Feed",
|
|
FeedURL: ts.URL,
|
|
Tier: 1,
|
|
PollIntervalMinutes: 20,
|
|
Enabled: false,
|
|
},
|
|
},
|
|
}
|
|
|
|
runSeed(cfg)
|
|
|
|
var count int
|
|
storage.Get().QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&count)
|
|
if count != 0 {
|
|
t.Errorf("expected 0 stories for disabled source, got %d", count)
|
|
}
|
|
}
|
|
|
|
func TestRunSeed_BadFeedURL(t *testing.T) {
|
|
setupTestDB(t)
|
|
|
|
cfg := &config.Config{
|
|
Sources: []config.SourceConfig{
|
|
{
|
|
Name: "Bad Feed",
|
|
FeedURL: "http://127.0.0.1:1/nonexistent",
|
|
Tier: 1,
|
|
PollIntervalMinutes: 20,
|
|
Enabled: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
// Should not panic — just log and skip
|
|
runSeed(cfg)
|
|
|
|
var count int
|
|
storage.Get().QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&count)
|
|
if count != 0 {
|
|
t.Errorf("expected 0 stories for bad feed, got %d", count)
|
|
}
|
|
}
|