9e5ba5aafc
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
90 lines
3.5 KiB
Go
90 lines
3.5 KiB
Go
package storage
|
|
|
|
const schema = `
|
|
CREATE TABLE IF NOT EXISTS stories (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
guid TEXT UNIQUE NOT NULL,
|
|
headline TEXT NOT NULL,
|
|
lede TEXT,
|
|
image_url TEXT,
|
|
article_url TEXT NOT NULL,
|
|
url_canonical TEXT,
|
|
headline_norm TEXT,
|
|
source TEXT NOT NULL,
|
|
platforms TEXT,
|
|
channel TEXT,
|
|
classified INTEGER NOT NULL DEFAULT 0,
|
|
paywalled INTEGER NOT NULL DEFAULT 0,
|
|
seen_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS post_log (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
guid TEXT NOT NULL,
|
|
channel TEXT NOT NULL,
|
|
event_id TEXT,
|
|
url_canonical TEXT,
|
|
posted_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS round_robin_state (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
last_channel TEXT,
|
|
last_tick_at INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS reactions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
post_guid TEXT NOT NULL,
|
|
channel TEXT NOT NULL,
|
|
event_id TEXT NOT NULL,
|
|
emoji TEXT NOT NULL,
|
|
user_id TEXT NOT NULL,
|
|
reacted_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_canonical_channel ON post_log(url_canonical, channel, posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_stories_classified_source ON stories(classified, source);
|
|
CREATE INDEX IF NOT EXISTS idx_stories_channel_classified_seen ON stories(channel, classified, seen_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_stories_classified_seen ON stories(classified, seen_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_stories_image_url ON stories(image_url) WHERE image_url IS NOT NULL AND image_url <> '';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canonical) WHERE url_canonical IS NOT NULL AND url_canonical <> '';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
|
|
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
|
|
`
|
|
|
|
const ftsSchema = `
|
|
CREATE VIRTUAL TABLE stories_fts USING fts5(
|
|
guid UNINDEXED,
|
|
headline,
|
|
lede,
|
|
source UNINDEXED,
|
|
platforms UNINDEXED,
|
|
content='stories',
|
|
content_rowid='id'
|
|
);
|
|
`
|
|
|
|
const ftsTriggers = `
|
|
CREATE TRIGGER stories_fts_insert AFTER INSERT ON stories BEGIN
|
|
INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
|
|
VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
|
|
END;
|
|
|
|
CREATE TRIGGER stories_fts_delete AFTER DELETE ON stories BEGIN
|
|
INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
|
|
VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
|
|
END;
|
|
|
|
CREATE TRIGGER stories_fts_update AFTER UPDATE ON stories BEGIN
|
|
INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
|
|
VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
|
|
INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
|
|
VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
|
|
END;
|
|
`
|