509a0fc7a7
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD gating scoped to Article-typed nodes, HTTP 402 treated as explicit paywall, Wayback freshness filter (30d cap), archive.today as secondary archive fallback, and transport failures no longer trigger snapshot swaps. When gating is detected and no archive workaround succeeds, the story is stored with paywalled=1 and the web card renders a diagonal red rubber-stamp overlay so readers know the link is gated.
87 lines
3.2 KiB
Go
87 lines
3.2 KiB
Go
package storage
|
|
|
|
const schema = `
|
|
CREATE TABLE IF NOT EXISTS stories (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
guid TEXT UNIQUE NOT NULL,
|
|
headline TEXT NOT NULL,
|
|
lede TEXT,
|
|
image_url TEXT,
|
|
article_url TEXT NOT NULL,
|
|
url_canonical TEXT,
|
|
headline_norm TEXT,
|
|
source TEXT NOT NULL,
|
|
platforms TEXT,
|
|
channel TEXT,
|
|
classified INTEGER NOT NULL DEFAULT 0,
|
|
paywalled INTEGER NOT NULL DEFAULT 0,
|
|
seen_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS post_log (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
guid TEXT NOT NULL,
|
|
channel TEXT NOT NULL,
|
|
event_id TEXT,
|
|
url_canonical TEXT,
|
|
posted_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS round_robin_state (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
last_channel TEXT,
|
|
last_tick_at INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS reactions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
post_guid TEXT NOT NULL,
|
|
channel TEXT NOT NULL,
|
|
event_id TEXT NOT NULL,
|
|
emoji TEXT NOT NULL,
|
|
user_id TEXT NOT NULL,
|
|
reacted_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_post_log_canonical_channel ON post_log(url_canonical, channel, posted_at);
|
|
CREATE INDEX IF NOT EXISTS idx_stories_classified_source ON stories(classified, source);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canonical) WHERE url_canonical IS NOT NULL AND url_canonical <> '';
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
|
|
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
|
|
`
|
|
|
|
const ftsSchema = `
|
|
CREATE VIRTUAL TABLE stories_fts USING fts5(
|
|
guid UNINDEXED,
|
|
headline,
|
|
lede,
|
|
source UNINDEXED,
|
|
platforms UNINDEXED,
|
|
content='stories',
|
|
content_rowid='id'
|
|
);
|
|
`
|
|
|
|
const ftsTriggers = `
|
|
CREATE TRIGGER stories_fts_insert AFTER INSERT ON stories BEGIN
|
|
INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
|
|
VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
|
|
END;
|
|
|
|
CREATE TRIGGER stories_fts_delete AFTER DELETE ON stories BEGIN
|
|
INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
|
|
VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
|
|
END;
|
|
|
|
CREATE TRIGGER stories_fts_update AFTER UPDATE ON stories BEGIN
|
|
INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
|
|
VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
|
|
INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
|
|
VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
|
|
END;
|
|
`
|