drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden paywall detection + render paywalled stamp on cards

Browse Source 
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
prosolis
2026-05-25 11:23:22 -07:00
parent 9bc8743b5e
commit 509a0fc7a7
11 changed files with 316 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/storage/db.go
View File

@@ -80,6 +80,7 @@ func runMigrations(d *sql.DB) error {
// we swallow that specifically.
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")

22
internal/storage/queries.go
View File

@@ -28,10 +28,14 @@ func InsertStory(s *Story) error {
if s.Classified {
classified = 1
}
paywalled := 0
if s.Paywalled {
paywalled = 1
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
)
return err
}
@@ -220,7 +224,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -234,7 +238,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -246,7 +250,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
// channels, newest first. Sentinel channels are excluded.
func ListAllClassified(limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -261,7 +265,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -273,7 +277,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
FROM stories s
JOIN post_log p ON p.guid = s.guid
GROUP BY s.guid
@@ -286,7 +290,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
return nil, err
}
s.Posted = true

1
internal/storage/schema.go
View File

@@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS stories (
platforms TEXT,
channel TEXT,
classified INTEGER NOT NULL DEFAULT 0,
paywalled INTEGER NOT NULL DEFAULT 0,
seen_at INTEGER NOT NULL
);

1
internal/storage/types.go
View File

@@ -14,6 +14,7 @@ type Story struct {
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
Channel string
Classified bool
Paywalled bool // source article is gated; ArticleURL may be an archive snapshot
SeenAt int64
Posted bool // true if this story has been posted to Matrix (joined from post_log)
}