drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden paywall detection + render paywalled stamp on cards

Browse Source 
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
prosolis
2026-05-25 11:23:22 -07:00
parent 9bc8743b5e
commit 509a0fc7a7
11 changed files with 316 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/storage/queries.go
View File

@@ -28,10 +28,14 @@ func InsertStory(s *Story) error {
if s.Classified {
classified = 1
}
paywalled := 0
if s.Paywalled {
paywalled = 1
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
)
return err
}
@@ -220,7 +224,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -234,7 +238,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -246,7 +250,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
// channels, newest first. Sentinel channels are excluded.
func ListAllClassified(limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -261,7 +265,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -273,7 +277,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
FROM stories s
JOIN post_log p ON p.guid = s.guid
GROUP BY s.guid
@@ -286,7 +290,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
return nil, err
}
s.Posted = true