Harden paywall detection + render paywalled stamp on cards
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD gating scoped to Article-typed nodes, HTTP 402 treated as explicit paywall, Wayback freshness filter (30d cap), archive.today as secondary archive fallback, and transport failures no longer trigger snapshot swaps. When gating is detected and no archive workaround succeeds, the story is stored with paywalled=1 and the web card renders a diagonal red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
@@ -48,54 +48,128 @@ func resolveURL(base, ref string) string {
|
||||
// as paywalled / gated, and the caller should fall back to an archive snapshot.
|
||||
const PaywallBodyThreshold = 500
|
||||
|
||||
// googlebotUA is what many metered publishers grant first-click access to.
|
||||
// Re-tried automatically when the default-UA fetch looks gated.
|
||||
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
|
||||
// ArticleMeta is what we can learn from fetching an article page directly.
|
||||
type ArticleMeta struct {
|
||||
ImageURL string // og:image or twitter:image, absolute URL
|
||||
BodyChars int // length of extracted visible body text
|
||||
Fetched bool // true if we got an HTTP 200 with HTML
|
||||
FetchError bool // true if the fetch failed at the network/HTTP layer
|
||||
Paywalled bool // true if the page explicitly declares gated access
|
||||
Status int // last HTTP status seen (0 on transport error)
|
||||
}
|
||||
|
||||
// Gated reports whether the response carries a strong gating signal: an
|
||||
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
|
||||
// or a 403 Forbidden after the bypass retry. A short body alone is not
|
||||
// considered gating — that's a heuristic used by callers separately.
|
||||
func (m ArticleMeta) Gated() bool {
|
||||
if m.Paywalled {
|
||||
return true
|
||||
}
|
||||
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
|
||||
}
|
||||
|
||||
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
||||
|
||||
// FetchArticleMeta fetches an article URL and returns its og:image plus the
|
||||
// length of the visible body text (concatenation of <p> tags under <article>
|
||||
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
|
||||
// network/HTTP failure so callers can branch on accessibility.
|
||||
// FetchArticleMeta fetches an article URL with the default UA. If the result
|
||||
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
|
||||
// retries once with a Googlebot UA + Google referer — the combination most
|
||||
// metered publishers grant first-click access to. Returns the best of the
|
||||
// two attempts.
|
||||
func FetchArticleMeta(articleURL string) ArticleMeta {
|
||||
if articleURL == "" {
|
||||
return ArticleMeta{}
|
||||
}
|
||||
first := fetchArticleMetaOnce(articleURL, userAgent, "")
|
||||
if !shouldRetryAsBot(first) {
|
||||
return first
|
||||
}
|
||||
second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
|
||||
return pickBetter(first, second)
|
||||
}
|
||||
|
||||
// shouldRetryAsBot returns true when the first attempt looks gated or too
|
||||
// thin to be the real article body.
|
||||
func shouldRetryAsBot(m ArticleMeta) bool {
|
||||
if m.FetchError {
|
||||
return false // transport failure won't be fixed by a different UA
|
||||
}
|
||||
if m.Gated() {
|
||||
return true
|
||||
}
|
||||
if m.Fetched && m.BodyChars < PaywallBodyThreshold {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pickBetter chooses the more useful of two fetch attempts: prefer the one
|
||||
// that isn't gated, then the one with more body, then the one that fetched
|
||||
// at all.
|
||||
func pickBetter(a, b ArticleMeta) ArticleMeta {
|
||||
aGated, bGated := a.Gated(), b.Gated()
|
||||
if aGated != bGated {
|
||||
if bGated {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
if a.Fetched != b.Fetched {
|
||||
if b.Fetched {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
if b.BodyChars > a.BodyChars {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
|
||||
if err != nil {
|
||||
return ArticleMeta{}
|
||||
return ArticleMeta{FetchError: true}
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
req.Header.Set("User-Agent", ua)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
|
||||
resp, err := articleClient.Do(req)
|
||||
if err != nil {
|
||||
return ArticleMeta{}
|
||||
return ArticleMeta{FetchError: true}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
meta := ArticleMeta{Status: resp.StatusCode}
|
||||
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
|
||||
// We still try to parse the body for whatever signals it contains.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ArticleMeta{}
|
||||
if resp.StatusCode == http.StatusPaymentRequired {
|
||||
meta.Paywalled = true
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return ArticleMeta{}
|
||||
return meta
|
||||
}
|
||||
|
||||
return ArticleMeta{
|
||||
ImageURL: extractOGImage(doc, articleURL),
|
||||
BodyChars: extractBodyChars(doc),
|
||||
Fetched: true,
|
||||
Paywalled: detectPaywall(doc),
|
||||
}
|
||||
meta.Fetched = true
|
||||
meta.ImageURL = extractOGImage(doc, articleURL)
|
||||
meta.BodyChars = extractBodyChars(doc)
|
||||
meta.Paywalled = detectPaywall(doc)
|
||||
return meta
|
||||
}
|
||||
|
||||
// detectPaywall checks the page for explicit gating signals that publishers
|
||||
@@ -121,9 +195,11 @@ func detectPaywall(doc *goquery.Document) bool {
|
||||
return gated
|
||||
}
|
||||
|
||||
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
|
||||
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
|
||||
// structure. Publishers vary wildly in shape, so we walk generically.
|
||||
// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
|
||||
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
|
||||
// with "isAccessibleForFree" set falsy. We restrict to Article types so
|
||||
// that embedded related-content or breadcrumb markup doesn't flip a free
|
||||
// article to gated.
|
||||
func jsonLDDeclaresGated(raw string) bool {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
@@ -136,22 +212,28 @@ func jsonLDDeclaresGated(raw string) bool {
|
||||
return walkJSONLDForGated(v)
|
||||
}
|
||||
|
||||
// articleSchemaTypes are the schema.org @type values we treat as "the main
|
||||
// article" for paywall purposes.
|
||||
var articleSchemaTypes = map[string]bool{
|
||||
"article": true,
|
||||
"newsarticle": true,
|
||||
"report": true,
|
||||
"reportagenewsarticle": true,
|
||||
"blogposting": true,
|
||||
"scholarlyarticle": true,
|
||||
"techarticle": true,
|
||||
"opinionnewsarticle": true,
|
||||
"analysisnewsarticle": true,
|
||||
"backgroundnewsarticle": true,
|
||||
"reviewnewsarticle": true,
|
||||
}
|
||||
|
||||
func walkJSONLDForGated(v any) bool {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
if raw, ok := x["isAccessibleForFree"]; ok {
|
||||
switch r := raw.(type) {
|
||||
case bool:
|
||||
if !r {
|
||||
if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
|
||||
return true
|
||||
}
|
||||
case string:
|
||||
s := strings.ToLower(strings.TrimSpace(r))
|
||||
if s == "false" || s == "no" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, vv := range x {
|
||||
if walkJSONLDForGated(vv) {
|
||||
return true
|
||||
@@ -167,6 +249,31 @@ func walkJSONLDForGated(v any) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func isArticleType(v any) bool {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
|
||||
case []any:
|
||||
for _, e := range t {
|
||||
if isArticleType(e) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func declaresGated(v any) bool {
|
||||
switch r := v.(type) {
|
||||
case bool:
|
||||
return !r
|
||||
case string:
|
||||
s := strings.ToLower(strings.TrimSpace(r))
|
||||
return s == "false" || s == "no"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
|
||||
// that only care about the image. Returns "" when not found.
|
||||
func FetchOGImage(articleURL string) string {
|
||||
|
||||
@@ -126,32 +126,47 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
continue
|
||||
}
|
||||
|
||||
// Genuinely new story — fetch the article page once. We use it for:
|
||||
// Genuinely new story — fetch the article page once (with an internal
|
||||
// bypass-UA retry on gating signals). We use it for:
|
||||
// 1. og:image fallback when the feed didn't give us one.
|
||||
// 2. Paywall detection: visible body text below the threshold means
|
||||
// the article is gated, so swap in a Wayback snapshot for both
|
||||
// Pete (image) and the reader (posted link).
|
||||
// 2. Paywall detection: explicit gating signals or a body too short
|
||||
// to be a real article means we swap in an archive snapshot for
|
||||
// both Pete (image) and the reader (posted link).
|
||||
//
|
||||
// Pure transport failures (FetchError) are NOT treated as gating —
|
||||
// swapping in a stale snapshot for a transient blip is worse than
|
||||
// posting the live link.
|
||||
meta := FetchArticleMeta(originalURL)
|
||||
paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold
|
||||
if paywalled {
|
||||
if snap := ResolveWayback(originalURL); snap != "" {
|
||||
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
|
||||
// paywalled tracks whether the link the user will click is still
|
||||
// gated — i.e. gating was detected AND no archive workaround worked.
|
||||
// When a snapshot swap succeeds, the reader gets a readable page, so
|
||||
// we don't stamp it.
|
||||
paywalled := false
|
||||
if gated {
|
||||
workedAround := false
|
||||
if snap := resolveArchive(originalURL); snap != "" {
|
||||
snapMeta := FetchArticleMeta(snap)
|
||||
if snapMeta.Fetched {
|
||||
items[i].ArticleURL = snap
|
||||
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
|
||||
items[i].ImageURL = snapMeta.ImageURL
|
||||
}
|
||||
slog.Info("paywall detected, using wayback snapshot",
|
||||
workedAround = true
|
||||
slog.Info("paywall detected, using archive snapshot",
|
||||
"guid", items[i].GUID, "original", originalURL,
|
||||
"snapshot", snap, "body_chars", meta.BodyChars)
|
||||
"snapshot", snap, "body_chars", meta.BodyChars,
|
||||
"status", meta.Status)
|
||||
} else {
|
||||
slog.Debug("paywall detected but snapshot fetch failed",
|
||||
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
|
||||
}
|
||||
} else {
|
||||
slog.Debug("paywall detected but no wayback snapshot available",
|
||||
"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
|
||||
slog.Debug("paywall detected but no archive snapshot available",
|
||||
"guid", items[i].GUID, "url", originalURL,
|
||||
"body_chars", meta.BodyChars, "status", meta.Status)
|
||||
}
|
||||
paywalled = !workedAround
|
||||
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
|
||||
items[i].ImageURL = meta.ImageURL
|
||||
slog.Debug("og:image fallback used",
|
||||
@@ -172,6 +187,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
URLCanonical: canonical,
|
||||
HeadlineNorm: headlineNorm,
|
||||
Source: items[i].Source,
|
||||
Paywalled: paywalled,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}); err != nil {
|
||||
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
||||
@@ -188,3 +204,13 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveArchive tries Wayback first (well-behaved availability API, fresh
|
||||
// snapshots filtered) and falls back to archive.today, which often has
|
||||
// captures Wayback doesn't — especially for hard-walled publishers.
|
||||
func resolveArchive(articleURL string) string {
|
||||
if snap := ResolveWayback(articleURL); snap != "" {
|
||||
return snap
|
||||
}
|
||||
return ResolveArchiveToday(articleURL)
|
||||
}
|
||||
|
||||
@@ -5,29 +5,40 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
|
||||
// it as useless for a freshly-published news article. The "closest" snapshot
|
||||
// returned by the availability API can otherwise be years old.
|
||||
const maxSnapshotAge = 30 * 24 * time.Hour
|
||||
|
||||
type waybackResp struct {
|
||||
ArchivedSnapshots struct {
|
||||
Closest struct {
|
||||
Available bool `json:"available"`
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss
|
||||
} `json:"closest"`
|
||||
} `json:"archived_snapshots"`
|
||||
}
|
||||
|
||||
// ResolveWayback asks the Internet Archive's Wayback availability API for the
|
||||
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
|
||||
// if no snapshot exists or the request fails.
|
||||
// ResolveWayback asks the Internet Archive's Wayback availability API for
|
||||
// the closest snapshot of the given URL. Returns the snapshot URL (https)
|
||||
// or "" if no snapshot exists, the request fails, or the closest snapshot
|
||||
// is older than maxSnapshotAge.
|
||||
func ResolveWayback(articleURL string) string {
|
||||
if articleURL == "" {
|
||||
return ""
|
||||
}
|
||||
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
|
||||
// Anchor the lookup to "now" so we get the freshest snapshot rather
|
||||
// than whichever happens to be Wayback's default closest match.
|
||||
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) +
|
||||
"×tamp=" + time.Now().UTC().Format("20060102150405")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -54,9 +65,76 @@ func ResolveWayback(articleURL string) string {
|
||||
if !snap.Available || snap.Status != "200" || snap.URL == "" {
|
||||
return ""
|
||||
}
|
||||
if !snapshotFreshEnough(snap.Timestamp) {
|
||||
return ""
|
||||
}
|
||||
// Wayback sometimes returns http:// even when https is available.
|
||||
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
|
||||
if strings.HasPrefix(snap.URL, "http://") {
|
||||
return "https://" + snap.URL[7:]
|
||||
}
|
||||
return snap.URL
|
||||
}
|
||||
|
||||
func snapshotFreshEnough(ts string) bool {
|
||||
if ts == "" {
|
||||
return false
|
||||
}
|
||||
t, err := time.Parse("20060102150405", ts)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return time.Since(t) <= maxSnapshotAge
|
||||
}
|
||||
|
||||
var archiveTodayClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
// Don't follow the redirect — we just want the Location header pointing
|
||||
// at the most recent capture.
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
|
||||
// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
|
||||
// for the given URL. archive.ph has no public availability API, but its
|
||||
// `/newest/<url>` endpoint redirects (HTTP 302) to the most recent capture
|
||||
// when one exists, or returns a non-redirect response otherwise. Returns
|
||||
// "" on any failure or when no snapshot exists.
|
||||
func ResolveArchiveToday(articleURL string) string {
|
||||
if articleURL == "" {
|
||||
return ""
|
||||
}
|
||||
api := "https://archive.ph/newest/" + articleURL
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := archiveTodayClient.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 300 || resp.StatusCode >= 400 {
|
||||
return ""
|
||||
}
|
||||
loc := strings.TrimSpace(resp.Header.Get("Location"))
|
||||
if loc == "" {
|
||||
return ""
|
||||
}
|
||||
// archive.ph occasionally returns a relative Location; absolutize it.
|
||||
if strings.HasPrefix(loc, "/") {
|
||||
loc = "https://archive.ph" + loc
|
||||
}
|
||||
// Bounce back the input as Location means "no snapshot, here's the form"
|
||||
// — distinguish a real capture URL (contains /YYYY/ or a short hash path).
|
||||
if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") {
|
||||
return loc
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -80,6 +80,7 @@ func runMigrations(d *sql.DB) error {
|
||||
// we swallow that specifically.
|
||||
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
|
||||
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
|
||||
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
||||
|
||||
|
||||
@@ -28,10 +28,14 @@ func InsertStory(s *Story) error {
|
||||
if s.Classified {
|
||||
classified = 1
|
||||
}
|
||||
paywalled := 0
|
||||
if s.Paywalled {
|
||||
paywalled = 1
|
||||
}
|
||||
_, err := Get().Exec(
|
||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
|
||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -220,7 +224,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
// channels (_discarded, _duplicate) are excluded.
|
||||
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
@@ -234,7 +238,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -246,7 +250,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
||||
// channels, newest first. Sentinel channels are excluded.
|
||||
func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
@@ -261,7 +265,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -273,7 +277,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
// Matrix, ordered by post time (newest first). Posted is always true.
|
||||
func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
|
||||
FROM stories s
|
||||
JOIN post_log p ON p.guid = s.guid
|
||||
GROUP BY s.guid
|
||||
@@ -286,7 +290,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Posted = true
|
||||
|
||||
@@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS stories (
|
||||
platforms TEXT,
|
||||
channel TEXT,
|
||||
classified INTEGER NOT NULL DEFAULT 0,
|
||||
paywalled INTEGER NOT NULL DEFAULT 0,
|
||||
seen_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ type Story struct {
|
||||
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
|
||||
Channel string
|
||||
Classified bool
|
||||
Paywalled bool // source article is gated; ArticleURL may be an archive snapshot
|
||||
SeenAt int64
|
||||
Posted bool // true if this story has been posted to Matrix (joined from post_log)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ type StoryView struct {
|
||||
Source string
|
||||
SeenAt time.Time
|
||||
Posted bool
|
||||
Paywalled bool // source is gated and no archive workaround succeeded
|
||||
Channel string // channel slug; also the theme key
|
||||
}
|
||||
|
||||
@@ -34,6 +35,7 @@ func toView(s storage.Story) StoryView {
|
||||
Source: s.Source,
|
||||
SeenAt: time.Unix(s.SeenAt, 0),
|
||||
Posted: s.Posted,
|
||||
Paywalled: s.Paywalled,
|
||||
Channel: s.Channel,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,4 +92,36 @@ html[data-phase="night"] {
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* "PAYWALLED" diagonal rubber stamp overlay — slapped across the whole
|
||||
card when the source is gated and no archive snapshot worked. The card
|
||||
stays clickable (link still goes to the original); the stamp is just
|
||||
visual warning. pointer-events:none keeps it from eating clicks. */
|
||||
.paywall-stamp {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
pointer-events: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 5;
|
||||
background: rgba(255, 250, 240, 0.18);
|
||||
}
|
||||
.paywall-stamp::before {
|
||||
content: "PAYWALLED";
|
||||
font-family: var(--font-display, ui-sans-serif), system-ui, sans-serif;
|
||||
font-weight: 900;
|
||||
font-size: clamp(1rem, 3.2vw, 1.75rem);
|
||||
letter-spacing: 0.12em;
|
||||
color: rgba(180, 30, 30, 0.85);
|
||||
border: 0.28rem double rgba(180, 30, 30, 0.85);
|
||||
padding: 0.3rem 0.9rem;
|
||||
transform: rotate(-15deg);
|
||||
text-shadow: 0 1px 0 rgba(255,255,255,0.4);
|
||||
box-shadow: inset 0 0 0 2px rgba(255,255,255,0.15);
|
||||
background: rgba(255, 240, 230, 0.55);
|
||||
border-radius: 0.25rem;
|
||||
/* slight roughness to mimic a real ink stamp */
|
||||
filter: contrast(1.05);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
|
||||
{{define "card"}}
|
||||
<a href="{{.Story.ArticleURL}}" target="_blank" rel="noopener noreferrer"
|
||||
class="group block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
|
||||
class="group relative block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
|
||||
{{if .Story.ImageURL}}
|
||||
<div class="aspect-[16/10] w-full overflow-hidden bg-[color:var(--ink)]/5">
|
||||
<img src="{{thumb .Story.ImageURL}}" alt="" loading="lazy" decoding="async"
|
||||
@@ -28,5 +28,8 @@
|
||||
<p class="text-sm text-[color:var(--ink)]/70 line-clamp-3">{{.Story.Lede}}</p>
|
||||
{{end}}
|
||||
</div>
|
||||
{{if .Story.Paywalled}}
|
||||
<div class="paywall-stamp" aria-label="Paywalled article"></div>
|
||||
{{end}}
|
||||
</a>
|
||||
{{end}}
|
||||
|
||||
Reference in New Issue
Block a user