Harden paywall detection + render paywalled stamp on cards
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD gating scoped to Article-typed nodes, HTTP 402 treated as explicit paywall, Wayback freshness filter (30d cap), archive.today as secondary archive fallback, and transport failures no longer trigger snapshot swaps. When gating is detected and no archive workaround succeeds, the story is stored with paywalled=1 and the web card renders a diagonal red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
@@ -48,54 +48,128 @@ func resolveURL(base, ref string) string {
|
|||||||
// as paywalled / gated, and the caller should fall back to an archive snapshot.
|
// as paywalled / gated, and the caller should fall back to an archive snapshot.
|
||||||
const PaywallBodyThreshold = 500
|
const PaywallBodyThreshold = 500
|
||||||
|
|
||||||
|
// googlebotUA is what many metered publishers grant first-click access to.
|
||||||
|
// Re-tried automatically when the default-UA fetch looks gated.
|
||||||
|
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||||
|
|
||||||
// ArticleMeta is what we can learn from fetching an article page directly.
|
// ArticleMeta is what we can learn from fetching an article page directly.
|
||||||
type ArticleMeta struct {
|
type ArticleMeta struct {
|
||||||
ImageURL string // og:image or twitter:image, absolute URL
|
ImageURL string // og:image or twitter:image, absolute URL
|
||||||
BodyChars int // length of extracted visible body text
|
BodyChars int // length of extracted visible body text
|
||||||
Fetched bool // true if we got an HTTP 200 with HTML
|
Fetched bool // true if we got an HTTP 200 with HTML
|
||||||
|
FetchError bool // true if the fetch failed at the network/HTTP layer
|
||||||
Paywalled bool // true if the page explicitly declares gated access
|
Paywalled bool // true if the page explicitly declares gated access
|
||||||
|
Status int // last HTTP status seen (0 on transport error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gated reports whether the response carries a strong gating signal: an
|
||||||
|
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
|
||||||
|
// or a 403 Forbidden after the bypass retry. A short body alone is not
|
||||||
|
// considered gating — that's a heuristic used by callers separately.
|
||||||
|
func (m ArticleMeta) Gated() bool {
|
||||||
|
if m.Paywalled {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
|
||||||
}
|
}
|
||||||
|
|
||||||
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
||||||
|
|
||||||
// FetchArticleMeta fetches an article URL and returns its og:image plus the
|
// FetchArticleMeta fetches an article URL with the default UA. If the result
|
||||||
// length of the visible body text (concatenation of <p> tags under <article>
|
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
|
||||||
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
|
// retries once with a Googlebot UA + Google referer — the combination most
|
||||||
// network/HTTP failure so callers can branch on accessibility.
|
// metered publishers grant first-click access to. Returns the best of the
|
||||||
|
// two attempts.
|
||||||
func FetchArticleMeta(articleURL string) ArticleMeta {
|
func FetchArticleMeta(articleURL string) ArticleMeta {
|
||||||
if articleURL == "" {
|
if articleURL == "" {
|
||||||
return ArticleMeta{}
|
return ArticleMeta{}
|
||||||
}
|
}
|
||||||
|
first := fetchArticleMetaOnce(articleURL, userAgent, "")
|
||||||
|
if !shouldRetryAsBot(first) {
|
||||||
|
return first
|
||||||
|
}
|
||||||
|
second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
|
||||||
|
return pickBetter(first, second)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shouldRetryAsBot returns true when the first attempt looks gated or too
|
||||||
|
// thin to be the real article body.
|
||||||
|
func shouldRetryAsBot(m ArticleMeta) bool {
|
||||||
|
if m.FetchError {
|
||||||
|
return false // transport failure won't be fixed by a different UA
|
||||||
|
}
|
||||||
|
if m.Gated() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if m.Fetched && m.BodyChars < PaywallBodyThreshold {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// pickBetter chooses the more useful of two fetch attempts: prefer the one
|
||||||
|
// that isn't gated, then the one with more body, then the one that fetched
|
||||||
|
// at all.
|
||||||
|
func pickBetter(a, b ArticleMeta) ArticleMeta {
|
||||||
|
aGated, bGated := a.Gated(), b.Gated()
|
||||||
|
if aGated != bGated {
|
||||||
|
if bGated {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
if a.Fetched != b.Fetched {
|
||||||
|
if b.Fetched {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
if b.BodyChars > a.BodyChars {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArticleMeta{}
|
return ArticleMeta{FetchError: true}
|
||||||
}
|
}
|
||||||
req.Header.Set("User-Agent", userAgent)
|
req.Header.Set("User-Agent", ua)
|
||||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||||
|
if referer != "" {
|
||||||
|
req.Header.Set("Referer", referer)
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := articleClient.Do(req)
|
resp, err := articleClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArticleMeta{}
|
return ArticleMeta{FetchError: true}
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
meta := ArticleMeta{Status: resp.StatusCode}
|
||||||
|
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
|
||||||
|
// We still try to parse the body for whatever signals it contains.
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return ArticleMeta{}
|
if resp.StatusCode == http.StatusPaymentRequired {
|
||||||
|
meta.Paywalled = true
|
||||||
|
}
|
||||||
|
return meta
|
||||||
}
|
}
|
||||||
|
|
||||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArticleMeta{}
|
return meta
|
||||||
}
|
}
|
||||||
|
|
||||||
return ArticleMeta{
|
meta.Fetched = true
|
||||||
ImageURL: extractOGImage(doc, articleURL),
|
meta.ImageURL = extractOGImage(doc, articleURL)
|
||||||
BodyChars: extractBodyChars(doc),
|
meta.BodyChars = extractBodyChars(doc)
|
||||||
Fetched: true,
|
meta.Paywalled = detectPaywall(doc)
|
||||||
Paywalled: detectPaywall(doc),
|
return meta
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// detectPaywall checks the page for explicit gating signals that publishers
|
// detectPaywall checks the page for explicit gating signals that publishers
|
||||||
@@ -121,9 +195,11 @@ func detectPaywall(doc *goquery.Document) bool {
|
|||||||
return gated
|
return gated
|
||||||
}
|
}
|
||||||
|
|
||||||
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
|
// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
|
||||||
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
|
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
|
||||||
// structure. Publishers vary wildly in shape, so we walk generically.
|
// with "isAccessibleForFree" set falsy. We restrict to Article types so
|
||||||
|
// that embedded related-content or breadcrumb markup doesn't flip a free
|
||||||
|
// article to gated.
|
||||||
func jsonLDDeclaresGated(raw string) bool {
|
func jsonLDDeclaresGated(raw string) bool {
|
||||||
raw = strings.TrimSpace(raw)
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
@@ -136,22 +212,28 @@ func jsonLDDeclaresGated(raw string) bool {
|
|||||||
return walkJSONLDForGated(v)
|
return walkJSONLDForGated(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// articleSchemaTypes are the schema.org @type values we treat as "the main
|
||||||
|
// article" for paywall purposes.
|
||||||
|
var articleSchemaTypes = map[string]bool{
|
||||||
|
"article": true,
|
||||||
|
"newsarticle": true,
|
||||||
|
"report": true,
|
||||||
|
"reportagenewsarticle": true,
|
||||||
|
"blogposting": true,
|
||||||
|
"scholarlyarticle": true,
|
||||||
|
"techarticle": true,
|
||||||
|
"opinionnewsarticle": true,
|
||||||
|
"analysisnewsarticle": true,
|
||||||
|
"backgroundnewsarticle": true,
|
||||||
|
"reviewnewsarticle": true,
|
||||||
|
}
|
||||||
|
|
||||||
func walkJSONLDForGated(v any) bool {
|
func walkJSONLDForGated(v any) bool {
|
||||||
switch x := v.(type) {
|
switch x := v.(type) {
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
if raw, ok := x["isAccessibleForFree"]; ok {
|
if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
|
||||||
switch r := raw.(type) {
|
|
||||||
case bool:
|
|
||||||
if !r {
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
case string:
|
|
||||||
s := strings.ToLower(strings.TrimSpace(r))
|
|
||||||
if s == "false" || s == "no" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, vv := range x {
|
for _, vv := range x {
|
||||||
if walkJSONLDForGated(vv) {
|
if walkJSONLDForGated(vv) {
|
||||||
return true
|
return true
|
||||||
@@ -167,6 +249,31 @@ func walkJSONLDForGated(v any) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isArticleType(v any) bool {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case string:
|
||||||
|
return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
|
||||||
|
case []any:
|
||||||
|
for _, e := range t {
|
||||||
|
if isArticleType(e) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func declaresGated(v any) bool {
|
||||||
|
switch r := v.(type) {
|
||||||
|
case bool:
|
||||||
|
return !r
|
||||||
|
case string:
|
||||||
|
s := strings.ToLower(strings.TrimSpace(r))
|
||||||
|
return s == "false" || s == "no"
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
|
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
|
||||||
// that only care about the image. Returns "" when not found.
|
// that only care about the image. Returns "" when not found.
|
||||||
func FetchOGImage(articleURL string) string {
|
func FetchOGImage(articleURL string) string {
|
||||||
|
|||||||
@@ -126,32 +126,47 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Genuinely new story — fetch the article page once. We use it for:
|
// Genuinely new story — fetch the article page once (with an internal
|
||||||
|
// bypass-UA retry on gating signals). We use it for:
|
||||||
// 1. og:image fallback when the feed didn't give us one.
|
// 1. og:image fallback when the feed didn't give us one.
|
||||||
// 2. Paywall detection: visible body text below the threshold means
|
// 2. Paywall detection: explicit gating signals or a body too short
|
||||||
// the article is gated, so swap in a Wayback snapshot for both
|
// to be a real article means we swap in an archive snapshot for
|
||||||
// Pete (image) and the reader (posted link).
|
// both Pete (image) and the reader (posted link).
|
||||||
|
//
|
||||||
|
// Pure transport failures (FetchError) are NOT treated as gating —
|
||||||
|
// swapping in a stale snapshot for a transient blip is worse than
|
||||||
|
// posting the live link.
|
||||||
meta := FetchArticleMeta(originalURL)
|
meta := FetchArticleMeta(originalURL)
|
||||||
paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold
|
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
|
||||||
if paywalled {
|
// paywalled tracks whether the link the user will click is still
|
||||||
if snap := ResolveWayback(originalURL); snap != "" {
|
// gated — i.e. gating was detected AND no archive workaround worked.
|
||||||
|
// When a snapshot swap succeeds, the reader gets a readable page, so
|
||||||
|
// we don't stamp it.
|
||||||
|
paywalled := false
|
||||||
|
if gated {
|
||||||
|
workedAround := false
|
||||||
|
if snap := resolveArchive(originalURL); snap != "" {
|
||||||
snapMeta := FetchArticleMeta(snap)
|
snapMeta := FetchArticleMeta(snap)
|
||||||
if snapMeta.Fetched {
|
if snapMeta.Fetched {
|
||||||
items[i].ArticleURL = snap
|
items[i].ArticleURL = snap
|
||||||
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
|
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
|
||||||
items[i].ImageURL = snapMeta.ImageURL
|
items[i].ImageURL = snapMeta.ImageURL
|
||||||
}
|
}
|
||||||
slog.Info("paywall detected, using wayback snapshot",
|
workedAround = true
|
||||||
|
slog.Info("paywall detected, using archive snapshot",
|
||||||
"guid", items[i].GUID, "original", originalURL,
|
"guid", items[i].GUID, "original", originalURL,
|
||||||
"snapshot", snap, "body_chars", meta.BodyChars)
|
"snapshot", snap, "body_chars", meta.BodyChars,
|
||||||
|
"status", meta.Status)
|
||||||
} else {
|
} else {
|
||||||
slog.Debug("paywall detected but snapshot fetch failed",
|
slog.Debug("paywall detected but snapshot fetch failed",
|
||||||
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
|
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
slog.Debug("paywall detected but no wayback snapshot available",
|
slog.Debug("paywall detected but no archive snapshot available",
|
||||||
"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
|
"guid", items[i].GUID, "url", originalURL,
|
||||||
|
"body_chars", meta.BodyChars, "status", meta.Status)
|
||||||
}
|
}
|
||||||
|
paywalled = !workedAround
|
||||||
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
|
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
|
||||||
items[i].ImageURL = meta.ImageURL
|
items[i].ImageURL = meta.ImageURL
|
||||||
slog.Debug("og:image fallback used",
|
slog.Debug("og:image fallback used",
|
||||||
@@ -172,6 +187,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
|||||||
URLCanonical: canonical,
|
URLCanonical: canonical,
|
||||||
HeadlineNorm: headlineNorm,
|
HeadlineNorm: headlineNorm,
|
||||||
Source: items[i].Source,
|
Source: items[i].Source,
|
||||||
|
Paywalled: paywalled,
|
||||||
SeenAt: time.Now().Unix(),
|
SeenAt: time.Now().Unix(),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
||||||
@@ -188,3 +204,13 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveArchive tries Wayback first (well-behaved availability API, fresh
|
||||||
|
// snapshots filtered) and falls back to archive.today, which often has
|
||||||
|
// captures Wayback doesn't — especially for hard-walled publishers.
|
||||||
|
func resolveArchive(articleURL string) string {
|
||||||
|
if snap := ResolveWayback(articleURL); snap != "" {
|
||||||
|
return snap
|
||||||
|
}
|
||||||
|
return ResolveArchiveToday(articleURL)
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,29 +5,40 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
||||||
|
|
||||||
|
// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
|
||||||
|
// it as useless for a freshly-published news article. The "closest" snapshot
|
||||||
|
// returned by the availability API can otherwise be years old.
|
||||||
|
const maxSnapshotAge = 30 * 24 * time.Hour
|
||||||
|
|
||||||
type waybackResp struct {
|
type waybackResp struct {
|
||||||
ArchivedSnapshots struct {
|
ArchivedSnapshots struct {
|
||||||
Closest struct {
|
Closest struct {
|
||||||
Available bool `json:"available"`
|
Available bool `json:"available"`
|
||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
|
Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss
|
||||||
} `json:"closest"`
|
} `json:"closest"`
|
||||||
} `json:"archived_snapshots"`
|
} `json:"archived_snapshots"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResolveWayback asks the Internet Archive's Wayback availability API for the
|
// ResolveWayback asks the Internet Archive's Wayback availability API for
|
||||||
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
|
// the closest snapshot of the given URL. Returns the snapshot URL (https)
|
||||||
// if no snapshot exists or the request fails.
|
// or "" if no snapshot exists, the request fails, or the closest snapshot
|
||||||
|
// is older than maxSnapshotAge.
|
||||||
func ResolveWayback(articleURL string) string {
|
func ResolveWayback(articleURL string) string {
|
||||||
if articleURL == "" {
|
if articleURL == "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
|
// Anchor the lookup to "now" so we get the freshest snapshot rather
|
||||||
|
// than whichever happens to be Wayback's default closest match.
|
||||||
|
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) +
|
||||||
|
"×tamp=" + time.Now().UTC().Format("20060102150405")
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
@@ -54,9 +65,76 @@ func ResolveWayback(articleURL string) string {
|
|||||||
if !snap.Available || snap.Status != "200" || snap.URL == "" {
|
if !snap.Available || snap.Status != "200" || snap.URL == "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
if !snapshotFreshEnough(snap.Timestamp) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
// Wayback sometimes returns http:// even when https is available.
|
// Wayback sometimes returns http:// even when https is available.
|
||||||
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
|
if strings.HasPrefix(snap.URL, "http://") {
|
||||||
return "https://" + snap.URL[7:]
|
return "https://" + snap.URL[7:]
|
||||||
}
|
}
|
||||||
return snap.URL
|
return snap.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func snapshotFreshEnough(ts string) bool {
|
||||||
|
if ts == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
t, err := time.Parse("20060102150405", ts)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return time.Since(t) <= maxSnapshotAge
|
||||||
|
}
|
||||||
|
|
||||||
|
var archiveTodayClient = &http.Client{
|
||||||
|
Timeout: 10 * time.Second,
|
||||||
|
// Don't follow the redirect — we just want the Location header pointing
|
||||||
|
// at the most recent capture.
|
||||||
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
|
||||||
|
// for the given URL. archive.ph has no public availability API, but its
|
||||||
|
// `/newest/<url>` endpoint redirects (HTTP 302) to the most recent capture
|
||||||
|
// when one exists, or returns a non-redirect response otherwise. Returns
|
||||||
|
// "" on any failure or when no snapshot exists.
|
||||||
|
func ResolveArchiveToday(articleURL string) string {
|
||||||
|
if articleURL == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
api := "https://archive.ph/newest/" + articleURL
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", userAgent)
|
||||||
|
|
||||||
|
resp, err := archiveTodayClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode < 300 || resp.StatusCode >= 400 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
loc := strings.TrimSpace(resp.Header.Get("Location"))
|
||||||
|
if loc == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// archive.ph occasionally returns a relative Location; absolutize it.
|
||||||
|
if strings.HasPrefix(loc, "/") {
|
||||||
|
loc = "https://archive.ph" + loc
|
||||||
|
}
|
||||||
|
// Bounce back the input as Location means "no snapshot, here's the form"
|
||||||
|
// — distinguish a real capture URL (contains /YYYY/ or a short hash path).
|
||||||
|
if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") {
|
||||||
|
return loc
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ func runMigrations(d *sql.DB) error {
|
|||||||
// we swallow that specifically.
|
// we swallow that specifically.
|
||||||
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
|
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
|
||||||
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
|
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
|
||||||
|
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
|
||||||
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
||||||
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
||||||
|
|
||||||
|
|||||||
@@ -28,10 +28,14 @@ func InsertStory(s *Story) error {
|
|||||||
if s.Classified {
|
if s.Classified {
|
||||||
classified = 1
|
classified = 1
|
||||||
}
|
}
|
||||||
|
paywalled := 0
|
||||||
|
if s.Paywalled {
|
||||||
|
paywalled = 1
|
||||||
|
}
|
||||||
_, err := Get().Exec(
|
_, err := Get().Exec(
|
||||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
|
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
|
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -220,7 +224,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
|||||||
// channels (_discarded, _duplicate) are excluded.
|
// channels (_discarded, _duplicate) are excluded.
|
||||||
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
|
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
|
||||||
rows, err := Get().Query(
|
rows, err := Get().Query(
|
||||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
|
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||||
FROM stories s
|
FROM stories s
|
||||||
WHERE s.classified = 1
|
WHERE s.classified = 1
|
||||||
@@ -234,7 +238,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
|||||||
var out []Story
|
var out []Story
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var s Story
|
var s Story
|
||||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
|
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
out = append(out, s)
|
out = append(out, s)
|
||||||
@@ -246,7 +250,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
|||||||
// channels, newest first. Sentinel channels are excluded.
|
// channels, newest first. Sentinel channels are excluded.
|
||||||
func ListAllClassified(limit, offset int) ([]Story, error) {
|
func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||||
rows, err := Get().Query(
|
rows, err := Get().Query(
|
||||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
|
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||||
FROM stories s
|
FROM stories s
|
||||||
WHERE s.classified = 1
|
WHERE s.classified = 1
|
||||||
@@ -261,7 +265,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
|||||||
var out []Story
|
var out []Story
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var s Story
|
var s Story
|
||||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
|
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
out = append(out, s)
|
out = append(out, s)
|
||||||
@@ -273,7 +277,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
|||||||
// Matrix, ordered by post time (newest first). Posted is always true.
|
// Matrix, ordered by post time (newest first). Posted is always true.
|
||||||
func ListRecentlyPosted(limit int) ([]Story, error) {
|
func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||||
rows, err := Get().Query(
|
rows, err := Get().Query(
|
||||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at
|
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
|
||||||
FROM stories s
|
FROM stories s
|
||||||
JOIN post_log p ON p.guid = s.guid
|
JOIN post_log p ON p.guid = s.guid
|
||||||
GROUP BY s.guid
|
GROUP BY s.guid
|
||||||
@@ -286,7 +290,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
|
|||||||
var out []Story
|
var out []Story
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var s Story
|
var s Story
|
||||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
s.Posted = true
|
s.Posted = true
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS stories (
|
|||||||
platforms TEXT,
|
platforms TEXT,
|
||||||
channel TEXT,
|
channel TEXT,
|
||||||
classified INTEGER NOT NULL DEFAULT 0,
|
classified INTEGER NOT NULL DEFAULT 0,
|
||||||
|
paywalled INTEGER NOT NULL DEFAULT 0,
|
||||||
seen_at INTEGER NOT NULL
|
seen_at INTEGER NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ type Story struct {
|
|||||||
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
|
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
|
||||||
Channel string
|
Channel string
|
||||||
Classified bool
|
Classified bool
|
||||||
|
Paywalled bool // source article is gated; ArticleURL may be an archive snapshot
|
||||||
SeenAt int64
|
SeenAt int64
|
||||||
Posted bool // true if this story has been posted to Matrix (joined from post_log)
|
Posted bool // true if this story has been posted to Matrix (joined from post_log)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ type StoryView struct {
|
|||||||
Source string
|
Source string
|
||||||
SeenAt time.Time
|
SeenAt time.Time
|
||||||
Posted bool
|
Posted bool
|
||||||
|
Paywalled bool // source is gated and no archive workaround succeeded
|
||||||
Channel string // channel slug; also the theme key
|
Channel string // channel slug; also the theme key
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,6 +35,7 @@ func toView(s storage.Story) StoryView {
|
|||||||
Source: s.Source,
|
Source: s.Source,
|
||||||
SeenAt: time.Unix(s.SeenAt, 0),
|
SeenAt: time.Unix(s.SeenAt, 0),
|
||||||
Posted: s.Posted,
|
Posted: s.Posted,
|
||||||
|
Paywalled: s.Paywalled,
|
||||||
Channel: s.Channel,
|
Channel: s.Channel,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,4 +92,36 @@ html[data-phase="night"] {
|
|||||||
-webkit-box-orient: vertical;
|
-webkit-box-orient: vertical;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* "PAYWALLED" diagonal rubber stamp overlay — slapped across the whole
|
||||||
|
card when the source is gated and no archive snapshot worked. The card
|
||||||
|
stays clickable (link still goes to the original); the stamp is just
|
||||||
|
visual warning. pointer-events:none keeps it from eating clicks. */
|
||||||
|
.paywall-stamp {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
pointer-events: none;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
z-index: 5;
|
||||||
|
background: rgba(255, 250, 240, 0.18);
|
||||||
|
}
|
||||||
|
.paywall-stamp::before {
|
||||||
|
content: "PAYWALLED";
|
||||||
|
font-family: var(--font-display, ui-sans-serif), system-ui, sans-serif;
|
||||||
|
font-weight: 900;
|
||||||
|
font-size: clamp(1rem, 3.2vw, 1.75rem);
|
||||||
|
letter-spacing: 0.12em;
|
||||||
|
color: rgba(180, 30, 30, 0.85);
|
||||||
|
border: 0.28rem double rgba(180, 30, 30, 0.85);
|
||||||
|
padding: 0.3rem 0.9rem;
|
||||||
|
transform: rotate(-15deg);
|
||||||
|
text-shadow: 0 1px 0 rgba(255,255,255,0.4);
|
||||||
|
box-shadow: inset 0 0 0 2px rgba(255,255,255,0.15);
|
||||||
|
background: rgba(255, 240, 230, 0.55);
|
||||||
|
border-radius: 0.25rem;
|
||||||
|
/* slight roughness to mimic a real ink stamp */
|
||||||
|
filter: contrast(1.05);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
|
|||||||
{{define "card"}}
|
{{define "card"}}
|
||||||
<a href="{{.Story.ArticleURL}}" target="_blank" rel="noopener noreferrer"
|
<a href="{{.Story.ArticleURL}}" target="_blank" rel="noopener noreferrer"
|
||||||
class="group block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
|
class="group relative block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
|
||||||
{{if .Story.ImageURL}}
|
{{if .Story.ImageURL}}
|
||||||
<div class="aspect-[16/10] w-full overflow-hidden bg-[color:var(--ink)]/5">
|
<div class="aspect-[16/10] w-full overflow-hidden bg-[color:var(--ink)]/5">
|
||||||
<img src="{{thumb .Story.ImageURL}}" alt="" loading="lazy" decoding="async"
|
<img src="{{thumb .Story.ImageURL}}" alt="" loading="lazy" decoding="async"
|
||||||
@@ -28,5 +28,8 @@
|
|||||||
<p class="text-sm text-[color:var(--ink)]/70 line-clamp-3">{{.Story.Lede}}</p>
|
<p class="text-sm text-[color:var(--ink)]/70 line-clamp-3">{{.Story.Lede}}</p>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
{{if .Story.Paywalled}}
|
||||||
|
<div class="paywall-stamp" aria-label="Paywalled article"></div>
|
||||||
|
{{end}}
|
||||||
</a>
|
</a>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|||||||
Reference in New Issue
Block a user