Paywall signal detection, sqlite driver registration, /img route fix

- Detect explicit paywall markers (article:content_tier meta, JSON-LD
  isAccessibleForFree) so metered articles fall back to Wayback even
  when body length is above threshold
- Blank-import go.mau.fi/util/dbutil/litestream so the sqlite3-fk-wal
  driver mautrix's cryptohelper depends on is registered
- Fix /img route: Go ServeMux requires {wildcard} to be a whole segment,
  so capture {name} and strip the .avif suffix in the handler
This commit is contained in:
prosolis
2026-05-25 07:50:36 -07:00
parent bddd15f7d1
commit 5ea64c1d32
5 changed files with 82 additions and 3 deletions

View File

@@ -2,6 +2,7 @@ package ingestion
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
@@ -52,6 +53,7 @@ type ArticleMeta struct {
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
Paywalled bool // true if the page explicitly declares gated access
}
var articleClient = &http.Client{Timeout: 12 * time.Second}
@@ -92,9 +94,79 @@ func FetchArticleMeta(articleURL string) ArticleMeta {
ImageURL: extractOGImage(doc, articleURL),
BodyChars: extractBodyChars(doc),
Fetched: true,
Paywalled: detectPaywall(doc),
}
}
// detectPaywall checks the page for explicit gating signals that publishers
// expose for Google News and crawlers. We treat the article as paywalled if:
// - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
// Hearst, many WordPress VIP sites including Wired)
// - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by
// NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers)
func detectPaywall(doc *goquery.Document) bool {
tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content")
switch strings.ToLower(strings.TrimSpace(tier)) {
case "metered", "locked":
return true
}
gated := false
doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
if jsonLDDeclaresGated(s.Text()) {
gated = true
return false
}
return true
})
return gated
}
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
// structure. Publishers vary wildly in shape, so we walk generically.
func jsonLDDeclaresGated(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
return false
}
var v any
if err := json.Unmarshal([]byte(raw), &v); err != nil {
return false
}
return walkJSONLDForGated(v)
}
func walkJSONLDForGated(v any) bool {
switch x := v.(type) {
case map[string]any:
if raw, ok := x["isAccessibleForFree"]; ok {
switch r := raw.(type) {
case bool:
if !r {
return true
}
case string:
s := strings.ToLower(strings.TrimSpace(r))
if s == "false" || s == "no" {
return true
}
}
}
for _, vv := range x {
if walkJSONLDForGated(vv) {
return true
}
}
case []any:
for _, vv := range x {
if walkJSONLDForGated(vv) {
return true
}
}
}
return false
}
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {