Paywall signal detection, sqlite driver registration, /img route fix
- Detect explicit paywall markers (article:content_tier meta, JSON-LD
isAccessibleForFree) so metered articles fall back to Wayback even
when body length is above threshold
- Blank-import go.mau.fi/util/dbutil/litestream so the sqlite3-fk-wal
driver mautrix's cryptohelper depends on is registered
- Fix /img route: Go ServeMux requires {wildcard} to be a whole segment,
so capture {name} and strip the .avif suffix in the handler
This commit is contained in:
@@ -2,6 +2,7 @@ package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -52,6 +53,7 @@ type ArticleMeta struct {
|
||||
ImageURL string // og:image or twitter:image, absolute URL
|
||||
BodyChars int // length of extracted visible body text
|
||||
Fetched bool // true if we got an HTTP 200 with HTML
|
||||
Paywalled bool // true if the page explicitly declares gated access
|
||||
}
|
||||
|
||||
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
||||
@@ -92,9 +94,79 @@ func FetchArticleMeta(articleURL string) ArticleMeta {
|
||||
ImageURL: extractOGImage(doc, articleURL),
|
||||
BodyChars: extractBodyChars(doc),
|
||||
Fetched: true,
|
||||
Paywalled: detectPaywall(doc),
|
||||
}
|
||||
}
|
||||
|
||||
// detectPaywall checks the page for explicit gating signals that publishers
|
||||
// expose for Google News and crawlers. We treat the article as paywalled if:
|
||||
// - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
|
||||
// Hearst, many WordPress VIP sites including Wired)
|
||||
// - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by
|
||||
// NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers)
|
||||
func detectPaywall(doc *goquery.Document) bool {
|
||||
tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content")
|
||||
switch strings.ToLower(strings.TrimSpace(tier)) {
|
||||
case "metered", "locked":
|
||||
return true
|
||||
}
|
||||
gated := false
|
||||
doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
|
||||
if jsonLDDeclaresGated(s.Text()) {
|
||||
gated = true
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return gated
|
||||
}
|
||||
|
||||
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
|
||||
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
|
||||
// structure. Publishers vary wildly in shape, so we walk generically.
|
||||
func jsonLDDeclaresGated(raw string) bool {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal([]byte(raw), &v); err != nil {
|
||||
return false
|
||||
}
|
||||
return walkJSONLDForGated(v)
|
||||
}
|
||||
|
||||
func walkJSONLDForGated(v any) bool {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
if raw, ok := x["isAccessibleForFree"]; ok {
|
||||
switch r := raw.(type) {
|
||||
case bool:
|
||||
if !r {
|
||||
return true
|
||||
}
|
||||
case string:
|
||||
s := strings.ToLower(strings.TrimSpace(r))
|
||||
if s == "false" || s == "no" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, vv := range x {
|
||||
if walkJSONLDForGated(vv) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, vv := range x {
|
||||
if walkJSONLDForGated(vv) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
|
||||
// that only care about the image. Returns "" when not found.
|
||||
func FetchOGImage(articleURL string) string {
|
||||
|
||||
Reference in New Issue
Block a user