diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go index 07b2501..482eef0 100644 --- a/internal/ingestion/article.go +++ b/internal/ingestion/article.go @@ -2,6 +2,7 @@ package ingestion import ( "context" + "encoding/json" "fmt" "net/http" "strings" @@ -52,6 +53,7 @@ type ArticleMeta struct { ImageURL string // og:image or twitter:image, absolute URL BodyChars int // length of extracted visible body text Fetched bool // true if we got an HTTP 200 with HTML + Paywalled bool // true if the page explicitly declares gated access } var articleClient = &http.Client{Timeout: 12 * time.Second} @@ -92,9 +94,79 @@ func FetchArticleMeta(articleURL string) ArticleMeta { ImageURL: extractOGImage(doc, articleURL), BodyChars: extractBodyChars(doc), Fetched: true, + Paywalled: detectPaywall(doc), } } +// detectPaywall checks the page for explicit gating signals that publishers +// expose for Google News and crawlers. We treat the article as paywalled if: +// - (Conde Nast, +// Hearst, many WordPress VIP sites including Wired) +// - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by +// NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers) +func detectPaywall(doc *goquery.Document) bool { + tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content") + switch strings.ToLower(strings.TrimSpace(tier)) { + case "metered", "locked": + return true + } + gated := false + doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool { + if jsonLDDeclaresGated(s.Text()) { + gated = true + return false + } + return true + }) + return gated +} + +// jsonLDDeclaresGated returns true if the JSON-LD payload contains +// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed) +// structure. Publishers vary wildly in shape, so we walk generically. +func jsonLDDeclaresGated(raw string) bool { + raw = strings.TrimSpace(raw) + if raw == "" { + return false + } + var v any + if err := json.Unmarshal([]byte(raw), &v); err != nil { + return false + } + return walkJSONLDForGated(v) +} + +func walkJSONLDForGated(v any) bool { + switch x := v.(type) { + case map[string]any: + if raw, ok := x["isAccessibleForFree"]; ok { + switch r := raw.(type) { + case bool: + if !r { + return true + } + case string: + s := strings.ToLower(strings.TrimSpace(r)) + if s == "false" || s == "no" { + return true + } + } + } + for _, vv := range x { + if walkJSONLDForGated(vv) { + return true + } + } + case []any: + for _, vv := range x { + if walkJSONLDForGated(vv) { + return true + } + } + } + return false +} + // FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers // that only care about the image. Returns "" when not found. func FetchOGImage(articleURL string) string { diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go index a6bc13e..82de688 100644 --- a/internal/ingestion/poller.go +++ b/internal/ingestion/poller.go @@ -132,7 +132,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e // the article is gated, so swap in a Wayback snapshot for both // Pete (image) and the reader (posted link). meta := FetchArticleMeta(originalURL) - paywalled := !meta.Fetched || meta.BodyChars < PaywallBodyThreshold + paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold if paywalled { if snap := ResolveWayback(originalURL); snap != "" { snapMeta := FetchArticleMeta(snap) diff --git a/internal/matrix/client.go b/internal/matrix/client.go index 865de52..f045c43 100644 --- a/internal/matrix/client.go +++ b/internal/matrix/client.go @@ -20,6 +20,8 @@ import ( "pete/internal/config" + _ "go.mau.fi/util/dbutil/litestream" // registers "sqlite3-fk-wal" driver used by cryptohelper + "maunium.net/go/mautrix" "maunium.net/go/mautrix/crypto/cryptohelper" "maunium.net/go/mautrix/event" diff --git a/internal/web/server.go b/internal/web/server.go index 9369715..3c0bab1 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -69,7 +69,7 @@ func New(cfg config.WebConfig) (*Server, error) { return nil, err } mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticSub)))) - mux.HandleFunc("GET /img/{hash}.avif", s.handleImg) + mux.HandleFunc("GET /img/{name}", s.handleImg) mux.HandleFunc("GET /{$}", s.handleIndex) for _, ch := range channels { diff --git a/internal/web/thumbs.go b/internal/web/thumbs.go index add2326..063db1a 100644 --- a/internal/web/thumbs.go +++ b/internal/web/thumbs.go @@ -56,7 +56,12 @@ func thumbURL(src string) string { } func (s *Server) handleImg(w http.ResponseWriter, r *http.Request) { - hash := r.PathValue("hash") + name := r.PathValue("name") + hash, ok := strings.CutSuffix(name, ".avif") + if !ok { + http.NotFound(w, r) + return + } src := r.URL.Query().Get("u") if src == "" || thumbKey(src) != hash { http.NotFound(w, r)