Pete/internal/ingestion/article.go

package ingestion

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"pete/internal/safehttp"
)

// articleBodyMax bounds how much of an article response goquery will
// ever buffer. ~5 MiB comfortably fits any real news article while
// preventing a hostile origin from OOMing the process by streaming an
// endless body within the request timeout window.
const articleBodyMax = 5 << 20

// resolveURL turns a possibly-relative URL into an absolute one using
// the base URL. Returns the raw input on parse failure.
func resolveURL(base, ref string) string {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return ""
	}
	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
		return ref
	}
	if strings.HasPrefix(ref, "//") {
		if i := strings.Index(base, "://"); i > 0 {
			return base[:i+1] + ref
		}
		return "https:" + ref
	}
	i := strings.Index(base, "://")
	if i < 0 {
		return ref
	}
	rest := base[i+3:]
	slash := strings.Index(rest, "/")
	if slash < 0 {
		return fmt.Sprintf("%s%s", base, ref)
	}
	host := base[:i+3+slash]
	if strings.HasPrefix(ref, "/") {
		return host + ref
	}
	return host + "/" + ref
}

// PaywallBodyThreshold is the minimum visible body length (in characters)
// for an article to be considered accessible. Anything below this is treated
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500

// googlebotUA is what many metered publishers grant first-click access to.
// Re-tried automatically when the default-UA fetch looks gated.
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
	ImageURL   string // og:image or twitter:image, absolute URL
	BodyChars  int    // length of extracted visible body text
	Fetched    bool   // true if we got an HTTP 200 with HTML
	FetchError bool   // true if the fetch failed at the network/HTTP layer
	Paywalled  bool   // true if the page explicitly declares gated access
	// SubscriberOnly is true when the source publishes the article in a
	// form that genuinely has no public version — no archive snapshot, no
	// bypass UA, nothing. Callers should drop these from view entirely
	// rather than stamp them as paywalled.
	SubscriberOnly bool
	Status         int // last HTTP status seen (0 on transport error)
}

// Gated reports whether the response carries a strong gating signal: an
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
// or a 403 Forbidden after the bypass retry. A short body alone is not
// considered gating — that's a heuristic used by callers separately.
func (m ArticleMeta) Gated() bool {
	if m.Paywalled {
		return true
	}
	return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}

var articleClient = safehttp.NewClient(12 * time.Second)

// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
// retries once with a Googlebot UA + Google referer — the combination most
// metered publishers grant first-click access to. Returns the best of the
// two attempts.
func FetchArticleMeta(articleURL string) ArticleMeta {
	if articleURL == "" {
		return ArticleMeta{}
	}
	first := fetchArticleMetaOnce(articleURL, userAgent, "")
	if !shouldRetryAsBot(first) {
		return first
	}
	second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
	return pickBetter(first, second)
}

// shouldRetryAsBot returns true when the first attempt looks gated or too
// thin to be the real article body.
func shouldRetryAsBot(m ArticleMeta) bool {
	if m.FetchError {
		return false // transport failure won't be fixed by a different UA
	}
	if m.SubscriberOnly {
		return false // genuinely subscriber-only; no UA changes that
	}
	if m.Gated() {
		return true
	}
	if m.Fetched && m.BodyChars < PaywallBodyThreshold {
		return true
	}
	return false
}

// pickBetter chooses the more useful of two fetch attempts: prefer the one
// that isn't gated, then the one with more body, then the one that fetched
// at all.
func pickBetter(a, b ArticleMeta) ArticleMeta {
	aGated, bGated := a.Gated(), b.Gated()
	if aGated != bGated {
		if bGated {
			return a
		}
		return b
	}
	if a.Fetched != b.Fetched {
		if b.Fetched {
			return b
		}
		return a
	}
	if b.BodyChars > a.BodyChars {
		return b
	}
	return a
}

func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ArticleMeta{FetchError: true}
	}
	req.Header.Set("User-Agent", ua)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")
	if referer != "" {
		req.Header.Set("Referer", referer)
	}

	resp, err := articleClient.Do(req)
	if err != nil {
		return ArticleMeta{FetchError: true}
	}
	defer resp.Body.Close()

	meta := ArticleMeta{Status: resp.StatusCode}
	// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
	// But many sites 403 scrapers via Cloudflare while serving humans fine —
	// peek at the body and treat a Cloudflare challenge as a transport failure
	// rather than a paywall, so we don't stamp readable articles.
	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
			if isCloudflareChallenge(resp) {
				return ArticleMeta{FetchError: true}
			}
		}
		if resp.StatusCode == http.StatusPaymentRequired {
			meta.Paywalled = true
		}
		return meta
	}

	doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
	if err != nil {
		return meta
	}

	meta.Fetched = true
	meta.ImageURL = extractOGImage(doc, articleURL)
	meta.BodyChars = extractBodyChars(doc)
	meta.Paywalled = detectPaywall(doc)
	meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
	return meta
}

// detectSubscriberOnly returns true for sources known to publish articles
// with no public version reachable by any workaround we have (archive
// snapshots, bypass UAs, etc). Currently only LWN's subscriber-only
// articles, which display a fixed marker in the article body.
func detectSubscriberOnly(articleURL string, doc *goquery.Document) bool {
	u, err := url.Parse(articleURL)
	if err != nil {
		return false
	}
	host := strings.ToLower(u.Hostname())
	if host != "lwn.net" && !strings.HasSuffix(host, ".lwn.net") {
		return false
	}
	body := strings.ToLower(doc.Find("body").Text())
	return strings.Contains(body, "available only to lwn subscribers")
}

// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
// bot-block or JS-challenge page rather than a real publisher gate. The
// signatures are stable: cf-* response headers, the cf-error-details body
// markup, and a handful of recognizable titles. We read at most 8 KiB so a
// huge body can't stall the poller.
func isCloudflareChallenge(resp *http.Response) bool {
	if resp == nil {
		return false
	}
	if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" {
		// Header alone isn't enough — plenty of real publishers sit behind
		// Cloudflare. Confirm with a body marker below.
	}
	buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192))
	if err != nil {
		return false
	}
	s := strings.ToLower(string(buf))
	markers := []string{
		"/cdn-cgi/styles/cf",
		"cf-error-details",
		"attention required! | cloudflare",
		"<title>just a moment",
		"sorry, you have been blocked",
		"cloudflare ray id",
	}
	for _, m := range markers {
		if strings.Contains(s, m) {
			return true
		}
	}
	return false
}

// detectPaywall checks the page for explicit gating signals that publishers
// expose for Google News and crawlers. We treat the article as paywalled if:
//   - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
//     Hearst, many WordPress VIP sites including Wired)
//   - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by
//     NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers)
func detectPaywall(doc *goquery.Document) bool {
	tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content")
	switch strings.ToLower(strings.TrimSpace(tier)) {
	case "metered", "locked":
		return true
	}
	gated := false
	doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		if jsonLDDeclaresGated(s.Text()) {
			gated = true
			return false
		}
		return true
	})
	return gated
}

// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
// with "isAccessibleForFree" set falsy. We restrict to Article types so
// that embedded related-content or breadcrumb markup doesn't flip a free
// article to gated.
func jsonLDDeclaresGated(raw string) bool {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return false
	}
	var v any
	if err := json.Unmarshal([]byte(raw), &v); err != nil {
		return false
	}
	return walkJSONLDForGated(v)
}

// articleSchemaTypes are the schema.org @type values we treat as "the main
// article" for paywall purposes.
var articleSchemaTypes = map[string]bool{
	"article":            true,
	"newsarticle":        true,
	"report":             true,
	"reportagenewsarticle": true,
	"blogposting":        true,
	"scholarlyarticle":   true,
	"techarticle":        true,
	"opinionnewsarticle": true,
	"analysisnewsarticle": true,
	"backgroundnewsarticle": true,
	"reviewnewsarticle":  true,
}

func walkJSONLDForGated(v any) bool {
	switch x := v.(type) {
	case map[string]any:
		if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
			return true
		}
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	case []any:
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	}
	return false
}

func isArticleType(v any) bool {
	switch t := v.(type) {
	case string:
		return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
	case []any:
		for _, e := range t {
			if isArticleType(e) {
				return true
			}
		}
	}
	return false
}

func declaresGated(v any) bool {
	switch r := v.(type) {
	case bool:
		return !r
	case string:
		s := strings.ToLower(strings.TrimSpace(r))
		return s == "false" || s == "no"
	}
	return false
}

// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {
	return FetchArticleMeta(articleURL).ImageURL
}

// MaxBodyChars is the cap on body text returned by FetchArticleBody. Keeps
// LLM prompts bounded; most news articles fit well under this.
const MaxBodyChars = 8000

// FetchArticleBody fetches the article and returns the concatenated visible
// body text (<article>/<main> <p> tags, falling back to all <p>), trimmed
// and capped at MaxBodyChars. Returns "" on any fetch failure.
func FetchArticleBody(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ""
	}

	doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
	if err != nil {
		return ""
	}
	return extractBodyText(doc)
}

func extractBodyText(doc *goquery.Document) string {
	out := joinParagraphText(doc.Find("article p, main p"))
	if len(out) < PaywallBodyThreshold {
		if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) {
			out = alt
		}
	}
	// Sites like Phoronix use <br>-separated text inside <article> rather
	// than <p> tags, and sites like Anime News Network skip semantic
	// article/main tags entirely and put the body in a custom div. Fall
	// back to the raw text of the longest known container.
	if len(out) < PaywallBodyThreshold {
		if alt := bestContainerText(doc); len(alt) > len(out) {
			out = alt
		}
	}
	if len(out) > MaxBodyChars {
		out = out[:MaxBodyChars]
	}
	return out
}

// bodyContainerSelectors is the list of selectors we try when <article>/<main>
// aren't present or are too thin. Ordered loosely from semantic to generic;
// we take the longest match across all of them.
var bodyContainerSelectors = []string{
	"article",
	"main",
	`[itemprop="articleBody"]`,
	`[role="main"]`,
	".article-body",
	".article-content",
	".entry-content",
	".post-content",
	".story-body",
	".KonaBody", // Anime News Network
	"#maincontent",
	"#content",
}

func bestContainerText(doc *goquery.Document) string {
	var best string
	for _, sel := range bodyContainerSelectors {
		if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
			best = t
		}
	}
	return best
}

func joinParagraphText(sel *goquery.Selection) string {
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteString("\n\n")
		}
		b.WriteString(t)
	})
	return strings.TrimSpace(b.String())
}

func containerText(sel *goquery.Selection) string {
	if sel.Length() == 0 {
		return ""
	}
	// Collapse whitespace runs so <br>-laden markup doesn't produce a wall
	// of blank lines, but preserve enough structure to look like prose.
	return strings.Join(strings.Fields(sel.Text()), " ")
}

func extractOGImage(doc *goquery.Document, base string) string {
	selectors := []string{
		`meta[property="og:image:secure_url"]`,
		`meta[property="og:image:url"]`,
		`meta[property="og:image"]`,
		`meta[name="twitter:image:src"]`,
		`meta[name="twitter:image"]`,
	}
	for _, sel := range selectors {
		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
			return resolveURL(base, strings.TrimSpace(v))
		}
	}
	// Sites like Hardcore Gaming 101 publish no og:image — fall back to the
	// WordPress featured image (.wp-post-image), then the first reasonably-
	// sized <img> inside <article>/<main>.
	if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok {
		if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" {
			return resolveURL(base, url)
		}
	}
	var found string
	doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		raw, _ := s.Attr("src")
		url := pickImageSrc(s, raw)
		if url == "" || !looksLikeContentImage(s, url) {
			return true
		}
		found = url
		return false
	})
	if found != "" {
		return resolveURL(base, found)
	}
	return ""
}

// pickImageSrc returns the most useful image URL from an <img> selection,
// preferring data-src/data-lazy-src (lazy-load) over src when src is a
// placeholder (empty or data: URI).
func pickImageSrc(s *goquery.Selection, src string) string {
	src = strings.TrimSpace(src)
	if !strings.HasPrefix(src, "data:") && src != "" {
		return src
	}
	for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
		if v, ok := s.Attr(attr); ok {
			v = strings.TrimSpace(v)
			if v != "" && !strings.HasPrefix(v, "data:") {
				return v
			}
		}
	}
	return ""
}

// looksLikeContentImage filters out obvious non-article images: gravatars,
// social-share icons, tracking pixels. Anything with declared dimensions
// under 100px on either side is treated as chrome.
func looksLikeContentImage(s *goquery.Selection, url string) bool {
	low := strings.ToLower(url)
	for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
		if strings.Contains(low, bad) {
			return false
		}
	}
	if w := atoiAttr(s, "width"); w > 0 && w < 100 {
		return false
	}
	if h := atoiAttr(s, "height"); h > 0 && h < 100 {
		return false
	}
	return true
}

func atoiAttr(s *goquery.Selection, name string) int {
	v, ok := s.Attr(name)
	if !ok {
		return 0
	}
	n := 0
	for _, c := range strings.TrimSpace(v) {
		if c < '0' || c > '9' {
			break
		}
		n = n*10 + int(c-'0')
	}
	return n
}

// extractBodyChars returns the visible body length used to gauge whether a
// page looks like a real article. It tries <p> tags inside <article>/<main>,
// then all <p> tags, then the raw text inside <article>/<main> — that last
// fallback catches sites like Phoronix that separate paragraphs with <br>
// instead of <p>.
func extractBodyChars(doc *goquery.Document) int {
	best := len(joinParagraphText(doc.Find("article p, main p")))
	if best < PaywallBodyThreshold {
		if n := len(joinParagraphText(doc.Find("p"))); n > best {
			best = n
		}
	}
	if best < PaywallBodyThreshold {
		if n := len(bestContainerText(doc)); n > best {
			best = n
		}
	}
	return best
}