Pete/internal/ingestion/article.go

package ingestion

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// resolveURL turns a possibly-relative URL into an absolute one using
// the base URL. Returns the raw input on parse failure.
func resolveURL(base, ref string) string {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return ""
	}
	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
		return ref
	}
	if strings.HasPrefix(ref, "//") {
		if i := strings.Index(base, "://"); i > 0 {
			return base[:i+1] + ref
		}
		return "https:" + ref
	}
	i := strings.Index(base, "://")
	if i < 0 {
		return ref
	}
	rest := base[i+3:]
	slash := strings.Index(rest, "/")
	if slash < 0 {
		return fmt.Sprintf("%s%s", base, ref)
	}
	host := base[:i+3+slash]
	if strings.HasPrefix(ref, "/") {
		return host + ref
	}
	return host + "/" + ref
}

// PaywallBodyThreshold is the minimum visible body length (in characters)
// for an article to be considered accessible. Anything below this is treated
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500

// googlebotUA is what many metered publishers grant first-click access to.
// Re-tried automatically when the default-UA fetch looks gated.
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
	ImageURL   string // og:image or twitter:image, absolute URL
	BodyChars  int    // length of extracted visible body text
	Fetched    bool   // true if we got an HTTP 200 with HTML
	FetchError bool   // true if the fetch failed at the network/HTTP layer
	Paywalled  bool   // true if the page explicitly declares gated access
	Status     int    // last HTTP status seen (0 on transport error)
}

// Gated reports whether the response carries a strong gating signal: an
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
// or a 403 Forbidden after the bypass retry. A short body alone is not
// considered gating — that's a heuristic used by callers separately.
func (m ArticleMeta) Gated() bool {
	if m.Paywalled {
		return true
	}
	return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}

var articleClient = &http.Client{Timeout: 12 * time.Second}

// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
// retries once with a Googlebot UA + Google referer — the combination most
// metered publishers grant first-click access to. Returns the best of the
// two attempts.
func FetchArticleMeta(articleURL string) ArticleMeta {
	if articleURL == "" {
		return ArticleMeta{}
	}
	first := fetchArticleMetaOnce(articleURL, userAgent, "")
	if !shouldRetryAsBot(first) {
		return first
	}
	second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
	return pickBetter(first, second)
}

// shouldRetryAsBot returns true when the first attempt looks gated or too
// thin to be the real article body.
func shouldRetryAsBot(m ArticleMeta) bool {
	if m.FetchError {
		return false // transport failure won't be fixed by a different UA
	}
	if m.Gated() {
		return true
	}
	if m.Fetched && m.BodyChars < PaywallBodyThreshold {
		return true
	}
	return false
}

// pickBetter chooses the more useful of two fetch attempts: prefer the one
// that isn't gated, then the one with more body, then the one that fetched
// at all.
func pickBetter(a, b ArticleMeta) ArticleMeta {
	aGated, bGated := a.Gated(), b.Gated()
	if aGated != bGated {
		if bGated {
			return a
		}
		return b
	}
	if a.Fetched != b.Fetched {
		if b.Fetched {
			return b
		}
		return a
	}
	if b.BodyChars > a.BodyChars {
		return b
	}
	return a
}

func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ArticleMeta{FetchError: true}
	}
	req.Header.Set("User-Agent", ua)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")
	if referer != "" {
		req.Header.Set("Referer", referer)
	}

	resp, err := articleClient.Do(req)
	if err != nil {
		return ArticleMeta{FetchError: true}
	}
	defer resp.Body.Close()

	meta := ArticleMeta{Status: resp.StatusCode}
	// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
	// We still try to parse the body for whatever signals it contains.
	if resp.StatusCode != http.StatusOK {
		if resp.StatusCode == http.StatusPaymentRequired {
			meta.Paywalled = true
		}
		return meta
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return meta
	}

	meta.Fetched = true
	meta.ImageURL = extractOGImage(doc, articleURL)
	meta.BodyChars = extractBodyChars(doc)
	meta.Paywalled = detectPaywall(doc)
	return meta
}

// detectPaywall checks the page for explicit gating signals that publishers
// expose for Google News and crawlers. We treat the article as paywalled if:
//   - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
//     Hearst, many WordPress VIP sites including Wired)
//   - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by
//     NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers)
func detectPaywall(doc *goquery.Document) bool {
	tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content")
	switch strings.ToLower(strings.TrimSpace(tier)) {
	case "metered", "locked":
		return true
	}
	gated := false
	doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		if jsonLDDeclaresGated(s.Text()) {
			gated = true
			return false
		}
		return true
	})
	return gated
}

// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
// with "isAccessibleForFree" set falsy. We restrict to Article types so
// that embedded related-content or breadcrumb markup doesn't flip a free
// article to gated.
func jsonLDDeclaresGated(raw string) bool {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return false
	}
	var v any
	if err := json.Unmarshal([]byte(raw), &v); err != nil {
		return false
	}
	return walkJSONLDForGated(v)
}

// articleSchemaTypes are the schema.org @type values we treat as "the main
// article" for paywall purposes.
var articleSchemaTypes = map[string]bool{
	"article":            true,
	"newsarticle":        true,
	"report":             true,
	"reportagenewsarticle": true,
	"blogposting":        true,
	"scholarlyarticle":   true,
	"techarticle":        true,
	"opinionnewsarticle": true,
	"analysisnewsarticle": true,
	"backgroundnewsarticle": true,
	"reviewnewsarticle":  true,
}

func walkJSONLDForGated(v any) bool {
	switch x := v.(type) {
	case map[string]any:
		if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
			return true
		}
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	case []any:
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	}
	return false
}

func isArticleType(v any) bool {
	switch t := v.(type) {
	case string:
		return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
	case []any:
		for _, e := range t {
			if isArticleType(e) {
				return true
			}
		}
	}
	return false
}

func declaresGated(v any) bool {
	switch r := v.(type) {
	case bool:
		return !r
	case string:
		s := strings.ToLower(strings.TrimSpace(r))
		return s == "false" || s == "no"
	}
	return false
}

// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {
	return FetchArticleMeta(articleURL).ImageURL
}

// MaxBodyChars is the cap on body text returned by FetchArticleBody. Keeps
// LLM prompts bounded; most news articles fit well under this.
const MaxBodyChars = 8000

// FetchArticleBody fetches the article and returns the concatenated visible
// body text (<article>/<main> <p> tags, falling back to all <p>), trimmed
// and capped at MaxBodyChars. Returns "" on any fetch failure.
func FetchArticleBody(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ""
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return ""
	}
	return extractBodyText(doc)
}

func extractBodyText(doc *goquery.Document) string {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteString("\n\n")
		}
		b.WriteString(t)
		if b.Len() >= MaxBodyChars {
			return
		}
	})
	out := strings.TrimSpace(b.String())
	if len(out) > MaxBodyChars {
		out = out[:MaxBodyChars]
	}
	return out
}

func extractOGImage(doc *goquery.Document, base string) string {
	selectors := []string{
		`meta[property="og:image:secure_url"]`,
		`meta[property="og:image:url"]`,
		`meta[property="og:image"]`,
		`meta[name="twitter:image:src"]`,
		`meta[name="twitter:image"]`,
	}
	for _, sel := range selectors {
		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
			return resolveURL(base, strings.TrimSpace(v))
		}
	}
	return ""
}

// extractBodyChars concatenates the text of <p> tags inside <article> or
// <main>, falling back to all <p> tags, and returns the trimmed length.
func extractBodyChars(doc *goquery.Document) int {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteByte(' ')
		}
		b.WriteString(t)
	})
	return len(strings.TrimSpace(b.String()))
}