Pete/internal/ingestion/article.go

package ingestion

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// resolveURL turns a possibly-relative URL into an absolute one using
// the base URL. Returns the raw input on parse failure.
func resolveURL(base, ref string) string {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return ""
	}
	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
		return ref
	}
	if strings.HasPrefix(ref, "//") {
		if i := strings.Index(base, "://"); i > 0 {
			return base[:i+1] + ref
		}
		return "https:" + ref
	}
	i := strings.Index(base, "://")
	if i < 0 {
		return ref
	}
	rest := base[i+3:]
	slash := strings.Index(rest, "/")
	if slash < 0 {
		return fmt.Sprintf("%s%s", base, ref)
	}
	host := base[:i+3+slash]
	if strings.HasPrefix(ref, "/") {
		return host + ref
	}
	return host + "/" + ref
}

// PaywallBodyThreshold is the minimum visible body length (in characters)
// for an article to be considered accessible. Anything below this is treated
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500

// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
	ImageURL  string // og:image or twitter:image, absolute URL
	BodyChars int    // length of extracted visible body text
	Fetched   bool   // true if we got an HTTP 200 with HTML
	Paywalled bool   // true if the page explicitly declares gated access
}

var articleClient = &http.Client{Timeout: 12 * time.Second}

// FetchArticleMeta fetches an article URL and returns its og:image plus the
// length of the visible body text (concatenation of <p> tags under <article>
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
// network/HTTP failure so callers can branch on accessibility.
func FetchArticleMeta(articleURL string) ArticleMeta {
	if articleURL == "" {
		return ArticleMeta{}
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ArticleMeta{}
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ArticleMeta{}
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ArticleMeta{}
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return ArticleMeta{}
	}

	return ArticleMeta{
		ImageURL:  extractOGImage(doc, articleURL),
		BodyChars: extractBodyChars(doc),
		Fetched:   true,
		Paywalled: detectPaywall(doc),
	}
}

// detectPaywall checks the page for explicit gating signals that publishers
// expose for Google News and crawlers. We treat the article as paywalled if:
//   - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
//     Hearst, many WordPress VIP sites including Wired)
//   - JSON-LD with "isAccessibleForFree": false (schema.org standard, used by
//     NYT, WaPo, Bloomberg, FT, WSJ, and most metered publishers)
func detectPaywall(doc *goquery.Document) bool {
	tier, _ := doc.Find(`meta[name="article:content_tier"]`).First().Attr("content")
	switch strings.ToLower(strings.TrimSpace(tier)) {
	case "metered", "locked":
		return true
	}
	gated := false
	doc.Find(`script[type="application/ld+json"]`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		if jsonLDDeclaresGated(s.Text()) {
			gated = true
			return false
		}
		return true
	})
	return gated
}

// jsonLDDeclaresGated returns true if the JSON-LD payload contains
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
// structure. Publishers vary wildly in shape, so we walk generically.
func jsonLDDeclaresGated(raw string) bool {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return false
	}
	var v any
	if err := json.Unmarshal([]byte(raw), &v); err != nil {
		return false
	}
	return walkJSONLDForGated(v)
}

func walkJSONLDForGated(v any) bool {
	switch x := v.(type) {
	case map[string]any:
		if raw, ok := x["isAccessibleForFree"]; ok {
			switch r := raw.(type) {
			case bool:
				if !r {
					return true
				}
			case string:
				s := strings.ToLower(strings.TrimSpace(r))
				if s == "false" || s == "no" {
					return true
				}
			}
		}
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	case []any:
		for _, vv := range x {
			if walkJSONLDForGated(vv) {
				return true
			}
		}
	}
	return false
}

// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {
	return FetchArticleMeta(articleURL).ImageURL
}

// MaxBodyChars is the cap on body text returned by FetchArticleBody. Keeps
// LLM prompts bounded; most news articles fit well under this.
const MaxBodyChars = 8000

// FetchArticleBody fetches the article and returns the concatenated visible
// body text (<article>/<main> <p> tags, falling back to all <p>), trimmed
// and capped at MaxBodyChars. Returns "" on any fetch failure.
func FetchArticleBody(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ""
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return ""
	}
	return extractBodyText(doc)
}

func extractBodyText(doc *goquery.Document) string {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteString("\n\n")
		}
		b.WriteString(t)
		if b.Len() >= MaxBodyChars {
			return
		}
	})
	out := strings.TrimSpace(b.String())
	if len(out) > MaxBodyChars {
		out = out[:MaxBodyChars]
	}
	return out
}

func extractOGImage(doc *goquery.Document, base string) string {
	selectors := []string{
		`meta[property="og:image:secure_url"]`,
		`meta[property="og:image:url"]`,
		`meta[property="og:image"]`,
		`meta[name="twitter:image:src"]`,
		`meta[name="twitter:image"]`,
	}
	for _, sel := range selectors {
		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
			return resolveURL(base, strings.TrimSpace(v))
		}
	}
	return ""
}

// extractBodyChars concatenates the text of <p> tags inside <article> or
// <main>, falling back to all <p> tags, and returns the trimmed length.
func extractBodyChars(doc *goquery.Document) int {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteByte(' ')
		}
		b.WriteString(t)
	})
	return len(strings.TrimSpace(b.String()))
}