Pete/internal/ingestion/article.go

package ingestion

import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// resolveURL turns a possibly-relative URL into an absolute one using
// the base URL. Returns the raw input on parse failure.
func resolveURL(base, ref string) string {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return ""
	}
	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
		return ref
	}
	if strings.HasPrefix(ref, "//") {
		if i := strings.Index(base, "://"); i > 0 {
			return base[:i+1] + ref
		}
		return "https:" + ref
	}
	i := strings.Index(base, "://")
	if i < 0 {
		return ref
	}
	rest := base[i+3:]
	slash := strings.Index(rest, "/")
	if slash < 0 {
		return fmt.Sprintf("%s%s", base, ref)
	}
	host := base[:i+3+slash]
	if strings.HasPrefix(ref, "/") {
		return host + ref
	}
	return host + "/" + ref
}

// PaywallBodyThreshold is the minimum visible body length (in characters)
// for an article to be considered accessible. Anything below this is treated
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500

// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
	ImageURL  string // og:image or twitter:image, absolute URL
	BodyChars int    // length of extracted visible body text
	Fetched   bool   // true if we got an HTTP 200 with HTML
}

var articleClient = &http.Client{Timeout: 12 * time.Second}

// FetchArticleMeta fetches an article URL and returns its og:image plus the
// length of the visible body text (concatenation of <p> tags under <article>
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
// network/HTTP failure so callers can branch on accessibility.
func FetchArticleMeta(articleURL string) ArticleMeta {
	if articleURL == "" {
		return ArticleMeta{}
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ArticleMeta{}
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ArticleMeta{}
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ArticleMeta{}
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return ArticleMeta{}
	}

	return ArticleMeta{
		ImageURL:  extractOGImage(doc, articleURL),
		BodyChars: extractBodyChars(doc),
		Fetched:   true,
	}
}

// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {
	return FetchArticleMeta(articleURL).ImageURL
}

// MaxBodyChars is the cap on body text returned by FetchArticleBody. Keeps
// LLM prompts bounded; most news articles fit well under this.
const MaxBodyChars = 8000

// FetchArticleBody fetches the article and returns the concatenated visible
// body text (<article>/<main> <p> tags, falling back to all <p>), trimmed
// and capped at MaxBodyChars. Returns "" on any fetch failure.
func FetchArticleBody(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := articleClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ""
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return ""
	}
	return extractBodyText(doc)
}

func extractBodyText(doc *goquery.Document) string {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteString("\n\n")
		}
		b.WriteString(t)
		if b.Len() >= MaxBodyChars {
			return
		}
	})
	out := strings.TrimSpace(b.String())
	if len(out) > MaxBodyChars {
		out = out[:MaxBodyChars]
	}
	return out
}

func extractOGImage(doc *goquery.Document, base string) string {
	selectors := []string{
		`meta[property="og:image:secure_url"]`,
		`meta[property="og:image:url"]`,
		`meta[property="og:image"]`,
		`meta[name="twitter:image:src"]`,
		`meta[name="twitter:image"]`,
	}
	for _, sel := range selectors {
		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
			return resolveURL(base, strings.TrimSpace(v))
		}
	}
	return ""
}

// extractBodyChars concatenates the text of <p> tags inside <article> or
// <main>, falling back to all <p> tags, and returns the trimmed length.
func extractBodyChars(doc *goquery.Document) int {
	sel := doc.Find("article p, main p")
	if sel.Length() == 0 {
		sel = doc.Find("p")
	}
	var b strings.Builder
	sel.Each(func(_ int, s *goquery.Selection) {
		t := strings.TrimSpace(s.Text())
		if t == "" {
			return
		}
		if b.Len() > 0 {
			b.WriteByte(' ')
		}
		b.WriteString(t)
	})
	return len(strings.TrimSpace(b.String()))
}