Pete/internal/ingestion/parser.go

package ingestion

import (
	"context"
	"fmt"
	"html"
	"log/slog"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/mmcdole/gofeed"
	ext "github.com/mmcdole/gofeed/extensions"

	"pete/internal/safehttp"
)

const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"

// maxFeedBytes caps how much of a feed body we'll buffer into the parser, so a
// hostile feed streaming an endless body within the request timeout can't OOM
// the process. Generous headroom — real RSS/Atom feeds are well under this.
const maxFeedBytes = 16 << 20

var feedClient = safehttp.NewClient(30 * time.Second)

// FeedItem represents a parsed RSS item ready for routing.
type FeedItem struct {
	GUID        string
	Headline    string
	Lede        string
	Content     string // full article text from content:encoded, block structure preserved; "" when the feed ships only a snippet
	ImageURL    string
	ArticleURL  string
	Source      string
	DirectRoute string
	Language    string // per-item <language> (or dc:language) when present; "" otherwise
	PublishedAt int64  // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
}

var (
	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
	wsRe      = regexp.MustCompile(`\s+`)
	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)

	// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
	// tags AND their text — before the generic tag strip runs. htmlTagRe alone
	// removes only the tags, which would leave inline JavaScript (e.g. a
	// Datawrapper embed resizer) as literal text in reader mode.
	nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)

	// Used by extractContentText to preserve paragraph structure when turning
	// content:encoded HTML into plain text.
	blockCloseRe  = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
	brRe          = regexp.MustCompile(`(?i)<br\s*/?>`)
	hspaceRe      = regexp.MustCompile(`[ \t\f\r]+`)
	manyNewlineRe = regexp.MustCompile(`\n{3,}`)
)

// maxContentChars bounds the stored article text. Generous enough for any real
// article while keeping row size and the reader payload sane.
const maxContentChars = 20000

// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
// the User-Agent header; pass "" to use Pete's honest default bot UA.
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
	if err != nil {
		return nil, err
	}
	if ua == "" {
		ua = userAgent
	}
	req.Header.Set("User-Agent", ua)
	resp, err := feedClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("feed fetch %s: status %d", feedURL, resp.StatusCode)
	}

	feed, err := gofeed.NewParser().Parse(safehttp.LimitedBody(resp.Body, maxFeedBytes))
	if err != nil {
		return nil, err
	}

	items := make([]FeedItem, 0, len(feed.Items))
	for _, item := range feed.Items {
		fi := FeedItem{
			GUID:        itemGUID(item),
			Headline:    strings.TrimSpace(item.Title),
			Lede:        extractLede(item.Description),
			Content:     extractContentText(item.Content),
			ImageURL:    NormalizeImageURL(extractImageURL(item)),
			ArticleURL:  strings.TrimSpace(item.Link),
			Language:    itemLanguage(item),
			PublishedAt: itemPublished(item),
		}
		if fi.GUID == "" || fi.ArticleURL == "" {
			continue
		}
		items = append(items, fi)
	}

	slog.Debug("fetched feed", "url", feedURL, "items", len(items))
	return items, nil
}

// itemLanguage returns the per-item language tag if the feed provides one.
// Politico Europe puts a raw <language> inside each <item>; gofeed parks
// unknown item-level elements in item.Custom. We also check the standard
// dc:language extension for feeds that use Dublin Core.
func itemLanguage(item *gofeed.Item) string {
	if item.Custom != nil {
		if v, ok := item.Custom["language"]; ok {
			return strings.ToLower(strings.TrimSpace(v))
		}
	}
	if item.DublinCoreExt != nil && len(item.DublinCoreExt.Language) > 0 {
		return strings.ToLower(strings.TrimSpace(item.DublinCoreExt.Language[0]))
	}
	return ""
}

func itemGUID(item *gofeed.Item) string {
	if item.GUID != "" {
		return item.GUID
	}
	return item.Link
}

// itemPublished returns the item's publish time as unix seconds, falling back
// to the updated time if pubDate was absent. Returns 0 when neither parses;
// the poller clamps future timestamps to ingest time.
func itemPublished(item *gofeed.Item) int64 {
	if item.PublishedParsed != nil {
		return item.PublishedParsed.Unix()
	}
	if item.UpdatedParsed != nil {
		return item.UpdatedParsed.Unix()
	}
	return 0
}

// extractLede returns the feed description verbatim, with HTML stripped.
func extractLede(desc string) string {
	if desc == "" {
		return ""
	}
	// Drop <script>/<style>/<noscript> (tags and their inner text) before the
	// generic tag strip, which would otherwise leave the inner text behind.
	desc = nonContentElemRe.ReplaceAllString(desc, " ")
	// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
	text := htmlTagRe.ReplaceAllString(desc, " ")
	text = html.UnescapeString(text)
	text = wsRe.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}

// extractContentText converts a feed's content:encoded HTML into plain text
// with paragraph breaks preserved, for reader mode. Block-level boundaries
// (</p>, </div>, <br>, list items, headings…) become newlines before the
// remaining tags are stripped, so the reader can re-wrap the text into
// paragraphs. Returns "" when the feed carried no content:encoded.
func extractContentText(raw string) string {
	if strings.TrimSpace(raw) == "" {
		return ""
	}
	// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
	// first — otherwise htmlTagRe below strips only the tags and leaves inline
	// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
	s := nonContentElemRe.ReplaceAllString(raw, "\n")
	s = brRe.ReplaceAllString(s, "\n")
	s = blockCloseRe.ReplaceAllString(s, "\n\n")
	s = htmlTagRe.ReplaceAllString(s, "")
	s = html.UnescapeString(s)
	lines := strings.Split(s, "\n")
	for i, ln := range lines {
		lines[i] = strings.TrimSpace(hspaceRe.ReplaceAllString(ln, " "))
	}
	s = manyNewlineRe.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
	s = strings.TrimSpace(s)
	return truncateUTF8(s, maxContentChars)
}

// truncateUTF8 caps s at max bytes without splitting a multi-byte rune. A plain
// byte slice can leave a partial trailing rune, which is invalid UTF-8; that
// corrupt byte then breaks the RSS content:encoded XML (Go's xml encoder writes
// it raw) and shows as a replacement char in JSON. Trim back to a rune boundary.
func truncateUTF8(s string, max int) string {
	if len(s) <= max {
		return s
	}
	s = s[:max]
	for len(s) > 0 {
		if r, size := utf8.DecodeLastRuneInString(s); r == utf8.RuneError && size <= 1 {
			s = s[:len(s)-1] // strip one byte of an incomplete trailing rune
			continue
		}
		break
	}
	return s
}

// extractImageURL tries to find an image URL from feed item metadata.
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
// scraped from content:encoded / description (catches feeds like Ars that
// embed the lead image in the article body but not as a media:* element).
func extractImageURL(item *gofeed.Item) string {
	// Check media content (common in RSS 2.0 with media namespace).
	// When multiple media:content variants are advertised (Guardian RSS lists
	// 140/460/700/1200…) we want the widest signed URL we can find.
	if item.Extensions != nil {
		if media, ok := item.Extensions["media"]; ok {
			if contents, ok := media["content"]; ok {
				if url := widestURL(contents); url != "" {
					return url
				}
			}
			if thumbs, ok := media["thumbnail"]; ok {
				if url := widestURL(thumbs); url != "" {
					return url
				}
			}
			// media:group wraps nested media:content / media:thumbnail in some feeds
			if groups, ok := media["group"]; ok {
				for _, g := range groups {
					for _, child := range g.Children {
						for _, n := range child {
							if url := n.Attrs["url"]; url != "" {
								return url
							}
						}
					}
				}
			}
		}
	}

	// Check enclosures
	for _, enc := range item.Enclosures {
		if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
			return enc.URL
		}
	}

	// Check item image
	if item.Image != nil && item.Image.URL != "" {
		return item.Image.URL
	}

	// Scrape first <img src> from content:encoded or description
	if url := firstImgSrc(item.Content); url != "" {
		return url
	}
	if url := firstImgSrc(item.Description); url != "" {
		return url
	}

	return ""
}

// widestURL picks the entry with the largest declared width attribute, falling
// back to the first URL if none have a parsable width.
func widestURL(entries []ext.Extension) string {
	best := ""
	bestW := -1
	for _, e := range entries {
		url := e.Attrs["url"]
		if url == "" {
			continue
		}
		w, _ := strconv.Atoi(e.Attrs["width"])
		if w > bestW {
			best = url
			bestW = w
		}
	}
	return best
}

func firstImgSrc(htmlBody string) string {
	if htmlBody == "" {
		return ""
	}
	m := imgSrcRe.FindStringSubmatch(htmlBody)
	if len(m) < 2 {
		return ""
	}
	return html.UnescapeString(m[1])
}