package ingestion
import (
"context"
"html"
"log/slog"
"net/http"
"regexp"
"strings"
"time"
"github.com/mmcdole/gofeed"
)
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
var feedClient = &http.Client{Timeout: 30 * time.Second}
// FeedItem represents a parsed RSS item ready for routing.
type FeedItem struct {
GUID string
Headline string
Lede string
ImageURL string
ArticleURL string
Source string
DirectRoute string
}
var (
htmlTagRe = regexp.MustCompile(`<[^>]*>`)
imgSrcRe = regexp.MustCompile(`(?i)
]+src=["']([^"']+)["']`)
)
// FetchFeed fetches and parses an RSS/Atom feed, returning items.
func FetchFeed(feedURL string) ([]FeedItem, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fp := gofeed.NewParser()
fp.Client = feedClient
fp.UserAgent = userAgent
feed, err := fp.ParseURLWithContext(feedURL, ctx)
if err != nil {
return nil, err
}
items := make([]FeedItem, 0, len(feed.Items))
for _, item := range feed.Items {
fi := FeedItem{
GUID: itemGUID(item),
Headline: strings.TrimSpace(item.Title),
Lede: extractLede(item.Description),
ImageURL: NormalizeImageURL(extractImageURL(item)),
ArticleURL: strings.TrimSpace(item.Link),
}
if fi.GUID == "" || fi.ArticleURL == "" {
continue
}
items = append(items, fi)
}
slog.Debug("fetched feed", "url", feedURL, "items", len(items))
return items, nil
}
func itemGUID(item *gofeed.Item) string {
if item.GUID != "" {
return item.GUID
}
return item.Link
}
// extractLede returns the feed description verbatim, with HTML stripped.
func extractLede(desc string) string {
if desc == "" {
return ""
}
text := htmlTagRe.ReplaceAllString(desc, "")
text = html.UnescapeString(text)
return strings.TrimSpace(text)
}
// extractImageURL tries to find an image URL from feed item metadata.
// Order: media:content/thumbnail, enclosures, item.Image, then
tags
// scraped from content:encoded / description (catches feeds like Ars that
// embed the lead image in the article body but not as a media:* element).
func extractImageURL(item *gofeed.Item) string {
// Check media content (common in RSS 2.0 with media namespace)
if item.Extensions != nil {
if media, ok := item.Extensions["media"]; ok {
if contents, ok := media["content"]; ok {
for _, c := range contents {
if url := c.Attrs["url"]; url != "" {
return url
}
}
}
if thumbs, ok := media["thumbnail"]; ok {
for _, t := range thumbs {
if url := t.Attrs["url"]; url != "" {
return url
}
}
}
// media:group wraps nested media:content / media:thumbnail in some feeds
if groups, ok := media["group"]; ok {
for _, g := range groups {
for _, child := range g.Children {
for _, n := range child {
if url := n.Attrs["url"]; url != "" {
return url
}
}
}
}
}
}
}
// Check enclosures
for _, enc := range item.Enclosures {
if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
return enc.URL
}
}
// Check item image
if item.Image != nil && item.Image.URL != "" {
return item.Image.URL
}
// Scrape first
from content:encoded or description
if url := firstImgSrc(item.Content); url != "" {
return url
}
if url := firstImgSrc(item.Description); url != "" {
return url
}
return ""
}
func firstImgSrc(htmlBody string) string {
if htmlBody == "" {
return ""
}
m := imgSrcRe.FindStringSubmatch(htmlBody)
if len(m) < 2 {
return ""
}
return html.UnescapeString(m[1])
}