package ingestion import ( "context" "html" "log/slog" "regexp" "strconv" "strings" "time" "github.com/mmcdole/gofeed" ext "github.com/mmcdole/gofeed/extensions" "pete/internal/safehttp" ) const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)" var feedClient = safehttp.NewClient(30 * time.Second) // FeedItem represents a parsed RSS item ready for routing. type FeedItem struct { GUID string Headline string Lede string ImageURL string ArticleURL string Source string DirectRoute string } var ( htmlTagRe = regexp.MustCompile(`<[^>]*>`) imgSrcRe = regexp.MustCompile(`(?i)]+src=["']([^"']+)["']`) ) // FetchFeed fetches and parses an RSS/Atom feed, returning items. func FetchFeed(feedURL string) ([]FeedItem, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() fp := gofeed.NewParser() fp.Client = feedClient fp.UserAgent = userAgent feed, err := fp.ParseURLWithContext(feedURL, ctx) if err != nil { return nil, err } items := make([]FeedItem, 0, len(feed.Items)) for _, item := range feed.Items { fi := FeedItem{ GUID: itemGUID(item), Headline: strings.TrimSpace(item.Title), Lede: extractLede(item.Description), ImageURL: NormalizeImageURL(extractImageURL(item)), ArticleURL: strings.TrimSpace(item.Link), } if fi.GUID == "" || fi.ArticleURL == "" { continue } items = append(items, fi) } slog.Debug("fetched feed", "url", feedURL, "items", len(items)) return items, nil } func itemGUID(item *gofeed.Item) string { if item.GUID != "" { return item.GUID } return item.Link } // extractLede returns the feed description verbatim, with HTML stripped. func extractLede(desc string) string { if desc == "" { return "" } text := htmlTagRe.ReplaceAllString(desc, "") text = html.UnescapeString(text) return strings.TrimSpace(text) } // extractImageURL tries to find an image URL from feed item metadata. // Order: media:content/thumbnail, enclosures, item.Image, then tags // scraped from content:encoded / description (catches feeds like Ars that // embed the lead image in the article body but not as a media:* element). func extractImageURL(item *gofeed.Item) string { // Check media content (common in RSS 2.0 with media namespace). // When multiple media:content variants are advertised (Guardian RSS lists // 140/460/700/1200…) we want the widest signed URL we can find. if item.Extensions != nil { if media, ok := item.Extensions["media"]; ok { if contents, ok := media["content"]; ok { if url := widestURL(contents); url != "" { return url } } if thumbs, ok := media["thumbnail"]; ok { if url := widestURL(thumbs); url != "" { return url } } // media:group wraps nested media:content / media:thumbnail in some feeds if groups, ok := media["group"]; ok { for _, g := range groups { for _, child := range g.Children { for _, n := range child { if url := n.Attrs["url"]; url != "" { return url } } } } } } } // Check enclosures for _, enc := range item.Enclosures { if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" { return enc.URL } } // Check item image if item.Image != nil && item.Image.URL != "" { return item.Image.URL } // Scrape first from content:encoded or description if url := firstImgSrc(item.Content); url != "" { return url } if url := firstImgSrc(item.Description); url != "" { return url } return "" } // widestURL picks the entry with the largest declared width attribute, falling // back to the first URL if none have a parsable width. func widestURL(entries []ext.Extension) string { best := "" bestW := -1 for _, e := range entries { url := e.Attrs["url"] if url == "" { continue } w, _ := strconv.Atoi(e.Attrs["width"]) if w > bestW { best = url bestW = w } } return best } func firstImgSrc(htmlBody string) string { if htmlBody == "" { return "" } m := imgSrcRe.FindStringSubmatch(htmlBody) if len(m) < 2 { return "" } return html.UnescapeString(m[1]) }