Initial commit

internal/ingestion/parser.go

@@ -0,0 +1,157 @@
+package ingestion
+
+import (
+	"context"
+	"html"
+	"log/slog"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+)
+
+const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
+
+var feedClient = &http.Client{Timeout: 30 * time.Second}
+
+// FeedItem represents a parsed RSS item ready for classification.
+type FeedItem struct {
+	GUID        string
+	Headline    string
+	Lede        string
+	ImageURL    string
+	ArticleURL  string
+	Source      string
+	FeedHint    string
+	Tier        int
+	DirectRoute *string
+}
+
+var (
+	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
+	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
+)
+
+
+// FetchFeed fetches and parses an RSS/Atom feed, returning items.
+func FetchFeed(feedURL string) ([]FeedItem, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	fp := gofeed.NewParser()
+	fp.Client = feedClient
+	fp.UserAgent = userAgent
+
+	feed, err := fp.ParseURLWithContext(feedURL, ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	items := make([]FeedItem, 0, len(feed.Items))
+	for _, item := range feed.Items {
+		fi := FeedItem{
+			GUID:       itemGUID(item),
+			Headline:   strings.TrimSpace(item.Title),
+			Lede:       extractLede(item.Description),
+			ImageURL:   extractImageURL(item),
+			ArticleURL: strings.TrimSpace(item.Link),
+		}
+		if fi.GUID == "" || fi.ArticleURL == "" {
+			continue
+		}
+		items = append(items, fi)
+	}
+
+	slog.Debug("fetched feed", "url", feedURL, "items", len(items))
+	return items, nil
+}
+
+func itemGUID(item *gofeed.Item) string {
+	if item.GUID != "" {
+		return item.GUID
+	}
+	return item.Link
+}
+
+// extractLede returns the feed description verbatim, with HTML stripped.
+func extractLede(desc string) string {
+	if desc == "" {
+		return ""
+	}
+	text := htmlTagRe.ReplaceAllString(desc, "")
+	text = html.UnescapeString(text)
+	return strings.TrimSpace(text)
+}
+
+// extractImageURL tries to find an image URL from feed item metadata.
+// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
+// scraped from content:encoded / description (catches feeds like Ars that
+// embed the lead image in the article body but not as a media:* element).
+func extractImageURL(item *gofeed.Item) string {
+	// Check media content (common in RSS 2.0 with media namespace)
+	if item.Extensions != nil {
+		if media, ok := item.Extensions["media"]; ok {
+			if contents, ok := media["content"]; ok {
+				for _, c := range contents {
+					if url := c.Attrs["url"]; url != "" {
+						return url
+					}
+				}
+			}
+			if thumbs, ok := media["thumbnail"]; ok {
+				for _, t := range thumbs {
+					if url := t.Attrs["url"]; url != "" {
+						return url
+					}
+				}
+			}
+			// media:group wraps nested media:content / media:thumbnail in some feeds
+			if groups, ok := media["group"]; ok {
+				for _, g := range groups {
+					for _, child := range g.Children {
+						for _, n := range child {
+							if url := n.Attrs["url"]; url != "" {
+								return url
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Check enclosures
+	for _, enc := range item.Enclosures {
+		if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
+			return enc.URL
+		}
+	}
+
+	// Check item image
+	if item.Image != nil && item.Image.URL != "" {
+		return item.Image.URL
+	}
+
+	// Scrape first <img src> from content:encoded or description
+	if url := firstImgSrc(item.Content); url != "" {
+		return url
+	}
+	if url := firstImgSrc(item.Description); url != "" {
+		return url
+	}
+
+	return ""
+}
+
+func firstImgSrc(htmlBody string) string {
+	if htmlBody == "" {
+		return ""
+	}
+	m := imgSrcRe.FindStringSubmatch(htmlBody)
+	if len(m) < 2 {
+		return ""
+	}
+	return html.UnescapeString(m[1])
+}