Pete/internal/ingestion/parser.go

package ingestion

import (
	"context"
	"html"
	"log/slog"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/mmcdole/gofeed"
)

const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"

var feedClient = &http.Client{Timeout: 30 * time.Second}

// FeedItem represents a parsed RSS item ready for routing.
type FeedItem struct {
	GUID        string
	Headline    string
	Lede        string
	ImageURL    string
	ArticleURL  string
	Source      string
	DirectRoute string
}

var (
	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
)


// FetchFeed fetches and parses an RSS/Atom feed, returning items.
func FetchFeed(feedURL string) ([]FeedItem, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	fp := gofeed.NewParser()
	fp.Client = feedClient
	fp.UserAgent = userAgent

	feed, err := fp.ParseURLWithContext(feedURL, ctx)
	if err != nil {
		return nil, err
	}

	items := make([]FeedItem, 0, len(feed.Items))
	for _, item := range feed.Items {
		fi := FeedItem{
			GUID:       itemGUID(item),
			Headline:   strings.TrimSpace(item.Title),
			Lede:       extractLede(item.Description),
			ImageURL:   NormalizeImageURL(extractImageURL(item)),
			ArticleURL: strings.TrimSpace(item.Link),
		}
		if fi.GUID == "" || fi.ArticleURL == "" {
			continue
		}
		items = append(items, fi)
	}

	slog.Debug("fetched feed", "url", feedURL, "items", len(items))
	return items, nil
}

func itemGUID(item *gofeed.Item) string {
	if item.GUID != "" {
		return item.GUID
	}
	return item.Link
}

// extractLede returns the feed description verbatim, with HTML stripped.
func extractLede(desc string) string {
	if desc == "" {
		return ""
	}
	text := htmlTagRe.ReplaceAllString(desc, "")
	text = html.UnescapeString(text)
	return strings.TrimSpace(text)
}

// extractImageURL tries to find an image URL from feed item metadata.
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
// scraped from content:encoded / description (catches feeds like Ars that
// embed the lead image in the article body but not as a media:* element).
func extractImageURL(item *gofeed.Item) string {
	// Check media content (common in RSS 2.0 with media namespace)
	if item.Extensions != nil {
		if media, ok := item.Extensions["media"]; ok {
			if contents, ok := media["content"]; ok {
				for _, c := range contents {
					if url := c.Attrs["url"]; url != "" {
						return url
					}
				}
			}
			if thumbs, ok := media["thumbnail"]; ok {
				for _, t := range thumbs {
					if url := t.Attrs["url"]; url != "" {
						return url
					}
				}
			}
			// media:group wraps nested media:content / media:thumbnail in some feeds
			if groups, ok := media["group"]; ok {
				for _, g := range groups {
					for _, child := range g.Children {
						for _, n := range child {
							if url := n.Attrs["url"]; url != "" {
								return url
							}
						}
					}
				}
			}
		}
	}

	// Check enclosures
	for _, enc := range item.Enclosures {
		if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
			return enc.URL
		}
	}

	// Check item image
	if item.Image != nil && item.Image.URL != "" {
		return item.Image.URL
	}

	// Scrape first <img src> from content:encoded or description
	if url := firstImgSrc(item.Content); url != "" {
		return url
	}
	if url := firstImgSrc(item.Description); url != "" {
		return url
	}

	return ""
}

func firstImgSrc(htmlBody string) string {
	if htmlBody == "" {
		return ""
	}
	m := imgSrcRe.FindStringSubmatch(htmlBody)
	if len(m) < 2 {
		return ""
	}
	return html.UnescapeString(m[1])
}