Initial commit
This commit is contained in:
prosolis
63
internal/ingestion/image.go
Normal file
63
internal/ingestion/image.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var imageClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// ValidateImageURL checks that an image URL returns a valid image response.
|
||||
// Returns false (with no error) on any failure — story posts without image.
|
||||
func ValidateImageURL(url string) bool {
|
||||
if url == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("HEAD", url, nil)
|
||||
if err != nil {
|
||||
slog.Debug("image validation: bad url", "url", url, "err", err)
|
||||
return false
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := imageClient.Do(req)
|
||||
if err != nil {
|
||||
slog.Warn("image validation: request failed", "url", url, "err", err)
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode)
|
||||
return false
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
slog.Warn("image validation: not an image", "url", url, "content_type", contentType)
|
||||
return false
|
||||
}
|
||||
|
||||
// Filter tracking pixels: Content-Length must be > 5KB if present
|
||||
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
||||
size, err := strconv.ParseInt(cl, 10, 64)
|
||||
if err == nil && size <= 5120 {
|
||||
slog.Warn("image validation: too small (likely tracking pixel)", "url", url, "size", size)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
98
internal/ingestion/og.go
Normal file
98
internal/ingestion/og.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Match <meta property="og:image" content="..."> in either attribute order,
|
||||
// and the twitter:image variant.
|
||||
var metaImageRe = regexp.MustCompile(
|
||||
`(?is)<meta\s+[^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
|
||||
`|<meta\s+[^>]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
|
||||
|
||||
var ogClient = &http.Client{Timeout: 8 * time.Second}
|
||||
|
||||
// FetchOGImage fetches the article URL and returns the first og:image or
|
||||
// twitter:image found in the <head>. Returns "" on any failure — callers
|
||||
// should treat this as best-effort.
|
||||
func FetchOGImage(articleURL string) string {
|
||||
if articleURL == "" {
|
||||
return ""
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
|
||||
resp, err := ogClient.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ""
|
||||
}
|
||||
|
||||
// og:image lives in <head>; cap read at 512KB to avoid pulling whole articles.
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
m := metaImageRe.FindSubmatch(body)
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
// One of the two capture groups will be non-empty depending on attr order.
|
||||
for i := 1; i < len(m); i++ {
|
||||
if len(m[i]) > 0 {
|
||||
return resolveURL(articleURL, html.UnescapeString(string(m[i])))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolveURL turns a possibly-relative image URL into an absolute one using
|
||||
// the article URL as the base. Returns the raw input on parse failure.
|
||||
func resolveURL(base, ref string) string {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
|
||||
return ref
|
||||
}
|
||||
if strings.HasPrefix(ref, "//") {
|
||||
if i := strings.Index(base, "://"); i > 0 {
|
||||
return base[:i+1] + ref
|
||||
}
|
||||
return "https:" + ref
|
||||
}
|
||||
// relative path — naive join: scheme://host + ref
|
||||
i := strings.Index(base, "://")
|
||||
if i < 0 {
|
||||
return ref
|
||||
}
|
||||
rest := base[i+3:]
|
||||
slash := strings.Index(rest, "/")
|
||||
if slash < 0 {
|
||||
return fmt.Sprintf("%s%s", base, ref)
|
||||
}
|
||||
host := base[:i+3+slash]
|
||||
if strings.HasPrefix(ref, "/") {
|
||||
return host + ref
|
||||
}
|
||||
return host + "/" + ref
|
||||
}
|
||||
72
internal/ingestion/og_test.go
Normal file
72
internal/ingestion/og_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchOGImage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
body string
|
||||
want string // suffix match
|
||||
}{
|
||||
{
|
||||
name: "og:image property",
|
||||
body: `<html><head><meta property="og:image" content="https://example.com/lead.jpg"></head></html>`,
|
||||
want: "https://example.com/lead.jpg",
|
||||
},
|
||||
{
|
||||
name: "content before property",
|
||||
body: `<html><head><meta content="https://example.com/swap.jpg" property="og:image"/></head></html>`,
|
||||
want: "https://example.com/swap.jpg",
|
||||
},
|
||||
{
|
||||
name: "twitter:image fallback",
|
||||
body: `<html><head><meta name="twitter:image" content="https://example.com/twit.jpg"></head></html>`,
|
||||
want: "https://example.com/twit.jpg",
|
||||
},
|
||||
{
|
||||
name: "relative URL resolved",
|
||||
body: `<html><head><meta property="og:image" content="/img/lead.jpg"></head></html>`,
|
||||
want: "/img/lead.jpg",
|
||||
},
|
||||
{
|
||||
name: "no og tags",
|
||||
body: `<html><head><title>nothing</title></head></html>`,
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte(c.body))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
got := FetchOGImage(srv.URL + "/article")
|
||||
if c.want == "" {
|
||||
if got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !strings.HasSuffix(got, c.want) {
|
||||
t.Errorf("got %q, want suffix %q", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstImgSrc(t *testing.T) {
|
||||
in := `<p>Body <img alt="x" src="https://cdn.example.com/lead.png" width="600"/> more</p>`
|
||||
if got := firstImgSrc(in); got != "https://cdn.example.com/lead.png" {
|
||||
t.Errorf("got %q", got)
|
||||
}
|
||||
if got := firstImgSrc(""); got != "" {
|
||||
t.Errorf("empty input should yield empty, got %q", got)
|
||||
}
|
||||
}
|
||||
157
internal/ingestion/parser.go
Normal file
157
internal/ingestion/parser.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||
|
||||
var feedClient = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// FeedItem represents a parsed RSS item ready for classification.
|
||||
type FeedItem struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
Source string
|
||||
FeedHint string
|
||||
Tier int
|
||||
DirectRoute *string
|
||||
}
|
||||
|
||||
var (
|
||||
htmlTagRe = regexp.MustCompile(`<[^>]*>`)
|
||||
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
|
||||
)
|
||||
|
||||
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items.
|
||||
func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
fp.Client = feedClient
|
||||
fp.UserAgent = userAgent
|
||||
|
||||
feed, err := fp.ParseURLWithContext(feedURL, ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items := make([]FeedItem, 0, len(feed.Items))
|
||||
for _, item := range feed.Items {
|
||||
fi := FeedItem{
|
||||
GUID: itemGUID(item),
|
||||
Headline: strings.TrimSpace(item.Title),
|
||||
Lede: extractLede(item.Description),
|
||||
ImageURL: extractImageURL(item),
|
||||
ArticleURL: strings.TrimSpace(item.Link),
|
||||
}
|
||||
if fi.GUID == "" || fi.ArticleURL == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, fi)
|
||||
}
|
||||
|
||||
slog.Debug("fetched feed", "url", feedURL, "items", len(items))
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func itemGUID(item *gofeed.Item) string {
|
||||
if item.GUID != "" {
|
||||
return item.GUID
|
||||
}
|
||||
return item.Link
|
||||
}
|
||||
|
||||
// extractLede returns the feed description verbatim, with HTML stripped.
|
||||
func extractLede(desc string) string {
|
||||
if desc == "" {
|
||||
return ""
|
||||
}
|
||||
text := htmlTagRe.ReplaceAllString(desc, "")
|
||||
text = html.UnescapeString(text)
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// extractImageURL tries to find an image URL from feed item metadata.
|
||||
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
|
||||
// scraped from content:encoded / description (catches feeds like Ars that
|
||||
// embed the lead image in the article body but not as a media:* element).
|
||||
func extractImageURL(item *gofeed.Item) string {
|
||||
// Check media content (common in RSS 2.0 with media namespace)
|
||||
if item.Extensions != nil {
|
||||
if media, ok := item.Extensions["media"]; ok {
|
||||
if contents, ok := media["content"]; ok {
|
||||
for _, c := range contents {
|
||||
if url := c.Attrs["url"]; url != "" {
|
||||
return url
|
||||
}
|
||||
}
|
||||
}
|
||||
if thumbs, ok := media["thumbnail"]; ok {
|
||||
for _, t := range thumbs {
|
||||
if url := t.Attrs["url"]; url != "" {
|
||||
return url
|
||||
}
|
||||
}
|
||||
}
|
||||
// media:group wraps nested media:content / media:thumbnail in some feeds
|
||||
if groups, ok := media["group"]; ok {
|
||||
for _, g := range groups {
|
||||
for _, child := range g.Children {
|
||||
for _, n := range child {
|
||||
if url := n.Attrs["url"]; url != "" {
|
||||
return url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check enclosures
|
||||
for _, enc := range item.Enclosures {
|
||||
if strings.HasPrefix(enc.Type, "image/") && enc.URL != "" {
|
||||
return enc.URL
|
||||
}
|
||||
}
|
||||
|
||||
// Check item image
|
||||
if item.Image != nil && item.Image.URL != "" {
|
||||
return item.Image.URL
|
||||
}
|
||||
|
||||
// Scrape first <img src> from content:encoded or description
|
||||
if url := firstImgSrc(item.Content); url != "" {
|
||||
return url
|
||||
}
|
||||
if url := firstImgSrc(item.Description); url != "" {
|
||||
return url
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstImgSrc(htmlBody string) string {
|
||||
if htmlBody == "" {
|
||||
return ""
|
||||
}
|
||||
m := imgSrcRe.FindStringSubmatch(htmlBody)
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
return html.UnescapeString(m[1])
|
||||
}
|
||||
66
internal/ingestion/parser_test.go
Normal file
66
internal/ingestion/parser_test.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package ingestion
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestExtractLede(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"plain text verbatim",
|
||||
"The president signed a new bill. More details followed in the afternoon session.",
|
||||
"The president signed a new bill. More details followed in the afternoon session.",
|
||||
},
|
||||
{
|
||||
"single sentence",
|
||||
"Breaking news from the capital.",
|
||||
"Breaking news from the capital.",
|
||||
},
|
||||
{
|
||||
"empty",
|
||||
"",
|
||||
"",
|
||||
},
|
||||
{
|
||||
"html stripped",
|
||||
"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
|
||||
"The company announced a major acquisition. Shares rose 5%.",
|
||||
},
|
||||
{
|
||||
"nested html",
|
||||
"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
|
||||
"Apple revealed the iPhone 16. It features a new chip.",
|
||||
},
|
||||
{
|
||||
"whitespace trimmed",
|
||||
" \n Some news happened today. More to come. \n ",
|
||||
"Some news happened today. More to come.",
|
||||
},
|
||||
{
|
||||
"html entities decoded",
|
||||
"The U.S. & EU agreed on a "landmark" deal worth >$1bn.",
|
||||
`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
|
||||
},
|
||||
{
|
||||
"numeric entities",
|
||||
"It's a new era—one of change.",
|
||||
"It's a new era\u2014one of change.",
|
||||
},
|
||||
{
|
||||
"tags and entities combined",
|
||||
"<p>Oil prices rose & gas fell by >5%.</p>",
|
||||
"Oil prices rose & gas fell by >5%.",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := extractLede(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
197
internal/ingestion/poller.go
Normal file
197
internal/ingestion/poller.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/dedup"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// ProcessFunc is called for each new feed item that needs classification.
|
||||
type ProcessFunc func(item *FeedItem)
|
||||
|
||||
// Poller manages per-source RSS polling goroutines.
|
||||
type Poller struct {
|
||||
sources []config.SourceConfig
|
||||
process ProcessFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewPoller creates a poller for the given sources.
|
||||
func NewPoller(sources []config.SourceConfig, process ProcessFunc) *Poller {
|
||||
return &Poller{
|
||||
sources: sources,
|
||||
process: process,
|
||||
}
|
||||
}
|
||||
|
||||
// Start launches a goroutine per enabled source. Blocks until ctx is cancelled.
|
||||
func (p *Poller) Start(ctx context.Context) {
|
||||
for _, src := range p.sources {
|
||||
if !src.Enabled {
|
||||
slog.Info("source disabled, skipping", "source", src.Name)
|
||||
continue
|
||||
}
|
||||
p.wg.Add(1)
|
||||
go p.pollSource(ctx, src)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait blocks until all polling goroutines have exited.
|
||||
func (p *Poller) Wait() {
|
||||
p.wg.Wait()
|
||||
}
|
||||
|
||||
func (p *Poller) pollSource(ctx context.Context, src config.SourceConfig) {
|
||||
defer p.wg.Done()
|
||||
|
||||
interval := time.Duration(src.PollIntervalMinutes) * time.Minute
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
slog.Info("starting poller", "source", src.Name, "interval", interval)
|
||||
|
||||
// Poll immediately on start, then on ticker
|
||||
p.pollOnce(src)
|
||||
|
||||
consecutiveFailures := 0
|
||||
const adminWarningThreshold = 5
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("poller stopping", "source", src.Name)
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := p.pollOnceWithErr(src); err != nil {
|
||||
consecutiveFailures++
|
||||
slog.Error("feed poll failed",
|
||||
"source", src.Name,
|
||||
"err", err,
|
||||
"consecutive_failures", consecutiveFailures,
|
||||
)
|
||||
if consecutiveFailures == adminWarningThreshold {
|
||||
slog.Warn("persistent feed failure",
|
||||
"source", src.Name,
|
||||
"failures", consecutiveFailures,
|
||||
)
|
||||
// TODO: send admin room warning via matrix client
|
||||
}
|
||||
} else {
|
||||
consecutiveFailures = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnce(src config.SourceConfig) {
|
||||
if err := p.pollOnceWithErr(src); err != nil {
|
||||
slog.Error("feed poll failed", "source", src.Name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
|
||||
items, err := FetchFeed(src.FeedURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newCount := 0
|
||||
for i := range items {
|
||||
if storage.IsGUIDSeen(items[i].GUID) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Last-resort image fallback: feed gave us nothing, scrape og:image
|
||||
// from the article page. Only runs for genuinely new items.
|
||||
if items[i].ImageURL == "" {
|
||||
if og := FetchOGImage(items[i].ArticleURL); og != "" {
|
||||
items[i].ImageURL = og
|
||||
slog.Debug("og:image fallback used",
|
||||
"guid", items[i].GUID, "url", items[i].ArticleURL, "image", og)
|
||||
}
|
||||
}
|
||||
|
||||
canonical := dedup.CanonicalURL(items[i].ArticleURL)
|
||||
headlineNorm := dedup.NormalizeHeadline(items[i].Headline)
|
||||
|
||||
if storage.IsCanonicalSeen(canonical) {
|
||||
slog.Info("dropping duplicate story (canonical URL match)",
|
||||
"guid", items[i].GUID, "url_canonical", canonical, "source", src.Name)
|
||||
continue
|
||||
}
|
||||
if storage.IsHeadlineSeen(src.Name, headlineNorm) {
|
||||
slog.Info("dropping duplicate story (headline match)",
|
||||
"guid", items[i].GUID, "headline_norm", headlineNorm, "source", src.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
// Stamp source metadata onto the item
|
||||
items[i].Source = src.Name
|
||||
items[i].FeedHint = src.FeedHint
|
||||
items[i].Tier = src.Tier
|
||||
items[i].DirectRoute = src.DirectRoute
|
||||
|
||||
// Store immediately (before classification) to prevent re-ingestion
|
||||
if err := storage.InsertStory(&storage.Story{
|
||||
GUID: items[i].GUID,
|
||||
Headline: items[i].Headline,
|
||||
Lede: items[i].Lede,
|
||||
ImageURL: items[i].ImageURL,
|
||||
ArticleURL: items[i].ArticleURL,
|
||||
URLCanonical: canonical,
|
||||
HeadlineNorm: headlineNorm,
|
||||
Source: items[i].Source,
|
||||
FeedHint: items[i].FeedHint,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}); err != nil {
|
||||
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
p.process(&items[i])
|
||||
newCount++
|
||||
}
|
||||
|
||||
if newCount > 0 {
|
||||
slog.Info("ingested new stories", "source", src.Name, "count", newCount)
|
||||
}
|
||||
|
||||
// Retry unclassified stories from this source only (cap at 20 to avoid runaway retries)
|
||||
unclassified, err := storage.GetUnclassifiedStories(src.Name)
|
||||
if err != nil {
|
||||
slog.Error("failed to get unclassified stories", "err", err)
|
||||
return nil
|
||||
}
|
||||
for _, s := range unclassified {
|
||||
// Skip stories we just ingested (they're already being processed above)
|
||||
alreadyProcessed := false
|
||||
for _, item := range items {
|
||||
if item.GUID == s.GUID {
|
||||
alreadyProcessed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if alreadyProcessed {
|
||||
continue
|
||||
}
|
||||
|
||||
p.process(&FeedItem{
|
||||
GUID: s.GUID,
|
||||
Headline: s.Headline,
|
||||
Lede: s.Lede,
|
||||
ImageURL: s.ImageURL,
|
||||
ArticleURL: s.ArticleURL,
|
||||
Source: s.Source,
|
||||
FeedHint: s.FeedHint,
|
||||
Tier: src.Tier,
|
||||
DirectRoute: src.DirectRoute,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user