tags). Returns Fetched=false on any // network/HTTP failure so callers can branch on accessibility. func FetchArticleMeta(articleURL string) ArticleMeta { if articleURL == "" { return ArticleMeta{} } ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil) if err != nil { return ArticleMeta{} } req.Header.Set("User-Agent", userAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml") resp, err := articleClient.Do(req) if err != nil { return ArticleMeta{} } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return ArticleMeta{} } doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return ArticleMeta{} } return ArticleMeta{ ImageURL: extractOGImage(doc, articleURL), BodyChars: extractBodyChars(doc), Fetched: true, } } // FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers // that only care about the image. Returns "" when not found. func FetchOGImage(articleURL string) string { return FetchArticleMeta(articleURL).ImageURL } func extractOGImage(doc *goquery.Document, base string) string { selectors := []string{ `meta[property="og:image:secure_url"]`, `meta[property="og:image:url"]`, `meta[property="og:image"]`, `meta[name="twitter:image:src"]`, `meta[name="twitter:image"]`, } for _, sel := range selectors { if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" { return resolveURL(base, strings.TrimSpace(v)) } } return "" } // extractBodyChars concatenates the text of
tags inside tags, and returns the trimmed length.
func extractBodyChars(doc *goquery.Document) int {
sel := doc.Find("article p, main p")
if sel.Length() == 0 {
sel = doc.Find("p")
}
var b strings.Builder
sel.Each(func(_ int, s *goquery.Selection) {
t := strings.TrimSpace(s.Text())
if t == "" {
return
}
if b.Len() > 0 {
b.WriteByte(' ')
}
b.WriteString(t)
})
return len(strings.TrimSpace(b.String()))
}