99 lines
2.5 KiB
Go
99 lines
2.5 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Match <meta property="og:image" content="..."> in either attribute order,
|
|
// and the twitter:image variant.
|
|
var metaImageRe = regexp.MustCompile(
|
|
`(?is)<meta\s+[^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
|
|
`|<meta\s+[^>]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
|
|
|
|
var ogClient = &http.Client{Timeout: 8 * time.Second}
|
|
|
|
// FetchOGImage fetches the article URL and returns the first og:image or
|
|
// twitter:image found in the <head>. Returns "" on any failure — callers
|
|
// should treat this as best-effort.
|
|
func FetchOGImage(articleURL string) string {
|
|
if articleURL == "" {
|
|
return ""
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
|
|
|
resp, err := ogClient.Do(req)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return ""
|
|
}
|
|
|
|
// og:image lives in <head>; cap read at 512KB to avoid pulling whole articles.
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
|
|
m := metaImageRe.FindSubmatch(body)
|
|
if len(m) == 0 {
|
|
return ""
|
|
}
|
|
// One of the two capture groups will be non-empty depending on attr order.
|
|
for i := 1; i < len(m); i++ {
|
|
if len(m[i]) > 0 {
|
|
return resolveURL(articleURL, html.UnescapeString(string(m[i])))
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// resolveURL turns a possibly-relative image URL into an absolute one using
|
|
// the article URL as the base. Returns the raw input on parse failure.
|
|
func resolveURL(base, ref string) string {
|
|
ref = strings.TrimSpace(ref)
|
|
if ref == "" {
|
|
return ""
|
|
}
|
|
if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
|
|
return ref
|
|
}
|
|
if strings.HasPrefix(ref, "//") {
|
|
if i := strings.Index(base, "://"); i > 0 {
|
|
return base[:i+1] + ref
|
|
}
|
|
return "https:" + ref
|
|
}
|
|
// relative path — naive join: scheme://host + ref
|
|
i := strings.Index(base, "://")
|
|
if i < 0 {
|
|
return ref
|
|
}
|
|
rest := base[i+3:]
|
|
slash := strings.Index(rest, "/")
|
|
if slash < 0 {
|
|
return fmt.Sprintf("%s%s", base, ref)
|
|
}
|
|
host := base[:i+3+slash]
|
|
if strings.HasPrefix(ref, "/") {
|
|
return host + ref
|
|
}
|
|
return host + "/" + ref
|
|
}
|