Initial commit

2026-05-22 17:25:27 -07:00
commit 652d6dfa38
40 changed files with 5855 additions and 0 deletions
--- a/internal/ingestion/og.go
+++ b/internal/ingestion/og.go
@@ -0,0 +1,98 @@
+package ingestion
+
+import (
+	"context"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// Match <meta property="og:image" content="..."> in either attribute order,
+// and the twitter:image variant.
+var metaImageRe = regexp.MustCompile(
+	`(?is)<meta\s+[^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
+		`|<meta\s+[^>]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
+
+var ogClient = &http.Client{Timeout: 8 * time.Second}
+
+// FetchOGImage fetches the article URL and returns the first og:image or
+// twitter:image found in the <head>. Returns "" on any failure — callers
+// should treat this as best-effort.
+func FetchOGImage(articleURL string) string {
+	if articleURL == "" {
+		return ""
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
+	if err != nil {
+		return ""
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := ogClient.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ""
+	}
+
+	// og:image lives in <head>; cap read at 512KB to avoid pulling whole articles.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
+	if err != nil {
+		return ""
+	}
+
+	m := metaImageRe.FindSubmatch(body)
+	if len(m) == 0 {
+		return ""
+	}
+	// One of the two capture groups will be non-empty depending on attr order.
+	for i := 1; i < len(m); i++ {
+		if len(m[i]) > 0 {
+			return resolveURL(articleURL, html.UnescapeString(string(m[i])))
+		}
+	}
+	return ""
+}
+
+// resolveURL turns a possibly-relative image URL into an absolute one using
+// the article URL as the base. Returns the raw input on parse failure.
+func resolveURL(base, ref string) string {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return ""
+	}
+	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		return ref
+	}
+	if strings.HasPrefix(ref, "//") {
+		if i := strings.Index(base, "://"); i > 0 {
+			return base[:i+1] + ref
+		}
+		return "https:" + ref
+	}
+	// relative path — naive join: scheme://host + ref
+	i := strings.Index(base, "://")
+	if i < 0 {
+		return ref
+	}
+	rest := base[i+3:]
+	slash := strings.Index(rest, "/")
+	if slash < 0 {
+		return fmt.Sprintf("%s%s", base, ref)
+	}
+	host := base[:i+3+slash]
+	if strings.HasPrefix(ref, "/") {
+		return host + ref
+	}
+	return host + "/" + ref
+}