9e5ba5aafc
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
81 lines
2.0 KiB
Go
81 lines
2.0 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"pete/internal/safehttp"
|
|
)
|
|
|
|
func init() {
|
|
// httptest servers bind to 127.0.0.1; allow the safe-dial check to
|
|
// accept them in tests only.
|
|
safehttp.AllowPrivate = true
|
|
}
|
|
|
|
func TestFetchOGImage(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
body string
|
|
want string // suffix match
|
|
}{
|
|
{
|
|
name: "og:image property",
|
|
body: `<html><head><meta property="og:image" content="https://example.com/lead.jpg"></head></html>`,
|
|
want: "https://example.com/lead.jpg",
|
|
},
|
|
{
|
|
name: "content before property",
|
|
body: `<html><head><meta content="https://example.com/swap.jpg" property="og:image"/></head></html>`,
|
|
want: "https://example.com/swap.jpg",
|
|
},
|
|
{
|
|
name: "twitter:image fallback",
|
|
body: `<html><head><meta name="twitter:image" content="https://example.com/twit.jpg"></head></html>`,
|
|
want: "https://example.com/twit.jpg",
|
|
},
|
|
{
|
|
name: "relative URL resolved",
|
|
body: `<html><head><meta property="og:image" content="/img/lead.jpg"></head></html>`,
|
|
want: "/img/lead.jpg",
|
|
},
|
|
{
|
|
name: "no og tags",
|
|
body: `<html><head><title>nothing</title></head></html>`,
|
|
want: "",
|
|
},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write([]byte(c.body))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
got := FetchOGImage(srv.URL + "/article")
|
|
if c.want == "" {
|
|
if got != "" {
|
|
t.Errorf("expected empty, got %q", got)
|
|
}
|
|
return
|
|
}
|
|
if !strings.HasSuffix(got, c.want) {
|
|
t.Errorf("got %q, want suffix %q", got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFirstImgSrc(t *testing.T) {
|
|
in := `<p>Body <img alt="x" src="https://cdn.example.com/lead.png" width="600"/> more</p>`
|
|
if got := firstImgSrc(in); got != "https://cdn.example.com/lead.png" {
|
|
t.Errorf("got %q", got)
|
|
}
|
|
if got := firstImgSrc(""); got != "" {
|
|
t.Errorf("empty input should yield empty, got %q", got)
|
|
}
|
|
}
|