mirror of
https://github.com/prosolis/gogobee.git
synced 2026-07-15 08:32:41 +00:00
ba7b20dfe5
Two independent causes, both silent: LimitedBody returned an error once the cap was hit, so scrapeOG failed the whole goquery parse on any page over 2 MiB — even though og: tags live in <head>, near the top. Hitting the cap is a truncation, not a failure: return io.EOF and let the parser decide whether it found what it needed. Sized against the pages actually posted here (n=14): og:title landed within the first 7 KiB on twelve, worst case ~600 KiB. validateImageURL HEAD-probed the image and bailed on non-200. Some publisher CDNs (dims.apnews.com among them) answer HEAD with 403 while serving the same URL over GET, so their thumbnails were always dropped. Probe with HEAD first, fall back to a ranged GET asking for the first KiB. A 206 declares the full size in Content-Range's "/119070" suffix rather than Content-Length, so the tracking-pixel filter reads size from there.
56 lines
1.4 KiB
Go
56 lines
1.4 KiB
Go
package safehttp
|
|
|
|
import (
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A body under the cap is passed through untouched.
|
|
func TestLimitedBodyUnderCap(t *testing.T) {
|
|
got, err := io.ReadAll(LimitedBody(strings.NewReader("hello"), 1024))
|
|
if err != nil {
|
|
t.Fatalf("ReadAll: %v", err)
|
|
}
|
|
if string(got) != "hello" {
|
|
t.Fatalf("got %q, want %q", got, "hello")
|
|
}
|
|
}
|
|
|
|
// A body over the cap truncates cleanly at EOF rather than failing the read.
|
|
// Callers parse whatever fits (an HTML <head>, an image header) instead of
|
|
// losing the whole document to an oversized tail.
|
|
func TestLimitedBodyTruncatesAtEOF(t *testing.T) {
|
|
body := strings.Repeat("x", 5000)
|
|
got, err := io.ReadAll(LimitedBody(strings.NewReader(body), 100))
|
|
if err != nil {
|
|
t.Fatalf("ReadAll returned an error instead of truncating: %v", err)
|
|
}
|
|
if len(got) != 100 {
|
|
t.Fatalf("read %d bytes, want the 100-byte cap", len(got))
|
|
}
|
|
}
|
|
|
|
// The cap bounds total bytes across many small reads, not just a single one.
|
|
func TestLimitedBodyCapsAcrossReads(t *testing.T) {
|
|
r := LimitedBody(strings.NewReader(strings.Repeat("x", 5000)), 10)
|
|
total := 0
|
|
buf := make([]byte, 3)
|
|
for {
|
|
n, err := r.Read(buf)
|
|
total += n
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("Read: %v", err)
|
|
}
|
|
if total > 10 {
|
|
t.Fatalf("read %d bytes past the 10-byte cap", total)
|
|
}
|
|
}
|
|
if total != 10 {
|
|
t.Fatalf("read %d bytes, want 10", total)
|
|
}
|
|
}
|