drwily/gogobee
1
0
Fork 0
mirror of https://github.com/prosolis/gogobee.git synced 2026-07-16 08:52:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

url previews: stop dropping thumbnails on body cap and HEAD-403 CDNs

Browse Source 
Two independent causes, both silent:

LimitedBody returned an error once the cap was hit, so scrapeOG failed the
whole goquery parse on any page over 2 MiB — even though og: tags live in
<head>, near the top. Hitting the cap is a truncation, not a failure: return
io.EOF and let the parser decide whether it found what it needed. Sized
against the pages actually posted here (n=14): og:title landed within the
first 7 KiB on twelve, worst case ~600 KiB.

validateImageURL HEAD-probed the image and bailed on non-200. Some publisher
CDNs (dims.apnews.com among them) answer HEAD with 403 while serving the same
URL over GET, so their thumbnails were always dropped. Probe with HEAD first,
fall back to a ranged GET asking for the first KiB. A 206 declares the full
size in Content-Range's "/119070" suffix rather than Content-Length, so the
tracking-pixel filter reads size from there.
This commit is contained in:
prosolis
2026-07-09 18:52:32 -07:00
parent b5493a0e79
commit ba7b20dfe5
5 changed files with 201 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
internal/safehttp/safehttp.go
View File

@@ -120,10 +120,16 @@ func NewClient(timeout time.Duration) *http.Client {
}
}
// LimitedBody wraps r in a reader that errors once more than max bytes have
// been read. Use to cap how much of a response body downstream parsers
// (goquery, image.Decode) will ever see — a hostile origin streaming an
// endless body otherwise OOMs the process.
// LimitedBody wraps r in a reader that reports EOF once max bytes have been
// read. Use to cap how much of a response body downstream parsers (goquery,
// image.Decode) will ever see — a hostile origin streaming an endless body
// otherwise OOMs the process.
//
// Hitting the cap is a truncation, not an error: parsers get a short-but-valid
// body and decide for themselves whether they found what they needed. Returning
// an error here instead would fail the whole parse on any oversized page, even
// when the interesting bytes (an HTML <head>, an image header) sit well inside
// the cap.
func LimitedBody(r io.Reader, max int64) io.Reader {
return &limitedReader{R: r, N: max}
}
@@ -135,7 +141,7 @@ type limitedReader struct {
func (l *limitedReader) Read(p []byte) (int, error) {
if l.N <= 0 {
return 0, fmt.Errorf("safehttp: response body exceeded cap")
return 0, io.EOF
}
if int64(len(p)) > l.N {
p = p[:l.N]

55
internal/safehttp/safehttp_test.go Normal file
View File

@@ -0,0 +1,55 @@
package safehttp
import (
"io"
"strings"
"testing"
)
// A body under the cap is passed through untouched.
func TestLimitedBodyUnderCap(t *testing.T) {
got, err := io.ReadAll(LimitedBody(strings.NewReader("hello"), 1024))
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
if string(got) != "hello" {
t.Fatalf("got %q, want %q", got, "hello")
}
}
// A body over the cap truncates cleanly at EOF rather than failing the read.
// Callers parse whatever fits (an HTML <head>, an image header) instead of
// losing the whole document to an oversized tail.
func TestLimitedBodyTruncatesAtEOF(t *testing.T) {
body := strings.Repeat("x", 5000)
got, err := io.ReadAll(LimitedBody(strings.NewReader(body), 100))
if err != nil {
t.Fatalf("ReadAll returned an error instead of truncating: %v", err)
}
if len(got) != 100 {
t.Fatalf("read %d bytes, want the 100-byte cap", len(got))
}
}
// The cap bounds total bytes across many small reads, not just a single one.
func TestLimitedBodyCapsAcrossReads(t *testing.T) {
r := LimitedBody(strings.NewReader(strings.Repeat("x", 5000)), 10)
total := 0
buf := make([]byte, 3)
for {
n, err := r.Read(buf)
total += n
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("Read: %v", err)
}
if total > 10 {
t.Fatalf("read %d bytes past the 10-byte cap", total)
}
}
if total != 10 {
t.Fatalf("read %d bytes, want 10", total)
}
}