drwily/gogobee
1
0
Fork 0
mirror of https://github.com/prosolis/gogobee.git synced 2026-07-16 08:52:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

url previews: stop dropping thumbnails on body cap and HEAD-403 CDNs

Browse Source 
Two independent causes, both silent:

LimitedBody returned an error once the cap was hit, so scrapeOG failed the
whole goquery parse on any page over 2 MiB — even though og: tags live in
<head>, near the top. Hitting the cap is a truncation, not a failure: return
io.EOF and let the parser decide whether it found what it needed. Sized
against the pages actually posted here (n=14): og:title landed within the
first 7 KiB on twelve, worst case ~600 KiB.

validateImageURL HEAD-probed the image and bailed on non-200. Some publisher
CDNs (dims.apnews.com among them) answer HEAD with 403 while serving the same
URL over GET, so their thumbnails were always dropped. Probe with HEAD first,
fall back to a ranged GET asking for the first KiB. A 206 declares the full
size in Content-Range's "/119070" suffix rather than Content-Length, so the
tracking-pixel filter reads size from there.
This commit is contained in:
prosolis
2026-07-09 18:52:32 -07:00
parent b5493a0e79
commit ba7b20dfe5
5 changed files with 201 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
internal/plugin/link_thumbnail_test.go
View File

@@ -1,6 +1,9 @@
package plugin
import "testing"
import (
"net/http"
"testing"
)
func TestResolveURL(t *testing.T) {
cases := []struct {
@@ -38,3 +41,51 @@ func TestNormalizeImageURL(t *testing.T) {
t.Errorf("normalizeImageURL unsigned = %q, want width=1200", got)
}
}
func TestDeclaredSize(t *testing.T) {
cases := []struct {
name string
status int
header map[string]string
want int64
}{
{"content-length on a full response", http.StatusOK,
map[string]string{"Content-Length": "119070"}, 119070},
// A ranged probe's Content-Length covers only the slice we asked for, so
// the resource's real size has to come from Content-Range's total.
{"content-range total on a partial response", http.StatusPartialContent,
map[string]string{"Content-Length": "1024", "Content-Range": "bytes 0-1023/119070"}, 119070},
{"unknown total in content-range", http.StatusPartialContent,
map[string]string{"Content-Range": "bytes 0-1023/*"}, -1},
{"partial response missing content-range", http.StatusPartialContent,
map[string]string{"Content-Length": "1024"}, -1},
{"no size declared", http.StatusOK, map[string]string{}, -1},
{"malformed content-length", http.StatusOK,
map[string]string{"Content-Length": "banana"}, -1},
}
for _, c := range cases {
resp := &http.Response{StatusCode: c.status, Header: http.Header{}}
for k, v := range c.header {
resp.Header.Set(k, v)
}
if got := declaredSize(resp); got != c.want {
t.Errorf("%s: declaredSize() = %d, want %d", c.name, got, c.want)
}
}
}
// An undeclared size (-1) must not be mistaken for a zero-byte image and dropped
// as a tracking pixel — plenty of CDNs omit the length entirely.
func TestTrackingPixelFilterSkipsUnknownSize(t *testing.T) {
rejected := func(size int64) bool { return size >= 0 && size <= trackingPixelBytes }
for _, size := range []int64{-1, trackingPixelBytes + 1, 119070} {
if rejected(size) {
t.Errorf("size %d was rejected as a tracking pixel, want kept", size)
}
}
for _, size := range []int64{0, 1, trackingPixelBytes} {
if !rejected(size) {
t.Errorf("size %d was kept, want rejected as a tracking pixel", size)
}
}
}