mirror of
https://github.com/prosolis/gogobee.git
synced 2026-07-15 08:32:41 +00:00
ba7b20dfe5
Two independent causes, both silent: LimitedBody returned an error once the cap was hit, so scrapeOG failed the whole goquery parse on any page over 2 MiB — even though og: tags live in <head>, near the top. Hitting the cap is a truncation, not a failure: return io.EOF and let the parser decide whether it found what it needed. Sized against the pages actually posted here (n=14): og:title landed within the first 7 KiB on twelve, worst case ~600 KiB. validateImageURL HEAD-probed the image and bailed on non-200. Some publisher CDNs (dims.apnews.com among them) answer HEAD with 403 while serving the same URL over GET, so their thumbnails were always dropped. Probe with HEAD first, fall back to a ranged GET asking for the first KiB. A 206 declares the full size in Content-Range's "/119070" suffix rather than Content-Length, so the tracking-pixel filter reads size from there.
92 lines
3.3 KiB
Go
92 lines
3.3 KiB
Go
package plugin
|
|
|
|
import (
|
|
"net/http"
|
|
"testing"
|
|
)
|
|
|
|
func TestResolveURL(t *testing.T) {
|
|
cases := []struct {
|
|
base, ref, want string
|
|
}{
|
|
{"https://e.com/news/story", "https://cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"},
|
|
{"https://e.com/news/story", "//cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"},
|
|
{"https://e.com/news/story", "/img/a.jpg", "https://e.com/img/a.jpg"},
|
|
{"https://e.com/news/story", "a.jpg", "https://e.com/news/a.jpg"},
|
|
{"https://e.com/news/story", "", ""},
|
|
}
|
|
for _, c := range cases {
|
|
if got := resolveURL(c.base, c.ref); got != c.want {
|
|
t.Errorf("resolveURL(%q, %q) = %q, want %q", c.base, c.ref, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeImageURL(t *testing.T) {
|
|
cases := []struct {
|
|
name, in, want string
|
|
}{
|
|
{"non-guardian passthrough", "https://e.com/a.jpg?width=140", "https://e.com/a.jpg?width=140"},
|
|
{"signed left alone", "https://i.guim.co.uk/x.jpg?width=140&s=abc", "https://i.guim.co.uk/x.jpg?width=140&s=abc"},
|
|
{"no width passthrough", "https://i.guim.co.uk/x.jpg", "https://i.guim.co.uk/x.jpg"},
|
|
{"empty", "", ""},
|
|
}
|
|
for _, c := range cases {
|
|
if got := normalizeImageURL(c.in); got != c.want {
|
|
t.Errorf("%s: normalizeImageURL(%q) = %q, want %q", c.name, c.in, got, c.want)
|
|
}
|
|
}
|
|
// Unsigned Guardian thumbnail gets bumped to width=1200.
|
|
if got := normalizeImageURL("https://i.guim.co.uk/x.jpg?width=140"); got != "https://i.guim.co.uk/x.jpg?width=1200" {
|
|
t.Errorf("normalizeImageURL unsigned = %q, want width=1200", got)
|
|
}
|
|
}
|
|
|
|
func TestDeclaredSize(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
status int
|
|
header map[string]string
|
|
want int64
|
|
}{
|
|
{"content-length on a full response", http.StatusOK,
|
|
map[string]string{"Content-Length": "119070"}, 119070},
|
|
// A ranged probe's Content-Length covers only the slice we asked for, so
|
|
// the resource's real size has to come from Content-Range's total.
|
|
{"content-range total on a partial response", http.StatusPartialContent,
|
|
map[string]string{"Content-Length": "1024", "Content-Range": "bytes 0-1023/119070"}, 119070},
|
|
{"unknown total in content-range", http.StatusPartialContent,
|
|
map[string]string{"Content-Range": "bytes 0-1023/*"}, -1},
|
|
{"partial response missing content-range", http.StatusPartialContent,
|
|
map[string]string{"Content-Length": "1024"}, -1},
|
|
{"no size declared", http.StatusOK, map[string]string{}, -1},
|
|
{"malformed content-length", http.StatusOK,
|
|
map[string]string{"Content-Length": "banana"}, -1},
|
|
}
|
|
for _, c := range cases {
|
|
resp := &http.Response{StatusCode: c.status, Header: http.Header{}}
|
|
for k, v := range c.header {
|
|
resp.Header.Set(k, v)
|
|
}
|
|
if got := declaredSize(resp); got != c.want {
|
|
t.Errorf("%s: declaredSize() = %d, want %d", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An undeclared size (-1) must not be mistaken for a zero-byte image and dropped
|
|
// as a tracking pixel — plenty of CDNs omit the length entirely.
|
|
func TestTrackingPixelFilterSkipsUnknownSize(t *testing.T) {
|
|
rejected := func(size int64) bool { return size >= 0 && size <= trackingPixelBytes }
|
|
for _, size := range []int64{-1, trackingPixelBytes + 1, 119070} {
|
|
if rejected(size) {
|
|
t.Errorf("size %d was rejected as a tracking pixel, want kept", size)
|
|
}
|
|
}
|
|
for _, size := range []int64{0, 1, trackingPixelBytes} {
|
|
if !rejected(size) {
|
|
t.Errorf("size %d was kept, want rejected as a tracking pixel", size)
|
|
}
|
|
}
|
|
}
|