9e5ba5aafc
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
102 lines
2.3 KiB
Go
102 lines
2.3 KiB
Go
// Package dedup provides deterministic normalization helpers for detecting
|
|
// duplicate stories across feeds.
|
|
package dedup
|
|
|
|
import (
|
|
"net/url"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// trackingParams are query-string keys stripped during URL canonicalization.
|
|
// Anything starting with "utm_" is also stripped.
|
|
var trackingParams = map[string]struct{}{
|
|
"fbclid": {},
|
|
"gclid": {},
|
|
"mc_cid": {},
|
|
"mc_eid": {},
|
|
"ref": {},
|
|
"ref_src": {},
|
|
"ref_url": {},
|
|
"share": {},
|
|
"shared": {},
|
|
"cmpid": {},
|
|
"cmp": {},
|
|
"icid": {},
|
|
"ito": {},
|
|
"yclid": {},
|
|
"_ga": {},
|
|
"igshid": {},
|
|
"feature": {},
|
|
"si": {},
|
|
"smid": {},
|
|
"twclid": {},
|
|
"mibextid": {},
|
|
"src": {},
|
|
}
|
|
|
|
// CanonicalURL returns a stable form of a story URL suitable for dedup.
|
|
// It lowercases the scheme/host, drops fragments and common tracking params,
|
|
// and trims trailing slashes. If parsing fails, the input is returned unchanged.
|
|
func CanonicalURL(raw string) string {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return ""
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host == "" {
|
|
return raw
|
|
}
|
|
|
|
u.Scheme = strings.ToLower(u.Scheme)
|
|
u.Host = strings.ToLower(u.Host)
|
|
u.Host = strings.TrimPrefix(u.Host, "www.")
|
|
u.Host = strings.TrimPrefix(u.Host, "m.")
|
|
u.Host = strings.TrimPrefix(u.Host, "amp.")
|
|
u.Fragment = ""
|
|
|
|
if q := u.Query(); len(q) > 0 {
|
|
for k := range q {
|
|
lk := strings.ToLower(k)
|
|
if strings.HasPrefix(lk, "utm_") {
|
|
q.Del(k)
|
|
continue
|
|
}
|
|
if _, drop := trackingParams[lk]; drop {
|
|
q.Del(k)
|
|
}
|
|
}
|
|
u.RawQuery = q.Encode()
|
|
}
|
|
|
|
u.Path = strings.TrimRight(u.Path, "/")
|
|
if u.Path == "" {
|
|
u.Path = "/"
|
|
}
|
|
return u.String()
|
|
}
|
|
|
|
// NormalizeHeadline returns a comparable form of a headline: lowercase,
|
|
// punctuation stripped, whitespace collapsed. Empty input returns "".
|
|
func NormalizeHeadline(h string) string {
|
|
if h == "" {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
b.Grow(len(h))
|
|
prevSpace := true
|
|
for _, r := range h {
|
|
switch {
|
|
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
|
b.WriteRune(unicode.ToLower(r))
|
|
prevSpace = false
|
|
case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
|
|
if !prevSpace {
|
|
b.WriteByte(' ')
|
|
prevSpace = true
|
|
}
|
|
}
|
|
}
|
|
return strings.TrimSpace(b.String())
|
|
}
|