Initial commit

internal/dedup/dedup.go

@@ -0,0 +1,101 @@
+// Package dedup provides deterministic normalization helpers for detecting
+// duplicate stories across feeds.
+package dedup
+
+import (
+	"net/url"
+	"strings"
+	"unicode"
+)
+
+// trackingParams are query-string keys stripped during URL canonicalization.
+// Anything starting with "utm_" is also stripped.
+var trackingParams = map[string]struct{}{
+	"fbclid":      {},
+	"gclid":       {},
+	"mc_cid":      {},
+	"mc_eid":      {},
+	"ref":         {},
+	"ref_src":     {},
+	"ref_url":     {},
+	"share":       {},
+	"shared":      {},
+	"cmpid":       {},
+	"CMP":         {},
+	"icid":        {},
+	"ito":         {},
+	"yclid":       {},
+	"_ga":         {},
+	"igshid":      {},
+	"feature":     {},
+	"si":          {},
+	"smid":        {},
+	"twclid":      {},
+	"mibextid":    {},
+	"src":         {},
+}
+
+// CanonicalURL returns a stable form of a story URL suitable for dedup.
+// It lowercases the scheme/host, drops fragments and common tracking params,
+// and trims trailing slashes. If parsing fails, the input is returned unchanged.
+func CanonicalURL(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	u, err := url.Parse(raw)
+	if err != nil || u.Host == "" {
+		return raw
+	}
+
+	u.Scheme = strings.ToLower(u.Scheme)
+	u.Host = strings.ToLower(u.Host)
+	u.Host = strings.TrimPrefix(u.Host, "www.")
+	u.Host = strings.TrimPrefix(u.Host, "m.")
+	u.Host = strings.TrimPrefix(u.Host, "amp.")
+	u.Fragment = ""
+
+	if q := u.Query(); len(q) > 0 {
+		for k := range q {
+			lk := strings.ToLower(k)
+			if strings.HasPrefix(lk, "utm_") {
+				q.Del(k)
+				continue
+			}
+			if _, drop := trackingParams[lk]; drop {
+				q.Del(k)
+			}
+		}
+		u.RawQuery = q.Encode()
+	}
+
+	u.Path = strings.TrimRight(u.Path, "/")
+	if u.Path == "" {
+		u.Path = "/"
+	}
+	return u.String()
+}
+
+// NormalizeHeadline returns a comparable form of a headline: lowercase,
+// punctuation stripped, whitespace collapsed. Empty input returns "".
+func NormalizeHeadline(h string) string {
+	if h == "" {
+		return ""
+	}
+	var b strings.Builder
+	b.Grow(len(h))
+	prevSpace := true
+	for _, r := range h {
+		switch {
+		case unicode.IsLetter(r) || unicode.IsDigit(r):
+			b.WriteRune(unicode.ToLower(r))
+			prevSpace = false
+		case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
+			if !prevSpace {
+				b.WriteByte(' ')
+				prevSpace = true
+			}
+		}
+	}
+	return strings.TrimSpace(b.String())
+}