Initial commit

internal/dedup/dedup.go

@@ -0,0 +1,101 @@
+// Package dedup provides deterministic normalization helpers for detecting
+// duplicate stories across feeds.
+package dedup
+
+import (
+	"net/url"
+	"strings"
+	"unicode"
+)
+
+// trackingParams are query-string keys stripped during URL canonicalization.
+// Anything starting with "utm_" is also stripped.
+var trackingParams = map[string]struct{}{
+	"fbclid":      {},
+	"gclid":       {},
+	"mc_cid":      {},
+	"mc_eid":      {},
+	"ref":         {},
+	"ref_src":     {},
+	"ref_url":     {},
+	"share":       {},
+	"shared":      {},
+	"cmpid":       {},
+	"CMP":         {},
+	"icid":        {},
+	"ito":         {},
+	"yclid":       {},
+	"_ga":         {},
+	"igshid":      {},
+	"feature":     {},
+	"si":          {},
+	"smid":        {},
+	"twclid":      {},
+	"mibextid":    {},
+	"src":         {},
+}
+
+// CanonicalURL returns a stable form of a story URL suitable for dedup.
+// It lowercases the scheme/host, drops fragments and common tracking params,
+// and trims trailing slashes. If parsing fails, the input is returned unchanged.
+func CanonicalURL(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return ""
+	}
+	u, err := url.Parse(raw)
+	if err != nil || u.Host == "" {
+		return raw
+	}
+
+	u.Scheme = strings.ToLower(u.Scheme)
+	u.Host = strings.ToLower(u.Host)
+	u.Host = strings.TrimPrefix(u.Host, "www.")
+	u.Host = strings.TrimPrefix(u.Host, "m.")
+	u.Host = strings.TrimPrefix(u.Host, "amp.")
+	u.Fragment = ""
+
+	if q := u.Query(); len(q) > 0 {
+		for k := range q {
+			lk := strings.ToLower(k)
+			if strings.HasPrefix(lk, "utm_") {
+				q.Del(k)
+				continue
+			}
+			if _, drop := trackingParams[lk]; drop {
+				q.Del(k)
+			}
+		}
+		u.RawQuery = q.Encode()
+	}
+
+	u.Path = strings.TrimRight(u.Path, "/")
+	if u.Path == "" {
+		u.Path = "/"
+	}
+	return u.String()
+}
+
+// NormalizeHeadline returns a comparable form of a headline: lowercase,
+// punctuation stripped, whitespace collapsed. Empty input returns "".
+func NormalizeHeadline(h string) string {
+	if h == "" {
+		return ""
+	}
+	var b strings.Builder
+	b.Grow(len(h))
+	prevSpace := true
+	for _, r := range h {
+		switch {
+		case unicode.IsLetter(r) || unicode.IsDigit(r):
+			b.WriteRune(unicode.ToLower(r))
+			prevSpace = false
+		case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
+			if !prevSpace {
+				b.WriteByte(' ')
+				prevSpace = true
+			}
+		}
+	}
+	return strings.TrimSpace(b.String())
+}

internal/dedup/dedup_test.go

@@ -0,0 +1,43 @@
+package dedup
+
+import "testing"
+
+func TestCanonicalURL(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"https://example.com/foo", "https://example.com/foo"},
+		{"https://www.example.com/foo/", "https://example.com/foo"},
+		{"HTTPS://Example.com/Foo?utm_source=x&utm_medium=y", "https://example.com/Foo"},
+		{"https://example.com/foo?id=1&utm_campaign=z&fbclid=abc", "https://example.com/foo?id=1"},
+		{"https://example.com/foo#section", "https://example.com/foo"},
+		{"https://m.bbc.com/news/123/", "https://bbc.com/news/123"},
+		{"https://amp.cnn.com/article", "https://cnn.com/article"},
+		{"not a url", "not a url"},
+	}
+	for _, c := range cases {
+		got := CanonicalURL(c.in)
+		if got != c.want {
+			t.Errorf("CanonicalURL(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestNormalizeHeadline(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"Hello, World!", "hello world"},
+		{"  Multiple   Spaces  ", "multiple spaces"},
+		{"Nintendo's Switch 2 — Day One!", "nintendo s switch 2 day one"},
+		{"BREAKING: Foo bar", "breaking foo bar"},
+	}
+	for _, c := range cases {
+		got := NormalizeHeadline(c.in)
+		if got != c.want {
+			t.Errorf("NormalizeHeadline(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}