Initial commit

internal/dedup/dedup_test.go

@@ -0,0 +1,43 @@
+package dedup
+
+import "testing"
+
+func TestCanonicalURL(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"https://example.com/foo", "https://example.com/foo"},
+		{"https://www.example.com/foo/", "https://example.com/foo"},
+		{"HTTPS://Example.com/Foo?utm_source=x&utm_medium=y", "https://example.com/Foo"},
+		{"https://example.com/foo?id=1&utm_campaign=z&fbclid=abc", "https://example.com/foo?id=1"},
+		{"https://example.com/foo#section", "https://example.com/foo"},
+		{"https://m.bbc.com/news/123/", "https://bbc.com/news/123"},
+		{"https://amp.cnn.com/article", "https://cnn.com/article"},
+		{"not a url", "not a url"},
+	}
+	for _, c := range cases {
+		got := CanonicalURL(c.in)
+		if got != c.want {
+			t.Errorf("CanonicalURL(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestNormalizeHeadline(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"Hello, World!", "hello world"},
+		{"  Multiple   Spaces  ", "multiple spaces"},
+		{"Nintendo's Switch 2 — Day One!", "nintendo s switch 2 day one"},
+		{"BREAKING: Foo bar", "breaking foo bar"},
+	}
+	for _, c := range cases {
+		got := NormalizeHeadline(c.in)
+		if got != c.want {
+			t.Errorf("NormalizeHeadline(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}