package suggestions import ( "strings" "testing" ) func texts(chunks []chunk) []string { out := make([]string, 0, len(chunks)) for _, c := range chunks { out = append(out, strings.TrimSpace(c.text)) } return out } func TestSplitChunks(t *testing.T) { cases := []struct { name string in string want []string }{ { name: "plain sentences", in: "I has two apple. She go to market yesterday! Why?", want: []string{"I has two apple.", "She go to market yesterday!", "Why?"}, }, { // A decimal must not split, or the sentence's identity would churn // while she types the number. name: "decimals stay whole", in: "It costs 3.50 today. Tomorrow, more.", want: []string{"It costs 3.50 today.", "Tomorrow, more."}, }, { name: "closing quote travels with its sentence", in: `He said "early," and left. She stayed.`, want: []string{`He said "early," and left.`, "She stayed."}, }, { // Chinese runs sentences together with no space after 。 — she writes // in both languages in one document. name: "cjk terminators split without a space", in: "我想说这句话。但是不知道用英语怎么说。", want: []string{"我想说这句话。", "但是不知道用英语怎么说。"}, }, { name: "newlines break chunks", in: "A list item\nAnother item\n", want: []string{"A list item", "Another item"}, }, { name: "blank runs are dropped", in: "\n\n \nOnly this.\n\n", want: []string{"Only this."}, }, { name: "trailing fragment is its own chunk", in: "Done. Still writing", want: []string{"Done.", "Still writing"}, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := texts(splitChunks(tc.in, "")) if len(got) != len(tc.want) { t.Fatalf("want %q, got %q", tc.want, got) } for i := range got { if got[i] != tc.want[i] { t.Fatalf("chunk %d: want %q, got %q", i, tc.want[i], got[i]) } } }) } } // A sentence's identity survives the churn that doesn't change what it says: // the editor rewrites quotes as she types, and a paragraph reflows. func TestChunkIdentityIgnoresCosmeticChurn(t *testing.T) { a := splitChunks(`She said "hello" softly.`, "") b := splitChunks("She said “hello” softly.", "") if len(a) != 1 || len(b) != 1 { t.Fatalf("want one chunk each, got %d and %d", len(a), len(b)) } if a[0].hash != b[0].hash { t.Fatalf("quote/whitespace churn changed the sentence's identity") } if same := splitChunks(`She said "hello" softly.`, "academic"); same[0].hash == a[0].hash { t.Fatalf("a different tone must be a different reading of the sentence") } } func TestChangedChunksAndLookup(t *testing.T) { chunks := splitChunks("One thing. Another thing. One thing.", "") if len(chunks) != 3 { t.Fatalf("want 3 chunks, got %d", len(chunks)) } // A repeated sentence is one question, not two. if got := changedChunks(chunks, nil); len(got) != 2 { t.Fatalf("want 2 distinct changed chunks, got %d", len(got)) } checked := hashSet(chunks[:1]) changed := changedChunks(chunks, checked) if len(changed) != 1 || strings.TrimSpace(changed[0].text) != "Another thing." { t.Fatalf("want only the unread sentence, got %q", texts(changed)) } if chunkFor("Another", chunks) != chunks[1].hash { t.Fatalf("span was attributed to the wrong sentence") } // A span the document doesn't contain has no sentence, so it is never cached. if chunkFor("nowhere in here", chunks) != "" { t.Fatalf("unanchorable span should have no chunk") } }