petal/internal/suggestions/chunk_test.go

package suggestions

import (
	"strings"
	"testing"
)

func texts(chunks []chunk) []string {
	out := make([]string, 0, len(chunks))
	for _, c := range chunks {
		out = append(out, strings.TrimSpace(c.text))
	}
	return out
}

func TestSplitChunks(t *testing.T) {
	cases := []struct {
		name string
		in   string
		want []string
	}{
		{
			name: "plain sentences",
			in:   "I has two apple. She go to market yesterday! Why?",
			want: []string{"I has two apple.", "She go to market yesterday!", "Why?"},
		},
		{
			// A decimal must not split, or the sentence's identity would churn
			// while she types the number.
			name: "decimals stay whole",
			in:   "It costs 3.50 today. Tomorrow, more.",
			want: []string{"It costs 3.50 today.", "Tomorrow, more."},
		},
		{
			name: "closing quote travels with its sentence",
			in:   `He said "early," and left. She stayed.`,
			want: []string{`He said "early," and left.`, "She stayed."},
		},
		{
			// Chinese runs sentences together with no space after 。 — she writes
			// in both languages in one document.
			name: "cjk terminators split without a space",
			in:   "我想说这句话。但是不知道用英语怎么说。",
			want: []string{"我想说这句话。", "但是不知道用英语怎么说。"},
		},
		{
			name: "newlines break chunks",
			in:   "A list item\nAnother item\n",
			want: []string{"A list item", "Another item"},
		},
		{
			name: "blank runs are dropped",
			in:   "\n\n  \nOnly this.\n\n",
			want: []string{"Only this."},
		},
		{
			name: "trailing fragment is its own chunk",
			in:   "Done. Still writing",
			want: []string{"Done.", "Still writing"},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got := texts(splitChunks(tc.in, ""))
			if len(got) != len(tc.want) {
				t.Fatalf("want %q, got %q", tc.want, got)
			}
			for i := range got {
				if got[i] != tc.want[i] {
					t.Fatalf("chunk %d: want %q, got %q", i, tc.want[i], got[i])
				}
			}
		})
	}
}

// A sentence's identity survives the churn that doesn't change what it says:
// the editor rewrites quotes as she types, and a paragraph reflows.
func TestChunkIdentityIgnoresCosmeticChurn(t *testing.T) {
	a := splitChunks(`She said "hello" softly.`, "")
	b := splitChunks("She said “hello”   softly.", "")
	if len(a) != 1 || len(b) != 1 {
		t.Fatalf("want one chunk each, got %d and %d", len(a), len(b))
	}
	if a[0].hash != b[0].hash {
		t.Fatalf("quote/whitespace churn changed the sentence's identity")
	}
	if same := splitChunks(`She said "hello" softly.`, "academic"); same[0].hash == a[0].hash {
		t.Fatalf("a different tone must be a different reading of the sentence")
	}
}

func TestChangedChunksAndLookup(t *testing.T) {
	chunks := splitChunks("One thing. Another thing. One thing.", "")
	if len(chunks) != 3 {
		t.Fatalf("want 3 chunks, got %d", len(chunks))
	}

	// A repeated sentence is one question, not two.
	if got := changedChunks(chunks, nil); len(got) != 2 {
		t.Fatalf("want 2 distinct changed chunks, got %d", len(got))
	}

	checked := hashSet(chunks[:1])
	changed := changedChunks(chunks, checked)
	if len(changed) != 1 || strings.TrimSpace(changed[0].text) != "Another thing." {
		t.Fatalf("want only the unread sentence, got %q", texts(changed))
	}

	if chunkFor("Another", chunks) != chunks[1].hash {
		t.Fatalf("span was attributed to the wrong sentence")
	}
	// A span the document doesn't contain has no sentence, so it is never cached.
	if chunkFor("nowhere in here", chunks) != "" {
		t.Fatalf("unanchorable span should have no chunk")
	}
}