petal/internal/suggestions/chunk.go

package suggestions

import (
	"crypto/sha256"
	"encoding/hex"
	"strings"
	"unicode"
)

// Chunking splits a document into sentence-sized units so a re-check can ask the
// model only about the sentences that actually changed. Accepting one edit used
// to re-run the whole document: every card vanished, came back with a new id and
// a freshly-worded explanation, and spans re-merged into different shapes. The
// sentences she didn't touch have nothing new to say about themselves, so their
// suggestions are simply kept (see reconcilePending).
//
// A chunk's identity is its hash, not its position — she inserts a paragraph at
// the top and every sentence below keeps its suggestions.

// chunk is one sentence of the document, with the hash that identifies it.
type chunk struct {
	text string
	hash string
}

// asciiTerminators end a sentence only when whitespace (or the end of the text)
// follows, so "3.5" and "Ms." don't split mid-word — a wrong split costs only a
// slightly smaller chunk, but a split inside a number would churn its hash on
// every keystroke around it.
const asciiTerminators = ".!?"

// cjkTerminators end a sentence outright: Chinese runs sentences together with
// no space after 。, and she writes in both languages in one document.
const cjkTerminators = "。！？"

// closers are swallowed into the sentence they close, so the quote mark travels
// with the sentence rather than opening the next one.
const closers = `)]}"'’”」』`

// splitChunks divides text into sentences, dropping whitespace-only runs.
// Newlines always break a chunk, so a list or a line of dialogue is its own unit.
//
// `salt` distinguishes two *readings* of the same sentence. The grammar
// checkpoint's advice depends on the document's tone — the same line gets
// different notes as an academic essay than as a journal entry — so switching
// tone must re-open every sentence rather than serve back advice written for the
// old register.
func splitChunks(text, salt string) []chunk {
	var out []chunk
	runes := []rune(text)
	start := 0
	add := func(end int) {
		if s := string(runes[start:end]); strings.TrimSpace(s) != "" {
			out = append(out, chunk{text: s, hash: hashChunk(s, salt)})
		}
		start = end
	}

	for i := 0; i < len(runes); i++ {
		r := runes[i]
		if r == '\n' {
			add(i + 1)
			continue
		}
		cjk := strings.ContainsRune(cjkTerminators, r)
		if !cjk && !strings.ContainsRune(asciiTerminators, r) {
			continue
		}
		// Swallow a run of terminators ("?!", "…") and any closing punctuation.
		j := i + 1
		for j < len(runes) && (strings.ContainsRune(asciiTerminators+cjkTerminators+closers, runes[j])) {
			j++
		}
		if cjk || j >= len(runes) || unicode.IsSpace(runes[j]) {
			add(j)
			i = j - 1
		}
	}
	if start < len(runes) {
		add(len(runes))
	}
	return out
}

// hashChunk identifies a sentence by its content under the same normalization
// the suppression logic uses: quote style and whitespace runs churn constantly
// (the editor rewrites quotes as she types, a paragraph reflows) and none of
// that changes what the sentence says, so none of it should cost a re-check.
func hashChunk(s, salt string) string {
	sum := sha256.Sum256([]byte(salt + "\x00" + normalizeForDedup(s)))
	return hex.EncodeToString(sum[:])[:16]
}

// hashSet indexes chunks by hash — "is this sentence in the document?"
func hashSet(chunks []chunk) map[string]bool {
	out := make(map[string]bool, len(chunks))
	for _, c := range chunks {
		out[c.hash] = true
	}
	return out
}

// changedChunks returns the chunks whose hash wasn't in the last checked set,
// in document order and deduplicated — a sentence repeated verbatim is one
// question, not two.
func changedChunks(chunks []chunk, checked map[string]bool) []chunk {
	seen := make(map[string]bool, len(chunks))
	var out []chunk
	for _, c := range chunks {
		if checked[c.hash] || seen[c.hash] {
			continue
		}
		seen[c.hash] = true
		out = append(out, c)
	}
	return out
}

// joinChunks renders a chunk set as the text to hand the model: one sentence per
// line, so two sentences pulled from opposite ends of the document don't read as
// one run-on.
func joinChunks(chunks []chunk) string {
	parts := make([]string, 0, len(chunks))
	for _, c := range chunks {
		parts = append(parts, strings.TrimSpace(c.text))
	}
	return strings.Join(parts, "\n")
}

// chunkFor names the sentence a suggestion belongs to: the first chunk whose
// text contains the flagged span. Returns "" when the span straddles a sentence
// boundary or the model paraphrased what it quoted — such a row is re-examined
// on every pass rather than cached, which is the safe direction.
func chunkFor(original string, chunks []chunk) string {
	o := normalizeForDedup(original)
	if o == "" {
		return ""
	}
	for _, c := range chunks {
		if strings.Contains(normalizeForDedup(c.text), o) {
			return c.hash
		}
	}
	return ""
}