package suggestions import ( "crypto/sha256" "encoding/hex" "strings" "unicode" ) // Chunking splits a document into sentence-sized units so a re-check can ask the // model only about the sentences that actually changed. Accepting one edit used // to re-run the whole document: every card vanished, came back with a new id and // a freshly-worded explanation, and spans re-merged into different shapes. The // sentences she didn't touch have nothing new to say about themselves, so their // suggestions are simply kept (see reconcilePending). // // A chunk's identity is its hash, not its position — she inserts a paragraph at // the top and every sentence below keeps its suggestions. // chunk is one sentence of the document, with the hash that identifies it. type chunk struct { text string hash string } // asciiTerminators end a sentence only when whitespace (or the end of the text) // follows, so "3.5" and "Ms." don't split mid-word — a wrong split costs only a // slightly smaller chunk, but a split inside a number would churn its hash on // every keystroke around it. const asciiTerminators = ".!?" // cjkTerminators end a sentence outright: Chinese runs sentences together with // no space after 。, and she writes in both languages in one document. const cjkTerminators = "。!?" // closers are swallowed into the sentence they close, so the quote mark travels // with the sentence rather than opening the next one. const closers = `)]}"'’”」』` // splitChunks divides text into sentences, dropping whitespace-only runs. // Newlines always break a chunk, so a list or a line of dialogue is its own unit. // // `salt` distinguishes two *readings* of the same sentence. The grammar // checkpoint's advice depends on the document's tone — the same line gets // different notes as an academic essay than as a journal entry — so switching // tone must re-open every sentence rather than serve back advice written for the // old register. func splitChunks(text, salt string) []chunk { var out []chunk runes := []rune(text) start := 0 add := func(end int) { if s := string(runes[start:end]); strings.TrimSpace(s) != "" { out = append(out, chunk{text: s, hash: hashChunk(s, salt)}) } start = end } for i := 0; i < len(runes); i++ { r := runes[i] if r == '\n' { add(i + 1) continue } cjk := strings.ContainsRune(cjkTerminators, r) if !cjk && !strings.ContainsRune(asciiTerminators, r) { continue } // Swallow a run of terminators ("?!", "…") and any closing punctuation. j := i + 1 for j < len(runes) && (strings.ContainsRune(asciiTerminators+cjkTerminators+closers, runes[j])) { j++ } if cjk || j >= len(runes) || unicode.IsSpace(runes[j]) { add(j) i = j - 1 } } if start < len(runes) { add(len(runes)) } return out } // hashChunk identifies a sentence by its content under the same normalization // the suppression logic uses: quote style and whitespace runs churn constantly // (the editor rewrites quotes as she types, a paragraph reflows) and none of // that changes what the sentence says, so none of it should cost a re-check. func hashChunk(s, salt string) string { sum := sha256.Sum256([]byte(salt + "\x00" + normalizeForDedup(s))) return hex.EncodeToString(sum[:])[:16] } // hashSet indexes chunks by hash — "is this sentence in the document?" func hashSet(chunks []chunk) map[string]bool { out := make(map[string]bool, len(chunks)) for _, c := range chunks { out[c.hash] = true } return out } // changedChunks returns the chunks whose hash wasn't in the last checked set, // in document order and deduplicated — a sentence repeated verbatim is one // question, not two. func changedChunks(chunks []chunk, checked map[string]bool) []chunk { seen := make(map[string]bool, len(chunks)) var out []chunk for _, c := range chunks { if checked[c.hash] || seen[c.hash] { continue } seen[c.hash] = true out = append(out, c) } return out } // joinChunks renders a chunk set as the text to hand the model: one sentence per // line, so two sentences pulled from opposite ends of the document don't read as // one run-on. func joinChunks(chunks []chunk) string { parts := make([]string, 0, len(chunks)) for _, c := range chunks { parts = append(parts, strings.TrimSpace(c.text)) } return strings.Join(parts, "\n") } // chunkFor names the sentence a suggestion belongs to: the first chunk whose // text contains the flagged span. Returns "" when the span straddles a sentence // boundary or the model paraphrased what it quoted — such a row is re-examined // on every pass rather than cached, which is the safe direction. func chunkFor(original string, chunks []chunk) string { o := normalizeForDedup(original) if o == "" { return "" } for _, c := range chunks { if strings.Contains(normalizeForDedup(c.text), o) { return c.hash } } return "" }