Phase 3: LLM grammar checkpoint
Backend (internal/llm): backend-agnostic LLMClient interface + factory
with vLLM (OpenAI-compat) and Ollama (native) clients, each Complete +
Stream. prompts.go holds the checkpoint and Ask Petal templates;
checkpoint.go salvages JSON from model output (brace-matched), enforces a
per-doc 30s RateLimiter, and truncates the doc to a latency cap.
internal/suggestions: POST /api/docs/:id/check runs a checkpoint and
replaces the doc's pending suggestions in one tx (accepted/rejected kept
as history); GET /api/docs/:id/suggestions lists pending;
POST /api/suggestions/:id/{accept,dismiss} resolves one. Throttled checks
return the current set rather than erroring.
Frontend: useCheckpoint (4s debounce, loads existing on open, stale-guard
tokens); SuggestionHighlight renders ProseMirror decorations re-anchored
by the `original` string on every doc change (not stored marks), with
precise textblock-offset→PM-position mapping; SuggestionCard shows the
type tag + diff + explanation and applies the replacement in-editor on
accept; breathing rose checkpoint dot in the StatusBar; fade-float +
breathe animations.
Tests: llm parse/rate-limit/truncate; suggestions full flow + rate-limit
over httptest with a stub client. Smoke-tested end-to-end against a fake
vLLM endpoint (anchoring verified) and the LLM-unreachable 502 path.
Claude-Session: https://claude.ai/code/session_016Yr6jELuRc7hyzYLccQKZd
This commit is contained in:
124
internal/llm/ollama.go
Normal file
124
internal/llm/ollama.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// OllamaClient talks to Ollama's native /api/chat endpoint. Selected when
|
||||
// LLM_BACKEND=ollama.
|
||||
type OllamaClient struct {
|
||||
backend
|
||||
}
|
||||
|
||||
// ollamaRequest mirrors Ollama's /api/chat body. Sampling parameters live under
|
||||
// "options" (see the cheatsheet in the spec).
|
||||
type ollamaRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []Message `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
Options ollamaOptions `json:"options"`
|
||||
}
|
||||
|
||||
type ollamaOptions struct {
|
||||
NumPredict int `json:"num_predict"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
RepeatPenalty float64 `json:"repeat_penalty"`
|
||||
TopP float64 `json:"top_p"`
|
||||
Stop []string `json:"stop,omitempty"`
|
||||
}
|
||||
|
||||
func (c *OllamaClient) body(req CompletionRequest) ollamaRequest {
|
||||
return ollamaRequest{
|
||||
Model: c.model(req),
|
||||
Messages: req.Messages,
|
||||
Stream: req.Stream,
|
||||
Options: ollamaOptions{
|
||||
NumPredict: req.MaxTokens,
|
||||
Temperature: req.Temperature,
|
||||
RepeatPenalty: req.RepetitionPenalty,
|
||||
TopP: req.TopP,
|
||||
Stop: req.Stop,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Complete sends a non-streaming request and returns the full message content.
|
||||
func (c *OllamaClient) Complete(ctx context.Context, req CompletionRequest) (string, error) {
|
||||
req.Stream = false
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
resp, err := c.post(ctx, c.body(req))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", httpError("ollama", resp)
|
||||
}
|
||||
|
||||
var out struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return "", fmt.Errorf("ollama: decode response: %w", err)
|
||||
}
|
||||
return out.Message.Content, nil
|
||||
}
|
||||
|
||||
// Stream sends a streaming request. Ollama emits newline-delimited JSON objects;
|
||||
// we forward each message.content and stop when done:true.
|
||||
func (c *OllamaClient) Stream(ctx context.Context, req CompletionRequest) (<-chan string, error) {
|
||||
req.Stream = true
|
||||
resp, err := c.post(ctx, c.body(req))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
return nil, httpError("ollama", resp)
|
||||
}
|
||||
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
defer resp.Body.Close()
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := sc.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var chunk struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
}
|
||||
if err := json.Unmarshal(line, &chunk); err != nil {
|
||||
continue
|
||||
}
|
||||
if chunk.Message.Content != "" {
|
||||
select {
|
||||
case ch <- chunk.Message.Content:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if chunk.Done {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (c *OllamaClient) post(ctx context.Context, body any) (*http.Response, error) {
|
||||
return postJSON(ctx, c.endpoint+"/api/chat", body)
|
||||
}
|
||||
Reference in New Issue
Block a user