a4069d5755
Backend (internal/llm): backend-agnostic LLMClient interface + factory
with vLLM (OpenAI-compat) and Ollama (native) clients, each Complete +
Stream. prompts.go holds the checkpoint and Ask Petal templates;
checkpoint.go salvages JSON from model output (brace-matched), enforces a
per-doc 30s RateLimiter, and truncates the doc to a latency cap.
internal/suggestions: POST /api/docs/:id/check runs a checkpoint and
replaces the doc's pending suggestions in one tx (accepted/rejected kept
as history); GET /api/docs/:id/suggestions lists pending;
POST /api/suggestions/:id/{accept,dismiss} resolves one. Throttled checks
return the current set rather than erroring.
Frontend: useCheckpoint (4s debounce, loads existing on open, stale-guard
tokens); SuggestionHighlight renders ProseMirror decorations re-anchored
by the `original` string on every doc change (not stored marks), with
precise textblock-offset→PM-position mapping; SuggestionCard shows the
type tag + diff + explanation and applies the replacement in-editor on
accept; breathing rose checkpoint dot in the StatusBar; fade-float +
breathe animations.
Tests: llm parse/rate-limit/truncate; suggestions full flow + rate-limit
over httptest with a stub client. Smoke-tested end-to-end against a fake
vLLM endpoint (anchoring verified) and the LLM-unreachable 502 path.
Claude-Session: https://claude.ai/code/session_016Yr6jELuRc7hyzYLccQKZd
125 lines
3.0 KiB
Go
125 lines
3.0 KiB
Go
package llm
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
)
|
|
|
|
// OllamaClient talks to Ollama's native /api/chat endpoint. Selected when
|
|
// LLM_BACKEND=ollama.
|
|
type OllamaClient struct {
|
|
backend
|
|
}
|
|
|
|
// ollamaRequest mirrors Ollama's /api/chat body. Sampling parameters live under
|
|
// "options" (see the cheatsheet in the spec).
|
|
type ollamaRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []Message `json:"messages"`
|
|
Stream bool `json:"stream"`
|
|
Options ollamaOptions `json:"options"`
|
|
}
|
|
|
|
type ollamaOptions struct {
|
|
NumPredict int `json:"num_predict"`
|
|
Temperature float64 `json:"temperature"`
|
|
RepeatPenalty float64 `json:"repeat_penalty"`
|
|
TopP float64 `json:"top_p"`
|
|
Stop []string `json:"stop,omitempty"`
|
|
}
|
|
|
|
func (c *OllamaClient) body(req CompletionRequest) ollamaRequest {
|
|
return ollamaRequest{
|
|
Model: c.model(req),
|
|
Messages: req.Messages,
|
|
Stream: req.Stream,
|
|
Options: ollamaOptions{
|
|
NumPredict: req.MaxTokens,
|
|
Temperature: req.Temperature,
|
|
RepeatPenalty: req.RepetitionPenalty,
|
|
TopP: req.TopP,
|
|
Stop: req.Stop,
|
|
},
|
|
}
|
|
}
|
|
|
|
// Complete sends a non-streaming request and returns the full message content.
|
|
func (c *OllamaClient) Complete(ctx context.Context, req CompletionRequest) (string, error) {
|
|
req.Stream = false
|
|
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
|
defer cancel()
|
|
resp, err := c.post(ctx, c.body(req))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", httpError("ollama", resp)
|
|
}
|
|
|
|
var out struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
} `json:"message"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
|
return "", fmt.Errorf("ollama: decode response: %w", err)
|
|
}
|
|
return out.Message.Content, nil
|
|
}
|
|
|
|
// Stream sends a streaming request. Ollama emits newline-delimited JSON objects;
|
|
// we forward each message.content and stop when done:true.
|
|
func (c *OllamaClient) Stream(ctx context.Context, req CompletionRequest) (<-chan string, error) {
|
|
req.Stream = true
|
|
resp, err := c.post(ctx, c.body(req))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
defer resp.Body.Close()
|
|
return nil, httpError("ollama", resp)
|
|
}
|
|
|
|
ch := make(chan string)
|
|
go func() {
|
|
defer close(ch)
|
|
defer resp.Body.Close()
|
|
sc := bufio.NewScanner(resp.Body)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var chunk struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
} `json:"message"`
|
|
Done bool `json:"done"`
|
|
}
|
|
if err := json.Unmarshal(line, &chunk); err != nil {
|
|
continue
|
|
}
|
|
if chunk.Message.Content != "" {
|
|
select {
|
|
case ch <- chunk.Message.Content:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
if chunk.Done {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
return ch, nil
|
|
}
|
|
|
|
func (c *OllamaClient) post(ctx context.Context, body any) (*http.Response, error) {
|
|
return postJSON(ctx, c.endpoint+"/api/chat", body)
|
|
}
|