llm: route every LLM caller through a shared backend client

Replaces the per-plugin Ollama HTTP calls with internal/llm, which picks a backend from the environment (vLLM or Ollama) behind one Chat interface, plus internal/plugin/llm_client.go as the plugin-facing wrapper. Startup now logs llm_backend/llm_endpoint/llm_model instead of the two OLLAMA_* vars, which no longer describe where inference actually goes. These files were already running in prod from the vLLM migration but had never been committed; this is that live state, byte-for-byte.
2026-09-14 19:01:09 +00:00 · 2026-07-26 10:16:33 -07:00
parent 3f9c338e67
commit 583616f9d0
17 changed files with 603 additions and 434 deletions
@@ -0,0 +1,161 @@
+// Package llm wraps the local inference endpoint behind a backend-agnostic
+// interface. Two concrete backends — Ollama (native /api/generate) and vLLM
+// (OpenAI-compatible /v1/chat/completions) — implement Client; plugin code
+// calls the interface only and never knows which one is active.
+//
+// Deliberately not routed through internal/safehttp: that client blocks
+// RFC1918 and loopback destinations to defend against SSRF from feed-supplied
+// URLs, and the inference endpoint is precisely such a destination. The URL
+// here comes from our own config, never from user input.
+package llm
+
+import (
+	"context"
+	"os"
+	"strings"
+	"time"
+)
+
+// Request is the backend-neutral generation request. Zero-valued fields fall
+// back to backend defaults.
+type Request struct {
+	// Prompt is a single raw instruction. The vLLM backend wraps it as one
+	// user message so the model's chat template still applies; sending it to
+	// /v1/completions instead would bypass the template and degrade an
+	// instruction-tuned model badly.
+	Prompt string
+	// System is an optional system message. Empty means none, which keeps the
+	// single-message shape the majority of callers use.
+	System string
+	// NumCtx is the per-request context window. Ollama honours it directly;
+	// vLLM fixes the window server-side at launch (--max-model-len), so this
+	// is ignored there rather than silently misapplied.
+	NumCtx int
+	// MaxTokens caps the completion length. 0 means the backend default.
+	MaxTokens int
+	// Temperature is passed through when non-zero.
+	Temperature float64
+	// Timeout overrides the client's default per-request budget.
+	Timeout time.Duration
+}
+
+// Client is the single surface plugin code depends on.
+type Client interface {
+	// Generate returns the full completion in one shot. Reasoning blocks are
+	// stripped before returning (see StripThink) — every caller in this repo
+	// wants the visible answer, not the chain of thought.
+	Generate(ctx context.Context, req Request) (string, error)
+	// Model reports the configured model id, for logging and /botinfo.
+	Model() string
+	// Ping reports the model ids the backend is currently serving. Used by
+	// /botinfo for a liveness line; the two backends expose this on different
+	// paths (/api/tags vs /v1/models), which is exactly the sort of difference
+	// this interface exists to hide.
+	Ping(ctx context.Context) ([]string, error)
+	// Backend reports "ollama" or "vllm", for logging and /botinfo.
+	Backend() string
+}
+
+// Config selects and configures a backend.
+type Config struct {
+	Backend  string // "ollama" | "vllm"
+	Endpoint string
+	Model    string
+	Timeout  time.Duration
+}
+
+// DefaultTimeout matches the budget the pre-refactor callOllama used.
+const DefaultTimeout = 120 * time.Second
+
+// ConfigFromEnv reads backend settings, preferring the new LLM_* names and
+// falling back to the legacy OLLAMA_* pair so an existing deployment keeps
+// working untouched after this refactor.
+func ConfigFromEnv() Config {
+	backend := strings.ToLower(strings.TrimSpace(os.Getenv("LLM_BACKEND")))
+	if backend == "" {
+		backend = "ollama"
+	}
+
+	endpoint := firstNonEmpty(os.Getenv("LLM_ENDPOINT"), os.Getenv("OLLAMA_HOST"))
+	model := firstNonEmpty(os.Getenv("LLM_MODEL"), os.Getenv("OLLAMA_MODEL"))
+
+	timeout := DefaultTimeout
+	if d, err := time.ParseDuration(os.Getenv("LLM_TIMEOUT")); err == nil && d > 0 {
+		timeout = d
+	}
+
+	return Config{Backend: backend, Endpoint: endpoint, Model: model, Timeout: timeout}
+}
+
+// Configured reports whether enough config is present to talk to a backend.
+// Plugins check this to stay dormant rather than erroring on every invocation,
+// which is what the old `if ollamaHost == "" || ollamaModel == ""` guards did.
+func (c Config) Configured() bool {
+	return c.Endpoint != "" && c.Model != ""
+}
+
+// New builds the client for cfg.Backend. An unrecognised backend falls back to
+// Ollama, which is what every existing deployment runs.
+func New(cfg Config) Client {
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = DefaultTimeout
+	}
+	base := backend{
+		endpoint: strings.TrimRight(cfg.Endpoint, "/"),
+		model:    cfg.Model,
+		timeout:  cfg.Timeout,
+	}
+	switch cfg.Backend {
+	case "vllm":
+		return &VLLMClient{base}
+	default:
+		return &OllamaClient{base}
+	}
+}
+
+// backend holds the fields shared by both concrete clients.
+type backend struct {
+	endpoint string
+	model    string
+	timeout  time.Duration
+}
+
+func (b backend) Model() string { return b.model }
+
+// timeoutFor lets a single call widen or narrow the client default. The two
+// dispatch-voice callers rely on this: a dispatch is authored on a game
+// chokepoint and must not stall it, while a run summary rides a background
+// ticker and can afford a bigger model.
+func (b backend) timeoutFor(req Request) time.Duration {
+	if req.Timeout > 0 {
+		return req.Timeout
+	}
+	return b.timeout
+}
+
+// StripThink removes a leading <think>...</think> reasoning block, which Qwen
+// models emit even when thinking is disabled by some backends. Callers that
+// parse JSON out of the completion depend on this running first.
+func StripThink(s string) string {
+	for {
+		i := strings.Index(s, "<think>")
+		if i < 0 {
+			break
+		}
+		j := strings.Index(s, "</think>")
+		if j < 0 || j < i {
+			break
+		}
+		s = s[:i] + s[j+len("</think>"):]
+	}
+	return strings.TrimSpace(s)
+}
+
+func firstNonEmpty(vals ...string) string {
+	for _, v := range vals {
+		if v = strings.TrimSpace(v); v != "" {
+			return v
+		}
+	}
+	return ""
+}
@@ -0,0 +1,103 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// OllamaClient talks to Ollama's native /api/generate endpoint.
+type OllamaClient struct {
+	backend
+}
+
+func (c *OllamaClient) Backend() string { return "ollama" }
+
+type ollamaOptions struct {
+	NumCtx      int     `json:"num_ctx,omitempty"`
+	NumPredict  int     `json:"num_predict,omitempty"`
+	Temperature float64 `json:"temperature,omitempty"`
+}
+
+type ollamaRequest struct {
+	Model   string        `json:"model"`
+	Prompt  string        `json:"prompt"`
+	System  string        `json:"system,omitempty"`
+	Stream  bool          `json:"stream"`
+	Think   bool          `json:"think"`
+	Options ollamaOptions `json:"options,omitempty"`
+}
+
+// Generate posts a single non-streaming generation and returns the completion.
+func (c *OllamaClient) Generate(ctx context.Context, req Request) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.timeoutFor(req))
+	defer cancel()
+
+	body, err := json.Marshal(ollamaRequest{
+		Model:  c.model,
+		Prompt: req.Prompt,
+		System: req.System,
+		Stream: false,
+		Think:  false,
+		Options: ollamaOptions{
+			NumCtx:      req.NumCtx,
+			NumPredict:  req.MaxTokens,
+			Temperature: req.Temperature,
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("ollama: marshal payload: %w", err)
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		c.endpoint+"/api/generate", bytes.NewReader(body))
+	if err != nil {
+		return "", fmt.Errorf("ollama: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("ollama request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("ollama: read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("ollama HTTP %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var result struct {
+		Response string `json:"response"`
+	}
+	if err := json.Unmarshal(respBody, &result); err != nil {
+		return "", fmt.Errorf("ollama: parse response: %w", err)
+	}
+	return StripThink(result.Response), nil
+}
+
+// Ping lists locally installed models via Ollama's native /api/tags.
+func (c *OllamaClient) Ping(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, pingTimeout)
+	defer cancel()
+
+	var out struct {
+		Models []struct {
+			Name string `json:"name"`
+		} `json:"models"`
+	}
+	if err := getJSON(ctx, c.endpoint+"/api/tags", &out); err != nil {
+		return nil, err
+	}
+	names := make([]string, 0, len(out.Models))
+	for _, m := range out.Models {
+		names = append(names, m.Name)
+	}
+	return names, nil
+}
@@ -0,0 +1,29 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+// pingTimeout keeps a liveness probe short — /botinfo renders synchronously and
+// a hung endpoint must not hold the reply.
+const pingTimeout = 5 * time.Second
+
+func getJSON(ctx context.Context, url string, out any) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}
@@ -0,0 +1,116 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// VLLMClient talks to an OpenAI-compatible /v1/chat/completions endpoint.
+type VLLMClient struct {
+	backend
+}
+
+func (c *VLLMClient) Backend() string { return "vllm" }
+
+type vllmMessage struct {
+	Role    string `json:"role"`
+	Content string `json:"content"`
+}
+
+type vllmRequest struct {
+	Model       string        `json:"model"`
+	Messages    []vllmMessage `json:"messages"`
+	MaxTokens   int           `json:"max_tokens,omitempty"`
+	Temperature float64       `json:"temperature,omitempty"`
+	Stream      bool          `json:"stream"`
+	// ChatTemplateKwargs is a vLLM extension to the OpenAI schema. It is how
+	// Qwen3-family reasoning is switched off; the Ollama backend spells the
+	// same intent as its native "think": false.
+	ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
+}
+
+// Generate posts a single non-streaming completion and returns the message
+// content. The raw prompt is sent as one user message so the server-side chat
+// template still wraps it.
+func (c *VLLMClient) Generate(ctx context.Context, req Request) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.timeoutFor(req))
+	defer cancel()
+
+	msgs := make([]vllmMessage, 0, 2)
+	if req.System != "" {
+		msgs = append(msgs, vllmMessage{Role: "system", Content: req.System})
+	}
+	msgs = append(msgs, vllmMessage{Role: "user", Content: req.Prompt})
+
+	body, err := json.Marshal(vllmRequest{
+		Model:              c.model,
+		Messages:           msgs,
+		MaxTokens:          req.MaxTokens,
+		Temperature:        req.Temperature,
+		Stream:             false,
+		ChatTemplateKwargs: map[string]any{"enable_thinking": false},
+	})
+	if err != nil {
+		return "", fmt.Errorf("vllm: marshal payload: %w", err)
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		c.endpoint+"/v1/chat/completions", bytes.NewReader(body))
+	if err != nil {
+		return "", fmt.Errorf("vllm: build request: %w", err)
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("vllm request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("vllm: read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("vllm HTTP %d: %s", resp.StatusCode, string(respBody))
+	}
+
+	var result struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+	}
+	if err := json.Unmarshal(respBody, &result); err != nil {
+		return "", fmt.Errorf("vllm: parse response: %w", err)
+	}
+	if len(result.Choices) == 0 {
+		return "", fmt.Errorf("vllm: empty choices in response")
+	}
+	return StripThink(result.Choices[0].Message.Content), nil
+}
+
+// Ping lists served models via the OpenAI-compatible /v1/models.
+func (c *VLLMClient) Ping(ctx context.Context) ([]string, error) {
+	ctx, cancel := context.WithTimeout(ctx, pingTimeout)
+	defer cancel()
+
+	var out struct {
+		Data []struct {
+			ID string `json:"id"`
+		} `json:"data"`
+	}
+	if err := getJSON(ctx, c.endpoint+"/v1/models", &out); err != nil {
+		return nil, err
+	}
+	names := make([]string, 0, len(out.Data))
+	for _, m := range out.Data {
+		names = append(names, m.ID)
+	}
+	return names, nil
+}