// Package llm wraps the local inference endpoint behind a backend-agnostic
// interface. Two concrete backends — Ollama (native /api/generate) and vLLM
// (OpenAI-compatible /v1/chat/completions) — implement Client; plugin code
// calls the interface only and never knows which one is active.
//
// Deliberately not routed through internal/safehttp: that client blocks
// RFC1918 and loopback destinations to defend against SSRF from feed-supplied
// URLs, and the inference endpoint is precisely such a destination. The URL
// here comes from our own config, never from user input.
package llm
import (
"context"
"os"
"strings"
"time"
)
// Request is the backend-neutral generation request. Zero-valued fields fall
// back to backend defaults.
type Request struct {
// Prompt is a single raw instruction. The vLLM backend wraps it as one
// user message so the model's chat template still applies; sending it to
// /v1/completions instead would bypass the template and degrade an
// instruction-tuned model badly.
Prompt string
// System is an optional system message. Empty means none, which keeps the
// single-message shape the majority of callers use.
System string
// NumCtx is the per-request context window. Ollama honours it directly;
// vLLM fixes the window server-side at launch (--max-model-len), so this
// is ignored there rather than silently misapplied.
NumCtx int
// MaxTokens caps the completion length. 0 means the backend default.
MaxTokens int
// Temperature is passed through when non-zero.
Temperature float64
// Timeout overrides the client's default per-request budget.
Timeout time.Duration
}
// Client is the single surface plugin code depends on.
type Client interface {
// Generate returns the full completion in one shot. Reasoning blocks are
// stripped before returning (see StripThink) — every caller in this repo
// wants the visible answer, not the chain of thought.
Generate(ctx context.Context, req Request) (string, error)
// Model reports the configured model id, for logging and /botinfo.
Model() string
// Ping reports the model ids the backend is currently serving. Used by
// /botinfo for a liveness line; the two backends expose this on different
// paths (/api/tags vs /v1/models), which is exactly the sort of difference
// this interface exists to hide.
Ping(ctx context.Context) ([]string, error)
// Backend reports "ollama" or "vllm", for logging and /botinfo.
Backend() string
}
// Config selects and configures a backend.
type Config struct {
Backend string // "ollama" | "vllm"
Endpoint string
Model string
Timeout time.Duration
}
// DefaultTimeout matches the budget the pre-refactor callOllama used.
const DefaultTimeout = 120 * time.Second
// ConfigFromEnv reads backend settings, preferring the new LLM_* names and
// falling back to the legacy OLLAMA_* pair so an existing deployment keeps
// working untouched after this refactor.
func ConfigFromEnv() Config {
backend := strings.ToLower(strings.TrimSpace(os.Getenv("LLM_BACKEND")))
if backend == "" {
backend = "ollama"
}
endpoint := firstNonEmpty(os.Getenv("LLM_ENDPOINT"), os.Getenv("OLLAMA_HOST"))
model := firstNonEmpty(os.Getenv("LLM_MODEL"), os.Getenv("OLLAMA_MODEL"))
timeout := DefaultTimeout
if d, err := time.ParseDuration(os.Getenv("LLM_TIMEOUT")); err == nil && d > 0 {
timeout = d
}
return Config{Backend: backend, Endpoint: endpoint, Model: model, Timeout: timeout}
}
// Configured reports whether enough config is present to talk to a backend.
// Plugins check this to stay dormant rather than erroring on every invocation,
// which is what the old `if ollamaHost == "" || ollamaModel == ""` guards did.
func (c Config) Configured() bool {
return c.Endpoint != "" && c.Model != ""
}
// New builds the client for cfg.Backend. An unrecognised backend falls back to
// Ollama, which is what every existing deployment runs.
func New(cfg Config) Client {
if cfg.Timeout <= 0 {
cfg.Timeout = DefaultTimeout
}
base := backend{
endpoint: strings.TrimRight(cfg.Endpoint, "/"),
model: cfg.Model,
timeout: cfg.Timeout,
}
switch cfg.Backend {
case "vllm":
return &VLLMClient{base}
default:
return &OllamaClient{base}
}
}
// backend holds the fields shared by both concrete clients.
type backend struct {
endpoint string
model string
timeout time.Duration
}
func (b backend) Model() string { return b.model }
// timeoutFor lets a single call widen or narrow the client default. The two
// dispatch-voice callers rely on this: a dispatch is authored on a game
// chokepoint and must not stall it, while a run summary rides a background
// ticker and can afford a bigger model.
func (b backend) timeoutFor(req Request) time.Duration {
if req.Timeout > 0 {
return req.Timeout
}
return b.timeout
}
// StripThink removes a leading ... reasoning block, which Qwen
// models emit even when thinking is disabled by some backends. Callers that
// parse JSON out of the completion depend on this running first.
func StripThink(s string) string {
for {
i := strings.Index(s, "")
if i < 0 {
break
}
j := strings.Index(s, "")
if j < 0 || j < i {
break
}
s = s[:i] + s[j+len(""):]
}
return strings.TrimSpace(s)
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if v = strings.TrimSpace(v); v != "" {
return v
}
}
return ""
}