mirror of
https://github.com/prosolis/gogobee.git
synced 2026-09-14 10:51:09 +00:00
llm: route every LLM caller through a shared backend client
Replaces the per-plugin Ollama HTTP calls with internal/llm, which picks a backend from the environment (vLLM or Ollama) behind one Chat interface, plus internal/plugin/llm_client.go as the plugin-facing wrapper. Startup now logs llm_backend/llm_endpoint/llm_model instead of the two OLLAMA_* vars, which no longer describe where inference actually goes. These files were already running in prod from the vLLM migration but had never been committed; this is that live state, byte-for-byte.
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"gogobee/internal/llm"
|
||||
)
|
||||
|
||||
// The inference backend is process-wide: one endpoint, one model, selected by
|
||||
// env at startup. Plugins share a single client rather than each rebuilding one
|
||||
// per invocation, and read config through llmConfigured/llmGenerate so that
|
||||
// swapping Ollama for vLLM is a config change rather than a code change.
|
||||
var (
|
||||
llmOnce sync.Once
|
||||
llmShared llm.Client
|
||||
llmCfg llm.Config
|
||||
)
|
||||
|
||||
func llmInit() {
|
||||
llmOnce.Do(func() {
|
||||
llmCfg = llm.ConfigFromEnv()
|
||||
llmShared = llm.New(llmCfg)
|
||||
})
|
||||
}
|
||||
|
||||
// llmConfigured reports whether an endpoint and model are set. Plugins call
|
||||
// this to stay dormant instead of erroring on every invocation — the same role
|
||||
// the old `if ollamaHost == "" || ollamaModel == ""` guards played.
|
||||
func llmConfigured() bool {
|
||||
llmInit()
|
||||
return llmCfg.Configured()
|
||||
}
|
||||
|
||||
// llmClient returns the shared backend client.
|
||||
func llmClient() llm.Client {
|
||||
llmInit()
|
||||
return llmShared
|
||||
}
|
||||
|
||||
// llmGenerate is the one-line path for the common case: a raw prompt in,
|
||||
// visible completion out, reasoning blocks already stripped.
|
||||
func llmGenerate(ctx context.Context, req llm.Request) (string, error) {
|
||||
return llmClient().Generate(ctx, req)
|
||||
}
|
||||
Reference in New Issue
Block a user