Disable thinking on the vLLM backend
Qwen3-family models reason by default and prepend a plain-text preamble
ahead of the answer — not a <think> block, so it cannot be stripped
after the fact. Every Petal pass parses a JSON object out of the
completion, so an unsuppressed preamble fails the parse outright.
Send chat_template_kwargs {"enable_thinking": false} on every request,
matching the unconditional think:false the Ollama backend already sends.
This commit is contained in:
@@ -28,6 +28,12 @@ type vllmRequest struct {
|
|||||||
TopP float64 `json:"top_p"`
|
TopP float64 `json:"top_p"`
|
||||||
Stop []string `json:"stop,omitempty"`
|
Stop []string `json:"stop,omitempty"`
|
||||||
Stream bool `json:"stream"`
|
Stream bool `json:"stream"`
|
||||||
|
// ChatTemplateKwargs is a vLLM extension. Qwen3-family models reason by
|
||||||
|
// default and prepend a plain-text preamble ("Here's a thinking process:")
|
||||||
|
// ahead of the answer — not a <think> block, so it cannot be stripped after
|
||||||
|
// the fact. Every Petal pass parses a JSON object out of the completion, so
|
||||||
|
// an unsuppressed preamble fails the parse outright.
|
||||||
|
ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *VLLMClient) body(req CompletionRequest) vllmRequest {
|
func (c *VLLMClient) body(req CompletionRequest) vllmRequest {
|
||||||
@@ -40,6 +46,8 @@ func (c *VLLMClient) body(req CompletionRequest) vllmRequest {
|
|||||||
TopP: req.TopP,
|
TopP: req.TopP,
|
||||||
Stop: req.Stop,
|
Stop: req.Stop,
|
||||||
Stream: req.Stream,
|
Stream: req.Stream,
|
||||||
|
|
||||||
|
ChatTemplateKwargs: map[string]any{"enable_thinking": false},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user