diff --git a/api/types.go b/api/types.go index 699dba428..e2c63b622 100644 --- a/api/types.go +++ b/api/types.go @@ -85,10 +85,11 @@ type GenerateRequest struct { Options map[string]any `json:"options"` // Think controls whether thinking/reasoning models will think before - // responding. Needs to be a pointer so we can distinguish between false + // responding. Can be a boolean (true/false) or a string ("high", "medium", "low") + // for supported models. Needs to be a pointer so we can distinguish between false // (request that thinking _not_ be used) and unset (use the old behavior // before this option was introduced) - Think *bool `json:"think,omitempty"` + Think *ThinkValue `json:"think,omitempty"` } // ChatRequest describes a request sent by [Client.Chat]. @@ -116,8 +117,9 @@ type ChatRequest struct { Options map[string]any `json:"options"` // Think controls whether thinking/reasoning models will think before - // responding - Think *bool `json:"think,omitempty"` + // responding. Can be a boolean (true/false) or a string ("high", "medium", "low") + // for supported models. + Think *ThinkValue `json:"think,omitempty"` } type Tools []Tool @@ -508,6 +510,8 @@ type GenerateResponse struct { Context []int `json:"context,omitempty"` Metrics + + ToolCalls []ToolCall `json:"tool_calls,omitempty"` } // ModelDetails provides details about a model. @@ -677,6 +681,113 @@ func DefaultOptions() Options { } } +// ThinkValue represents a value that can be a boolean or a string ("high", "medium", "low") +type ThinkValue struct { + // Value can be a bool or string + Value interface{} +} + +// IsValid checks if the ThinkValue is valid +func (t *ThinkValue) IsValid() bool { + if t == nil || t.Value == nil { + return true // nil is valid (means not set) + } + + switch v := t.Value.(type) { + case bool: + return true + case string: + return v == "high" || v == "medium" || v == "low" + default: + return false + } +} + +// IsBool returns true if the value is a boolean +func (t *ThinkValue) IsBool() bool { + if t == nil || t.Value == nil { + return false + } + _, ok := t.Value.(bool) + return ok +} + +// IsString returns true if the value is a string +func (t *ThinkValue) IsString() bool { + if t == nil || t.Value == nil { + return false + } + _, ok := t.Value.(string) + return ok +} + +// AsBool returns the value as a bool (true if enabled in any way) +func (t *ThinkValue) AsBool() bool { + if t == nil || t.Value == nil { + return false + } + + switch v := t.Value.(type) { + case bool: + return v + case string: + // Any string value ("high", "medium", "low") means thinking is enabled + return v == "high" || v == "medium" || v == "low" + default: + return false + } +} + +// AsString returns the value as a string +func (t *ThinkValue) AsString() string { + if t == nil || t.Value == nil { + return "" + } + + switch v := t.Value.(type) { + case string: + return v + case bool: + if v { + return "medium" // Default level when just true + } + return "" + default: + return "" + } +} + +// UnmarshalJSON implements json.Unmarshaler +func (t *ThinkValue) UnmarshalJSON(data []byte) error { + // Try to unmarshal as bool first + var b bool + if err := json.Unmarshal(data, &b); err == nil { + t.Value = b + return nil + } + + // Try to unmarshal as string + var s string + if err := json.Unmarshal(data, &s); err == nil { + // Validate string values + if s != "high" && s != "medium" && s != "low" { + return fmt.Errorf("invalid think value: %q (must be \"high\", \"medium\", \"low\", true, or false)", s) + } + t.Value = s + return nil + } + + return fmt.Errorf("think must be a boolean or string (\"high\", \"medium\", \"low\")") +} + +// MarshalJSON implements json.Marshaler +func (t *ThinkValue) MarshalJSON() ([]byte, error) { + if t == nil || t.Value == nil { + return []byte("null"), nil + } + return json.Marshal(t.Value) +} + type Duration struct { time.Duration } diff --git a/api/types_test.go b/api/types_test.go index 9c2fb1f11..841853808 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -374,24 +374,21 @@ func TestPropertyType_MarshalJSON(t *testing.T) { } func TestThinking_UnmarshalJSON(t *testing.T) { - trueVal := true - falseVal := false - tests := []struct { name string input string - expectedThinking *bool + expectedThinking *ThinkValue expectedError bool }{ { name: "true", input: `{ "think": true }`, - expectedThinking: &trueVal, + expectedThinking: &ThinkValue{Value: true}, }, { name: "false", input: `{ "think": false }`, - expectedThinking: &falseVal, + expectedThinking: &ThinkValue{Value: false}, }, { name: "unset", @@ -399,8 +396,23 @@ func TestThinking_UnmarshalJSON(t *testing.T) { expectedThinking: nil, }, { - name: "invalid", - input: `{ "think": "true" }`, + name: "string_high", + input: `{ "think": "high" }`, + expectedThinking: &ThinkValue{Value: "high"}, + }, + { + name: "string_medium", + input: `{ "think": "medium" }`, + expectedThinking: &ThinkValue{Value: "medium"}, + }, + { + name: "string_low", + input: `{ "think": "low" }`, + expectedThinking: &ThinkValue{Value: "low"}, + }, + { + name: "invalid_string", + input: `{ "think": "invalid" }`, expectedThinking: nil, expectedError: true, }, @@ -414,7 +426,12 @@ func TestThinking_UnmarshalJSON(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - assert.Equal(t, test.expectedThinking, req.Think) + if test.expectedThinking == nil { + assert.Nil(t, req.Think) + } else { + require.NotNil(t, req.Think) + assert.Equal(t, test.expectedThinking.Value, req.Think.Value) + } } }) } diff --git a/cmd/cmd.go b/cmd/cmd.go index 1d1d116ba..3ef96f064 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -322,11 +322,23 @@ func RunHandler(cmd *cobra.Command, args []string) error { thinkFlag := cmd.Flags().Lookup("think") if thinkFlag.Changed { - think, err := cmd.Flags().GetBool("think") + thinkStr, err := cmd.Flags().GetString("think") if err != nil { return err } - opts.Think = &think + + // Handle different values for --think + switch thinkStr { + case "", "true": + // --think or --think=true + opts.Think = &api.ThinkValue{Value: true} + case "false": + opts.Think = &api.ThinkValue{Value: false} + case "high", "medium", "low": + opts.Think = &api.ThinkValue{Value: thinkStr} + default: + return fmt.Errorf("invalid value for --think: %q (must be true, false, high, medium, or low)", thinkStr) + } } else { opts.Think = nil } @@ -977,7 +989,7 @@ type runOptions struct { Options map[string]any MultiModal bool KeepAlive *api.Duration - Think *bool + Think *api.ThinkValue HideThinking bool } @@ -1017,10 +1029,11 @@ func displayResponse(content string, wordWrap bool, state *displayResponseState) } switch ch { - case ' ': + case ' ', '\t': state.wordBuffer = "" - case '\n': + case '\n', '\r': state.lineLength = 0 + state.wordBuffer = "" default: state.wordBuffer += string(ch) } @@ -1078,6 +1091,7 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { }() var state *displayResponseState = &displayResponseState{} + var thinkingContent strings.Builder var latest api.ChatResponse var fullResponse strings.Builder var thinkTagOpened bool = false @@ -1097,14 +1111,21 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { if !thinkTagOpened { fmt.Print(thinkingOutputOpeningText(false)) thinkTagOpened = true + thinkTagClosed = false } + thinkingContent.WriteString(response.Message.Thinking) displayResponse(response.Message.Thinking, opts.WordWrap, state) } content := response.Message.Content - if thinkTagOpened && !thinkTagClosed && content != "" { + if thinkTagOpened && !thinkTagClosed && (content != "" || len(response.Message.ToolCalls) > 0) { + if !strings.HasSuffix(thinkingContent.String(), "\n") { + fmt.Println() + } fmt.Print(thinkingOutputClosingText(false)) + thinkTagOpened = false thinkTagClosed = true + state = &displayResponseState{} } // purposefully not putting thinking blocks in the response, which would // only be needed if we later added tool calling to the cli (they get @@ -1112,6 +1133,13 @@ func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) { // about to finish some tool calls) fullResponse.WriteString(content) + if response.Message.ToolCalls != nil { + toolCalls := response.Message.ToolCalls + if len(toolCalls) > 0 { + fmt.Print(renderToolCalls(toolCalls, false)) + } + } + displayResponse(content, opts.WordWrap, state) return nil @@ -1196,6 +1224,7 @@ func generate(cmd *cobra.Command, opts runOptions) error { }() var state *displayResponseState = &displayResponseState{} + var thinkingContent strings.Builder var thinkTagOpened bool = false var thinkTagClosed bool = false @@ -1213,17 +1242,30 @@ func generate(cmd *cobra.Command, opts runOptions) error { if !thinkTagOpened { fmt.Print(thinkingOutputOpeningText(plainText)) thinkTagOpened = true + thinkTagClosed = false } + thinkingContent.WriteString(response.Thinking) displayResponse(response.Thinking, opts.WordWrap, state) } - if thinkTagOpened && !thinkTagClosed && content != "" { + if thinkTagOpened && !thinkTagClosed && (content != "" || len(response.ToolCalls) > 0) { + if !strings.HasSuffix(thinkingContent.String(), "\n") { + fmt.Println() + } fmt.Print(thinkingOutputClosingText(plainText)) + thinkTagOpened = false thinkTagClosed = true } displayResponse(content, opts.WordWrap, state) + if response.ToolCalls != nil { + toolCalls := response.ToolCalls + if len(toolCalls) > 0 { + fmt.Print(renderToolCalls(toolCalls, plainText)) + } + } + return nil } @@ -1463,7 +1505,8 @@ func NewCLI() *cobra.Command { runCmd.Flags().Bool("insecure", false, "Use an insecure registry") runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically") runCmd.Flags().String("format", "", "Response format (e.g. json)") - runCmd.Flags().Bool("think", false, "Whether to use thinking mode for supported models") + runCmd.Flags().String("think", "", "Enable thinking mode: true/false or high/medium/low for supported models") + runCmd.Flags().Lookup("think").NoOptDefVal = "true" runCmd.Flags().Bool("hidethinking", false, "Hide thinking output (if provided)") stopCmd := &cobra.Command{ @@ -1613,7 +1656,7 @@ func NewCLI() *cobra.Command { // to false). // // If capabilities are not provided, we fetch them from the server. -func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicitlySetByUser bool) (*bool, error) { +func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicitlySetByUser bool) (*api.ThinkValue, error) { if explicitlySetByUser { return runOpts.Think, nil } @@ -1640,9 +1683,34 @@ func inferThinkingOption(caps *[]model.Capability, runOpts *runOptions, explicit } if thinkingSupported { - thinking := true - return &thinking, nil + return &api.ThinkValue{Value: true}, nil } return nil, nil } + +func renderToolCalls(toolCalls []api.ToolCall, plainText bool) string { + out := "" + formatExplanation := "" + formatValues := "" + if !plainText { + formatExplanation = readline.ColorGrey + readline.ColorBold + formatValues = readline.ColorDefault + out += formatExplanation + } + for i, toolCall := range toolCalls { + argsAsJSON, err := json.Marshal(toolCall.Function.Arguments) + if err != nil { + return "" + } + if i > 0 { + out += "\n" + } + // all tool calls are unexpected since we don't currently support registering any in the CLI + out += fmt.Sprintf(" Model called a non-existent function '%s()' with arguments: %s", formatValues+toolCall.Function.Name+formatExplanation, formatValues+string(argsAsJSON)+formatExplanation) + } + if !plainText { + out += readline.ColorDefault + } + return out +} diff --git a/cmd/interactive.go b/cmd/interactive.go index 08ab4947b..e290d84ce 100644 --- a/cmd/interactive.go +++ b/cmd/interactive.go @@ -272,16 +272,29 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { } fmt.Println("Set 'quiet' mode.") case "think": - think := true - opts.Think = &think + thinkValue := api.ThinkValue{Value: true} + var maybeLevel string + if len(args) > 2 { + maybeLevel = args[2] + } + if maybeLevel != "" { + // TODO(drifkin): validate the level, could be model dependent + // though... It will also be validated on the server once a call is + // made. + thinkValue.Value = maybeLevel + } + opts.Think = &thinkValue thinkExplicitlySet = true if client, err := api.ClientFromEnvironment(); err == nil { ensureThinkingSupport(cmd.Context(), client, opts.Model) } - fmt.Println("Set 'think' mode.") + if maybeLevel != "" { + fmt.Printf("Set 'think' mode to '%s'.\n", maybeLevel) + } else { + fmt.Println("Set 'think' mode.") + } case "nothink": - think := false - opts.Think = &think + opts.Think = &api.ThinkValue{Value: false} thinkExplicitlySet = true if client, err := api.ClientFromEnvironment(); err == nil { ensureThinkingSupport(cmd.Context(), client, opts.Model) @@ -478,7 +491,8 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error { assistant, err := chat(cmd, opts) if err != nil { - if strings.Contains(err.Error(), "does not support thinking") { + if strings.Contains(err.Error(), "does not support thinking") || + strings.Contains(err.Error(), "invalid think value") { fmt.Printf("error: %v\n", err) sb.Reset() continue diff --git a/openai/openai.go b/openai/openai.go index 35b8b9a01..d065de8f1 100644 --- a/openai/openai.go +++ b/openai/openai.go @@ -36,6 +36,7 @@ type ErrorResponse struct { type Message struct { Role string `json:"role"` Content any `json:"content"` + Reasoning string `json:"reasoning,omitempty"` ToolCalls []ToolCall `json:"tool_calls,omitempty"` } @@ -81,6 +82,10 @@ type StreamOptions struct { IncludeUsage bool `json:"include_usage"` } +type Reasoning struct { + Effort *string `json:"effort,omitempty"` +} + type ChatCompletionRequest struct { Model string `json:"model"` Messages []Message `json:"messages"` @@ -95,6 +100,7 @@ type ChatCompletionRequest struct { TopP *float64 `json:"top_p"` ResponseFormat *ResponseFormat `json:"response_format"` Tools []api.Tool `json:"tools"` + Reasoning *Reasoning `json:"reasoning,omitempty"` } type ChatCompletion struct { @@ -253,7 +259,7 @@ func toChatCompletion(id string, r api.ChatResponse) ChatCompletion { SystemFingerprint: "fp_ollama", Choices: []Choice{{ Index: 0, - Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls}, + Message: Message{Role: r.Message.Role, Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking}, FinishReason: func(reason string) *string { if len(toolCalls) > 0 { reason = "tool_calls" @@ -278,10 +284,10 @@ func toChunk(id string, r api.ChatResponse, toolCallSent bool) ChatCompletionChu SystemFingerprint: "fp_ollama", Choices: []ChunkChoice{{ Index: 0, - Delta: Message{Role: "assistant", Content: r.Message.Content, ToolCalls: toolCalls}, + Delta: Message{Role: "assistant", Content: r.Message.Content, ToolCalls: toolCalls, Reasoning: r.Message.Thinking}, FinishReason: func(reason string) *string { if len(reason) > 0 { - if toolCallSent { + if toolCallSent || len(toolCalls) > 0 { return &finishReasonToolCalls } return &reason @@ -397,7 +403,7 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) { for _, msg := range r.Messages { switch content := msg.Content.(type) { case string: - messages = append(messages, api.Message{Role: msg.Role, Content: content}) + messages = append(messages, api.Message{Role: msg.Role, Content: content, Thinking: msg.Reasoning}) case []any: for _, c := range content { data, ok := c.(map[string]any) @@ -508,6 +514,10 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) { options["top_p"] = 1.0 } + if r.Reasoning != nil { + options["reasoning"] = *r.Reasoning.Effort + } + var format json.RawMessage if r.ResponseFormat != nil { switch strings.ToLower(strings.TrimSpace(r.ResponseFormat.Type)) { @@ -521,6 +531,13 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) { } } + var think *api.ThinkValue + if r.Reasoning != nil { + think = &api.ThinkValue{ + Value: *r.Reasoning.Effort, + } + } + return &api.ChatRequest{ Model: r.Model, Messages: messages, @@ -528,6 +545,7 @@ func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) { Options: options, Stream: &r.Stream, Tools: r.Tools, + Think: think, }, nil } diff --git a/server/harmonyparser.go b/server/harmonyparser.go new file mode 100644 index 000000000..fd6c64e73 --- /dev/null +++ b/server/harmonyparser.go @@ -0,0 +1,379 @@ +package server + +import ( + "context" + "log/slog" + "strings" + "unicode" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/logutil" +) + +type harmonyParserState int + +const ( + harmonyParserState_LookingForMessageStart harmonyParserState = iota + harmonyParserState_ParsingHeader + harmonyParserState_ParsingContent +) + +func shouldUseHarmony(model Model) bool { + if model.Config.ModelFamily == "gptoss" { + // heuristic to check whether the template expects to be parsed via harmony: + // search for harmony tags that are nearly always used + if model.Template.Contains("<|start|>") && model.Template.Contains("<|end|>") { + return true + } + } + + return false +} + +func (s harmonyParserState) String() string { + switch s { + // we're looking for the message start tag + case harmonyParserState_LookingForMessageStart: + return "LookingForMessageStart" + case harmonyParserState_ParsingHeader: + return "ParsingHeader" + case harmonyParserState_ParsingContent: + return "ParsingContent" + default: + return "Unknown" + } +} + +type HarmonyParser struct { + state harmonyParserState + MessageStartTag string + MessageEndTag string + HeaderEndTag string + acc strings.Builder + lifetimeAcc strings.Builder +} + +type HarmonyEvent interface { + isHarmonyEvent() +} + +type HarmonyEventMessageStart struct{} + +func (HarmonyEventMessageStart) isHarmonyEvent() {} + +type HarmonyEventHeaderComplete struct { + Header HarmonyHeader +} + +func (HarmonyEventHeaderComplete) isHarmonyEvent() {} + +type HarmonyEventContentEmitted struct { + Content string +} + +func (HarmonyEventContentEmitted) isHarmonyEvent() {} + +type HarmonyEventMessageEnd struct{} + +func (HarmonyEventMessageEnd) isHarmonyEvent() {} + +type HarmonyHeader struct { + Role string + Channel string + Recipient string +} + +func (s *HarmonyParser) AddImplicitStart() { + s.acc.WriteString("<|start|>assistant") +} + +func (s *HarmonyParser) AddImplicitStartOrPrefill(lastMessage *api.Message) { + if lastMessage != nil && lastMessage.Role == "assistant" { + // handle prefilling conditions + if lastMessage.Content != "" { + s.acc.WriteString("<|start|>assistant<|channel|>final<|message|>") + return + } else if lastMessage.Thinking != "" { + s.acc.WriteString("<|start|>assistant<|channel|>analysis<|message|>") + return + } + } + s.AddImplicitStart() +} + +func (s *HarmonyParser) AddContent(content string) []HarmonyEvent { + s.lifetimeAcc.WriteString(content) + s.acc.WriteString(content) + + var events []HarmonyEvent + + keepLooping := true + // we loop because we might pass through multiple parsing states in a single + // call to addContent, and we want to make sure callers don't have to wait for + // data that's already unambiguous + for keepLooping { + var newEvents []HarmonyEvent + newEvents, keepLooping = eat(s) + events = append(events, newEvents...) + } + + return events +} + +// the additional bool return is true iff we should continue eating +func eat(s *HarmonyParser) ([]HarmonyEvent, bool) { + switch s.state { + case harmonyParserState_LookingForMessageStart: + // does the acc contain the message start tag? + if strings.Contains(s.acc.String(), s.MessageStartTag) { + // split the acc into the message start tag and the rest + split := strings.SplitN(s.acc.String(), s.MessageStartTag, 2) + before := split[0] + if before != "" { + slog.Warn("harmony parser: found message start tag in the middle of the content", "content", s.acc.String()) + } + after := split[1] + s.acc.Reset() + s.acc.WriteString(after) + s.state = harmonyParserState_ParsingHeader + return []HarmonyEvent{HarmonyEventMessageStart{}}, true + } + + // no match, so we keep accumulating + return nil, false + case harmonyParserState_ParsingHeader: + if strings.Contains(s.acc.String(), s.HeaderEndTag) { + split := strings.SplitN(s.acc.String(), s.HeaderEndTag, 2) + header := split[0] + after := split[1] + s.acc.Reset() + s.acc.WriteString(after) + s.state = harmonyParserState_ParsingContent + return []HarmonyEvent{HarmonyEventHeaderComplete{Header: s.parseHeader(header)}}, true + } + return nil, false + case harmonyParserState_ParsingContent: + if strings.Contains(s.acc.String(), s.MessageEndTag) { + // if we already have the message end tag, we can emit the content up to it + split := strings.SplitN(s.acc.String(), s.MessageEndTag, 2) + content := split[0] + after := split[1] + s.acc.Reset() + s.acc.WriteString(after) + s.state = harmonyParserState_LookingForMessageStart + events := []HarmonyEvent{} + if content != "" { + events = append(events, HarmonyEventContentEmitted{Content: content}) + } + events = append(events, HarmonyEventMessageEnd{}) + return events, true + } else if overlapLen := overlap(s.acc.String(), s.MessageEndTag); overlapLen > 0 { + // if our suffix contains the start of the message end tag, we can emit + // the content up to the start of the message end tag + content := s.acc.String()[:len(s.acc.String())-overlapLen] + remaining := s.acc.String()[len(s.acc.String())-overlapLen:] + s.acc.Reset() + s.acc.WriteString(remaining) + // emit the content we know isn't part of the message end tag, and keep + // accumulating to disambiguate the rest + if content == "" { + return nil, false + } + return []HarmonyEvent{HarmonyEventContentEmitted{Content: content}}, false + } else { + // no end tag, so it's still normal content that we can immediately emit + content := s.acc.String() + if content == "" { + return nil, false + } + s.acc.Reset() + return []HarmonyEvent{HarmonyEventContentEmitted{Content: content}}, false + } + } + + return nil, false +} + +func (s *HarmonyParser) parseHeader(raw string) HarmonyHeader { + harmonyHeader := HarmonyHeader{} + + // if `<|constrain|>` is present, ensure it has a space before it so it gets + // parsed as a separate token, even if the model didn't include the space + if strings.Contains(raw, "<|constrain|>") { + raw = strings.Replace(raw, "<|constrain|>", " <|constrain|>", 1) + raw = strings.TrimSpace(raw) + } + + // look for the optional channel tag, which is `<|channel|>` followed by the + // channel name, all without any whitespace + channelIndex := strings.Index(raw, "<|channel|>") + if channelIndex != -1 { + before := raw[:channelIndex] + after := raw[channelIndex+len("<|channel|>"):] + // the channel name is `after` all the way up to the first (if any) whitespace character + idx := strings.IndexFunc(after, func(r rune) bool { + return unicode.IsSpace(r) + }) + if idx == -1 { + idx = len(after) + } + harmonyHeader.Channel = after[:idx] + after = after[idx:] + // now we remove the channel tag from the raw string to further process + raw = before + after + raw = strings.TrimSpace(raw) + } + + // split the header into whitespace-separated tokens + tokens := strings.Fields(raw) + + // the first token is treated as the role + if len(tokens) == 0 { + slog.Error("harmony parser: missing role in header", "header", raw) + return harmonyHeader + } + role := tokens[0] + tokens = tokens[1:] + // special case: if role starts with to= then it's a tool call + if strings.HasPrefix(role, "to=") { + harmonyHeader.Recipient = role[3:] + harmonyHeader.Role = "tool" + } else { + harmonyHeader.Role = role + } + + // the recipient (if any) can be specified before or after the channel tag, so + // we check it at the end once we've already parsed the channel and role + if harmonyHeader.Recipient == "" && len(tokens) > 0 && strings.HasPrefix(tokens[0], "to=") { + harmonyHeader.Recipient = tokens[0][3:] + } + + return harmonyHeader +} + +// longest overlap between suffix of s and prefix of delim +func overlap(s, delim string) int { + max := min(len(delim), len(s)) + for i := max; i > 0; i-- { + if strings.HasSuffix(s, delim[:i]) { + return i + } + } + return 0 +} + +// harmonyMessageState represents the current state of message processing +type harmonyMessageState int + +const ( + harmonyMessageState_Normal harmonyMessageState = iota + harmonyMessageState_Thinking + harmonyMessageState_ToolCalling +) + +// HarmonyMessageHandler processes harmony events and accumulates content appropriately. +// This is a higher level interface that maps harmony concepts into ollama concepts +type HarmonyMessageHandler struct { + state harmonyMessageState + harmonyParser *HarmonyParser +} + +// NewHarmonyMessageHandler creates a new message handler +func NewHarmonyMessageHandler() *HarmonyMessageHandler { + return &HarmonyMessageHandler{ + state: harmonyMessageState_Normal, + harmonyParser: &HarmonyParser{ + MessageStartTag: "<|start|>", + MessageEndTag: "<|end|>", + HeaderEndTag: "<|message|>", + }, + } +} + +// AddContent processes the content and returns the content, thinking, and tool content. +// content and thinking are already fully parsed, but tool content still needs to be passed to the tool parser +func (h *HarmonyMessageHandler) AddContent(content string, toolParser *HarmonyToolCallAccumulator) (string, string, string) { + contentSb := strings.Builder{} + thinkingSb := strings.Builder{} + toolContentSb := strings.Builder{} + + events := h.harmonyParser.AddContent(content) + for _, event := range events { + switch event := event.(type) { + case HarmonyEventHeaderComplete: + slog.Log(context.TODO(), logutil.LevelTrace, "harmony event header complete", "header", event.Header) + switch event.Header.Channel { + case "analysis": + if event.Header.Recipient != "" { + h.state = harmonyMessageState_ToolCalling + // event.Header.Recipient is the tool name, something like + // "browser.search" for a built-in, or "functions.calc" for a + // custom one + toolParser.SetToolName(event.Header.Recipient) + } else { + h.state = harmonyMessageState_Thinking + } + case "commentary": + if event.Header.Recipient != "" { + h.state = harmonyMessageState_ToolCalling + toolParser.SetToolName(event.Header.Recipient) + } else { + h.state = harmonyMessageState_Normal + } + case "final": + h.state = harmonyMessageState_Normal + } + case HarmonyEventContentEmitted: + slog.Log(context.TODO(), logutil.LevelTrace, "harmony event content", "content", event.Content, "state", h.state) + if h.state == harmonyMessageState_Normal { + contentSb.WriteString(event.Content) + } else if h.state == harmonyMessageState_Thinking { + thinkingSb.WriteString(event.Content) + } else if h.state == harmonyMessageState_ToolCalling { + toolContentSb.WriteString(event.Content) + } + case HarmonyEventMessageEnd: + h.state = harmonyMessageState_Normal + } + } + return contentSb.String(), thinkingSb.String(), toolContentSb.String() +} + +func (h *HarmonyMessageHandler) CreateToolParser() *HarmonyToolCallAccumulator { + return &HarmonyToolCallAccumulator{ + state: harmonyToolCallState_Normal, + currentToolName: nil, + } +} + +type harmonyToolCallState int + +const ( + harmonyToolCallState_Normal harmonyToolCallState = iota + harmonyToolCallState_ToolCalling +) + +type HarmonyToolCallAccumulator struct { + state harmonyToolCallState + acc strings.Builder + currentToolName *string +} + +func (a *HarmonyToolCallAccumulator) SetToolName(toolName string) { + a.currentToolName = &toolName +} + +func (a *HarmonyToolCallAccumulator) Add(content string) { + a.acc.WriteString(content) +} + +func (a *HarmonyToolCallAccumulator) Drain() (*string, string) { + str := a.acc.String() + a.state = harmonyToolCallState_Normal + a.acc.Reset() + return a.currentToolName, str +} + +func (a *HarmonyToolCallAccumulator) Content() string { + return a.acc.String() +} diff --git a/server/harmonyparser_test.go b/server/harmonyparser_test.go new file mode 100644 index 000000000..cd1743e1c --- /dev/null +++ b/server/harmonyparser_test.go @@ -0,0 +1,469 @@ +package server + +import ( + "fmt" + "reflect" + "testing" +) + +func TestHeaderParsing(t *testing.T) { + tests := []struct { + in, wantRole, wantChannel, wantRecipient string + }{ + { + in: "assistant<|channel|>analysis", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "", + }, + { + in: "assistant<|channel|>analysis to=functions.get_weather", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "functions.get_weather", + }, + { + in: "assistant to=functions.get_weather<|channel|>analysis", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "functions.get_weather", + }, + // special case where the role is replaced by the recipient (matches reference code) + { + in: "to=functions.get_weather<|channel|>analysis", + wantRole: "tool", + wantChannel: "analysis", + wantRecipient: "functions.get_weather", + }, + // extra token after the recipient is ignored + { + in: "assistant to=functions.get_weather abc<|channel|>analysis", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "functions.get_weather", + }, + // with constrain tag, recipient after channel tag + { + in: "assistant<|channel|>commentary to=functions.get_weather <|constrain|>json", + wantRole: "assistant", + wantChannel: "commentary", + wantRecipient: "functions.get_weather", + }, + // with constrain tag, recipient before channel tag + { + in: "assistant to=functions.get_weather<|channel|>commentary <|constrain|>json", + wantRole: "assistant", + wantChannel: "commentary", + wantRecipient: "functions.get_weather", + }, + // constrain tag without space + { + in: "assistant<|channel|>commentary to=functions.get_weather<|constrain|>json", + wantRole: "assistant", + wantChannel: "commentary", + wantRecipient: "functions.get_weather", + }, + // constrain tag without space, different order + { + in: "assistant to=functions.get_weather<|channel|>commentary<|constrain|>json", + wantRole: "assistant", + wantChannel: "commentary", + wantRecipient: "functions.get_weather", + }, + } + for i, tt := range tests { + parser := HarmonyParser{ + MessageStartTag: "<|start|>", + MessageEndTag: "<|end|>", + HeaderEndTag: "<|message|>", + } + header := parser.parseHeader(tt.in) + + if header.Role != tt.wantRole { + t.Errorf("case %d: got role \"%s\", want \"%s\"", i, header.Role, tt.wantRole) + } + if header.Channel != tt.wantChannel { + t.Errorf("case %d: got channel \"%s\", want \"%s\"", i, header.Channel, tt.wantChannel) + } + if header.Recipient != tt.wantRecipient { + t.Errorf("case %d: got recipient \"%s\", want \"%s\"", i, header.Recipient, tt.wantRecipient) + } + } +} + +func TestHarmonyParserHeaderEvent(t *testing.T) { + tests := []struct { + in, wantRole, wantChannel, wantRecipient string + implicitStart bool + }{ + { + in: "<|start|>user<|message|>What is 2 + 2?<|end|>", + wantRole: "user", + wantChannel: "", + wantRecipient: "", + }, + { + in: "<|start|>assistant<|channel|>analysis<|message|>What is 2 + 2?<|end|>", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "", + }, + { + in: "<|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"location\":\"San Francisco\"}<|call|><|start|>functions.get_weather to=assistant<|message|>{\"sunny\": true, \"temperature\": 20}<|end|>", + wantRole: "assistant", + wantChannel: "commentary", + wantRecipient: "functions.get_weather", + }, + { + in: "<|channel|>analysis<|message|>User asks weather in SF. We need location. Use get_current_weather with location \"San Francisco, CA\".<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\":\"San Francisco, CA\"}<|call|>", + wantRole: "assistant", + wantChannel: "analysis", + wantRecipient: "", + implicitStart: true, + }, + } + for i, tt := range tests { + parser := HarmonyParser{ + MessageStartTag: "<|start|>", + MessageEndTag: "<|end|>", + HeaderEndTag: "<|message|>", + } + if tt.implicitStart { + parser.AddImplicitStart() + } + gotEvents := parser.AddContent(tt.in) + if len(gotEvents) == 0 { + t.Errorf("case %d: got no events, want at least one", i) + } + + var firstHeaderEvent *HarmonyEventHeaderComplete + // print events + for _, event := range gotEvents { + fmt.Printf("event: %+v\n", event) + } + for _, event := range gotEvents { + if event, ok := event.(HarmonyEventHeaderComplete); ok { + firstHeaderEvent = &event + break + } + } + + if firstHeaderEvent == nil { + t.Errorf("case %d: got no header complete event, want one", i) + continue + } + gotHeader := firstHeaderEvent.Header + if gotHeader.Role != tt.wantRole || gotHeader.Channel != tt.wantChannel || gotHeader.Recipient != tt.wantRecipient { + t.Errorf("case %d: got header %+v, want role=%s channel=%s recipient=%s", i, gotHeader, tt.wantRole, tt.wantChannel, tt.wantRecipient) + } + } +} + +func TestHarmonyParserNonStreaming(t *testing.T) { + tests := []struct { + in string + implicitStart bool + wantEvents []HarmonyEvent + }{ + { + in: "<|start|>user<|message|>What is 2 + 2?<|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "What is 2 + 2?"}, + HarmonyEventMessageEnd{}, + }, + }, + { + in: "<|start|>assistant<|channel|>analysis<|message|>The answer is 4<|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "The answer is 4"}, + HarmonyEventMessageEnd{}, + }, + }, + { + in: "<|start|>assistant<|channel|>commentary to=functions.calc<|message|>Computing...<|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}}, + HarmonyEventContentEmitted{Content: "Computing..."}, + HarmonyEventMessageEnd{}, + }, + }, + { + in: "<|start|>user<|message|><|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventMessageEnd{}, + }, + }, + { + in: "<|start|>user<|message|>Hello<|end|><|start|>assistant<|message|>Hi!<|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "Hello"}, + HarmonyEventMessageEnd{}, + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "Hi!"}, + HarmonyEventMessageEnd{}, + }, + }, + { + in: "<|channel|>analysis<|message|>Thinking about the request<|end|>", + implicitStart: true, + wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}, HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}}, HarmonyEventContentEmitted{Content: "Thinking about the request"}, HarmonyEventMessageEnd{}}, + }, + } + for i, tt := range tests { + parser := HarmonyParser{ + MessageStartTag: "<|start|>", + MessageEndTag: "<|end|>", + HeaderEndTag: "<|message|>", + } + if tt.implicitStart { + parser.AddImplicitStart() + } + gotEvents := parser.AddContent(tt.in) + if !reflect.DeepEqual(gotEvents, tt.wantEvents) { + t.Errorf("case %d: got events %#v, want %#v", i, gotEvents, tt.wantEvents) + } + } +} + +func TestHarmonyParserStreaming(t *testing.T) { + type step struct { + input string + wantEvents []HarmonyEvent + } + + cases := []struct { + desc string + implicitStart bool + steps []step + }{ + { + desc: "simple message streamed character by character", + steps: []step{ + { + input: "<", + wantEvents: nil, + }, + { + input: "|", + wantEvents: nil, + }, + { + input: "start|>u", + wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}}, + }, + { + input: "ser<|mess", + wantEvents: nil, + }, + { + input: "age|>Hi", + wantEvents: []HarmonyEvent{ + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "Hi"}, + }, + }, + { + input: " there", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: " there"}}, + }, + { + input: "<|e", + wantEvents: nil, + }, + { + input: "nd|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "message with channel streamed", + steps: []step{ + { + input: "<|start|>assistant", + wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}}, + }, + { + input: "<|chan", + wantEvents: nil, + }, + { + input: "nel|>analysis", + wantEvents: nil, + }, + { + input: "<|message|>", + wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "analysis", Recipient: ""}}}, + }, + { + input: "Thinking", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Thinking"}}, + }, + { + input: "...", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "..."}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "message with channel and recipient", + steps: []step{ + { + input: "<|start|>assistant<|channel|>commentary to=functions.calc<|message|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}}, + }, + }, + { + input: "{\"x\": 5}", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "{\"x\": 5}"}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "message with channel and recipient (receipient before channel)", + steps: []step{ + { + input: "<|start|>assistant to=functions.calc<|channel|>commentary<|message|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "commentary", Recipient: "functions.calc"}}, + }, + }, + { + input: "{\"x\": 5}", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "{\"x\": 5}"}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "implicit start with channel", + implicitStart: true, + steps: []step{ + { + input: "<|channel|>thinking", + wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}}, + }, + { + input: "<|message|>", + wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "thinking", Recipient: ""}}}, + }, + { + input: "Processing request", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Processing request"}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "multiple messages streamed", + steps: []step{ + { + input: "<|start|>user<|message|>Hello<|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "Hello"}, + HarmonyEventMessageEnd{}, + }, + }, + { + input: "<|start|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageStart{}}, + }, + { + input: "assistant<|message|>", + wantEvents: []HarmonyEvent{HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "assistant", Channel: "", Recipient: ""}}}, + }, + { + input: "Hi!", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "Hi!"}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + { + desc: "empty message", + steps: []step{ + { + input: "<|start|>system<|message|><|end|>", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "system", Channel: "", Recipient: ""}}, + HarmonyEventMessageEnd{}, + }, + }, + }, + }, + { + desc: "partial tag that looks like end but isn't", + steps: []step{ + { + input: "<|start|>user<|message|>test<|e", + wantEvents: []HarmonyEvent{ + HarmonyEventMessageStart{}, + HarmonyEventHeaderComplete{Header: HarmonyHeader{Role: "user", Channel: "", Recipient: ""}}, + HarmonyEventContentEmitted{Content: "test"}, + }, + }, + { + input: "xample|>more", + wantEvents: []HarmonyEvent{HarmonyEventContentEmitted{Content: "<|example|>more"}}, + }, + { + input: "<|end|>", + wantEvents: []HarmonyEvent{HarmonyEventMessageEnd{}}, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + parser := HarmonyParser{ + MessageStartTag: "<|start|>", + MessageEndTag: "<|end|>", + HeaderEndTag: "<|message|>", + } + if tc.implicitStart { + parser.AddImplicitStart() + } + + for i, step := range tc.steps { + gotEvents := parser.AddContent(step.input) + if !reflect.DeepEqual(gotEvents, step.wantEvents) { + t.Errorf("step %d: input %q: got events %#v, want %#v", i, step.input, gotEvents, step.wantEvents) + } + } + }) + } +} diff --git a/server/images.go b/server/images.go index 38505cc51..0c16dd435 100644 --- a/server/images.go +++ b/server/images.go @@ -111,7 +111,8 @@ func (m *Model) Capabilities() []model.Capability { // Check for thinking capability openingTag, closingTag := thinking.InferTags(m.Template.Template) - if openingTag != "" && closingTag != "" { + hasTags := openingTag != "" && closingTag != "" + if hasTags || m.Config.ModelFamily == "gptoss" { capabilities = append(capabilities, model.CapabilityThinking) } diff --git a/server/prompt.go b/server/prompt.go index f8c895d71..5d6c3e27c 100644 --- a/server/prompt.go +++ b/server/prompt.go @@ -19,7 +19,7 @@ type tokenizeFunc func(context.Context, string) ([]int, error) // chatPrompt accepts a list of messages and returns the prompt and images that should be used for the next chat turn. // chatPrompt truncates any messages that exceed the context window of the model, making sure to always include 1) the // latest message and 2) system messages -func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *bool) (prompt string, images []llm.ImageData, _ error) { +func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api.Options, msgs []api.Message, tools []api.Tool, think *api.ThinkValue) (prompt string, images []llm.ImageData, _ error) { var system []api.Message // TODO: Ideally we would compute this from the projector metadata but some pieces are implementation dependent @@ -42,11 +42,13 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. } thinkVal := false + thinkLevel := "" if think != nil { - thinkVal = *think + thinkVal = think.AsBool() + thinkLevel = think.AsString() } var b bytes.Buffer - if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil { + if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[i:]...), Tools: tools, Think: thinkVal, ThinkLevel: thinkLevel, IsThinkSet: think != nil}); err != nil { return "", nil, err } @@ -101,10 +103,12 @@ func chatPrompt(ctx context.Context, m *Model, tokenize tokenizeFunc, opts *api. // truncate any messages that do not fit into the context window var b bytes.Buffer thinkVal := false + thinkLevel := "" if think != nil { - thinkVal = *think + thinkVal = think.AsBool() + thinkLevel = think.AsString() } - if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, IsThinkSet: think != nil}); err != nil { + if err := m.Template.Execute(&b, template.Values{Messages: append(system, msgs[currMsgIdx:]...), Tools: tools, Think: thinkVal, ThinkLevel: thinkLevel, IsThinkSet: think != nil}); err != nil { return "", nil, err } diff --git a/server/prompt_test.go b/server/prompt_test.go index 0043b9a47..659e64084 100644 --- a/server/prompt_test.go +++ b/server/prompt_test.go @@ -209,7 +209,7 @@ func TestChatPrompt(t *testing.T) { model := tt.model opts := api.Options{Runner: api.Runner{NumCtx: tt.limit}} think := false - prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil, &think) + prompt, images, err := chatPrompt(t.Context(), &model, mockRunner{}.Tokenize, &opts, tt.msgs, nil, &api.ThinkValue{Value: think}) if tt.error == nil && err != nil { t.Fatal(err) } else if tt.error != nil && err != tt.error { diff --git a/server/routes.go b/server/routes.go index b78f4b266..991e92003 100644 --- a/server/routes.go +++ b/server/routes.go @@ -187,11 +187,26 @@ func (s *Server) GenerateHandler(c *gin.Context) { return } + useHarmony := shouldUseHarmony(*m) && !req.Raw + var harmonyMessageHandler *HarmonyMessageHandler + var harmonyToolParser *HarmonyToolCallAccumulator + if useHarmony { + harmonyMessageHandler = NewHarmonyMessageHandler() + harmonyMessageHandler.harmonyParser.AddImplicitStart() + harmonyToolParser = harmonyMessageHandler.CreateToolParser() + } + + // Validate Think value: string values currently only allowed for gptoss models + if req.Think != nil && req.Think.IsString() && !useHarmony { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.AsString())}) + return + } + caps := []model.Capability{model.CapabilityCompletion} if req.Suffix != "" { caps = append(caps, model.CapabilityInsert) } - if req.Think != nil && *req.Think { + if req.Think != nil && req.Think.AsBool() { caps = append(caps, model.CapabilityThinking) // TODO(drifkin): consider adding a warning if it's false and the model // doesn't support thinking. It's not strictly required, but it can be a @@ -266,7 +281,11 @@ func (s *Server) GenerateHandler(c *gin.Context) { values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt}) } - values.Think = req.Think != nil && *req.Think + values.Think = req.Think != nil && req.Think.AsBool() + values.ThinkLevel = "" + if req.Think != nil { + values.ThinkLevel = req.Think.AsString() + } values.IsThinkSet = req.Think != nil var b bytes.Buffer @@ -289,11 +308,13 @@ func (s *Server) GenerateHandler(c *gin.Context) { } var thinkingState *thinking.Parser - openingTag, closingTag := thinking.InferTags(m.Template.Template) - if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { - thinkingState = &thinking.Parser{ - OpeningTag: openingTag, - ClosingTag: closingTag, + if !useHarmony { + openingTag, closingTag := thinking.InferTags(m.Template.Template) + if req.Think != nil && req.Think.AsBool() && openingTag != "" && closingTag != "" { + thinkingState = &thinking.Parser{ + OpeningTag: openingTag, + ClosingTag: closingTag, + } } } @@ -321,7 +342,12 @@ func (s *Server) GenerateHandler(c *gin.Context) { }, } - if thinkingState != nil { + if useHarmony { + content, thinking, toolContent := harmonyMessageHandler.AddContent(cr.Content, harmonyToolParser) + res.Response = content + res.Thinking = thinking + harmonyToolParser.Add(toolContent) + } else if thinkingState != nil { thinking, content := thinkingState.AddContent(cr.Content) res.Thinking = thinking res.Response = content @@ -332,6 +358,25 @@ func (s *Server) GenerateHandler(c *gin.Context) { } if cr.Done { + if useHarmony { + toolName, toolContent := harmonyToolParser.Drain() + if toolName != nil { + *toolName = strings.TrimPrefix(*toolName, "functions.") + var args api.ToolCallFunctionArguments + if err := json.Unmarshal([]byte(toolContent), &args); err != nil { + ch <- gin.H{"error parsing tool call": err.Error()} + return + } + + res.ToolCalls = append(res.ToolCalls, api.ToolCall{ + Function: api.ToolCallFunction{ + Name: *toolName, + Arguments: args, + }, + }) + } + } + res.DoneReason = cr.DoneReason.String() res.TotalDuration = time.Since(checkpointStart) res.LoadDuration = checkpointLoaded.Sub(checkpointStart) @@ -346,6 +391,15 @@ func (s *Server) GenerateHandler(c *gin.Context) { } } + if useHarmony { + // only send messages with meaningful content (empty messages confuse clients) + if res.Response != "" || res.Thinking != "" || res.Done || len(res.ToolCalls) > 0 { + ch <- res + } + + return + } + ch <- res }); err != nil { ch <- gin.H{"error": err.Error()} @@ -1476,7 +1530,7 @@ func (s *Server) ChatHandler(c *gin.Context) { if len(req.Tools) > 0 { caps = append(caps, model.CapabilityTools) } - if req.Think != nil && *req.Think { + if req.Think != nil && req.Think.AsBool() { caps = append(caps, model.CapabilityThinking) } @@ -1526,9 +1580,30 @@ func (s *Server) ChatHandler(c *gin.Context) { return } + useHarmony := shouldUseHarmony(*m) + + // Validate Think value: string values currently only allowed for gptoss models + if req.Think != nil && req.Think.IsString() && !useHarmony { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.AsString())}) + return + } + + var harmonyMessageHandler *HarmonyMessageHandler + var harmonyToolParser *HarmonyToolCallAccumulator + + if useHarmony { + harmonyMessageHandler = NewHarmonyMessageHandler() + var lastMessage *api.Message + if len(msgs) > 0 { + lastMessage = &msgs[len(msgs)-1] + } + harmonyMessageHandler.harmonyParser.AddImplicitStartOrPrefill(lastMessage) + harmonyToolParser = harmonyMessageHandler.CreateToolParser() + } + var thinkingState *thinking.Parser openingTag, closingTag := thinking.InferTags(m.Template.Template) - if req.Think != nil && *req.Think && openingTag != "" && closingTag != "" { + if req.Think != nil && req.Think.AsBool() && openingTag != "" && closingTag != "" { thinkingState = &thinking.Parser{ OpeningTag: openingTag, ClosingTag: closingTag, @@ -1536,7 +1611,7 @@ func (s *Server) ChatHandler(c *gin.Context) { } var toolParser *tools.Parser - if len(req.Tools) > 0 { + if len(req.Tools) > 0 && !useHarmony { toolParser = tools.NewParser(m.Template.Template, req.Tools) } @@ -1562,6 +1637,38 @@ func (s *Server) ChatHandler(c *gin.Context) { EvalDuration: r.EvalDuration, }, } + if r.Done { + res.DoneReason = r.DoneReason.String() + res.TotalDuration = time.Since(checkpointStart) + res.LoadDuration = checkpointLoaded.Sub(checkpointStart) + } + + if useHarmony { + content, thinking, toolContent := harmonyMessageHandler.AddContent(r.Content, harmonyToolParser) + res.Message.Content = content + res.Message.Thinking = thinking + harmonyToolParser.Add(toolContent) + + if r.Done { + toolName, toolContent := harmonyToolParser.Drain() + if toolName != nil { + *toolName = strings.TrimPrefix(*toolName, "functions.") + var args api.ToolCallFunctionArguments + if err := json.Unmarshal([]byte(toolContent), &args); err != nil { + ch <- gin.H{"error parsing tool call": err.Error()} + return + } + res.Message.ToolCalls = []api.ToolCall{{Function: api.ToolCallFunction{Name: *toolName, Arguments: args}}} + } + } + + // only send messages with meaningful content (empty messages confuse clients) + if res.Message.Content != "" || res.Message.Thinking != "" || len(res.Message.ToolCalls) > 0 || res.Done { + ch <- res + } + + return + } if thinkingState != nil { thinkingContent, remainingContent := thinkingState.AddContent(res.Message.Content) @@ -1573,12 +1680,6 @@ func (s *Server) ChatHandler(c *gin.Context) { res.Message.Thinking = thinkingContent } - if r.Done { - res.DoneReason = r.DoneReason.String() - res.TotalDuration = time.Since(checkpointStart) - res.LoadDuration = checkpointLoaded.Sub(checkpointStart) - } - if len(req.Tools) > 0 { toolCalls, content := toolParser.Add(res.Message.Content) if len(content) > 0 { diff --git a/server/routes_generate_test.go b/server/routes_generate_test.go index 75a246fc6..477d6b814 100644 --- a/server/routes_generate_test.go +++ b/server/routes_generate_test.go @@ -150,7 +150,7 @@ func TestGenerateChat(t *testing.T) { Messages: []api.Message{ {Role: "user", Content: "Hello!"}, }, - Think: &think, + Think: &api.ThinkValue{Value: think}, }) if w.Code != http.StatusBadRequest { diff --git a/server/routes_harmony_streaming_test.go b/server/routes_harmony_streaming_test.go new file mode 100644 index 000000000..424e65d27 --- /dev/null +++ b/server/routes_harmony_streaming_test.go @@ -0,0 +1,713 @@ +package server + +// this test file is to test integration of harmony parser into routes.go (as +// opposed to harmonyparser_test.go, which tests the parser in isolation) + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/discover" + "github.com/ollama/ollama/fs/ggml" + "github.com/ollama/ollama/llm" +) + +func getTestTools() []api.Tool { + return []api.Tool{ + { + Type: "function", + Function: api.ToolFunction{ + Name: "get_weather", + Description: "Get the current weather in a given location", + Parameters: struct { + Type string `json:"type"` + Defs any `json:"$defs,omitempty"` + Items any `json:"items,omitempty"` + Required []string `json:"required"` + Properties map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + } `json:"properties"` + }{ + Type: "object", + Required: []string{"location"}, + Properties: map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + }{ + "location": { + Type: api.PropertyType{"string"}, + Description: "The city and state, e.g. San Francisco, CA", + }, + }, + }, + }, + }, + { + Type: "function", + Function: api.ToolFunction{ + Name: "calculate", + Description: "Calculate a mathematical expression", + Parameters: struct { + Type string `json:"type"` + Defs any `json:"$defs,omitempty"` + Items any `json:"items,omitempty"` + Required []string `json:"required"` + Properties map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + } `json:"properties"` + }{ + Type: "object", + Required: []string{"expression"}, + Properties: map[string]struct { + Type api.PropertyType `json:"type"` + Items any `json:"items,omitempty"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + }{ + "expression": { + Type: api.PropertyType{"string"}, + Description: "The mathematical expression to calculate", + }, + }, + }, + }, + }, + } +} + +func createHarmonyTestModel(t *testing.T) (string, string) { + t.Helper() + + return createBinFile(t, ggml.KV{ + "general.architecture": "gptoss", + "llama.block_count": uint32(1), + "llama.context_length": uint32(8192), + "llama.embedding_length": uint32(4096), + "llama.attention.head_count": uint32(32), + "llama.attention.head_count_kv": uint32(8), + "tokenizer.ggml.tokens": []string{""}, + "tokenizer.ggml.scores": []float32{0}, + "tokenizer.ggml.token_type": []int32{0}, + }, []*ggml.Tensor{ + {Name: "token_embd.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.attn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.ffn_down.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.ffn_gate.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.ffn_up.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.ffn_norm.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.attn_k.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.attn_output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.attn_q.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "blk.0.attn_v.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + {Name: "output.weight", Shape: []uint64{1}, WriterTo: bytes.NewReader(make([]byte, 4))}, + }) +} + +// TestChatHarmonyParserStreamingRealtime verifies that chunks are emitted as soon as they're available +func TestChatHarmonyParserStreamingRealtime(t *testing.T) { + gin.SetMode(gin.TestMode) + + type step struct { + input llm.CompletionResponse + wantContent string + wantThinking string + wantToolCalls []api.ToolCall + } + + testCases := []struct { + name string + steps []step + only bool + }{ + { + name: "content streams as it arrives", + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|message|>Hello", Done: false}, + wantContent: "Hello", + }, + { + input: llm.CompletionResponse{Content: ", world", Done: false}, + wantContent: ", world", + }, + { + input: llm.CompletionResponse{Content: "!<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + wantContent: "!", + }, + }, + }, + { + name: "thinking streams separately from content", + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|channel|>analysis<|message|>Thinking...", Done: false}, + wantThinking: "Thinking...", + }, + { + input: llm.CompletionResponse{Content: "<|end|>", Done: false}, + // No output expected - just closes the analysis message and resets state to normal + }, + { + input: llm.CompletionResponse{Content: "<|start|>assistant<|message|>Answer", Done: false}, + wantContent: "Answer", // After message end, state is reset to normal + }, + { + input: llm.CompletionResponse{Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + // No output expected - just closes the assistant message + }, + }, + }, + { + name: "partial tags buffer until complete", + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|chan", Done: false}, + // No output - partial tag + }, + { + input: llm.CompletionResponse{Content: "nel|>analysis<|mess", Done: false}, + // No output - still building tags + }, + { + input: llm.CompletionResponse{Content: "age|>Deep ", Done: false}, + wantThinking: "Deep ", + }, + { + input: llm.CompletionResponse{Content: "thought<|end|>", Done: false}, + wantThinking: "thought", + }, + { + input: llm.CompletionResponse{Content: "<|start|>assistant<|message|>Done<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + wantContent: "Done", // After message end, state is reset to normal + }, + }, + }, + { + name: "simple assistant after analysis", + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|channel|>analysis<|message|>Think<|end|><|start|>assistant<|message|>Answer<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + wantContent: "Answer", + wantThinking: "Think", + }, + }, + }, + { + name: "tool call parsed and returned correctly", + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|channel|>commentary to=functions.get_weather<|message|>{\"location\":\"San Francisco\"}<|end|><|start|>assistant<|message|>The weather is sunny<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + wantContent: "The weather is sunny", + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "get_weather", + Arguments: api.ToolCallFunctionArguments{ + "location": "San Francisco", + }, + }, + }, + }, + }, + }, + }, + { + name: "tool call with streaming JSON across chunks", + only: true, + steps: []step{ + { + input: llm.CompletionResponse{Content: "<|channel|>commentary to=functions.calculate<|message|>{\"expr", Done: false}, + // No output yet - incomplete JSON + }, + { + input: llm.CompletionResponse{Content: "ession\":\"2+", Done: false}, + // Still no output - incomplete JSON + }, + { + input: llm.CompletionResponse{Content: "2\"}", Done: true}, + wantToolCalls: []api.ToolCall{ + { + Function: api.ToolCallFunction{ + Name: "calculate", + Arguments: api.ToolCallFunctionArguments{ + "expression": "2+2", + }, + }, + }, + }, + }, + }, + }, + } + + anyOnlies := false + for _, tc := range testCases { + if tc.only { + anyOnlies = true + } + } + + for _, tc := range testCases { + if anyOnlies && !tc.only { + continue + } + + t.Run(tc.name, func(t *testing.T) { + var chunks []api.ChatResponse + chunkIdx := 0 + + mockResponses := make([]llm.CompletionResponse, len(tc.steps)) + for i, step := range tc.steps { + mockResponses[i] = step.input + } + + mock := mockRunner{ + CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error { + for _, resp := range mockResponses { + fn(resp) + // Give the handler time to process each response + time.Sleep(30 * time.Millisecond) + } + return nil + }, + } + + s := Server{ + sched: &Scheduler{ + pendingReqCh: make(chan *LlmRequest, 1), + finishedReqCh: make(chan *LlmRequest, 1), + expiredCh: make(chan *runnerRef, 1), + unloadedCh: make(chan any, 1), + loaded: make(map[string]*runnerRef), + newServerFn: newMockServer(&mock), + getGpuFn: discover.GetGPUInfo, + getCpuFn: discover.GetCPUInfo, + reschedDelay: 100 * time.Millisecond, + loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) { + req.successCh <- &runnerRef{ + llama: &mock, + } + }, + }, + } + + go s.sched.Run(t.Context()) + + // Create a simple test model + _, digest := createHarmonyTestModel(t) + + streamFalse := false + w := createRequest(t, s.CreateHandler, api.CreateRequest{ + Model: "harmony-test-streaming", + Files: map[string]string{"test.gguf": digest}, + Template: `{{ with .Tools }}{{ end }}{{ .Prompt }}`, + Stream: &streamFalse, + }) + + if w.Code != 200 { + t.Fatalf("failed to create model: %d", w.Code) + } + + // Test chat endpoint with streaming + streamTrue := true + w = createRequest(t, s.ChatHandler, api.ChatRequest{ + Model: "harmony-test-streaming", + Messages: []api.Message{{Role: "user", Content: "Hello"}}, + Stream: &streamTrue, + Tools: getTestTools(), + }) + + if w.Code != 200 { + t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String()) + } + + // Parse all chunks + decoder := json.NewDecoder(w.Body) + for decoder.More() { + var chunk api.ChatResponse + if err := decoder.Decode(&chunk); err != nil { + t.Fatalf("failed to decode chunk: %v", err) + } + if chunk.Message.Content != "" || chunk.Message.Thinking != "" || len(chunk.Message.ToolCalls) > 0 { + chunks = append(chunks, chunk) + } + } + + // Log received chunks for debugging + if t.Failed() || len(chunks) == 0 { + t.Logf("Received %d chunks:", len(chunks)) + for i, chunk := range chunks { + t.Logf(" Chunk %d: content=%q thinking=%q", i, chunk.Message.Content, chunk.Message.Thinking) + } + } + + // Verify chunks match expected steps + for i, step := range tc.steps { + // Skip steps that don't expect any output + if step.wantContent == "" && step.wantThinking == "" && len(step.wantToolCalls) == 0 { + continue + } + + if chunkIdx >= len(chunks) { + t.Errorf("step %d: expected chunk not received (wanted content=%q thinking=%q)", + i, step.wantContent, step.wantThinking) + continue + } + + chunk := chunks[chunkIdx] + if chunk.Message.Content != step.wantContent || chunk.Message.Thinking != step.wantThinking { + t.Errorf("step %d: chunk mismatch: got (content=%q, thinking=%q), want (content=%q, thinking=%q)", + i, chunk.Message.Content, chunk.Message.Thinking, step.wantContent, step.wantThinking) + } + + // Check tool calls if expected + if len(step.wantToolCalls) > 0 { + if len(chunk.Message.ToolCalls) != len(step.wantToolCalls) { + t.Errorf("step %d: tool calls count mismatch: got %d, want %d", + i, len(chunk.Message.ToolCalls), len(step.wantToolCalls)) + } else { + for j, wantCall := range step.wantToolCalls { + if j >= len(chunk.Message.ToolCalls) { + break + } + gotCall := chunk.Message.ToolCalls[j] + if gotCall.Function.Name != wantCall.Function.Name { + t.Errorf("step %d, tool call %d: name mismatch: got %q, want %q", + i, j, gotCall.Function.Name, wantCall.Function.Name) + } + // Compare arguments as JSON strings for simplicity + gotArgs, _ := json.Marshal(gotCall.Function.Arguments) + wantArgs, _ := json.Marshal(wantCall.Function.Arguments) + if string(gotArgs) != string(wantArgs) { + t.Errorf("step %d, tool call %d: arguments mismatch: got %s, want %s", + i, j, string(gotArgs), string(wantArgs)) + } + } + } + } + chunkIdx++ + } + + // Check if we have extra chunks + if chunkIdx < len(chunks) { + t.Errorf("received %d extra chunks", len(chunks)-chunkIdx) + for i := chunkIdx; i < len(chunks); i++ { + t.Logf(" extra chunk %d: content=%q thinking=%q", + i-chunkIdx, chunks[i].Message.Content, chunks[i].Message.Thinking) + } + } + }) + } +} + +// TestChatHarmonyParserStreamingSimple is a simpler test that just verifies basic streaming +func TestChatHarmonyParserStreamingSimple(t *testing.T) { + gin.SetMode(gin.TestMode) + + mockResponses := []llm.CompletionResponse{ + {Content: "<|start|>assistant<|message|>First ", Done: false}, + {Content: "chunk ", Done: false}, + {Content: "here<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + } + + mock := mockRunner{ + CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error { + t.Logf("Mock received prompt: %q", r.Prompt) + t.Logf("Mock sending %d responses", len(mockResponses)) + for i, resp := range mockResponses { + t.Logf("Sending response %d: %q", i, resp.Content) + fn(resp) + } + return nil + }, + } + + s := Server{ + sched: &Scheduler{ + pendingReqCh: make(chan *LlmRequest, 1), + finishedReqCh: make(chan *LlmRequest, 1), + expiredCh: make(chan *runnerRef, 1), + unloadedCh: make(chan any, 1), + loaded: make(map[string]*runnerRef), + newServerFn: newMockServer(&mock), + getGpuFn: discover.GetGPUInfo, + getCpuFn: discover.GetCPUInfo, + reschedDelay: 100 * time.Millisecond, + loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) { + req.successCh <- &runnerRef{ + llama: &mock, + } + }, + }, + } + + go s.sched.Run(t.Context()) + + // Create model + _, digest := createHarmonyTestModel(t) + streamFalse := false + w := createRequest(t, s.CreateHandler, api.CreateRequest{ + Model: "harmony-test-simple", + Files: map[string]string{"test.gguf": digest}, + Template: `{{ .Tools }}{{ .Prompt }}`, + Stream: &streamFalse, + }) + + if w.Code != 200 { + t.Fatalf("failed to create model: %d", w.Code) + } + + // Test streaming + streamTrue := true + w = createRequest(t, s.ChatHandler, api.ChatRequest{ + Model: "harmony-test-simple", + Messages: []api.Message{{Role: "user", Content: "Hello"}}, + Stream: &streamTrue, + Tools: getTestTools(), + }) + + if w.Code != 200 { + t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String()) + } + + // Parse chunks + var chunks []api.ChatResponse + decoder := json.NewDecoder(w.Body) + for decoder.More() { + var chunk api.ChatResponse + if err := decoder.Decode(&chunk); err != nil { + t.Fatalf("failed to decode chunk: %v", err) + } + chunks = append(chunks, chunk) + t.Logf("Received chunk %d: content=%q thinking=%q done=%v", + len(chunks), chunk.Message.Content, chunk.Message.Thinking, chunk.Done) + } + + // Verify we got chunks + if len(chunks) == 0 { + t.Fatal("expected streaming chunks, got none") + } + + // Verify content + var content strings.Builder + for _, chunk := range chunks { + content.WriteString(chunk.Message.Content) + } + + expectedContent := "First chunk here" + if content.String() != expectedContent { + t.Errorf("content mismatch: got %q, want %q", content.String(), expectedContent) + } + + // Verify we got multiple chunks (streaming) + contentChunks := 0 + for _, chunk := range chunks { + if chunk.Message.Content != "" { + contentChunks++ + } + } + + if contentChunks < 2 { + t.Errorf("expected at least 2 content chunks for streaming, got %d", contentChunks) + } +} + +func TestChatHarmonyParserStreaming(t *testing.T) { + gin.SetMode(gin.TestMode) + + type expectedChunk struct { + afterResponse int // Which mock response this chunk should appear after + content string // Expected content in this chunk + thinking string // Expected thinking in this chunk + } + + testCases := []struct { + name string + mockResponses []llm.CompletionResponse + expectedChunks []expectedChunk + wantContent string + wantThinking string + }{ + { + name: "simple message without thinking", + mockResponses: []llm.CompletionResponse{ + {Content: "<|start|>assistant<|message|>Hello, ", Done: false}, + {Content: "how can I help?", Done: false}, + {Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + }, + expectedChunks: []expectedChunk{ + {afterResponse: 1, content: "Hello, "}, + {afterResponse: 2, content: "how can I help?"}, + }, + wantContent: "Hello, how can I help?", + }, + { + name: "message with analysis channel for thinking", + mockResponses: []llm.CompletionResponse{ + {Content: "<|channel|>analysis<|message|>", Done: false}, + {Content: "Let me think ", Done: false}, + {Content: "about this problem...", Done: false}, + {Content: "<|end|>", Done: false}, + {Content: "<|start|>assistant<|message|>", Done: false}, + {Content: "The answer ", Done: false}, + {Content: "is 42", Done: false}, + {Content: "<|end|>", Done: true, DoneReason: llm.DoneReasonStop}, + }, + expectedChunks: []expectedChunk{ + {afterResponse: 2, thinking: "Let me think "}, + {afterResponse: 3, thinking: "about this problem..."}, + {afterResponse: 6, content: "The answer "}, + {afterResponse: 7, content: "is 42"}, + }, + wantContent: "The answer is 42", + wantThinking: "Let me think about this problem...", + }, + { + name: "streaming with partial tags across boundaries", + mockResponses: []llm.CompletionResponse{ + {Content: "<|chan", Done: false}, + {Content: "nel|>analy", Done: false}, + {Content: "sis<|mess", Done: false}, + {Content: "age|>Think", Done: false}, + {Content: "ing deeply...<|end|>", Done: false}, + {Content: "<|start|>assi", Done: false}, + {Content: "stant<|message|>Result ", Done: false}, + {Content: "computed<|e", Done: false}, + {Content: "nd|>", Done: true, DoneReason: llm.DoneReasonStop}, + }, + expectedChunks: []expectedChunk{ + {afterResponse: 4, thinking: "Think"}, + {afterResponse: 5, thinking: "ing deeply..."}, + {afterResponse: 7, content: "Result "}, + {afterResponse: 8, content: "computed"}, + }, + wantContent: "Result computed", + wantThinking: "Thinking deeply...", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Channel to synchronize mock responses with chunk verification + responsesSent := make(chan int, len(tc.mockResponses)) + + mock := mockRunner{ + CompletionFn: func(ctx context.Context, r llm.CompletionRequest, fn func(llm.CompletionResponse)) error { + // Send mock responses one at a time, notifying when each is sent + for i, resp := range tc.mockResponses { + fn(resp) + responsesSent <- i + 1 + } + close(responsesSent) + return nil + }, + } + + s := Server{ + sched: &Scheduler{ + pendingReqCh: make(chan *LlmRequest, 1), + finishedReqCh: make(chan *LlmRequest, 1), + expiredCh: make(chan *runnerRef, 1), + unloadedCh: make(chan any, 1), + loaded: make(map[string]*runnerRef), + newServerFn: newMockServer(&mock), + getGpuFn: discover.GetGPUInfo, + getCpuFn: discover.GetCPUInfo, + reschedDelay: 250 * time.Millisecond, + loadFn: func(req *LlmRequest, _ *ggml.GGML, _ discover.GpuInfoList, _ int) { + req.successCh <- &runnerRef{ + llama: &mock, + } + }, + }, + } + + go s.sched.Run(t.Context()) + + // Create a minimal model + _, digest := createHarmonyTestModel(t) + + // Create model with passthrough template + stream := false + w := createRequest(t, s.CreateHandler, api.CreateRequest{ + Model: "harmony-test", + Files: map[string]string{"file.gguf": digest}, + Template: `{{ with .Tools }}{{ end }}{{ .Prompt }}`, + Stream: &stream, + }) + + if w.Code != http.StatusOK { + t.Fatalf("failed to create model: %d", w.Code) + } + + // Test chat endpoint with streaming + streamTrue := true + w = createRequest(t, s.ChatHandler, api.ChatRequest{ + Model: "harmony-test", + Messages: []api.Message{{Role: "user", Content: "Hello"}}, + Stream: &streamTrue, + Tools: getTestTools(), + }) + + if w.Code != http.StatusOK { + t.Fatalf("chat request failed: %d - %s", w.Code, w.Body.String()) + } + + // Parse streaming response + var chunks []api.ChatResponse + var content, thinking strings.Builder + + decoder := json.NewDecoder(w.Body) + for decoder.More() { + var chunk api.ChatResponse + if err := decoder.Decode(&chunk); err != nil { + t.Fatalf("failed to decode chunk: %v", err) + } + chunks = append(chunks, chunk) + + // Accumulate content and thinking from each chunk + content.WriteString(chunk.Message.Content) + thinking.WriteString(chunk.Message.Thinking) + + // Debug output + t.Logf("Chunk %d: content=%q thinking=%q done=%v", len(chunks), chunk.Message.Content, chunk.Message.Thinking, chunk.Done) + } + + // Verify we got streaming chunks + if len(chunks) == 0 { + t.Fatal("expected streaming chunks, got none") + } + + gotContent := content.String() + gotThinking := thinking.String() + + if gotContent != tc.wantContent { + t.Errorf("content mismatch: got %q, want %q", gotContent, tc.wantContent) + } + if gotThinking != tc.wantThinking { + t.Errorf("thinking mismatch: got %q, want %q", gotThinking, tc.wantThinking) + } + + // Verify last chunk has done=true + lastChunk := chunks[len(chunks)-1] + if !lastChunk.Done { + t.Error("expected last chunk to have done=true") + } + }) + } +} diff --git a/template/template.go b/template/template.go index d28ace413..bfd02a92d 100644 --- a/template/template.go +++ b/template/template.go @@ -13,6 +13,7 @@ import ( "sync" "text/template" "text/template/parse" + "time" "github.com/agnivade/levenshtein" @@ -121,6 +122,11 @@ var funcs = template.FuncMap{ b, _ := json.Marshal(v) return string(b) }, + "currentDate": func(args ...string) string { + // Currently ignoring the format argument, but accepting it for future use + // Default format is YYYY-MM-DD + return time.Now().Format("2006-01-02") + }, } func Parse(s string) (*Template, error) { @@ -160,12 +166,18 @@ func (t *Template) Vars() []string { return slices.Sorted(maps.Keys(set)) } +func (t *Template) Contains(s string) bool { + return strings.Contains(t.raw, s) +} + type Values struct { Messages []api.Message api.Tools Prompt string Suffix string Think bool + // ThinkLevel contains the thinking level if Think is true and a string value was provided + ThinkLevel string // whether or not the user explicitly set the thinking flag (vs. it being // implicitly false). Templates can't see whether `Think` is nil IsThinkSet bool @@ -228,6 +240,7 @@ func (t *Template) Execute(w io.Writer, v Values) error { "Suffix": v.Suffix, "Response": "", "Think": v.Think, + "ThinkLevel": v.ThinkLevel, "IsThinkSet": v.IsThinkSet, }) } else if !v.forceLegacy && slices.Contains(t.Vars(), "messages") { @@ -237,6 +250,7 @@ func (t *Template) Execute(w io.Writer, v Values) error { "Tools": v.Tools, "Response": "", "Think": v.Think, + "ThinkLevel": v.ThinkLevel, "IsThinkSet": v.IsThinkSet, }) } @@ -251,6 +265,7 @@ func (t *Template) Execute(w io.Writer, v Values) error { "Prompt": prompt, "Response": response, "Think": v.Think, + "ThinkLevel": v.ThinkLevel, "IsThinkSet": v.IsThinkSet, }); err != nil { return err @@ -298,6 +313,7 @@ func (t *Template) Execute(w io.Writer, v Values) error { "Prompt": prompt, "Response": response, "Think": v.Think, + "ThinkLevel": v.ThinkLevel, "IsThinkSet": v.IsThinkSet, }); err != nil { return err diff --git a/tmp-devon-modelsfiles/Modelfile b/tmp-devon-modelsfiles/Modelfile new file mode 100644 index 000000000..0d396bb1b --- /dev/null +++ b/tmp-devon-modelsfiles/Modelfile @@ -0,0 +1,132 @@ +FROM gpt-oss:120b-tmpl + +PARAMETER num_predict 4000 + +TEMPLATE """{{- if .Tools }}{{ end }}<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: {{ currentDate }} +{{- if and .IsThinkSet .Think (ne .ThinkLevel "") }} + +reasoning: {{ .ThinkLevel }} +{{- else if or (not .IsThinkSet) (and .IsThinkSet .Think) }} + +reasoning: medium +{{- end }} + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.{{- $hasNonBuiltinTools := false }}{{- if .Tools }}{{/* Check for all tools */}} +{{- $hasBrowserSearch := false }} +{{- $hasBrowserOpen := false }} +{{- $hasBrowserFind := false }} +{{- $hasPython := false }} +{{- range .Tools }} + {{- if eq .Function.Name "browser.search" }}{{ $hasBrowserSearch = true }} + {{- else if eq .Function.Name "browser.open" }}{{ $hasBrowserOpen = true }} + {{- else if eq .Function.Name "browser.find" }}{{ $hasBrowserFind = true }} + {{- else if eq .Function.Name "python" }}{{ $hasPython = true }} + {{- else }}{{ $hasNonBuiltinTools = true }} + {{- end }} +{{- end }} +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind $hasPython }} + +# Tools +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind }} + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// sources=web_with_bing (default: web_with_bing) +namespace browser { +{{- if $hasBrowserSearch }} + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserOpen }} + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserFind }} + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; +{{- end }} + +} // namespace browser +{{- end }}{{/* end if has browser tools */}} +{{- if $hasPython }} + +## python + +Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). + +When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster. +{{- end }}{{/* end if hasPython */}} +{{- end }}{{/* end if has any built-in tools */}} +{{- end }}{{/* end if .Tools */}}<|end|>{{/* end of system */ -}} +<|start|>developer<|message|>{{- if $hasNonBuiltinTools }}# Tools + +## functions + +namespace functions { +{{- range .Tools }} +{{- if not (or (eq .Function.Name "browser.search") (eq .Function.Name "browser.open") (eq .Function.Name "browser.find") (eq .Function.Name "python")) }} +{{if .Function.Description }} +// {{ .Function.Description }} +{{- end }} +{{- if and .Function.Parameters.Properties (gt (len .Function.Parameters.Properties) 0) }} +type {{ .Function.Name }} = (_: { +{{- range $name, $prop := .Function.Parameters.Properties }} +{{- if $prop.Description }} + // {{ $prop.Description }} +{{- end }} + {{ $name }}: {{ if gt (len $prop.Type) 1 }}{{ range $i, $t := $prop.Type }}{{ if $i }} | {{ end }}{{ $t }}{{ end }}{{ else }}{{ index $prop.Type 0 }}{{ end }}, +{{- end }} +}) => any; +{{- else }} +type {{ .Function.Name }} = () => any; +{{- end }} +{{- end }}{{/* end if not browser tool */}} +{{- end }}{{/* end of range .Tools */}} + +} // namespace functions +{{- end }}{{/* end if hasNonBuiltinTools */}} +{{- if .System}} + +# Instructions + +{{ .System }} +{{- end }}{{/* end of if system */}}<|end|> +{{- range $i, $msg := .Messages }}{{- if (ne $msg.Role "system") }}{{- if eq $msg.Role "tool" }}{{- if or (eq $msg.ToolName "python") (eq $msg.ToolName "browser.search") (eq $msg.ToolName "browser.open") (eq $msg.ToolName "browser.find") }}<|start|>{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> +{{- else }}<|start|>functions.{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> +{{- end }} +{{- else }}<|start|>{{ $msg.Role }}<|message|>{{ $msg.Content }}<|end|> +{{- end }}{{- end }}{{- end }}{{/* end of range .Messages */}} +{{- /* end of developer */ -}}<|start|>assistant""" +SYSTEM "hey there" +PARAMETER stop <|return|> +PARAMETER stop <|call|> +PARAMETER temperature 0 \ No newline at end of file diff --git a/tmp-devon-modelsfiles/Modelfile-120b b/tmp-devon-modelsfiles/Modelfile-120b new file mode 100644 index 000000000..5e4e112ba --- /dev/null +++ b/tmp-devon-modelsfiles/Modelfile-120b @@ -0,0 +1,169 @@ +FROM gpt-oss:120b-weights + +PARAMETER num_predict 8000 + +TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: {{ currentDate }} +{{- if and .IsThinkSet .Think (ne .ThinkLevel "") }} + +reasoning: {{ .ThinkLevel }} +{{- else if or (not .IsThinkSet) (and .IsThinkSet .Think) }} + +reasoning: medium +{{- end }} + +{{- $hasNonBuiltinTools := false }} +{{- if .Tools -}} +{{- $hasBrowserSearch := false }} +{{- $hasBrowserOpen := false }} +{{- $hasBrowserFind := false }} +{{- $hasPython := false }} + {{- range .Tools }} + {{- if eq .Function.Name "browser.search" -}}{{- $hasBrowserSearch = true -}} + {{- else if eq .Function.Name "browser.open" -}}{{- $hasBrowserOpen = true -}} + {{- else if eq .Function.Name "browser.find" -}}{{- $hasBrowserFind = true -}} + {{- else if eq .Function.Name "python" -}}{{- $hasPython = true -}} + {{- else }}{{ $hasNonBuiltinTools = true -}} + {{- end }} + {{- end }} +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind $hasPython }} + +# Tools +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind }} + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { +{{- if $hasBrowserSearch }} + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserOpen }} + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserFind }} + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; +{{- end }} + +} // namespace browser +{{- end }}{{/* end if has browser tools */}} +{{- if $hasPython }} + +## python + +Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). + +When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster. +{{- end }}{{/* end if hasPython */}} +{{- end }}{{/* end if has any built-in tools */}} +{{- end }}{{/* end if .Tools */}} + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|>{{/* end of system */ -}} +{{- if or $hasNonBuiltinTools .System -}} +<|start|>developer<|message|>{{- if $hasNonBuiltinTools }}# Tools + +## functions + +namespace functions { +{{- range .Tools }} +{{- if not (or (eq .Function.Name "browser.search") (eq .Function.Name "browser.open") (eq .Function.Name "browser.find") (eq .Function.Name "python")) }} +{{if .Function.Description }} +// {{ .Function.Description }} +{{- end }} +{{- if and .Function.Parameters.Properties (gt (len .Function.Parameters.Properties) 0) }} +type {{ .Function.Name }} = (_: { +{{- range $name, $prop := .Function.Parameters.Properties }} +{{- if $prop.Description }} + // {{ $prop.Description }} +{{- end }} + {{ $name }}: {{ if gt (len $prop.Type) 1 }}{{ range $i, $t := $prop.Type }}{{ if $i }} | {{ end }}{{ $t }}{{ end }}{{ else }}{{ index $prop.Type 0 }}{{ end }}, +{{- end }} +}) => any; +{{- else }} +type {{ .Function.Name }} = () => any; +{{- end }} +{{- end }}{{/* end if not browser tool */}} +{{- end }}{{/* end of range .Tools */}} + +} // namespace functions +{{- end }}{{/* end if hasNonBuiltinTools */}} +{{- if .System}} + +# Instructions + +{{ .System }} +{{- end -}} +<|end|> +{{- end -}} +{{- /* Find the index of the last user message */ -}} +{{- $lastUserIdx := -1 }} +{{- range $i, $msg := .Messages }} + {{- if eq $msg.Role "user" }} + {{- $lastUserIdx = $i }} + {{- end -}} +{{- end -}} +{{- /* Now render messages */ -}} +{{- range $i, $msg := .Messages }} + {{- if (ne $msg.Role "system") -}} + {{- if eq $msg.Role "tool" -}} + {{- if or (eq $msg.ToolName "python") (eq $msg.ToolName "browser.search") (eq $msg.ToolName "browser.open") (eq $msg.ToolName "browser.find") -}} + <|start|>{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- else -}} + <|start|>functions.{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- end -}} + {{- else if eq $msg.Role "assistant" -}} + {{- if and $msg.Thinking (gt $i $lastUserIdx) -}}{{- /* Show thinking only after last user message */ -}} + <|start|>assistant<|channel|>analysis<|message|>{{ $msg.Thinking }}<|end|> + {{- end -}} + {{- if gt (len $msg.Content) 0 -}} + <|start|>assistant<|channel|>final<|message|>{{ $msg.Content }}<|end|> + {{- end -}} + {{- if gt (len $msg.ToolCalls) 0 -}} + {{- range $j, $toolCall := $msg.ToolCalls -}} + {{- $isBuiltin := or (eq $toolCall.Function.Name "python") (eq $toolCall.Function.Name "browser.search") (eq $toolCall.Function.Name "browser.open") (eq $toolCall.Function.Name "browser.find") -}} + <|start|>assistant<|channel|>{{ if $isBuiltin }}analysis{{ else }}commentary{{ end }} to={{ if not $isBuiltin}}functions.{{end}}{{ $toolCall.Function.Name }} <|constrain|>json<|message|>{{ $toolCall.Function.Arguments }}<|call|> + {{- end -}} + {{- end -}} + {{- else if eq $msg.Role "user" -}} + <|start|>{{ $msg.Role }}<|message|>{{ $msg.Content }}<|end|> + {{- end }} + {{- else }} + {{- end }} +{{- end -}} +<|start|>assistant""" +SYSTEM "" +PARAMETER stop <|return|> +PARAMETER stop <|call|> +PARAMETER temperature 0 diff --git a/tmp-devon-modelsfiles/Modelfile-local b/tmp-devon-modelsfiles/Modelfile-local new file mode 100644 index 000000000..947151b1b --- /dev/null +++ b/tmp-devon-modelsfiles/Modelfile-local @@ -0,0 +1,181 @@ +FROM gpt-oss-local-weights + +PARAMETER num_predict 4000 + +TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: {{ currentDate }} +{{- if and .IsThinkSet .Think (ne .ThinkLevel "") }} + +Reasoning: {{ .ThinkLevel }} +{{- else if or (not .IsThinkSet) (and .IsThinkSet .Think) }} + +Reasoning: medium +{{- end }} + +{{- $hasNonBuiltinTools := false }} +{{- if .Tools -}} +{{- $hasBrowserSearch := false }} +{{- $hasBrowserOpen := false }} +{{- $hasBrowserFind := false }} +{{- $hasPython := false }} + {{- range .Tools }} + {{- if eq .Function.Name "browser.search" -}}{{- $hasBrowserSearch = true -}} + {{- else if eq .Function.Name "browser.open" -}}{{- $hasBrowserOpen = true -}} + {{- else if eq .Function.Name "browser.find" -}}{{- $hasBrowserFind = true -}} + {{- else if eq .Function.Name "python" -}}{{- $hasPython = true -}} + {{- else }}{{ $hasNonBuiltinTools = true -}} + {{- end }} + {{- end }} +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind $hasPython }} + +# Tools +{{- if or $hasBrowserSearch $hasBrowserOpen $hasBrowserFind }} + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { +{{- if $hasBrowserSearch }} + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserOpen }} + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; +{{- end }} +{{- if $hasBrowserFind }} + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; +{{- end }} + +} // namespace browser +{{- end }}{{/* end if has browser tools */}} +{{- if $hasPython }} + +## python + +Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). + +When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster. +{{- end }}{{/* end if hasPython */}} +{{- end }}{{/* end if has any built-in tools */}} +{{- end }}{{/* end if .Tools */}} + +# Valid channels: analysis, commentary, final. Channel must be included for every message.{{ if $hasNonBuiltinTools }} +Calls to these tools must go to the commentary channel: 'functions'. +{{- end -}}<|end|>{{/* end of system */ -}} +{{- if or $hasNonBuiltinTools .System -}} +<|start|>developer<|message|>{{- if $hasNonBuiltinTools }}# Tools + +## functions + +namespace functions { +{{- range .Tools }} +{{- if not (or (eq .Function.Name "browser.search") (eq .Function.Name "browser.open") (eq .Function.Name "browser.find") (eq .Function.Name "python")) }} +{{if .Function.Description }} +// {{ .Function.Description }} +{{- end }} +{{- if and .Function.Parameters.Properties (gt (len .Function.Parameters.Properties) 0) }} +type {{ .Function.Name }} = (_: { +{{- range $name, $prop := .Function.Parameters.Properties }} +{{- if $prop.Description }} + // {{ $prop.Description }} +{{- end }} + {{ $name }}: {{ if gt (len $prop.Type) 1 }}{{ range $i, $t := $prop.Type }}{{ if $i }} | {{ end }}{{ $t }}{{ end }}{{ else }}{{ index $prop.Type 0 }}{{ end }}, +{{- end }} +}) => any; +{{- else }} +type {{ .Function.Name }} = () => any; +{{- end }} +{{- end }}{{/* end if not browser tool */}} +{{- end }}{{/* end of range .Tools */}} + +} // namespace functions +{{- end }}{{/* end if hasNonBuiltinTools */}} +{{- if .System}} + +# Instructions + +{{ .System }} +{{- end -}} +<|end|> +{{- end -}} +{{- /* Find the index of the last user message */ -}} +{{- $lastUserIdx := -1 }} +{{- $prefillingContent := false }} +{{- $prefillingThinkingOnly := false }} +{{- range $i, $msg := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{- if eq $msg.Role "user" }} + {{- $lastUserIdx = $i }} + {{- end -}} + {{- if and $last (eq $msg.Role "assistant") (gt (len $msg.Content) 0) }} + {{- $prefillingContent = true }} + {{- else if and $last (eq $msg.Role "assistant") (gt (len $msg.Thinking) 0) }} + {{- $prefillingThinkingOnly = true }} + {{- end }} +{{- end -}} +{{- /* Now render messages */ -}} +{{- range $i, $msg := .Messages }} + {{- $last := eq (len (slice $.Messages $i)) 1 -}} + {{- if (ne $msg.Role "system") -}} + {{- if eq $msg.Role "tool" -}} + {{- if or (eq $msg.ToolName "python") (eq $msg.ToolName "browser.search") (eq $msg.ToolName "browser.open") (eq $msg.ToolName "browser.find") -}} + <|start|>{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- else -}} + <|start|>functions.{{ $msg.ToolName }} to=assistant<|message|>{{ $msg.Content }}<|end|> + {{- end -}} + {{- else if eq $msg.Role "assistant" -}} + {{- if and $msg.Thinking (gt $i $lastUserIdx) -}}{{- /* Show thinking only after last user message */ -}} + <|start|>assistant<|channel|>analysis<|message|>{{ $msg.Thinking }}{{- if not $prefillingThinkingOnly -}}<|end|>{{- end -}} + {{- end -}} + {{- if gt (len $msg.Content) 0 -}} + <|start|>assistant<|channel|>final<|message|>{{ $msg.Content }}{{- if not $prefillingContent -}}<|end|>{{- end -}} + {{- end -}} + {{- if gt (len $msg.ToolCalls) 0 -}} + {{- range $j, $toolCall := $msg.ToolCalls -}} + {{- $isBuiltin := or (eq $toolCall.Function.Name "python") (eq $toolCall.Function.Name "browser.search") (eq $toolCall.Function.Name "browser.open") (eq $toolCall.Function.Name "browser.find") -}} + <|start|>assistant<|channel|>{{ if $isBuiltin }}analysis{{ else }}commentary{{ end }} to={{ if not $isBuiltin}}functions.{{end}}{{ $toolCall.Function.Name }} <|constrain|>json<|message|>{{ $toolCall.Function.Arguments }}<|call|> + {{- end -}} + {{- end -}} + {{- else if eq $msg.Role "user" -}} + <|start|>{{ $msg.Role }}<|message|>{{ $msg.Content }}<|end|> + {{- end }} + {{- else }} + {{- end }} +{{- end -}} +{{- if not (or $prefillingContent $prefillingThinkingOnly) -}} +<|start|>assistant +{{- end -}}""" +SYSTEM "" +PARAMETER stop <|return|> +PARAMETER stop <|call|> +PARAMETER temperature 1 diff --git a/tmp-devon-modelsfiles/Modelfile-local-weights b/tmp-devon-modelsfiles/Modelfile-local-weights new file mode 100644 index 000000000..ed6e950cd --- /dev/null +++ b/tmp-devon-modelsfiles/Modelfile-local-weights @@ -0,0 +1,6 @@ +FROM /Users/drifkin/Downloads/sha256-14c7c5e8e3b9a312ae89fabe670b711c27a3ff22f778dc8bbe0e8d856ca9fe1e + +SYSTEM "" +PARAMETER stop <|return|> +PARAMETER stop <|call|> +PARAMETER temperature 0 diff --git a/tools/tools.go b/tools/tools.go index f473ab6a6..f9ca15530 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -26,6 +26,10 @@ type Parser struct { n int } +func (p *Parser) GetBuffer() []byte { + return p.buffer +} + // NewParser creates a new tool call parser from a model's chat // template and a list of provided tools. func NewParser(tmpl *template.Template, tools []api.Tool) *Parser {