mirror of
https://github.com/ollama/ollama.git
synced 2026-04-17 15:53:27 +02:00
* bench: add prompt calibration, context size flag, and NumCtx reporting Add --num-ctx flag to set context size, and report NumCtx in model info header. Calibrate tokens-per-word ratio during warmup using actual tokenization metrics from the model, replacing the fixed 1.3 heuristic. This produces more accurate prompt token counts for --prompt-tokens. Also add fetchContextLength() to query running model context via /api/ps. * integration: improve vision test robustness and add thinking tests Add skipIfNoVisionOverride() to skip vision tests when OLLAMA_TEST_MODEL is set to a non-vision model. Add Think:false to context exhaustion test to prevent thinking models from using all context before the test can measure it. Add third test image (ollama homepage) and replace OCR test with ImageDescription test using it. Relax match strings for broader model compatibility. Add TestThinkingEnabled and TestThinkingSuppressed to verify thinking output and channel tag handling. * gemma4: add Gemma 4 GGML model support Add full Gemma 4 model family support (E2B, E4B, 26B MoE, 31B Dense) for the GGML backend including text, vision, converter, parser, and renderer. Text model features: - Sliding window + full attention with per-layer patterns - KV sharing across layers with donor map - Per-layer embeddings (PLE) with learned projections - MoE routing with RMSNorm + learned scale - Proportional RoPE with freq_factors for global attention - Final logit softcapping Vision model features: - SigLIP vision encoder with 2D RoPE - ClippableLinear with input/output clamping via packed v.clamp_data - Adaptive average pooling with nMerge kernel - Multi-modal projection with unweighted RMSNorm Converter: - Safetensors to GGUF with vision tensor renaming - Fused MoE gate_up_proj splitting - Vision patch embedding reshape (HF to Conv2D layout) - Packed clamp data tensor for ClippableLinear bounds - Proportional RoPE freq_factors generation Also includes: - BackendGet() on ml.Tensor for reading weight tensor data - Q6_K CUDA get_rows kernel support - MoE-aware ffn_down quantization layer counting - Gemma4 parser with tool calling and thinking support - Gemma4 renderer with structured tool format - Architecture-based auto-detection of renderer/parser/stop tokens - Integration test gemma4 model list additions * gemma4: add audio support with USM conformer encoder Add audio encoding for Gemma 4 using the USM conformer architecture: - Converter: audio tensor mapping, SSCP/conformer/embedder name replacements, softplus repacker for per_dim_scale, F32 enforcement for conv weights - GGML backend: Conv1DDW and PadExt tensor ops - Audio encoder: SSCP Conv2D, 12 conformer blocks (FFW + block-local attention with relative position embeddings + LightConv1d + FFW), output projection, audio-to-text embedding projector - Audio preprocessing: WAV decode, mel spectrogram, FFT (pure Go) - Model wiring: WAV detection, audio token handling, unified PostTokenize Correctly transcribes "why is the sky blue" from test audio. * integration: add gemma4 audio tests including OpenAI API coverage Test audio transcription and response via the Ollama native API, plus two new tests exercising the OpenAI-compatible endpoints: - /v1/audio/transcriptions (multipart form upload) - /v1/chat/completions with input_audio content type All tests use capability checks and skip models without audio support. * gemma4: add OpenAI audio API support and capability detection - Add CapabilityAudio and detect from audio.block_count in GGUF - Add /v1/audio/transcriptions endpoint with TranscriptionMiddleware - Add input_audio content type support in /v1/chat/completions - Add TranscriptionRequest/Response types in openai package * gemma4: add audio input support for run command - /audio toggle in interactive mode for voice chat - Platform-specific microphone recording (AVFoundation on macOS, PulseAudio/ALSA on Linux, WASAPI on Windows) - Space to start/stop recording, automatic chunking for long audio * gemma4: add transcribe command (ollama transcribe MODEL) - Interactive mode with readline prompt and slash commands - Non-interactive mode for piped audio or record-until-Ctrl+C - Chunked streaming transcription for long recordings - Word-wrapped output matching run command style * gemma4: add parser, renderer, and integration test plumbing * gemma4: fix renderer to emit BOS token * gemma4: add OpenAI audio transcription API and input_audio support * gemma4: update converter for new weight drop naming * gemma4: add per_expert_scale to MoE router and fix moe_intermediate_size config * gemma4: rewrite renderer to match HF Jinja2 template exactly Fix 8 bugs found by building 55 reference tests verified against the HF Jinja2 chat template (VERIFY_JINJA2=1 shells out to Python): - Tool responses use separate <|turn>tool turns (not inline tags) - Tool calls emitted before content in assistant messages - Thinking content stripped from assistant history (strip_thinking) - User, tool, and system content trimmed (template does | trim) - Empty system message still emits system turn (check role, not content) - Nested object properties rendered recursively with required field - Array items specification rendered for array-type properties - OBJECT/ARRAY type-specific rendering comma logic matches template Also adds Required field to api.ToolProperty for nested object schemas, replaces old gemma4_test.go with comprehensive gemma4_reference_test.go, and commits the Jinja2 template as testdata for verification. * gemma4: fix MoE fused gate_up split and multiline tool-call arg parsing - Text MoE: split `ffn_gate_up_exps` into contiguous `[gate|up]` halves instead of stride-2 slices. - Parser: escape control characters in `<|"|>...<|"|>` string literals when converting tool-call args to JSON. - Fixes warnings like `invalid character '\n' in string literal` for multiline tool arguments. - Add Gemma4 parser regressions for multiline tool-call args and `gemma4ArgsToJSON`. * cmd: simplify audio input to dropped file attachments * gemma4: use full SWA memory for better cache reuse * gemma4: initialize clamps after backend load * convert: align gemma4 audio tensor renames with llama.cpp * Remove redundant comments in gemma4 vision model * Format Gemma4 MoE block field alignment * use 4096 kvcache.NewSWAMemCache * convert: support new Gemma4 audio_tower tensor naming (#15221) Co-authored-by: jmorganca <jmorganca@gmail.com> * fix integration test defaults for audio * review comments and lint fixes * remove unused audio/video files --------- Co-authored-by: jmorganca <jmorganca@gmail.com>
438 lines
9.1 KiB
Go
438 lines
9.1 KiB
Go
package readline
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
)
|
|
|
|
type Prompt struct {
|
|
Prompt string
|
|
AltPrompt string
|
|
Placeholder string
|
|
AltPlaceholder string
|
|
UseAlt bool
|
|
}
|
|
|
|
func (p *Prompt) prompt() string {
|
|
if p.UseAlt {
|
|
return p.AltPrompt
|
|
}
|
|
return p.Prompt
|
|
}
|
|
|
|
func (p *Prompt) placeholder() string {
|
|
if p.UseAlt {
|
|
return p.AltPlaceholder
|
|
}
|
|
return p.Placeholder
|
|
}
|
|
|
|
type Terminal struct {
|
|
reader *bufio.Reader
|
|
rawmode bool
|
|
termios any
|
|
}
|
|
|
|
type Instance struct {
|
|
Prompt *Prompt
|
|
Terminal *Terminal
|
|
History *History
|
|
Pasting bool
|
|
Prefill string
|
|
pastedLines []string
|
|
}
|
|
|
|
func New(prompt Prompt) (*Instance, error) {
|
|
term, err := NewTerminal()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
history, err := NewHistory()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Instance{
|
|
Prompt: &prompt,
|
|
Terminal: term,
|
|
History: history,
|
|
}, nil
|
|
}
|
|
|
|
func (i *Instance) Readline() (string, error) {
|
|
if !i.Terminal.rawmode {
|
|
fd := os.Stdin.Fd()
|
|
termios, err := SetRawMode(fd)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
i.Terminal.rawmode = true
|
|
i.Terminal.termios = termios
|
|
}
|
|
|
|
prompt := i.Prompt.prompt()
|
|
if i.Pasting {
|
|
// force alt prompt when pasting
|
|
prompt = i.Prompt.AltPrompt
|
|
}
|
|
fmt.Print(prompt)
|
|
|
|
defer func() {
|
|
fd := os.Stdin.Fd()
|
|
//nolint:errcheck
|
|
UnsetRawMode(fd, i.Terminal.termios)
|
|
i.Terminal.rawmode = false
|
|
}()
|
|
|
|
buf, _ := NewBuffer(i.Prompt)
|
|
|
|
// Prefill the buffer with any text that we received from an external editor
|
|
if i.Prefill != "" {
|
|
lines := strings.Split(i.Prefill, "\n")
|
|
i.Prefill = ""
|
|
for idx, l := range lines {
|
|
for _, r := range l {
|
|
buf.Add(r)
|
|
}
|
|
if idx < len(lines)-1 {
|
|
i.pastedLines = append(i.pastedLines, buf.String())
|
|
buf.Buf.Clear()
|
|
buf.Pos = 0
|
|
buf.DisplayPos = 0
|
|
buf.LineHasSpace.Clear()
|
|
fmt.Println()
|
|
fmt.Print(i.Prompt.AltPrompt)
|
|
i.Prompt.UseAlt = true
|
|
}
|
|
}
|
|
}
|
|
|
|
var esc bool
|
|
var escex bool
|
|
var metaDel bool
|
|
|
|
var currentLineBuf []rune
|
|
|
|
// draining tracks if we're processing buffered input from cooked mode.
|
|
// In cooked mode Enter sends \n, but in raw mode Ctrl+J sends \n.
|
|
// We treat \n from cooked mode as submit, not multiline.
|
|
// We check Buffered() after the first read since the bufio buffer is
|
|
// empty until then. This is compatible with """ multiline mode in
|
|
// interactive.go since each Readline() call is independent.
|
|
var draining, stopDraining bool
|
|
|
|
for {
|
|
// Apply deferred state change from previous iteration
|
|
if stopDraining {
|
|
draining = false
|
|
stopDraining = false
|
|
}
|
|
|
|
// don't show placeholder when pasting unless we're in multiline mode
|
|
showPlaceholder := !i.Pasting || i.Prompt.UseAlt
|
|
if buf.IsEmpty() && showPlaceholder {
|
|
ph := i.Prompt.placeholder()
|
|
fmt.Print(ColorGrey + ph + CursorLeftN(len(ph)) + ColorDefault)
|
|
}
|
|
|
|
r, err := i.Terminal.Read()
|
|
|
|
// After reading, check if there's more buffered data. If so, we're
|
|
// processing cooked-mode input. Once buffer empties, the current
|
|
// char is the last buffered one (still drain it), then stop next iteration.
|
|
if i.Terminal.reader.Buffered() > 0 {
|
|
draining = true
|
|
} else if draining {
|
|
stopDraining = true
|
|
}
|
|
|
|
if buf.IsEmpty() {
|
|
fmt.Print(ClearToEOL)
|
|
}
|
|
|
|
if err != nil {
|
|
return "", io.EOF
|
|
}
|
|
|
|
if escex {
|
|
escex = false
|
|
|
|
switch r {
|
|
case KeyUp:
|
|
i.historyPrev(buf, ¤tLineBuf)
|
|
case KeyDown:
|
|
i.historyNext(buf, ¤tLineBuf)
|
|
case KeyLeft:
|
|
buf.MoveLeft()
|
|
case KeyRight:
|
|
buf.MoveRight()
|
|
case CharBracketedPaste:
|
|
var code string
|
|
for range 3 {
|
|
r, err = i.Terminal.Read()
|
|
if err != nil {
|
|
return "", io.EOF
|
|
}
|
|
|
|
code += string(r)
|
|
}
|
|
if code == CharBracketedPasteStart {
|
|
i.Pasting = true
|
|
} else if code == CharBracketedPasteEnd {
|
|
i.Pasting = false
|
|
}
|
|
case KeyDel:
|
|
if buf.DisplaySize() > 0 {
|
|
buf.Delete()
|
|
}
|
|
metaDel = true
|
|
case MetaStart:
|
|
buf.MoveToStart()
|
|
case MetaEnd:
|
|
buf.MoveToEnd()
|
|
default:
|
|
// skip any keys we don't know about
|
|
continue
|
|
}
|
|
continue
|
|
} else if esc {
|
|
esc = false
|
|
|
|
switch r {
|
|
case 'b':
|
|
buf.MoveLeftWord()
|
|
case 'f':
|
|
buf.MoveRightWord()
|
|
case CharBackspace:
|
|
buf.DeleteWord()
|
|
case CharEscapeEx:
|
|
escex = true
|
|
}
|
|
continue
|
|
}
|
|
|
|
switch r {
|
|
case CharNull:
|
|
continue
|
|
case CharEsc:
|
|
esc = true
|
|
case CharInterrupt:
|
|
i.pastedLines = nil
|
|
i.Prompt.UseAlt = false
|
|
return "", ErrInterrupt
|
|
case CharPrev:
|
|
i.historyPrev(buf, ¤tLineBuf)
|
|
case CharNext:
|
|
i.historyNext(buf, ¤tLineBuf)
|
|
case CharLineStart:
|
|
buf.MoveToStart()
|
|
case CharLineEnd:
|
|
buf.MoveToEnd()
|
|
case CharBackward:
|
|
buf.MoveLeft()
|
|
case CharForward:
|
|
buf.MoveRight()
|
|
case CharBackspace, CharCtrlH:
|
|
if buf.IsEmpty() && len(i.pastedLines) > 0 {
|
|
lastIdx := len(i.pastedLines) - 1
|
|
prevLine := i.pastedLines[lastIdx]
|
|
i.pastedLines = i.pastedLines[:lastIdx]
|
|
fmt.Print(CursorBOL + ClearToEOL + CursorUp + CursorBOL + ClearToEOL)
|
|
if len(i.pastedLines) == 0 {
|
|
fmt.Print(i.Prompt.Prompt)
|
|
i.Prompt.UseAlt = false
|
|
} else {
|
|
fmt.Print(i.Prompt.AltPrompt)
|
|
}
|
|
for _, r := range prevLine {
|
|
buf.Add(r)
|
|
}
|
|
} else {
|
|
buf.Remove()
|
|
}
|
|
case CharTab:
|
|
// todo: convert back to real tabs
|
|
for range 8 {
|
|
buf.Add(' ')
|
|
}
|
|
case CharDelete:
|
|
if buf.DisplaySize() > 0 {
|
|
buf.Delete()
|
|
} else {
|
|
return "", io.EOF
|
|
}
|
|
case CharKill:
|
|
buf.DeleteRemaining()
|
|
case CharCtrlU:
|
|
buf.DeleteBefore()
|
|
case CharCtrlL:
|
|
buf.ClearScreen()
|
|
case CharCtrlW:
|
|
buf.DeleteWord()
|
|
case CharBell:
|
|
output := buf.String()
|
|
numPastedLines := len(i.pastedLines)
|
|
if numPastedLines > 0 {
|
|
output = strings.Join(i.pastedLines, "\n") + "\n" + output
|
|
i.pastedLines = nil
|
|
}
|
|
|
|
// Move cursor to the last display line of the current buffer
|
|
currLine := buf.DisplayPos / buf.LineWidth
|
|
lastLine := buf.DisplaySize() / buf.LineWidth
|
|
if lastLine > currLine {
|
|
fmt.Print(CursorDownN(lastLine - currLine))
|
|
}
|
|
|
|
// Clear all lines from bottom to top: buffer wrapped lines + pasted lines
|
|
for range lastLine + numPastedLines {
|
|
fmt.Print(CursorBOL + ClearToEOL + CursorUp)
|
|
}
|
|
fmt.Print(CursorBOL + ClearToEOL)
|
|
|
|
i.Prompt.UseAlt = false
|
|
return output, ErrEditPrompt
|
|
case CharCtrlZ:
|
|
fd := os.Stdin.Fd()
|
|
return handleCharCtrlZ(fd, i.Terminal.termios)
|
|
case CharCtrlJ:
|
|
// If not draining cooked-mode input, treat as multiline
|
|
if !draining {
|
|
i.pastedLines = append(i.pastedLines, buf.String())
|
|
buf.Buf.Clear()
|
|
buf.Pos = 0
|
|
buf.DisplayPos = 0
|
|
buf.LineHasSpace.Clear()
|
|
fmt.Println()
|
|
fmt.Print(i.Prompt.AltPrompt)
|
|
i.Prompt.UseAlt = true
|
|
continue
|
|
}
|
|
// Draining cooked-mode input: treat \n as submit
|
|
fallthrough
|
|
case CharEnter:
|
|
output := buf.String()
|
|
if len(i.pastedLines) > 0 {
|
|
output = strings.Join(i.pastedLines, "\n") + "\n" + output
|
|
i.pastedLines = nil
|
|
}
|
|
if output != "" {
|
|
i.History.Add(output)
|
|
}
|
|
buf.MoveToEnd()
|
|
fmt.Println()
|
|
i.Prompt.UseAlt = false
|
|
|
|
return output, nil
|
|
default:
|
|
if metaDel {
|
|
metaDel = false
|
|
continue
|
|
}
|
|
if r >= CharSpace || r == CharEnter || r == CharCtrlJ {
|
|
buf.Add(r)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (i *Instance) HistoryEnable() {
|
|
i.History.Enabled = true
|
|
}
|
|
|
|
func (i *Instance) HistoryDisable() {
|
|
i.History.Enabled = false
|
|
}
|
|
|
|
func (i *Instance) historyPrev(buf *Buffer, currentLineBuf *[]rune) {
|
|
if i.History.Pos > 0 {
|
|
if i.History.Pos == i.History.Size() {
|
|
*currentLineBuf = []rune(buf.String())
|
|
}
|
|
buf.Replace([]rune(i.History.Prev()))
|
|
}
|
|
}
|
|
|
|
func (i *Instance) historyNext(buf *Buffer, currentLineBuf *[]rune) {
|
|
if i.History.Pos < i.History.Size() {
|
|
buf.Replace([]rune(i.History.Next()))
|
|
if i.History.Pos == i.History.Size() {
|
|
buf.Replace(*currentLineBuf)
|
|
}
|
|
}
|
|
}
|
|
|
|
func NewTerminal() (*Terminal, error) {
|
|
fd := os.Stdin.Fd()
|
|
termios, err := SetRawMode(fd)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := UnsetRawMode(fd, termios); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
t := &Terminal{
|
|
reader: bufio.NewReader(os.Stdin),
|
|
}
|
|
|
|
return t, nil
|
|
}
|
|
|
|
func (t *Terminal) Read() (rune, error) {
|
|
r, _, err := t.reader.ReadRune()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// SetRawModeOn enables raw terminal mode and keeps it on.
|
|
// Call SetRawModeOff to restore when done.
|
|
func (i *Instance) SetRawModeOn() error {
|
|
if i.Terminal.rawmode {
|
|
return nil
|
|
}
|
|
fd := os.Stdin.Fd()
|
|
termios, err := SetRawMode(fd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
i.Terminal.rawmode = true
|
|
i.Terminal.termios = termios
|
|
return nil
|
|
}
|
|
|
|
// SetRawModeOff restores the terminal to its previous mode.
|
|
func (i *Instance) SetRawModeOff() {
|
|
if !i.Terminal.rawmode {
|
|
return
|
|
}
|
|
fd := os.Stdin.Fd()
|
|
//nolint:errcheck
|
|
UnsetRawMode(fd, i.Terminal.termios)
|
|
i.Terminal.rawmode = false
|
|
}
|
|
|
|
// ReadRaw reads a single rune. If the terminal is already in raw mode
|
|
// (via SetRawModeOn), it reads directly. Otherwise it temporarily enters
|
|
// raw mode for the read.
|
|
func (i *Instance) ReadRaw() (rune, error) {
|
|
if !i.Terminal.rawmode {
|
|
fd := os.Stdin.Fd()
|
|
termios, err := SetRawMode(fd)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer func() {
|
|
//nolint:errcheck
|
|
UnsetRawMode(fd, termios)
|
|
}()
|
|
}
|
|
return i.Terminal.Read()
|
|
}
|