Compare commits

...

2 Commits

Author SHA1 Message Date
Patrick Devine
7bcdb250b9 fix failing client2 unit tests 2026-04-21 13:56:39 -07:00
Patrick Devine
7bbcd2e6be server: add v2 manifest path
This change adds a new manifest-v2/ path for new models created with the
create/pull/copy commands. Under manifest-v2, manifests are now just blobs which are
content addressable similar to tensors/config files. The named tags instead
will symlink/hard link/contain a copy depending on what the file system supports.

Downgrades to older versions of ollama are still possible, but any create/pull/copy
done with the newer version will potentially have its blobs pruned by the older
version.

manifest-v2 also changes the default registry name to `ollama.com` instead of
`registry.ollama.ai`.
2026-04-21 12:05:54 -07:00
11 changed files with 1040 additions and 204 deletions

View File

@@ -1,18 +1,23 @@
package manifest package manifest
import ( import (
"bytes"
"crypto/sha256" "crypto/sha256"
"encoding/hex"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"strings"
"github.com/ollama/ollama/types/model" "github.com/ollama/ollama/types/model"
) )
var blobFilenamePattern = regexp.MustCompile(`^sha256-[0-9a-fA-F]{64}$`)
type Manifest struct { type Manifest struct {
SchemaVersion int `json:"schemaVersion"` SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"` MediaType string `json:"mediaType"`
@@ -22,6 +27,7 @@ type Manifest struct {
filepath string filepath string
fi os.FileInfo fi os.FileInfo
digest string digest string
name model.Name
} }
func (m *Manifest) Size() (size int64) { func (m *Manifest) Size() (size int64) {
@@ -36,6 +42,14 @@ func (m *Manifest) Digest() string {
return m.digest return m.digest
} }
func (m *Manifest) BlobDigest() string {
if m.digest == "" {
return ""
}
return "sha256:" + m.digest
}
func (m *Manifest) FileInfo() os.FileInfo { func (m *Manifest) FileInfo() os.FileInfo {
return m.fi return m.fi
} }
@@ -59,16 +73,7 @@ func (m *Manifest) ReadConfigJSON(configPath string, v any) error {
} }
func (m *Manifest) Remove() error { func (m *Manifest) Remove() error {
if err := os.Remove(m.filepath); err != nil { return removeNamedManifestPaths(m.name)
return err
}
manifests, err := Path()
if err != nil {
return err
}
return PruneDirectory(manifests)
} }
func (m *Manifest) RemoveLayers() error { func (m *Manifest) RemoveLayers() error {
@@ -80,6 +85,9 @@ func (m *Manifest) RemoveLayers() error {
// Build set of digests still in use by other manifests // Build set of digests still in use by other manifests
inUse := make(map[string]struct{}) inUse := make(map[string]struct{})
for _, other := range ms { for _, other := range ms {
if other.BlobDigest() != "" {
inUse[other.BlobDigest()] = struct{}{}
}
for _, layer := range append(other.Layers, other.Config) { for _, layer := range append(other.Layers, other.Config) {
if layer.Digest != "" { if layer.Digest != "" {
inUse[layer.Digest] = struct{}{} inUse[layer.Digest] = struct{}{}
@@ -87,20 +95,27 @@ func (m *Manifest) RemoveLayers() error {
} }
} }
// Remove layers not used by any other manifest digests := make([]string, 0, len(m.Layers)+2)
for _, layer := range append(m.Layers, m.Config) { digests = append(digests, m.BlobDigest())
if layer.Digest == "" { for _, layer := range m.Layers {
digests = append(digests, layer.Digest)
}
digests = append(digests, m.Config.Digest)
// Remove manifest and layer blobs not used by any other manifest
for _, digest := range digests {
if digest == "" {
continue continue
} }
if _, used := inUse[layer.Digest]; used { if _, used := inUse[digest]; used {
continue continue
} }
blob, err := BlobsPath(layer.Digest) blob, err := BlobsPath(digest)
if err != nil { if err != nil {
return err return err
} }
if err := os.Remove(blob); os.IsNotExist(err) { if err := os.Remove(blob); os.IsNotExist(err) {
slog.Debug("layer does not exist", "digest", layer.Digest) slog.Debug("blob does not exist", "digest", digest)
} else if err != nil { } else if err != nil {
return err return err
} }
@@ -114,15 +129,36 @@ func ParseNamedManifest(n model.Name) (*Manifest, error) {
return nil, model.Unqualified(n) return nil, model.Unqualified(n)
} }
manifests, err := Path() p, root, err := resolveManifestPath(n)
if err != nil { if err != nil {
return nil, err return nil, err
} }
p := filepath.Join(manifests, n.Filepath()) return parseManifestFile(normalizeLogicalName(n), p, root)
}
func ReadManifestData(n model.Name) ([]byte, error) {
if !n.IsFullyQualified() {
return nil, model.Unqualified(n)
}
p, root, err := resolveManifestPath(n)
if err != nil {
return nil, err
}
f, _, err := OpenVerifiedManifest(p, root)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
func parseManifestFile(name model.Name, path, root string) (*Manifest, error) {
var m Manifest var m Manifest
f, err := os.Open(p) f, digest, err := OpenVerifiedManifest(path, root)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -133,35 +169,19 @@ func ParseNamedManifest(n model.Name) (*Manifest, error) {
return nil, err return nil, err
} }
sha256sum := sha256.New() if err := json.NewDecoder(f).Decode(&m); err != nil {
if err := json.NewDecoder(io.TeeReader(f, sha256sum)).Decode(&m); err != nil {
return nil, err return nil, err
} }
m.filepath = p m.filepath = path
m.fi = fi m.fi = fi
m.digest = hex.EncodeToString(sha256sum.Sum(nil)) m.digest = digest
m.name = name
return &m, nil return &m, nil
} }
func WriteManifest(name model.Name, config Layer, layers []Layer) error { func WriteManifest(name model.Name, config Layer, layers []Layer) error {
manifests, err := Path()
if err != nil {
return err
}
p := filepath.Join(manifests, name.Filepath())
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
return err
}
f, err := os.Create(p)
if err != nil {
return err
}
defer f.Close()
m := Manifest{ m := Manifest{
SchemaVersion: 2, SchemaVersion: 2,
MediaType: "application/vnd.docker.distribution.manifest.v2+json", MediaType: "application/vnd.docker.distribution.manifest.v2+json",
@@ -169,33 +189,371 @@ func WriteManifest(name model.Name, config Layer, layers []Layer) error {
Layers: layers, Layers: layers,
} }
return json.NewEncoder(f).Encode(m) var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(m); err != nil {
return err
}
return WriteManifestData(name, b.Bytes())
} }
func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) { // WriteManifestData stores raw manifest bytes as a content-addressed blob and
// updates the v2 named manifest path to reference that blob. Any legacy named
// manifest for the same model is removed after the v2 write succeeds.
func WriteManifestData(name model.Name, data []byte) error {
if !name.IsFullyQualified() {
return model.Unqualified(name)
}
digest, err := writeManifestBlob(data)
if err != nil {
return err
}
if err := LinkManifest(name, digest); err != nil {
return err
}
return removeLegacyManifestPaths(name)
}
// LinkManifest updates the v2 named manifest path to reference an existing
// manifest blob. It prefers symlinks, then hardlinks, then a byte-for-byte copy
// for filesystems that do not support links.
func LinkManifest(name model.Name, digest string) error {
if !name.IsFullyQualified() {
return model.Unqualified(name)
}
manifestPath, err := V2PathForName(name)
if err != nil {
return err
}
blobPath, err := BlobsPath(digest)
if err != nil {
return err
}
if _, err := os.Stat(blobPath); err != nil {
return err
}
if err := checkBlobDigest(blobPath, digest); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
return err
}
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
return err
}
if rel, err := filepath.Rel(filepath.Dir(manifestPath), blobPath); err == nil {
if err := os.Symlink(rel, manifestPath); err == nil {
return nil
}
}
if err := os.Link(blobPath, manifestPath); err == nil {
return nil
}
return copyManifestFile(blobPath, manifestPath)
}
func writeManifestBlob(data []byte) (string, error) {
sum := sha256.Sum256(data)
digest := fmt.Sprintf("sha256:%x", sum)
blobPath, err := BlobsPath(digest)
if err != nil {
return "", err
}
if existing, err := os.ReadFile(blobPath); err == nil && bytes.Equal(existing, data) {
return digest, nil
}
blobs, err := BlobsPath("")
if err != nil {
return "", err
}
temp, err := os.CreateTemp(blobs, "sha256-")
if err != nil {
return "", err
}
tempName := temp.Name()
defer os.Remove(tempName)
if _, err := temp.Write(data); err != nil {
temp.Close()
return "", err
}
if err := temp.Close(); err != nil {
return "", err
}
if err := os.Chmod(tempName, 0o644); err != nil {
return "", err
}
if err := os.Rename(tempName, blobPath); err != nil {
if err := os.Remove(blobPath); err != nil && !os.IsNotExist(err) {
return "", err
}
if err := os.Rename(tempName, blobPath); err != nil {
return "", err
}
}
return digest, nil
}
func copyManifestFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
temp, err := os.CreateTemp(filepath.Dir(dst), ".manifest-*")
if err != nil {
return err
}
tempName := temp.Name()
defer os.Remove(tempName)
if _, err := io.Copy(temp, in); err != nil {
temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if err := os.Chmod(tempName, 0o644); err != nil {
return err
}
return os.Rename(tempName, dst)
}
// OpenVerifiedManifest opens a named manifest path rooted under root. Symlinks must resolve to a
// blob whose basename is sha256-<hex> and whose bytes hash to that digest.
// Regular-file manifests are treated as legacy/copy fallback manifests and are
// opened without mutating the local store.
func OpenVerifiedManifest(path, root string) (*os.File, string, error) {
resolvedRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return nil, "", err
}
info, err := os.Lstat(path)
if err != nil {
return nil, "", err
}
target, err := evalAbs(path)
if err != nil {
return nil, "", err
}
if info.Mode()&os.ModeSymlink != 0 {
base := filepath.Base(target)
if !blobFilenamePattern.MatchString(base) {
return nil, "", fmt.Errorf("manifest symlink target %q is not a sha256 blob", target)
}
digest := strings.ToLower(strings.TrimPrefix(base, "sha256-"))
blobPath, err := BlobsPath("sha256:" + digest)
if err != nil {
return nil, "", err
}
if !sameFile(target, blobPath) {
return nil, "", fmt.Errorf("manifest symlink target %q does not match blob %q", target, blobPath)
}
f, err := os.Open(path)
if err != nil {
return nil, "", err
}
if err := checkBlobDigestReader(f, "sha256:"+digest); err != nil {
f.Close()
return nil, "", err
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
f.Close()
return nil, "", err
}
return f, digest, nil
}
if !pathWithin(target, resolvedRoot) {
return nil, "", fmt.Errorf("manifest path %q resolves outside manifest directory", path)
}
f, err := os.Open(path)
if err != nil {
return nil, "", err
}
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
f.Close()
return nil, "", err
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
f.Close()
return nil, "", err
}
digest := fmt.Sprintf("%x", h.Sum(nil))
return f, digest, nil
}
// MigrateManifestLinks moves legacy named manifests into manifests-v2. This is currently unwired but
// will be added in the future.
func MigrateManifestLinks() (int, error) {
manifests, err := Path() manifests, err := Path()
if err != nil { if err != nil {
return nil, err return 0, err
} }
// TODO(mxyng): use something less brittle // TODO(mxyng): use something less brittle
matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*")) matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*"))
if err != nil { if err != nil {
return nil, err return 0, err
} }
ms := make(map[model.Name]*Manifest) var migrated int
for _, match := range matches { for _, match := range matches {
fi, err := os.Stat(match) fi, err := os.Stat(match)
if err != nil { if err != nil {
return nil, err return migrated, err
}
if fi.IsDir() {
continue
}
rel, err := filepath.Rel(manifests, match)
if err != nil {
return migrated, fmt.Errorf("%s %w", match, err)
}
n := model.ParseNameFromFilepath(rel)
if !n.IsFullyQualified() {
slog.Warn("bad manifest name", "path", rel)
continue
}
data, err := readManifestPath(match, manifests)
if err != nil {
return migrated, err
}
if err := WriteManifestData(normalizeLogicalName(n), data); err != nil {
return migrated, err
}
migrated++
}
return migrated, nil
}
func readManifestPath(path, root string) ([]byte, error) {
f, _, err := OpenVerifiedManifest(path, root)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
func pathWithin(path, root string) bool {
rel, err := filepath.Rel(root, path)
return err == nil && rel != "." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != ".."
}
func evalAbs(path string) (string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return "", err
}
return filepath.EvalSymlinks(abs)
}
func sameFile(a, b string) bool {
ai, err := os.Stat(a)
if err != nil {
return false
}
bi, err := os.Stat(b)
if err != nil {
return false
}
return os.SameFile(ai, bi)
}
func checkBlobDigest(path, digest string) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return checkBlobDigestReader(f, digest)
}
func checkBlobDigestReader(r io.Reader, digest string) error {
h := sha256.New()
if _, err := io.Copy(h, r); err != nil {
return err
}
got := fmt.Sprintf("sha256:%x", h.Sum(nil))
if got != strings.ToLower(strings.Replace(digest, "-", ":", 1)) {
return errors.New("digest mismatch")
}
return nil
}
func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
ms := make(map[model.Name]*Manifest)
manifestsV2, err := V2Path()
if err != nil {
return nil, err
}
if err := collectManifests(ms, manifestsV2, continueOnError); err != nil {
return nil, err
}
manifests, err := Path()
if err != nil {
return nil, err
}
if err := collectManifests(ms, manifests, continueOnError); err != nil {
return nil, err
}
return ms, nil
}
func collectManifests(ms map[model.Name]*Manifest, root string, continueOnError bool) error {
// TODO(mxyng): use something less brittle
matches, err := filepath.Glob(filepath.Join(root, "*", "*", "*", "*"))
if err != nil {
return err
}
for _, match := range matches {
fi, err := os.Lstat(match)
if err != nil {
return err
} }
if !fi.IsDir() { if !fi.IsDir() {
rel, err := filepath.Rel(manifests, match) rel, err := filepath.Rel(root, match)
if err != nil { if err != nil {
if !continueOnError { if !continueOnError {
return nil, fmt.Errorf("%s %w", match, err) return fmt.Errorf("%s %w", match, err)
} }
slog.Warn("bad filepath", "path", match, "error", err) slog.Warn("bad filepath", "path", match, "error", err)
continue continue
@@ -204,16 +562,21 @@ func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
n := model.ParseNameFromFilepath(rel) n := model.ParseNameFromFilepath(rel)
if !n.IsValid() { if !n.IsValid() {
if !continueOnError { if !continueOnError {
return nil, fmt.Errorf("%s %w", rel, err) return fmt.Errorf("invalid manifest name: %s", rel)
} }
slog.Warn("bad manifest name", "path", rel) slog.Warn("bad manifest name", "path", rel)
continue continue
} }
m, err := ParseNamedManifest(n) n = normalizeLogicalName(n)
if _, ok := ms[n]; ok {
continue
}
m, err := parseManifestFile(n, match, root)
if err != nil { if err != nil {
if !continueOnError { if !continueOnError {
return nil, fmt.Errorf("%s %w", n, err) return fmt.Errorf("%s %w", n, err)
} }
slog.Warn("bad manifest", "name", n, "error", err) slog.Warn("bad manifest", "name", n, "error", err)
continue continue
@@ -223,5 +586,5 @@ func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
} }
} }
return ms, nil return nil
} }

View File

@@ -1,19 +1,23 @@
package manifest package manifest
import ( import (
"bytes"
"crypto/sha256"
"encoding/json" "encoding/json"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"strings"
"testing" "testing"
"github.com/ollama/ollama/types/model" "github.com/ollama/ollama/types/model"
) )
func createManifest(t *testing.T, path, name string) { func createManifestAtRoot(t *testing.T, path, root, name string) {
t.Helper() t.Helper()
p := filepath.Join(path, "manifests", name) p := filepath.Join(path, root, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -29,6 +33,309 @@ func createManifest(t *testing.T, path, name string) {
} }
} }
func createManifest(t *testing.T, path, name string) {
t.Helper()
createManifestAtRoot(t, path, "manifests", name)
}
func TestWriteManifestStoresManifestAsBlob(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
name := model.ParseName("example")
config := Layer{
MediaType: "application/vnd.docker.container.image.v1+json",
Digest: "sha256:" + strings.Repeat("a", 64),
Size: 12,
}
if err := WriteManifest(name, config, nil); err != nil {
t.Fatal(err)
}
manifestPath, err := V2PathForName(name)
if err != nil {
t.Fatal(err)
}
manifestData, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(manifestData)
digest := fmt.Sprintf("sha256:%x", sum)
blobPath, err := BlobsPath(digest)
if err != nil {
t.Fatal(err)
}
blobData, err := os.ReadFile(blobPath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(blobData, manifestData) {
t.Fatal("manifest path and blob content differ")
}
m, err := ParseNamedManifest(name)
if err != nil {
t.Fatal(err)
}
if got := m.Digest(); got != fmt.Sprintf("%x", sum) {
t.Fatalf("digest = %q, want %x", got, sum)
}
if got := m.BlobDigest(); got != digest {
t.Fatalf("blob digest = %q, want %q", got, digest)
}
}
func TestParseNamedManifestLeavesLegacyManifestInPlace(t *testing.T) {
models := t.TempDir()
t.Setenv("OLLAMA_MODELS", models)
name := model.ParseName("example")
createManifest(t, models, name.Filepath())
manifestPath, err := PathForName(name)
if err != nil {
t.Fatal(err)
}
if _, err := ParseNamedManifest(name); err != nil {
t.Fatal(err)
}
fi, err := os.Lstat(manifestPath)
if err != nil {
t.Fatal(err)
}
if fi.Mode()&os.ModeSymlink != 0 {
t.Fatal("legacy manifest was converted to a symlink while reading")
}
data, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(data)
blobPath, err := BlobsPath(fmt.Sprintf("sha256:%x", sum))
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
t.Fatalf("legacy manifest read created blob: %v", err)
}
}
func TestMigrateManifestLinks(t *testing.T) {
models := t.TempDir()
t.Setenv("OLLAMA_MODELS", models)
name := model.ParseName("example")
createManifest(t, models, name.Filepath())
migrated, err := MigrateManifestLinks()
if err != nil {
t.Fatal(err)
}
if migrated != 1 {
t.Fatalf("migrated = %d, want 1", migrated)
}
manifestPath, err := V2PathForName(name)
if err != nil {
t.Fatal(err)
}
manifestData, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(manifestData)
blobPath, err := BlobsPath(fmt.Sprintf("sha256:%x", sum))
if err != nil {
t.Fatal(err)
}
blobData, err := os.ReadFile(blobPath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(blobData, manifestData) {
t.Fatal("migrated manifest path and blob content differ")
}
legacyPath, err := PathForName(name)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
t.Fatalf("legacy manifest still exists: %v", err)
}
migrated, err = MigrateManifestLinks()
if err != nil {
t.Fatal(err)
}
if migrated != 0 {
t.Fatalf("migrated on second run = %d, want 0", migrated)
}
if _, err := MigrateManifestLinks(); err != nil {
t.Fatal(err)
}
manifestDataAfter, err := os.ReadFile(manifestPath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(manifestDataAfter, manifestData) {
t.Fatal("second migration changed manifest content")
}
}
func TestRemoveLayersRemovesUnreferencedManifestBlob(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
name := model.ParseName("example")
if err := WriteManifest(name, Layer{}, nil); err != nil {
t.Fatal(err)
}
m, err := ParseNamedManifest(name)
if err != nil {
t.Fatal(err)
}
blobPath, err := BlobsPath(m.BlobDigest())
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(blobPath); err != nil {
t.Fatal(err)
}
if err := m.Remove(); err != nil {
t.Fatal(err)
}
if err := m.RemoveLayers(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
t.Fatalf("manifest blob still exists: %v", err)
}
}
func TestParseNamedManifestRejectsUnsafeSymlinks(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
name := model.ParseName("example")
manifestPath, err := PathForName(name)
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
t.Fatal(err)
}
t.Run("non blob basename", func(t *testing.T) {
target := filepath.Join(t.TempDir(), "not-a-blob")
if err := os.WriteFile(target, []byte(`{"schemaVersion":2}`), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
if err := os.Symlink(target, manifestPath); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
_, err := ParseNamedManifest(name)
if err == nil || !strings.Contains(err.Error(), "not a sha256 blob") {
t.Fatalf("err = %v, want not a sha256 blob", err)
}
})
t.Run("blob basename outside blob store", func(t *testing.T) {
data := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`)
sum := sha256.Sum256(data)
target := filepath.Join(t.TempDir(), fmt.Sprintf("sha256-%x", sum))
if err := os.WriteFile(target, data, 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
if err := os.Symlink(target, manifestPath); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
_, err := ParseNamedManifest(name)
if err == nil || !strings.Contains(err.Error(), "does not match blob") {
t.Fatalf("err = %v, want does not match blob", err)
}
})
}
func TestParseNamedManifestPrefersV2(t *testing.T) {
models := t.TempDir()
t.Setenv("OLLAMA_MODELS", models)
name := model.ParseName("example")
legacyPath, err := PathForName(name)
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(legacyPath, []byte(`{"schemaVersion":2,"mediaType":"legacy"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := WriteManifestData(name, []byte(`{"schemaVersion":2,"mediaType":"v2"}`)); err != nil {
t.Fatal(err)
}
m, err := ParseNamedManifest(name)
if err != nil {
t.Fatal(err)
}
if m.MediaType != "v2" {
t.Fatalf("media type = %q, want %q", m.MediaType, "v2")
}
}
func TestManifestsV2ShadowsLegacy(t *testing.T) {
models := t.TempDir()
t.Setenv("OLLAMA_MODELS", models)
name := model.ParseName("example")
createManifest(t, models, name.Filepath())
if err := WriteManifestData(name, []byte(`{"schemaVersion":2,"mediaType":"v2"}`)); err != nil {
t.Fatal(err)
}
ms, err := Manifests(true)
if err != nil {
t.Fatal(err)
}
if len(ms) != 1 {
t.Fatalf("manifest count = %d, want 1", len(ms))
}
var m *Manifest
for gotName, gotManifest := range ms {
if gotName.EqualFold(model.ParseName("example")) {
m = gotManifest
break
}
}
if m == nil {
t.Fatalf("missing v2 manifest for %s", name)
}
if m.MediaType != "v2" {
t.Fatalf("media type = %q, want %q", m.MediaType, "v2")
}
}
func TestManifests(t *testing.T) { func TestManifests(t *testing.T) {
cases := map[string]struct { cases := map[string]struct {
ps []string ps []string

View File

@@ -14,8 +14,23 @@ import (
var ErrInvalidDigestFormat = errors.New("invalid digest format") var ErrInvalidDigestFormat = errors.New("invalid digest format")
const (
legacyDirName = "manifests"
v2DirName = "manifests-v2"
defaultPublicHost = "registry.ollama.ai"
v2CanonicalHost = "ollama.com"
)
func Path() (string, error) { func Path() (string, error) {
path := filepath.Join(envconfig.Models(), "manifests") return manifestPath(legacyDirName)
}
func V2Path() (string, error) {
return manifestPath(v2DirName)
}
func manifestPath(dir string) (string, error) {
path := filepath.Join(envconfig.Models(), dir)
if err := os.MkdirAll(path, 0o755); err != nil { if err := os.MkdirAll(path, 0o755); err != nil {
return "", fmt.Errorf("%w: ensure path elements are traversable", err) return "", fmt.Errorf("%w: ensure path elements are traversable", err)
} }
@@ -25,6 +40,10 @@ func Path() (string, error) {
// PathForName returns the path to the manifest file for a specific model name. // PathForName returns the path to the manifest file for a specific model name.
func PathForName(n model.Name) (string, error) { func PathForName(n model.Name) (string, error) {
return LegacyPathForName(n)
}
func LegacyPathForName(n model.Name) (string, error) {
if !n.IsValid() { if !n.IsValid() {
return "", os.ErrNotExist return "", os.ErrNotExist
} }
@@ -37,6 +56,162 @@ func PathForName(n model.Name) (string, error) {
return filepath.Join(manifests, n.Filepath()), nil return filepath.Join(manifests, n.Filepath()), nil
} }
func V2PathForName(n model.Name) (string, error) {
if !n.IsValid() {
return "", os.ErrNotExist
}
manifests, err := V2Path()
if err != nil {
return "", err
}
return filepath.Join(manifests, canonicalV2Name(n).Filepath()), nil
}
func ResolvePathForName(n model.Name) (string, error) {
path, _, err := resolveManifestPath(n)
return path, err
}
func resolveManifestPath(n model.Name) (string, string, error) {
if !n.IsValid() {
return "", "", os.ErrNotExist
}
v2Path, err := V2PathForName(n)
if err != nil {
return "", "", err
}
if _, err := os.Lstat(v2Path); err == nil {
root, err := V2Path()
return v2Path, root, err
} else if !os.IsNotExist(err) {
return "", "", err
}
legacyRoot, err := Path()
if err != nil {
return "", "", err
}
for _, legacyName := range legacyNameCandidates(n) {
legacyPath := filepath.Join(legacyRoot, legacyName.Filepath())
if _, err := os.Lstat(legacyPath); err == nil {
return legacyPath, legacyRoot, nil
} else if !os.IsNotExist(err) {
return "", "", err
}
}
return "", "", os.ErrNotExist
}
func removeNamedManifestPaths(n model.Name) error {
candidates := legacyNameCandidates(n)
paths := make([]string, 0, 1+len(candidates))
v2Path, err := V2PathForName(n)
if err != nil {
return err
}
paths = append(paths, v2Path)
for _, legacyName := range candidates {
legacyPath, err := LegacyPathForName(legacyName)
if err != nil {
return err
}
paths = append(paths, legacyPath)
}
for _, path := range paths {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}
return pruneManifestRoots()
}
func removeLegacyManifestPaths(n model.Name) error {
for _, legacyName := range legacyNameCandidates(n) {
legacyPath, err := LegacyPathForName(legacyName)
if err != nil {
return err
}
if err := os.Remove(legacyPath); err != nil && !os.IsNotExist(err) {
return err
}
}
legacyRoot, err := Path()
if err != nil {
return err
}
if err := PruneDirectory(legacyRoot); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
func pruneManifestRoots() error {
roots := []func() (string, error){Path, V2Path}
for _, rootFn := range roots {
root, err := rootFn()
if err != nil {
return err
}
if err := PruneDirectory(root); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
// normalizeLogicalName maps any public host to the legacy default
// so that map keys use a single identity regardless of on-disk host.
func normalizeLogicalName(n model.Name) model.Name {
if isDefaultPublicHost(n.Host) {
n.Host = defaultPublicHost
}
return n
}
// canonicalV2Name maps any public host to the v2 canonical host
// for use in manifests-v2/ on-disk paths.
func canonicalV2Name(n model.Name) model.Name {
if isDefaultPublicHost(n.Host) {
n.Host = v2CanonicalHost
}
return n
}
func legacyNameCandidates(n model.Name) []model.Name {
names := []model.Name{n}
if !isDefaultPublicHost(n.Host) {
return names
}
alt := n
switch {
case strings.EqualFold(n.Host, defaultPublicHost):
alt.Host = v2CanonicalHost
default:
alt.Host = defaultPublicHost
}
return append(names, alt)
}
func isDefaultPublicHost(host string) bool {
return strings.EqualFold(host, defaultPublicHost) || strings.EqualFold(host, v2CanonicalHost)
}
func BlobsPath(digest string) (string, error) { func BlobsPath(digest string) (string, error) {
// only accept actual sha256 digests // only accept actual sha256 digests
pattern := "^sha256[:-][0-9a-fA-F]{64}$" pattern := "^sha256[:-][0-9a-fA-F]{64}$"

View File

@@ -411,31 +411,12 @@ func CopyModel(src, dst model.Name) error {
return nil return nil
} }
manifests, err := manifest.Path() data, err := manifest.ReadManifestData(src)
if err != nil { if err != nil {
return err return err
} }
dstpath := filepath.Join(manifests, dst.Filepath()) return manifest.WriteManifestData(dst, data)
if err := os.MkdirAll(filepath.Dir(dstpath), 0o755); err != nil {
return err
}
srcpath := filepath.Join(manifests, src.Filepath())
srcfile, err := os.Open(srcpath)
if err != nil {
return err
}
defer srcfile.Close()
dstfile, err := os.Create(dstpath)
if err != nil {
return err
}
defer dstfile.Close()
_, err = io.Copy(dstfile, srcfile)
return err
} }
func deleteUnusedLayers(deleteMap map[string]struct{}) error { func deleteUnusedLayers(deleteMap map[string]struct{}) error {
@@ -446,6 +427,10 @@ func deleteUnusedLayers(deleteMap map[string]struct{}) error {
} }
for _, manifest := range manifests { for _, manifest := range manifests {
if manifest.BlobDigest() != "" {
delete(deleteMap, manifest.BlobDigest())
}
for _, layer := range manifest.Layers { for _, layer := range manifest.Layers {
delete(deleteMap, layer.Digest) delete(deleteMap, layer.Digest)
} }
@@ -549,11 +534,7 @@ func PushModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
// Use fast transfer for models with tensor layers (many small blobs) // Use fast transfer for models with tensor layers (many small blobs)
if hasTensorLayers(layers) { if hasTensorLayers(layers) {
// Read raw manifest JSON to preserve tensor metadata fields // Read raw manifest JSON to preserve tensor metadata fields
manifestPath, err := manifest.PathForName(n) manifestJSON, err := manifest.ReadManifestData(n)
if err != nil {
return err
}
manifestJSON, err := os.ReadFile(manifestPath)
if err != nil { if err != nil {
return err return err
} }
@@ -610,6 +591,14 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
if existingMf.Config.Digest != "" { if existingMf.Config.Digest != "" {
deleteMap[existingMf.Config.Digest] = struct{}{} deleteMap[existingMf.Config.Digest] = struct{}{}
} }
if existingMf.BlobDigest() != "" {
digest := existingMf.BlobDigest()
if blob, err := manifest.BlobsPath(digest); err == nil {
if _, err := os.Stat(blob); err == nil {
deleteMap[digest] = struct{}{}
}
}
}
} }
if n.ProtocolScheme == "http" && !regOpts.Insecure { if n.ProtocolScheme == "http" && !regOpts.Insecure {
@@ -679,21 +668,12 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
fn(api.ProgressResponse{Status: "writing manifest"}) fn(api.ProgressResponse{Status: "writing manifest"})
fp, err := manifest.PathForName(n) if err := manifest.WriteManifestData(n, manifestData); err != nil {
if err != nil { slog.Info(fmt.Sprintf("couldn't write manifest for %s", n.DisplayShortest()))
return err
}
if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
return err return err
} }
err = os.WriteFile(fp, manifestData, 0o644) slog.Debug("manifest written", "name", n.DisplayShortest(), "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
if err != nil {
slog.Info(fmt.Sprintf("couldn't write to %s", fp))
return err
}
slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
if !envconfig.NoPrune() && len(deleteMap) > 0 { if !envconfig.NoPrune() && len(deleteMap) > 0 {
fn(api.ProgressResponse{Status: "removing unused layers"}) fn(api.ProgressResponse{Status: "removing unused layers"})
@@ -776,19 +756,11 @@ func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer
// Write manifest // Write manifest
fn(api.ProgressResponse{Status: "writing manifest"}) fn(api.ProgressResponse{Status: "writing manifest"})
fp, err := manifest.PathForName(n) if err := manifest.WriteManifestData(n, manifestData); err != nil {
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
return err return err
} }
if err := os.WriteFile(fp, manifestData, 0o644); err != nil { slog.Debug("manifest written", "name", n.DisplayShortest(), "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
return err
}
slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
return nil return nil
} }

View File

@@ -116,6 +116,10 @@ func (s *Local) serveHTTP(rec *statusCodeRecorder, r *http.Request) {
proxied, err := func() (bool, error) { proxied, err := func() (bool, error) {
switch r.URL.Path { switch r.URL.Path {
case "/api/delete": case "/api/delete":
if s.Fallback != nil {
s.Fallback.ServeHTTP(rec, r)
return true, nil
}
return false, s.handleDelete(rec, r) return false, s.handleDelete(rec, r)
case "/api/pull": case "/api/pull":
return false, s.handlePull(rec, r) return false, s.handlePull(rec, r)

View File

@@ -1770,13 +1770,15 @@ func Serve(ln net.Listener) error {
return err return err
} }
manifestsPath, err := manifest.Path() for _, rootFn := range []func() (string, error){manifest.Path, manifest.V2Path} {
if err != nil { manifestsPath, err := rootFn()
return err if err != nil {
} return err
}
if err := manifest.PruneDirectory(manifestsPath); err != nil { if err := manifest.PruneDirectory(manifestsPath); err != nil && !os.IsNotExist(err) {
return err return err
}
} }
} }
} }

View File

@@ -109,12 +109,44 @@ func checkFileExists(t *testing.T, p string, expect []string) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if strings.HasSuffix(filepath.ToSlash(p), "/blobs/*") {
actual = slices.DeleteFunc(actual, isManifestBlobForTest)
}
if diff := gocmp.Diff(expect, actual, gocmpopts.SortSlices(strings.Compare), gocmpopts.EquateEmpty()); diff != "" { if diff := gocmp.Diff(expect, actual, gocmpopts.SortSlices(strings.Compare), gocmpopts.EquateEmpty()); diff != "" {
t.Errorf("file exists mismatch (-want +got):\n%s", diff) t.Errorf("file exists mismatch (-want +got):\n%s", diff)
} }
} }
func checkManifestFiles(t *testing.T, names ...string) {
t.Helper()
expect := make([]string, len(names))
for i, name := range names {
p, err := manifest.V2PathForName(model.ParseName(name))
if err != nil {
t.Fatal(err)
}
expect[i] = p
}
checkFileExists(t, filepath.Join(envconfig.Models(), "manifests-v2", "*", "*", "*", "*"), expect)
}
func isManifestBlobForTest(path string) bool {
data, err := os.ReadFile(path)
if err != nil {
return false
}
var m manifest.Manifest
if err := json.Unmarshal(data, &m); err != nil {
return false
}
return m.SchemaVersion != 0 && m.MediaType != "" && (m.Config.Digest != "" || len(m.Layers) > 0)
}
func TestCreateFromBin(t *testing.T) { func TestCreateFromBin(t *testing.T) {
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)
@@ -136,9 +168,7 @@ func TestCreateFromBin(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"), filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
@@ -196,9 +226,7 @@ func TestCreateFromModel(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
w = createRequest(t, s.CreateHandler, api.CreateRequest{ w = createRequest(t, s.CreateHandler, api.CreateRequest{
Name: "test2", Name: "test2",
@@ -210,10 +238,7 @@ func TestCreateFromModel(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test", "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"), filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
@@ -306,9 +331,7 @@ func TestCreateRemovesLayers(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-89a2116c3a82d6a97f59f748d86ed4417214353fd178ee54df418fde32495fad"), filepath.Join(p, "blobs", "sha256-89a2116c3a82d6a97f59f748d86ed4417214353fd178ee54df418fde32495fad"),
@@ -327,9 +350,7 @@ func TestCreateRemovesLayers(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"), filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
@@ -357,9 +378,7 @@ func TestCreateUnsetsSystem(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-0a666d113e8e0a3d27e9c7bd136a0bdfb6241037db50729d81568451ebfdbde8"), filepath.Join(p, "blobs", "sha256-0a666d113e8e0a3d27e9c7bd136a0bdfb6241037db50729d81568451ebfdbde8"),
@@ -378,9 +397,7 @@ func TestCreateUnsetsSystem(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"), filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
@@ -411,9 +428,7 @@ func TestCreateMergeParameters(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"), filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
@@ -436,10 +451,7 @@ func TestCreateMergeParameters(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test", "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
// Display contents of each blob in the directory // Display contents of each blob in the directory
blobDir := filepath.Join(p, "blobs") blobDir := filepath.Join(p, "blobs")
@@ -495,10 +507,7 @@ func TestCreateMergeParameters(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test", "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"), filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"),
@@ -555,9 +564,7 @@ func TestCreateReplacesMessages(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"), filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"),
@@ -589,10 +596,7 @@ func TestCreateReplacesMessages(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test", "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
// Old layers will not have been pruned // Old layers will not have been pruned
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
@@ -650,9 +654,7 @@ func TestCreateTemplateSystem(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-0a04d979734167da3b80811a1874d734697f366a689f3912589b99d2e86e7ad1"), filepath.Join(p, "blobs", "sha256-0a04d979734167da3b80811a1874d734697f366a689f3912589b99d2e86e7ad1"),
@@ -850,9 +852,7 @@ func TestCreateLicenses(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"), filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"),

View File

@@ -42,10 +42,7 @@ func TestDelete(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test", "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"), filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
@@ -60,9 +57,7 @@ func TestDelete(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "test2")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
})
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"), filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
@@ -76,7 +71,7 @@ func TestDelete(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{}) checkManifestFiles(t)
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{}) checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{})
} }
@@ -109,7 +104,7 @@ func TestDeleteDuplicateLayers(t *testing.T) {
t.Errorf("expected status code 200, actual %d", w.Code) t.Errorf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{}) checkManifestFiles(t)
} }
func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) { func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) {
@@ -129,14 +124,12 @@ func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) {
t.Fatalf("expected status code 200, actual %d", w.Code) t.Fatalf("expected status code 200, actual %d", w.Code)
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{ checkManifestFiles(t, "gpt-oss:20b-cloud")
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "gpt-oss", "20b-cloud"),
})
w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "gpt-oss:20b:cloud"}) w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "gpt-oss:20b:cloud"})
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Fatalf("expected status code 200, actual %d (%s)", w.Code, w.Body.String()) t.Fatalf("expected status code 200, actual %d (%s)", w.Code, w.Body.String())
} }
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{}) checkManifestFiles(t)
} }

View File

@@ -658,11 +658,14 @@ func TestManifestCaseSensitivity(t *testing.T) {
checkManifestList := func() { checkManifestList := func() {
t.Helper() t.Helper()
mandir := filepath.Join(os.Getenv("OLLAMA_MODELS"), "manifests/") mandir, err := manifest.V2Path()
if err != nil {
t.Fatalf("failed to resolve v2 manifest path: %v", err)
}
var entries []string var entries []string
t.Logf("dir entries:") t.Logf("dir entries:")
fsys := os.DirFS(mandir) fsys := os.DirFS(mandir)
err := fs.WalkDir(fsys, ".", func(path string, info fs.DirEntry, err error) error { err = fs.WalkDir(fsys, ".", func(path string, info fs.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
@@ -685,7 +688,14 @@ func TestManifestCaseSensitivity(t *testing.T) {
g := entries[0] // raw path g := entries[0] // raw path
g = filepath.ToSlash(g) g = filepath.ToSlash(g)
w := model.ParseName(wantStableName).Filepath() wp, err := manifest.V2PathForName(model.ParseName(wantStableName))
if err != nil {
t.Fatalf("failed to resolve expected manifest path: %v", err)
}
w, err := filepath.Rel(mandir, wp)
if err != nil {
t.Fatalf("failed to make expected manifest path relative: %v", err)
}
w = filepath.ToSlash(w) w = filepath.ToSlash(w)
if g != w { if g != w {
t.Errorf("\ngot: %s\nwant: %s", g, w) t.Errorf("\ngot: %s\nwant: %s", g, w)

View File

@@ -11,6 +11,8 @@ import (
"strings" "strings"
"github.com/ollama/ollama/envconfig" "github.com/ollama/ollama/envconfig"
rootmanifest "github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
) )
// ManifestLayer represents a layer in the manifest. // ManifestLayer represents a layer in the manifest.
@@ -49,9 +51,7 @@ func DefaultManifestDir() string {
// LoadManifest loads a manifest for the given model name. // LoadManifest loads a manifest for the given model name.
// Model name format: "modelname" or "modelname:tag" or "host/namespace/name:tag" // Model name format: "modelname" or "modelname:tag" or "host/namespace/name:tag"
func LoadManifest(modelName string) (*ModelManifest, error) { func LoadManifest(modelName string) (*ModelManifest, error) {
manifestPath := resolveManifestPath(modelName) data, err := rootmanifest.ReadManifestData(model.ParseName(modelName))
data, err := os.ReadFile(manifestPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("read manifest: %w", err) return nil, fmt.Errorf("read manifest: %w", err)
} }
@@ -67,36 +67,6 @@ func LoadManifest(modelName string) (*ModelManifest, error) {
}, nil }, nil
} }
// resolveManifestPath converts a model name to a manifest file path.
func resolveManifestPath(modelName string) string {
// Parse model name into components
// Default: registry.ollama.ai/library/<name>/<tag>
host := "registry.ollama.ai"
namespace := "library"
name := modelName
tag := "latest"
// Handle explicit tag
if idx := strings.LastIndex(name, ":"); idx != -1 {
tag = name[idx+1:]
name = name[:idx]
}
// Handle full path like "host/namespace/name"
parts := strings.Split(name, "/")
switch len(parts) {
case 3:
host = parts[0]
namespace = parts[1]
name = parts[2]
case 2:
namespace = parts[0]
name = parts[1]
}
return filepath.Join(DefaultManifestDir(), host, namespace, name, tag)
}
// BlobPath returns the full path to a blob given its digest. // BlobPath returns the full path to a blob given its digest.
func (m *ModelManifest) BlobPath(digest string) string { func (m *ModelManifest) BlobPath(digest string) string {
// Convert "sha256:abc123" to "sha256-abc123" // Convert "sha256:abc123" to "sha256-abc123"

View File

@@ -1,8 +1,12 @@
package manifest package manifest
import ( import (
"os"
"path/filepath" "path/filepath"
"testing" "testing"
rootmanifest "github.com/ollama/ollama/manifest"
"github.com/ollama/ollama/types/model"
) )
func TestTotalTensorSize(t *testing.T) { func TestTotalTensorSize(t *testing.T) {
@@ -55,3 +59,39 @@ func TestManifestAndBlobDirsRespectOLLAMAModels(t *testing.T) {
t.Fatalf("DefaultBlobDir() = %q, want %q", got, wantBlobs) t.Fatalf("DefaultBlobDir() = %q, want %q", got, wantBlobs)
} }
} }
func TestLoadManifestPrefersV2(t *testing.T) {
t.Setenv("OLLAMA_MODELS", t.TempDir())
name := model.ParseName("example")
legacyPath, err := rootmanifest.PathForName(name)
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(legacyPath, []byte(`{"schemaVersion":2,"mediaType":"legacy"}`), 0o644); err != nil {
t.Fatal(err)
}
v2Path, err := rootmanifest.V2PathForName(name)
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(v2Path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(v2Path, []byte(`{"schemaVersion":2,"mediaType":"v2"}`), 0o644); err != nil {
t.Fatal(err)
}
m, err := LoadManifest(name.String())
if err != nil {
t.Fatal(err)
}
if m.Manifest.MediaType != "v2" {
t.Fatalf("media type = %q, want %q", m.Manifest.MediaType, "v2")
}
}