mirror of
https://github.com/ollama/ollama.git
synced 2026-04-22 16:55:44 +02:00
Compare commits
2 Commits
jessegross
...
pdevine/ad
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7bcdb250b9 | ||
|
|
7bbcd2e6be |
@@ -1,18 +1,23 @@
|
|||||||
package manifest
|
package manifest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/ollama/ollama/types/model"
|
"github.com/ollama/ollama/types/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var blobFilenamePattern = regexp.MustCompile(`^sha256-[0-9a-fA-F]{64}$`)
|
||||||
|
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
SchemaVersion int `json:"schemaVersion"`
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
MediaType string `json:"mediaType"`
|
MediaType string `json:"mediaType"`
|
||||||
@@ -22,6 +27,7 @@ type Manifest struct {
|
|||||||
filepath string
|
filepath string
|
||||||
fi os.FileInfo
|
fi os.FileInfo
|
||||||
digest string
|
digest string
|
||||||
|
name model.Name
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manifest) Size() (size int64) {
|
func (m *Manifest) Size() (size int64) {
|
||||||
@@ -36,6 +42,14 @@ func (m *Manifest) Digest() string {
|
|||||||
return m.digest
|
return m.digest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Manifest) BlobDigest() string {
|
||||||
|
if m.digest == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return "sha256:" + m.digest
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manifest) FileInfo() os.FileInfo {
|
func (m *Manifest) FileInfo() os.FileInfo {
|
||||||
return m.fi
|
return m.fi
|
||||||
}
|
}
|
||||||
@@ -59,16 +73,7 @@ func (m *Manifest) ReadConfigJSON(configPath string, v any) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manifest) Remove() error {
|
func (m *Manifest) Remove() error {
|
||||||
if err := os.Remove(m.filepath); err != nil {
|
return removeNamedManifestPaths(m.name)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
manifests, err := Path()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return PruneDirectory(manifests)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manifest) RemoveLayers() error {
|
func (m *Manifest) RemoveLayers() error {
|
||||||
@@ -80,6 +85,9 @@ func (m *Manifest) RemoveLayers() error {
|
|||||||
// Build set of digests still in use by other manifests
|
// Build set of digests still in use by other manifests
|
||||||
inUse := make(map[string]struct{})
|
inUse := make(map[string]struct{})
|
||||||
for _, other := range ms {
|
for _, other := range ms {
|
||||||
|
if other.BlobDigest() != "" {
|
||||||
|
inUse[other.BlobDigest()] = struct{}{}
|
||||||
|
}
|
||||||
for _, layer := range append(other.Layers, other.Config) {
|
for _, layer := range append(other.Layers, other.Config) {
|
||||||
if layer.Digest != "" {
|
if layer.Digest != "" {
|
||||||
inUse[layer.Digest] = struct{}{}
|
inUse[layer.Digest] = struct{}{}
|
||||||
@@ -87,20 +95,27 @@ func (m *Manifest) RemoveLayers() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove layers not used by any other manifest
|
digests := make([]string, 0, len(m.Layers)+2)
|
||||||
for _, layer := range append(m.Layers, m.Config) {
|
digests = append(digests, m.BlobDigest())
|
||||||
if layer.Digest == "" {
|
for _, layer := range m.Layers {
|
||||||
|
digests = append(digests, layer.Digest)
|
||||||
|
}
|
||||||
|
digests = append(digests, m.Config.Digest)
|
||||||
|
|
||||||
|
// Remove manifest and layer blobs not used by any other manifest
|
||||||
|
for _, digest := range digests {
|
||||||
|
if digest == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, used := inUse[layer.Digest]; used {
|
if _, used := inUse[digest]; used {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
blob, err := BlobsPath(layer.Digest)
|
blob, err := BlobsPath(digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := os.Remove(blob); os.IsNotExist(err) {
|
if err := os.Remove(blob); os.IsNotExist(err) {
|
||||||
slog.Debug("layer does not exist", "digest", layer.Digest)
|
slog.Debug("blob does not exist", "digest", digest)
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -114,15 +129,36 @@ func ParseNamedManifest(n model.Name) (*Manifest, error) {
|
|||||||
return nil, model.Unqualified(n)
|
return nil, model.Unqualified(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifests, err := Path()
|
p, root, err := resolveManifestPath(n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
p := filepath.Join(manifests, n.Filepath())
|
return parseManifestFile(normalizeLogicalName(n), p, root)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ReadManifestData(n model.Name) ([]byte, error) {
|
||||||
|
if !n.IsFullyQualified() {
|
||||||
|
return nil, model.Unqualified(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, root, err := resolveManifestPath(n)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
f, _, err := OpenVerifiedManifest(p, root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return io.ReadAll(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseManifestFile(name model.Name, path, root string) (*Manifest, error) {
|
||||||
var m Manifest
|
var m Manifest
|
||||||
f, err := os.Open(p)
|
f, digest, err := OpenVerifiedManifest(path, root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -133,35 +169,19 @@ func ParseNamedManifest(n model.Name) (*Manifest, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
sha256sum := sha256.New()
|
if err := json.NewDecoder(f).Decode(&m); err != nil {
|
||||||
if err := json.NewDecoder(io.TeeReader(f, sha256sum)).Decode(&m); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
m.filepath = p
|
m.filepath = path
|
||||||
m.fi = fi
|
m.fi = fi
|
||||||
m.digest = hex.EncodeToString(sha256sum.Sum(nil))
|
m.digest = digest
|
||||||
|
m.name = name
|
||||||
|
|
||||||
return &m, nil
|
return &m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func WriteManifest(name model.Name, config Layer, layers []Layer) error {
|
func WriteManifest(name model.Name, config Layer, layers []Layer) error {
|
||||||
manifests, err := Path()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
p := filepath.Join(manifests, name.Filepath())
|
|
||||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.Create(p)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
m := Manifest{
|
m := Manifest{
|
||||||
SchemaVersion: 2,
|
SchemaVersion: 2,
|
||||||
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
|
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
|
||||||
@@ -169,33 +189,371 @@ func WriteManifest(name model.Name, config Layer, layers []Layer) error {
|
|||||||
Layers: layers,
|
Layers: layers,
|
||||||
}
|
}
|
||||||
|
|
||||||
return json.NewEncoder(f).Encode(m)
|
var b bytes.Buffer
|
||||||
|
if err := json.NewEncoder(&b).Encode(m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return WriteManifestData(name, b.Bytes())
|
||||||
}
|
}
|
||||||
|
|
||||||
func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
|
// WriteManifestData stores raw manifest bytes as a content-addressed blob and
|
||||||
|
// updates the v2 named manifest path to reference that blob. Any legacy named
|
||||||
|
// manifest for the same model is removed after the v2 write succeeds.
|
||||||
|
func WriteManifestData(name model.Name, data []byte) error {
|
||||||
|
if !name.IsFullyQualified() {
|
||||||
|
return model.Unqualified(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest, err := writeManifestBlob(data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := LinkManifest(name, digest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return removeLegacyManifestPaths(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LinkManifest updates the v2 named manifest path to reference an existing
|
||||||
|
// manifest blob. It prefers symlinks, then hardlinks, then a byte-for-byte copy
|
||||||
|
// for filesystems that do not support links.
|
||||||
|
func LinkManifest(name model.Name, digest string) error {
|
||||||
|
if !name.IsFullyQualified() {
|
||||||
|
return model.Unqualified(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestPath, err := V2PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
blobPath, err := BlobsPath(digest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(blobPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := checkBlobDigest(blobPath, digest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if rel, err := filepath.Rel(filepath.Dir(manifestPath), blobPath); err == nil {
|
||||||
|
if err := os.Symlink(rel, manifestPath); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Link(blobPath, manifestPath); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return copyManifestFile(blobPath, manifestPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeManifestBlob(data []byte) (string, error) {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
digest := fmt.Sprintf("sha256:%x", sum)
|
||||||
|
|
||||||
|
blobPath, err := BlobsPath(digest)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if existing, err := os.ReadFile(blobPath); err == nil && bytes.Equal(existing, data) {
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
blobs, err := BlobsPath("")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
temp, err := os.CreateTemp(blobs, "sha256-")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
tempName := temp.Name()
|
||||||
|
defer os.Remove(tempName)
|
||||||
|
|
||||||
|
if _, err := temp.Write(data); err != nil {
|
||||||
|
temp.Close()
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := temp.Close(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tempName, 0o644); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempName, blobPath); err != nil {
|
||||||
|
if err := os.Remove(blobPath); err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempName, blobPath); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyManifestFile(src, dst string) error {
|
||||||
|
in, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
|
||||||
|
temp, err := os.CreateTemp(filepath.Dir(dst), ".manifest-*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tempName := temp.Name()
|
||||||
|
defer os.Remove(tempName)
|
||||||
|
|
||||||
|
if _, err := io.Copy(temp, in); err != nil {
|
||||||
|
temp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := temp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tempName, 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.Rename(tempName, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenVerifiedManifest opens a named manifest path rooted under root. Symlinks must resolve to a
|
||||||
|
// blob whose basename is sha256-<hex> and whose bytes hash to that digest.
|
||||||
|
// Regular-file manifests are treated as legacy/copy fallback manifests and are
|
||||||
|
// opened without mutating the local store.
|
||||||
|
func OpenVerifiedManifest(path, root string) (*os.File, string, error) {
|
||||||
|
resolvedRoot, err := filepath.EvalSymlinks(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
target, err := evalAbs(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
base := filepath.Base(target)
|
||||||
|
if !blobFilenamePattern.MatchString(base) {
|
||||||
|
return nil, "", fmt.Errorf("manifest symlink target %q is not a sha256 blob", target)
|
||||||
|
}
|
||||||
|
|
||||||
|
digest := strings.ToLower(strings.TrimPrefix(base, "sha256-"))
|
||||||
|
blobPath, err := BlobsPath("sha256:" + digest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if !sameFile(target, blobPath) {
|
||||||
|
return nil, "", fmt.Errorf("manifest symlink target %q does not match blob %q", target, blobPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if err := checkBlobDigestReader(f, "sha256:"+digest); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return f, digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !pathWithin(target, resolvedRoot) {
|
||||||
|
return nil, "", fmt.Errorf("manifest path %q resolves outside manifest directory", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(h, f); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
digest := fmt.Sprintf("%x", h.Sum(nil))
|
||||||
|
|
||||||
|
return f, digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MigrateManifestLinks moves legacy named manifests into manifests-v2. This is currently unwired but
|
||||||
|
// will be added in the future.
|
||||||
|
func MigrateManifestLinks() (int, error) {
|
||||||
manifests, err := Path()
|
manifests, err := Path()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(mxyng): use something less brittle
|
// TODO(mxyng): use something less brittle
|
||||||
matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*"))
|
matches, err := filepath.Glob(filepath.Join(manifests, "*", "*", "*", "*"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
ms := make(map[model.Name]*Manifest)
|
var migrated int
|
||||||
for _, match := range matches {
|
for _, match := range matches {
|
||||||
fi, err := os.Stat(match)
|
fi, err := os.Stat(match)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return migrated, err
|
||||||
|
}
|
||||||
|
if fi.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(manifests, match)
|
||||||
|
if err != nil {
|
||||||
|
return migrated, fmt.Errorf("%s %w", match, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n := model.ParseNameFromFilepath(rel)
|
||||||
|
if !n.IsFullyQualified() {
|
||||||
|
slog.Warn("bad manifest name", "path", rel)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := readManifestPath(match, manifests)
|
||||||
|
if err != nil {
|
||||||
|
return migrated, err
|
||||||
|
}
|
||||||
|
if err := WriteManifestData(normalizeLogicalName(n), data); err != nil {
|
||||||
|
return migrated, err
|
||||||
|
}
|
||||||
|
migrated++
|
||||||
|
}
|
||||||
|
|
||||||
|
return migrated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readManifestPath(path, root string) ([]byte, error) {
|
||||||
|
f, _, err := OpenVerifiedManifest(path, root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return io.ReadAll(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathWithin(path, root string) bool {
|
||||||
|
rel, err := filepath.Rel(root, path)
|
||||||
|
return err == nil && rel != "." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != ".."
|
||||||
|
}
|
||||||
|
|
||||||
|
func evalAbs(path string) (string, error) {
|
||||||
|
abs, err := filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.EvalSymlinks(abs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameFile(a, b string) bool {
|
||||||
|
ai, err := os.Stat(a)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
bi, err := os.Stat(b)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return os.SameFile(ai, bi)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkBlobDigest(path, digest string) error {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return checkBlobDigestReader(f, digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkBlobDigestReader(r io.Reader, digest string) error {
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(h, r); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
got := fmt.Sprintf("sha256:%x", h.Sum(nil))
|
||||||
|
if got != strings.ToLower(strings.Replace(digest, "-", ":", 1)) {
|
||||||
|
return errors.New("digest mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
|
||||||
|
ms := make(map[model.Name]*Manifest)
|
||||||
|
|
||||||
|
manifestsV2, err := V2Path()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := collectManifests(ms, manifestsV2, continueOnError); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
manifests, err := Path()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := collectManifests(ms, manifests, continueOnError); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ms, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectManifests(ms map[model.Name]*Manifest, root string, continueOnError bool) error {
|
||||||
|
// TODO(mxyng): use something less brittle
|
||||||
|
matches, err := filepath.Glob(filepath.Join(root, "*", "*", "*", "*"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
fi, err := os.Lstat(match)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fi.IsDir() {
|
if !fi.IsDir() {
|
||||||
rel, err := filepath.Rel(manifests, match)
|
rel, err := filepath.Rel(root, match)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !continueOnError {
|
if !continueOnError {
|
||||||
return nil, fmt.Errorf("%s %w", match, err)
|
return fmt.Errorf("%s %w", match, err)
|
||||||
}
|
}
|
||||||
slog.Warn("bad filepath", "path", match, "error", err)
|
slog.Warn("bad filepath", "path", match, "error", err)
|
||||||
continue
|
continue
|
||||||
@@ -204,16 +562,21 @@ func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
|
|||||||
n := model.ParseNameFromFilepath(rel)
|
n := model.ParseNameFromFilepath(rel)
|
||||||
if !n.IsValid() {
|
if !n.IsValid() {
|
||||||
if !continueOnError {
|
if !continueOnError {
|
||||||
return nil, fmt.Errorf("%s %w", rel, err)
|
return fmt.Errorf("invalid manifest name: %s", rel)
|
||||||
}
|
}
|
||||||
slog.Warn("bad manifest name", "path", rel)
|
slog.Warn("bad manifest name", "path", rel)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
m, err := ParseNamedManifest(n)
|
n = normalizeLogicalName(n)
|
||||||
|
if _, ok := ms[n]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := parseManifestFile(n, match, root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !continueOnError {
|
if !continueOnError {
|
||||||
return nil, fmt.Errorf("%s %w", n, err)
|
return fmt.Errorf("%s %w", n, err)
|
||||||
}
|
}
|
||||||
slog.Warn("bad manifest", "name", n, "error", err)
|
slog.Warn("bad manifest", "name", n, "error", err)
|
||||||
continue
|
continue
|
||||||
@@ -223,5 +586,5 @@ func Manifests(continueOnError bool) (map[model.Name]*Manifest, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return ms, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
package manifest
|
package manifest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/ollama/ollama/types/model"
|
"github.com/ollama/ollama/types/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
func createManifest(t *testing.T, path, name string) {
|
func createManifestAtRoot(t *testing.T, path, root, name string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
p := filepath.Join(path, "manifests", name)
|
p := filepath.Join(path, root, name)
|
||||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -29,6 +33,309 @@ func createManifest(t *testing.T, path, name string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func createManifest(t *testing.T, path, name string) {
|
||||||
|
t.Helper()
|
||||||
|
createManifestAtRoot(t, path, "manifests", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteManifestStoresManifestAsBlob(t *testing.T) {
|
||||||
|
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
config := Layer{
|
||||||
|
MediaType: "application/vnd.docker.container.image.v1+json",
|
||||||
|
Digest: "sha256:" + strings.Repeat("a", 64),
|
||||||
|
Size: 12,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := WriteManifest(name, config, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestPath, err := V2PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifestData, err := os.ReadFile(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sum := sha256.Sum256(manifestData)
|
||||||
|
digest := fmt.Sprintf("sha256:%x", sum)
|
||||||
|
blobPath, err := BlobsPath(digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blobData, err := os.ReadFile(blobPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(blobData, manifestData) {
|
||||||
|
t.Fatal("manifest path and blob content differ")
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := ParseNamedManifest(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := m.Digest(); got != fmt.Sprintf("%x", sum) {
|
||||||
|
t.Fatalf("digest = %q, want %x", got, sum)
|
||||||
|
}
|
||||||
|
if got := m.BlobDigest(); got != digest {
|
||||||
|
t.Fatalf("blob digest = %q, want %q", got, digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNamedManifestLeavesLegacyManifestInPlace(t *testing.T) {
|
||||||
|
models := t.TempDir()
|
||||||
|
t.Setenv("OLLAMA_MODELS", models)
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
createManifest(t, models, name.Filepath())
|
||||||
|
|
||||||
|
manifestPath, err := PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := ParseNamedManifest(name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fi, err := os.Lstat(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fi.Mode()&os.ModeSymlink != 0 {
|
||||||
|
t.Fatal("legacy manifest was converted to a symlink while reading")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
blobPath, err := BlobsPath(fmt.Sprintf("sha256:%x", sum))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("legacy manifest read created blob: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigrateManifestLinks(t *testing.T) {
|
||||||
|
models := t.TempDir()
|
||||||
|
t.Setenv("OLLAMA_MODELS", models)
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
createManifest(t, models, name.Filepath())
|
||||||
|
|
||||||
|
migrated, err := MigrateManifestLinks()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if migrated != 1 {
|
||||||
|
t.Fatalf("migrated = %d, want 1", migrated)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestPath, err := V2PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifestData, err := os.ReadFile(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(manifestData)
|
||||||
|
blobPath, err := BlobsPath(fmt.Sprintf("sha256:%x", sum))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blobData, err := os.ReadFile(blobPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(blobData, manifestData) {
|
||||||
|
t.Fatal("migrated manifest path and blob content differ")
|
||||||
|
}
|
||||||
|
|
||||||
|
legacyPath, err := PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("legacy manifest still exists: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
migrated, err = MigrateManifestLinks()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if migrated != 0 {
|
||||||
|
t.Fatalf("migrated on second run = %d, want 0", migrated)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := MigrateManifestLinks(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifestDataAfter, err := os.ReadFile(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(manifestDataAfter, manifestData) {
|
||||||
|
t.Fatal("second migration changed manifest content")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveLayersRemovesUnreferencedManifestBlob(t *testing.T) {
|
||||||
|
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
if err := WriteManifest(name, Layer{}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := ParseNamedManifest(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blobPath, err := BlobsPath(m.BlobDigest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(blobPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.Remove(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := m.RemoveLayers(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("manifest blob still exists: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNamedManifestRejectsUnsafeSymlinks(t *testing.T) {
|
||||||
|
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
manifestPath, err := PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("non blob basename", func(t *testing.T) {
|
||||||
|
target := filepath.Join(t.TempDir(), "not-a-blob")
|
||||||
|
if err := os.WriteFile(target, []byte(`{"schemaVersion":2}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, manifestPath); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ParseNamedManifest(name)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "not a sha256 blob") {
|
||||||
|
t.Fatalf("err = %v, want not a sha256 blob", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("blob basename outside blob store", func(t *testing.T) {
|
||||||
|
data := []byte(`{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`)
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
target := filepath.Join(t.TempDir(), fmt.Sprintf("sha256-%x", sum))
|
||||||
|
if err := os.WriteFile(target, data, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, manifestPath); err != nil {
|
||||||
|
t.Skipf("symlink unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ParseNamedManifest(name)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "does not match blob") {
|
||||||
|
t.Fatalf("err = %v, want does not match blob", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNamedManifestPrefersV2(t *testing.T) {
|
||||||
|
models := t.TempDir()
|
||||||
|
t.Setenv("OLLAMA_MODELS", models)
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
|
||||||
|
legacyPath, err := PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(legacyPath, []byte(`{"schemaVersion":2,"mediaType":"legacy"}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := WriteManifestData(name, []byte(`{"schemaVersion":2,"mediaType":"v2"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := ParseNamedManifest(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.MediaType != "v2" {
|
||||||
|
t.Fatalf("media type = %q, want %q", m.MediaType, "v2")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestsV2ShadowsLegacy(t *testing.T) {
|
||||||
|
models := t.TempDir()
|
||||||
|
t.Setenv("OLLAMA_MODELS", models)
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
createManifest(t, models, name.Filepath())
|
||||||
|
if err := WriteManifestData(name, []byte(`{"schemaVersion":2,"mediaType":"v2"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ms, err := Manifests(true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ms) != 1 {
|
||||||
|
t.Fatalf("manifest count = %d, want 1", len(ms))
|
||||||
|
}
|
||||||
|
|
||||||
|
var m *Manifest
|
||||||
|
for gotName, gotManifest := range ms {
|
||||||
|
if gotName.EqualFold(model.ParseName("example")) {
|
||||||
|
m = gotManifest
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m == nil {
|
||||||
|
t.Fatalf("missing v2 manifest for %s", name)
|
||||||
|
}
|
||||||
|
if m.MediaType != "v2" {
|
||||||
|
t.Fatalf("media type = %q, want %q", m.MediaType, "v2")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManifests(t *testing.T) {
|
func TestManifests(t *testing.T) {
|
||||||
cases := map[string]struct {
|
cases := map[string]struct {
|
||||||
ps []string
|
ps []string
|
||||||
|
|||||||
@@ -14,8 +14,23 @@ import (
|
|||||||
|
|
||||||
var ErrInvalidDigestFormat = errors.New("invalid digest format")
|
var ErrInvalidDigestFormat = errors.New("invalid digest format")
|
||||||
|
|
||||||
|
const (
|
||||||
|
legacyDirName = "manifests"
|
||||||
|
v2DirName = "manifests-v2"
|
||||||
|
defaultPublicHost = "registry.ollama.ai"
|
||||||
|
v2CanonicalHost = "ollama.com"
|
||||||
|
)
|
||||||
|
|
||||||
func Path() (string, error) {
|
func Path() (string, error) {
|
||||||
path := filepath.Join(envconfig.Models(), "manifests")
|
return manifestPath(legacyDirName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func V2Path() (string, error) {
|
||||||
|
return manifestPath(v2DirName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestPath(dir string) (string, error) {
|
||||||
|
path := filepath.Join(envconfig.Models(), dir)
|
||||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||||
return "", fmt.Errorf("%w: ensure path elements are traversable", err)
|
return "", fmt.Errorf("%w: ensure path elements are traversable", err)
|
||||||
}
|
}
|
||||||
@@ -25,6 +40,10 @@ func Path() (string, error) {
|
|||||||
|
|
||||||
// PathForName returns the path to the manifest file for a specific model name.
|
// PathForName returns the path to the manifest file for a specific model name.
|
||||||
func PathForName(n model.Name) (string, error) {
|
func PathForName(n model.Name) (string, error) {
|
||||||
|
return LegacyPathForName(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func LegacyPathForName(n model.Name) (string, error) {
|
||||||
if !n.IsValid() {
|
if !n.IsValid() {
|
||||||
return "", os.ErrNotExist
|
return "", os.ErrNotExist
|
||||||
}
|
}
|
||||||
@@ -37,6 +56,162 @@ func PathForName(n model.Name) (string, error) {
|
|||||||
return filepath.Join(manifests, n.Filepath()), nil
|
return filepath.Join(manifests, n.Filepath()), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func V2PathForName(n model.Name) (string, error) {
|
||||||
|
if !n.IsValid() {
|
||||||
|
return "", os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
manifests, err := V2Path()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(manifests, canonicalV2Name(n).Filepath()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ResolvePathForName(n model.Name) (string, error) {
|
||||||
|
path, _, err := resolveManifestPath(n)
|
||||||
|
return path, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveManifestPath(n model.Name) (string, string, error) {
|
||||||
|
if !n.IsValid() {
|
||||||
|
return "", "", os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
v2Path, err := V2PathForName(n)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(v2Path); err == nil {
|
||||||
|
root, err := V2Path()
|
||||||
|
return v2Path, root, err
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
legacyRoot, err := Path()
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
for _, legacyName := range legacyNameCandidates(n) {
|
||||||
|
legacyPath := filepath.Join(legacyRoot, legacyName.Filepath())
|
||||||
|
if _, err := os.Lstat(legacyPath); err == nil {
|
||||||
|
return legacyPath, legacyRoot, nil
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", "", os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeNamedManifestPaths(n model.Name) error {
|
||||||
|
candidates := legacyNameCandidates(n)
|
||||||
|
paths := make([]string, 0, 1+len(candidates))
|
||||||
|
|
||||||
|
v2Path, err := V2PathForName(n)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
paths = append(paths, v2Path)
|
||||||
|
|
||||||
|
for _, legacyName := range candidates {
|
||||||
|
legacyPath, err := LegacyPathForName(legacyName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
paths = append(paths, legacyPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range paths {
|
||||||
|
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return pruneManifestRoots()
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeLegacyManifestPaths(n model.Name) error {
|
||||||
|
for _, legacyName := range legacyNameCandidates(n) {
|
||||||
|
legacyPath, err := LegacyPathForName(legacyName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(legacyPath); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
legacyRoot, err := Path()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PruneDirectory(legacyRoot); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pruneManifestRoots() error {
|
||||||
|
roots := []func() (string, error){Path, V2Path}
|
||||||
|
for _, rootFn := range roots {
|
||||||
|
root, err := rootFn()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := PruneDirectory(root); err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeLogicalName maps any public host to the legacy default
|
||||||
|
// so that map keys use a single identity regardless of on-disk host.
|
||||||
|
func normalizeLogicalName(n model.Name) model.Name {
|
||||||
|
if isDefaultPublicHost(n.Host) {
|
||||||
|
n.Host = defaultPublicHost
|
||||||
|
}
|
||||||
|
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// canonicalV2Name maps any public host to the v2 canonical host
|
||||||
|
// for use in manifests-v2/ on-disk paths.
|
||||||
|
func canonicalV2Name(n model.Name) model.Name {
|
||||||
|
if isDefaultPublicHost(n.Host) {
|
||||||
|
n.Host = v2CanonicalHost
|
||||||
|
}
|
||||||
|
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyNameCandidates(n model.Name) []model.Name {
|
||||||
|
names := []model.Name{n}
|
||||||
|
if !isDefaultPublicHost(n.Host) {
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
alt := n
|
||||||
|
switch {
|
||||||
|
case strings.EqualFold(n.Host, defaultPublicHost):
|
||||||
|
alt.Host = v2CanonicalHost
|
||||||
|
default:
|
||||||
|
alt.Host = defaultPublicHost
|
||||||
|
}
|
||||||
|
|
||||||
|
return append(names, alt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDefaultPublicHost(host string) bool {
|
||||||
|
return strings.EqualFold(host, defaultPublicHost) || strings.EqualFold(host, v2CanonicalHost)
|
||||||
|
}
|
||||||
|
|
||||||
func BlobsPath(digest string) (string, error) {
|
func BlobsPath(digest string) (string, error) {
|
||||||
// only accept actual sha256 digests
|
// only accept actual sha256 digests
|
||||||
pattern := "^sha256[:-][0-9a-fA-F]{64}$"
|
pattern := "^sha256[:-][0-9a-fA-F]{64}$"
|
||||||
|
|||||||
@@ -411,31 +411,12 @@ func CopyModel(src, dst model.Name) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
manifests, err := manifest.Path()
|
data, err := manifest.ReadManifestData(src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dstpath := filepath.Join(manifests, dst.Filepath())
|
return manifest.WriteManifestData(dst, data)
|
||||||
if err := os.MkdirAll(filepath.Dir(dstpath), 0o755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
srcpath := filepath.Join(manifests, src.Filepath())
|
|
||||||
srcfile, err := os.Open(srcpath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer srcfile.Close()
|
|
||||||
|
|
||||||
dstfile, err := os.Create(dstpath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer dstfile.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(dstfile, srcfile)
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteUnusedLayers(deleteMap map[string]struct{}) error {
|
func deleteUnusedLayers(deleteMap map[string]struct{}) error {
|
||||||
@@ -446,6 +427,10 @@ func deleteUnusedLayers(deleteMap map[string]struct{}) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, manifest := range manifests {
|
for _, manifest := range manifests {
|
||||||
|
if manifest.BlobDigest() != "" {
|
||||||
|
delete(deleteMap, manifest.BlobDigest())
|
||||||
|
}
|
||||||
|
|
||||||
for _, layer := range manifest.Layers {
|
for _, layer := range manifest.Layers {
|
||||||
delete(deleteMap, layer.Digest)
|
delete(deleteMap, layer.Digest)
|
||||||
}
|
}
|
||||||
@@ -549,11 +534,7 @@ func PushModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
|
|||||||
// Use fast transfer for models with tensor layers (many small blobs)
|
// Use fast transfer for models with tensor layers (many small blobs)
|
||||||
if hasTensorLayers(layers) {
|
if hasTensorLayers(layers) {
|
||||||
// Read raw manifest JSON to preserve tensor metadata fields
|
// Read raw manifest JSON to preserve tensor metadata fields
|
||||||
manifestPath, err := manifest.PathForName(n)
|
manifestJSON, err := manifest.ReadManifestData(n)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
manifestJSON, err := os.ReadFile(manifestPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -610,6 +591,14 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
|
|||||||
if existingMf.Config.Digest != "" {
|
if existingMf.Config.Digest != "" {
|
||||||
deleteMap[existingMf.Config.Digest] = struct{}{}
|
deleteMap[existingMf.Config.Digest] = struct{}{}
|
||||||
}
|
}
|
||||||
|
if existingMf.BlobDigest() != "" {
|
||||||
|
digest := existingMf.BlobDigest()
|
||||||
|
if blob, err := manifest.BlobsPath(digest); err == nil {
|
||||||
|
if _, err := os.Stat(blob); err == nil {
|
||||||
|
deleteMap[digest] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.ProtocolScheme == "http" && !regOpts.Insecure {
|
if n.ProtocolScheme == "http" && !regOpts.Insecure {
|
||||||
@@ -679,21 +668,12 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu
|
|||||||
|
|
||||||
fn(api.ProgressResponse{Status: "writing manifest"})
|
fn(api.ProgressResponse{Status: "writing manifest"})
|
||||||
|
|
||||||
fp, err := manifest.PathForName(n)
|
if err := manifest.WriteManifestData(n, manifestData); err != nil {
|
||||||
if err != nil {
|
slog.Info(fmt.Sprintf("couldn't write manifest for %s", n.DisplayShortest()))
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.WriteFile(fp, manifestData, 0o644)
|
slog.Debug("manifest written", "name", n.DisplayShortest(), "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
|
||||||
if err != nil {
|
|
||||||
slog.Info(fmt.Sprintf("couldn't write to %s", fp))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
|
|
||||||
|
|
||||||
if !envconfig.NoPrune() && len(deleteMap) > 0 {
|
if !envconfig.NoPrune() && len(deleteMap) > 0 {
|
||||||
fn(api.ProgressResponse{Status: "removing unused layers"})
|
fn(api.ProgressResponse{Status: "removing unused layers"})
|
||||||
@@ -776,19 +756,11 @@ func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer
|
|||||||
// Write manifest
|
// Write manifest
|
||||||
fn(api.ProgressResponse{Status: "writing manifest"})
|
fn(api.ProgressResponse{Status: "writing manifest"})
|
||||||
|
|
||||||
fp, err := manifest.PathForName(n)
|
if err := manifest.WriteManifestData(n, manifestData); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.WriteFile(fp, manifestData, 0o644); err != nil {
|
slog.Debug("manifest written", "name", n.DisplayShortest(), "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData))
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -116,6 +116,10 @@ func (s *Local) serveHTTP(rec *statusCodeRecorder, r *http.Request) {
|
|||||||
proxied, err := func() (bool, error) {
|
proxied, err := func() (bool, error) {
|
||||||
switch r.URL.Path {
|
switch r.URL.Path {
|
||||||
case "/api/delete":
|
case "/api/delete":
|
||||||
|
if s.Fallback != nil {
|
||||||
|
s.Fallback.ServeHTTP(rec, r)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
return false, s.handleDelete(rec, r)
|
return false, s.handleDelete(rec, r)
|
||||||
case "/api/pull":
|
case "/api/pull":
|
||||||
return false, s.handlePull(rec, r)
|
return false, s.handlePull(rec, r)
|
||||||
|
|||||||
@@ -1770,13 +1770,15 @@ func Serve(ln net.Listener) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestsPath, err := manifest.Path()
|
for _, rootFn := range []func() (string, error){manifest.Path, manifest.V2Path} {
|
||||||
if err != nil {
|
manifestsPath, err := rootFn()
|
||||||
return err
|
if err != nil {
|
||||||
}
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := manifest.PruneDirectory(manifestsPath); err != nil {
|
if err := manifest.PruneDirectory(manifestsPath); err != nil && !os.IsNotExist(err) {
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -109,12 +109,44 @@ func checkFileExists(t *testing.T, p string, expect []string) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
if strings.HasSuffix(filepath.ToSlash(p), "/blobs/*") {
|
||||||
|
actual = slices.DeleteFunc(actual, isManifestBlobForTest)
|
||||||
|
}
|
||||||
|
|
||||||
if diff := gocmp.Diff(expect, actual, gocmpopts.SortSlices(strings.Compare), gocmpopts.EquateEmpty()); diff != "" {
|
if diff := gocmp.Diff(expect, actual, gocmpopts.SortSlices(strings.Compare), gocmpopts.EquateEmpty()); diff != "" {
|
||||||
t.Errorf("file exists mismatch (-want +got):\n%s", diff)
|
t.Errorf("file exists mismatch (-want +got):\n%s", diff)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkManifestFiles(t *testing.T, names ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
expect := make([]string, len(names))
|
||||||
|
for i, name := range names {
|
||||||
|
p, err := manifest.V2PathForName(model.ParseName(name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
expect[i] = p
|
||||||
|
}
|
||||||
|
|
||||||
|
checkFileExists(t, filepath.Join(envconfig.Models(), "manifests-v2", "*", "*", "*", "*"), expect)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isManifestBlobForTest(path string) bool {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var m manifest.Manifest
|
||||||
|
if err := json.Unmarshal(data, &m); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return m.SchemaVersion != 0 && m.MediaType != "" && (m.Config.Digest != "" || len(m.Layers) > 0)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCreateFromBin(t *testing.T) {
|
func TestCreateFromBin(t *testing.T) {
|
||||||
gin.SetMode(gin.TestMode)
|
gin.SetMode(gin.TestMode)
|
||||||
|
|
||||||
@@ -136,9 +168,7 @@ func TestCreateFromBin(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
||||||
@@ -196,9 +226,7 @@ func TestCreateFromModel(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
w = createRequest(t, s.CreateHandler, api.CreateRequest{
|
w = createRequest(t, s.CreateHandler, api.CreateRequest{
|
||||||
Name: "test2",
|
Name: "test2",
|
||||||
@@ -210,10 +238,7 @@ func TestCreateFromModel(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test", "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
||||||
@@ -306,9 +331,7 @@ func TestCreateRemovesLayers(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-89a2116c3a82d6a97f59f748d86ed4417214353fd178ee54df418fde32495fad"),
|
filepath.Join(p, "blobs", "sha256-89a2116c3a82d6a97f59f748d86ed4417214353fd178ee54df418fde32495fad"),
|
||||||
@@ -327,9 +350,7 @@ func TestCreateRemovesLayers(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
||||||
@@ -357,9 +378,7 @@ func TestCreateUnsetsSystem(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-0a666d113e8e0a3d27e9c7bd136a0bdfb6241037db50729d81568451ebfdbde8"),
|
filepath.Join(p, "blobs", "sha256-0a666d113e8e0a3d27e9c7bd136a0bdfb6241037db50729d81568451ebfdbde8"),
|
||||||
@@ -378,9 +397,7 @@ func TestCreateUnsetsSystem(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
filepath.Join(p, "blobs", "sha256-6bcdb8859d417753645538d7bbfbd7ca91a3f0c191aef5379c53c05e86b669dd"),
|
||||||
@@ -411,9 +428,7 @@ func TestCreateMergeParameters(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
|
filepath.Join(p, "blobs", "sha256-1d0ad71299d48c2fb7ae2b98e683643e771f8a5b72be34942af90d97a91c1e37"),
|
||||||
@@ -436,10 +451,7 @@ func TestCreateMergeParameters(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test", "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
// Display contents of each blob in the directory
|
// Display contents of each blob in the directory
|
||||||
blobDir := filepath.Join(p, "blobs")
|
blobDir := filepath.Join(p, "blobs")
|
||||||
@@ -495,10 +507,7 @@ func TestCreateMergeParameters(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test", "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"),
|
filepath.Join(p, "blobs", "sha256-12f58bb75cb3042d69a7e013ab87fb3c3c7088f50ddc62f0c77bd332f0d44d35"),
|
||||||
@@ -555,9 +564,7 @@ func TestCreateReplacesMessages(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"),
|
filepath.Join(p, "blobs", "sha256-298baeaf6928a60cf666d88d64a1ba606feb43a2865687c39e40652e407bffc4"),
|
||||||
@@ -589,10 +596,7 @@ func TestCreateReplacesMessages(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test", "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
// Old layers will not have been pruned
|
// Old layers will not have been pruned
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
@@ -650,9 +654,7 @@ func TestCreateTemplateSystem(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-0a04d979734167da3b80811a1874d734697f366a689f3912589b99d2e86e7ad1"),
|
filepath.Join(p, "blobs", "sha256-0a04d979734167da3b80811a1874d734697f366a689f3912589b99d2e86e7ad1"),
|
||||||
@@ -850,9 +852,7 @@ func TestCreateLicenses(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"),
|
filepath.Join(p, "blobs", "sha256-2af71558e438db0b73a20beab92dc278a94e1bbe974c00c1a33e3ab62d53a608"),
|
||||||
|
|||||||
@@ -42,10 +42,7 @@ func TestDelete(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test", "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test", "latest"),
|
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
||||||
@@ -60,9 +57,7 @@ func TestDelete(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "test2")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "test2", "latest"),
|
|
||||||
})
|
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{
|
||||||
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
filepath.Join(p, "blobs", "sha256-136bf7c76bac2ec09d6617885507d37829e04b41acc47687d45e512b544e893a"),
|
||||||
@@ -76,7 +71,7 @@ func TestDelete(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{})
|
checkManifestFiles(t)
|
||||||
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{})
|
checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,7 +104,7 @@ func TestDeleteDuplicateLayers(t *testing.T) {
|
|||||||
t.Errorf("expected status code 200, actual %d", w.Code)
|
t.Errorf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{})
|
checkManifestFiles(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) {
|
func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) {
|
||||||
@@ -129,14 +124,12 @@ func TestDeleteCloudSourceNormalizesToLegacyName(t *testing.T) {
|
|||||||
t.Fatalf("expected status code 200, actual %d", w.Code)
|
t.Fatalf("expected status code 200, actual %d", w.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{
|
checkManifestFiles(t, "gpt-oss:20b-cloud")
|
||||||
filepath.Join(p, "manifests", "registry.ollama.ai", "library", "gpt-oss", "20b-cloud"),
|
|
||||||
})
|
|
||||||
|
|
||||||
w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "gpt-oss:20b:cloud"})
|
w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Name: "gpt-oss:20b:cloud"})
|
||||||
if w.Code != http.StatusOK {
|
if w.Code != http.StatusOK {
|
||||||
t.Fatalf("expected status code 200, actual %d (%s)", w.Code, w.Body.String())
|
t.Fatalf("expected status code 200, actual %d (%s)", w.Code, w.Body.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
checkFileExists(t, filepath.Join(p, "manifests", "*", "*", "*", "*"), []string{})
|
checkManifestFiles(t)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -658,11 +658,14 @@ func TestManifestCaseSensitivity(t *testing.T) {
|
|||||||
checkManifestList := func() {
|
checkManifestList := func() {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
mandir := filepath.Join(os.Getenv("OLLAMA_MODELS"), "manifests/")
|
mandir, err := manifest.V2Path()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to resolve v2 manifest path: %v", err)
|
||||||
|
}
|
||||||
var entries []string
|
var entries []string
|
||||||
t.Logf("dir entries:")
|
t.Logf("dir entries:")
|
||||||
fsys := os.DirFS(mandir)
|
fsys := os.DirFS(mandir)
|
||||||
err := fs.WalkDir(fsys, ".", func(path string, info fs.DirEntry, err error) error {
|
err = fs.WalkDir(fsys, ".", func(path string, info fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -685,7 +688,14 @@ func TestManifestCaseSensitivity(t *testing.T) {
|
|||||||
|
|
||||||
g := entries[0] // raw path
|
g := entries[0] // raw path
|
||||||
g = filepath.ToSlash(g)
|
g = filepath.ToSlash(g)
|
||||||
w := model.ParseName(wantStableName).Filepath()
|
wp, err := manifest.V2PathForName(model.ParseName(wantStableName))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to resolve expected manifest path: %v", err)
|
||||||
|
}
|
||||||
|
w, err := filepath.Rel(mandir, wp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to make expected manifest path relative: %v", err)
|
||||||
|
}
|
||||||
w = filepath.ToSlash(w)
|
w = filepath.ToSlash(w)
|
||||||
if g != w {
|
if g != w {
|
||||||
t.Errorf("\ngot: %s\nwant: %s", g, w)
|
t.Errorf("\ngot: %s\nwant: %s", g, w)
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/ollama/ollama/envconfig"
|
"github.com/ollama/ollama/envconfig"
|
||||||
|
rootmanifest "github.com/ollama/ollama/manifest"
|
||||||
|
"github.com/ollama/ollama/types/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ManifestLayer represents a layer in the manifest.
|
// ManifestLayer represents a layer in the manifest.
|
||||||
@@ -49,9 +51,7 @@ func DefaultManifestDir() string {
|
|||||||
// LoadManifest loads a manifest for the given model name.
|
// LoadManifest loads a manifest for the given model name.
|
||||||
// Model name format: "modelname" or "modelname:tag" or "host/namespace/name:tag"
|
// Model name format: "modelname" or "modelname:tag" or "host/namespace/name:tag"
|
||||||
func LoadManifest(modelName string) (*ModelManifest, error) {
|
func LoadManifest(modelName string) (*ModelManifest, error) {
|
||||||
manifestPath := resolveManifestPath(modelName)
|
data, err := rootmanifest.ReadManifestData(model.ParseName(modelName))
|
||||||
|
|
||||||
data, err := os.ReadFile(manifestPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read manifest: %w", err)
|
return nil, fmt.Errorf("read manifest: %w", err)
|
||||||
}
|
}
|
||||||
@@ -67,36 +67,6 @@ func LoadManifest(modelName string) (*ModelManifest, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveManifestPath converts a model name to a manifest file path.
|
|
||||||
func resolveManifestPath(modelName string) string {
|
|
||||||
// Parse model name into components
|
|
||||||
// Default: registry.ollama.ai/library/<name>/<tag>
|
|
||||||
host := "registry.ollama.ai"
|
|
||||||
namespace := "library"
|
|
||||||
name := modelName
|
|
||||||
tag := "latest"
|
|
||||||
|
|
||||||
// Handle explicit tag
|
|
||||||
if idx := strings.LastIndex(name, ":"); idx != -1 {
|
|
||||||
tag = name[idx+1:]
|
|
||||||
name = name[:idx]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle full path like "host/namespace/name"
|
|
||||||
parts := strings.Split(name, "/")
|
|
||||||
switch len(parts) {
|
|
||||||
case 3:
|
|
||||||
host = parts[0]
|
|
||||||
namespace = parts[1]
|
|
||||||
name = parts[2]
|
|
||||||
case 2:
|
|
||||||
namespace = parts[0]
|
|
||||||
name = parts[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
return filepath.Join(DefaultManifestDir(), host, namespace, name, tag)
|
|
||||||
}
|
|
||||||
|
|
||||||
// BlobPath returns the full path to a blob given its digest.
|
// BlobPath returns the full path to a blob given its digest.
|
||||||
func (m *ModelManifest) BlobPath(digest string) string {
|
func (m *ModelManifest) BlobPath(digest string) string {
|
||||||
// Convert "sha256:abc123" to "sha256-abc123"
|
// Convert "sha256:abc123" to "sha256-abc123"
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
package manifest
|
package manifest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
rootmanifest "github.com/ollama/ollama/manifest"
|
||||||
|
"github.com/ollama/ollama/types/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTotalTensorSize(t *testing.T) {
|
func TestTotalTensorSize(t *testing.T) {
|
||||||
@@ -55,3 +59,39 @@ func TestManifestAndBlobDirsRespectOLLAMAModels(t *testing.T) {
|
|||||||
t.Fatalf("DefaultBlobDir() = %q, want %q", got, wantBlobs)
|
t.Fatalf("DefaultBlobDir() = %q, want %q", got, wantBlobs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadManifestPrefersV2(t *testing.T) {
|
||||||
|
t.Setenv("OLLAMA_MODELS", t.TempDir())
|
||||||
|
|
||||||
|
name := model.ParseName("example")
|
||||||
|
|
||||||
|
legacyPath, err := rootmanifest.PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(legacyPath), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(legacyPath, []byte(`{"schemaVersion":2,"mediaType":"legacy"}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
v2Path, err := rootmanifest.V2PathForName(name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(v2Path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(v2Path, []byte(`{"schemaVersion":2,"mediaType":"v2"}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := LoadManifest(name.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Manifest.MediaType != "v2" {
|
||||||
|
t.Fatalf("media type = %q, want %q", m.Manifest.MediaType, "v2")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user