From 522d553b03b9a0246d1f89446412a7448d38cf07 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Fri, 27 Mar 2026 14:30:51 -0700 Subject: [PATCH] server: preserve raw manifest bytes during pull pullModelManifest unmarshals the registry response into a Go struct then re-marshals with json.Marshal before writing to disk. When the registry's JSON formatting or field ordering differs from Go's output, the local SHA256 won't match the registry's Ollama-Content-Digest header, causing false "out of date" warnings. Preserve the raw bytes from the registry response and write them directly to disk so the local manifest is byte-for-byte identical to what the registry serves. --- server/images.go | 43 +++++++++++++++-------------- server/images_test.go | 64 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 20 deletions(-) diff --git a/server/images.go b/server/images.go index fad4e102a..56cb4b898 100644 --- a/server/images.go +++ b/server/images.go @@ -578,7 +578,7 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu fn(api.ProgressResponse{Status: "pulling manifest"}) - mf, err := pullModelManifest(ctx, n, regOpts) + mf, manifestData, err := pullModelManifest(ctx, n, regOpts) if err != nil { return fmt.Errorf("pull model manifest: %s", err) } @@ -591,7 +591,7 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu // Use fast transfer for models with tensor layers (many small blobs) if hasTensorLayers(layers) { - if err := pullWithTransfer(ctx, n, layers, mf, regOpts, fn); err != nil { + if err := pullWithTransfer(ctx, n, layers, manifestData, regOpts, fn); err != nil { return err } fn(api.ProgressResponse{Status: "success"}) @@ -639,11 +639,6 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu fn(api.ProgressResponse{Status: "writing manifest"}) - manifestJSON, err := json.Marshal(mf) - if err != nil { - return err - } - fp, err := manifest.PathForName(n) if err != nil { return err @@ -652,12 +647,14 @@ func PullModel(ctx context.Context, name string, regOpts *registryOptions, fn fu return err } - err = os.WriteFile(fp, manifestJSON, 0o644) + err = os.WriteFile(fp, manifestData, 0o644) if err != nil { slog.Info(fmt.Sprintf("couldn't write to %s", fp)) return err } + slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData)) + if !envconfig.NoPrune() && len(deleteMap) > 0 { fn(api.ProgressResponse{Status: "removing unused layers"}) if err := deleteUnusedLayers(deleteMap); err != nil { @@ -681,7 +678,7 @@ func hasTensorLayers(layers []manifest.Layer) bool { } // pullWithTransfer uses the simplified x/transfer package for downloading blobs. -func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer, mf *manifest.Manifest, regOpts *registryOptions, fn func(api.ProgressResponse)) error { +func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer, manifestData []byte, regOpts *registryOptions, fn func(api.ProgressResponse)) error { blobs := make([]transfer.Blob, len(layers)) for i, layer := range layers { blobs[i] = transfer.Blob{ @@ -738,10 +735,6 @@ func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer // Write manifest fn(api.ProgressResponse{Status: "writing manifest"}) - manifestJSON, err := json.Marshal(mf) - if err != nil { - return err - } fp, err := manifest.PathForName(n) if err != nil { @@ -751,7 +744,12 @@ func pullWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer return err } - return os.WriteFile(fp, manifestJSON, 0o644) + if err := os.WriteFile(fp, manifestData, 0o644); err != nil { + return err + } + + slog.Debug("manifest written", "path", fp, "sha256", fmt.Sprintf("%x", sha256.Sum256(manifestData)), "size", len(manifestData)) + return nil } // pushWithTransfer uses the simplified x/transfer package for uploading blobs and manifest. @@ -812,23 +810,28 @@ func pushWithTransfer(ctx context.Context, n model.Name, layers []manifest.Layer }) } -func pullModelManifest(ctx context.Context, n model.Name, regOpts *registryOptions) (*manifest.Manifest, error) { +func pullModelManifest(ctx context.Context, n model.Name, regOpts *registryOptions) (*manifest.Manifest, []byte, error) { requestURL := n.BaseURL().JoinPath("v2", n.DisplayNamespaceModel(), "manifests", n.Tag) headers := make(http.Header) headers.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, headers, nil, regOpts) if err != nil { - return nil, err + return nil, nil, err } defer resp.Body.Close() - var m manifest.Manifest - if err := json.NewDecoder(resp.Body).Decode(&m); err != nil { - return nil, err + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err } - return &m, err + var m manifest.Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, nil, err + } + + return &m, data, err } // GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer diff --git a/server/images_test.go b/server/images_test.go index 639cf8662..88f1d07c5 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -1,6 +1,10 @@ package server import ( + "crypto/sha256" + "fmt" + "net/http" + "net/http/httptest" "strings" "testing" @@ -289,3 +293,63 @@ func TestModelCheckCapabilities(t *testing.T) { }) } } + +func TestPullModelManifest(t *testing.T) { + cases := []struct { + name string + manifest string + }{ + { + name: "pretty printed", + manifest: `{ "schemaVersion": 2, "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { "digest": "sha256:abc", "mediaType": "application/vnd.docker.container.image.v1+json", "size": 50 }, + "layers": [{ "digest": "sha256:t1", "mediaType": "application/vnd.ollama.image.tensor", "size": 1024, "name": "model.weight" }] +}`, + }, + { + name: "non-standard field order", + manifest: `{"layers":[{"size":999,"digest":"sha256:def","mediaType":"application/vnd.ollama.image.model"}],"schemaVersion":2,"config":{"size":50,"digest":"sha256:abc","mediaType":"application/vnd.docker.container.image.v1+json"},"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(tt.manifest)) + })) + defer ts.Close() + + n := model.ParseName("test/model:latest") + n.ProtocolScheme = "http" + n.Host = strings.TrimPrefix(ts.URL, "http://") + + mf, data, err := pullModelManifest(t.Context(), n, ®istryOptions{}) + if err != nil { + t.Fatal(err) + } + + // Raw bytes must be byte-for-byte identical to what the server sent + if string(data) != tt.manifest { + t.Fatalf("raw bytes differ from server response") + } + + // SHA256 of returned data must match the expected registry digest + expectedDigest := fmt.Sprintf("%x", sha256.Sum256([]byte(tt.manifest))) + gotDigest := fmt.Sprintf("%x", sha256.Sum256(data)) + if gotDigest != expectedDigest { + t.Fatalf("digest mismatch\ngot: %s\nwant: %s", gotDigest, expectedDigest) + } + + // Parsed manifest must still be usable + if mf.SchemaVersion != 2 { + t.Fatalf("schemaVersion = %d, want 2", mf.SchemaVersion) + } + if mf.Config.Digest == "" { + t.Fatal("config digest is empty") + } + if len(mf.Layers) == 0 { + t.Fatal("expected at least one layer") + } + }) + } +}