mirror of
https://github.com/ollama/ollama.git
synced 2026-04-17 15:53:27 +02:00
mlx: Improve M5 performance with NAX (#15345)
* mlx: Improve M5 performance with NAX This modifies the Mac release to now have 2 builds of MLX for broader compatibility while supporting the latest M5 hardware features. NAX requires building with xcode 26.2 and targetting support only for OS v26 and up. Since we want to support older MacOS versions as well, we now need 2 different MLX builds and runtime detection logic to select the optimal version. The newer build will detect NAX missing at runtime, so it is safe to run on pre M5 macs. * mac: prevent generate on cross-compiles For some versions of Xcode, cmake builds are failing due to header problems in cross-compiling during the generate phase. Since generate is producing arch independent generated output, we can skip this during cross-compiling.
This commit is contained in:
Daniel Hiltgen
GitHub
2
.github/workflows/release.yaml
vendored
2
.github/workflows/release.yaml
vendored
@@ -27,7 +27,7 @@ jobs:
|
||||
echo vendorsha=$(make -f Makefile.sync print-base) | tee -a $GITHUB_OUTPUT
|
||||
|
||||
darwin-build:
|
||||
runs-on: macos-14-xlarge
|
||||
runs-on: macos-26-xlarge
|
||||
environment: release
|
||||
needs: setup-environment
|
||||
env:
|
||||
|
||||
@@ -94,7 +94,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
|
||||
}
|
||||
var dirs []string
|
||||
if dir != "" {
|
||||
if requested != "" && filepath.Base(dir) != requested {
|
||||
if requested != "" && !strings.HasPrefix(requested, "mlx_") && filepath.Base(dir) != requested {
|
||||
slog.Debug("skipping available library at user's request", "requested", requested, "libDir", dir)
|
||||
continue
|
||||
} else if jetpack != "" && filepath.Base(dir) != "cuda_"+jetpack {
|
||||
|
||||
@@ -62,21 +62,61 @@ _build_darwin() {
|
||||
MLX_CGO_CFLAGS="-O3 -mmacosx-version-min=14.0"
|
||||
MLX_CGO_LDFLAGS="-ldl -lc++ -framework Accelerate -mmacosx-version-min=14.0"
|
||||
else
|
||||
BUILD_DIR=build
|
||||
cmake --preset MLX \
|
||||
-DOLLAMA_RUNNER_DIR=./ \
|
||||
# CPU backend (ggml-cpu, installed flat to lib/ollama/)
|
||||
BUILD_DIR_CPU=build/arm64-cpu
|
||||
status "Building arm64 CPU backend"
|
||||
cmake -S . -B $BUILD_DIR_CPU \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX
|
||||
cmake --build --preset MLX --parallel
|
||||
cmake --build $BUILD_DIR_CPU --target ggml-cpu --parallel
|
||||
cmake --install $BUILD_DIR_CPU --component CPU
|
||||
|
||||
# Build MLX twice for arm64
|
||||
# Metal 3.x build (backward compatible, macOS 14+)
|
||||
BUILD_DIR=build/metal-v3
|
||||
status "Building MLX Metal v3 (macOS 14+)"
|
||||
cmake -S . -B $BUILD_DIR \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DMLX_ENGINE=ON \
|
||||
-DOLLAMA_RUNNER_DIR=mlx_metal_v3 \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX
|
||||
cmake --build $BUILD_DIR --target mlx mlxc --parallel
|
||||
cmake --install $BUILD_DIR --component MLX
|
||||
# Use default CGO flags from mlx.go for arm64
|
||||
|
||||
# Metal 4.x build (NAX-enabled, macOS 26+)
|
||||
# Only possible with Xcode 26+ SDK; skip on older toolchains.
|
||||
SDK_MAJOR=$(xcrun --show-sdk-version 2>/dev/null | cut -d. -f1)
|
||||
if [ "${SDK_MAJOR:-0}" -ge 26 ]; then
|
||||
V3_DEPS=$BUILD_DIR/_deps
|
||||
BUILD_DIR_V4=build/metal-v4
|
||||
status "Building MLX Metal v4 (macOS 26+, NAX)"
|
||||
cmake -S . -B $BUILD_DIR_V4 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DMLX_ENGINE=ON \
|
||||
-DOLLAMA_RUNNER_DIR=mlx_metal_v4 \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=26.0 \
|
||||
-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
|
||||
-DFETCHCONTENT_SOURCE_DIR_MLX=$V3_DEPS/mlx-src \
|
||||
-DFETCHCONTENT_SOURCE_DIR_MLX-C=$V3_DEPS/mlx-c-src \
|
||||
-DFETCHCONTENT_SOURCE_DIR_JSON=$V3_DEPS/json-src \
|
||||
-DFETCHCONTENT_SOURCE_DIR_FMT=$V3_DEPS/fmt-src \
|
||||
-DFETCHCONTENT_SOURCE_DIR_METAL_CPP=$V3_DEPS/metal_cpp-src
|
||||
cmake --build $BUILD_DIR_V4 --target mlx mlxc --parallel
|
||||
cmake --install $BUILD_DIR_V4 --component MLX
|
||||
else
|
||||
status "Skipping MLX Metal v4 (SDK $SDK_MAJOR < 26, need Xcode 26+)"
|
||||
fi
|
||||
|
||||
# Use the v3 build for CGO linking (compatible with both)
|
||||
MLX_CGO_CFLAGS="-O3 -mmacosx-version-min=14.0"
|
||||
MLX_CGO_LDFLAGS="-lc++ -framework Metal -framework Foundation -framework Accelerate -mmacosx-version-min=14.0"
|
||||
fi
|
||||
GOOS=darwin GOARCH=$ARCH CGO_ENABLED=1 CGO_CFLAGS="$MLX_CGO_CFLAGS" CGO_LDFLAGS="$MLX_CGO_LDFLAGS" go build -o $INSTALL_PREFIX .
|
||||
# Copy MLX libraries to same directory as executable for dlopen
|
||||
cp $INSTALL_PREFIX/lib/ollama/libmlxc.dylib $INSTALL_PREFIX/
|
||||
cp $INSTALL_PREFIX/lib/ollama/libmlx.dylib $INSTALL_PREFIX/
|
||||
# MLX libraries stay in lib/ollama/ (flat or variant subdirs).
|
||||
# The runtime discovery in dynamic.go searches lib/ollama/ relative
|
||||
# to the executable, including mlx_* subdirectories.
|
||||
done
|
||||
}
|
||||
|
||||
@@ -87,8 +127,9 @@ _sign_darwin() {
|
||||
chmod +x dist/darwin/ollama
|
||||
|
||||
if [ -n "$APPLE_IDENTITY" ]; then
|
||||
for F in dist/darwin/ollama dist/darwin-*/lib/ollama/*; do
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime $F
|
||||
for F in dist/darwin/ollama dist/darwin-*/lib/ollama/* dist/darwin-*/lib/ollama/mlx_metal_v*/*; do
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime "$F"
|
||||
done
|
||||
|
||||
# create a temporary zip for notarization
|
||||
@@ -101,6 +142,7 @@ _sign_darwin() {
|
||||
status "Creating universal tarball..."
|
||||
tar -cf dist/ollama-darwin.tar --strip-components 2 dist/darwin/ollama
|
||||
tar -rf dist/ollama-darwin.tar --strip-components 4 dist/darwin-amd64/lib/
|
||||
tar -rf dist/ollama-darwin.tar --strip-components 4 dist/darwin-arm64/lib/
|
||||
gzip -9vc <dist/ollama-darwin.tar >dist/ollama-darwin.tgz
|
||||
}
|
||||
|
||||
@@ -154,31 +196,84 @@ _build_macapp() {
|
||||
mkdir -p dist/Ollama.app/Contents/Resources
|
||||
if [ -d dist/darwin-amd64 ]; then
|
||||
lipo -create -output dist/Ollama.app/Contents/Resources/ollama dist/darwin-amd64/ollama dist/darwin-arm64/ollama
|
||||
for F in dist/darwin-amd64/lib/ollama/*mlx*.dylib ; do
|
||||
lipo -create -output dist/darwin/$(basename $F) $F dist/darwin-arm64/lib/ollama/$(basename $F)
|
||||
|
||||
# Copy .so files from both architectures (names don't collide: arm64=libggml-cpu.so, amd64=libggml-cpu-*.so)
|
||||
cp dist/darwin-arm64/lib/ollama/*.so dist/Ollama.app/Contents/Resources/ 2>/dev/null || true
|
||||
cp dist/darwin-amd64/lib/ollama/*.so dist/Ollama.app/Contents/Resources/ 2>/dev/null || true
|
||||
# Lipo common dylibs into universal binaries, copy amd64-only ones as-is
|
||||
for F in dist/darwin-amd64/lib/ollama/*.dylib; do
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
BASE=$(basename "$F")
|
||||
if [ -f "dist/darwin-arm64/lib/ollama/$BASE" ]; then
|
||||
lipo -create -output "dist/Ollama.app/Contents/Resources/$BASE" "$F" "dist/darwin-arm64/lib/ollama/$BASE"
|
||||
else
|
||||
cp "$F" dist/Ollama.app/Contents/Resources/
|
||||
fi
|
||||
done
|
||||
# Recreate ggml-base symlinks
|
||||
(cd dist/Ollama.app/Contents/Resources && ln -sf libggml-base.0.0.0.dylib libggml-base.0.dylib && ln -sf libggml-base.0.dylib libggml-base.dylib) 2>/dev/null || true
|
||||
|
||||
# MLX Metal variant subdirs from arm64
|
||||
for VARIANT in dist/darwin-arm64/lib/ollama/mlx_metal_v*/; do
|
||||
[ -d "$VARIANT" ] || continue
|
||||
VNAME=$(basename "$VARIANT")
|
||||
DEST=dist/Ollama.app/Contents/Resources/$VNAME
|
||||
mkdir -p "$DEST"
|
||||
if [ "$VNAME" = "mlx_metal_v3" ]; then
|
||||
# v3: lipo amd64 flat + arm64 v3 into universal dylibs
|
||||
for LIB in libmlx.dylib libmlxc.dylib; do
|
||||
if [ -f "dist/darwin-amd64/lib/ollama/$LIB" ] && [ -f "$VARIANT$LIB" ]; then
|
||||
lipo -create -output "$DEST/$LIB" "dist/darwin-amd64/lib/ollama/$LIB" "$VARIANT$LIB"
|
||||
elif [ -f "$VARIANT$LIB" ]; then
|
||||
cp "$VARIANT$LIB" "$DEST/"
|
||||
fi
|
||||
done
|
||||
# Copy remaining files (metallib) from arm64 v3
|
||||
for F in "$VARIANT"*; do
|
||||
case "$(basename "$F")" in *.dylib) continue ;; esac
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
cp "$F" "$DEST/"
|
||||
done
|
||||
else
|
||||
# v4+: arm64-only, copy all non-symlink files
|
||||
for F in "$VARIANT"*; do
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
cp "$F" "$DEST/"
|
||||
done
|
||||
fi
|
||||
done
|
||||
cp dist/darwin-*/lib/ollama/*.so dist/darwin-*/lib/ollama/*.dylib dist/Ollama.app/Contents/Resources/
|
||||
cp dist/darwin/*.dylib dist/Ollama.app/Contents/Resources/
|
||||
# Copy MLX metallib (architecture-independent, just use arm64 version)
|
||||
cp dist/darwin-arm64/lib/ollama/*.metallib dist/Ollama.app/Contents/Resources/ 2>/dev/null || true
|
||||
else
|
||||
cp -a dist/darwin/ollama dist/Ollama.app/Contents/Resources/ollama
|
||||
cp dist/darwin/*.so dist/darwin/*.dylib dist/Ollama.app/Contents/Resources/
|
||||
# arm64-only build: copy variant subdirs directly
|
||||
for VARIANT in dist/darwin-arm64/lib/ollama/mlx_metal_v*/; do
|
||||
[ -d "$VARIANT" ] || continue
|
||||
VNAME=$(basename "$VARIANT")
|
||||
mkdir -p dist/Ollama.app/Contents/Resources/$VNAME
|
||||
cp "$VARIANT"* dist/Ollama.app/Contents/Resources/$VNAME/ 2>/dev/null || true
|
||||
done
|
||||
# CPU backend libs (ggml-base, ggml-cpu) are flat in lib/ollama/
|
||||
cp dist/darwin-arm64/lib/ollama/*.so dist/Ollama.app/Contents/Resources/ 2>/dev/null || true
|
||||
for F in dist/darwin-arm64/lib/ollama/*.dylib; do
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
cp "$F" dist/Ollama.app/Contents/Resources/
|
||||
done
|
||||
(cd dist/Ollama.app/Contents/Resources && ln -sf libggml-base.0.0.0.dylib libggml-base.0.dylib && ln -sf libggml-base.0.dylib libggml-base.dylib) 2>/dev/null || true
|
||||
fi
|
||||
chmod a+x dist/Ollama.app/Contents/Resources/ollama
|
||||
|
||||
# Sign
|
||||
if [ -n "$APPLE_IDENTITY" ]; then
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime dist/Ollama.app/Contents/Resources/ollama
|
||||
for lib in dist/Ollama.app/Contents/Resources/*.so dist/Ollama.app/Contents/Resources/*.dylib dist/Ollama.app/Contents/Resources/*.metallib ; do
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime ${lib}
|
||||
for lib in dist/Ollama.app/Contents/Resources/*.so dist/Ollama.app/Contents/Resources/*.dylib dist/Ollama.app/Contents/Resources/*.metallib dist/Ollama.app/Contents/Resources/mlx_metal_v*/*.dylib dist/Ollama.app/Contents/Resources/mlx_metal_v*/*.metallib dist/Ollama.app/Contents/Resources/mlx_metal_v*/*.so; do
|
||||
[ -f "$lib" ] || continue
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier ai.ollama.ollama --options=runtime "$lib"
|
||||
done
|
||||
codesign -f --timestamp -s "$APPLE_IDENTITY" --identifier com.electron.ollama --deep --options=runtime dist/Ollama.app
|
||||
fi
|
||||
|
||||
rm -f dist/Ollama-darwin.zip
|
||||
ditto -c -k --norsrc --keepParent dist/Ollama.app dist/Ollama-darwin.zip
|
||||
(cd dist/Ollama.app/Contents/Resources/; tar -cf - ollama *.so *.dylib *.metallib 2>/dev/null) | gzip -9vc > dist/ollama-darwin.tgz
|
||||
(cd dist/Ollama.app/Contents/Resources/; tar -cf - ollama *.so *.dylib *.metallib mlx_metal_v*/ 2>/dev/null) | gzip -9vc > dist/ollama-darwin.tgz
|
||||
|
||||
# Notarize and Staple
|
||||
if [ -n "$APPLE_IDENTITY" ]; then
|
||||
|
||||
@@ -99,25 +99,35 @@ file(GLOB _mlx_c_hdrs "${mlx-c_SOURCE_DIR}/mlx/c/*.h")
|
||||
file(COPY ${_mlx_c_hdrs} DESTINATION "${CMAKE_SOURCE_DIR}/x/mlxrunner/mlx/include/mlx/c/")
|
||||
|
||||
# Regenerate Go/C shim wrappers from the (possibly updated) headers.
|
||||
find_program(GO_EXECUTABLE go REQUIRED)
|
||||
message(STATUS "Regenerating MLX Go wrappers")
|
||||
# Skip during cross-compilation — the generated files are arch-independent.
|
||||
if(CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_SYSTEM_PROCESSOR OR NOT APPLE)
|
||||
find_program(GO_EXECUTABLE go REQUIRED)
|
||||
message(STATUS "Regenerating MLX Go wrappers")
|
||||
|
||||
# Go's cgo splits CC on whitespace, so a CC like "C:/Program Files/…/cl.exe"
|
||||
# (set by cmake on Windows) breaks with "C:/Program" not found. Clear CC
|
||||
# when it contains spaces so cgo falls back to its default (gcc).
|
||||
if(WIN32 AND "$ENV{CC}" MATCHES " ")
|
||||
# CGo's probe compilation is sensitive to CGO_CFLAGS/CGO_CXXFLAGS and CC.
|
||||
# Clear them so go generate uses default compiler settings:
|
||||
# - On Windows, CC may contain spaces (e.g., "C:/Program Files/.../cl.exe")
|
||||
# which breaks CGo's CC parsing.
|
||||
# - On macOS, CGO_CFLAGS with -mmacosx-version-min breaks header search
|
||||
# when cmake also sets CMAKE_OSX_DEPLOYMENT_TARGET.
|
||||
set(_SAVE_CC "$ENV{CC}")
|
||||
set(_SAVE_CGO_CFLAGS "$ENV{CGO_CFLAGS}")
|
||||
set(_SAVE_CGO_CXXFLAGS "$ENV{CGO_CXXFLAGS}")
|
||||
set(ENV{CC} "")
|
||||
endif()
|
||||
set(ENV{CGO_CFLAGS} "")
|
||||
set(ENV{CGO_CXXFLAGS} "")
|
||||
|
||||
execute_process(
|
||||
COMMAND ${GO_EXECUTABLE} generate ./x/...
|
||||
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
execute_process(
|
||||
COMMAND ${GO_EXECUTABLE} generate ./x/...
|
||||
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
|
||||
if(DEFINED _SAVE_CC)
|
||||
set(ENV{CC} "${_SAVE_CC}")
|
||||
set(ENV{CGO_CFLAGS} "${_SAVE_CGO_CFLAGS}")
|
||||
set(ENV{CGO_CXXFLAGS} "${_SAVE_CGO_CXXFLAGS}")
|
||||
else()
|
||||
message(STATUS "Skipping MLX Go wrapper generation (cross-compiling)")
|
||||
endif()
|
||||
|
||||
# For local dev builds, override MLX_VERSION with git describe output
|
||||
|
||||
@@ -12,23 +12,28 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var initError error
|
||||
var initLoadError string
|
||||
var initLoadedPath string
|
||||
|
||||
// CheckInit returns any error that occurred during MLX dynamic library initialization.
|
||||
// When initialization failed, detailed load errors are logged to help diagnose the issue.
|
||||
func CheckInit() error {
|
||||
if initLoadedPath != "" {
|
||||
slog.Debug("MLX dynamic library loaded", "path", initLoadedPath)
|
||||
}
|
||||
if initError != nil && initLoadError != "" {
|
||||
slog.Error(initLoadError)
|
||||
}
|
||||
return initError
|
||||
}
|
||||
|
||||
// tryLoadFromDir searches a directory for the mlxc shared library and tries to load it.
|
||||
// Returns true if the library was successfully loaded.
|
||||
// tryLoadFromDir searches a directory for the mlxc shared library and loads it.
|
||||
func tryLoadFromDir(dir string) bool {
|
||||
// On Windows, MSVC produces mlxc.dll (no lib prefix)
|
||||
// On Unix, it's libmlxc.so or libmlxc.dylib
|
||||
@@ -40,10 +45,8 @@ func tryLoadFromDir(dir string) bool {
|
||||
if err != nil || len(matches) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, match := range matches {
|
||||
path := filepath.Join(dir, match)
|
||||
|
||||
cPath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cPath))
|
||||
|
||||
@@ -52,92 +55,73 @@ func tryLoadFromDir(dir string) bool {
|
||||
initLoadError = fmt.Sprintf("failed to load MLX dynamic library: path=%s", path)
|
||||
continue
|
||||
}
|
||||
|
||||
if C.mlx_dynamic_load_symbols(handle) != 0 {
|
||||
initLoadError = fmt.Sprintf("failed to load MLX dynamic library symbols: path=%s", path)
|
||||
C.mlx_dynamic_unload(&handle)
|
||||
continue
|
||||
}
|
||||
|
||||
initLoadedPath = path
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// tryLoadByName attempts to load the library using just its name,
|
||||
// allowing the system to use rpath, LD_LIBRARY_PATH, or standard search paths.
|
||||
// Returns true if the library was successfully loaded.
|
||||
func tryLoadByName() bool {
|
||||
libraryName := "libmlxc.dylib"
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
libraryName = "mlxc.dll"
|
||||
case "linux":
|
||||
libraryName = "libmlxc.so"
|
||||
// libOllamaRoots returns candidate directories for MLX dynamic libraries.
|
||||
// Production: exe_dir/lib/ollama (dist tarball) and exe_dir (app bundle).
|
||||
// Development: build/lib/ollama and build/*/lib/ollama.
|
||||
func libOllamaRoots() []string {
|
||||
var roots []string
|
||||
|
||||
// Production paths relative to executable
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if eval, err := filepath.EvalSymlinks(exe); err == nil {
|
||||
exe = eval
|
||||
}
|
||||
exeDir := filepath.Dir(exe)
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
roots = append(roots, filepath.Join(exeDir, "lib", "ollama"))
|
||||
roots = append(roots, exeDir) // app bundle: Contents/Resources/
|
||||
case "linux":
|
||||
roots = append(roots, filepath.Join(exeDir, "..", "lib", "ollama"))
|
||||
case "windows":
|
||||
roots = append(roots, filepath.Join(exeDir, "lib", "ollama"))
|
||||
}
|
||||
}
|
||||
|
||||
cPath := C.CString(libraryName)
|
||||
defer C.free(unsafe.Pointer(cPath))
|
||||
|
||||
var handle C.mlx_dynamic_handle
|
||||
if C.mlx_dynamic_load(&handle, cPath) != 0 {
|
||||
return false
|
||||
}
|
||||
if C.mlx_dynamic_load_symbols(handle) != 0 {
|
||||
C.mlx_dynamic_unload(&handle)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func init() {
|
||||
switch runtime.GOOS {
|
||||
case "darwin", "linux", "windows":
|
||||
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
// Try OLLAMA_LIBRARY_PATH first, including mlx_* subdirectories
|
||||
if paths, ok := os.LookupEnv("OLLAMA_LIBRARY_PATH"); ok {
|
||||
for _, dir := range filepath.SplitList(paths) {
|
||||
if tryLoadFromDir(dir) {
|
||||
return
|
||||
}
|
||||
if mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx_*")); err == nil {
|
||||
for _, mlxDir := range mlxDirs {
|
||||
if tryLoadFromDir(mlxDir) {
|
||||
return
|
||||
}
|
||||
// Development paths: build/lib/ollama and build/*/lib/ollama.
|
||||
// Reverse-sort and filter the glob results so higher-versioned Metal
|
||||
// builds (e.g., metal-v4) are tried before lower ones (metal-v3),
|
||||
// and incompatible variants are skipped. Without this, alphabetical
|
||||
// order would always pick v3 over v4 in dev builds.
|
||||
for _, base := range repoBuildDirs() {
|
||||
roots = append(roots, filepath.Join(base, "lib", "ollama"))
|
||||
if matches, err := filepath.Glob(filepath.Join(base, "*", "lib", "ollama")); err == nil {
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(matches)))
|
||||
for _, m := range matches {
|
||||
// Extract the build dir name (e.g., "metal-v4" from "build/metal-v4/lib/ollama")
|
||||
rel, _ := filepath.Rel(base, m)
|
||||
variant := strings.SplitN(rel, string(filepath.Separator), 2)[0]
|
||||
if isCompatibleMLXVariant(variant) {
|
||||
roots = append(roots, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try loading via rpath/standard library search
|
||||
if tryLoadByName() {
|
||||
return
|
||||
}
|
||||
|
||||
// Build search paths: executable directory, then build directories
|
||||
var searchDirs []string
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if eval, err := filepath.EvalSymlinks(exe); err == nil {
|
||||
exe = eval
|
||||
}
|
||||
searchDirs = append(searchDirs, filepath.Dir(exe))
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
// repoBuildDirs returns candidate build/ directories relative to cwd and repo root.
|
||||
func repoBuildDirs() []string {
|
||||
var dirs []string
|
||||
if cwd, err := os.Getwd(); err == nil {
|
||||
searchDirs = append(searchDirs, filepath.Join(cwd, "build", "lib", "ollama"))
|
||||
|
||||
// Walk up from cwd to find the repo root (containing go.mod) so that
|
||||
// tests running from a package subdirectory can find the build output.
|
||||
dirs = append(dirs, filepath.Join(cwd, "build"))
|
||||
for dir := cwd; ; {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
if dir != cwd {
|
||||
searchDirs = append(searchDirs, filepath.Join(dir, "build", "lib", "ollama"))
|
||||
dirs = append(dirs, filepath.Join(dir, "build"))
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -148,22 +132,122 @@ func init() {
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
// Also scan mlx_* subdirectories within each search dir
|
||||
var expanded []string
|
||||
for _, dir := range searchDirs {
|
||||
expanded = append(expanded, dir)
|
||||
if mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx_*")); err == nil {
|
||||
expanded = append(expanded, mlxDirs...)
|
||||
}
|
||||
}
|
||||
|
||||
for _, dir := range expanded {
|
||||
if tryLoadFromDir(dir) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
initError = fmt.Errorf("failed to load MLX dynamic library (searched: %v)", searchDirs)
|
||||
slog.Debug("MLX dynamic library not available", "error", initError)
|
||||
return dirs
|
||||
}
|
||||
|
||||
// prependLibraryPath prepends dir to the platform's dynamic library search
|
||||
// path so the linker finds colocated libmlx before any stale copies.
|
||||
// Called once after successful library load.
|
||||
func prependLibraryPath(dir string) {
|
||||
var envVar string
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
envVar = "DYLD_LIBRARY_PATH"
|
||||
case "linux":
|
||||
envVar = "LD_LIBRARY_PATH"
|
||||
default:
|
||||
return
|
||||
}
|
||||
if existing := os.Getenv(envVar); existing != "" {
|
||||
os.Setenv(envVar, dir+string(filepath.ListSeparator)+existing)
|
||||
} else {
|
||||
os.Setenv(envVar, dir)
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
switch runtime.GOOS {
|
||||
case "darwin", "linux", "windows":
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
// OLLAMA_LLM_LIBRARY overrides variant selection (e.g., "mlx_metal_v3").
|
||||
// When set to an mlx_* value, only that specific subdir is tried.
|
||||
// The GGML runner ignores mlx_* values (see discover/runner.go).
|
||||
forcedVariant, _ := os.LookupEnv("OLLAMA_LLM_LIBRARY")
|
||||
if forcedVariant != "" && !strings.HasPrefix(forcedVariant, "mlx_") {
|
||||
forcedVariant = "" // not an MLX variant, ignore
|
||||
}
|
||||
|
||||
found := findMLXLibrary(forcedVariant)
|
||||
if !found {
|
||||
initError = fmt.Errorf("failed to load MLX dynamic library (searched: %v)", libOllamaRoots())
|
||||
return
|
||||
}
|
||||
|
||||
prependLibraryPath(filepath.Dir(initLoadedPath))
|
||||
}
|
||||
|
||||
func findMLXLibrary(forcedVariant string) bool {
|
||||
for _, root := range libOllamaRoots() {
|
||||
if forcedVariant != "" {
|
||||
if tryLoadFromDir(filepath.Join(root, forcedVariant)) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if tryLoadFromMLXSubdirs(root) {
|
||||
return true
|
||||
}
|
||||
if tryLoadFromDir(root) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// tryLoadFromMLXSubdirs globs for mlx_* subdirs within dir, filters out
|
||||
// incompatible variants, tries the remainder in reverse sorted order (so
|
||||
// higher-versioned variants are preferred), and returns true on first
|
||||
// successful load.
|
||||
func tryLoadFromMLXSubdirs(dir string) bool {
|
||||
mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx_*"))
|
||||
if err != nil || len(mlxDirs) == 0 {
|
||||
return false
|
||||
}
|
||||
// Reverse sort: mlx_metal_v4 before mlx_metal_v3, mlx_cuda_v13 before v12
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(mlxDirs)))
|
||||
for _, mlxDir := range mlxDirs {
|
||||
if !isCompatibleMLXVariant(filepath.Base(mlxDir)) {
|
||||
slog.Debug("skipping incompatible MLX variant", "dir", mlxDir)
|
||||
continue
|
||||
}
|
||||
if tryLoadFromDir(mlxDir) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isCompatibleMLXVariant checks whether an MLX variant directory is
|
||||
// compatible with the current OS. On macOS, dlopen does NOT enforce
|
||||
// the deployment target for dynamically loaded libraries, so we must
|
||||
// check compatibility ourselves to avoid loading Metal 4.x shaders
|
||||
// on a Metal 3.x driver.
|
||||
func isCompatibleMLXVariant(name string) bool {
|
||||
if runtime.GOOS != "darwin" {
|
||||
return true // non-macOS variants use dlopen failure for filtering
|
||||
}
|
||||
// Metal variant naming:
|
||||
// Production: mlx_metal_v3, mlx_metal_v4
|
||||
// Dev build: metal-v3, metal-v4
|
||||
var verStr string
|
||||
switch {
|
||||
case strings.HasPrefix(name, "mlx_metal_v"):
|
||||
verStr = strings.TrimPrefix(name, "mlx_metal_v")
|
||||
case strings.HasPrefix(name, "metal-v"):
|
||||
verStr = strings.TrimPrefix(name, "metal-v")
|
||||
}
|
||||
if verStr != "" {
|
||||
metalVer, err := strconv.Atoi(verStr)
|
||||
if err != nil {
|
||||
return true // unknown format, try it
|
||||
}
|
||||
// Metal 4.x requires macOS 26+
|
||||
if metalVer >= 4 && macOSMajorVersion() < 26 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
17
x/mlxrunner/mlx/dynamic_darwin.go
Normal file
17
x/mlxrunner/mlx/dynamic_darwin.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package mlx
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func macOSMajorVersion() int {
|
||||
ver, err := syscall.Sysctl("kern.osproductversion")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
parts := strings.SplitN(ver, ".", 2)
|
||||
major, _ := strconv.Atoi(parts[0])
|
||||
return major
|
||||
}
|
||||
5
x/mlxrunner/mlx/dynamic_other.go
Normal file
5
x/mlxrunner/mlx/dynamic_other.go
Normal file
@@ -0,0 +1,5 @@
|
||||
//go:build !darwin
|
||||
|
||||
package mlx
|
||||
|
||||
func macOSMajorVersion() int { return 0 }
|
||||
Reference in New Issue
Block a user