starred/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-04-24 17:55:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

improve scratch buffer estimates

Browse Source
This commit is contained in:
Jeffrey Morgan
2024-01-18 00:53:17 -05:00
parent dc88cc3981
commit 2789ed31a7
2 changed files with 8 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
gpu/gpu.go
View File

@@ -190,13 +190,7 @@ func getCPUMem() (memInfo, error) {
func CheckVRAM() (int64, error) {
gpuInfo := GetGPUInfo()
if gpuInfo.FreeMemory > 0 && (gpuInfo.Library == "cuda" || gpuInfo.Library == "rocm") {
// leave 10% or 512MiB of VRAM free per GPU to handle unaccounted for overhead
overhead := gpuInfo.FreeMemory / 10
gpus := uint64(gpuInfo.DeviceCount)
if overhead < gpus*512*1024*1024 {
overhead = gpus * 512 * 1024 * 1024
}
return int64(gpuInfo.FreeMemory - overhead), nil
return int64(gpuInfo.FreeMemory), nil
}
return 0, fmt.Errorf("no GPU detected") // TODO - better handling of CPU based memory determiniation