starred/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-04-26 10:45:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merged latest ollama 0.6.2 and nasrally's Flash Attention patches (#5)

Browse Source 
* readme: add Ellama to list of community integrations (#9800)

* readme: add screenpipe to community integrations (#9786)

* Add support for ROCm gfx1151 (#9773)

* conditionally enable parallel pipelines

* sample: make mutations in transforms explicit (#9743)

* updated minP to use early exit making use of sorted tokens

* ml/backend/ggml: allocate memory with malloc when loading model (#9822)

* runner: remove cache prompt flag from ollama runner (#9826)

We do not need to bypass the prompt caching in the ollama runner yet, as
only embedding models needed to bypass the prompt caching. When embedding
models are implemented they can skip initializing this cache completely.

* ollamarunner: Check for minBatch of context space when shifting

Models can specify that a group of inputs need to be handled a single
batch. However, context shifting didn't respect this and could trigger
a break anyways. In this case, we should instead trigger a context
shift earlier so that it occurs before the grouped batch.

Note that there still some corner cases:
 - A long prompt that exceeds the context window can get truncated
   in the middle of an image. With the current models, this will
   result in the model not recognizing the image at all, which is
   pretty much the expected result with truncation.
 - The context window is set less than the minimum batch size. The
   only solution to this is to refuse to load the model with these
   settings. However, this can never occur with current models and
   default settings.

Since users are unlikely to run into these scenarios, fixing them is
left as a follow up.

* Applied latest patches from McBane87

See this for details: https://github.com/whyvl/ollama-vulkan/issues/7#issuecomment-2708820861

Signed-off-by: Vadim Grinco <vadim@grinco.eu>

* Add ability to enable flash attention on vulkan (#4)

* discover: add flash attention handling for vulkan
* envconfig: fix typo in config.go

As part of the process some code was refactored and I added a new field
FlashAttention to GpuInfo since the previous solution didn't allow for a
granular check via vulkan extensions. As a side effect, this now allows
for granular per-device FA support checking in other places

---------

Signed-off-by: Vadim Grinco <vadim@grinco.eu>
Co-authored-by: zeo <108888572+zeozeozeo@users.noreply.github.com>
Co-authored-by: Louis Beaumont <louis.beaumont@gmail.com>
Co-authored-by: Daniel Hiltgen <dhiltgen@users.noreply.github.com>
Co-authored-by: Michael Yang <mxyng@pm.me>
Co-authored-by: Parth Sareen <parth.sareen@ollama.com>
Co-authored-by: Jeffrey Morgan <jmorganca@gmail.com>
Co-authored-by: Bruce MacDonald <brucewmacdonald@gmail.com>
Co-authored-by: Jesse Gross <jesse@ollama.com>
Co-authored-by: Nikita <50599445+nasrally@users.noreply.github.com>
This commit is contained in:
Vadim Grinco
2025-03-23 12:27:37 +01:00
committed by GitHub
parent 4aa7e5ed79
commit 45dbd14645
15 changed files with 348 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
discover/gpu_info_vulkan.c
View File

@@ -24,18 +24,32 @@ int check_perfmon(vk_handle_t* rh) {
return 0;
}
int support_memory_budget(vk_handle_t* rh, VkPhysicalDevice device) {
int is_extension_supported(vk_handle_t* rh, VkPhysicalDevice device, char* extension) {
VkPhysicalDeviceProperties properties;
(*rh->vkGetPhysicalDeviceProperties)(device, &properties);
uint32_t extensionCount;
(*rh->vkEnumerateDeviceExtensionProperties)(device, NULL, &extensionCount, NULL);
if (extensionCount == 0) {
return 0;
}
VkExtensionProperties* extensions = malloc(extensionCount * sizeof(VkExtensionProperties));
if (extensions == NULL) {
return 0;
}
(*rh->vkEnumerateDeviceExtensionProperties)(device, NULL, &extensionCount, extensions);
for (int j = 0; j < extensionCount; j++) {
if (strcmp(extensions[j].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME) == 0) {
if (strcmp(extensions[j].extensionName, extension) == 0) {
free(extensions);
return 1;
}
}
free(extensions);
return 0;
}
@@ -125,6 +139,7 @@ void vk_init(char* vk_lib_path, char* cap_lib_path, vk_init_resp_t *resp) {
}
VkInstance instance;
VkApplicationInfo appInfo = {};
appInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
appInfo.pNext = NULL;
@@ -133,6 +148,7 @@ void vk_init(char* vk_lib_path, char* cap_lib_path, vk_init_resp_t *resp) {
appInfo.pEngineName = "No Engine";
appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0);
appInfo.apiVersion = VK_API_VERSION_1_2;
VkInstanceCreateInfo createInfo = {};
createInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
createInfo.pNext = NULL;
@@ -141,6 +157,7 @@ void vk_init(char* vk_lib_path, char* cap_lib_path, vk_init_resp_t *resp) {
const char* extensions[] = { VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME };
createInfo.ppEnabledExtensionNames = extensions;
createInfo.pApplicationInfo = &appInfo;
VkResult result = (*resp->ch.vkCreateInstance)(&createInfo, NULL, &instance);
if (result != VK_SUCCESS) {
resp->err = strdup("failed to create instance");
@@ -160,25 +177,63 @@ void vk_init(char* vk_lib_path, char* cap_lib_path, vk_init_resp_t *resp) {
resp->num_devices = deviceCount;
}
int vk_check_flash_attention(vk_handle_t rh, int i) {
VkInstance instance = rh.vk;
uint32_t deviceCount = rh.num_devices;
VkPhysicalDevice* devices = malloc(deviceCount * sizeof(VkPhysicalDevice));
if (devices == NULL) {
return 0;
}
VkResult result = (*rh.vkEnumeratePhysicalDevices)(instance, &deviceCount, devices);
if (result != VK_SUCCESS) {
free(devices);
return 0;
}
VkPhysicalDeviceProperties properties;
(*rh.vkGetPhysicalDeviceProperties)(devices[i], &properties);
int supports_nv_coopmat2 = is_extension_supported(&rh, devices[i], VK_NV_COOPERATIVE_MATRIX_2_EXTENSION_NAME);
if (!supports_nv_coopmat2) {
free(devices);
return 1;
}
free(devices);
return 0;
}
void vk_check_vram(vk_handle_t rh, int i, mem_info_t *resp) {
VkInstance instance = rh.vk;
uint32_t deviceCount = rh.num_devices;
VkPhysicalDevice* devices = malloc(deviceCount * sizeof(VkPhysicalDevice));
if (devices == NULL) {
resp->err = strdup("memory allocation failed for devices array");
return;
}
VkResult result = (*rh.vkEnumeratePhysicalDevices)(instance, &deviceCount, devices);
if (result != VK_SUCCESS) {
free(devices);
resp->err = strdup("failed to enumerate physical devices");
return;
}
VkPhysicalDeviceProperties properties;
(*rh.vkGetPhysicalDeviceProperties)(devices[i], &properties);
int supports_budget = support_memory_budget(&rh, devices[i]);
int supports_budget = is_extension_supported(&rh, devices[i], VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
if (!supports_budget) {
free(devices);
resp->err = strdup("device does not support memory budget");
return;
}
if (properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) {
free(devices);
resp->err = strdup("device is a CPU");
return;
}
@@ -204,6 +259,8 @@ void vk_check_vram(vk_handle_t rh, int i, mem_info_t *resp) {
}
}
free(devices);
resp->err = NULL;
snprintf(&resp->gpu_id[0], GPU_ID_LEN, "%d", i);
resp->gpu_name[GPU_NAME_LEN - 1] = '\0';
@@ -220,6 +277,7 @@ void vk_release(vk_handle_t rh) {
(*rh.vkDestroyInstance)(rh.vk, NULL);
UNLOAD_LIBRARY(rh.vk_handle);
rh.vk_handle = NULL;
#ifdef __linux__
LOG(rh.verbose, "releasing libcap library\n");
UNLOAD_LIBRARY(rh.cap_handle);