diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 420571140..436bfe3bb 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3990,10 +3990,12 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); + static std::mutex mutex; + std::lock_guard lock(mutex); + if (vk_instance.devices[idx] == nullptr) { VK_LOG_DEBUG("Initializing new vk_device"); vk_device device = std::make_shared(); - vk_instance.devices[idx] = device; device->memory_logger = std::unique_ptr(new vk_memory_logger()); @@ -4891,14 +4893,14 @@ vk_device ggml_vk_get_device(size_t idx) { device->async_use_transfer_queue = false; } + device->fence = device->device.createFence({}); + device->buffer_type = { /* .iface = */ ggml_backend_vk_buffer_type_interface, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx), /* .context = */ new ggml_backend_vk_buffer_type_context{ device->name, device }, }; - device->fence = device->device.createFence({}); - device->idx = idx; device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr; @@ -4920,6 +4922,9 @@ vk_device ggml_vk_get_device(size_t idx) { device->mmvq_mode = 1; } + // publish the device only when it is fully built + vk_instance.devices[idx] = device; + return device; } @@ -16463,6 +16468,11 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy } vk_device_struct::~vk_device_struct() { + // the build failed before the device was created, nothing to destroy + if (device == VK_NULL_HANDLE) { + return; + } + VK_LOG_DEBUG("destroy device " << name); device.destroyFence(fence); diff --git a/tests/test-thread-safety.cpp b/tests/test-thread-safety.cpp index 4fbb1a270..0e735d684 100644 --- a/tests/test-thread-safety.cpp +++ b/tests/test-thread-safety.cpp @@ -1,5 +1,5 @@ // thread safety test -// - Loads a copy of the same model on each GPU, plus a copy on the CPU +// - Loads a copy of the same model on each GPU, plus a copy on the CPU, all in parallel // - Creates n_parallel (--parallel) contexts per model // - Runs inference in parallel on each context @@ -51,30 +51,43 @@ int main(int argc, char ** argv) { //const int num_models = std::max(1, gpu_dev_count); const int num_contexts = std::max(1, params.n_parallel); - std::vector models; + std::vector models(num_models); + std::vector load_threads; std::vector threads; std::atomic failed = false; for (int m = 0; m < num_models; ++m) { - auto mparams = common_model_params_to_llama(params); + load_threads.emplace_back([&, m]() { + auto mparams = common_model_params_to_llama(params); - if (m < gpu_dev_count) { - mparams.split_mode = LLAMA_SPLIT_MODE_NONE; - mparams.devices = gpus[m].data(); - } else if (m == gpu_dev_count) { - mparams.split_mode = LLAMA_SPLIT_MODE_NONE; - mparams.main_gpu = -1; // CPU model - } else { - mparams.split_mode = LLAMA_SPLIT_MODE_LAYER; - } + if (m < gpu_dev_count) { + mparams.split_mode = LLAMA_SPLIT_MODE_NONE; + mparams.devices = gpus[m].data(); + } else if (m == gpu_dev_count) { + mparams.split_mode = LLAMA_SPLIT_MODE_NONE; + mparams.main_gpu = -1; // CPU model + } else { + mparams.split_mode = LLAMA_SPLIT_MODE_LAYER; + } - llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams); - if (model == NULL) { - LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); - return 1; - } + llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams); + if (model == NULL) { + LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); + failed.store(true); + return; + } - models.emplace_back(model); + models[m].reset(model); + }); + } + + for (auto & thread : load_threads) { + thread.join(); + } + + if (failed) { + LOG_ERR("One or more models failed to load.\n"); + return 1; } for (int m = 0; m < num_models; ++m) {