vulkan : fix concurrent first device init (#29766)

* vulkan : fix concurrent first device init

* tests : load models in parallel in test-thread-safety

Loading them serially never exercised concurrent first backend init, so the crash reported in #29764 was invisible to this test.

* vulkan : create fence before retaining device in buffer context

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -3990,10 +3990,12 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
 vk_device ggml_vk_get_device(size_t idx) {
     VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")");
 
+    static std::mutex mutex;
+    std::lock_guard<std::mutex> lock(mutex);
+
     if (vk_instance.devices[idx] == nullptr) {
         VK_LOG_DEBUG("Initializing new vk_device");
         vk_device device = std::make_shared<vk_device_struct>();
-        vk_instance.devices[idx] = device;
 
         device->memory_logger = std::unique_ptr<vk_memory_logger>(new vk_memory_logger());
 
@@ -4891,14 +4893,14 @@ vk_device ggml_vk_get_device(size_t idx) {
             device->async_use_transfer_queue = false;
         }
 
+        device->fence = device->device.createFence({});
+
         device->buffer_type = {
             /* .iface    = */ ggml_backend_vk_buffer_type_interface,
             /* .device   = */ ggml_backend_reg_dev_get(ggml_backend_vk_reg(), idx),
             /* .context  = */ new ggml_backend_vk_buffer_type_context{ device->name, device },
         };
 
-        device->fence = device->device.createFence({});
-
         device->idx = idx;
 
         device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
@@ -4920,6 +4922,9 @@ vk_device ggml_vk_get_device(size_t idx) {
             device->mmvq_mode = 1;
         }
 
+        // publish the device only when it is fully built
+        vk_instance.devices[idx] = device;
+
         return device;
     }
 
@@ -16463,6 +16468,11 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo>
 }
 
 vk_device_struct::~vk_device_struct() {
+    // the build failed before the device was created, nothing to destroy
+    if (device == VK_NULL_HANDLE) {
+        return;
+    }
+
     VK_LOG_DEBUG("destroy device " << name);
 
     device.destroyFence(fence);

tests/test-thread-safety.cpp

@@ -1,5 +1,5 @@
 // thread safety test
-// - Loads a copy of the same model on each GPU, plus a copy on the CPU
+// - Loads a copy of the same model on each GPU, plus a copy on the CPU, all in parallel
 // - Creates n_parallel (--parallel) contexts per model
 // - Runs inference in parallel on each context
 
@@ -51,30 +51,43 @@ int main(int argc, char ** argv) {
     //const int num_models = std::max(1, gpu_dev_count);
     const int num_contexts = std::max(1, params.n_parallel);
 
-    std::vector<llama_model_ptr> models;
+    std::vector<llama_model_ptr> models(num_models);
+    std::vector<std::thread> load_threads;
     std::vector<std::thread> threads;
     std::atomic<bool> failed = false;
 
     for (int m = 0; m < num_models; ++m) {
-        auto mparams = common_model_params_to_llama(params);
+        load_threads.emplace_back([&, m]() {
+            auto mparams = common_model_params_to_llama(params);
 
-        if (m < gpu_dev_count) {
-            mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
-            mparams.devices = gpus[m].data();
-        } else if (m == gpu_dev_count) {
-            mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
-            mparams.main_gpu = -1; // CPU model
-        } else {
-            mparams.split_mode = LLAMA_SPLIT_MODE_LAYER;
-        }
+            if (m < gpu_dev_count) {
+                mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
+                mparams.devices = gpus[m].data();
+            } else if (m == gpu_dev_count) {
+                mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
+                mparams.main_gpu = -1; // CPU model
+            } else {
+                mparams.split_mode = LLAMA_SPLIT_MODE_LAYER;
+            }
 
-        llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
-        if (model == NULL) {
-            LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
-            return 1;
-        }
+            llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
+            if (model == NULL) {
+                LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
+                failed.store(true);
+                return;
+            }
 
-        models.emplace_back(model);
+            models[m].reset(model);
+        });
+    }
+
+    for (auto & thread : load_threads) {
+        thread.join();
+    }
+
+    if (failed) {
+        LOG_ERR("One or more models failed to load.\n");
+        return 1;
     }
 
     for  (int m = 0; m < num_models; ++m) {