diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 3113de017f0..b3fec628ecb 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -339,11 +339,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
     }
 
 #ifdef GGML_USE_NCCL
-    int dev_ids[GGML_CUDA_MAX_DEVICES];
-    for (int id = 0; id < info.device_count; ++id) {
-        dev_ids[id] = id;
+    if (info.device_count > 1) {
+        int dev_ids[GGML_CUDA_MAX_DEVICES];
+        for (int id = 0; id < info.device_count; ++id) {
+            dev_ids[id] = id;
+        }
+        NCCL_CHECK(ncclCommInitAll(info.comms, info.device_count, dev_ids));
     }
-    NCCL_CHECK(ncclCommInitAll(info.comms, info.device_count, dev_ids));
 #endif // GGML_USE_NCCL
 
     return info;