diff --git a/mllib-dal/src/main/java/org/apache/spark/ml/util/LibLoader.java b/mllib-dal/src/main/java/org/apache/spark/ml/util/LibLoader.java
index eada9b20c..c98d6dbb3 100644
--- a/mllib-dal/src/main/java/org/apache/spark/ml/util/LibLoader.java
+++ b/mllib-dal/src/main/java/org/apache/spark/ml/util/LibLoader.java
@@ -53,10 +53,13 @@ public static synchronized void loadLibraries() throws IOException {
    * Load oneCCL libs in dependency order
    */
   private static synchronized void loadLibCCL() throws IOException {
-    loadFromJar(subDir, "libfabric.so.1");
+    // Load libfabric from system first, if failed load from jar
+    if (!loadFromSystem("libfabric.so.1")) {
+      loadFromJar(subDir, "libfabric.so.1");
+      loadFromJar(subDir, "libsockets-fi.so");
+    }
     loadFromJar(subDir, "libmpi.so.12");
     loadFromJar(subDir, "libccl.so");
-    loadFromJar(subDir, "libsockets-fi.so");
   }
 
   /**
@@ -72,6 +75,15 @@ private static synchronized void loadLibMLlibDAL() throws IOException {
     loadFromJar(subDir, "libMLlibDAL.so");
   }
 
+  private static synchronized boolean loadFromSystem(String name) {
+    try {
+      System.loadLibrary(name);
+    } catch (UnsatisfiedLinkError e) {
+      return false;
+    }
+    return true;
+  }
+
   /**
    * Load lib as resource
    *