diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 3e580dfc109cf..d56609c123968 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -640,19 +640,23 @@ if(NOT WIN32 AND NOT APPLE)
       if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.1.0")
           set(MOLD_LINKER_FLAGS "-fuse-ld=mold")
-          message(STATUS "Using optional mold linker")
         else()
           message(STATUS "Need GCC 12.1.0 or later to use mold linker: ${CMAKE_CXX_COMPILER_VERSION}"
           )
         endif()
       elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-        set(MOLD_LINKER_FLAGS "--ld-path=${LD_MOLD}")
-        message(STATUS "Using optional mold linker")
+        if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.0.0")
+          set(MOLD_LINKER_FLAGS "--ld-path=${LD_MOLD}")
+        else()
+          message(STATUS "Need clang 12.0.0 or later to use mold linker: ${CMAKE_CXX_COMPILER_VERSION}"
+          )
+        endif()
       else()
         message(STATUS "Using the default linker because compiler doesn't support mold: ${CMAKE_CXX_COMPILER_ID}"
         )
       endif()
       if(MOLD_LINKER_FLAGS)
+        message(STATUS "Using optional mold linker")
         string(APPEND CMAKE_EXE_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
         string(APPEND CMAKE_MODULE_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
         string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
@@ -675,7 +679,12 @@ if(ARROW_USE_LLD)
         )
       endif()
     elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-      set(LLD_LINKER_FLAGS "--ld-path=${LD_LLD}")
+      if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.0.0")
+        set(LLD_LINKER_FLAGS "--ld-path=${LD_LLD}")
+      else()
+        message(STATUS "Need clang 12.0.0 or later to use LLD linker: ${CMAKE_CXX_COMPILER_VERSION}"
+        )
+      endif()
     else()
       message(STATUS "Using the default linker because compiler doesn't support LLD: ${CMAKE_CXX_COMPILER_ID}"
       )