diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 5ef27dd5fd02a..90decb4224ec6 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -629,23 +629,27 @@ if(NOT WIN32 AND NOT APPLE)
   if(ARROW_USE_MOLD)
     find_program(LD_MOLD ld.mold)
     if(LD_MOLD)
+      unset(MOLD_LINKER_FLAGS)
       if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "12.1.0")
-          set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fuse-ld=mold")
-          set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fuse-ld=mold")
+          set(MOLD_LINKER_FLAGS "-fuse-ld=mold")
           message(STATUS "Using optional mold linker")
         else()
           message(STATUS "Need GCC 12.1.0 or later to use mold linker: ${CMAKE_CXX_COMPILER_VERSION}"
           )
         endif()
       elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --ld-path=${LD_MOLD}")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --ld-path=${LD_MOLD}")
+        set(MOLD_LINKER_FLAGS "--ld-path=${LD_MOLD}")
         message(STATUS "Using optional mold linker")
       else()
         message(STATUS "Using the default linker because compiler doesn't support mold: ${CMAKE_CXX_COMPILER_ID}"
         )
       endif()
+      if(MOLD_LINKER_FLAGS)
+        string(APPEND CMAKE_EXE_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
+        string(APPEND CMAKE_MODULE_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
+        string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${MOLD_LINKER_FLAGS}")
+      endif()
     else()
       message(STATUS "Using the default linker because mold isn't found")
     endif()