Re-organize the compiler build/packaging to more closely match ROCM>= 6.4.

base/CMakeLists.txt

@@ -6,7 +6,7 @@ therock_cmake_subproject_declare(rocm-cmake
   EXTERNAL_SOURCE_DIR "rocm-cmake"
 )
 therock_cmake_subproject_provide_package(rocm-cmake
-  ROCmCMakeBuildToolsConfig share/rocmcmakebuildtools/cmake)
+  ROCmCMakeBuildTools share/rocmcmakebuildtools/cmake)
 therock_cmake_subproject_provide_package(rocm-cmake
   ROCM share/rocm/cmake)
 therock_cmake_subproject_activate(rocm-cmake)

cmake/therock_subproject.cmake

@@ -756,10 +756,23 @@ function(_therock_cmake_subproject_setup_toolchain compiler_toolchain toolchain_
   if(NOT compiler_toolchain)
     # Make any additional customizations if no toolchain specified.
   elseif(compiler_toolchain STREQUAL "amd-llvm" OR compiler_toolchain STREQUAL "amd-hip")
-    # amd-llvm toolchain: The private built LLVM in isolation.
-    _therock_assert_is_cmake_subproject("amd-llvm")
-    get_target_property(_amd_llvm_dist_dir amd-llvm THEROCK_DIST_DIR)
-    get_target_property(_amd_llvm_stamp_dir amd-llvm THEROCK_STAMP_DIR)
+    # The "amd-llvm" and "amd-hip" toolchains are configured very similarly so
+    # we commingle them, but they are different:
+    #   "amd-llvm": Just the base LLVM compiler and device libraries. This
+    #     doesn't know anything about hip (i.e. it doesn't have hipconfig, etc).
+    #   "amd-hip": Superset of "amd-llvm" which also includes hipcc, hip headers,
+    #     and hip version info. This has hipconfig in it.
+    # The main difference is that for "amd-llvm", we derive the configuration from
+    # the amd-llvm project's dist/ tree. And for "amd-hip", from the hip-clr
+    # project (which has runtime dependencies on the underlying toolchain).
+    if (compiler_toolchain STREQUAL "amd-hip")
+      set(_toolchain_subproject "hip-clr")
+    else()
+      set(_toolchain_subproject "amd-llvm")
+    endif()
+    _therock_assert_is_cmake_subproject("${_toolchain_subproject}")
+    get_target_property(_amd_llvm_dist_dir "${_toolchain_subproject}" THEROCK_DIST_DIR)
+    get_target_property(_amd_llvm_stamp_dir "${_toolchain_subproject}" THEROCK_STAMP_DIR)
     # Add a dependency on the toolchain's dist
     set(AMD_LLVM_C_COMPILER "${_amd_llvm_dist_dir}/lib/llvm/bin/clang")
     set(AMD_LLVM_CXX_COMPILER "${_amd_llvm_dist_dir}/lib/llvm/bin/clang++")

compiler/CMakeLists.txt

@@ -1,5 +1,8 @@
 ################################################################################
 # amd-llvm
+# Everything built as part of LLVM installs to lib/llvm
+# This includes the device-libs, since they must be findable by the compiler
+# frontend.
 ################################################################################
 
 set(_extra_llvm_cmake_args)
@@ -12,12 +15,28 @@ therock_cmake_subproject_declare(amd-llvm
   # Note that LLVM top level CMakeLists.txt is in the llvm subdir of the
   # monorepo.
   CMAKE_LISTS_RELPATH "llvm"
+  INTERFACE_PROGRAM_DIRS
+    lib/llvm/bin
   CMAKE_ARGS
+    # LIBCXX
+    -DLIBCXX_ENABLE_SHARED=OFF
+    -DLIBCXX_ENABLE_STATIC=ON
+    -DLIBCXX_INSTALL_LIBRARY=OFF
+    -DLIBCXX_INSTALL_HEADERS=OFF
+    -DLIBCXXABI_ENABLE_SHARED=OFF
+    -DLIBCXXABI_ENABLE_STATIC=ON
+
+    # Features
     -DLLVM_INCLUDE_TESTS=OFF
-    -DCOMGR_DISABLE_SPIRV=ON
+    -DLLVM_ENABLE_ZLIB=ON
+    -DLLVM_ENABLE_Z3_SOLVER=OFF
+    -DLLVM_AMDGPU_ALLOW_NPI_TARGETS=ON
+    -DCLANG_DEFAULT_LINKER=lld
+    -DCLANG_DEFAULT_RTLIB=compiler-rt
+    -DCLANG_DEFAULT_UNWINDLIB=libgcc
+    -DCLANG_ENABLE_AMDCLANG=ON
+
     ${_extra_llvm_cmake_args}
-  INTERFACE_PROGRAM_DIRS
-    lib/llvm/bin
   BUILD_DEPS
     rocm-cmake
   # The entire LLVM install tree is placed inside of the overall ROCM lib/llvm
@@ -30,14 +49,56 @@ therock_cmake_subproject_declare(amd-llvm
 # too large to glob like that. Consider having a project dev mode option for
 # enabling better ergonomics here.
 
-therock_cmake_subproject_provide_package(amd-llvm amd_comgr lib/llvm/lib/cmake/amd_comgr)
+therock_cmake_subproject_provide_package(amd-llvm AMDDeviceLibs lib/llvm/lib/cmake/AMDDeviceLibs)
 therock_cmake_subproject_provide_package(amd-llvm Clang lib/llvm/lib/cmake/clang)
 therock_cmake_subproject_provide_package(amd-llvm LLD lib/llvm/lib/cmake/lld)
 therock_cmake_subproject_provide_package(amd-llvm LLVM lib/llvm/lib/cmake/llvm)
-therock_cmake_subproject_provide_package(amd-llvm AMDDeviceLibs lib/llvm/lib/cmake/AMDDeviceLibs)
 therock_cmake_subproject_activate(amd-llvm)
 
 
+#################################################################################
+# comgr
+# A client of libLLVM which provides an in-process compiler API to the HIP
+# runtime.
+#################################################################################
+
+therock_cmake_subproject_declare(amd-comgr
+  EXTERNAL_SOURCE_DIR "amd-llvm/amd/comgr"
+  BINARY_DIR "amd-comgr"
+  BACKGROUND_BUILD
+  CMAKE_ARGS
+    # TODO: Currently unstable. Enable in >6.4.
+    -DCOMGR_DISABLE_SPIRV=ON
+    -DLLVM_LINK_LLVM_DYLIB=ON
+    # The comgr tests have a circular dependency on the HIP runtime.
+    # https://github.com/nod-ai/TheRock/issues/67
+    -DBUILD_TESTING=OFF
+  BUILD_DEPS
+    rocm-cmake
+  RUNTIME_DEPS
+    amd-llvm
+)
+therock_cmake_subproject_provide_package(amd-comgr amd_comgr lib/cmake/amd_comgr)
+therock_cmake_subproject_activate(amd-comgr)
+
+#################################################################################
+# hipcc
+# Provides hipcc and hipconfig
+#################################################################################
+
+therock_cmake_subproject_declare(hipcc
+  EXTERNAL_SOURCE_DIR "amd-llvm/amd/hipcc"
+  BINARY_DIR "hipcc"
+  BACKGROUND_BUILD
+  CMAKE_ARGS
+    -DHIPCC_BACKWARD_COMPATIBILITY=OFF
+  BUILD_DEPS
+    rocm-cmake
+  RUNTIME_DEPS
+    amd-llvm
+)
+therock_cmake_subproject_activate(hipcc)
+
 #################################################################################
 # HIPIFY
 #################################################################################
@@ -49,6 +110,8 @@ therock_cmake_subproject_declare(hipify
     bin
   CMAKE_ARGS
     -DHIPIFY_INSTALL_CLANG_HEADERS=OFF
+  BUILD_DEPS
+    rocm-cmake
   RUNTIME_DEPS
     amd-llvm
 )

compiler/artifact-amd-llvm.toml

@@ -1,3 +1,4 @@
+[components.dbg."compiler/amd-llvm/stage"]
 [components.dev."compiler/amd-llvm/stage"]
 exclude = [
   "lib/llvm/lib/clang/**",
@@ -15,3 +16,18 @@ include = [
   "lib/llvm/libexec/**",
   "lib/llvm/hip/**",
 ]
+
+# comgr and hipcc are considered part of the LLVM artifact (they are separated
+# in the build tree because they need to be built separately in order to
+# install to different prefixes).
+[components.dbg."compiler/amd-comgr/stage"]
+[components.dev."compiler/amd-comgr/stage"]
+[components.doc."compiler/amd-comgr/stage"]
+[components.lib."compiler/amd-comgr/stage"]
+[components.run."compiler/amd-comgr/stage"]
+
+[components.dbg."compiler/hipcc/stage"]
+[components.dev."compiler/hipcc/stage"]
+[components.doc."compiler/hipcc/stage"]
+[components.lib."compiler/hipcc/stage"]
+[components.run."compiler/hipcc/stage"]

compiler/post_hook_amd-llvm.cmake → compiler/post_hook_amd-comgr.cmake

@@ -8,5 +8,5 @@ therock_set_install_rpath(
   TARGETS
     amd_comgr
   PATHS
-    .
+    llvm/lib
 )

compiler/pre_hook_amd-llvm.cmake

@@ -1,16 +1,35 @@
+# Get access to LLVM_VERSION_MAJOR
+include("${ROCM_GIT_DIR}/llvm-project/cmake/Modules/LLVMVersion.cmake")
+
 # Build LLVM and the comgr dependency.
 # Note that in LLVM "BUILD_SHARED_LIBS" enables an unsupported development mode.
 # The flag you want for a shared library build is LLVM_BUILD_LLVM_DYLIB.
 set(BUILD_SHARED_LIBS OFF)
 set(LLVM_BUILD_LLVM_DYLIB ON)
 set(LLVM_LINK_LLVM_DYLIB ON)
+set(LLVM_ENABLE_LIBCXX ON)
 
 # Set the LLVM_ENABLE_PROJECTS variable before including LLVM's CMakeLists.txt
 set(BUILD_TESTING OFF CACHE BOOL "DISABLE BUILDING TESTS IN SUBPROJECTS" FORCE)
-set(LLVM_ENABLE_PROJECTS "compiler-rt;lld;clang" CACHE STRING "Enable LLVM projects" FORCE)
+set(LLVM_ENABLE_PROJECTS "clang;lld;clang-tools-extra" CACHE STRING "Enable LLVM projects" FORCE)
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libunwind;libcxx;libcxxabi" CACHE STRING "Enabled runtimes" FORCE)
 set(LLVM_TARGETS_TO_BUILD "AMDGPU;X86" CACHE STRING "Enable LLVM Targets" FORCE)
-set(LLVM_EXTERNAL_DEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs" CACHE STRING "Device libs path" FORCE)
-set(LLVM_EXTERNAL_PROJECTS "amddevice-libs;amdcomgr;hipcc" CACHE STRING "Enable extra projects" FORCE)
+
+# Packaging.
+set(PACKAGE_VENDOR "AMD" CACHE STRING "Vendor" FORCE)
+
+# Build the device-libs as part of the core compiler so that clang works by
+# default (as opposed to other components that are *users* of the compiler).
 set(LLVM_EXTERNAL_AMDDEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs")
-set(LLVM_EXTERNAL_AMDCOMGR_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/comgr")
-set(LLVM_EXTERNAL_HIPCC_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/hipcc")
+set(LLVM_EXTERNAL_PROJECTS "amddevice-libs" CACHE STRING "Enable extra projects" FORCE)
+
+# TODO: Arrange for the devicelibs to be installed to the clange resource dir
+# by default. This corresponds to the layout for ROCM>=7. However, not all
+# code (specifically the AMDDeviceLibs.cmake file) has adapted to the new
+# location, so we have to also make them available at amdgcn. There are cache
+# options to manage this transition but they require knowing the clange resource
+# dir. In order to avoid drift, we just fixate that too. This can all be
+# removed in a future version.
+set(CLANG_RESOURCE_DIR "../lib/clang/${LLVM_VERSION_MAJOR}" CACHE STRING "Resource dir" FORCE)
+set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW "lib/clang/${LLVM_VERSION_MAJOR}/amdgcn" CACHE STRING "New devicelibs loc" FORCE)
+set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD "amdgcn" CACHE STRING "Old devicelibs loc" FORCE)

core/CMakeLists.txt

@@ -26,10 +26,16 @@ therock_cmake_subproject_activate(ROCR-Runtime)
 
 ################################################################################
 # clr
+# The primary HIP compiler and runtime target. This is also the project used
+# to root the "amd-hip" compiler toolchain and is a superset of amd-llvm.
+# As a particularly sharp edge, the `hipconfig` tool is really only fully
+# functional from this project (which contributes version and metadata).
 ################################################################################
 
 therock_cmake_subproject_declare(hip-clr
   EXTERNAL_SOURCE_DIR "clr"
+  INTERFACE_PROGRAM_DIRS
+    bin
   BACKGROUND_BUILD
   CMAKE_ARGS
     -DHIP_PLATFORM=amd
@@ -42,6 +48,8 @@ therock_cmake_subproject_declare(hip-clr
     rocm-cmake
   RUNTIME_DEPS
     amd-llvm
+    amd-comgr
+    hipcc     # For hipconfig
     rocm-core
     rocprofiler-register
     ROCR-Runtime

math-libs/CMakeLists.txt

@@ -2,8 +2,8 @@
 # If a project is using an amd-hip or amd-llvm toolchain, then it will already
 # have an implicit dep on the toolchain, so it is safe to reference binaries
 # here without worrying about build deps.
-therock_cmake_subproject_dist_dir(_toolchain_dir amd-llvm)
-cmake_path(APPEND _toolchain_dir lib/llvm)
+therock_cmake_subproject_dist_dir(_hip_dir hip-clr)
+cmake_path(APPEND _hip_dir lib/llvm OUTPUT_VARIABLE _toolchain_dir)
 
 #################################################################################
 # hipBLAS-common
@@ -51,7 +51,7 @@ therock_cmake_subproject_declare(hipBLASLt
     -DTensile_CODE_OBJECT_VERSION=default
     -DTensile_CPU_THREADS=
     -DTensile_LIBRARY_FORMAT=msgpack
-    -DTensile_HIP_CONFIG=${_toolchain_dir}/bin/hipconfig
+    -DTensile_HIP_CONFIG=${_hip_dir}/bin/hipconfig
     "-DTensile_C_COMPILER=${_toolchain_dir}/bin/clang"
     "-DTensile_CXX_COMPILER=${_toolchain_dir}/bin/clang++"
     "-DTensile_ASSEMBLER=${_toolchain_dir}/bin/clang++"