Skip to content

Commit

Permalink
Fully parameterize the build. (#79)
Browse files Browse the repository at this point in the history
* Options are defined for every library.
* Dependencies are specified so that it is safe to enable/disable any
subset and have the build do the right thing.
* Updates tests to adapt to feature availability.
* Updates documentation, removing some obsolete docs and defining the
flags.
* Adds a THEROCK_VERBOSE=OFF flag and hides a lot of chatty (but useful
for developers) CMake output behind it.
  • Loading branch information
stellaraccident authored Feb 11, 2025
1 parent 6c4d1c6 commit 2225897
Show file tree
Hide file tree
Showing 13 changed files with 954 additions and 724 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/build_linux_packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ jobs:
cmake -B build -GNinja . \
-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DTHEROCK_AMDGPU_FAMILIES=gfx110X-dgpu \
-DTHEROCK_PACKAGE_VERSION="${package_version}"
-DTHEROCK_PACKAGE_VERSION="${package_version}" \
-DTHEROCK_VERBOSE=ON
./build_tools/watch_top_processes.sh &
cmake --build build --target therock-archives therock-dist
kill %1
Expand Down
95 changes: 85 additions & 10 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ include(CMakeDependentOption)
include(ExternalProject)
include(therock_amdgpu_targets)
include(therock_artifacts)
include(therock_features)
include(therock_subproject)
include(therock_job_pools)

Expand All @@ -38,10 +39,8 @@ message(STATUS "ROCM_GIT_DIR is set to: ${ROCM_GIT_DIR}")
set(THEROCK_ARTIFACT_ARCHIVE_SUFFIX "" CACHE STRING "Suffix to add to artifact archive file stem names")
set(THEROCK_ARTIFACT_ARCHIVE_TYPES "tar.xz" CACHE STRING "List of artifact archive types to generate")

# Library specific enable flags.
option(THEROCK_ENABLE_RCCL "Enable the comm_libs/rccl sub-project" ON)
option(THEROCK_ENABLE_MATH_LIBS "Enable building of math libraries" ON)
cmake_dependent_option(THEROCK_ENABLE_ML_LIBS "Enables building of ML frameworks" ON THEROCK_ENABLE_MATH_LIBS OFF)
# Overall build settings.
option(THEROCK_VERBOSE "Enables verbose CMake statuses" OFF)

# Initialize the install directory.
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
Expand All @@ -53,6 +52,86 @@ set(ROCM_MAJOR_VERSION 6)
set(ROCM_MINOR_VERSION 3)
set(ROCM_PATCH_VERSION 1)

################################################################################
# Feature selection
# Each feature added via therock_add_feature produces a boolean cache variable
# like `THEROCK_ENABLE_${feature_name}` that takes its default value from the
# `THEROCK_ENABLE_${group_name}` if a GROUP is present.
################################################################################
option(THEROCK_ENABLE_ALL "Enables building of all feature groups" ON)
cmake_dependent_option(THEROCK_ENABLE_CORE "Enable building of core libraries" ON "THEROCK_ENABLE_ALL" OFF)
cmake_dependent_option(THEROCK_ENABLE_COMM_LIBS "Enable building of comm libraries" ON "THEROCK_ENABLE_ALL" OFF)
cmake_dependent_option(THEROCK_ENABLE_MATH_LIBS "Enable building of math libraries" ON "THEROCK_ENABLE_ALL" OFF)
cmake_dependent_option(THEROCK_ENABLE_ML_LIBS "Enable building of ML libraries" ON "THEROCK_ENABLE_ALL" OFF)
option(THEROCK_RESET_FEATURES "One-shot flag which forces all feature flags to their default state for this configuration run" OFF)

# Base Features.
therock_add_feature(COMPILER
GROUP ALL
DESCRIPTION "Enables the AMDGPU+host compiler toolchain"
)
therock_add_feature(HIPIFY
GROUP ALL
DESCRIPTION "Enables the hipify tool"
REQUIRES COMPILER
)

# Core Features.
therock_add_feature(CORE_RUNTIME
GROUP CORE
DESCRIPTION "Enables the core device runtime and tools"
)
therock_add_feature(HIP_RUNTIME
GROUP CORE
DESCRIPTION "Enables the HIP runtime"
REQUIRES COMPILER CORE_RUNTIME
)

# Comm-libs Features.
therock_add_feature(RCCL
GROUP COMM_LIBS
DESCRIPTION "Enables rccl"
REQUIRES COMPILER HIP_RUNTIME HIPIFY
)

# Math-libs Features.
therock_add_feature(PRIM
GROUP MATH_LIBS
DESCRIPTION "Enables prim libraries (rocprim)"
REQUIRES COMPILER HIP_RUNTIME
)
therock_add_feature(BLAS
GROUP MATH_LIBS
DESCRIPTION "Enables blas libraries (hipblaslt, rocblas, hipblas)"
REQUIRES COMPILER HIP_RUNTIME
)
therock_add_feature(RAND
GROUP MATH_LIBS
DESCRIPTION "Enables rand libraries (hiprand, rocrand)"
REQUIRES COMPILER HIP_RUNTIME
)
therock_add_feature(SOLVER
GROUP MATH_LIBS
DESCRIPTION "Enables solver libraries (hipsolver, rocsolver)"
REQUIRES COMPILER HIP_RUNTIME BLAS PRIM
)
therock_add_feature(SPARSE
GROUP MATH_LIBS
DESCRIPTION "Enables sparse libraries (hipsparse, rocsparse)"
REQUIRES COMPILER HIP_RUNTIME BLAS PRIM
)

# ML-Libs Features.
therock_add_feature(MIOPEN
GROUP ML_LIBS
DESCRIPTION "Enables the MIOpen project (with minimal deps defaults to ON)"
REQUIRES COMPILER HIP_RUNTIME BLAS RAND
)

# Finalize all feature flags.
therock_finalize_features()
therock_report_features()

################################################################################
# GPU target selection
#
Expand Down Expand Up @@ -115,12 +194,8 @@ if(NOT WIN32)
add_subdirectory(compiler)
add_subdirectory(core)
add_subdirectory(comm-libs)
if(THEROCK_ENABLE_MATH_LIBS)
add_subdirectory(math-libs)
endif()
if(THEROCK_ENABLE_ML_LIBS)
add_subdirectory(ml-libs)
endif()
add_subdirectory(math-libs)
add_subdirectory(ml-libs)
else()
# TODO(#36): Enable more project builds on Windows and/or make the full list configurable.
endif()
Expand Down
63 changes: 34 additions & 29 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,45 +79,50 @@ cmake -B build -GNinja . -DTHEROCK_AMDGPU_FAMILIES=gfx110X-dgpu
# Or if iterating and wishing to cache:
# cmake -Bbuild -GNinja -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache .
cmake --build build
# Install per component.
cmake --install build --component amdgpu-runtime
cmake --install build --component amdgpu-runtime-dev
cmake --install build --component amdgpu-compiler
# Create archives.
(cd build && cpack -G TBZ2)
```

## Sanity Checks
## Feature Flags

The following does not replace a robust test suite. However, it will tell you
whether the toolchain you have just crafted is viable at all (as in can load
and enumerate devices).
By default, the project builds everything available. The following group flags
allow enable/disable of selected subsets:

```
./build/dlopen-hip install/lib/libamdhip64.so
```
- `-DTHEROCK_ENABLE_ALL=OFF`: Disables all optional components.
- `-DTHEROCK_ENABLE_CORE=OFF`: Disables all core components.
- `-DTHEROCK_ENABLE_COMM_LIBS=OFF`: Disables all communication libraries.
- `-DTHEROCK_ENABLE_MATH_LIBS=OFF`: Disables all math libraries.
- `-DTHEROCK_ENABLE_ML_LIBS=OFF`: Disables all ML libraries.

HIP enabled IREE can also be used to enumerate:
Individual features can be controlled separately (typically in combination with
`-DTHEROCK_ENABLE_ALL=OFF` or `-DTHEROCK_RESET_FEATURES=ON` to force a
minimal build):

```
LD_LIBRARY_PATH=install/lib iree-run-module --dump_devices=hip
```
- `-DTHEROCK_ENABLE_COMPILER=ON`: Enables the GPU+host compiler toolchain.
- `-DTHEROCK_ENABLE_HIPIFY=ON`: Enables the hipify tool.
- `-DTHEROCK_ENABLE_CORE_RUNTIME=ON`: Enables the core runtime components and tools.
- `-DTHEROCK_ENABLE_HIP_RUNTIME=ON`: Enables the HIP runtime components.
- `-DTHEROCK_ENABLE_RCCL=ON`: Enables RCCL.
- `-DTHEROCK_ENABLE_PRIM=ON`: Enables the PRIM library.
- `-DTHEROCK_ENABLE_BLAS=ON`: Enables the BLAS libraries.
- `-DTHEROCK_ENABLE_RAND=ON`: Enables the RAND libraries.
- `-DTHEROCK_ENABLE_SOLVER=ON`: Enables the SOLVER libraries.
- `-DTHEROCK_ENABLE_SPARSE=ON`: Enables the SPARSE libraries.
- `-DTHEROCK_ENABLE_MIOPEN=ON`: Enables MIOpen.

# Development Notes
Enabling any features will implicitly enable its *minimum* dependencies. Some
libraries (like MIOpen) have a number of *optional* dependencies, which must
be enabled manually if enabling/disabling individual features.

## Building in a manylinux container
A report of enabled/disabled features and flags will be printed on every
CMake configure.

Our CI builds run in such a container, and it can be useful to run locally.

```
docker run --rm -it -v $PWD:$PWD --entrypoint /bin/bash ghcr.io/nod-ai/manylinux_ghr_x86_64:main
```
## Sanity Checks

Packages needed:
Tests of the integrity of the build are enabled by default and can be run
with ctest:

```
yum install -y numactl-devel elfutils-libelf-devel vim-common git-lfs
pip install CppHeaderParser
ctest --test-dir build
```

Testing functionality on an actual GPU is in progress and will be documented
separately.
103 changes: 103 additions & 0 deletions cmake/therock_features.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
# therock_features.cmake
# Registry of features that can be enabled/disabled when building TheRock.
# In this context, "feature" can refer to either a top-level component/project
# or some optional configuration of it (typically as "project_feature").
# Features can have dependencies, which are a superset of the project
# dependencies and must be kept rationalized by hand.
#
# Each feature results in the following changes to the caller scope:
# THEROCK_ENABLE_${feature_name}: boolean of whether enabled/disabled
# THEROCK_REQUIRES_${feature_name}: List of features that are required to
# be enabled for this feature
# THEROCK_ALL_FEATURES: Added to this global list of features.
#
# A feature is enabled by default unless if the GROUP keyword is specified.
# In this case, it's default state is dependent on THEROCK_ENABLE_${group}.

function(therock_add_feature feature_name)
cmake_parse_arguments(PARSE_ARGV 1 ARG
""
"GROUP;DESCRIPTION"
"REQUIRES"
)

set(_default_enabled ON)
if(ARG_GROUP)
if(NOT "${THEROCK_ENABLE_${ARG_GROUP}}")
set(_default_enabled OFF)
endif()
endif()
if(THEROCK_RESET_FEATURES)
set(_force "FORCE")
endif()

# Validate.
if("${feature_name}" IN_LIST THEROCK_ALL_FEATURES)
message(FATAL_ERROR "CMake feature already defined: ${feature_name}")
endif()
foreach(require ${ARG_REQUIRES})
if(NOT DEFINED THEROCK_ENABLE_${require})
message(FATAL_ERROR "CMake feature order error: ${feature_name} requires ${require} which was not defined first")
endif()
endforeach()

# Set up the cache option and inject the effective value into the parent
# scope.
set(THEROCK_ENABLE_${feature_name} ${_default_enabled} CACHE BOOL "${ARG_DESCRIPTION}" ${_force})
set(_actual $CACHE{THEROCK_ENABLE_${feature_name}})
set(THEROCK_ENABLE_${feature_name} "${_actual}" PARENT_SCOPE)
set(THEROCK_REQUIRES_${feature_name} ${ARG_REQUIRES} PARENT_SCOPE)
set(_all_features ${THEROCK_ALL_FEATURES})
list(APPEND _all_features "${feature_name}")
set(THEROCK_ALL_FEATURES ${_all_features} PARENT_SCOPE)
endfunction()

function(therock_finalize_features)
# Force enable any features required of an enabled feature.
# These are processed in reverse declaration order, which ensures a DAG.
set(all_features_reversed ${THEROCK_ALL_FEATURES})
list(REVERSE all_features_reversed)
set(_implicit_features)
foreach(feature_name ${all_features_reversed})
if(THEROCK_ENABLE_${feature_name})
foreach(require ${THEROCK_REQUIRES_${feature_name}})
if(NOT THEROCK_ENABLE_${require})
set(THEROCK_ENABLE_${require} ON PARENT_SCOPE)
set(THEROCK_ENABLE_${require} ON)
list(APPEND _implicit_features ${require})
endif()
endforeach()
endif()
endforeach()

if(_implicit_features)
list(REMOVE_DUPLICATES _implicit_features)
list(JOIN _implicit_features " " _implicit_features_spaces)
message(STATUS "Implicitly enabled features: ${_implicit_features_spaces}")
endif()
endfunction()

function(therock_report_features)
# And report.
message(STATUS "Enabled features:")
set(_available_list)
foreach(feature_name ${THEROCK_ALL_FEATURES})
if(THEROCK_ENABLE_${feature_name})
message(STATUS " * ${feature_name} (-DTHEROCK_ENABLE_${feature_name}=ON)")
else()
list(APPEND _available_list "${feature_name}")
endif()
endforeach()
if(_available_list)
message(STATUS "Disabled features:")
foreach(feature_name ${_available_list})
message(STATUS " * ${feature_name} (-DTHEROCK_ENABLE_${feature_name}=OFF)")
endforeach()
endif()

# Reset the force disable flag.
if(THEROCK_RESET_FEATURES)
set(THEROCK_RESET_FEATURES OFF CACHE BOOL "" FORCE)
set(THEROCK_RESET_FEATURES OFF PARENT_SCOPE)
endif()
endfunction()
Loading

0 comments on commit 2225897

Please sign in to comment.