Use nproc --all to determine number of processors (#272)

* Use nproc --all to determine number of processors For some reason, nproc looks at the value (if any) of OMP_NUM_THREADS and OMP_THREAD_LIMIT in the environment to constrain the result. If we don't pass --all and we have one of those environment variables set, then build parallelism is incorrectly limited to nproc=1. * bump feature versions --------- Co-authored-by: ptaylor <paul.e.taylor@me.com>
rapidsai · Apr 19, 2024 · 7e84a15 · 7e84a15
1 parent d776a3b
commit 7e84a15
Show file tree

Hide file tree

Showing 13 changed files with 42 additions and 38 deletions.
diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.6.8",
+  "version": "24.6.9",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/checkout-same-branch.sh
@@ -24,7 +24,7 @@ checkout_same_branch() {
     eval "$(_parse_args "$@" <&0)";
 
     eval "$(                                              \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}            \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}      \
         rapids-get-num-archs-jobs-and-load --archs 0 "$@" \
     )";
 

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-num-archs-jobs-and-load.sh
@@ -41,7 +41,10 @@ get_num_archs_jobs_and_load() {
     # shellcheck disable=SC1091
     . devcontainer-utils-debug-output 'rapids_build_utils_debug' 'get-num-archs-jobs-and-load';
 
-    local -r n_cpus="$(nproc)";
+    # The return value of nproc is (who knew!) constrained by the
+    # values of OMP_NUM_THREADS and/or OMP_THREAD_LIMIT
+    # Since we want the physical number of processors here, pass --all
+    local -r n_cpus="$(nproc --all)";
 
     if test ${#j[@]} -gt 0 && test -z "${j:-}"; then
         j="${n_cpus}";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/merge-compile-commands-json.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/merge-compile-commands-json.sh
@@ -54,7 +54,7 @@ merge_compile_commands_json() {
 
     readarray -t dirs < <(                                             \
         _list_repo_paths                                               \
-      | xargs -r -I% -P$(nproc)                                        \
+      | xargs -r -I% -P$(nproc --all)                                  \
         rapids-get-cmake-build-dir --skip-links --skip-build-type -- % \
       | xargs -r -I% sh -c 'if test -e %; then echo %/; fi'            \
     );

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh
@@ -17,10 +17,10 @@ build_${CPP_LIB}_cpp() {
     local -;
     set -euo pipefail;
 
-    eval "$(                                    \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}  \
-        rapids-get-num-archs-jobs-and-load "$@" \
-        2>/dev/null                             \
+    eval "$(                                          \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
+        rapids-get-num-archs-jobs-and-load "$@"       \
+        2>/dev/null                                   \
     )";
 
     local -a cmake_args_="(${CMAKE_ARGS:-})";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.configure.tmpl.sh
@@ -19,10 +19,10 @@ configure_${CPP_LIB}_cpp() {
     local -;
     set -euo pipefail;
 
-    eval "$(                                    \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}  \
-        rapids-get-num-archs-jobs-and-load "$@" \
-        2>/dev/null                             \
+    eval "$(                                          \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
+        rapids-get-num-archs-jobs-and-load "$@"       \
+        2>/dev/null                                   \
     )";
 
     local -a cmake_args_="(${CMAKE_ARGS:-})";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.cpack.tmpl.sh
@@ -11,7 +11,7 @@
 #
 # Options that require values:
 #  -j,--parallel <num>                           Use <num> threads to compress in parallel
-#                                                (default: $(nproc))
+#                                                (default: $(nproc --all))
 #  -o,--out-dir <dir>                            copy cpack'd TGZ file into <dir>
 #                                                (default: none)
 # @_include_value_options rapids-select-cmake-install-args -h | tail -n-5 | head -n-2;
@@ -34,7 +34,7 @@ cpack_${CPP_LIB}_cpp() {
     fi
 
     eval "$(                                              \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}            \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}      \
         rapids-get-num-archs-jobs-and-load --archs 0 "$@" \
     )";
 

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh
@@ -18,10 +18,10 @@ build_${PY_LIB}_python_wheel() {
     local -;
     set -euo pipefail;
 
-    eval "$(                                    \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}  \
-        rapids-get-num-archs-jobs-and-load "$@" \
-        2>/dev/null                             \
+    eval "$(                                          \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
+        rapids-get-num-archs-jobs-and-load "$@"       \
+        2>/dev/null                                   \
     )";
 
     local py_lib="${PY_LIB}";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh
@@ -18,10 +18,10 @@ install_${PY_LIB}_python() {
     local -;
     set -euo pipefail;
 
-    eval "$(                                    \
-    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}  \
-        rapids-get-num-archs-jobs-and-load "$@" \
-        2>/dev/null                             \
+    eval "$(                                          \
+    PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
+        rapids-get-num-archs-jobs-and-load "$@"       \
+        2>/dev/null                                   \
     )";
 
     local py_lib="${PY_LIB}";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh
@@ -44,18 +44,18 @@ clone_${NAME}() {
 
     echo 'Cloning ${NAME}' 1>&2;
 
-    devcontainer-utils-clone-${GIT_HOST}-repo \
-        --tags                                \
-        --branch "${branch}"                  \
-        --ssh-url "${ssh_url}"                \
-        --https-url "${https_url}"            \
-        --recurse-submodules                  \
-        -j ${n_jobs:-$(nproc --ignore=1)}     \
-        -c checkout.defaultRemote=upstream    \
-        "${OPTS[@]}"                          \
-        --                                    \
-        "${upstream}"                         \
-        "${directory}"                        \
+    devcontainer-utils-clone-${GIT_HOST}-repo   \
+        --tags                                  \
+        --branch "${branch}"                    \
+        --ssh-url "${ssh_url}"                  \
+        --https-url "${https_url}"              \
+        --recurse-submodules                    \
+        -j ${n_jobs:-$(nproc --all --ignore=1)} \
+        -c checkout.defaultRemote=upstream      \
+        "${OPTS[@]}"                            \
+        --                                      \
+        "${upstream}"                           \
+        "${directory}"                          \
     ;
 
     git -C "${SRC_PATH}" config --add remote.upstream.fetch '^refs/heads/pull-request/*';

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.cpack.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.cpack.tmpl.sh
@@ -11,7 +11,7 @@
 #
 # Options that require values:
 #  -j,--parallel <num>                           Use <num> threads to compress in parallel.
-#                                                (default: $(nproc))
+#                                                (default: $(nproc --all))
 #  -o,--out-dir <dir>                            Copy cpack'd TGZ file into <dir>.
 #                                                (default: none)
 # @_include_value_options rapids-select-cmake-install-args -h | tail -n-5 | head -n-2;

diff --git a/features/src/ucx/devcontainer-feature.json b/features/src/ucx/devcontainer-feature.json
@@ -1,14 +1,15 @@
 {
   "name": "UCX",
   "id": "ucx",
-  "version": "24.6.0",
+  "version": "24.6.1",
   "description": "A feature to install UCX",
   "options": {
     "version": {
       "type": "string",
       "proposals": [
         "latest",
-        "1.15.0-rc3",
+        "1.16.0",
+        "1.15.0",
         "1.14.1"
       ],
       "default": "latest",

diff --git a/features/src/ucx/install.sh b/features/src/ucx/install.sh
@@ -87,7 +87,7 @@ build_and_install_ucx() {
             --with-rdmacm           \
             ${cuda:+--with-cuda=${CUDA_HOME:-/usr/local/cuda}};
 
-        make -j$(nproc);
+        make -j$(nproc --all);
         make install;
     )