[Build] Building onnxruntime_test_all

[gedoensmax]

Describe the issue

I am not able to build onnxruntime with all unittests and CUDA due to linking against cudart here:

onnxruntime/cmake/onnxruntime_unittests.cmake

Lines 67 to 71 in 91b2e66

	if(onnxruntime_USE_CUDA)
	#XXX: we should not need to do this. onnxruntime_test_all.exe should not have direct dependency on CUDA DLLs,
	# otherwise it will impact when CUDA DLLs can be unloaded.
	target_link_libraries(${_UT_TARGET} PRIVATE cudart)
	endif()

The solution to this is to add the line target_link_directories(${_UT_TARGET} PRIVATE ${onnxruntime_CUDA_HOME}/lib/x64)

I would like to understand why cudart linkin is managed without using CMake built in find_package(CUDAToolkit) and then linking to CUDA::cudart ? I am happy to integrate this change if it is something that would be welcome.

Urgency

No response

Target platform

Windows 11

Build script

.\build.bat --build_dir build --build_shared_lib
      --skip_tests --parallel 
      --use_dml --use_cuda --use_cache
      --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON 
      --cuda_home "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.3"
      --cudnn_home "C:/cudnn" --cmake_generator "Ninja"

Error / output

cmd.exe /C "cmd.exe /C "cd /D C:\runner\builds\z1SQuGxmk\0\devtechproviz\dl\onnxruntime\build\RelWithDebInfo && "C:\Program Files\CMake\bin\cmake.exe" -E copy_directory C:/runner/builds/z1SQuGxmk/0/devtechproviz/dl/onnxruntime/onnxruntime/test/testdata C:/runner/builds/z1SQuGxmk/0/devtechproviz/dl/onnxruntime/build/RelWithDebInfo/testdata && cd /D C:\runner\builds\z1SQuGxmk\0\devtechproviz\dl\onnxruntime\build\RelWithDebInfo && "C:\Program Files\CMake\bin\cmake.exe" -E copy_directory C:/runner/builds/z1SQuGxmk/0/devtechproviz/dl/onnxruntime/samples C:/runner/builds/z1SQuGxmk/0/devtechproviz/dl/onnxruntime/build/RelWithDebInfo/samples && cd C:\runner\builds\z1SQuGxmk\0\devtechproviz\dl\onnxruntime\build\RelWithDebInfo" && "C:\Program Files\CMake\bin\cmake.exe" -E vs_link_exe --intdir=CMakeFiles\onnxruntime_test_all.dir --rc=C:\PROGRA~2\WINDOW~4\10\bin\100226~1.0\x64\rc.exe --mt=C:\PROGRA~2\WINDOW~4\10\bin\100226~1.0\x64\mt.exe --manifests  -- C:\PROGRA~2\MICROS~2\2022\BUILDT~1\VC\Tools\MSVC\1439~1.335\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\onnxruntime_test_all.rsp  /out:onnxruntime_test_all.exe /implib:onnxruntime_test_all.lib /pdb:onnxruntime_test_all.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /OPT:REF,ICF,LBR /INCREMENTAL:NO /subsystem:console  /CETCOMPAT  && cd ."
LINK: command "C:\PROGRA~2\MICROS~2\2022\BUILDT~1\VC\Tools\MSVC\1439~1.335\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\onnxruntime_test_all.rsp /out:onnxruntime_test_all.exe /implib:onnxruntime_test_all.lib /pdb:onnxruntime_test_all.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /OPT:REF,ICF,LBR /INCREMENTAL:NO /subsystem:console /CETCOMPAT /MANIFEST:EMBED,ID=1" failed (exit code 1181) with the following output:
LINK : fatal error LNK1181: cannot open input file 'cudart.lib'

Visual Studio Version

2022

GCC / Compiler Version

No response

[skottmckay]

@snnn any reason not to use FindPackage for the CUDA libraries?

We might need to set CUDAToolkit_ROOT to the cuda_home value when calling cmake though.

https://cmake.org/cmake/help/latest/module/FindCUDAToolkit.html

[tianleiwu]

Example build script for Windows:

IF {%VCToolsVersion%}=={} call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64

set CudaToolkitDir=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2

.\build.bat --config Debug --build_dir .\build --build_shared_lib --parallel --build_wheel ^
            --cmake_generator "Visual Studio 17 2022" ^
            --use_cuda --cuda_version 12.2 --cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2" ^
            --cudnn_home d:\nvidia\cudnn-windows-x86_64-8.9.4.25_cuda12-archive ^
            --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=89 --skip_tests

[snnn]

@snnn any reason not to use FindPackage for the CUDA libraries?

We might need to set CUDAToolkit_ROOT to the cuda_home value when calling cmake though.

https://cmake.org/cmake/help/latest/module/FindCUDAToolkit.html

CMake has 3 offerings for this:

1. enable_language(CUDA)
2. [Deprecated] FindCUDA
3. FindCUDAToolkit

We are using the first one. And it is enough for all our needs. Therefore we are not using the third one. The issue @gedoensmax because as I put the in comments: onnxruntime_test_all.exe should not have direct dependency on CUDA DLL, but it does so. In my opinion we should revert #16161 and find another way to test the internal CUDA code.

[gedoensmax]

While I agree with that it is suboptimal to link cedar to the test, I would still resolve the CUDA libs with the find_package call. I'll push a PR.
This makes these link directories not needed:

onnxruntime/cmake/external/onnxruntime_external_deps.cmake

Lines 557 to 570 in 91b2e66

	if (onnxruntime_USE_CUDA)
	#TODO: combine onnxruntime_CUDNN_HOME and onnxruntime_CUDA_HOME, assume they are the same
	if (WIN32)
	if(onnxruntime_CUDNN_HOME)
	list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib ${onnxruntime_CUDNN_HOME}/lib/x64)
	endif()
	list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDA_HOME}/x64/lib64)
	else()
	if(onnxruntime_CUDNN_HOME)
	list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDNN_HOME}/lib ${onnxruntime_CUDNN_HOME}/lib64)
	endif()
	list(APPEND onnxruntime_LINK_DIRS ${onnxruntime_CUDA_HOME}/lib64)
	endif()
	endif()