Issues building CUDA with Cmake on kirkstone

[jswang]

We are migrating from gatesgarth to kirkstone and running into issues building CUDA with CMake on kirkstone. We are building on a Jetson AGX Xavier.

We are noticing that there seems to be a missing library for gcc, namely /usr/local/cuda-10.2/include/cuda/std/detail/libcxx/include isn't being included by gcc when building CMakeCUDACompilerId.cu. This previously worked on gatesgarth, but gatesgarth was using the native gcc version, whereas kirkstone is natively gcc-11.2 and includes gcc-8.5 for backwards compatibility with CUDA 10.2

Here's the top of our CMakeLists.txt:

cmake_minimum_required(VERSION 3.18)
set(CMAKE_SKIP_RPATH TRUE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--as-needed -Wl,--gc-sections")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--as-needed -Wl,--gc-sections")
set(CMAKE_CUDA_ARCHITECTURES "72")
set(CMAKE_C_COMPILER /usr/bin/aarch64-poky-linux-gcc-8.5.0)
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-poky-linux-g++-8.5.0)
set(CMAKE_CUDA_HOST_COMPILER /usr/bin/aarch64-poky-linux-gcc-8.5.0)
if (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Linux")
  enable_language(CUDA)
endif ()

The build is failing at this line:

CMake Error at /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake:724 (message):
  Compiling the CUDA compiler identification source file
  "CMakeCUDACompilerId.cu" failed.
  Compiler: /usr/local/cuda-10.2/bin/nvcc
  Build flags:
  Id flags:
  --keep;--keep-dir;tmp;-ccbin=/usr/bin/aarch64-poky-linux-gcc-8.5.0;-gencode=arch=compute_72,code=sm_72
  -v
  The output was:
  1
  #$ _NVVM_BRANCH_=nvvm
  #$ _SPACE_=
  #$ _CUDART_=cudart
  #$ _HERE_=/usr/local/cuda-10.2/bin
  #$ _THERE_=/usr/local/cuda-10.2/bin
  #$ _TARGET_SIZE_=
  #$ _TARGET_DIR_=
  #$ _TARGET_SIZE_=64
  #$ TOP=/usr/local/cuda-10.2/bin/..
  #$ NVVMIR_LIBRARY_DIR=/usr/local/cuda-10.2/bin/../nvvm/libdevice
  #$ LD_LIBRARY_PATH=/usr/local/cuda-10.2/bin/../lib:
  #$
  PATH=/usr/local/cuda-10.2/bin/../nvvm/bin:/usr/local/cuda-10.2/bin:/home/root/.cargo/bin:/usr/local/cuda-10.2/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin
  #$ INCLUDES="-I/usr/local/cuda-10.2/bin/..//include"
  #$ LIBRARIES= "-L/usr/local/cuda-10.2/bin/..//lib64/stubs"
  "-L/usr/local/cuda-10.2/bin/..//lib64"
  #$ CUDAFE_FLAGS=
  #$ PTXAS_FLAGS=
  #$ rm tmp/a_dlink.reg.c
  #$ "/usr/bin"/aarch64-poky-linux-gcc-8.5.0 -D__CUDA_ARCH__=720 -E -x c++
  -DCUDA_DOUBLE_MATH_FUNCTIONS -D__CUDACC__ -D__NVCC__
  "-I/usr/local/cuda-10.2/bin/..//include" -D__CUDACC_VER_MAJOR__=10
  -D__CUDACC_VER_MINOR__=2 -D__CUDACC_VER_BUILD__=300 -include
  "cuda_runtime.h" "CMakeCUDACompilerId.cu" -o
  "tmp/CMakeCUDACompilerId.cpp1.ii"
  In file included from
  /usr/local/cuda-10.2/bin/..//include/cuda_runtime.h:115,

                   from <command-line>:

  /usr/local/cuda-10.2/bin/..//include/crt/common_functions.h:103:10: fatal
  error: new: No such file or directory

   #include <new>
            ^~~~~

  compilation terminated.

  # --error 0x1 --
Call Stack (most recent call first):
  /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake:6 (CMAKE_DETERMINE_COMPILER_ID_BUILD)
  /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake:48 (__determine_compiler_id_test)
  /usr/share/cmake-3.22/Modules/CMakeDetermineCUDACompiler.cmake:298 (CMAKE_DETERMINE_COMPILER_ID)
  CMakeLists.txt:20 (enable_language)


   Called from: [5]     /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake
                [4]     /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake
                [3]     /usr/share/cmake-3.22/Modules/CMakeDetermineCompilerId.cmake
                [2]     /usr/share/cmake-3.22/Modules/CMakeDetermineCUDACompiler.cmake
                [1]     /ephemeral_data/monorail/perception/application/CMakeLists.txt

I am able to recreate it by running:

cd build/CMakeFiles/3.22.3/CompilerIdCUDA
gcc -D__CUDA_ARCH__=720 -E -x c++ -DCUDA_DOUBLE_MATH_FUNCTIONS   -D__CUDACC__ -D__NVCC__ “-I/usr/local/cuda-10.2/include”  -D__CUDACC_VER_MAJOR__=10 -D__CUDACC_VER_MINOR__=2   -D__CUDACC_VER_BUILD__=300 -include “cuda_runtime.h”   “CMakeCUDACompilerId.cu” -o “tmp/CMakeCUDACompilerId.cpp1.ii”

However, if I change the includes to include the path to the missing new file, compilation is successful:

gcc -D__CUDA_ARCH__=720 -E -x c++ -DCUDA_DOUBLE_MATH_FUNCTIONS   -D__CUDACC__ -D__NVCC__ “-I/usr/local/cuda-10.2/include” “-I/usr/local/cuda-10.2/include/cuda/std/detail/libcxx/include”  -D__CUDACC_VER_MAJOR__=10 -D__CUDACC_VER_MINOR__=2   -D__CUDACC_VER_BUILD__=300 -include “cuda_runtime.h”   “CMakeCUDACompilerId.cu” -o “tmp/CMakeCUDACompilerId.cpp1.ii”

Any idea what's going on here?

[madisongh]

Are you running this on the Jetson target?

If so, do you see the C++ 8.5.0 headers under /usr/include/c++/8.5.0? That's the directory where the g++ 8 compiler would look for them. If those headers aren't there, make sure you install gcc-8-runtime-dev into your target image.

[jackatparallel]

Thank you for your help so far!! If I swap these, I get the same error:

Compiling the CUDA compiler identification source file "CMakeCUDACompilerId.cu" failed.
Compiler: /usr/local/cuda-10.2/bin/nvcc 
Build flags: 
Id flags: --keep;--keep-dir;tmp;-ccbin=/usr/bin/aarch64-poky-linux-g++-8.5.0;-gencode=arch=compute_72,code=sm_72 -v

The output was:
1
#$ _NVVM_BRANCH_=nvvm
#$ _SPACE_= 
#$ _CUDART_=cudart
#$ _HERE_=/usr/local/cuda-10.2/bin
#$ _THERE_=/usr/local/cuda-10.2/bin
#$ _TARGET_SIZE_=
#$ _TARGET_DIR_=
#$ _TARGET_SIZE_=64
#$ TOP=/usr/local/cuda-10.2/bin/..
#$ NVVMIR_LIBRARY_DIR=/usr/local/cuda-10.2/bin/../nvvm/libdevice
#$ LD_LIBRARY_PATH=/usr/local/cuda-10.2/bin/../lib:
#$ PATH=/usr/local/cuda-10.2/bin/../nvvm/bin:/usr/local/cuda-10.2/bin:/home/root/.vscode-server/bin/57fd6d0195bb9b9d1b49f6da5db789060795de47/bin/remote-cli:/home/root/.cargo/bin:/usr/local/cuda-10.2/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin
#$ INCLUDES="-I/usr/local/cuda-10.2/bin/..//include"  
#$ LIBRARIES=  "-L/usr/local/cuda-10.2/bin/..//lib64/stubs" "-L/usr/local/cuda-10.2/bin/..//lib64"
#$ CUDAFE_FLAGS=
#$ PTXAS_FLAGS=
#$ rm tmp/a_dlink.reg.c
#$ "/usr/bin"/aarch64-poky-linux-g++-8.5.0 -D__CUDA_ARCH__=720 -E -x c++  -DCUDA_DOUBLE_MATH_FUNCTIONS -D__CUDACC__ -D__NVCC__  "-I/usr/local/cuda-10.2/bin/..//include"    -D__CUDACC_VER_MAJOR__=10 -D__CUDACC_VER_MINOR__=2 -D__CUDACC_VER_BUILD__=300 -include "cuda_runtime.h" "CMakeCUDACompilerId.cu" -o "tmp/CMakeCUDACompilerId.cpp1.ii" 
In file included from /usr/local/cuda-10.2/bin/..//include/cuda_runtime.h:115,
                 from <command-line>:
/usr/local/cuda-10.2/bin/..//include/crt/common_functions.h:103:10: fatal error: new: No such file or directory
 #include <new>
          ^~~~~
compilation terminated.
# --error 0x1 --

Adding the --compiler-options=... flag I mentioned before also has the same behavior as gcc. Here's a snippet from our development image's recipe file with all the CUDA and gcc/g++ related recipes:

IMAGE_INSTALL += "cuda-cudart \
                  cuda-nvcc \
                  cuda-nvcc-headers"
IMAGE_INSTALL += "cuda-path"
IMAGE_INSTALL += "cuda-toolkit"
IMAGE_INSTALL += "cudnn"
IMAGE_INSTALL += "cmake"
IMAGE_INSTALL += "deepstream-6.0"
IMAGE_INSTALL += "deepstream-6.0-sources"
IMAGE_INSTALL += "gcc-8 \
                  gcc-8-plugins \
                  gcc-8-symlinks \
                  gcc-8-dev \
                  gcc-8-runtime-dev \
                  libgcc-8-dev \
                  cpp-8 \
                  cpp-8-symlinks \
                  g++-8 \
                  g++-8-symlinks"
...
IMAGE_INSTALL += "libstdc++ \
                  libstdc++-staticdev"
...
EXTRA_IMAGE_FEATURES += "package-management \
                         tools-sdk \
                         tools-debug \
                         dbg-pkgs \
                         src-pkgs \
                         "

cuda_path is a custom recipe to add /usr/local/cuda-10.2/bin to PATH

[madisongh]

Hmm. What is the compiler reporting as its header search paths? Use:

$ $(/usr/bin/aarch64-poky-linux-g++-8.5.0 -print-prog-name=cc1plus) -v

and that should list out the include paths for "" and '<>' header searches.

[jackatparallel]

Oooh, good idea:

 $ $(/usr/bin/aarch64-poky-linux-g++-8.5.0 -print-prog-name=cc1plus) -v
ignoring nonexistent directory "/usr/lib/gcc/aarch64-poky-linux/8.5.0/include/c++"
ignoring nonexistent directory "/usr/lib/gcc/aarch64-poky-linux/8.5.0/include/c++/aarch64-poky-linux"
ignoring nonexistent directory "/usr/lib/gcc/aarch64-poky-linux/8.5.0/include/c++/backward"
ignoring nonexistent directory "/usr/local/include"
ignoring nonexistent directory "/usr/lib/gcc/aarch64-poky-linux/8.5.0/../../../../aarch64-poky-linux/include"
#include "..." search starts here:
#include <...> search starts here:
 /usr/lib/gcc/aarch64-poky-linux/8.5.0/include
 /usr/lib/aarch64-poky-linux/8.5.0/include
 /usr/lib/gcc/aarch64-poky-linux/8.5.0/include-fixed
 /usr/include
End of search list.

Looks like somehow g++ got configured to look for that include/c++ directory in the wrong place

[madisongh]

OK, I've found the configuration setting for the toolchain that was causing this, and have updated the recipes to fix it so the compiler gets the right path under /usr/include built in instead. Merged to both kirkstone and master.

[jackatparallel]

resolved! thank you