icpx: error: ptxas command failed with exit code 255

[Jervie]

Hello,

I'm trying to install cloudsc_sycl on Nvidia A100 GPU using OneAPI DPC++ compiler. 

First, I download cloudsc_sycl from GitHub - ecmwf-ifs/dwarf-p-cloudsc at nams_pvc_sycl

Then I installed OneAPI for Nvidia GPU following this website: Install oneAPI for NVIDIA GPUs - Guides - oneAPI for NVIDIA® GPUs - Products - Codeplay Developer

Below is my environment:

-----------------------------------

CUDA Version: 12.4

OneAPI: l_BaseKit_p_2024.1.0.596_offline.sh and l_HPCKit_p_2024.1.0.560_offline.sh

DPC++ plugin: oneapi-for-nvidia-gpus-2024.1.0-cuda-12.0-linux.sh

-----------------------------------

Following are some settings in env.sh and toolchain.cmake under dwarf-p-cloudsc-nams/arch/ecmwf/hpc2020/intel-sycl/2024.1 dir:

-----------------------------------

env.sh:

export IntelSYCL_DIR="/opt/intel/oneapi/compiler/2024.1/lib/cmake/IntelSYCL"
export HDF5_ROOT="/usr/local/cloudsc_sycl_libs"

toolchain.cmake:

set(CMAKE_CXX_COMPILER "/opt/intel/oneapi/compiler/2024.1/bin/icpx")
set(CMAKE_C_COMPILER "/opt/intel/oneapi/compiler/2024.1/bin/icx")
set(CMAKE_Fortran_COMPILER "/opt/intel/oneapi/compiler/2024.1/bin/ifort")

set(CMAKE_C_FLAGS "-O3 -L/opt/intel/oneapi/compiler/2024.1/lib -qopenmp -fgpu-rdc")
set(CMAKE_CXX_FLAGS "-O3 -L/opt/intel/oneapi/compiler/2024.1/lib -qopenmp -lstdc++ -fgpu-rdc")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-early-optimizations -fsycl")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=nvptx64-nvidia-cuda")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xsycl-target-backend --cuda-gpu-arch=sm_80")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/opt/intel/oneapi/compiler/2024.1/include -I/usr/local/cuda/include")

-----------------------------------

Then I started to install cloudsc_sycl by running "./cloudsc-bundle build --clean --build-dir=build-sycl --with-gpu --with-sycl --arch=arch/ecmwf/hpc2020/intel-sycl/2024.1 -j 1" and met the following error:

-----------------------------------

[ 69%] Building CXX object cloudsc-dwarf/src/cloudsc_sycl/CMakeFiles/dwarf-cloudsc-scc-sycl-lib.dir/cloudsc/mycpu.cpp.o
icpx: warning: -Z-reserved-lib-stdc++: 'linker' input unused [-Wunused-command-line-argument]
[ 70%] Linking CXX shared library ../../../lib/libdwarf-cloudsc-scc-sycl-lib.so
ptxas fatal : Unresolved extern function 'fmin'
llvm-foreach:
icpx: error: ptxas command failed with exit code 255 (use -v to see invocation)
Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.1.0.20240308)
Target: x86_64-unknown-linux-gnu
Thread model: posix
InstalledDir: /opt/intel/oneapi/compiler/2024.1/bin/compiler
Configuration file: /opt/intel/oneapi/compiler/2024.1/bin/compiler/../icpx.cfg
icpx: note: diagnostic msg: Error generating preprocessed source(s).
make[2]: *** [cloudsc-dwarf/src/cloudsc_sycl/CMakeFiles/dwarf-cloudsc-scc-sycl-lib.dir/build.make:147: lib/libdwarf-cloudsc-scc-sycl-lib.so] Error 1
make[1]: *** [CMakeFiles/Makefile2:1335: cloudsc-dwarf/src/cloudsc_sycl/CMakeFiles/dwarf-cloudsc-scc-sycl-lib.dir/all] Error 2
make: *** [Makefile:166: all] Error 2

-----------------------------------

Any suggestions?

Thanks!

[Alex_Y_Intel]

Looks like you need to understand why that "fmin" function is missing. Have you tried the -v to check the invocation? 

[Jervie]

Hello,

I added -v as below:
set(CMAKE_C_FLAGS "-O3 -fgpu-rdc -v -L/opt/intel/oneapi/compiler/2024.1/lib -qopenmp -L/usr/local/cuda/lib64")
set(CMAKE_CXX_FLAGS "-O3 -fgpu-rdc -v -L/opt/intel/oneapi/compiler/2024.1/lib -qopenmp -lstdc++ -L/usr/local/cuda/lib64")

And I got the follwoing error messages:
-----------------------------
"/opt/intel/oneapi/compiler/2024.1/bin/compiler/llvm-foreach" --out-ext=o --in-file-list=/tmp/icpx-bb44e32033/load_state-sm_80-d9163a.s --in-replace=/tmp/icpx-bb44e32033/load_state-sm_80-d9163a.s --out-file-list=/tmp/icpx-bb44e32033/load_state-sm_80-3ab3e6.cubin --out-replace=/tmp/icpx-bb44e32033/load_state-sm_80-3ab3e6.cubin -- /usr/local/cuda-12.0/bin/ptxas -m64 -O3 -v --gpu-name sm_80 --output-file /tmp/icpx-bb44e32033/load_state-sm_80-3ab3e6.cubin /tmp/icpx-bb44e32033/load_state-sm_80-d9163a.s
ptxas info : 0 bytes gmem
ptxas info : 0 bytes gmem
ptxas fatal : Unresolved extern function 'fmin'
llvm-foreach:
icpx: error: ptxas command failed with exit code 255 (use -v to see invocation)
Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.1.0.20240308)
Target: x86_64-unknown-linux-gnu
Thread model: posix

[Jervie]

Adding more error messages:
-----------------------------
/opt/intel/oneapi/compiler/2024.1/bin/compiler/clang-offload-bundler -type=o -input=CMakeFiles/dwarf-cloudsc-scc-sycl-lib.dir/cloudsc/mycpu.cpp.o -list
/opt/intel/oneapi/compiler/2024.1/bin/compiler/clang-offload-bundler -type=ao -targets=sycl-fpga_aocx-intel-unknown -input=/usr/local/cuda/lib64/libcudadevrt.a -check-section -base-temp-dir=/tmp/icpx-e2ed2974b6
/opt/intel/oneapi/compiler/2024.1/bin/compiler/clang-offload-bundler -type=ao -targets=sycl-fpga_aocr-intel-unknown -input=/usr/local/cuda/lib64/libcudadevrt.a -check-section -base-temp-dir=/tmp/icpx-e2ed2974b6
/opt/intel/oneapi/compiler/2024.1/bin/compiler/clang-offload-bundler -type=ao -targets=sycl-fpga_aocr_emu-intel-unknown -input=/usr/local/cuda/lib64/libcudadevrt.a -check-section -base-temp-dir=/tmp/icpx-e2ed2974b6
/opt/intel/oneapi/compiler/2024.1/bin/compiler/clang-offload-bundler -type=ao -input=/usr/local/cuda/lib64/libcudadevrt.a -list
"/opt/intel/oneapi/compiler/2024.1/bin/compiler/llvm-foreach" --out-ext=o --in-file-list=/tmp/icpx-c2c23b019a/load_state-sm_80-c9c042.s --in-replace=/tmp/icpx-c2c23b019a/load_state-sm_80-c9c042.s --out-file-list=/tmp/icpx-c2c23b019a/load_state-sm_80-146846.cubin --out-replace=/tmp/icpx-c2c23b019a/load_state-sm_80-146846.cubin -- /usr/local/cuda-12.0/bin/ptxas -m64 -O3 -v --gpu-name sm_80 --output-file /tmp/icpx-c2c23b019a/load_state-sm_80-146846.cubin /tmp/icpx-c2c23b019a/load_state-sm_80-c9c042.s
ptxas info : 0 bytes gmem
ptxas info : 0 bytes gmem
ptxas fatal : Unresolved extern function 'fmin'
llvm-foreach:
icpx: error: ptxas command failed with exit code 255 (use -v to see invocation)
Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.1.0.20240308)
Target: x86_64-unknown-linux-gnu
Thread model: posix

I guess it may be related to CUDA. When using CUDA nvcc, adding -rdc=true as an compiler option may solve the "Unresolved extern function 'xxxxx' " issue. As for icpx, I found the compiler option "-fgpu-rdc", but it didn't solve my problem.