InvalidFunctionCall: Unexpected llvm intrinsic: llvm.fabs.f64 [Src: ../lib/SPIRV/SPIRVWriter.cpp:1625 ]

[freemin7]

I attempted to run the first example from https://github.com/SciML/DiffEqGPU.jl but only with oneArray instead of cu to allocate the arrays in Intel GPU space.

I got the following stack trace trying to execute:
@time sol = solve(prob,Tsit5())

InvalidFunctionCall: Unexpected llvm intrinsic: llvm.fabs.f64 [Src: ../lib/SPIRV/SPIRVWriter.cpp:1625  ]
Stack dump:
0.	Program arguments: /home/joto/.julia/artifacts/6b3e339c94c460d95f35b5c6cc186f3d829b31de/bin/llvm-spirv -o /tmp/jl_5uXC9h /tmp/jl_NKMykh 
1.	Running pass 'LLVMToSPIRV' on module '/tmp/jl_NKMykh'.
/usr/bin/../lib64/julia/libLLVM-9jl.so(_ZN4llvm3sys15PrintStackTraceERNS_11raw_ostreamE+0x2e)[0x7f0e5986688e]
/usr/bin/../lib64/julia/libLLVM-9jl.so(_ZN4llvm3sys17RunSignalHandlersEv+0x34)[0x7f0e59864ec4]
/usr/bin/../lib64/julia/libLLVM-9jl.so(+0x7bf002)[0x7f0e59865002]
/usr/bin/../lib64/libpthread.so.0(+0x14a90)[0x7f0e5c787a90]
/usr/bin/../lib64/libc.so.6(gsignal+0x145)[0x7f0e58bc59e5]
/usr/bin/../lib64/libc.so.6(abort+0x127)[0x7f0e58bae895]
/home/joto/.julia/artifacts/6b3e339c94c460d95f35b5c6cc186f3d829b31de/lib/libLLVMSPIRVLib.so.9jl(_ZN5SPIRV13SPIRVErrorLog10checkErrorEbNS_14SPIRVErrorCodeERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKcSB_j+0x358)[0x7f0e5c463bb8]
ERROR: failed process: Process(`/home/joto/.julia/artifacts/6b3e339c94c460d95f35b5c6cc186f3d829b31de/bin/llvm-spirv -o /tmp/jl_5uXC9h /tmp/jl_NKMykh`, ProcessSignaled(6)) [0]

Stacktrace:
 [1] pipeline_error at ./process.jl:525 [inlined]
 [2] run(::Cmd; wait::Bool) at ./process.jl:440
 [3] run at ./process.jl:438 [inlined]
 [4] #28 at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/spirv.jl:77 [inlined]
 [5] #2 at /home/joto/.julia/packages/JLLWrappers/KuIwt/src/runtime.jl:49 [inlined]
 [6] withenv(::JLLWrappers.var"#2#3"{GPUCompiler.var"#28#31"{String,String,LLVM.API.LLVMCodeGenFileType},String}, ::Pair{String,String}, ::Vararg{Pair{String,String},N} where N) at ./env.jl:161
 [7] withenv_executable_wrapper(::GPUCompiler.var"#28#31"{String,String,LLVM.API.LLVMCodeGenFileType}, ::String, ::String, ::String, ::Bool, ::Bool) at /home/joto/.julia/packages/JLLWrappers/KuIwt/src/runtime.jl:48
 [8] #invokelatest#1 at ./essentials.jl:710 [inlined]
 [9] invokelatest at ./essentials.jl:709 [inlined]
 [10] #llvm_spirv#7 at /home/joto/.julia/packages/JLLWrappers/KuIwt/src/products/executable_generators.jl:7 [inlined]
 [11] llvm_spirv(::Function) at /home/joto/.julia/packages/JLLWrappers/KuIwt/src/products/executable_generators.jl:7
 [12] (::GPUCompiler.var"#27#30"{String,LLVM.API.LLVMCodeGenFileType})(::String, ::IOStream) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/spirv.jl:71
 [13] mktemp(::GPUCompiler.var"#27#30"{String,LLVM.API.LLVMCodeGenFileType}, ::String) at ./file.jl:659
 [14] mktemp at ./file.jl:657 [inlined]
 [15] #26 at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/spirv.jl:70 [inlined]
 [16] mktemp(::GPUCompiler.var"#26#29"{LLVM.Module,LLVM.API.LLVMCodeGenFileType}, ::String) at ./file.jl:659
 [17] mktemp at ./file.jl:657 [inlined]
 [18] mcgen(::GPUCompiler.CompilerJob{GPUCompiler.SPIRVCompilerTarget,oneAPI.oneAPICompilerParams}, ::LLVM.Module, ::LLVM.Function, ::LLVM.API.LLVMCodeGenFileType) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/spirv.jl:65
 [19] macro expansion at /home/joto/.julia/packages/TimerOutputs/ZmKD7/src/TimerOutput.jl:206 [inlined]
 [20] macro expansion at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/driver.jl:254 [inlined]
 [21] macro expansion at /home/joto/.julia/packages/TimerOutputs/ZmKD7/src/TimerOutput.jl:206 [inlined]
 [22] codegen(::Symbol, ::GPUCompiler.CompilerJob; libraries::Bool, deferred_codegen::Bool, optimize::Bool, strip::Bool, validate::Bool, only_entry::Bool) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/driver.jl:248
 [23] compile(::Symbol, ::GPUCompiler.CompilerJob; libraries::Bool, deferred_codegen::Bool, optimize::Bool, strip::Bool, validate::Bool, only_entry::Bool) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/driver.jl:39
 [24] compile at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/driver.jl:35 [inlined]
 [25] #zefunction_compile#53 at /home/joto/.julia/packages/oneAPI/UcBkX/src/compiler/execution.jl:128 [inlined]
 [26] zefunction_compile(::GPUCompiler.FunctionSpec{GPUArrays.var"#broadcast_kernel#12",Tuple{oneAPI.oneKernelContext,oneDeviceArray{Float64,1,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(muladd),Tuple{Base.Broadcast.Broadcasted{oneAPI.oneArrayStyle{1},Nothing,typeof(DiffEqBase.ODE_DEFAULT_NORM),Tuple{Base.Broadcast.Extruded{oneDeviceArray{Float64,1,1},Tuple{Bool},Tuple{Int64}},Float32}},Float64,Float64}},Int64}}) at /home/joto/.julia/packages/oneAPI/UcBkX/src/compiler/execution.jl:125
 [27] check_cache(::Dict{UInt64,Any}, ::Any, ::Any, ::GPUCompiler.FunctionSpec{GPUArrays.var"#broadcast_kernel#12",Tuple{oneAPI.oneKernelContext,oneDeviceArray{Float64,1,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(muladd),Tuple{Base.Broadcast.Broadcasted{oneAPI.oneArrayStyle{1},Nothing,typeof(DiffEqBase.ODE_DEFAULT_NORM),Tuple{Base.Broadcast.Extruded{oneDeviceArray{Float64,1,1},Tuple{Bool},Tuple{Int64}},Float32}},Float64,Float64}},Int64}}, ::UInt64; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/cache.jl:40
 [28] broadcast_kernel at /home/joto/.julia/packages/GPUArrays/ZxsKE/src/host/broadcast.jl:60 [inlined]
 [29] cached_compilation(::Dict{UInt64,Any}, ::typeof(oneAPI.zefunction_compile), ::typeof(oneAPI.zefunction_link), ::GPUCompiler.FunctionSpec{GPUArrays.var"#broadcast_kernel#12",Tuple{oneAPI.oneKernelContext,oneDeviceArray{Float64,1,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(muladd),Tuple{Base.Broadcast.Broadcasted{oneAPI.oneArrayStyle{1},Nothing,typeof(DiffEqBase.ODE_DEFAULT_NORM),Tuple{Base.Broadcast.Extruded{oneDeviceArray{Float64,1,1},Tuple{Bool},Tuple{Int64}},Float32}},Float64,Float64}},Int64}}; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/cache.jl:0
 [30] cached_compilation(::Dict{UInt64,Any}, ::Function, ::Function, ::GPUCompiler.FunctionSpec{GPUArrays.var"#broadcast_kernel#12",Tuple{oneAPI.oneKernelContext,oneDeviceArray{Float64,1,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(muladd),Tuple{Base.Broadcast.Broadcasted{oneAPI.oneArrayStyle{1},Nothing,typeof(DiffEqBase.ODE_DEFAULT_NORM),Tuple{Base.Broadcast.Extruded{oneDeviceArray{Float64,1,1},Tuple{Bool},Tuple{Int64}},Float32}},Float64,Float64}},Int64}}) at /home/joto/.julia/packages/GPUCompiler/uTpNx/src/cache.jl:65
 [31] zefunction(::Function, ::Type{T} where T; name::Nothing, kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /home/joto/.julia/packages/oneAPI/UcBkX/src/compiler/execution.jl:118
 [32] macro expansion at /home/joto/.julia/packages/oneAPI/UcBkX/src/compiler/execution.jl:34 [inlined]
 [33] #gpu_call#81 at /home/joto/.julia/packages/oneAPI/UcBkX/src/gpuarrays.jl:29 [inlined]
 [34] #gpu_call#1 at /home/joto/.julia/packages/GPUArrays/ZxsKE/src/device/execution.jl:67 [inlined]
 [35] copyto! at /home/joto/.julia/packages/GPUArrays/ZxsKE/src/host/broadcast.jl:68 [inlined]
 [36] copyto! at ./broadcast.jl:886 [inlined]
 [37] materialize! at ./broadcast.jl:848 [inlined]
 [38] materialize! at ./broadcast.jl:845 [inlined]
 [39] macro expansion at /home/joto/.julia/packages/DiffEqBase/V7P18/src/diffeqfastbc.jl:88 [inlined]
 [40] ode_determine_initdt(::oneArray{Float64,1}, ::Float32, ::Float32, ::Float32, ::Float64, ::Float64, ::typeof(DiffEqBase.ODE_DEFAULT_NORM), ::ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}, ::OrdinaryDiffEq.ODEIntegrator{Tsit5,true,oneArray{Float64,1},Nothing,Float32,DiffEqBase.NullParameters,Float32,Float32,Float32,Array{oneArray{Float64,1},1},ODESolution{Float64,2,Array{oneArray{Float64,1},1},Nothing,Nothing,Array{Float32,1},Array{Array{oneArray{Float64,1},1},1},ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem},Tsit5,OrdinaryDiffEq.InterpolationData{ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Array{oneArray{Float64,1},1},Array{Float32,1},Array{Array{oneArray{Float64,1},1},1},OrdinaryDiffEq.Tsit5Cache{oneArray{Float64,1},oneArray{Float64,1},oneArray{Float64,1},OrdinaryDiffEq.Tsit5ConstantCache{Float64,Float32}}},DiffEqBase.DEStats},ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},OrdinaryDiffEq.Tsit5Cache{oneArray{Float64,1},oneArray{Float64,1},oneArray{Float64,1},OrdinaryDiffEq.Tsit5ConstantCache{Float64,Float32}},OrdinaryDiffEq.DEOptions{Float64,Float64,Float32,Float32,typeof(DiffEqBase.ODE_DEFAULT_NORM),typeof(opnorm),CallbackSet{Tuple{},Tuple{}},typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN),typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE),typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK),DataStructures.BinaryHeap{Float32,DataStructures.LessThan},DataStructures.BinaryHeap{Float32,DataStructures.LessThan},Nothing,Nothing,Int64,Tuple{},Tuple{},Tuple{}},oneArray{Float64,1},Float64,Nothing,OrdinaryDiffEq.DefaultInit}) at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/initdt.jl:11
 [41] auto_dt_reset! at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/integrators/integrator_interface.jl:297 [inlined]
 [42] handle_dt!(::OrdinaryDiffEq.ODEIntegrator{Tsit5,true,oneArray{Float64,1},Nothing,Float32,DiffEqBase.NullParameters,Float32,Float32,Float32,Array{oneArray{Float64,1},1},ODESolution{Float64,2,Array{oneArray{Float64,1},1},Nothing,Nothing,Array{Float32,1},Array{Array{oneArray{Float64,1},1},1},ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem},Tsit5,OrdinaryDiffEq.InterpolationData{ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Array{oneArray{Float64,1},1},Array{Float32,1},Array{Array{oneArray{Float64,1},1},1},OrdinaryDiffEq.Tsit5Cache{oneArray{Float64,1},oneArray{Float64,1},oneArray{Float64,1},OrdinaryDiffEq.Tsit5ConstantCache{Float64,Float32}}},DiffEqBase.DEStats},ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},OrdinaryDiffEq.Tsit5Cache{oneArray{Float64,1},oneArray{Float64,1},oneArray{Float64,1},OrdinaryDiffEq.Tsit5ConstantCache{Float64,Float32}},OrdinaryDiffEq.DEOptions{Float64,Float64,Float32,Float32,typeof(DiffEqBase.ODE_DEFAULT_NORM),typeof(opnorm),CallbackSet{Tuple{},Tuple{}},typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN),typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE),typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK),DataStructures.BinaryHeap{Float32,DataStructures.LessThan},DataStructures.BinaryHeap{Float32,DataStructures.LessThan},Nothing,Nothing,Int64,Tuple{},Tuple{},Tuple{}},oneArray{Float64,1},Float64,Nothing,OrdinaryDiffEq.DefaultInit}) at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/solve.jl:453
 [43] __init(::ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}, ::Tsit5, ::Tuple{}, ::Tuple{}, ::Tuple{}, ::Type{Val{true}}; saveat::Tuple{}, tstops::Tuple{}, d_discontinuities::Tuple{}, save_idxs::Nothing, save_everystep::Bool, save_on::Bool, save_start::Bool, save_end::Bool, callback::Nothing, dense::Bool, calck::Bool, dt::Float32, dtmin::Nothing, dtmax::Float32, force_dtmin::Bool, adaptive::Bool, gamma::Rational{Int64}, abstol::Nothing, reltol::Nothing, qmin::Rational{Int64}, qmax::Int64, qsteady_min::Int64, qsteady_max::Int64, qoldinit::Rational{Int64}, fullnormalize::Bool, failfactor::Int64, beta1::Nothing, beta2::Nothing, maxiters::Int64, internalnorm::typeof(DiffEqBase.ODE_DEFAULT_NORM), internalopnorm::typeof(opnorm), isoutofdomain::typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), unstable_check::typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), verbose::Bool, timeseries_errors::Bool, dense_errors::Bool, advance_to_tstop::Bool, stop_at_next_tstop::Bool, initialize_save::Bool, progress::Bool, progress_steps::Int64, progress_name::String, progress_message::typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), userdata::Nothing, allow_extrapolation::Bool, initialize_integrator::Bool, alias_u0::Bool, alias_du0::Bool, initializealg::OrdinaryDiffEq.DefaultInit, kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/solve.jl:416
 [44] __init(::ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}, ::Tsit5, ::Tuple{}, ::Tuple{}, ::Tuple{}, ::Type{Val{true}}) at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/solve.jl:66 (repeats 5 times)
 [45] #__solve#391 at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/solve.jl:4 [inlined]
 [46] __solve at /home/joto/.julia/packages/OrdinaryDiffEq/VPJBD/src/solve.jl:4 [inlined]
 [47] solve_call(::ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}, ::Tsit5; merge_callbacks::Bool, kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:92
 [48] solve_call at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:65 [inlined]
 [49] #solve_up#461 at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:114 [inlined]
 [50] solve_up at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:107 [inlined]
 [51] #solve#460 at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:102 [inlined]
 [52] solve(::ODEProblem{oneArray{Float64,1},Tuple{Float32,Float32},true,DiffEqBase.NullParameters,ODEFunction{true,typeof(f),UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}, ::Tsit5) at /home/joto/.julia/packages/DiffEqBase/V7P18/src/solve.jl:100
 [53] top-level scope at ./timing.jl:174 [inlined]
 [54] top-level scope at ./REPL[8]:0

[maleadt]

Dup of JuliaGPU/GPUCompiler.jl#112. You need to use oneAPI.abs instead of abs; this is the same as with CUDA.jl. If DiffEqGPU.jl would use regular broadcast, we'd detect this situation (for simple cases). I assume there's some support there for CUDA.jl functions, but not for oneAPI.jl.

Anyway, see the linked issue, we hope to resolve this in a better way soon. For now, you (or the packages you use) need to call oneAPI.abs instead of abs (the same applies for other essential math functions).