Add support for Float32 (#187)

* float32 on cpu works

* tests passing

* general precision

* gpu test passing

* everything in HSL works

* first draft done

* export linear solvers

* krylov subtyping

* hsl build improvement, hopefully final

* HSL build improvment - no fakemetis

* addressed francois's comments

* remove unncessary

* typo fix

lib/MadNLPGPU/src/MadNLPGPU.jl

@@ -2,19 +2,23 @@ module MadNLPGPU
 
 import LinearAlgebra
 # CUDA
-import CUDA: CUBLAS, CUSOLVER, CuVector, CuMatrix, CuArray, R_64F, has_cuda, @allowscalar, runtime_version
+import CUDA: CUDA, CUBLAS, CUSOLVER, CuVector, CuMatrix, CuArray, R_64F, has_cuda, @allowscalar, runtime_version
+import .CUSOLVER:
+    libcusolver, cusolverStatus_t, CuPtr, cudaDataType, cublasFillMode_t, cusolverDnHandle_t, dense_handle
+import .CUBLAS: handle, CUBLAS_DIAG_NON_UNIT,
+    CUBLAS_FILL_MODE_LOWER, CUBLAS_FILL_MODE_UPPER, CUBLAS_SIDE_LEFT, CUBLAS_OP_N, CUBLAS_OP_T
+
 # Kernels
 import KernelAbstractions: @kernel, @index, wait, Event
 import CUDAKernels: CUDADevice
 
 import MadNLP
-
 import MadNLP:
     @kwdef, Logger, @debug, @warn, @error,
     AbstractOptions, AbstractLinearSolver, AbstractNLPModel, set_options!,
     SymbolicException,FactorizationException,SolveException,InertiaException,
     introduce, factorize!, solve!, improve!, is_inertia, inertia, tril_to_full!,
-    LapackOptions, input_type
+    LapackOptions, input_type, is_supported
 
 
 

lib/MadNLPGPU/src/interface.jl

@@ -1,22 +1,22 @@
 
-function CuInteriorPointSolver(nlp::AbstractNLPModel;
+function CuInteriorPointSolver(nlp::AbstractNLPModel{T};
     option_dict::Dict{Symbol,Any}=Dict{Symbol,Any}(), kwargs...
-)
+) where T
     opt = MadNLP.Options(linear_solver=LapackGPUSolver)
     MadNLP.set_options!(opt,option_dict,kwargs)
     MadNLP.check_option_sanity(opt)
 
     KKTSystem = if (opt.kkt_system == MadNLP.SPARSE_KKT_SYSTEM) || (opt.kkt_system == MadNLP.SPARSE_UNREDUCED_KKT_SYSTEM)
         error("Sparse KKT system are currently not supported on CUDA GPU.\n" *
-              "Please use `DENSE_KKT_SYSTEM` or `DENSE_CONDENSED_KKT_SYSTEM` instead.")
+            "Please use `DENSE_KKT_SYSTEM` or `DENSE_CONDENSED_KKT_SYSTEM` instead.")
     elseif opt.kkt_system == MadNLP.DENSE_KKT_SYSTEM
-        MT = CuMatrix{Float64}
-        VT = CuVector{Float64}
-        MadNLP.DenseKKTSystem{Float64, VT, MT}
+        MT = CuMatrix{T}
+        VT = CuVector{T}
+        MadNLP.DenseKKTSystem{T, VT, MT}
     elseif opt.kkt_system == MadNLP.DENSE_CONDENSED_KKT_SYSTEM
-        MT = CuMatrix{Float64}
-        VT = CuVector{Float64}
-        MadNLP.DenseCondensedKKTSystem{Float64, VT, MT}
+        MT = CuMatrix{T}
+        VT = CuVector{T}
+        MadNLP.DenseCondensedKKTSystem{T, VT, MT}
     end
-    return MadNLP.InteriorPointSolver{KKTSystem}(nlp, opt; option_linear_solver=option_dict)
+    return MadNLP.InteriorPointSolver{T,KKTSystem}(nlp, opt; option_linear_solver=option_dict)
 end

lib/MadNLPGPU/test/densekkt_gpu.jl

@@ -9,32 +9,32 @@ function _compare_gpu_with_cpu(KKTSystem, n, m, ind_fixed)
     elseif (KKTSystem == MadNLP.DenseCondensedKKTSystem)
         MadNLP.DENSE_CONDENSED_KKT_SYSTEM
     end
-    # Define options
-    madnlp_options = Dict{Symbol, Any}(
-        :kkt_system=>opt_kkt,
-        :linear_solver=>LapackGPUSolver,
-        :print_level=>MadNLP.ERROR,
-    )
-
-    nlp = MadNLPTests.DenseDummyQP(; n=n, m=m, fixed_variables=ind_fixed)
-
-    # Solve on CPU
-    h_ips = MadNLP.InteriorPointSolver(nlp; option_dict=copy(madnlp_options))
-    MadNLP.optimize!(h_ips)
-
-    # Solve on GPU
-    d_ips = MadNLPGPU.CuInteriorPointSolver(nlp; option_dict=copy(madnlp_options))
-    MadNLP.optimize!(d_ips)
-
-    T = Float64
-    VT = CuVector{T}
-    MT = CuMatrix{T}
-    @test isa(d_ips.kkt, KKTSystem{T, VT, MT})
-    # # Check that both results match exactly
-    @test h_ips.cnt.k == d_ips.cnt.k
-    @test h_ips.obj_val ≈ d_ips.obj_val atol=1e-10
-    @test h_ips.x ≈ d_ips.x atol=1e-10
-    @test h_ips.l ≈ d_ips.l atol=1e-10
+
+    for (T,tol,atol) in [(Float32,1e-3,1e-1), (Float64,1e-8,1e-6)]
+        madnlp_options = Dict{Symbol, Any}(
+            :kkt_system=>opt_kkt,
+            :linear_solver=>LapackGPUSolver,
+            :print_level=>MadNLP.ERROR,
+            :tol=>tol
+        )
+
+        nlp = MadNLPTests.DenseDummyQP{T}(; n=n, m=m, fixed_variables=ind_fixed)
+        
+        # Solve on CPU
+        h_ips = MadNLP.InteriorPointSolver(nlp; option_dict=copy(madnlp_options))
+        MadNLP.optimize!(h_ips)
+
+        # Solve on GPU
+        d_ips = MadNLPGPU.CuInteriorPointSolver(nlp; option_dict=copy(madnlp_options))
+        MadNLP.optimize!(d_ips)
+
+        @test isa(d_ips.kkt, KKTSystem{T, CuVector{T}, CuMatrix{T}})
+        # # Check that both results match exactly
+        @test h_ips.cnt.k == d_ips.cnt.k
+        @test h_ips.obj_val ≈ d_ips.obj_val atol=atol
+        @test h_ips.x ≈ d_ips.x atol=atol
+        @test h_ips.l ≈ d_ips.l atol=atol
+    end
 end
 
 @testset "MadNLPGPU ($(kkt_system))" for kkt_system in [

lib/MadNLPGPU/test/runtests.jl

@@ -35,13 +35,16 @@ testset = [
     ],
 ]
 
-# Test LapackGPU wrapper
-@testset "LapackGPU test" begin
+@testset "MadNLPGPU test" begin
+
+    MadNLPTests.test_linear_solver(LapackGPUSolver,Float32)
+    MadNLPTests.test_linear_solver(LapackGPUSolver,Float64)
+
+    # Test LapackGPU wrapper
     for (name,optimizer_constructor,exclude) in testset
         test_madnlp(name,optimizer_constructor,exclude)
     end
 end
 
 # Test DenseKKTSystem on GPU
 include("densekkt_gpu.jl")
-

lib/MadNLPHSL/README.md

@@ -7,11 +7,20 @@ pkg> add MadNLPHSL
 
 To build MadNLP with HSL linear solvers (Ma27, Ma57, Ma77, Ma86, Ma97), the source codes need to be obtained by the user from <http://www.hsl.rl.ac.uk/ipopt/> under Coin-HSL Full (Stable). The source codes are distribted as a tarball file `coinhsl-*.tar.gz`. The absolute path to the extracted source code or the complied library should be provided to the user. If the user has an already compiled HSL sovler library, one can simply provide a path to that shared library.In this case, the source code is not compiled and the provided shared library is directly used.
 ```julia
-# either one of the following should be given
+# at least one of the following should be given
 julia> ENV["MADNLP_HSL_SOURCE_PATH"] = "/opt/coinhsl" 
-julia> ENV["MADNLP_HSL_SOURCE_PATH"] = "/opt/coinhsl-archive-2021.05.05" 
-julia> ENV["MADNLP_HSL_SOURCE_PATH"] = "/opt/ma57-3.11.0/" 
+julia> ENV["MADNLP_MA27_SOURCE_PATH"] = "/opt/coinhsl-archive-2021.05.05" 
+julia> ENV["MADNLP_MA57_SOURCE_PATH"] = "/opt/ma57-3.11.0/" 
+julia> ENV["MADNLP_MA77_SOURCE_PATH"] = "/opt/hsl_ma77-6.3.0" 
+julia> ENV["MADNLP_MA86_SOURCE_PATH"] = "/opt/hsl_ma86-1.7.2" 
+julia> ENV["MADNLP_MA97_SOURCE_PATH"] = "/opt/hsl_ma97-2.7.1" 
+
 julia> ENV["MADNLP_HSL_LIBRARY_PATH"] = "/usr/lib/libcoinhsl.so"
+julia> ENV["MADNLP_MA27_LIBRARY_PATH"] = "/usr/lib/libma27.so"
+julia> ENV["MADNLP_MA57_LIBRARY_PATH"] = "/usr/lib/libma57.so"
+julia> ENV["MADNLP_MA77_LIBRARY_PATH"] = "/usr/lib/libma77.so"
+julia> ENV["MADNLP_MA86_LIBRARY_PATH"] = "/usr/lib/libma86.so"
+julia> ENV["MADNLP_MA97_LIBRARY_PATH"] = "/usr/lib/libma97.so"
 # optionally, one can specify
 julia> ENV["MADNLP_HSL_BLAS"] = "mkl" # default is "openblas"
 ```

lib/MadNLPHSL/deps/build.jl

@@ -13,8 +13,6 @@ const rpath = `-Wl,-rpath,`
 const whole_archive= Sys.isapple() ? `-Wl,-all_load` : `-Wl,--whole-archive`
 const no_whole_archive = Sys.isapple() ? `-Wl,-noall_load` : `-Wl,--no-whole-archive`
 const libdir     = mkpath(joinpath(@__DIR__, "lib"))
-const hsl_library_path = haskey(ENV,"MADNLP_HSL_LIBRARY_PATH") ? ENV["MADNLP_HSL_LIBRARY_PATH"] : ""
-const hsl_source_path = haskey(ENV,"MADNLP_HSL_SOURCE_PATH") ? ENV["MADNLP_HSL_SOURCE_PATH"] : ""
 const FC = haskey(ENV,"MADNLP_FC") ? ENV["MADNLP_FC"] : `gfortran`
 const libmetis_dir = joinpath(artifact"METIS", "lib")
 const with_metis = `-L$libmetis_dir $rpath$libmetis_dir -lmetis`
@@ -26,74 +24,120 @@ else
     const libblas_dir = joinpath(artifact"OpenBLAS32","lib")
     const with_blas = `-L$libblas_dir $rpath$libblas_dir -lopenblas`
 end
-
-const targets =[
-    [
-        "deps.f", "deps90.f90",
+
+const supported_library = [
+    (:libhsl, "MADNLP_HSL_LIBRARY_PATH", "MADNLP_HSL_SOURCE_PATH")
+    (:libma27, "MADNLP_MA27_LIBRARY_PATH", "MADNLP_MA27_SOURCE_PATH")
+    (:libma57, "MADNLP_MA57_LIBRARY_PATH", "MADNLP_MA57_SOURCE_PATH")
+    (:libma77, "MADNLP_MA77_LIBRARY_PATH", "MADNLP_MA77_SOURCE_PATH")
+    (:libma86, "MADNLP_MA86_LIBRARY_PATH", "MADNLP_MA86_SOURCE_PATH")
+    (:libma97, "MADNLP_MA97_LIBRARY_PATH", "MADNLP_MA97_SOURCE_PATH")
+]
+
+const targets_dict = Dict(
+    :libhsl=> [
+        "deps.f",
+        "deps90.f90",
+        "ma27d.f",
+        "ma57d.f",
+        "hsl_ma77d.f90",
+        "hsl_ma86d.f90",
+        "hsl_ma97d.f90",
+        "hsl_mc68i_ciface.f90",
+        "hsl_ma77d_ciface.f90",
+        "hsl_ma86d_ciface.f90",
+        "hsl_ma97d_ciface.f90",
     ],
-    [
-        "ma27d.f", "ma27s.f",
-        "ma57d.f", "ma57s.f",
-        "hsl_ma77d.f90", "hsl_ma77s.f90", 
-        "hsl_ma86d.f90", "hsl_ma86s.f90", 
-        "hsl_ma97d.f90", "hsl_ma97s.f90",
+    :libma27 => [
+        "deps.f",
+        "ma27d.f",
+        "ma27s.f",
     ],
-    [
-        "hsl_mc68i_ciface.f90",
+    :libma57 => [
+        "sdeps.f", "ddeps.f",
+        "ma57d.f", "ma57s.f", 
+    ],
+    :libma77 => [
+        "common.f", "common90.f90",
+        "ddeps90.f90", "sdeps90.f90", 
+        "hsl_ma77d.f90", "hsl_ma77s.f90",
         "hsl_ma77d_ciface.f90", "hsl_ma77s_ciface.f90",
+    ],
+    :libma86 => [
+        "common.f", "common90.f90",
+        "sdeps90.f90",
+        "hsl_ma86d.f90", "hsl_ma86s.f90",
         "hsl_ma86d_ciface.f90", "hsl_ma86s_ciface.f90",
-        "hsl_ma97d_ciface.f90", "hsl_ma97s_ciface.f90", 
+        "hsl_mc68i_ciface.f90",
+    ],
+    :libma97 => [
+        "common.f", "common90.f90",
+        "sdeps90.f90", "ddeps90.f90",
+        "hsl_ma97d.f90", "hsl_ma97s.f90",
+        "hsl_ma97d_ciface.f90", "hsl_ma97s_ciface.f90",
     ]
-]
+)
 
 rm(libdir;recursive=true,force=true)
 mkpath(libdir)
 isvalid(cmd::Cmd)=(try run(cmd) catch e return false end; return true)
 
 
 # HSL
-if hsl_source_path != ""
-    if isvalid(`$FC --version`)
-        OC = OutputCollector[]
-        cd(hsl_source_path)
+attempted = Tuple{Symbol,Product}[]
 
-        names_succeeded = []
-        for i=1:3
-            names = []
-            for (root, dirs, files) in walkdir(hsl_source_path)
-                for file in files;
-                    if file in targets[i];
-                        filter!(x->x != file,files)
-                        name = splitext(relpath(joinpath(root,file),hsl_source_path))
-                        push!(names, name)
-                        @info "$(name[1])$(name[2]) source code detected."
-                    end
+for (lib, envlib, envsrc) in supported_library
+    if haskey(ENV,envlib)
+        push!(attempted, (lib,FileProduct(ENV[envlib], lib)))
+    elseif haskey(ENV,envsrc) && isvalid(`$FC --version`)
+        @info "Compiling $lib"
+        source_path = ENV[envsrc]
+        targets = targets_dict[lib]
+
+        cd(source_path)
+
+        list = []
+        for (root, dir, files) in walkdir(source_path)
+            for file in files
+                if file in targets
+                    @info "$file source code detected."
+                    push!(list, (root, dir, file))
                 end
             end
-            succeeded = wait.(
-                [OutputCollector(`$FC -fopenmp -fPIC -c -O3 -o $name.o $name$ext`,verbose=verbose)
-                 for (name,ext) in names])
-            append!(names_succeeded, names[succeeded])
         end
 
-        cmd = `$FC -o$(libdir)/libhsl.$so -shared -fPIC -O3 -fopenmp`
-        append!(cmd.exec, ["$name.o" for (name,ext) in names_succeeded])
+        succeeded = []
+        for target in targets
+            for (root, dir, file) in list
+                if file == target
+                    name, ext = splitext(relpath(joinpath(root,file),source_path))
+                    isvalid(`$FC -fopenmp -fPIC -c -O3 -o $name.o $name$ext`)
+                    push!(succeeded, (name, ext))
+                end
+            end
+        end
+
+
+        cmd = `$FC -o$(libdir)/$lib.$so -shared -fPIC -O3 -fopenmp`
+        append!(cmd.exec, ["$name.o" for (name,ext) in succeeded])
         append!(cmd.exec, with_metis.exec)
         append!(cmd.exec, with_blas.exec)
 
         run(cmd)
         cd("$(@__DIR__)")
-        product = FileProduct(prefix,joinpath(libdir,"libhsl.$so"), :libhsl)
+        push!(attempted, (lib,FileProduct(prefix,joinpath(libdir,"$lib.$so"), lib)))
     end
-else
-    product = FileProduct(hsl_library_path, :libhsl)
 end
 
 # write deps.jl
-if satisfied(product)
-    @info "Building HSL succeeded."
-    write_deps_file(joinpath(@__DIR__, "deps.jl"),Product[product], verbose=verbose)
-else
-    @error "Building HSL failed."
-    write_deps_file(joinpath(@__DIR__, "deps.jl"),Product[], verbose=verbose)
+succeeded = Product[]
+for (lib, product) in attempted
+    if satisfied(product)
+        @info "Building $lib succeeded."
+        push!(succeeded, product)
+    else
+        @error "Building $lib failed."
+    end
 end
+
+write_deps_file(joinpath(@__DIR__, "deps.jl"), succeeded, verbose=verbose)

lib/MadNLPHSL/src/MadNLPHSL.jl

@@ -5,25 +5,56 @@ import MadNLP: @kwdef, Logger, @debug, @warn, @error,
     AbstractOptions, AbstractLinearSolver, set_options!, SparseMatrixCSC, SubVector,
     SymbolicException,FactorizationException,SolveException,InertiaException,
     introduce, factorize!, solve!, improve!, is_inertia, inertia, findIJ, nnz,
-    get_tril_to_full, transfer!, input_type, _madnlp_unsafe_wrap
+    get_tril_to_full, transfer!, input_type, _madnlp_unsafe_wrap,
+    is_supported
 
 include(joinpath("..","deps","deps.jl"))
 
+include("common.jl")
+include("mc68.jl")
+
 if @isdefined(libhsl)
-    include("common.jl")
-    include("mc68.jl")
+    @isdefined(libma27) || const libma27 = libhsl
+    @isdefined(libma57) || const libma57 = libhsl
+    @isdefined(libma77) || const libma77 = libhsl
+    @isdefined(libma86) || const libma86 = libhsl
+    @isdefined(libma97) || const libma97 = libhsl
+end
+
+if @isdefined(libma27)
     include("ma27.jl")
+    export Ma27Solver
+end
+
+if @isdefined(libma57)
     include("ma57.jl")
+    export Ma57Solver
+end
+
+if @isdefined(libma77)
     include("ma77.jl")
+    export Ma77Solver
+end
+
+if @isdefined(libma86)
     include("ma86.jl")
+    export Ma86Solver
+end
+
+if @isdefined(libma97)
     include("ma97.jl")
-    export Ma27Solver, Ma57Solver, Ma77Solver, Ma86Solver, Ma97Solver
+    export Ma97Solver
 end
 
 function __init__()
     check_deps()
     try
-        @isdefined(libhsl) && dlopen(libhsl,RTLD_DEEPBIND)
+        @isdefined(libhsl)  && dlopen(libhsl,RTLD_DEEPBIND)
+        @isdefined(libma27) && dlopen(libma27,RTLD_DEEPBIND)
+        @isdefined(libma77) && dlopen(libma57,RTLD_DEEPBIND)
+        @isdefined(libma77) && dlopen(libma77,RTLD_DEEPBIND)
+        @isdefined(libma86) && dlopen(libma77,RTLD_DEEPBIND)
+        @isdefined(libma97) && dlopen(libma97,RTLD_DEEPBIND)
     catch e
         println("HSL shared library cannot be loaded")
     end