Progress towards leaky tests

CliMA · Jul 3, 2024 · fd4e1b9 · fd4e1b9
1 parent f9d110a
commit fd4e1b9
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 49 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -289,6 +289,16 @@ steps:
         key: unit_sphere
         command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl"
 
+      - label: "Unit: sphere gpu"
+        key: unit_sphere_gpu
+        command:
+          - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
       - label: "Unit: terrain warp"
         key: unit_terrain_warp
         command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl"
@@ -556,6 +566,16 @@ steps:
           - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl"
           - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl"
 
+      - label: "Unit: column gpu"
+        key: unit_column_gpu
+        command:
+          - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
       - label: "Unit: wfact"
         key: unit_wfact
         command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/wfact.jl"
@@ -1190,6 +1210,16 @@ steps:
         key: perf_FD
         command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl"
 
+      - label: "Perf: FD operators gpu"
+        key: perf_FD_gpu
+        command:
+          - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/opt.jl"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+        agents:
+          slurm_gpus: 1
+
         # TODO: combine this with FD operators above
       - label: "Perf: FD operators from the wild"
         key: perf_FD_ops_examples

diff --git a/test/Operators/finitedifference/opt.jl b/test/Operators/finitedifference/opt.jl
@@ -1,3 +1,8 @@
+#=
+julia --check-bounds=yes --project
+julia --project
+using Revise; include(joinpath("test", "Operators", "finitedifference", "opt.jl"))
+=#
 using Test
 using JET
 
@@ -209,6 +214,7 @@ end
 @static if @isdefined(var"@test_opt")
     @testset "Scalar Field FiniteDifferenceSpaces optimizations" begin
         device = ClimaComms.device()
+        is_gpu = device isa ClimaComms.CUDADevice
         for FT in (Float64,)
             domain = Domains.IntervalDomain(
                 Geometry.ZPoint{FT}(0.0),
@@ -231,74 +237,89 @@ end
             filter(@nospecialize f) = f !== Base.mapreduce_empty
 
             # face space operators
-            @test_opt function_filter = filter sum(ones(FT, face_space))
-            @test_opt function_filter = filter sum(sin.(faces))
+            @test_opt broken = is_gpu function_filter = filter sum(
+                ones(FT, face_space),
+            )
+            @test_opt broken = is_gpu function_filter = filter sum(sin.(faces))
 
-            @test_opt opt_InterpolateF2C(faces)
-            @test_opt opt_WeightedInterpolateF2C(face_values, faces)
+            @test_opt broken = is_gpu opt_InterpolateF2C(faces)
+            @test_opt broken = is_gpu opt_WeightedInterpolateF2C(
+                face_values,
+                faces,
+            )
 
-            @test_opt opt_LeftBiasedF2C(faces)
-            @test_opt opt_RightBiasedF2C(faces)
+            @test_opt broken = is_gpu opt_LeftBiasedF2C(faces)
+            @test_opt broken = is_gpu opt_RightBiasedF2C(faces)
 
-            # @test_opt opt_AdvectionF2F(face_velocities, faces)
+            @test_opt broken = is_gpu opt_AdvectionF2F(face_velocities, faces)
 
-            @test_opt opt_FluxCorrectionF2F_Extrapolate(
+            @test_opt broken = is_gpu opt_FluxCorrectionF2F_Extrapolate(
                 center_velocities,
                 faces,
             )
 
-            @test_opt opt_GradientF2C(faces)
-            @test_opt opt_DivergenceF2C(faces)
+            @test_opt broken = is_gpu opt_GradientF2C(faces)
+            @test_opt broken = is_gpu opt_DivergenceF2C(faces)
 
-            @test_opt opt_SetBoundary_SetValue(faces)
+            @test_opt broken = is_gpu opt_SetBoundary_SetValue(faces)
 
             # center space operators
-            @test_opt function_filter = filter sum(ones(FT, center_space))
-            @test_opt function_filter = filter sum(sin.(centers))
+            @test_opt broken = is_gpu function_filter = filter sum(
+                ones(FT, center_space),
+            )
+            @test_opt broken = is_gpu function_filter = filter sum(
+                sin.(centers),
+            )
 
-            @test_opt opt_InterpolateC2F_SetValue(centers)
-            @test_opt opt_InterpolateC2F_SetGradient(centers)
-            @test_opt opt_InterpolateC2F_Extrapolate(centers)
+            @test_opt broken = is_gpu opt_InterpolateC2F_SetValue(centers)
+            @test_opt broken = is_gpu opt_InterpolateC2F_SetGradient(centers)
+            @test_opt broken = is_gpu opt_InterpolateC2F_Extrapolate(centers)
 
-            @test_opt opt_WeightedInterpolateC2F_SetValue(
+            @test_opt broken = is_gpu opt_WeightedInterpolateC2F_SetValue(
                 center_values,
                 centers,
             )
-            @test_opt opt_WeightedInterpolateC2F_SetGradient(
+            @test_opt broken = is_gpu opt_WeightedInterpolateC2F_SetGradient(
                 center_values,
                 centers,
             )
-            @test_opt opt_WeightedInterpolateC2F_Extrapolate(
+            @test_opt broken = is_gpu opt_WeightedInterpolateC2F_Extrapolate(
                 center_values,
                 centers,
             )
 
-            @test_opt opt_LeftBiasedC2F(centers)
-            @test_opt opt_RightBiasedC2F(centers)
+            @test_opt broken = is_gpu opt_LeftBiasedC2F(centers)
+            @test_opt broken = is_gpu opt_RightBiasedC2F(centers)
 
-            @test_opt opt_UpwindBiasedProductC2F_SetValue(
+            @test_opt broken = is_gpu opt_UpwindBiasedProductC2F_SetValue(
                 face_velocities,
                 centers,
             )
-            @test_opt opt_UpwindBiasedProductC2F_Extrapolate(
+            @test_opt broken = is_gpu opt_UpwindBiasedProductC2F_Extrapolate(
                 face_velocities,
                 centers,
             )
 
-            @test_opt opt_AdvectionC2C_SetValue(face_velocities, centers)
-            @test_opt opt_AdvectionC2C_Extrapolate(face_velocities, centers)
+            @test_opt broken = is_gpu opt_AdvectionC2C_SetValue(
+                face_velocities,
+                centers,
+            )
+            @test_opt broken = is_gpu opt_AdvectionC2C_Extrapolate(
+                face_velocities,
+                centers,
+            )
 
-            @test_opt opt_FluxCorrectionC2C_Extrapolate(
+            @test_opt broken = is_gpu opt_FluxCorrectionC2C_Extrapolate(
                 face_velocities,
                 centers,
             )
 
-            @test_opt opt_GradientC2F_SetValue(centers)
-            @test_opt opt_GradientC2F_SetGradient(centers)
+            @test_opt broken = is_gpu opt_GradientC2F_SetValue(centers)
+            @test_opt broken = is_gpu opt_GradientC2F_SetGradient(centers)
 
-            @test_opt opt_DivergenceC2F_SetValue(centers)
-            @test_opt opt_DivergenceC2F_SetDivergence(centers)
-            @test_opt opt_CurlC2F_SetValue(centers)
+            @test_opt broken = is_gpu opt_DivergenceC2F_SetValue(centers)
+            @test_opt broken = is_gpu opt_DivergenceC2F_SetDivergence(centers)
+            @test_opt broken = is_gpu opt_CurlC2F_SetValue(centers)
         end
     end
 end
diff --git a/test/Operators/finitedifference/unit_column.jl b/test/Operators/finitedifference/unit_column.jl
@@ -1,3 +1,8 @@
+#=
+julia --check-bounds=yes --project
+julia --project
+using Revise; include(joinpath("test", "Operators", "finitedifference", "unit_column.jl"))
+=#
 using Test
 using StaticArrays, IntervalSets, LinearAlgebra
 
@@ -26,7 +31,9 @@ device = ClimaComms.device()
         face_space = Spaces.FaceFiniteDifferenceSpace(center_space)
 
         @test sum(ones(FT, center_space)) ≈ pi
-        @test sum(ones(FT, face_space)) ≈ pi
+        ClimaComms.allowscalar(device) do
+            @test sum(ones(FT, face_space)) ≈ pi
+        end
 
         centers = getproperty(Fields.coordinate_field(center_space), :z)
         @test sum(sin.(centers)) ≈ FT(2.0) atol = 1e-2
@@ -102,20 +109,22 @@ end
         @test sum(sinz_f) ≈ FT(0.0) atol = 1e-2
 
         ∇ᶜ = Operators.GradientF2C()
-        ∂sin = Geometry.WVector.(∇ᶜ.(sinz_f))
-        @test ∂sin ≈ Geometry.WVector.(cosz_c) atol = 1e-2
+        @test begin
+            ∂sin = Geometry.WVector.(∇ᶜ.(sinz_f))
+            @test ∂sin ≈ Geometry.WVector.(cosz_c) atol = 1e-2
 
-        divᶜ = Operators.DivergenceF2C()
-        ∂sin = divᶜ.(Geometry.WVector.(sinz_f))
-        @test ∂sin ≈ cosz_c atol = 1e-2
+            divᶜ = Operators.DivergenceF2C()
+            ∂sin = divᶜ.(Geometry.WVector.(sinz_f))
+            @test ∂sin ≈ cosz_c atol = 1e-2
 
-        ∇ᶠ = Operators.GradientC2F()
-        ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c))
-        @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-1
+            ∇ᶠ = Operators.GradientC2F()
+            ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c))
+            @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-1
 
-        ∇ᶠ = Operators.GradientC2F()
-        ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c))
-        @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-2
+            ∇ᶠ = Operators.GradientC2F()
+            ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c))
+            @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-2
+        end broken = device isa ClimaComms.CUDADevice
 
         # test that broadcasting into incorrect field space throws an error
         empty_centers = zeros(FT, center_space)
@@ -269,32 +278,32 @@ end
     cy = cfield.y
     fy = ffield.y
 
-    cyp = parent(cy)
-    fyp = parent(fy)
+    cyp = Array(parent(cy))
+    fyp = Array(parent(fy))
 
     # C2F biased operators
     LBC2F = Operators.LeftBiasedC2F(; bottom = Operators.SetValue(10))
     @. cy = cos(zc)
     @. fy = LBC2F(cy)
     fy_ref = [FT(10), [cyp[i] for i in 1:length(cyp)]...]
-    @test all(fy_ref .== fyp)
+    @test all(fy_ref .== fyp) broken = device isa ClimaComms.CUDADevice
 
     RBC2F = Operators.RightBiasedC2F(; top = Operators.SetValue(10))
     @. cy = cos(zc)
     @. fy = RBC2F(cy)
     fy_ref = [[cyp[i] for i in 1:length(cyp)]..., FT(10)]
-    @test all(fy_ref .== fyp)
+    @test all(fy_ref .== fyp) broken = device isa ClimaComms.CUDADevice
 
     # F2C biased operators
     LBF2C = Operators.LeftBiasedF2C(; bottom = Operators.SetValue(10))
     @. cy = cos(zc)
     @. cy = LBF2C(fy)
     cy_ref = [i == 1 ? FT(10) : fyp[i] for i in 1:length(cyp)]
-    @test all(cy_ref .== cyp)
+    @test all(cy_ref .== cyp) broken = device isa ClimaComms.CUDADevice
 
     RBF2C = Operators.RightBiasedF2C(; top = Operators.SetValue(10))
     @. cy = cos(zc)
     @. cy = RBF2C(fy)
     cy_ref = [i == length(cyp) ? FT(10) : fyp[i + 1] for i in 1:length(cyp)]
-    @test all(cy_ref .== cyp)
+    @test all(cy_ref .== cyp) broken = device isa ClimaComms.CUDADevice
 end