Horizontal Limiter GPU

Examples/testAdvectionCartGPU.jl

@@ -1,13 +1,16 @@
-include("../src/CGDycore.jl")
+import CGDycore:
+  Examples, Parallels, Models, Grids, Outputs, Integration,  GPU, DyCore
 using MPI
 using Base
 using CUDA
+using AMDGPU
+using Metal
 using KernelAbstractions
-using KernelAbstractions: @atomic, @atomicswap, @atomicreplace
 using StaticArrays
-
+using ArgParse
+using MPI
 # Model
-parsed_args = CGDycore.parse_commandline()
+parsed_args = DyCore.parse_commandline()
 Problem = parsed_args["Problem"]
 ProfRho = parsed_args["ProfRho"]
 ProfTheta = parsed_args["ProfTheta"]
@@ -84,19 +87,23 @@ PrintTime = parsed_args["PrintTime"]
 Device = parsed_args["Device"]
 GPUType = parsed_args["GPUType"]
 FloatTypeBackend = parsed_args["FloatTypeBackend"]
-Param = CGDycore.Parameters(Problem)
-
 
 if Device == "CPU"
   backend = CPU()
 elseif Device == "GPU"
   if GPUType == "CUDA"
-    backend = CUDA.CUDABackend()
-    CUDA.allowscalar(true)
-  end
+    backend = CUDABackend()
+    CUDA.allowscalar(false)
+#   CUDA.device!(MPI.Comm_rank(MPI.COMM_WORLD))
+  elseif GPUType == "AMD"
+    backend = ROCBackend()
+    AMDGPU.allowscalar(false)
+  elseif GPUType == "Metal"
+    backend = MetalBackend()
+    Metal.allowscalar(true)
+  end 
 else
-  @show "False device"
-  stop
+  backend = CPU()
 end
 
 if FloatTypeBackend == "Float64"
@@ -108,6 +115,8 @@ else
   stop
 end
 
+Param = Examples.Parameters(FTB,Problem)
+
 KernelAbstractions.synchronize(backend)
 
 MPI.Init()
@@ -116,10 +125,10 @@ OrdPolyZ=1
 Parallel = true
 
 # Physical parameters
-Phys=CGDycore.PhysParameters{FTB}()
+Phys=DyCore.PhysParameters{FTB}()
 
 #ModelParameters
-Model = CGDycore.Model()
+Model = DyCore.ModelStruct{FTB}()
 # Initial conditions
 Model.Equation="Compressible"
 Model.NumV=NumV
@@ -182,7 +191,7 @@ Model.HyperDDiv = HyperDDiv # =7.e15
 
 
 
-Boundary = CGDycore.Boundary()
+Boundary = Grids.Boundary()
 Boundary.WE = BoundaryWE
 Boundary.SN = BoundarySN
 Boundary.BT = BoundaryBT
@@ -195,14 +204,13 @@ Topography=(TopoS=TopoS,
            )
 
   @show "vor InitCart"
-  (CG, Metric, Global) = CGDycore.InitCart(backend,FTB,OrdPoly,OrdPolyZ,nx,ny,Lx,Ly,x0,y0,nz,H,
+  (CG, Metric, Exchange, Global) = DyCore.InitCart(backend,FTB,OrdPoly,OrdPolyZ,nx,ny,Lx,Ly,x0,y0,nz,H,
   Boundary,GridType,Topography,Decomp,Model,Phys)
-  @show CG.NumG,Global.Grid.nz
 
-  Profile = CGDycore.RotationalCartExample()(Param,Phys)
+  Profile = Examples.RotationalCartExample()(Param,Phys)
 
 
-  U = CGDycore.InitialConditionsAdvection(backend,FTB,CG,Metric,Phys,Global,Profile,Param)
+  U = GPU.InitialConditionsAdvection(backend,FTB,CG,Metric,Phys,Global,Profile,Param)
 
 
 # Output
@@ -220,7 +228,7 @@ Topography=(TopoS=TopoS,
   Global.Output.PrintStartTime = 0
   Global.Output.OrdPrint=CG.OrdPoly
   @show "vor Global.vtkCache"
-  Global.vtkCache = CGDycore.vtkStruct{FTB}(backend,Global.Output.OrdPrint,CGDycore.TransCartX!,CG,Metric,Global)
+  Global.vtkCache = Outputs.vtkStruct{FTB}(backend,Global.Output.OrdPrint,Grids.TransCartX!,CG,Metric,Global)
 
 
   # TimeStepper
@@ -235,10 +243,10 @@ Topography=(TopoS=TopoS,
   Global.TimeStepper.SimTime = SimTime
 
   nT = NumV + NumTr
-# CGDycore.InitExchangeData3D(nz,nT,Global.Exchange)
+  Parallels.InitExchangeData3D(backend,FTB,nz,nT,Exchange)  
   @show "vor TimeStepperGPUAdvection!"
-  if Device == "CPU" 
-    CGDycore.TimeStepperAdvection!(U,CGDycore.TransCartX,CG,Metric,Phys,Global,Param)
+  if Device == "CPU" || Device == "CPU" 
+    Integration.TimeStepperGPUAdvection!(U,GPU.FcnAdvectionGPU!,Grids.TransCartX,CG,Metric,Phys,Exchange,Global,Param,Profile)
   else
-    CGDycore.TimeStepperGPUAdvection!(U,CGDycore.TransCartX,CG,Metric,Phys,Global,Param,Profile)
+    Integration.TimeStepperAdvection!(U,DyCore.FcnTracer!,Grids.TransCartX,CG,Metric,Phys,Exchange,Global,Param,Profile)
   end

Jobs/JobAdvectionCubeCart

@@ -1,31 +1,32 @@
-mpirun -n 6 julia --project --check-bounds=yes Examples/testAdvectionCart.jl \
-  --Problem="AdvectionCubeCart" \
-  --NumV=1 \
+mpirun -n 1 julia --project Examples/testAdvectionCartGPU.jl \
+  --Problem="AdvectionCubeRotCart" \
+  --Device="" \
+  --FloatTypeBackend="Float64" \
+  --NumV=5 \
   --NumTr=1 \
-  --ProfTr="AdvectionCubeCart" \
+  --ProfTr="AdvectionCubeRotCart" \
   --ProfVel="Const" \
-  --HorLimit=false \
-  --vtkFileName="AdvectionCubeCart" \
+  --HorLimit=true \
+  --Upwind=false \
+  --vtkFileName="AdvectionCubeRotCart" \
   --SimTime=1000.0 \
-  --PrintTime=10.0 \
-  --dtau=0.25\
+  --PrintTime=100.0 \
+  --dtau=1\
   --IntMethod="SSPRungeKutta" \
   --Table="SSP32" \
   --Lx=1000.0 \
   --Ly=1000.0 \
-  --H=1.0 \
+  --H=1000.0 \
   --x0=0.0 \
   --y0=0.0 \
-  --nx=40 \
-  --ny=40 \
+  --nx=20 \
+  --ny=20 \
+  --nz=1 \
   --OrdPoly=4 \
   --BoundaryWE="Period" \
   --BoundarySN="Period" \
   --BoundaryBT="" \
   --HyperVisc=true \
   --HyperDCurl=1.e4 \
   --HyperDGrad=1.e4 \
-  --HyperDDiv=1.e4
-
-
-
+  --HyperDDiv=0.e0

Jobs/JobAdvectionCubeCartCPU

@@ -2,31 +2,31 @@ mpirun -n 1 julia --project Examples/testAdvectionCartGPU.jl \
   --Problem="AdvectionCubeRotCart" \
   --Device="CPU" \
   --FloatTypeBackend="Float64" \
-  --NumV=4 \
+  --NumV=5 \
   --NumTr=1 \
   --ProfTr="AdvectionCubeRotCart" \
   --ProfVel="Const" \
-  --HorLimit=false \
+  --HorLimit=true \
   --Upwind=false \
   --vtkFileName="AdvectionCubeRotCart" \
   --SimTime=1000.0 \
   --PrintTime=100.0 \
-  --dtau=0.1\
+  --dtau=.25\
   --IntMethod="SSPRungeKutta" \
   --Table="SSP32" \
   --Lx=1000.0 \
   --Ly=1000.0 \
   --H=1000.0 \
   --x0=0.0 \
   --y0=0.0 \
-  --nx=20 \
-  --ny=20 \
-  --nz=64 \
+  --nx=80 \
+  --ny=80 \
+  --nz=1 \
   --OrdPoly=4 \
   --BoundaryWE="Period" \
   --BoundarySN="Period" \
   --BoundaryBT="" \
   --HyperVisc=true \
   --HyperDCurl=1.e4 \
   --HyperDGrad=1.e4 \
-  --HyperDDiv=1.e0
+  --HyperDDiv=1.e1

Jobs/JobNHHeldSuarezMoistSphere

@@ -18,7 +18,7 @@ mpirun -n 6 julia --project Examples/testNHSphere.jl \
   --SurfaceFlux=true \
   --Coriolis=true \
   --Upwind=true \
-  --HorLimit=false \
+  --HorLimit=true \
   --Equation=CompressibleMoist \
   --Microphysics=true \
   --TypeMicrophysics="SimpleMicrophysics" \

src/DyCore/DiscretizationCG.jl

@@ -30,8 +30,11 @@ function DiscretizationCG(backend,FT,Jacobi,CG,Exchange,Global,zs)
     F[4,3,iF] = Grid.Faces[iF].P[4].z  
   end  
   copyto!(FGPU,F)
-  Grids.JacobiSphere3GPU!(Metric.X,Metric.dXdxI,Metric.J,CG,FGPU,Grid.z,zs,Grid.Rad)
-# Grids.JacobiSphere3CPU!(Metric.X,Metric.dXdxI,Metric.J,CG,Grid,zs,Grids.Topo)
+  if Global.Grid.Form == "Sphere"
+    Grids.JacobiSphere3GPU!(Metric.X,Metric.dXdxI,Metric.J,CG,FGPU,Grid.z,zs,Grid.Rad)
+  else
+    Grids.JacobiDG3GPU!(Metric.X,Metric.dXdxI,Metric.J,CG,FGPU,Grid.z,zs)
+  end  
 
   MassCGGPU!(CG,Metric.J,CG.Glob,Exchange,Global)
 
@@ -58,6 +61,16 @@ function DiscretizationCG(backend,FT,Jacobi,CG,Exchange,Global,zs)
   KSurfaceNormalKernel! = SurfaceNormalKernel!(backend,group)
   KSurfaceNormalKernel!(Metric.FS,Metric.nS,Metric.dXdxI,ndrange=ndrange)
 
+  if Global.Model.HorLimit
+    Metric.JC = KernelAbstractions.zeros(backend,FT,size(Metric.J,1),size(Metric.J,3),size(Metric.J,4))  
+    Metric.JCW = KernelAbstractions.zeros(backend,FT,size(Metric.J,1),size(Metric.J,3),size(Metric.J,4))  
+#   NFG = min(div(NumberThreadGPU,),NF)  
+    group = (nz, 1)
+    ndrange = (nz, NF)
+    KCenterJacobiansKernel! = CenterJacobiansKernel!(backend,group)
+    KCenterJacobiansKernel!(CG.OrdPoly+1,Metric.JC,Metric.JCW,Metric.J,CG.w,ndrange=ndrange)
+  end    
+
   return (CG,Metric)
 end
 
@@ -82,6 +95,23 @@ end
   end
 end
 
+@kernel function CenterJacobiansKernel!(N,JC,JCW,@Const(J),@Const(w))
+  Iz,IF = @index(Global, NTuple)
+
+  FT = eltype(JC)
+  Nz = @uniform @ndrange()[1]
+  if Iz <= Nz
+    @inbounds @views sumJ = sum(J[:,:,Iz,IF])
+    for j = 1 : N
+      for i = 1 : N
+        ID = i + (j - 1) * N  
+        @inbounds JC[ID,Iz,IF] = J[ID,1,Iz,IF] + J[ID,2,Iz,IF] 
+        @inbounds JCW[ID,Iz,IF] = JC[ID,Iz,IF] * w[i] * w[j] / sumJ
+      end
+    end  
+  end  
+end
+
 @kernel function GridSizeSphereKernel!(lat,zP,dz,@Const(X),@Const(Glob),Rad)
 
   ID,Iz,IF = @index(Global, NTuple)

src/DyCore/DyCore.jl

@@ -13,6 +13,8 @@ using LinearAlgebra
 using SparseArrays
 using ArgParse
 using UnPack
+using StaticArrays
+using Statistics
 using StrideArraysCore: @gc_preserve, StrideArray, StaticInt
 using KernelAbstractions
 using KernelAbstractions: @atomic, @atomicswap, @atomicreplace