TKe is working

CliMA · Jul 3, 2024 · 11a8880 · 11a8880
1 parent 7788d61
commit 11a8880
Show file tree

Hide file tree

Showing 23 changed files with 415 additions and 120 deletions.
diff --git a/Batch/batch.sh b/Batch/batch.sh
@@ -0,0 +1,71 @@
+!/bin/bash
+#SBATCH --account=project_465000863
+#SBATCH --partition=small-g
+#SBATCH --time=00:15:00
+#SBATCH --nodes=1
+#SBATCH --gpus-per-node=2
+#SBATCH --ntasks-per-node=2
+#SBATCH --cpus-per-task=2
+#SBATCH --mem-per-cpu=0
+
+module use /appl/local/csc/modulefiles
+module load julia
+module load julia-mpi
+module load julia-amdgpu
+julia --project=. -e 'using Pkg; Pkg.instantiate()'
+julia --project Examples/testNHSphere.jl \
+  --Problem="HeldSuarezMoistSphere" \
+  --Device="CPU" \
+  --GPUType="Metal" \
+  --NumberThreadGPU=512 \
+  --FloatTypeBackend="Float32" \
+  --NumV=5 \
+  --NumTr=2 \
+  --TkePos=0 \
+  --RhoVPos=6 \
+  --RhoCPos=7 \
+  --ProfpBGrd="" \
+  --ProfRhoBGrd="" \
+  --Source=true \
+  --Forcing=true \
+  --Curl=false \
+  --ModelType="VectorInvariant" \
+  --VerticalDiffusion=true \
+  --SurfaceFlux=true \
+  --SurfaceScheme="" \
+  --Coriolis=true \
+  --Upwind=true \
+  --HorLimit=false \
+  --Equation="CompressibleShallow" \
+  --State="Moist" \
+  --Microphysics=true \
+  --TypeMicrophysics="SimpleMicrophysics" \
+  --Buoyancy=true \
+  --Damping=true \
+  --StrideDamp=10000 \
+  --Relax=1.0e-3 \
+  --Decomp="EqualArea" \
+  --SimSeconds=0 \
+  --SimDays=10 \
+  --PrintSeconds=0 \
+  --PrintMinutes=0 \
+  --PrintHours=12 \
+  --PrintDays=0 \
+  --StartAverageDays=100 \
+  --Flat=true \
+  --dtau=150 \
+  --IntMethod="Rosenbrock" \
+  --Table="SSP-Knoth" \
+  --TopoS="" \
+  --Stretch=true \
+  --StretchType="Exp" \
+  --GridType="CubedSphere" \
+  --nz=64 \
+  --nPanel=30 \
+  --H=45000.0 \
+  --OrdPoly=3 \
+  --HyperVisc=true \
+  --HyperDCurl=2.e15 \
+  --HyperDGrad=2.e15 \
+  --HyperDDiv=2.e15 \
+  --HyperDDivW=0.e15
diff --git a/Examples/testNHCart.jl b/Examples/testNHCart.jl
@@ -1,5 +1,5 @@
 import CGDycore:
-  Thermodynamics, Examples, Parallels, Models, Grids, Outputs, Integration,  GPU, DyCore
+  Thermodynamics, Examples, Parallels, Models, Grids, Surfaces,  Outputs, Integration,  GPU, DyCore
 using MPI
 using Base
 using CUDA
@@ -20,6 +20,11 @@ PertTh = parsed_args["PertTh"]
 ProfVel = parsed_args["ProfVel"]
 ProfpBGrd = parsed_args["ProfpBGrd"]
 ProfRhoBGrd = parsed_args["ProfRhoBGrd"]
+RhoVPos = parsed_args["RhoVPos"]
+RhoCPos = parsed_args["RhoCPos"]
+RhoIPos = parsed_args["RhoIPos"]
+RhoRPos = parsed_args["RhoRPos"]
+TkePos = parsed_args["TkePos"]
 HorLimit = parsed_args["HorLimit"]
 Upwind = parsed_args["Upwind"]
 Damping = parsed_args["Damping"]
@@ -63,13 +68,15 @@ AdaptGridType = parsed_args["AdaptGridType"]
 Coriolis = parsed_args["Coriolis"]
 CoriolisType = parsed_args["CoriolisType"]
 Buoyancy = parsed_args["Buoyancy"]
+Turbulence = parsed_args["Turbulence"]
 Source = parsed_args["Source"]
 VerticalDiffusion = parsed_args["VerticalDiffusion"]
 JacVerticalDiffusion = parsed_args["JacVerticalDiffusion"]
 JacVerticalAdvection = parsed_args["JacVerticalAdvection"]
 VerticalDiffusionMom = parsed_args["VerticalDiffusionMom"]
 SurfaceFlux = parsed_args["SurfaceFlux"]
 SurfaceFluxMom = parsed_args["SurfaceFluxMom"]
+SurfaceScheme = parsed_args["SurfaceScheme"]
 # Grid
 nx = parsed_args["nx"]
 ny = parsed_args["ny"]
@@ -88,6 +95,7 @@ HyperDCurl = parsed_args["HyperDCurl"]
 HyperDGrad = parsed_args["HyperDGrad"]
 HyperDDiv = parsed_args["HyperDDiv"]
 # Output
+OrdPrint = parsed_args["OrdPrint"]
 PrintDays = parsed_args["PrintDays"]
 PrintHours = parsed_args["PrintHours"]
 PrintMinutes = parsed_args["PrintMinutes"]
@@ -175,17 +183,20 @@ Model.uPos=2
 Model.vPos=3
 Model.wPos=4
 Model.ThPos=5
-if Model.Equation == "CompressibleMoist"
-  Model.RhoVPos = 1
-  Model.RhoCPos = 2
-end
+Model.RhoVPos  = RhoVPos
+Model.RhoCPos  = RhoCPos
+Model.RhoIPos  = RhoIPos
+Model.RhoRPos  = RhoRPos
+Model.TkePos  = TkePos
 Model.HorLimit = HorLimit
 Model.Upwind = Upwind
 Model.Damping = Damping
 Model.StrideDamp = StrideDamp
 Model.Relax = Relax
 Model.Coriolis = Coriolis
 Model.CoriolisType = CoriolisType
+Model.Buoyancy = Buoyancy
+Model.Turbulence = Turbulence
 Model.VerticalDiffusion = VerticalDiffusion
 Model.JacVerticalDiffusion = JacVerticalDiffusion
 Model.JacVerticalAdvection = JacVerticalAdvection
@@ -306,6 +317,7 @@ elseif State == "Moist"
   Model.Pressure = Pressure
 end
 
+#Microphysics
 if Microphysics
   if TypeMicrophysics == "SimpleMicrophysics"
     MicrophysicsSource  = Models.SimpleMicrophysics()(Phys,Model.RhoPos,Model.ThPos,
@@ -316,6 +328,46 @@ if Microphysics
   end
 end
 
+# Surface flux
+Global.SurfaceData = Surfaces.SurfaceData{FTB}(backend,CG.NumG)
+Global.LandUseData = Surfaces.LandUseData{FTB}(backend,CG.NumG)
+@. Global.LandUseData.z0M = 0.01
+@. Global.LandUseData.z0H = 0.01
+@. Global.LandUseData.LandClass = 5
+if Model.SurfaceFlux || Model.VerticalDiffusion || Model.SurfaceFluxMom || Model.VerticalDiffusionMom
+  if SurfaceScheme == ""
+    @show "Warning: No surface scheme"  
+  elseif SurfaceScheme == "MOST"
+    @show "SurfaceScheme MOST"
+    SurfaceValues, SurfaceFluxValues = Surfaces.MOSurface()(Surfaces.Businger(),Phys,Model.RhoPos,Model.uPos,
+      Model.vPos,Model.wPos,Model.ThPos)
+    Model.SurfaceValues = SurfaceValues
+    Model.SurfaceFluxValues = SurfaceFluxValues
+  end
+end
+if Model.SurfaceFlux
+  if RhoVPos > 0  
+    Model.SurfaceFluxRhs = Surfaces.SurfaceFlux(Phys,Param,Model.ThPos,Model.RhoPos,Model.RhoVPos)
+  else  
+    Model.SurfaceFluxRhs = Surfaces.SurfaceFlux(Phys,Param,Model.ThPos,Model.RhoPos)
+  end  
+end
+
+#Vertical Diffusion
+if Model.VerticalDiffusion || Model.VerticalDiffusionMom
+  if Model.Turbulence
+    Model.Eddy = Examples.TkeKoefficient()(Param,Phys,TkePos,Model.RhoPos)
+  else
+    Model.Eddy = Examples.SimpleKoefficient()(Param,Phys)
+  end
+end
+
+#Turbulence
+if Model.Turbulence
+  Model.TurbulenceSource = Examples.TKEModel()(Param,Phys,Model.RhoPos,Model.uPos,
+    Model.vPos,Model.ThPos,Model.TkePos)
+end
+
 # Forcing
 Force =  Examples.NoForcing()(Param,Phys)
 # Damping
@@ -339,6 +391,12 @@ Global.Output.vtk=0
         "Th",
         "Pres",
         ]
+      if TkePos > 0
+        push!(Global.Output.cNames,"Tke")
+      end
+      if VerticalDiffusion
+        push!(Global.Output.cNames,"DiffKoeff")
+      end
     elseif State == "Moist"
       Global.Output.cNames = [
         "Rho",
@@ -369,7 +427,11 @@ Global.Output.vtk=0
   Global.Output.PrintSeconds = PrintSeconds
   Global.Output.PrintTime = PrintTime
   Global.Output.PrintStartTime = PrintStartTime
-  Global.Output.OrdPrint=CG.OrdPoly
+  if OrdPrint == 0
+    Global.Output.OrdPrint = CG.OrdPoly
+  else  
+    Global.Output.OrdPrint = OrdPrint
+  end  
   Global.vtkCache = Outputs.vtkStruct{FTB}(backend,Global.Output.OrdPrint,Grids.TransCartX!,CG,Metric,Global)
 
   # TimeStepper

diff --git a/Examples/testNHSphere.jl b/Examples/testNHSphere.jl
@@ -22,7 +22,6 @@ RhoCPos = parsed_args["RhoCPos"]
 RhoIPos = parsed_args["RhoIPos"]
 RhoRPos = parsed_args["RhoRPos"]
 TkePos = parsed_args["TkePos"]
-@show TkePos,RhoVPos,RhoCPos
 HorLimit = parsed_args["HorLimit"]
 Upwind = parsed_args["Upwind"]
 Damping = parsed_args["Damping"]
@@ -93,6 +92,7 @@ HyperDRhoDiv = parsed_args["HyperDRhoDiv"]
 HyperDDiv = parsed_args["HyperDDiv"]
 HyperDDivW = parsed_args["HyperDDivW"]
 # Output
+OrdPrint = parsed_args["OrdPrint"]
 PrintDays = parsed_args["PrintDays"]
 PrintHours = parsed_args["PrintHours"]
 PrintMinutes = parsed_args["PrintMinutes"]
@@ -191,7 +191,6 @@ Model.RhoVPos  = RhoVPos
 Model.RhoCPos  = RhoCPos
 Model.RhoIPos  = RhoIPos
 Model.RhoRPos  = RhoRPos
-@show TkePos
 Model.TkePos  = TkePos
 Model.HorLimit = HorLimit
 Model.Upwind = Upwind
@@ -355,6 +354,10 @@ if Model.SurfaceFlux || Model.VerticalDiffusion || Model.SurfaceFluxMom || Model
       Model.vPos,Model.wPos,Model.ThPos)
     Model.SurfaceValues = SurfaceValues
     Model.SurfaceFluxValues = SurfaceFluxValues
+    if Problem == "HeldSuarezMoistSphere" || Problem == "HeldSuarezMoistSphereOro" 
+      SurfaceValues, SurfaceFluxValues = Surfaces.HeldSuarezMoistSurface()(Phys,Param,Model.uPos,Model.vPos,Model.wPos)
+      Model.SurfaceValues = SurfaceValues
+    end   
   end  
 end
 if Model.SurfaceFlux
@@ -364,6 +367,7 @@ end
 #Vertical Diffusion
 if Model.VerticalDiffusion || Model.VerticalDiffusionMom
   if Model.Turbulence
+    Model.Eddy = Examples.TkeKoefficient()(Param,Phys,TkePos,Model.RhoPos)
   else  
     Model.Eddy = Examples.SimpleKoefficient()(Param,Phys)
   end
@@ -400,10 +404,6 @@ if ModelType == "VectorInvariant" || ModelType == "Advection"
     if TkePos > 0
       push!(Global.Output.cNames,"Tke")
     end  
-    @show RhoVPos
-    if RhoVPos > 0
-      push!(Global.Output.cNames,"Tr1")
-    end  
     if VerticalDiffusion
       push!(Global.Output.cNames,"DiffKoeff")
     end  
@@ -414,11 +414,19 @@ if ModelType == "VectorInvariant" || ModelType == "Advection"
       "v",
       "wB",
       "Th",
-#     "Vort",
-#     "Pres",
+      "Vort",
       "Tr1",
       "Tr2",
       ]
+    Model.TrPos = NumV + 1
+    if TkePos > 0
+      push!(Global.Output.cNames,"Tke")
+      Model.TrPos += 1
+    end  
+    if VerticalDiffusion
+      push!(Global.Output.cNames,"DiffKoeff")
+    end  
+
   elseif State == "ShallowWater"  
     Global.Output.cNames = [
       "Rho",
@@ -447,7 +455,11 @@ Global.Output.PrintMinutes = PrintMinutes
 Global.Output.PrintSeconds = PrintSeconds
 Global.Output.StartAverageDays = StartAverageDays
 Global.Output.PrintStartTime = PrintStartTime
-Global.Output.OrdPrint = CG.OrdPoly
+if OrdPrint == 0
+  Global.Output.OrdPrint = CG.OrdPoly
+else  
+  Global.Output.OrdPrint = OrdPrint
+end  
 
 Global.vtkCache = Outputs.vtkStruct{FTB}(backend,Global.Output.OrdPrint,Grids.TransSphereX!,CG,Metric,Global)
 
@@ -470,11 +482,9 @@ end
 
 Global.ParallelCom.NumberThreadGPU = NumberThreadGPU   
 nT = max(7 + NumTr, NumV + NumTr) + Model.NDEDMF*(4 + NumTr)
-@show nT
 Parallels.InitExchangeData3D(backend,FTB,nz,nT,Exchange)
 if ParallelCom.Proc == 1
   @show "vor Timestepper"
 end  
-@show TkePos
 Integration.TimeStepper!(U,GPU.FcnGPU!,GPU.FcnPrepareGPU!,DyCore.JacSchurGPU!,
   Grids.TransSphereX,CG,Metric,Phys,Exchange,Global,Param,Model.Equation)
diff --git a/Jobs/NHCart/JobNHTkeCart b/Jobs/NHCart/JobNHTkeCart
@@ -0,0 +1,58 @@
+mpirun -n 1 julia --project Examples/testNHCart.jl \
+  --Problem="HillAgnesiXCart" \
+  --Device="CPU" \
+  --GPUType="CUDA" \
+  --NumberThreadGPU=512 \
+  --FloatTypeBackend="Float64" \
+  --NumV=5 \
+  --TkePos=6 \
+  --NumTr=0 \
+  --RefProfile=false \
+  --Equation="CompressibleShallow" \
+  --State="Dry" \
+  --Buoyancy=true \
+  --Coriolis=false \
+  --Curl=true \
+  --ModelType="VectorInvariant" \
+  --Turbulence=true \
+  --SurfaceSchem="MOST" \
+  --SurfaceFluxMom=true \
+  --SurfaceFlux=true \
+  --VerticalDiffusionMom=true \
+  --VerticalDiffusion=true \
+  --Damping=true \
+  --StrideDamp=10000 \
+  --Relax=1.0e-2 \
+  --Upwind=true \
+  --BoundaryWE="Period" \
+  --BoundarySN="Period" \
+  --BoundaryBT="" \
+  --Thermo="" \
+  --SimMinutes=0 \
+  --SimHours=20 \
+  --PrintSeconds=0 \
+  --PrintMinutes=20 \
+  --PrintTime=0 \
+  --dtau=3 \
+  --IntMethod="Rosenbrock" \
+  --Table="SSP-Knoth" \
+  --TopoS="Flat" \
+  `#Topography `        \
+  --P1=0.0\
+  --P2=1000.0\
+  --P3=0.0\
+  --Stretch=true \
+  --StretchType="ICON" \
+  --H=19600.0 \
+  --Lx=40000.0 \
+  --Ly=40000.0 \
+  --x0=-20000.0 \
+  --y0=-20000.0 \
+  --nz=80 \
+  --nx=2 \
+  --ny=2 \
+  --OrdPoly=4 \
+  --HyperVisc=true \
+  --HyperDCurl=1.e6 \
+  --HyperDGrad=1.e6 \
+  --HyperDDiv=1.e6