From 0a2259f85a8d5b6f379ee16f906d9a3fc3fa00ee Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Mon, 5 Jun 2023 18:02:07 +0200 Subject: [PATCH] Ruisdael: rebase open boundary conditions onto v4.4.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open boundary conditions by Frans Liqui Lung Poisson solver with FFTW and OpenBC from CaNS by Pedro Simões Costa Merged and updated for mixed precision by Fredrik Jansson Squashed commit of the following: commit 9f10ba2115fef1385d0e890656895ae0ebb924e8 Merge: bec7d8c3 cc84edf8 Author: Fredrik Jansson Date: Wed Dec 14 15:52:02 2022 +0100 Merge remote-tracking branch 'frans/ruisdael' into ruisdael commit bec7d8c3b0327b8d4b85c666bc79f65c5317aa29 Merge: 8d64687c a77d2e7e Author: Fredrik Jansson Date: Tue Dec 13 17:17:00 2022 +0100 Merge branch 'ruisdael_exner' into ruisdael commit cc84edf89eac4499295ffab8adb3fb0fceec93aa Author: Frans Liqui Lung Date: Tue Dec 13 11:44:49 2022 +1100 removed old code to allow for different initial pressure solver commit 41edb3088c4b63ab51465874dd4e2ada411861ff Author: Frans Liqui Lung Date: Tue Dec 13 11:36:33 2022 +1100 adjusted initial convergence correction to be compatible with cosine fourier solver, implemented in modstartup to avoid cross dependency conflict between modpois and modopenboundaries commit 09498f6fc97b870da885605509f4d78746051bec Author: Frans Liqui Lung Date: Sun Oct 2 20:26:12 2022 +0000 removed print from modsynturb and removed hard set top layer of temperature in modboundary commit 031b97b17eb16425e6cf45129907dd03e3a65c2f Author: Frans Liqui Lung Date: Thu Sep 15 10:29:21 2022 +0000 fixed timing issue in modsynturb commit 8d64687cb40634704ced330a6dc59084d1106760 Author: Fredrik Jansson Date: Thu Nov 17 14:43:12 2022 +0100 Open boundary conditions and FFTW FFTW can be used also with open lateral boundary conditions. Pressure solver boundary condition is Neumann on the open boundaries. The following modifications are made for the direction(s) with open boundaries: * use Discrete Cosine Transform instead of FFT. - FFTW_REDFT_10 forward - FFTW_REDFT_01 back * normalization constant is different, a factor 2 per direction * the eigenvalues xrt, yrt are different By Pedro Simões Costa and Fredrik Jansson commit a77d2e7e229a9445bcd1faf141ad4c899a73c0fc Author: Fredrik Jansson Date: Fri Sep 16 15:07:13 2022 +0200 allow multiple planes for crossxz, crossyz and in any MPI tile commit 98dbc566a7a4417cb0957ddfcb5f3c4c5e054b28 Author: Fredrik Jansson Date: Tue Mar 2 11:22:12 2021 +0100 Added option to disable certain cross-sections (xy, xz, yz) commit 22174b0cfc459df085cb5b225cced99497070332 Author: Fredrik Jansson Date: Fri Sep 16 09:37:54 2022 +0200 add PHYSICS/lconstexner flag, output exnf, exnh to profiles.nc commit 854ca46a0d37ef835ce88afba23c483a520c839c Author: Fredrik Jansson Date: Thu Sep 15 14:44:10 2022 +0200 try using the initial pressure profile in the exner function commit 0fd18d20ce6fd7a8bc25b772fb031159d3df869b Merge: f2fe301c 41bd2209 Author: Fredrik Jansson Date: Mon Aug 8 15:51:58 2022 +0200 Merge pull request #77 from natalieth/patch-1 rename randomNumbers commit f2fe301c55d0b97035c4bcd735462a3115153fcf Merge: 4b762277 8784487b Author: Fredrik Jansson Date: Mon Aug 8 15:51:29 2022 +0200 Merge pull request #76 from natalieth/patch-2 remove comma (typo) commit 8784487bf059aa4a9d071cc1c0d7e13654ee38ff Author: Natalie Theeuwes <19662980+natalieth@users.noreply.github.com> Date: Mon Aug 8 15:25:22 2022 +0200 remove comma (typo) commit 41bd2209891112c3d1af162ccca71967ca4ae5d3 Author: Natalie Theeuwes <19662980+natalieth@users.noreply.github.com> Date: Mon Aug 8 15:23:19 2022 +0200 rename randomNumbers commit 4b762277b182edfedde296144bfe71d096f01172 Author: Fredrik Jansson Date: Wed Apr 6 20:15:07 2022 +0200 hypre: switch PFMG-preconditioner to use red-black smoother, faster commit bf320039d492b30e4a6a374a193728dc7d3dc55d Author: Fredrik Jansson Date: Mon Apr 4 18:33:38 2022 +0200 add SOLVER namelist variable hypre_logging for setting hypre printing amount commit 2db79933f4a07450f6a3bb855ee20fb81d63c100 Author: Fredrik Jansson Date: Mon Apr 4 18:32:33 2022 +0200 add alternative name for hypre library for Ubuntu commit 94f360af3625b6fefa17d47f322edf8321ac7940 Author: Fredrik Jansson Date: Sat Apr 2 09:11:15 2022 +0200 hypre singularity-fix: fix off-by-one, move anchor point cose to top of domain commit 729ce43fcdf01b6ab7da00243a659cbffd693835 Author: Fredrik Jansson Date: Fri Apr 1 16:35:02 2022 +0200 print wall time of poisson solver, try without modifying neighbors of the special element, zero guess in RK step 1 commit 2f59d561472ec92ed13c9632f0c5e1d034333895 Author: Fredrik Jansson Date: Tue Mar 29 21:28:30 2022 +0200 iterative solver: try making the matrix non-singular for better convergence commit 28717a26ebfd4cdfac53d6b479a2c7d2b847bbb7 Author: Frans Liqui Lung Date: Mon Mar 7 15:34:02 2022 +1100 added flag for compatibility periodic simulations commit 371ec7e90db521c0c360ffbfcb3addf5e04d7aed Author: Fredrik Jansson Date: Mon Feb 14 18:25:38 2022 +0100 hypre: update dummy routine in modnohypre to have same parameters as real routine commit 4b186a06fcffe58bf7dc5ceaca3d837dea9cae77 Author: Frans Liqui Lung Date: Fri Jan 28 15:40:14 2022 +1100 finalised extra hypre input for preconditioner commit 9401ce6cb2443a5152f73e3ca2c7904bb9366e21 Author: Frans Liqui Lung Date: Tue Jan 11 16:43:16 2022 +1100 disconnected iterations preconditionar from max iterations total of pressure solver (HYPRE) commit 7247c000f70aec9ba571b67487692310d338fd5c Author: Frans Liqui Lung Date: Mon Dec 20 17:09:06 2021 +0100 fixed mistake in initial divergence correction commit e9c5dd873e2971bdae3d6f94ba50b4362d045cc4 Author: Frans Liqui Lung Date: Mon Dec 20 14:11:34 2021 +0100 added divergence correction for initial field when a heterogneous initialisation is used commit 55e7fe48861e49a0d784ad0e155024b911cf21d3 Author: Frans Liqui Lung Date: Mon Dec 20 13:18:01 2021 +0100 Changed hypre coding to take solver as an input to the hypre subroutines, such that multiple solvers can be created and coexist. This is a preparation to be able to use a different solver to remove any divergence from a heterogneous starting field. Doesn't change anything to the performance/results of the solvers. commit 3b4aade871642d6389a2ddcb5b70608953001a20 Merge: a4289f0f 3f428245 Author: Fredrik Jansson Date: Mon Feb 14 18:47:40 2022 +0100 Merge commit '3f428245ff' into ruisdael commit 3f428245ffd1d7c57d1666041263303d17b9ebd4 Author: Frans Liqui Lung Date: Wed Dec 8 11:48:44 2021 +0100 fixed rhointi bug found in modopenboundary during tests commit 1ae67096610915518395c341c8e3d0b7130c4974 Author: Frans Liqui Lung Date: Wed Dec 8 09:52:33 2021 +0100 Adjusted program for openboundaries commit 73e4926507a65930e0557c3fd6667652290a93f8 Author: Frans Liqui Lung Date: Wed Dec 8 09:37:57 2021 +0100 fixed bugs found in modstartup during compilation commit e864a045edd2fa78f9884819efd98a77cd8eb0b4 Author: Frans Liqui Lung Date: Wed Dec 8 09:32:28 2021 +0100 fixed bugs found in advection during compilation commit c9c9a10e26a7cb9a43d8feedd4372132361b05b2 Author: Frans Liqui Lung Date: Wed Dec 8 09:23:13 2021 +0100 Adjusted modmpi for openboundaries commit 4d9fe7f80bda278ad19892d3723694b9cbccd4e3 Author: Frans Liqui Lung Date: Wed Dec 8 09:20:24 2021 +0100 fixed bugs found in modopenboundary during compilation commit 8242f567a853ee3797e0b518f2c4d34744864e8e Author: Frans Liqui Lung Date: Wed Dec 8 09:09:56 2021 +0100 Adjusted modlsm for openboundaries commit 78de429736e72cacf42fa10e2a581c69b3165c99 Author: Frans Liqui Lung Date: Wed Dec 8 09:04:31 2021 +0100 Adjusted modsurface for openboundaries commit c7a8aae90d086b0994eb4b6e78f311ca0f982962 Author: Frans Liqui Lung Date: Wed Dec 8 08:46:29 2021 +0100 Adjusted modstartup for openboundaries commit 356c2afd184e1b02fc884f1872786b8e3e4aff8a Author: Frans Liqui Lung Date: Tue Dec 7 16:46:56 2021 +0100 Adjusted modglobal for openboundaries commit b7216699c2695e0d318a81d86782a8f38cdb4e6d Author: Frans Liqui Lung Date: Tue Dec 7 16:32:08 2021 +0100 Adjusted modpois for openboundaries commit 5b6810bc89a276c07335c8b2863ab7df5c46fa1c Author: Frans Liqui Lung Date: Tue Dec 7 16:25:55 2021 +0100 Adjusted modgenstat for openboundaries, covariances take non-zero w profile into account commit 21512040d9ff851944d3cb2f69befffcb00ac68f Author: Frans Liqui Lung Date: Tue Dec 7 16:18:28 2021 +0100 Adjusted modforces for openboundaries commit 24db68a5b7c575b3c17fb5688f49ddc827211d17 Author: Frans Liqui Lung Date: Tue Dec 7 16:16:47 2021 +0100 Adjusted modsubgrid for openboundaries commit d250ed8b78bea478e816b0269579045c0a47eea9 Author: Frans Liqui Lung Date: Tue Dec 7 16:14:36 2021 +0100 Adjusted advection for openboundaries commit e174ff4e927eea60b65eb8213115af5e15ffcd3d Author: Frans Liqui Lung Date: Tue Dec 7 16:04:57 2021 +0100 Adjusted advec_2nd for openboundaries commit dc2300dd9ff48d05e325bc586d02b424fadcf4dd Author: Frans Liqui Lung Date: Tue Dec 7 15:40:53 2021 +0100 Adjusted modhypre for openboundaries commit 6d9b23afaf65f602e41f6177005e34af8c9da637 Author: Frans Liqui Lung Date: Tue Dec 7 15:35:30 2021 +0100 Added modopenboundary.f90 and modsynturb.f90 commit a4289f0f245cce5065c03d53074054853a376a8d Author: Bart van Stratum Date: Wed Sep 8 14:54:23 2021 +0200 Added CO2 scalar for A-Gs net assimilation, for offline budget checks commit 02372b533fef59a8f8b264dd564356981ba9998d Author: Bart van Stratum Date: Tue Sep 7 09:43:17 2021 +0200 Added A-Gs option without emission module commit 8c7881f60a1acab294e0f2664248088ae9c49a1b Author: Bart van Stratum Date: Tue Sep 7 08:08:14 2021 +0200 Added CO2 respiration/assimilation to cross-section and time series. commit 4dc4a0b1f2805b6fde7f47bac67de3f7138617aa Author: Bart van Stratum Date: Mon Sep 6 16:36:23 2021 +0200 Added A-Gs to the new land surface model commit 30507044028ecfc5d342b2a868838511e89fad3d Author: Bart van Stratum Date: Mon Sep 6 12:05:31 2021 +0200 Added basic infrastructure for A-Gs and communication with emission module commit bdfa25e428da0d836f5b37a34e026bfa97b0ba0a Author: Marco de Bruine Date: Tue Jun 8 16:56:12 2021 +0200 AG-s <-> emission interface indicators now relative to full list of scalar fields commit 0424c3cf59cbd06bbaf41bc713bef413eb6c93f7 Author: Marco de Bruine Date: Tue Jun 8 15:53:49 2021 +0200 Emission <-> AGS interface commit c05eee046767528d7dbe82ab5028557367fda59e Merge: d329736e 002a8623 Author: Fredrik Jansson Date: Mon Nov 9 12:13:43 2020 +0100 Merge branch 'ruisdael_lsm' into ruisdael commit 002a86233d9b01504ad30001abe9631509055ac1 Author: Bart van Stratum Date: Fri Nov 6 11:30:33 2020 +0100 Calculate obukhov length from tile averaged fluxes, instead of calculating a tiled mean commit 01a48d8892ea5f40c161a3cc8d07b72026aeb0a3 Author: Bart van Stratum Date: Tue Nov 3 09:12:35 2020 +0100 Bugfix harmonic mean calculation (which is currently not used in the code..) commit d329736ee90b2721ac232ab513c320a1d5331c87 Author: Fredrik Jansson Date: Tue Oct 27 11:25:58 2020 +0100 update fix for #58 - allow lsmcrossection also for new LSM isurf=11 commit 8b42f82e6c665ec4e0c563dea6d09a2bb276908c Author: Fredrik Jansson Date: Fri Oct 23 11:03:29 2020 +0200 emission: remove MPI-parallel .nc read, add better namelist error reporting commit fa1a4b53f2744e79655cf5c13bdb7f1976d65c5d Merge: 15a81a37 8230865e Author: Fredrik Jansson Date: Fri Oct 16 15:42:00 2020 +0200 Merge branch 'ruisdael_lsm' into ruisdael commit 8230865e7e998597a393d203c53b6158dbca4fd7 Author: Bart van Stratum Date: Fri Oct 16 10:18:03 2020 +0200 Added NetCDF file with van Genuchten parameters commit 15a81a37a765b813ef4300b310823dee596b4842 Merge: 6eddb8fd 58f9d331 Author: Fredrik Jansson Date: Thu Oct 15 14:57:17 2020 +0200 Merge branch 'to4.3_Fredrik' into ruisdael commit 58f9d33166b7bc3bfba9345c83fe7e5d55fe421d Author: Fredrik Jansson Date: Thu Oct 15 14:56:53 2020 +0200 advec_kappa : remove debug write() commit 6eddb8fd5aecc480e5b1b578813595c9a627e952 Author: Fredrik Jansson Date: Thu Oct 15 14:43:57 2020 +0200 modnudgebounday: use random_number() instead of rand() for better intel/gnu compatibility commit b70f80e807365c0927f986fe3b3a2a6c17339a43 Merge: 796eaaed 6b4a3ab1 Author: Fredrik Jansson Date: Thu Oct 15 14:04:57 2020 +0200 Merge branch '4.2_emission' into ruisdael commit 6b4a3ab1ed64339df0e33d82d61dd1b64a3d460c Author: Marco de Bruine Date: Thu Oct 15 13:43:10 2020 +0200 Renaming of emission related variables commit 6b199b38589c883e99cfc23b8d9087cd5e0a6614 Author: Fredrik Jansson Date: Mon Oct 12 17:01:12 2020 +0200 advec_kappa: do advection only for the k-range with non-zero input field commit 85b2be8179eac90029cdab284a371fb881fc461f Author: Jisk Attema Date: Thu Oct 1 13:57:37 2020 +0200 Add intent to slabsum arguments commit 5887bd07d50e2a816da5fa402cd8bb568cea9aec Author: Jisk Attema Date: Tue Sep 29 15:13:12 2020 +0200 gfortran switches when using default-real-8, constants and variables explicitly declared as double will become quadruples, which we dont want for DALES. quadruple precision could even result in software fallback, which is much slower. Adding the default-double-8 keeps doubles at double precision. Aliging stuff on 64-bit boundaries could help a bit with vectorization. Not initializing variables will speed things up when using large arrays commit 7ebd462b02287c7cdbcb7b15e5d2aee124f860ff Author: Fredrik Jansson Date: Wed Oct 7 18:59:33 2020 +0200 fix error in previous fix of #48 - used wrong time array in ls forcing commit e295c3560ef7c1336958842c018755eeda7f5851 Author: Fredrik Jansson Date: Tue Oct 6 10:01:40 2020 +0200 Update benchmark cases to use the standard kappa scheme 7 instead of the removed experimental one commit d088437e68ae94f879dc30cc115412d24a00f42e Author: Fredrik Jansson Date: Tue Oct 6 09:56:14 2020 +0200 Remove experimental kappa scheme iadv=77 which is now redundant. The non-uniform grid support has been merged into the main kappa scheme. commit 20b27f822ad2c1bd0c0499b1d44988b2dbde3134 Author: Fredrik Jansson Date: Tue Oct 6 09:45:00 2020 +0200 Optimized advec_kappa scheme - rlim function inlined and rewritten without division and eps1 - both branches of if uvw0 > 0 calculated, then selected in order to enable vectorization. by Jisk Attema and Fredrik Jansson commit d307f11af17b784f445dfe3a26d54971269b486a Merge: a73ad934 7f86be40 Author: Marco de Bruine Date: Tue Sep 22 10:53:16 2020 +0200 Merge to4.3 into 4.2_emission commit a73ad9341d43dc7a112e2a14976f3cd3e3a76dc7 Author: Marco de Bruine Date: Tue Sep 22 10:43:18 2020 +0200 Included all scalar fields in crosssection output (similar to fielddump) commit 78e6594993985f17ebfe1d06f3ffd45e7a5eb2ac Author: Marco de Bruine Date: Tue Sep 15 15:13:54 2020 +0200 Improved bookkeeping of scalar field numbers and names commit 7f2396041d6a906002cb8b904be647f2c945bf6f Merge: 038d2b6d c2db0eeb Author: julietbravo Date: Mon Sep 7 11:05:02 2020 +0200 Merge branch 'to4.3_lsm' into ruisdael_lsm commit c2db0eeb640ac22afdead487e221f6d4b7a2c781 Merge: 704d145e 7f86be40 Author: julietbravo Date: Mon Sep 7 11:02:57 2020 +0200 Merge remote-tracking branch 'upstream/to4.3' into to4.3_lsm commit 704d145ee245c3153e36a21fc0916bdc99ed79db Author: julietbravo Date: Mon Sep 7 10:31:11 2020 +0200 Small fix in calculation liq. water fraction, plus added cleanup of arrays commit c1eb5a43dd4c8bfe1135c68bcaa4675fe8c319ea Author: julietbravo Date: Fri Sep 4 15:02:36 2020 +0200 Fixed dynamic tile fraction for cases with open water commit 4aec697b06afb4ac4892aa722620eea14d5924ca Author: julietbravo Date: Fri Sep 4 11:33:18 2020 +0200 Added water tile to time series statistics commit 13e4129d13b609cc2c3127c052eb168845319497 Author: Marco de Bruine Date: Tue Sep 1 15:05:35 2020 +0200 Removed temporary forcing at lateral boundaries commit 36730a88c0dd34e1f84417d8edca46bca4b8d123 Author: Marco de Bruine Date: Tue Sep 1 15:02:30 2020 +0200 Added boundary nudging of scalar fields commit 0069bf457f3f25530bae30931caad3f6994e53d9 Author: julietbravo Date: Thu Jul 30 15:43:29 2020 +0200 Added additional variables to surface cross-sections commit 663791693c98267c075f49ee4d51d06b6e089dae Author: julietbravo Date: Tue Jul 21 14:12:40 2020 +0200 Modified modlsmcrosssection for new LSM commit 45c6b5efa2f84aa51b567930fe3216ffeae5ebdb Author: julietbravo Date: Tue Jul 21 12:45:14 2020 +0200 Store/load the water tiles for restarts (results = bitwise identical now) commit 5925ee216ad89ba44f24da3598df7071416b8de6 Author: julietbravo Date: Tue Jul 21 09:19:12 2020 +0200 Fixed some MPI issues in new LSM commit 3f8c6acd40aa18e299445adf92443f0452faecd6 Author: julietbravo Date: Mon Jul 20 11:55:18 2020 +0200 First ~working version with water tile (still has some issues...) commit e4c0755b49e9c486d61f5a2bfb324f1f21d3daff Merge: a6ca5f1c f470b418 Author: julietbravo Date: Fri Jul 17 15:39:12 2020 +0200 Merge remote-tracking branch 'upstream/to4.3' into to4.3_lsm_water commit a6ca5f1cf560df782fac24acbd7b5a3120c8f8af Author: julietbravo Date: Tue Jul 7 17:28:34 2020 +0200 Added option to read the spatial data for heterogeneous land surfaces commit b6249221bb89e298237f8e46e8fc6db1712b1512 Author: julietbravo Date: Tue Jul 7 11:53:21 2020 +0200 Added restart functionality to new LSM commit cfc205ed523a6231acf95ecd90f99d7770f76f33 Author: julietbravo Date: Tue Jul 7 11:00:32 2020 +0200 Added old soil grid definition, different soil interpolation types, plus fixed bug in buoyancy calculation commit a2e51fbee5cf9af33e742db938884f40a6d4310c Author: julietbravo Date: Mon Jul 6 16:29:59 2020 +0200 Added calculation of liquid water reservoir (plus bugfix ra calculation) commit 491b3933dea52377d2fcdbb6f0e35e46603d1f5b Author: julietbravo Date: Mon Jul 6 12:21:21 2020 +0200 Added root water extraction to soil water budget commit 4891bb5c931e5f551d3b102ccefef3963281013c Author: julietbravo Date: Mon Jul 6 11:23:16 2020 +0200 Added bulk (grid point mean) quantities needed for SGS model commit 027b03b04beae696910fd87417e6de2900e13da0 Author: julietbravo Date: Mon Jul 6 09:01:42 2020 +0200 Added calculation of bulk (grid point mean) scalar BCs commit a0d4cbfd713c13cde0e6c5ca545bb7daaf46b74d Author: julietbravo Date: Fri Jul 3 18:02:56 2020 +0200 Working surface solver, with fluxes per tile commit 12dd3e347d95892ca09358d0cd673912164524cb Author: Marco de Bruine Date: Fri Jul 3 14:13:06 2020 +0200 3D emission and namelist options; emission switch (l_emission) and vertical upper limit to apply emission (kemis) commit 3184f6b6cdcb1c0b052abfa5ee174b28900c3364 Author: julietbravo Date: Thu Jul 2 16:32:56 2020 +0200 Bugfix calculation canopy resistances commit 656839f90dfe463bcd0e7bdc4d3db4b99740dcd8 Author: julietbravo Date: Thu Jul 2 16:30:10 2020 +0200 Added calculation canopy and soil resistance commit c56c4781f83501b0c6bf047d5f7dd8ddf691b072 Author: julietbravo Date: Thu Jul 2 12:13:05 2020 +0200 Working tiled surface layer solver commit 8def726de1eae040453995e28caeae4f199f39d2 Author: julietbravo Date: Wed Jul 1 16:25:10 2020 +0200 Start of tiled surface layer solver commit 2b3e8262e99246170484294cb421381e795bf47b Author: julietbravo Date: Wed Jul 1 10:00:08 2020 +0200 Added calculation hydraulic properties (van Genuchten) and diffusion solver for soil moisture commit f215e669ca592ad25874cc90a91a0c6546bb4664 Author: julietbravo Date: Tue Jun 30 16:51:10 2020 +0200 Added calculation of soil thermal properties, and (explicit) temperature diffusion solver commit 61de721e4f10d9b608776c28ca991b1a09dbf647 Author: julietbravo Date: Tue Jun 30 14:39:31 2020 +0200 Added calculation derived van Genuchten parameters commit 32614223236160ad6ead3250552aec171ef6bc43 Author: julietbravo Date: Tue Jun 30 11:38:47 2020 +0200 Added minimal set of time series statistics for the new LSM commit 6b5d8bea7a80934f920816d57c7b7e8d66e3474d Author: julietbravo Date: Tue Jun 30 11:08:45 2020 +0200 Modified modlsmstat to work with the new LSM commit 693964183b27ac3efd5f7af62e4590cd48b0c5be Author: julietbravo Date: Tue Jun 30 09:52:26 2020 +0200 Minor changes new LSM commit 7818aeb7f67d0eb02306d20107207c8c1385921a Author: Marco de Bruine Date: Mon Jun 29 16:49:25 2020 +0200 Typo in modemisdata.f90 commit 0772796637359909c91d59d943d3233a3319f2d2 Author: Marco de Bruine Date: Mon Jun 29 16:13:57 2020 +0200 Corrected emission scalar list to work with microphysics, i.e. skip first 2 scalar fields commit 1d141d49e3532d459cae5ab815a3d6f6c9d7590a Author: Marco de Bruine Date: Mon Jun 29 16:12:01 2020 +0200 Fixed incorrect NORTH/EAST boundary indices commit b7dd889e706cbda0c90471b3b2bf680684413807 Author: julietbravo Date: Mon Jun 29 15:45:35 2020 +0200 Added soil grid and initialisation of soil commit 9e78c3d813a6e940f2a5658c3a5b0f0f47f5b6d9 Author: julietbravo Date: Mon Jun 29 13:52:11 2020 +0200 Added homogeneous initialisation of LSM commit db40554172aa43424117c3df69d4c791d17efec3 Author: julietbravo Date: Mon Jun 29 12:23:47 2020 +0200 Added data structures for land-surface tiles commit 796eaaed91ff40fae56a39c507f76dbc506663a7 Merge: 7860bb97 e5da4f84 Author: Fredrik Jansson Date: Thu Jun 25 12:01:06 2020 +0200 Merge remote-tracking branch 'origin/4.2_emission' into ruisdael Merge surface emission map code by M de Bruine commit 7860bb97be22a432cdb70494ad8a9455404eb7dd Merge: 038d2b6d f0fb85d5 Author: Fredrik Jansson Date: Thu Jun 25 11:58:33 2020 +0200 Merge branch 'to4.3' into ruisdael merging for gfortran 10 compatibility commit e5da4f84ed160e08e4798293648e0cff14e14e21 Author: Marco de Bruine Date: Thu Jun 25 11:27:58 2020 +0200 Add file moddatetime for reading emission files. commit 5e47c9393baa1263ff023c5efad3574987fc1b5b Author: julietbravo Date: Mon Jun 15 12:06:48 2020 +0200 Start at implemention realistic LSM commit 5cf1affb255346a1e86046df943341e819517266 Merge: c6c8324f 038d2b6d Author: Marco de Bruine Date: Mon Jun 8 11:30:33 2020 +0200 Merge with ruisdael branch commit c6c8324f9826b9d56f9aaf2c0e8192453ced6c7a Merge: 1407034a 9278ce35 Author: Marco de Bruine Date: Fri Jun 5 16:10:02 2020 +0200 Merge with master commit 1407034a8e38d052f7704ab5cfbe2fd0f6446367 Author: Marco de Bruine Date: Fri Jun 5 15:17:20 2020 +0200 Added scaling for model resolution and domain dboundary 'nullifier' commit 038d2b6daaa01806ca2664124dd65d6b6f5ab825 Merge: 89b48301 1cc8a3cf Author: Fredrik Jansson Date: Wed Jun 3 16:20:56 2020 +0200 Merge branch 'to4.3' into ruisdael for PR #53 - Only set `tauField` when A-gs is enabled commit baeab6543ed317ee618a12f51f2a8adaeb67b116 Author: Marco de Bruine Date: Tue Apr 28 10:38:29 2020 +0200 Changes in documentation and compiler settings commit 89b4830189ff6e3352da65ca7a15b104d7765afc Author: Fredrik Jansson Date: Tue Apr 14 15:43:41 2020 +0200 boundary nudging: add check for tile size being sufficient to contain the nudging domain commit 8cd006c530e132b06246d770f68bce083e2ca509 Author: Fredrik Jansson Date: Fri Apr 10 14:56:32 2020 +0200 Print warning when unspecified iadv_sv values are set to iadv_mom commit 6e208f5b524cc2a4ed0282efb407eb7f31f0b16f Author: Fredrik Jansson Date: Tue Apr 7 13:11:48 2020 +0200 Add boundary nudging by BvS commit e61fddcfea673866de97705b6de2c2cbb024b22b Author: Marco de Bruine Date: Thu Feb 27 15:03:53 2020 +0100 Realistic surface emissions (WIP) commit 4b9dec3551c69f75d2724e954ea357dfe536c343 Author: Marco de Bruine Date: Mon Dec 9 15:37:52 2019 +0100 New emission routines: Reading (parallel netCDF) of 2D surface emission on native grid --- src/advec_2nd.f90 | 31 +- src/modadvection.f90 | 31 +- src/modboundary.f90 | 15 +- src/modchecksim.f90 | 9 +- src/modfftw.f90 | 77 ++- src/modforces.f90 | 97 ++- src/modglobal.f90 | 33 +- src/modhypre.f90 | 528 ++++++++++------ src/modlsm.f90 | 15 +- src/modmpi.f90 | 237 +++++++- src/modopenboundary.f90 | 1192 +++++++++++++++++++++++++++++++++++++ src/modpois.f90 | 93 ++- src/modstartup.f90 | 247 +++++--- src/modsurface.f90 | 17 +- src/modsynturb.f90 | 757 +++++++++++++++++++++++ src/modthermodynamics.f90 | 31 +- src/program.f90 | 42 +- 17 files changed, 3026 insertions(+), 426 deletions(-) create mode 100644 src/modopenboundary.f90 create mode 100644 src/modsynturb.f90 diff --git a/src/advec_2nd.f90 b/src/advec_2nd.f90 index b2457bba..836e0655 100644 --- a/src/advec_2nd.f90 +++ b/src/advec_2nd.f90 @@ -81,7 +81,7 @@ subroutine advecc_2nd(a_in,a_out) end do do j=2,j1 - do k=2,kmax + do k=2,kmax do i=2,i1 a_out(i,j,k) = a_out(i,j,k)- (1./rhobf(k))*( & w0(i,j,k+1) * (rhobf(k+1) * a_in(i,j,k+1) + rhobf(k) * a_in(i,j,k)) & @@ -118,12 +118,12 @@ end subroutine advecc_2nd !> Advection at the u point. -subroutine advecu_2nd(a_in, a_out) - +subroutine advecu_2nd(a_in, a_out, sx) use modglobal, only : i1,ih,j1,jh,k1,kmax,dxiq,dyiq,dziq,dzf,dzh,leq use modfields, only : u0, v0, w0, rhobf implicit none + integer, intent(in) :: sx real(field_r), dimension(2-ih:i1+ih,2-jh:j1+jh,k1), intent(in) :: a_in real(field_r), dimension(2-ih:i1+ih,2-jh:j1+jh,k1), intent(inout) :: a_out ! real, dimension(2-ih:i1+ih,2-jh:j1+jh,k1) :: rho_a_in @@ -144,7 +144,7 @@ subroutine advecu_2nd(a_in, a_out) do j=2,j1 jm=j-1 jp=j+1 - do i=2,i1 + do i=sx,i1 im=i-1 ip=i+1 a_out(i,j,k) = a_out(i,j,k)- ( & @@ -166,7 +166,7 @@ subroutine advecu_2nd(a_in, a_out) do j=2,j1 jm=j-1 jp=j+1 - do i=2,i1 + do i=sx,i1 im=i-1 ip=i+1 a_out(i,j,1) = a_out(i,j,1)-(1./rhobf(1))*( & @@ -181,7 +181,7 @@ subroutine advecu_2nd(a_in, a_out) do k=2,kmax km=k-1 kp=k+1 - do i=2,i1 + do i=sx,i1 im=i-1 ip=i+1 a_out(i,j,k) = a_out(i,j,k)- (1./rhobf(k))*( & @@ -197,7 +197,7 @@ subroutine advecu_2nd(a_in, a_out) do j=2,j1 jm=j-1 jp=j+1 - do i=2,i1 + do i=sx,i1 im=i-1 ip=i+1 a_out(i,j,1) = a_out(i,j,1)- (1./rhobf(1))*( & @@ -212,7 +212,7 @@ subroutine advecu_2nd(a_in, a_out) do k=2,kmax km=k-1 kp=k+1 - do i=2,i1 + do i=sx,i1 im=i-1 ip=i+1 a_out(i,j,k) = a_out(i,j,k)- (1./rhobf(k))*( & @@ -234,12 +234,13 @@ end subroutine advecu_2nd !> Advection at the v point. -subroutine advecv_2nd(a_in, a_out) - +subroutine advecv_2nd(a_in, a_out, sy) use modglobal, only : i1,ih,j1,jh,k1,kmax,dxiq,dyiq,dziq,dzf,dzh,leq use modfields, only : u0, v0, w0, rhobf implicit none + + integer,intent(in) :: sy real(field_r), dimension(2-ih:i1+ih,2-jh:j1+jh,k1), intent(in) :: a_in !< Input: the v-field real(field_r), dimension(2-ih:i1+ih,2-jh:j1+jh,k1), intent(inout) :: a_out !< Output: the tendency ! real, dimension(2-ih:i1+ih,2-jh:j1+jh,k1) :: rho_a_in @@ -257,7 +258,7 @@ subroutine advecv_2nd(a_in, a_out) do k=1,kmax km=k-1 kp=k+1 - do j=2,j1 + do j=sy,j1 jm=j-1 jp=j+1 do i=2,i1 @@ -279,7 +280,7 @@ subroutine advecv_2nd(a_in, a_out) if (leq) then - do j=2,j1 + do j=sy,j1 jm=j-1 jp=j+1 do i=2,i1 @@ -291,7 +292,7 @@ subroutine advecv_2nd(a_in, a_out) end do end do - do j=2,j1 + do j=sy,j1 jm=j-1 jp=j+1 do k=2,kmax @@ -309,7 +310,7 @@ subroutine advecv_2nd(a_in, a_out) end do else - do j=2,j1 + do j=sy,j1 jm=j-1 jp=j+1 do i=2,i1 @@ -322,7 +323,7 @@ subroutine advecv_2nd(a_in, a_out) end do end do - do j=2,j1 + do j=sy,j1 jm=j-1 jp=j+1 do k=2,kmax diff --git a/src/modadvection.f90 b/src/modadvection.f90 index 4bb644a8..ade47bd8 100644 --- a/src/modadvection.f90 +++ b/src/modadvection.f90 @@ -30,7 +30,9 @@ module modadvection subroutine advection use modglobal, only : lmoist, nsv, iadv_mom,iadv_tke,iadv_thl,iadv_qt,iadv_sv, & - iadv_cd2,iadv_5th,iadv_52,iadv_cd6,iadv_62,iadv_kappa,iadv_upw,iadv_hybrid,iadv_hybrid_f,iadv_null,leq + iadv_cd2,iadv_5th,iadv_52,iadv_cd6,iadv_62,iadv_kappa,& + iadv_upw,iadv_hybrid,iadv_hybrid_f,iadv_null,leq,& + lopenbc,lboundary,lperiodic use modfields, only : u0,up,v0,vp,w0,wp,e120,e12p,thl0,thlp,qt0,qtp,sv0,svp use modsubgrid, only : lsmagorinsky use advec_2nd, only : advecu_2nd, advecv_2nd, advecw_2nd, advecc_2nd @@ -43,15 +45,20 @@ subroutine advection use advec_kappa, only : advecc_kappa use advec_upw, only : advecc_upw implicit none - integer :: n + integer :: n,sx = 2,sy = 2 ! leq = .false. ! for testing that the non-uniform advection routines agree with the uniform ones ! when the grid is uniform - + + if(lopenbc) then ! Calculate tendencies only for non-domain boundary cells if openboundaries are used + if(lboundary(1).and. .not. lperiodic(1)) sx = 3 + if(lboundary(3).and. .not. lperiodic(3)) sy = 3 + endif + select case(iadv_mom) case(iadv_cd2) - call advecu_2nd(u0,up) - call advecv_2nd(v0,vp) + call advecu_2nd(u0,up,sx) + call advecv_2nd(v0,vp,sy) call advecw_2nd(w0,wp) case(iadv_5th) !if (.not. leq) stop "advec_5th does not support a non-uniform vertical grid." @@ -82,7 +89,7 @@ subroutine advection call advecv_5th(v0,vp) call advecw_5th(w0,wp) case(iadv_null) - ! null advection scheme + ! null advection scheme stop "Null advection scheme selected for iadv_mom - probably a bad idea." case default stop "Unknown advection scheme " @@ -109,9 +116,9 @@ subroutine advection call advecc_hybrid(e120,e12p) case(iadv_hybrid_f) !if (.not. leq) stop "advec_hybrid_f does not support a non-uniform vertical grid." - call advecc_hybrid_f(e120,e12p) + call advecc_hybrid_f(e120,e12p) case(iadv_null) - ! null advection scheme + ! null advection scheme stop "Null advection scheme selected for iadv_tke - probably a bad idea." case default stop "Unknown advection scheme " @@ -143,8 +150,8 @@ subroutine advection !if (.not. leq) stop "advec_hybrid_f does not support a non-uniform vertical grid." call advecc_hybrid_f(thl0,thlp,1.0) case(iadv_null) - ! null advection scheme - stop "Null advection scheme selected for iadv_thl - probably a bad idea." + ! null advection scheme + stop "Null advection scheme selected for iadv_thl - probably a bad idea." case default stop "Unknown advection scheme " end select @@ -174,7 +181,7 @@ subroutine advection !if (.not. leq) stop "advec_hybrid_f does not support a non-uniform vertical grid." call advecc_hybrid_f(qt0,qtp,1e-3) case(iadv_null) - ! null advection scheme + ! null advection scheme stop "Null advection scheme selected for iadv_qt - probably a bad idea." case default stop "Unknown advection scheme " @@ -204,7 +211,7 @@ subroutine advection call advecc_hybrid(sv0(:,:,:,n),svp(:,:,:,n)) case(iadv_hybrid_f) !if (.not. leq) stop "advec_hybrid_f does not support a non-uniform vertical grid." - call advecc_hybrid_f(sv0(:,:,:,n),svp(:,:,:,n)) + call advecc_hybrid_f(sv0(:,:,:,n),svp(:,:,:,n)) case(iadv_null) ! null advection scheme - do nothing case default diff --git a/src/modboundary.f90 b/src/modboundary.f90 index 7d5e73c6..9a60fc5f 100644 --- a/src/modboundary.f90 +++ b/src/modboundary.f90 @@ -140,7 +140,7 @@ end subroutine cyclicm !! to infinity at the bottom of the sponge layer. !! \endlatexonly subroutine grwdamp - use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf + use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf,lopenbc use modfields, only : up,vp,wp,thlp,qtp,u0,v0,w0,thl0,qt0,sv0,ug,vg & ,thl0av,qt0av,sv0av,u0av,v0av implicit none @@ -189,12 +189,13 @@ subroutine grwdamp ! Additional to gravity wave damping, set qt, thl and sv0(:) equal to slabaverage ! at level kmax. ! Originally done in subroutine tqaver, now using averages from modthermodynamics - - thl0(2:i1,2:j1,kmax) = thl0av(kmax) - qt0 (2:i1,2:j1,kmax) = qt0av(kmax) - do n=1,nsv - sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n) - end do + if(.not.lopenbc) then + thl0(2:i1,2:j1,kmax) = thl0av(kmax) + qt0 (2:i1,2:j1,kmax) = qt0av(kmax) + do n=1,nsv + sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n) + end do + endif return end subroutine grwdamp diff --git a/src/modchecksim.f90 b/src/modchecksim.f90 index c9ee78b8..f7510727 100644 --- a/src/modchecksim.f90 +++ b/src/modchecksim.f90 @@ -31,7 +31,7 @@ module modchecksim implicit none private - public initchecksim,checksim + public initchecksim,checksim,chkdiv real :: tcheck = 0. integer(kind=longint) :: tnext = 3600.,itcheck @@ -39,7 +39,7 @@ module modchecksim ! explanations for dt_limit, determined in tstep_update() character (len=15) :: dt_reasons(0:5) = [character(len=15):: "initial step", "timee", "dt_lim" , "idtmax", "velocity", "diffusion"] - + save contains !> Initializing Checksim. Read out the namelist, initializing the variables @@ -200,9 +200,8 @@ subroutine chkdiv write(6 ,'(A,2ES11.2,A,A)')'divmax, divtot = ', divmax, divtot, ' dt limited by ', dt_reasons(dt_reason) end if - return - + return + end subroutine chkdiv end module modchecksim - diff --git a/src/modfftw.f90 b/src/modfftw.f90 index 91fd950d..ebb38343 100644 --- a/src/modfftw.f90 +++ b/src/modfftw.f90 @@ -28,7 +28,7 @@ module modfftw use, intrinsic :: iso_c_binding use modprecision, only : pois_r use modglobal, only : itot, jtot, imax, jmax, i1, j1, ih, jh, kmax, ijtot & - , dxi,dyi,pi + , dxi,dyi,pi, lperiodic use modmpi, only : commcol, commrow, mpierr, nprocx, D_MPI_ALLTOALL, nprocy & , myidx, myidy implicit none @@ -170,7 +170,12 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) ! Prepare 1d FFT transforms ! TODO: in plan_many, skip part where k > kmax embed(1) = itot - kinds(1) = FFTW_R2HC + if (lperiodic(1)) then + kinds(1) = FFTW_R2HC + else + kinds(1) = FFTW_REDFT10 ! DCT for Neumann BC for pressure + end if + planx = fftw_plan_many_r2r_if( & 1, & ! rank embed, & ! n (size) [array] @@ -188,7 +193,12 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) ) embed(1) = itot - kinds(1) = FFTW_HC2R + if (lperiodic(1)) then + kinds(1) = FFTW_HC2R + else + kinds(1) = FFTW_REDFT01 ! Inverse DCT + end if + planxi = fftw_plan_many_r2r_if( & 1, & ! rank embed, & ! n (size) [array] @@ -206,7 +216,12 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) ) embed(1) = jtot - kinds(1) = FFTW_R2HC + if (lperiodic(3)) then + kinds(1) = FFTW_R2HC + else + kinds(1) = FFTW_REDFT10 + end if + plany = fftw_plan_many_r2r_if( & 1, & ! rank embed, & ! n (size) [array] @@ -222,9 +237,13 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) kinds, & ! kind FFTW_MEASURE & ! flags (FFTW_MEASURE or FFTW_ESTIMATE) ) - + embed(1) = jtot - kinds(1) = FFTW_HC2R + if (lperiodic(3)) then + kinds(1) = FFTW_HC2R + else + kinds(1) = FFTW_REDFT01 ! Inverse DCT + end if planyi = fftw_plan_many_r2r_if( & 1, & ! rank embed, & ! n (size) [array] @@ -276,6 +295,9 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) kinds(1) = FFTW_R2HC kinds(2) = FFTW_R2HC + if (.not. lperiodic(1)) kinds(1) = FFTW_REDFT10 + if (.not. lperiodic(3)) kinds(2) = FFTW_REDFT10 + planxy = fftw_plan_guru_r2r_if(& 2, & ! rank dimij, & ! dims @@ -289,6 +311,9 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe) kinds(1) = FFTW_HC2R kinds(2) = FFTW_HC2R + if (.not. lperiodic(1)) kinds(1) = FFTW_REDFT01 + if (.not. lperiodic(3)) kinds(2) = FFTW_REDFT01 + planxyi = fftw_plan_guru_r2r_if(& 2, & ! rank dimij, & ! dims @@ -656,7 +681,7 @@ subroutine fftwf(p, Fp) stop 'Illegal method in fftwsolver.' endif - Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot) + !Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot) ! moving normalization to fftwb end subroutine subroutine fftwb(p, Fp) @@ -664,8 +689,9 @@ subroutine fftwb(p, Fp) real(pois_r), pointer :: p(:,:,:) real(pois_r), pointer :: Fp(:,:,:) - - Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot) + real :: norm + + !Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot) if (method == 1) then call transpose_a3inv(p201, Fp) @@ -680,19 +706,26 @@ subroutine fftwb(p, Fp) call fftw_execute_r2r_if(planxyi, p_nohalo, p_nohalo) else stop 'Illegal method in fftwsolver.' - endif - end subroutine + endif + norm = 1.0/ijtot + if (.not. lperiodic(1)) norm = norm / 2 ! different normalization for the DCT + if (.not. lperiodic(3)) norm = norm / 2 + p(:,:,:) = p(:,:,:) * norm ! do all normalization at once + end subroutine ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! subroutine fftwinit_factors(xyrt) + use modglobal, only : i1,j1,kmax,imax,jmax,itot,jtot,dxi,dyi,pi,ih,jh,lperiodic + use modmpi, only : myidx, myidy + implicit none real(pois_r), allocatable :: xyrt(:,:) - integer :: i,j,iv,jv - real(pois_r) :: fac - real(pois_r) :: xrt(itot), yrt(jtot) + integer :: i,j,iv,jv + real(pois_r) :: fac + real(pois_r) :: xrt(itot), yrt(jtot) ! Generate Eigenvalues xrt and yrt resulting from d**2/dx**2 F = a**2 F @@ -712,27 +745,19 @@ subroutine fftwinit_factors(xyrt) ! I --> direction fac = 1./(2.*itot) - do i=2,(itot/2) + if (.not. lperiodic(1)) fac = 1./(4.*itot) + do i=2,itot xrt(i)=-4.*dxi*dxi*(sin(float(2*(i-1))*pi*fac))**2 - xrt(itot - i + 2) = xrt(i) end do xrt(1) = 0. - if (mod(itot,2) == 0) then - ! Nyquist frequency - xrt(1 + itot/2) = -4.*dxi*dxi - endif ! J --> direction fac = 1./(2.*jtot) - do j=2,(jtot/2) + if (.not. lperiodic(3)) fac = 1./(4.*jtot) + do j=2,jtot yrt(j)=-4.*dyi*dyi*(sin(float(2*(j-1))*pi*fac))**2 - yrt(jtot - j + 2) = yrt(j) end do yrt(1) = 0. - if (mod(jtot,2) == 0) then - ! Nyquist frequency - yrt(1 + jtot/2) = -4.*dyi*dyi - endif ! Combine I and J directions ! Note that: diff --git a/src/modforces.f90 b/src/modforces.f90 index 09bbe322..01915ec5 100644 --- a/src/modforces.f90 +++ b/src/modforces.f90 @@ -112,63 +112,56 @@ subroutine coriolis ! | !-----------------------------------------------------------------| - use modglobal, only : i1,j1,kmax,dzh,dzf,cu,cv,om22,om23,lcoriol + use modglobal, only : i1,j1,kmax,dzh,dzf,cu,cv,om22,om23,lcoriol,lopenbc,lboundary,lperiodic use modfields, only : u0,v0,w0,up,vp,wp implicit none - integer i, j, k, jm, jp, km, kp + integer :: i, j, k,im,ip, jm, jp, km, kp,sx=2,sy=2 if (lcoriol .eqv. .false.) return - do k=2,kmax - kp=k+1 - km=k-1 - do j=2,j1 - jp=j+1 - jm=j-1 - do i=2,i1 - - up(i,j,k) = up(i,j,k)+ cv*om23 & - +(v0(i,j,k)+v0(i,jp,k)+v0(i-1,j,k)+v0(i-1,jp,k))*om23*0.25 & - -(w0(i,j,k)+w0(i,j,kp)+w0(i-1,j,kp)+w0(i-1,j,k))*om22*0.25 - - vp(i,j,k) = vp(i,j,k) - cu*om23 & - -(u0(i,j,k)+u0(i,jm,k)+u0(i+1,jm,k)+u0(i+1,j,k))*om23*0.25 - - - wp(i,j,k) = wp(i,j,k) + cu*om22 +( (dzf(km) * (u0(i,j,k) + u0(i+1,j,k) ) & - + dzf(k) * (u0(i,j,km) + u0(i+1,j,km)) ) / dzh(k) ) & - * om22*0.25 - end do - end do -! -------------------------------------------end i&j-loop + ! Only calculate interior tendencies when open boundaries are used + if(lopenbc) then + if(lboundary(1).and..not.lperiodic(1)) sx = 3 + if(lboundary(3).and..not.lperiodic(3)) sy = 3 + endif + + ! u tendency + do k = 1,kmax + kp = k+1 + do j = 2,j1 + jp = j+1 + do i = sx,i1 + im = i-1 + up(i,j,k) = up(i,j,k)+ cv*om23 & + +(v0(i,j,k)+v0(i,jp,k)+v0(im,j,k)+v0(im,jp,k))*om23*0.25 & + -(w0(i,j,k)+w0(i,j,kp)+w0(im,j,kp)+w0(im,j,k))*om22*0.25 + end do + end do end do -! -------------------------------------------end k-loop - -! -------------------------------------------- -! special treatment for lowest full level: k=1 -! -------------------------------------------- - - do j=2,j1 - jp = j+1 - jm = j-1 - do i=2,i1 - - up(i,j,1) = up(i,j,1) + cv*om23 & - +(v0(i,j,1)+v0(i,jp,1)+v0(i-1,j,1)+v0(i-1,jp,1))*om23*0.25 & - -(w0(i,j,1)+w0(i,j ,2)+w0(i-1,j,2)+w0(i-1,j ,1))*om22*0.25 - - vp(i,j,1) = vp(i,j,1) - cu*om23 & - -(u0(i,j,1)+u0(i,jm,1)+u0(i+1,jm,1)+u0(i+1,j,1))*om23*0.25 - - wp(i,j,1) = 0.0 - + ! v tendency + do k = 1,kmax + do j = sy,j1 + jm = j -1 + do i = 2,i1 + ip = i+1 + vp(i,j,k) = vp(i,j,k) - cu*om23 & + -(u0(i,j,k)+u0(i,jm,k)+u0(ip,jm,k)+u0(ip,j,k))*om23*0.25 + end do + end do end do + ! w tendency + do k=2,kmax ! wp(:,:,1)=0 + km=k-1 + do j=2,j1 + do i=2,i1 + ip = i+1 + wp(i,j,k) = wp(i,j,k) + cu*om22 +( (dzf(km) * (u0(i,j,k) + u0(ip,j,k) ) & + + dzf(k) * (u0(i,j,km) + u0(ip,j,km)) ) / dzh(k) ) & + * om22*0.25 + end do + end do end do -! ----------------------------------------------end i,j-loop - - - return end subroutine coriolis subroutine lstend @@ -211,17 +204,16 @@ subroutine lstend kp=k+1 km=k-1 - if (whls(kp).lt.0) then !downwind scheme for subsidence + if (whls(kp).lt.0) then !upwind scheme for subsidence thlp(2:i1,2:j1,k) = thlp(2:i1,2:j1,k) - whls(kp) * (thl0(2:i1,2:j1,kp) - thl0(2:i1,2:j1,k))/dzh(kp) qtp (2:i1,2:j1,k) = qtp (2:i1,2:j1,k) - whls(kp) * (qt0 (2:i1,2:j1,kp) - qt0 (2:i1,2:j1,k))/dzh(kp) if (lmomsubs) then up(2:i1,2:j1,k) = up(2:i1,2:j1,k) - whls(kp) * (u0(2:i1,2:j1,kp) - u0(2:i1,2:j1,k))/dzh(kp) vp(2:i1,2:j1,k) = vp(2:i1,2:j1,k) - whls(kp) * (v0(2:i1,2:j1,kp) - v0(2:i1,2:j1,k))/dzh(kp) endif - svp(2:i1,2:j1,k,:) = svp(2:i1,2:j1,k,:) - whls(kp) * (sv0(2:i1,2:j1,kp,:) - sv0(2:i1,2:j1,k,:))/dzh(kp) - else !downwind scheme for mean upward motions + else !upwind scheme for mean upward motions if (k > 1) then !neglect effect of mean ascending on tendencies at the lowest full level thlp(2:i1,2:j1,k) = thlp(2:i1,2:j1,k) - whls(k) * (thl0(2:i1,2:j1,k) - thl0(2:i1,2:j1,km))/dzh(k) qtp (2:i1,2:j1,k) = qtp (2:i1,2:j1,k) - whls(k) * (qt0 (2:i1,2:j1,k) - qt0 (2:i1,2:j1,km))/dzh(k) @@ -237,10 +229,7 @@ subroutine lstend qtp (2:i1,2:j1,k) = qtp (2:i1,2:j1,k)-u0av(k)*dqtdxls (k)-v0av(k)*dqtdyls (k) + dqtdtls(k) up (2:i1,2:j1,k) = up (2:i1,2:j1,k)-u0av(k)*dudxls (k)-v0av(k)*dudyls (k) + dudtls(k) vp (2:i1,2:j1,k) = vp (2:i1,2:j1,k)-u0av(k)*dvdxls (k)-v0av(k)*dvdyls (k) + dvdtls(k) - enddo - - return end subroutine lstend end module modforces diff --git a/src/modglobal.f90 b/src/modglobal.f90 index 518e2e44..f61a4ccc 100644 --- a/src/modglobal.f90 +++ b/src/modglobal.f90 @@ -22,6 +22,7 @@ ! Copyright 1993-2009 Delft University of Technology, Wageningen University, Utrecht University, KNMI ! module modglobal +use iso_c_binding use modprecision implicit none save @@ -148,6 +149,7 @@ module modglobal logical :: lnoclouds = .false. !< switch to enable/disable thl calculations logical :: lfast_thermo = .false. !< switch to enable faster icethermo scheme logical :: lsgbucorr= .false. !< switch to enable subgrid buoyancy flux + logical :: lconstexner = .false. !< switch to use the initial pressure profile in the exner function ! Poisson solver: modpois / modhypre integer :: solver_id = 0 ! Identifier for nummerical solver: 0 1 2 3 4 @@ -156,7 +158,16 @@ module modglobal real(real64):: tolerance = 1E-8! Convergence threshold . X X X X integer :: n_pre = 1 ! Number of pre and post relaxations . X X X X integer :: n_post =1 ! Number of pre and post relaxations . X X X X - integer :: precond = 1 ! Preconditioner ID . . 12 0189 0189 + integer :: precond_id = 1 ! Preconditioner ID . . 12 0189 0189 + integer :: maxiter_precond = 1 ! Number of iterations for precondition per iteration + integer :: hypre_logging = 1 ! HYPRE logging and print level - set higher value for more messages + type solver_type + !integer*8 solver,precond + type(c_ptr) solver, precond + integer solver_id, precond_id, maxiter, n_post, n_pre, maxiter_precond + real tolerance + end type + type(solver_type) :: psolver ! Global variables (modvar.f90) integer :: xyear = 0 !< * year, only for time units in netcdf @@ -197,9 +208,29 @@ module modglobal integer :: iexpnr = 0 !< * number of the experiment character(3) cexpnr + logical :: loutdirs = .false. !< if true, create output directories using myidy character(20) :: output_prefix = '' !< prefix for output files e.g. for an output directory + ! Variables for modopenboundary.f90 + logical :: lopenbc = .false., lsynturb = .false.,linithetero = .false. + type boundary_type + integer :: nx1,nx2,nx1patch,nx2patch,nx1u,nx2u,nx1v,nx2v,nx1w,nx2w + real(field_r), allocatable, dimension(:,:,:) :: u,v,w,thl,qt,e12, & + & u2,v2,w2,uv,uw,vw,thl2,qt2,wthl,wqt,ci,svturb + real(field_r), allocatable, dimension(:,:,:,:) :: sv + real(field_r), allocatable, dimension(:,:) :: radcorr,uphase,uphasesingle, & + radcorrsingle,uturb,vturb,wturb,thlturb,qtturb,e12turb!,randqt,randthl + real(field_r), allocatable, dimension(:,:,:,:) :: eigvec + character (len=:), allocatable :: name + end type + type(boundary_type), dimension(5) :: boundary + logical, dimension(5) :: lboundary = .false. + logical, dimension(5) :: lperiodic = (/.true., .true., .true., .true., .false./) + real :: dxint=-1.,dyint=-1.,dzint=-1.,tauh=60.,taum=0.,tau=60.,lambda,lambdas=-1.,lambdas_x=-1.,lambdas_y=-1.,lambdas_z=-1. + integer :: nmodes=100,ntboundary=1,pbc = 3,iturb=0 + real,dimension(:),allocatable :: tboundary + ! modphsgrd.f90 real :: dx !< grid spacing in x-direction diff --git a/src/modhypre.f90 b/src/modhypre.f90 index 366c6249..ba37b136 100644 --- a/src/modhypre.f90 +++ b/src/modhypre.f90 @@ -26,22 +26,22 @@ module modhypre use modmpi, only : myid, myidx, myidy, nprocx, nprocy, MPI_COMM_WORLD & , MPI_Wtime use modglobal, only : i1, j1, ih, jh, imax, jmax, kmax, solver_id, maxiter & - , n_pre, n_post, tolerance, precond, dzf, dzh, dx, dy & - , itot, jtot + , n_pre, n_post, tolerance, dzf, dzh, dx, dy & + , itot, jtot, solver_type, hypre_logging use modfields, only : rhobf, rhobh - implicit none #ifdef USE_HYPRE private -public :: inithypre, solve_hypre, exithypre, set_initial_guess +public :: inithypre_grid, inithypre_solver, solve_hypre, exithypre_grid, exithypre_solver, set_initial_guess, set_zero_guess save - integer(c_int) mpi_comm_hypre + integer(c_int) mpi_comm_hypre integer(c_int) ierr - type(c_ptr) grid, stencil, solver, precond_solver + + type(c_ptr) grid, stencil !, solver, precond_solver type(c_ptr) matrixA, vectorX, vectorB ! Solve Ax = b integer ilower(3), iupper(3), periodic(3) @@ -345,6 +345,14 @@ subroutine HYPRE_StructLGMRESSetup ( solver, matrixA, vectorB, vectorX, ierr) & type(c_ptr) :: solver, matrixA, vectorB, vectorX integer(c_int) ierr end subroutine + subroutine HYPRE_StructPCGSetRelChange ( solver, tol, ierr) & + bind(c, name="hypre_structpcgsetrelchange_") + use iso_c_binding + implicit none + type(c_ptr) :: solver + real(c_double) :: tol + integer(c_int) ierr + end subroutine HYPRE_StructPCGSetRelChange subroutine HYPRE_StructPCGSetTwoNorm ( solver, two_norm, ierr ) & bind(c, name="hypre_structpcgsettwonorm_") use iso_c_binding @@ -543,6 +551,16 @@ subroutine HYPRE_StructMatrixSetBoxValues ( matrix, ilower, iupper, & real(c_double) :: values(*) integer(c_int) num_stencil_indices, ierr end subroutine + subroutine HYPRE_StructMatrixGetBoxValues ( matrix, ilower, iupper, & + num_stencil_indices, stencil_indices, values, ierr ) & + bind(c, name="hypre_structmatrixgetboxvalues_") + use iso_c_binding + implicit none + type(c_ptr) :: matrix + integer(c_int) :: ilower(*), iupper(*), stencil_indices(*) + real(c_double) :: values(*) + integer(c_int) num_stencil_indices, ierr + end subroutine subroutine HYPRE_StructMatrixInitialize ( matrix, ierr ) & bind(c, name="hypre_structmatrixinitialize_") use iso_c_binding @@ -725,9 +743,10 @@ subroutine HYPRE_StructPCGDestroy ( obj , ierr ) & end interface contains - subroutine initprecond + subroutine initprecond(solver) ! Setup a preconditioner for the BiCGSTAB or GMRES solvers implicit none + type(solver_type), intent(inout) :: solver ! The precond_id flags mean : ! 0 - setup a smg preconditioner @@ -735,56 +754,61 @@ subroutine initprecond ! 7 - setup a jacobi preconditioner ! 8 - setup a ds preconditioner ! 9 - dont setup a preconditioner - if (precond == 0) then - call HYPRE_StructSMGCreate(mpi_comm_hypre, precond_solver, ierr) - call HYPRE_StructSMGSetMemoryUse(precond_solver, zero, ierr) - call HYPRE_StructSMGSetMaxIter(precond_solver, maxiter, ierr) - call HYPRE_StructSMGSetNumPreRelax(precond_solver, n_pre, ierr) - call HYPRE_StructSMGSetNumPostRelax(precond_solver, n_post, ierr) - call HYPRE_StructSMGSetTol(precond_solver, 0.0d0, ierr) - else if (precond == 1) then - call HYPRE_StructPFMGCreate(mpi_comm_hypre, precond_solver, ierr) - call HYPRE_StructPFMGSetMaxIter(precond_solver, maxiter, ierr) - ! weighted Jacobi = 1; red-black GS = 2 - call HYPRE_StructPFMGSetRelaxType(precond_solver, 2, ierr) - call HYPRE_StructPFMGSetNumPreRelax(precond_solver, n_pre, ierr) - call HYPRE_StructPFMGSetNumPostRelax(precond_solver, n_post, ierr) - call HYPRE_StructPFMGSetTol(precond_solver, 0.0d0, ierr) - ! call HYPRE_StructPFMGSetDxyz(precond_solver, dxyz, ierr) - ! call HYPRE_StructPFMGSetSkipRelax(precond, 1, ierr) - ! call HYPRE_StructPFMGSetRAPType(precond, 1, ierr) - else if (precond == 7 .and. solver_id == 5) then - call HYPRE_StructJacobiCreate(mpi_comm_hypre, precond_solver, ierr) - call HYPRE_StructJacobiSetMaxIter(precond_solver, maxiter, ierr) - else if (precond == 8) then - precond_solver = c_null_ptr - else if (precond == 9) then - precond_solver = c_null_ptr + if (solver%precond_id == 0) then + call HYPRE_StructSMGCreate(mpi_comm_hypre, solver%precond, ierr) + call HYPRE_StructSMGSetMemoryUse(solver%precond, zero, ierr) + call HYPRE_StructSMGSetMaxIter(solver%precond, solver%maxiter_precond, ierr) + call HYPRE_StructSMGSetNumPreRelax(solver%precond, solver%n_pre, ierr) + call HYPRE_StructSMGSetNumPostRelax(solver%precond, solver%n_post, ierr) + call HYPRE_StructSMGSetTol(solver%precond, 0.0d0, ierr) + else if (solver%precond_id == 1) then + call HYPRE_StructPFMGCreate(mpi_comm_hypre, solver%precond, ierr) + call HYPRE_StructPFMGSetMaxIter(solver%precond, solver%maxiter_precond, ierr) + ! weighted Jacobi = 1; red-black GS = 2 2 seems faster + call HYPRE_StructPFMGSetRelaxType(solver%precond, 2, ierr) + call HYPRE_StructPFMGSetNumPreRelax(solver%precond, solver%n_pre, ierr) + call HYPRE_StructPFMGSetNumPostRelax(solver%precond, solver%n_post, ierr) + call HYPRE_StructPFMGSetTol(solver%precond, 0.0d0, ierr) + ! call HYPRE_StructPFMGSetDxyz(precond, dxyz, ierr) + ! call HYPRE_StructPFMGSetSkipRelax(precond_id, 1, ierr) + ! call HYPRE_StructPFMGSetRAPType(precond_id, 1, ierr) + else if (solver%precond_id == 7 .and. solver%solver_id == 5) then + call HYPRE_StructJacobiCreate(mpi_comm_hypre, solver%precond, ierr) + call HYPRE_StructJacobiSetMaxIter(solver%precond, solver%maxiter_precond, ierr) + else if (solver%precond_id == 8) then + solver%precond = c_null_ptr + else if (solver%precond_id == 9) then + solver%precond = c_null_ptr else - write (*,*) 'Invalid preconditioner in inithypre', precond + write (*,*) 'Invalid preconditioner in inithypre', solver%precond_id write (*,*) 'Possbile values are (0) SMG (1) PFMG (8) DS (9) None' call exit(-1) endif end subroutine - subroutine exitprecond + subroutine exitprecond(solver) implicit none + type(solver_type), intent(inout) :: solver - if (precond == 0) then - call HYPRE_StructSMGDestroy(precond_solver, ierr) - else if (precond == 1) then - call HYPRE_StructPFMGDestroy(precond_solver, ierr) - else if (precond == 7) then - call HYPRE_StructJacobiDestroy(precond_solver, ierr) + if (solver%precond_id == 0) then + call HYPRE_StructSMGDestroy(solver%precond, ierr) + else if (solver%precond_id == 1) then + call HYPRE_StructPFMGDestroy(solver%precond, ierr) + else if (solver%precond_id == 7) then + call HYPRE_StructJacobiDestroy(solver%precond, ierr) endif end subroutine - subroutine inithypre + subroutine inithypre_grid + use modmpi, only : myid, myidx, myidy, nprocx, nprocy + use modglobal, only : imax, jmax, kmax, dzf, dzh, dx, dy, itot, jtot, lopenbc,lperiodic,lboundary + + use modfields, only : rhobf, rhobh implicit none ! NOTE: we assume kmax is larger than 7, ! when we make the stencil entries - real(real64), allocatable :: values(:) + real(real64), allocatable :: values(:),temp(:) real cx, cy, cz_down, cz_up, cc ! integer num_ghost(6) @@ -795,7 +819,7 @@ subroutine inithypre ! data num_ghost / 3, 3, 3, 3, 0, 0 / - allocate(values(imax*jmax*kmax)) + allocate(values(imax*jmax*7),temp(max(imax,jmax))) ! Have hypre reuse the comm world mpi_comm_hypre = MPI_COMM_WORLD%MPI_VAL @@ -820,6 +844,9 @@ subroutine inithypre periodic(1) = itot periodic(2) = jtot periodic(3) = 0 + ! Remove periodicity if openboundaries are used + if(lopenbc.and..not.lperiodic(1)) periodic(1)=0 + if(lopenbc.and..not.lperiodic(3)) periodic(2)=0 call HYPRE_StructGridSetPeriodic(grid, periodic, ierr) ! This is a collective call finalizing the grid assembly @@ -931,8 +958,85 @@ subroutine inithypre values(i+6) = cc ! center enddo + ! Modify the matrix at a single point to fix the over-all p + ! makes the matrix non-singular which may improve convergence + ! matrix is made symmetric by not coupling to the special element + ! if that element is 0, the couplings of other elements can be set to 0 + ! note: assumes kmax > 10, just to put the special point well inside the domain + if (k == kmax-10 .and. myidx == 0 .and. myidy == 0) then + i = imax/2 + j = jmax/2 + values( (i+j*imax)*7 + 1) = 0 + values( (i+j*imax)*7 + 2) = 0 + values( (i+j*imax)*7 + 3) = 0 + values( (i+j*imax)*7 + 4) = 0 + values( (i+j*imax)*7 + 5) = 0 + values( (i+j*imax)*7 + 6) = 0 + values( (i+j*imax)*7 + 7) = cc ! value here doesn't matter in theory, probably better conditioning if + ! similar to other diagonal values + + !! from here on, it's the neighbors of the special element + values( (i+1+j*imax)*7 + 1) = 0 ! west link of east neighbor + values( (i-1+j*imax)*7 + 2) = 0 ! east link of west neighbor + values( (i+(j+1)*imax)*7 + 3) = 0 ! south link of north neighbor + values( (i+(j-1)*imax)*7 + 4) = 0 ! north link of south neighbor + end if + if (k == kmax-9 .and. myidx == 0 .and. myidy == 0) then + i = imax/2 + j = jmax/2 + values( (i+j*imax)*7 + 5) = 0 ! down link of above neighbor + end if + if (k == kmax-11 .and. myidx == 0 .and. myidy == 0) then + i = imax/2 + j = jmax/2 + values( (i+j*imax)*7 + 6) = 0 ! up link of below neighbor + end if + + call HYPRE_StructMatrixSetBoxValues(matrixA, & ilower, iupper, 7, stencil_indices, values, ierr) + if(lopenbc) then ! Set potential neuman lateral boundary conditions + if(lboundary(1).and. .not. lperiodic(1)) then + call HYPRE_StructMatrixGetBoxValues(matrixA, & + (/0,myidy*jmax,k-1/),(/0,(myidy+1)*jmax-1,k-1/),1,(/6/),temp,ierr) + do i = 1, jmax*2,2 + values(i+0) = 0. + values(i+1) = temp((i+1)/2)+cx + end do + call HYPRE_StructMatrixSetBoxValues(matrixA, & + (/0,myidy*jmax,k-1/),(/0,(myidy+1)*jmax-1,k-1/),2,(/0,6/),values,ierr) + endif + if(lboundary(2).and. .not. lperiodic(2)) then + call HYPRE_StructMatrixGetBoxValues(matrixA, & + (/itot-1,myidy*jmax,k-1/),(/itot-1,(myidy+1)*jmax-1,k-1/),1,(/6/),temp,ierr) + do i = 1, jmax*2,2 + values(i+0) = 0. + values(i+1) = temp((i+1)/2)+cx + end do + call HYPRE_StructMatrixSetBoxValues(matrixA, & + (/itot-1,myidy*jmax,k-1/),(/itot-1,(myidy+1)*jmax-1,k-1/),2,(/1,6/),values,ierr) + endif + if(lboundary(3).and. .not. lperiodic(3)) then + call HYPRE_StructMatrixGetBoxValues(matrixA, & + (/myidx*imax,0,k-1/),(/(myidx+1)*imax-1,0,k-1/),1,(/6/),temp,ierr) + do i = 1,imax*2,2 + values(i+0) = 0. + values(i+1) = temp((i+1)/2)+cy + end do + call HYPRE_StructMatrixSetBoxValues(matrixA, & + (/myidx*imax,0,k-1/),(/(myidx+1)*imax-1,0,k-1/),2,(/2,6/),values,ierr) + endif + if(lboundary(4).and. .not. lperiodic(4)) then + call HYPRE_StructMatrixGetBoxValues(matrixA, & + (/myidx*imax,jtot-1,k-1/),(/(myidx+1)*imax-1,jtot-1,k-1/),1,(/6/),temp,ierr) + do i = 1,imax*2,2 + values(i+0) = 0. + values(i+1) = temp((i+1)/2)+cy + end do + call HYPRE_StructMatrixSetBoxValues(matrixA, & + (/myidx*imax,jtot-1,k-1/),(/(myidx+1)*imax-1,jtot-1,k-1/),2,(/3,6/),values,ierr) + endif + endif enddo call HYPRE_StructMatrixAssemble(matrixA, ierr) @@ -956,7 +1060,7 @@ subroutine inithypre ! initialize some values as starting point for the iterative solver do i=1,imax*jmax - values(i) = 1e-5 + values(i) = 0 !1e-5 enddo do k=1,kmax ilower(3) = k - 1 @@ -965,120 +1069,133 @@ subroutine inithypre enddo call HYPRE_StructVectorAssemble(vectorX, ierr) + deallocate(values,temp) + + end subroutine inithypre_grid + + subroutine inithypre_solver(solver,solver_id,maxiter,tolerance,precond_id,n_pre,n_post,maxiter_precond) + use modmpi, only : myid + implicit none + type(solver_type), intent(inout) :: solver + integer, intent(in) :: solver_id, maxiter, precond_id, n_pre, n_post, maxiter_precond + real, intent(in) :: tolerance !----------------------------------------------------------------------- ! 5. Choose a solver and initialize it !----------------------------------------------------------------------- - - if (solver_id == 1) then + solver%solver_id = solver_id + solver%maxiter = maxiter + solver%tolerance = tolerance + solver%precond_id = precond_id + solver%n_pre = n_pre + solver%n_post = n_post + solver%maxiter_precond = maxiter_precond + if (solver%solver_id == 1) then ! Solve the system using SMG if (myid == 0) then - write (*,*) 'Selected solver 1 (SMG) with parameters:', maxiter, tolerance, n_pre, n_post + write (*,*) 'Selected solver 1 (SMG) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post endif - call HYPRE_StructSMGCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructSMGSetMemoryUse(solver, zero, ierr) - call HYPRE_StructSMGSetMaxIter(solver, maxiter, ierr) - call HYPRE_StructSMGSetTol(solver, tolerance, ierr) - call HYPRE_StructSMGSetRelChange(solver, zero, ierr) - call HYPRE_StructSMGSetNumPreRelax(solver, n_pre, ierr) - call HYPRE_StructSMGSetNumPostRelax(solver, n_post, ierr) - call HYPRE_StructSMGSetLogging(solver, 1, ierr) - call HYPRE_StructSMGSetup(solver, matrixA, vectorB, vectorX, ierr) - else if (solver_id == 2) then + call HYPRE_StructSMGCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructSMGSetMemoryUse(solver%solver, zero, ierr) + call HYPRE_StructSMGSetMaxIter(solver%solver, solver%maxiter, ierr) + call HYPRE_StructSMGSetTol(solver%solver, solver%tolerance, ierr) + call HYPRE_StructSMGSetRelChange(solver%solver, zero, ierr) + call HYPRE_StructSMGSetNumPreRelax(solver%solver, solver%n_pre, ierr) + call HYPRE_StructSMGSetNumPostRelax(solver%solver, solver%n_post, ierr) + call HYPRE_StructSMGSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructSMGSetup(solver%solver, matrixA, vectorB, vectorX, ierr) + else if (solver%solver_id == 2) then ! Solve the system using PFMG if (myid == 0) then - write (*,*) 'Selected solver 2 (PFMG) with parameters:', maxiter, tolerance, n_pre, n_post, precond + write (*,*) 'Selected solver 2 (PFMG) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post, solver%precond_id endif - call HYPRE_StructPFMGCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructPFMGSetMaxIter(solver, maxiter, ierr) - call HYPRE_StructPFMGSetTol(solver, tolerance, ierr) - call HYPRE_StructPFMGSetRelChange(solver, zero, ierr) + call HYPRE_StructPFMGCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructPFMGSetMaxIter(solver%solver, solver%maxiter, ierr) + call HYPRE_StructPFMGSetTol(solver%solver, solver%tolerance, ierr) + call HYPRE_StructPFMGSetRelChange(solver%solver, zero, ierr) ! weighted Jacobi = 1; red-black GS = 2 - if (precond == 1 .or. precond == 2) then - call HYPRE_StructPFMGSetRelaxType(solver, precond, ierr) + if (solver%precond_id == 1 .or. solver%precond_id == 2) then + call HYPRE_StructPFMGSetRelaxType(solver%solver, solver%precond_id, ierr) else - write (*,*) 'Invalid preconditioner in inithypre', precond + write (*,*) 'Invalid preconditioner in inithypre', solver%precond_id write (*,*) 'Possbile values are (1) weighted jacobi (2) red-black GS' call exit(-1) endif - call HYPRE_StructPFMGSetNumPreRelax(solver, n_pre, ierr) - call HYPRE_StructPFMGSetNumPostRelax(solver, n_post, ierr) + call HYPRE_StructPFMGSetNumPreRelax(solver%solver, solver%n_pre, ierr) + call HYPRE_StructPFMGSetNumPostRelax(solver%solver, solver%n_post, ierr) ! call HYPRE_StructPFMGSetSkipRelax(solver, one) ! call HYPRE_StructPFMGSetDxyz(solver, dxyz, ierr) - call HYPRE_StructPFMGSetLogging(solver, 2, ierr) - call HYPRE_StructPFMGSetup(solver, matrixA, vectorB, vectorX, ierr) + call HYPRE_StructPFMGSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructPFMGSetup(solver%solver, matrixA, vectorB, vectorX, ierr) - else if (solver_id == 3) then + else if (solver%solver_id == 3) then ! Solve the system using BiCGSTAB if (myid == 0) then - write (*,*) 'Selected solver 3 (BiCGSTAB) with parameters:', maxiter, tolerance, n_pre, n_post, precond + write (*,*) 'Selected solver 3 (BiCGSTAB) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post, solver%precond_id endif - call HYPRE_StructBiCGSTABCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructBiCGSTABSetMaxIter(solver, maxiter, ierr) - call HYPRE_StructBiCGSTABSetTol(solver, tolerance, ierr) - call initprecond - call HYPRE_StructBiCGSTABSetPrecond(solver, precond, precond_solver, ierr) + call HYPRE_StructBiCGSTABCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructBiCGSTABSetMaxIter(solver%solver, solver%maxiter, ierr) + call HYPRE_StructBiCGSTABSetTol(solver%solver, solver%tolerance, ierr) + call initprecond(solver) + call HYPRE_StructBiCGSTABSetPrecond(solver%solver, solver%precond_id, solver%precond, ierr) - call HYPRE_StructBiCGSTABSetLogging(solver, 2, ierr) - call HYPRE_StructBiCGSTABSetup(solver, matrixA, vectorB, vectorX, ierr) + call HYPRE_StructBiCGSTABSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructBiCGSTABSetup(solver%solver, matrixA, vectorB, vectorX, ierr) - else if (solver_id == 4) then + else if (solver%solver_id == 4) then ! Solve the system using GMRES if (myid == 0) then - write (*,*) 'Selected solver 4 (GMRES) with parameters:', maxiter, tolerance, n_pre, n_post, precond + write (*,*) 'Selected solver 4 (GMRES) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post, solver%precond_id endif - call HYPRE_StructGMRESCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructGMRESSetTol(solver, tolerance, ierr) - call HYPRE_StructGMRESSetMaxIter(solver, maxiter, ierr) - call initprecond - call HYPRE_StructGMRESSetPrecond(solver, precond, precond_solver, ierr) + call HYPRE_StructGMRESCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructGMRESSetTol(solver%solver, solver%tolerance, ierr) + call HYPRE_StructGMRESSetMaxIter(solver%solver, solver%maxiter, ierr) + call initprecond(solver) + call HYPRE_StructGMRESSetPrecond(solver%solver, solver%precond_id, solver%precond, ierr) - call HYPRE_StructGMRESSetLogging(solver, 2, ierr) - call HYPRE_StructGMRESSetup(solver, matrixA, vectorB, vectorX, ierr) + call HYPRE_StructGMRESSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructGMRESSetup(solver%solver, matrixA, vectorB, vectorX, ierr) - else if (solver_id == 5) then + else if (solver%solver_id == 5) then ! Solve the system using PCG if (myid == 0) then - write (*,*) 'Selected solver 5 (PCG) with parameters:', maxiter, tolerance, n_pre, n_post, precond + write (*,*) 'Selected solver 5 (PCG) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post, solver%precond_id endif - call HYPRE_StructPCGCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructPCGSetTol(solver, tolerance, ierr) - call HYPRE_StructPCGSetTwoNorm(solver, 1, ierr) - call HYPRE_StructPCGSetMaxIter(solver, maxiter, ierr) - call initprecond - call HYPRE_StructPCGSetPrecond(solver, precond, precond_solver, ierr) - - call HYPRE_StructPCGSetLogging(solver, 2, ierr) - call HYPRE_StructPCGSetPrintLevel(solver, 0, ierr) - call HYPRE_StructPCGSetup(solver, matrixA, vectorB, vectorX, ierr) - - else if (solver_id == 6) then + call HYPRE_StructPCGCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructPCGSetRelChange(solver%solver, solver%tolerance, ierr) + call HYPRE_StructPCGSetTwoNorm(solver%solver, 1, ierr) + call HYPRE_StructPCGSetMaxIter(solver%solver, solver%maxiter, ierr) + call initprecond(solver) + call HYPRE_StructPCGSetPrecond(solver%solver, solver%precond_id, solver%precond, ierr) + + call HYPRE_StructPCGSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructPCGSetPrintLevel(solver%solver, hypre_logging, ierr) + call HYPRE_StructPCGSetup(solver%solver, matrixA, vectorB, vectorX, ierr) + + else if (solver%solver_id == 6) then ! Solve the system using LGMRES if (myid == 0) then - write (*,*) 'Selected solver 6 (LGMRES) with parameters:', maxiter, tolerance, n_pre, n_post, precond + write (*,*) 'Selected solver 6 (LGMRES) with parameters:', solver%maxiter, solver%tolerance, solver%n_pre, solver%n_post, solver%precond_id endif - call HYPRE_StructLGMRESCreate(mpi_comm_hypre, solver, ierr) - call HYPRE_StructLGMRESSetTol(solver, tolerance, ierr) - call HYPRE_StructLGMRESSetMaxIter(solver, maxiter, ierr) - call initprecond - call HYPRE_StructLGMRESSetPrecond(solver, precond, precond_solver, ierr) + call HYPRE_StructLGMRESCreate(mpi_comm_hypre, solver%solver, ierr) + call HYPRE_StructLGMRESSetTol(solver%solver, solver%tolerance, ierr) + call HYPRE_StructLGMRESSetMaxIter(solver%solver, solver%maxiter, ierr) + call initprecond(solver) + call HYPRE_StructLGMRESSetPrecond(solver%solver, solver%precond_id, solver%precond, ierr) - call HYPRE_StructLGMRESSetLogging(solver, 2, ierr) - call HYPRE_StructLGMRESSetPrintLevel(solver, 0, ierr) - call HYPRE_StructLGMRESSetup(solver, matrixA, vectorB, vectorX, ierr) + call HYPRE_StructLGMRESSetLogging(solver%solver, hypre_logging, ierr) + call HYPRE_StructLGMRESSetPrintLevel(solver%solver, hypre_logging, ierr) + call HYPRE_StructLGMRESSetup(solver%solver, matrixA, vectorB, vectorX, ierr) else if (myid == 0) then - write (*,*) 'Invalid solver in inithypre', solver_id + write (*,*) 'Invalid solver in inithypre', solver%solver_id endif call exit(-1) endif - - deallocate(values) - - end subroutine + end subroutine inithypre_solver subroutine set_initial_guess(p) @@ -1101,13 +1218,35 @@ subroutine set_initial_guess(p) call HYPRE_StructVectorAssemble(vectorX, ierr) end subroutine - subroutine solve_hypre(p, converged) + subroutine set_zero_guess() + use modglobal, only : i1, j1, ih, jh, imax, jmax, kmax + implicit none + real values(imax,jmax) + + integer i,j,k + do k=1,kmax + do j=1,jmax + do i=1,imax + values(i,j) = 0. + enddo + enddo + ilower(3) = k - 1 + iupper(3) = k - 1 + call HYPRE_StructVectorSetBoxValues(vectorX, ilower, iupper, values, ierr) + enddo + call HYPRE_StructVectorAssemble(vectorX, ierr) + end subroutine + + subroutine solve_hypre(solver, p, converged) + use modmpi, only : myid, myidx, myidy + use modglobal, only : i1, j1, ih, jh, imax, jmax, kmax, rk3step implicit none real(pois_r), intent(inout) :: p(2-ih:i1+ih,2-jh:j1+jh,kmax) logical, intent(out) :: converged - real(real64) values(imax,jmax) + type(solver_type), intent(inout) :: solver + real values(imax,jmax) real(real64) final_res_norm integer i,j,k, num_iterations, stat @@ -1123,6 +1262,14 @@ subroutine solve_hypre(p, converged) enddo enddo + ! special point anchoring P + ! note should match the special point in the matrix + if (k == kmax-10 .and. myidx == 0 .and. myidy == 0) then + i = imax/2 + j = jmax/2 + values(i+1,j+1) = 0 ! values indexed from 1 here + end if + ilower(3) = k - 1 iupper(3) = k - 1 call HYPRE_StructVectorSetBoxValues(vectorB, ilower, iupper, values, ierr) @@ -1131,49 +1278,57 @@ subroutine solve_hypre(p, converged) ! use current values (ie. the solution to the previous call) as starting point + ! ...except in the beginning of a full RK step + !if(rk3step == 1) then + ! if (myid == 0) then + ! write(*,*) "Poisson solver: initial guess 0" + ! end if + ! call set_zero_guess() + !end if + !----------------------------------------------------------------------- ! 2. Call a solver !----------------------------------------------------------------------- - - if (solver_id == 1) then + stat = 0 + if (solver%solver_id == 1) then ! Solve the system using SMG - call HYPRE_StructSMGSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructSMGGetNumIterations(solver, num_iterations, ierr) - call HYPRE_StructSMGGetFinalRelative(solver, final_res_norm, ierr) + call HYPRE_StructSMGSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructSMGGetNumIterations(solver%solver, num_iterations, ierr) + call HYPRE_StructSMGGetFinalRelative(solver%solver, final_res_norm, ierr) - else if (solver_id == 2) then + else if (solver%solver_id == 2) then ! Solve the system using PFMG - call HYPRE_StructPFMGSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructPFMGGetNumIteration(solver, num_iterations, ierr) - call HYPRE_StructPFMGGetFinalRelativ(solver, final_res_norm, ierr) + call HYPRE_StructPFMGSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructPFMGGetNumIteration(solver%solver, num_iterations, ierr) + call HYPRE_StructPFMGGetFinalRelativ(solver%solver, final_res_norm, ierr) - else if (solver_id == 3) then + else if (solver%solver_id == 3) then ! Solve the system using BiCGSTAB - call HYPRE_StructBiCGSTABSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructBiCGSTABGetNumItera(solver, num_iterations, ierr) - call HYPRE_StructBiCGSTABGetFinalRel(solver, final_res_norm, ierr) + call HYPRE_StructBiCGSTABSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructBiCGSTABGetNumItera(solver%solver, num_iterations, ierr) + call HYPRE_StructBiCGSTABGetFinalRel(solver%solver, final_res_norm, ierr) - else if (solver_id == 4) then + else if (solver%solver_id == 4) then ! Solve the system using GMRES - call HYPRE_StructGMRESSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructGMRESGetNumIteratio(solver, num_iterations, ierr) - call HYPRE_StructGMRESGetFinalRelati(solver, final_res_norm, ierr) + call HYPRE_StructGMRESSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructGMRESGetNumIteratio(solver%solver, num_iterations, ierr) + call HYPRE_StructGMRESGetFinalRelati(solver%solver, final_res_norm, ierr) - else if (solver_id == 5) then + else if (solver%solver_id == 5) then ! Solve the system using PCG - call HYPRE_StructPCGSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructPCGGetNumIterations(solver, num_iterations, ierr) - call HYPRE_StructPCGGetFinalRelative(solver, final_res_norm, ierr) + call HYPRE_StructPCGSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructPCGGetNumIterations(solver%solver, num_iterations, ierr) + call HYPRE_StructPCGGetFinalRelative(solver%solver, final_res_norm, ierr) - else if (solver_id == 6) then + else if (solver%solver_id == 6) then ! Solve the system using LGMRES - call HYPRE_StructLGMRESSolve(solver, matrixA, vectorB, vectorX, stat) - call HYPRE_StructLGMRESGetNumIter(solver, num_iterations, ierr) - call HYPRE_StructLGMRESGetFinalRel(solver, final_res_norm, ierr) + call HYPRE_StructLGMRESSolve(solver%solver, matrixA, vectorB, vectorX, stat) + call HYPRE_StructLGMRESGetNumIter(solver%solver, num_iterations, ierr) + call HYPRE_StructLGMRESGetFinalRel(solver%solver, final_res_norm, ierr) else - write (*,*) 'Invalid solver in solve_hypre', solver_id + write (*,*) 'Invalid solver in solve_hypre', solver%solver_id call exit(-1) endif @@ -1195,7 +1350,7 @@ subroutine solve_hypre(p, converged) call exit(-1) endif - if (num_iterations >= maxiter) then + if (num_iterations >= solver%maxiter) then converged = .false. return else @@ -1219,35 +1374,38 @@ subroutine solve_hypre(p, converged) enddo end subroutine - subroutine exithypre - + subroutine exithypre_solver(solver) implicit none - - if (solver_id == 1) then - call HYPRE_StructSMGDestroy(solver, ierr) - else if (solver_id == 2) then - call HYPRE_StructPFMGDestroy(solver, ierr) - else if (solver_id == 3) then - if (precond == 0) then - call HYPRE_StructSMGDestroy(precond_solver, ierr) - else if (precond == 1) then - call HYPRE_StructPFMGDestroy(precond_solver, ierr) + type(solver_type), intent(inout) :: solver + if (solver%solver_id == 1) then + call HYPRE_StructSMGDestroy(solver%solver, ierr) + else if (solver%solver_id == 2) then + call HYPRE_StructPFMGDestroy(solver%solver, ierr) + else if (solver%solver_id == 3) then + if (solver%precond_id == 0) then + call HYPRE_StructSMGDestroy(solver%precond, ierr) + else if (solver%precond_id == 1) then + call HYPRE_StructPFMGDestroy(solver%precond, ierr) endif - call HYPRE_StructBiCGSTABDestroy(solver, ierr) - else if (solver_id == 4) then - call exitprecond - call HYPRE_StructGMRESDestroy(solver, ierr) - else if (solver_id == 5) then - call exitprecond - call HYPRE_StructPCGDestroy(solver, ierr) - else if (solver_id == 6) then - call exitprecond - call HYPRE_StructLGMRESDestroy(solver, ierr) + call HYPRE_StructBiCGSTABDestroy(solver%solver, ierr) + else if (solver%solver_id == 4) then + call exitprecond(solver) + call HYPRE_StructGMRESDestroy(solver%solver, ierr) + else if (solver%solver_id == 5) then + call exitprecond(solver) + call HYPRE_StructPCGDestroy(solver%solver, ierr) + else if (solver%solver_id == 6) then + call exitprecond(solver) + call HYPRE_StructLGMRESDestroy(solver%solver, ierr) else - write (*,*) 'Invalid solver in exit_hypre', solver_id + write (*,*) 'Invalid solver in exit_hypre', solver%solver_id call exit(-1) endif + end subroutine exithypre_solver + + subroutine exithypre_grid + implicit none call HYPRE_StructGridDestroy(grid, ierr) call HYPRE_StructStencilDestroy(stencil, ierr) call HYPRE_StructMatrixDestroy(matrixA, ierr) @@ -1258,35 +1416,53 @@ subroutine exithypre #else private -public :: inithypre, solve_hypre, exithypre, set_initial_guess - +public :: inithypre_grid, inithypre_solver, solve_hypre, exithypre_grid, exithypre_solver, set_initial_guess, set_zero_guess contains - subroutine inithypre + subroutine inithypre_grid implicit none call error_and_exit() - end subroutine + end subroutine inithypre_grid - subroutine exithypre + subroutine inithypre_solver(solver,solver_id,maxiter,tolerance,precond_id,n_pre,n_post,maxiter_precond) implicit none + type(solver_type), intent(inout) :: solver + integer, intent(in) :: solver_id, maxiter, precond_id, n_pre, n_post, maxiter_precond + real, intent(in) :: tolerance call error_and_exit() - end subroutine + end subroutine inithypre_solver - subroutine set_initial_guess(p) + subroutine exithypre_grid + implicit none + + call error_and_exit() + end subroutine exithypre_grid + + subroutine exithypre_solver(solver) implicit none + type(solver_type), intent(inout) :: solver + call error_and_exit() + end subroutine exithypre_solver + subroutine set_initial_guess(p) + implicit none real(pois_r), intent(inout) :: p(2-ih:i1+ih,2-jh:j1+jh,kmax) call error_and_exit() end subroutine - subroutine solve_hypre(p, converged) - implicit none + subroutine set_zero_guess() + call error_and_exit() + end subroutine set_zero_guess + + subroutine solve_hypre(solver, p, converged) + implicit none real(pois_r), intent(inout) :: p(2-ih:i1+ih,2-jh:j1+jh,kmax) logical, intent(out) :: converged + type(solver_type), intent(inout) :: solver call error_and_exit() converged = .false. ! suppress warnings about intent(out) variable diff --git a/src/modlsm.f90 b/src/modlsm.f90 index a788eef2..f4e56d89 100644 --- a/src/modlsm.f90 +++ b/src/modlsm.f90 @@ -831,10 +831,11 @@ end subroutine calc_water_bcs ! the diffusion scheme, thermodynamics, ... ! subroutine calc_bulk_bcs - use modglobal, only : i1, j1, cp, rlv, fkar, zf, cu, cv, grav, rv, rd + use modglobal, only : i1, j1, i2, j2, cp, rlv, fkar, zf, cu, cv, grav, rv, rd, lopenbc,lboundary,lperiodic use modfields, only : rhof, thl0, qt0, u0, v0, thvh use modsurface, only : phim, phih use modmpi, only : excjs + use modopenboundary, only : openboundary_excjs use modsurfdata, only : & H, LE, G0, tskin, qskin, thlflux, qtflux, dthldz, dqtdz, & dudz, dvdz, ustar, obl, cliq, ra, rsveg, rssoil @@ -915,9 +916,15 @@ subroutine calc_bulk_bcs dvdz(i,j) = ustar(i,j) / (fkar * zf(1)) * phim(zf(1)/obl(i,j)) * (vcv/du_tot(i,j)) ! Cyclic BCs where needed. - !call excjs(ustar,2,i1,2,j1,1,1,1,1) - ustar_3D(1:size(ustar,1),1:size(ustar,2),1:1) => ustar - call excjs(ustar_3D,2,i1,2,j1,1,1,1,1) + ustar_3D(1:i2,1:j2,1:1) => ustar + + if(lopenbc) then ! Only use periodicity for non-domain boundaries when openboundaries are used + call openboundary_excjs(ustar_3D, 2,i1,2,j1,1,1,1,1, & + & (.not.lboundary(1:4)).or.lperiodic(1:4)) + else + !call excjs(ustar,2,i1,2,j1,1,1,1,1) + call excjs(ustar_3D,2,i1,2,j1,1,1,1,1) + endif ! Just for diagnostics (modlsmcrosssection) cliq(i,j) = tile_ws%frac(i,j) / land_frac(i,j) diff --git a/src/modmpi.f90 b/src/modmpi.f90 index 504ba9ac..2760fe6c 100644 --- a/src/modmpi.f90 +++ b/src/modmpi.f90 @@ -49,6 +49,7 @@ module modmpi integer :: nprocx = 1 integer :: nprocy = 0 integer :: mpierr + logical :: periods(2) = .true. real :: CPU_program !end time real :: CPU_program0 !start time @@ -141,7 +142,13 @@ module modmpi procedure :: excjs_real32 procedure :: excjs_real64 procedure :: excjs_logical - end interface + end interface excjs + + interface openboundary_excjs + procedure :: openboundary_excjs_real32 + procedure :: openboundary_excjs_real64 + end interface openboundary_excjs + interface slabsum procedure :: slabsum_real32 procedure :: slabsum_real64 @@ -444,6 +451,234 @@ subroutine excjs_real32(a,sx,ex,sy,ey,sz,ez,ih,jh) endif end subroutine excjs_real32 + subroutine openboundary_excjs_real32(a,sx,ex,sy,ey,sz,ez,ih,jh,switch) + implicit none + integer sx, ex, sy, ey, sz, ez, ih, jh + real(real32) a(sx-ih:ex+ih, sy-jh:ey+jh, sz:ez) + logical, dimension(4), intent(in) :: switch ! flags for which boundaries need exchange + type(MPI_STATUS) :: status + integer :: xl, yl, zl + type(MPI_REQUEST) :: reqn, reqs, reqe, reqw + type(MPI_REQUEST) :: reqrn, reqrs, reqre, reqrw + integer nssize, ewsize + real(real32),allocatable, dimension(:) :: sendn,recvn & + , sends,recvs & + , sende,recve & + , sendw,recvw + +! Calulate buffer lengths + xl = size(a,1) + yl = size(a,2) + zl = size(a,3) + +! Calculate buffer size + nssize = xl*jh*zl + ewsize = ih*yl*zl + + + if(nprocy .gt. 1)then + + ! Allocate send / receive buffers + allocate(sendn(nssize),sends(nssize),recvn(nssize),recvs(nssize)) + + if(switch(4)) sendn = reshape(a(:,ey-jh+1:ey,:),(/nssize/)) + if(switch(3)) sends = reshape(a(:,sy:sy+jh-1,:),(/nssize/)) + + ! Send north/south + if(switch(4)) call D_MPI_ISEND(sendn, nssize, nbrnorth, 4, comm3d, reqn, mpierr) + if(switch(3)) call D_MPI_ISEND(sends, nssize, nbrsouth, 5, comm3d, reqs, mpierr) + + ! Receive south/north + if(switch(3)) call D_MPI_IRECV(recvs, nssize, nbrsouth, 4, comm3d, reqrs, mpierr) + if(switch(4)) call D_MPI_IRECV(recvn, nssize, nbrnorth, 5, comm3d, reqrn, mpierr) + + ! Wait until data is received + if(switch(3)) call MPI_WAIT(reqrs, status, mpierr) + if(switch(4)) call MPI_WAIT(reqrn, status, mpierr) + + ! Write back buffers + if(switch(3)) a(:,sy-jh:sy-1,:) = reshape(recvs,(/xl,jh,zl/)) + if(switch(4)) a(:,ey+1:ey+jh,:) = reshape(recvn,(/xl,jh,zl/)) + + else + + ! Single processor, make sure the field is periodic + if(switch(3)) a(:,sy-jh:sy-1,:) = a(:,ey-jh+1:ey,:) + if(switch(3)) a(:,ey+1:ey+jh,:) = a(:,sy:sy+jh-1,:) + + endif + + if(nprocx .gt. 1)then + + ! Allocate send / receive buffers + allocate(sende(ewsize),sendw(ewsize),recve(ewsize),recvw(ewsize)) + + if(switch(2)) sende = reshape(a(ex-ih+1:ex,:,:),(/ewsize/)) + if(switch(1)) sendw = reshape(a(sx:sx+ih-1,:,:),(/ewsize/)) + + ! Send east/west + if(switch(2)) call D_MPI_ISEND(sende, ewsize, nbreast, 6, comm3d, reqe, mpierr) + if(switch(1)) call D_MPI_ISEND(sendw, ewsize, nbrwest, 7, comm3d, reqw, mpierr) + + ! Receive west/east + if(switch(1)) call D_MPI_IRECV(recvw, ewsize, nbrwest, 6, comm3d, reqrw, mpierr) + if(switch(2)) call D_MPI_IRECV(recve, ewsize, nbreast, 7, comm3d, reqre, mpierr) + + ! Wait until data is received + if(switch(1)) call MPI_WAIT(reqrw, status, mpierr) + if(switch(2)) call MPI_WAIT(reqre, status, mpierr) + + ! Write back buffers + if(switch(1)) a(sx-ih:sx-1,:,:) = reshape(recvw,(/ih,yl,zl/)) + if(switch(2)) a(ex+1:ex+ih,:,:) = reshape(recve,(/ih,yl,zl/)) + + else + + ! Single processor, make sure the field is periodic + if(switch(1)) a(sx-ih:sx-1,:,:) = a(ex-ih+1:ex,:,:) + if(switch(1)) a(ex+1:ex+ih,:,:) = a(sx:sx+ih-1,:,:) + + endif + + if(nprocy.gt.1)then + + ! Make sure data is sent + if(switch(4)) call MPI_WAIT(reqn, status, mpierr) + if(switch(3)) call MPI_WAIT(reqs, status, mpierr) + + deallocate (sendn, sends) + deallocate (recvn, recvs) + + endif + + if(nprocx.gt.1)then + + ! Make sure data is sent + if(switch(2)) call MPI_WAIT(reqe, status, mpierr) + if(switch(1)) call MPI_WAIT(reqw, status, mpierr) + + ! Deallocate buffers + deallocate (sende, sendw) + deallocate (recve, recvw) + + endif + end subroutine openboundary_excjs_real32 + + + subroutine openboundary_excjs_real64(a,sx,ex,sy,ey,sz,ez,ih,jh,switch) + implicit none + integer sx, ex, sy, ey, sz, ez, ih, jh + real(real64) a(sx-ih:ex+ih, sy-jh:ey+jh, sz:ez) + logical, dimension(4), intent(in) :: switch ! flags for which boundaries need exchange + type(MPI_STATUS) :: status + integer :: xl, yl, zl + type(MPI_REQUEST) :: reqn, reqs, reqe, reqw + type(MPI_REQUEST) :: reqrn, reqrs, reqre, reqrw + integer nssize, ewsize + real(real32),allocatable, dimension(:) :: sendn,recvn & + , sends,recvs & + , sende,recve & + , sendw,recvw + +! Calulate buffer lengths + xl = size(a,1) + yl = size(a,2) + zl = size(a,3) + +! Calculate buffer size + nssize = xl*jh*zl + ewsize = ih*yl*zl + + + if(nprocy .gt. 1)then + + ! Allocate send / receive buffers + allocate(sendn(nssize),sends(nssize),recvn(nssize),recvs(nssize)) + + if(switch(4)) sendn = reshape(a(:,ey-jh+1:ey,:),(/nssize/)) + if(switch(3)) sends = reshape(a(:,sy:sy+jh-1,:),(/nssize/)) + + ! Send north/south + if(switch(4)) call D_MPI_ISEND(sendn, nssize, nbrnorth, 4, comm3d, reqn, mpierr) + if(switch(3)) call D_MPI_ISEND(sends, nssize, nbrsouth, 5, comm3d, reqs, mpierr) + + ! Receive south/north + if(switch(3)) call D_MPI_IRECV(recvs, nssize, nbrsouth, 4, comm3d, reqrs, mpierr) + if(switch(4)) call D_MPI_IRECV(recvn, nssize, nbrnorth, 5, comm3d, reqrn, mpierr) + + ! Wait until data is received + if(switch(3)) call MPI_WAIT(reqrs, status, mpierr) + if(switch(4)) call MPI_WAIT(reqrn, status, mpierr) + + ! Write back buffers + if(switch(3)) a(:,sy-jh:sy-1,:) = reshape(recvs,(/xl,jh,zl/)) + if(switch(4)) a(:,ey+1:ey+jh,:) = reshape(recvn,(/xl,jh,zl/)) + + else + + ! Single processor, make sure the field is periodic + if(switch(3)) a(:,sy-jh:sy-1,:) = a(:,ey-jh+1:ey,:) + if(switch(3)) a(:,ey+1:ey+jh,:) = a(:,sy:sy+jh-1,:) + + endif + + if(nprocx .gt. 1)then + + ! Allocate send / receive buffers + allocate(sende(ewsize),sendw(ewsize),recve(ewsize),recvw(ewsize)) + + if(switch(2)) sende = reshape(a(ex-ih+1:ex,:,:),(/ewsize/)) + if(switch(1)) sendw = reshape(a(sx:sx+ih-1,:,:),(/ewsize/)) + + ! Send east/west + if(switch(2)) call D_MPI_ISEND(sende, ewsize, nbreast, 6, comm3d, reqe, mpierr) + if(switch(1)) call D_MPI_ISEND(sendw, ewsize, nbrwest, 7, comm3d, reqw, mpierr) + + ! Receive west/east + if(switch(1)) call D_MPI_IRECV(recvw, ewsize, nbrwest, 6, comm3d, reqrw, mpierr) + if(switch(2)) call D_MPI_IRECV(recve, ewsize, nbreast, 7, comm3d, reqre, mpierr) + + ! Wait until data is received + if(switch(1)) call MPI_WAIT(reqrw, status, mpierr) + if(switch(2)) call MPI_WAIT(reqre, status, mpierr) + + ! Write back buffers + if(switch(1)) a(sx-ih:sx-1,:,:) = reshape(recvw,(/ih,yl,zl/)) + if(switch(2)) a(ex+1:ex+ih,:,:) = reshape(recve,(/ih,yl,zl/)) + + else + + ! Single processor, make sure the field is periodic + if(switch(1)) a(sx-ih:sx-1,:,:) = a(ex-ih+1:ex,:,:) + if(switch(1)) a(ex+1:ex+ih,:,:) = a(sx:sx+ih-1,:,:) + + endif + + if(nprocy.gt.1)then + + ! Make sure data is sent + if(switch(4)) call MPI_WAIT(reqn, status, mpierr) + if(switch(3)) call MPI_WAIT(reqs, status, mpierr) + + deallocate (sendn, sends) + deallocate (recvn, recvs) + + endif + + if(nprocx.gt.1)then + + ! Make sure data is sent + if(switch(2)) call MPI_WAIT(reqe, status, mpierr) + if(switch(1)) call MPI_WAIT(reqw, status, mpierr) + + ! Deallocate buffers + deallocate (sende, sendw) + deallocate (recve, recvw) + + endif + end subroutine openboundary_excjs_real64 + + subroutine excjs_real64(a,sx,ex,sy,ey,sz,ez,ih,jh) implicit none integer sx, ex, sy, ey, sz, ez, ih, jh diff --git a/src/modopenboundary.f90 b/src/modopenboundary.f90 new file mode 100644 index 00000000..2eaf5b93 --- /dev/null +++ b/src/modopenboundary.f90 @@ -0,0 +1,1192 @@ +!> \file modopenboundary.f90 +!! Sets ghost cells at domain boundary and implements radiation +!! conditions for the boundary normal velocity components +!> +!! Sets ghost cells at domain boundary and implements radiation +!! conditions for the boundary normal velocity components +!> +!! \author Frans Liqui Lung +! This file is part of DALES. +! To do: +! - Allow for different zint and adjust rhointi accordingly +! - Correct non divergence free input more elegently +! - Change definition uphase for division by 0 +! - When to use nextval and currentval for nudging and check rtimee +! - How to handle vertical derivative in top boundary condition (full levels) now obtained from profile (horizontal average) +! - Use correct velocity level to determine in or outflow in full levels, half levels are used now +! - Check rtimee and half level nudgin and correction term +! - Use um or u0 in half levels +! - Add possibility for higher order integration schemes +! - Extent turbulent pertubation generation options +! +! DALES is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! DALES is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License +! along with this program. If not, see . +! +! Copyright 1993-2009 Delft University of Technology, Wageningen University, Utrecht University, KNMI +! +module modopenboundary +use modglobal, only : boundary_type,boundary,lopenbc,linithetero,lboundary,lperiodic,lsynturb,dzint,dxint,dyint,ntboundary,tboundary +use modsynturb, only : synturb,initsynturb,exitsynturb +use netcdf +use modprecision, only : field_r +use modmpi, only : openboundary_excjs + +implicit none +integer :: nxpatch, nypatch, nzpatch +real, dimension(:,:), allocatable :: uturbtemp,vturbtemp,wturbtemp +real, dimension(:), allocatable :: rhointi +real, dimension(:,:,:), allocatable :: thls_hetero,ps_hetero +integer :: nx1max,nx2max + +contains + subroutine initopenboundary + ! Initialisation routine for openboundaries + use modmpi, only : myidx, myidy, nprocx, nprocy,myid + use modglobal, only : imax,jmax,kmax,i1,j1,k1,dx,dy,dzf,itot,jtot,zf,zh,solver_id, & + & iadv_mom,iadv_thl,iadv_qt,iadv_tke,iadv_sv,nsv,cu,cv + use modsurfdata, only : isurf + implicit none + integer :: i,j + + if(.not.lopenbc) return + ! Check for conflicting options + if(solver_id == 0) stop 'Openboundaries only possible with HYPRE or FFTW pressure solver, change solver_id' + if(iadv_mom /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_mom to 2' + if(iadv_thl /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_thl to 2' + if(iadv_qt /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_qt to 2' + if(iadv_tke /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_tke to 2' + if(any(iadv_sv(1:nsv)/=2)) stop 'Only second order advection scheme supported with openboundaries, change iadv_sv to 2' + if(cu/=0.) stop 'Translation velocity not allowed in combination with open boundaries, set cu to 0' + if(cv/=0.) stop 'Translation velocity not allowed in combination with open boundaries, set cv to 0' + ! Check if boundary is present on process + if(myidx==0) lboundary(1) = .true. + if(myidx==nprocx-1) lboundary(2) = .true. + if(myidy==0) lboundary(3) = .true. + if(myidy==nprocy-1) lboundary(4) = .true. + lboundary(5) = .true. + ! Set boundary names + boundary(1)%name = "west" + boundary(2)%name = "east" + boundary(3)%name = "south" + boundary(4)%name = "north" + boundary(5)%name = "top" + ! Set dimensions for each boundary + boundary(1)%nx1 = jmax; boundary(1)%nx2 = kmax + boundary(2)%nx1 = jmax; boundary(2)%nx2 = kmax + boundary(3)%nx1 = imax; boundary(3)%nx2 = kmax + boundary(4)%nx1 = imax; boundary(4)%nx2 = kmax + boundary(5)%nx1 = imax; boundary(5)%nx2 = jmax + boundary(1)%nx1u = jmax; boundary(1)%nx2u = kmax + boundary(2)%nx1u = jmax; boundary(2)%nx2u = kmax + boundary(3)%nx1u = i1; boundary(3)%nx2u = kmax + boundary(4)%nx1u = i1; boundary(4)%nx2u = kmax + boundary(5)%nx1u = i1; boundary(5)%nx2u = jmax + boundary(1)%nx1v = j1; boundary(1)%nx2v = kmax + boundary(2)%nx1v = j1; boundary(2)%nx2v = kmax + boundary(3)%nx1v = imax; boundary(3)%nx2v = kmax + boundary(4)%nx1v = imax; boundary(4)%nx2v = kmax + boundary(5)%nx1v = imax; boundary(5)%nx2v = j1 + boundary(1)%nx1w = jmax; boundary(1)%nx2w = k1 + boundary(2)%nx1w = jmax; boundary(2)%nx2w = k1 + boundary(3)%nx1w = imax; boundary(3)%nx2w = k1 + boundary(4)%nx1w = imax; boundary(4)%nx2w = k1 + boundary(5)%nx1w = imax; boundary(5)%nx2w = jmax + ! Set number of patches for correction factor for radiation boundary conditions + if(dxint == -1.) dxint = real(itot)*dx ! Set dxint to entire width as default + if(dyint == -1.) dxint = real(jtot)*dy ! Set dyint to entire width as default + nxpatch = int(dx/dxint*real(itot)); + nypatch = int(dy/dyint*real(jtot)); + nzpatch = kmax ! For now vertical integration scale is set equal to dz + if(mod(dxint,dx)/=0 .or. mod(dyint,dy)/=0) stop 'dxint and dyint should be multiples of dx and dy respectively.' + boundary(1)%nx1patch = nypatch; boundary(1)%nx2patch = nzpatch + boundary(2)%nx1patch = nypatch; boundary(2)%nx2patch = nzpatch + boundary(3)%nx1patch = nxpatch; boundary(3)%nx2patch = nzpatch + boundary(4)%nx1patch = nxpatch; boundary(4)%nx2patch = nzpatch + boundary(5)%nx1patch = nxpatch; boundary(5)%nx2patch = nypatch + if(myid==0) print *,"dxint/dx,dyint/dy,nxpatch,nypatch",int(dxint/dx),int(dyint/dy),nxpatch,nypatch + ! Allocate phase velocity, correction term radiation boundaries and pertubation fields + do i = 1,5 + if(.not.lboundary(i) .or. lperiodic(i)) cycle ! Open boundary not present + allocate(boundary(i)%radcorr(boundary(i)%nx1patch,boundary(i)%nx2patch), & + boundary(i)%radcorrsingle(boundary(i)%nx1patch,boundary(i)%nx2patch), & + boundary(i)%uphase(boundary(i)%nx1patch,boundary(i)%nx2patch), & + boundary(i)%uphasesingle(boundary(i)%nx1patch,boundary(i)%nx2patch), & + boundary(i)%uturb(boundary(i)%nx1u,boundary(i)%nx2u), & + boundary(i)%vturb(boundary(i)%nx1v,boundary(i)%nx2v), & + boundary(i)%wturb(boundary(i)%nx1w,boundary(i)%nx2w), & + boundary(i)%thlturb(boundary(i)%nx1,boundary(i)%nx2), & + boundary(i)%qtturb(boundary(i)%nx1,boundary(i)%nx2), & + boundary(i)%e12turb(boundary(i)%nx1,boundary(i)%nx2)) + boundary(i)%uturb = 0.; boundary(i)%vturb = 0.; boundary(i)%wturb = 0. + boundary(i)%thlturb = 0.; boundary(i)%qtturb = 0.; boundary(i)%e12turb = 0. + if(nsv>0) then + allocate(boundary(i)%svturb(boundary(i)%nx1,boundary(i)%nx2,nsv)) + boundary(i)%svturb = 0. + endif + end do + call initsynturb + end subroutine initopenboundary + + subroutine exitopenboundary + ! Exit routine for openboundaries + use modglobal, only : nsv + implicit none + integer :: i + if(.not.lopenbc) return + deallocate(tboundary) + do i = 1,5 + if(.not.lboundary(i) .or. lperiodic(i)) cycle + deallocate(boundary(i)%thl,boundary(i)%qt,boundary(i)%e12, & + boundary(i)%u,boundary(i)%v,boundary(i)%w,boundary(i)%uphasesingle,boundary(i)%uphase, & + boundary(i)%radcorr,boundary(i)%radcorrsingle,boundary(i)%uturb,boundary(i)%vturb, & + boundary(i)%wturb,boundary(i)%thlturb,boundary(i)%qtturb,boundary(i)%name,boundary(i)%e12turb) + if(nsv>0) deallocate(boundary(i)%sv,boundary(i)%svturb) + end do + deallocate(rhointi) + call exitsynturb + end subroutine exitopenboundary + + subroutine openboundary_initfields + ! Routine that reads the fields for a heterogneous initialisation + ! Variables not present in the input file are initialised to their profiles + use modglobal, only : imax,jmax,kmax,i1,j1,cexpnr,nsv,i2,j2,k1 + use modfields, only : u0,um,v0,vm,w0,wm,thl0,thlm,qt0,qtm,e120,e12m, sv0, svm, & + uprof,vprof,thlprof,qtprof,e12prof,svprof + use modmpi, only : myidx,myidy,myid + implicit none + integer :: VARID,STATUS,NCID,mpierr,timeID,n + integer, dimension(3) :: istart + if(.not.lopenbc) return + if(.not.linithetero) return + u0 = 0.; um = 0.; v0 = 0.; vm = 0.; w0 = 0.; wm = 0.; + thl0 = 0.; thlm = 0.; qt0 = 0; qtm = 0; e120 = 0.; e12m = 0. + !--- open initfields.input.xxx.nc --- + STATUS = NF90_OPEN('initfields.inp.'//cexpnr//'.nc', nf90_nowrite, NCID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + istart = (/myidx*imax+1,myidy*jmax+1,1/) + STATUS = NF90_INQ_VARID(NCID,'u0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + call take_prof(u0,um,uprof) + if(myid==0) print *, "u initialized with prof.inp" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,u0(2:i2,2:j1,1:kmax),start=istart,count=(/i1,jmax,kmax/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + um = u0 + endif + STATUS = NF90_INQ_VARID(NCID,'v0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + call take_prof(v0,vm,vprof) + if(myid==0) print *, "v initialized with prof.inp" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,v0(2:i1,2:j2,1:kmax),start=istart,count=(/imax,j1,kmax/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + vm = v0 + endif + STATUS = NF90_INQ_VARID(NCID,'w0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with 0. + wm = 0. + w0 = 0. + if(myid==0) print *, "w initilised with 0." + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,w0(2:i1,2:j1,1:k1),start=istart,count=(/imax,jmax,k1/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + wm = w0 + endif + STATUS = NF90_INQ_VARID(NCID,'thl0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + call take_prof(thl0,thlm,thlprof) + if(myid==0) print *, "thl initialized with prof.inp" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,thl0(2:i1,2:j1,1:kmax),start=istart,count=(/imax,jmax,kmax/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + thlm = thl0 + endif + STATUS = NF90_INQ_VARID(NCID,'qt0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + call take_prof(qt0,qtm,qtprof) + if(myid==0) print *, "qt initialized with prof.inp" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,qt0(2:i1,2:j1,1:kmax),start=istart,count=(/imax,jmax,kmax/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + qtm = qt0 + endif + STATUS = NF90_INQ_VARID(NCID,'e120', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + call take_prof(e120,e12m,e12prof) + if(myid==0) print *, "e12 initialized with prof.inp" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,e120(2:i1,2:j1,1:kmax),start=istart,count=(/imax,jmax,kmax/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + e12m = e120 + endif + if(nsv>0) then + STATUS = NF90_INQ_VARID(NCID,'sv0', VARID) + if(STATUS == NF90_ENOTVAR) then ! If not available initialise with profile + do n = 1,nsv + call take_prof(sv0(:,:,:,n),svm(:,:,:,n),svprof(:,n)) + end do + if(myid==0) print *, "sv initialized with scalar.inp" + else ! Initial field taken from initfields.inp + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID,sv0(2:i1,2:j1,1:kmax,1),start=istart,count=(/imax,jmax,kmax,nsv/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + svm = sv0 + endif + endif + status = nf90_close(ncid) + if (status /= nf90_noerr) call handle_err(status) + end subroutine openboundary_initfields + + subroutine openboundary_readboundary + ! Routine reads the boundary input for all time steps + use modglobal, only : kmax,cexpnr,imax,jmax,itot,jtot,k1,ntboundary, & + tboundary,i1,j1,i2,j2,kmax,nsv,iturb + use modmpi, only : myidx,myidy + implicit none + integer :: it,i,j,k,ib + character(len = nf90_max_name) :: RecordDimName + integer :: VARID,STATUS,NCID,mpierr,timeID + integer, dimension(3) :: istart + + if(.not.lopenbc) return + !--- open openboundaries.inp.xxx.nc --- + STATUS = NF90_OPEN('openboundaries.inp.'//cexpnr//'.nc', nf90_nowrite, NCID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + !--- get time dimensions + status = nf90_inq_dimid(ncid, "time", timeID) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_inquire_dimension(NCID, timeID, len=ntboundary, name=RecordDimName) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + !--- read time + allocate(tboundary(ntboundary)) + STATUS = NF90_INQ_VARID(NCID, 'time', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, tboundary, start=(/1/), count=(/ntboundary/) ) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + do ib = 1,5 ! loop over boundaries + ! Allocate input fields + if(.not.lboundary(ib) .or. lperiodic(ib)) cycle ! Open boundary not present + allocate(boundary(ib)%thl(boundary(ib)%nx1,boundary(ib)%nx2,ntboundary), & + boundary(ib)%qt(boundary(ib)%nx1,boundary(ib)%nx2,ntboundary), & + boundary(ib)%e12(boundary(ib)%nx1,boundary(ib)%nx2,ntboundary), & + boundary(ib)%u(boundary(ib)%nx1u,boundary(ib)%nx2u,ntboundary), & + boundary(ib)%v(boundary(ib)%nx1v,boundary(ib)%nx2v,ntboundary), & + boundary(ib)%w(boundary(ib)%nx1w,boundary(ib)%nx2w,ntboundary)) + if(nsv>0) then + allocate(boundary(ib)%sv(boundary(ib)%nx1,boundary(ib)%nx2,ntboundary,nsv)) + endif + if(lsynturb .and. iturb<10) then ! Allocate turbulent input fields + allocate(boundary(ib)%u2(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%v2(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%w2(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%uv(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%uw(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%vw(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%thl2(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary),& + & boundary(ib)%qt2(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%wthl(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary), & + & boundary(ib)%wqt(boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary)) + endif + ! Read fields + select case(ib) + case(1,2) + istart = (/myidy*jmax+1,1,1/) + case(3,4) + istart = (/myidx*imax+1,1,1/) + case(5) + istart = (/myidx*imax+1,myidy*jmax+1,1/) + end select + ! Read u + STATUS = NF90_INQ_VARID(NCID,'u'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%u, start=istart, & + & count=(/boundary(ib)%nx1u,boundary(ib)%nx2u,ntboundary/)) + ! Read v + STATUS = NF90_INQ_VARID(NCID, 'v'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%v, start=istart, & + & count=(/boundary(ib)%nx1v,boundary(ib)%nx2v,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read w + STATUS = NF90_INQ_VARID(NCID, 'w'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%w, start=istart, & + & count=(/boundary(ib)%nx1w,boundary(ib)%nx2w,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read thl + STATUS = NF90_INQ_VARID(NCID, 'thl'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%thl, start=istart, & + & count=(/boundary(ib)%nx1,boundary(ib)%nx2,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read qt + STATUS = NF90_INQ_VARID(NCID, 'qt'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%qt, start=istart, & + & count=(/boundary(ib)%nx1,boundary(ib)%nx2,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read e12 + STATUS = NF90_INQ_VARID(NCID, 'e12'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%e12, start=istart, & + & count=(/boundary(ib)%nx1,boundary(ib)%nx2,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read sv + if(nsv>0) then + STATUS = NF90_INQ_VARID(NCID, 'sv'//boundary(ib)%name, VARID) + if(STATUS == NF90_ENOTVAR) then + boundary(ib)%sv = 0. + if(myidx==0 .and. myidy==0) print *, "No boundary information for sv at boundary",ib,"Values set to 0" + else + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%sv, start=(/istart,1/), & + & count=(/boundary(ib)%nx1,boundary(ib)%nx2,ntboundary,nsv/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + endif + endif + ! Read input for turbulent pertubations + if(lsynturb .and. iturb <10) then + ! Read u2 + STATUS = NF90_INQ_VARID(NCID, 'u2'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%u2, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read v2 + STATUS = NF90_INQ_VARID(NCID, 'v2'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%v2, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read w2 + STATUS = NF90_INQ_VARID(NCID, 'w2'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%w2, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read uv + STATUS = NF90_INQ_VARID(NCID, 'uv'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%uv, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read uw + STATUS = NF90_INQ_VARID(NCID, 'uw'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%uw, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read vw + STATUS = NF90_INQ_VARID(NCID, 'vw'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%vw, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read thl2 + STATUS = NF90_INQ_VARID(NCID, 'thl2'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%thl2, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read qt2 + STATUS = NF90_INQ_VARID(NCID, 'qt2'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%qt2, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read wthl + STATUS = NF90_INQ_VARID(NCID, 'wthl'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%wthl, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! Read wqt + STATUS = NF90_INQ_VARID(NCID, 'wqt'//boundary(ib)%name, VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, boundary(ib)%wqt, start=(/1,1,1/), & + & count=(/boundary(ib)%nx1patch,boundary(ib)%nx2patch,ntboundary/)) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + endif + end do + status = nf90_close(ncid) + if (status /= nf90_noerr) call handle_err(status) + end subroutine openboundary_readboundary + + subroutine openboundary_divcorr() + ! Correct for any integrated divergence present in the boundary input + use modmpi, only : myid,comm3d,mpierr,d_mpi_allreduce,MPI_SUM + use modglobal, only : imax,jmax,kmax,dzf,dy,dx,xsize,ysize,zh,k1,lwarmstart, & + i1,i2,j1,j2,k1,dzh + use modfields, only : u0,um,v0,vm,w0,wm,rhobf,rhobh + implicit none + real(field_r) :: sumdiv,div,divpart,divnew,divold + integer :: i,j,k,it,icalc + ! Create 1/int(rho) + allocate(rhointi(k1)) + rhointi = 1./(rhobf*dzf) + ! Divergence correction + if(myid==0) print *, "Start divergence correction boundaries" + do it = 1,ntboundary + do icalc=1,2 + ! Calculate divergence + div = 0. + if(lboundary(1)) then + do j = 1,jmax + do k = 1,kmax + div = div - rhobf(k)*boundary(1)%u(j,k,it)*dzf(k)*dy + end do + end do + endif + if(lboundary(2)) then + do j = 1,jmax + do k = 1,kmax + div = div + rhobf(k)*boundary(2)%u(j,k,it)*dzf(k)*dy + end do + end do + endif + if(lboundary(3)) then + do i = 1,imax + do k = 1,kmax + div = div - rhobf(k)*boundary(3)%v(i,k,it)*dzf(k)*dx + end do + end do + endif + if(lboundary(4)) then + do i = 1,imax + do k = 1,kmax + div = div + rhobf(k)*boundary(4)%v(i,k,it)*dzf(k)*dx + end do + end do + endif + if(lboundary(5)) then + do i = 1,imax + do j = 1,jmax + div = div + rhobh(k1)*boundary(5)%w(i,j,it)*dx*dy + end do + end do + endif + call D_MPI_ALLREDUCE(div,sumdiv,1,MPI_SUM,comm3d,mpierr) + if(icalc==1) then + divold=sumdiv + else + divnew=sumdiv + if(myid==0) print *, 't,input,corrected;',tboundary(it),divold,divnew + exit + endif + ! Apply correction, spread divergence over lateral boundaries + if(lboundary(1)) then + do k = 1,kmax + divpart = sumdiv*ysize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1)) + boundary(1)%u(:,k,it)=boundary(1)%u(:,k,it)+divpart/(rhobf(k)*ysize*dzf(k)) + end do + endif + if(lboundary(2)) then + do k = 1,kmax + divpart = sumdiv*ysize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1)) + boundary(2)%u(:,k,it)=boundary(2)%u(:,k,it)-divpart/(rhobf(k)*ysize*dzf(k)) + end do + endif + if(lboundary(3)) then + do k = 1,kmax + divpart = sumdiv*xsize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1)) + boundary(3)%v(:,k,it)=boundary(3)%v(:,k,it)+divpart/(rhobf(k)*dzf(k)*xsize) + end do + endif + if(lboundary(4)) then + do k = 1,kmax + divpart = sumdiv*xsize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1)) + boundary(4)%v(:,k,it)=boundary(4)%v(:,k,it)-divpart/(rhobf(k)*xsize*dzf(k)) + enddo + endif + end do + end do + if(myid==0) print *, "Finished divergence correction boundaries" + ! Copy data to boundary information + if(.not.lwarmstart) then + if(lboundary(1).and..not.lperiodic(1)) then + do j = 2,j1 + do k = 1,kmax + u0(2,j,k) = boundary(1)%u(j-1,k,1) + um(2,j,k) = boundary(1)%u(j-1,k,1) + end do + end do + endif + if(lboundary(2).and..not.lperiodic(2)) then + do j = 2,j1 + do k = 1,kmax + u0(i2,j,k) = boundary(2)%u(j-1,k,1) + um(i2,j,k) = boundary(2)%u(j-1,k,1) + end do + end do + endif + if(lboundary(3).and..not.lperiodic(3)) then + do i = 2,i1 + do k = 1,kmax + v0(i,2,k) = boundary(3)%v(i-1,k,1) + vm(i,2,k) = boundary(3)%v(i-1,k,1) + end do + end do + endif + if(lboundary(4).and..not.lperiodic(4)) then + do i = 2,i1 + do k = 1,kmax + v0(i,j2,k) = boundary(4)%v(i-1,k,1) + vm(i,j2,k) = boundary(4)%v(i-1,k,1) + end do + end do + endif + if(lboundary(5).and..not.lperiodic(5)) then + do i = 2,i1 + do j = 2,j1 + w0(i,j,k1) = boundary(5)%w(i-1,j-1,1) + wm(i,j,k1) = boundary(5)%w(i-1,j-1,1) + end do + end do + endif + endif + end subroutine openboundary_divcorr + + subroutine openboundary_ghost + ! Subroutine that fills the ghost cells for the cell centred variables at the boundary + use modglobal, only : i1,j1,k1,ih,jh,nsv + use modfields, only : um,u0,vm,v0,wm,w0,e12m,e120,thlm,thl0,qtm,qt0,svm,sv0,thl0av,qt0av,u0av,v0av + implicit none + integer :: i,n + if(.not.lopenbc) return + ! Apply non domain boundaries and periodic boundaries + call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(u0 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(v0 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(w0 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(e12m , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(e120 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(thlm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(thl0 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(qtm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(qt0 , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + do n = 1,nsv + call openboundary_excjs(svm(:,:,:,n), 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(sv0(:,:,:,n), 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + end do + ! Apply open boundaries for domain boundaries for full levels (ghost cells) + do i = 1,5 ! Loop over boundaries + if(.not.lboundary(i).or.lperiodic(i)) cycle + call applyboundaryf(thlm ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%thl,boundary(i)%nx1,boundary(i)%nx2,0,boundary(i)%thlturb,profile=thl0av) + call applyboundaryf(thl0 ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%thl,boundary(i)%nx1,boundary(i)%nx2,0,boundary(i)%thlturb,profile=thl0av) + call applyboundaryf(qtm ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%qt,boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%qtturb,profile=qt0av) + call applyboundaryf(qt0 ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%qt,boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%qtturb,profile=qt0av) + call applyboundaryf(e12m ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%e12,boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%e12turb) + call applyboundaryf(e120 ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%e12,boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%e12turb) + do n = 1,nsv + call applyboundaryf(svm(:,:,:,n) ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%sv(:,:,:,n),boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%svturb(:,:,n)) + call applyboundaryf(sv0(:,:,:,n) ,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%sv(:,:,:,n),boundary(i)%nx1,boundary(i)%nx2,1,boundary(i)%svturb(:,:,n)) + end do + if(i/=1.and.i/=2) call applyboundaryf(um,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%u,boundary(i)%nx1u,boundary(i)%nx2u,0,boundary(i)%uturb,profile=u0av) + if(i/=1.and.i/=2) call applyboundaryf(u0,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%u,boundary(i)%nx1u,boundary(i)%nx2u,0,boundary(i)%uturb,profile=u0av) + if(i/=3.and.i/=4) call applyboundaryf(vm,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%v,boundary(i)%nx1v,boundary(i)%nx2v,0,boundary(i)%vturb,profile=v0av) + if(i/=3.and.i/=4) call applyboundaryf(v0,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%v,boundary(i)%nx1v,boundary(i)%nx2v,0,boundary(i)%vturb,profile=v0av) + if(i/=5) call applyboundaryf(wm,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%w,boundary(i)%nx1w,boundary(i)%nx2w,0,boundary(i)%wturb) + if(i/=5) call applyboundaryf(w0,2,i1,2,j1,1,k1,ih,jh,i,boundary(i)%w,boundary(i)%nx1w,boundary(i)%nx2w,0,boundary(i)%wturb) + end do + end subroutine openboundary_ghost + + subroutine openboundary_tend + ! Subroutine that handles the tendencies of the boundary normal velocity components + ! Radiation boundary conditions are used for outflow boundaries and nudging + ! boundary conditions for the inflow boundaries. + ! Outflow: + ! du/dt = -uphase*du/dx + ! Inflow: + ! du/dt = (u-uboundary)/tau + implicit none + integer :: ib + + if(.not.lopenbc) return + if(lboundary(1).and..not.lperiodic(1)) call applyboundaryh(1,boundary(1)%nx1,boundary(1)%nx2,boundary(1)%uturb) + if(lboundary(2).and..not.lperiodic(2)) call applyboundaryh(2,boundary(2)%nx1,boundary(2)%nx2,boundary(2)%uturb) + if(lboundary(3).and..not.lperiodic(3)) call applyboundaryh(3,boundary(3)%nx1,boundary(3)%nx2,boundary(3)%vturb) + if(lboundary(4).and..not.lperiodic(4)) call applyboundaryh(4,boundary(4)%nx1,boundary(4)%nx2,boundary(4)%vturb) + if(lboundary(5).and..not.lperiodic(5)) call applyboundaryh(5,boundary(5)%nx1,boundary(5)%nx2,boundary(5)%wturb) + ! Calculate and add correction term to guarantee conservation of mass + do ib = 1,5 + if(.not. lboundary(ib).or.lperiodic(ib)) cycle + call radcorrection(ib) + end do + end subroutine openboundary_tend + + subroutine openboundary_phasevelocity + ! Subroutine that calculates the phase velocity that is required for the + ! radiation outflow boundary. The phase velocity is calculated from the phase + ! velocity of one gridcell to the interior at the prior time and is averaged + ! over the integration length scales. Lower limit is given by input boundary + ! velocity and upper limit by cfd criterium. + use modmpi, only : comm3d,commrow,commcol,myidx,myidy,mpierr,D_MPI_ALLREDUCE,MPI_SUM + use modglobal, only : imax,jmax,kmax,i1,j1,dx,dy,dzf + use modfields, only : u0,up,v0,vp,w0,wp,rhobh,rhobf + implicit none + integer :: ib,i,j,k,ipatch,jpatch,kpatch + real :: ipos,jpos + + if(.not.lopenbc) return + do ib = 1,5 ! Loop over boundaries + if(.not.lboundary(ib).or.lperiodic(ib)) cycle + select case(ib) ! Select boundary + case(1) ! West + boundary(1)%uphasesingle=0. + do j = 1,jmax + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + boundary(1)%uphasesingle(jpatch,kpatch) = boundary(1)%uphasesingle(jpatch,kpatch) + & + (-up(3,j+1,k)*dx/sign(max(abs(u0(4,j+1,k)-u0(3,j+1,k)),real(1e-10, field_r)),u0(4,j+1,k)-u0(3,j+1,k))) & + *dy*rhobf(k)*dzf(k)/dyint*rhointi(kpatch) + end do + end do + ! Integrate over processes + call D_MPI_ALLREDUCE(boundary(1)%uphasesingle,boundary(1)%uphase,nypatch*nzpatch, & + MPI_SUM, commcol,mpierr) + case(2) ! East + boundary(2)%uphasesingle=0. + do j = 1,jmax + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + boundary(2)%uphasesingle(jpatch,kpatch) = boundary(2)%uphasesingle(jpatch,kpatch) + & + (-up(i1,j+1,k)*dx/sign(max(abs(u0(i1,j+1,k)-u0(i1-1,j+1,k)),real(1e-10, field_r)),u0(i1,j+1,k)-u0(i1-1,j+1,k))) & + *dy*rhobf(k)*dzf(k)/dyint*rhointi(kpatch) + end do + end do + ! Integrate over processes + call D_MPI_ALLREDUCE(boundary(2)%uphasesingle,boundary(2)%uphase,nypatch*nzpatch, & + MPI_SUM, commcol,mpierr) + case(3) ! South + boundary(3)%uphasesingle=0. + do i = 1,imax + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + boundary(3)%uphasesingle(ipatch,kpatch) = boundary(3)%uphasesingle(ipatch,kpatch) + & + (-vp(i+1,3,k)*dy/sign(max(abs(v0(i+1,4,k)-v0(i+1,3,k)),real(1e-10, field_r)),v0(i+1,4,k)-v0(i+1,3,k))) & + *dx*rhobf(k)*dzf(k)/dxint*rhointi(kpatch) + end do + end do + ! Integrate over processes + call D_MPI_ALLREDUCE(boundary(3)%uphasesingle,boundary(3)%uphase,nxpatch*nzpatch, & + MPI_SUM, commrow,mpierr) + case(4) ! North + boundary(4)%uphasesingle=0. + do i = 1,imax + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + boundary(4)%uphasesingle(ipatch,kpatch) = boundary(4)%uphasesingle(ipatch,kpatch) + & + (-vp(i+1,j1,k)*dy/sign(max(abs(v0(i+1,j1,k)-v0(i+1,j1-1,k)),real(1e-10, field_r)),v0(i+1,j1,k)-v0(i+1,j1-1,k))) & + *dx*rhobf(k)/dxint*rhointi(kpatch) + end do + end do + ! Integrate over processes + call D_MPI_ALLREDUCE(boundary(4)%uphasesingle,boundary(4)%uphase,nxpatch*nzpatch, & + MPI_SUM, commrow,mpierr) + case(5) ! Top + boundary(5)%uphasesingle=0. + do i = 1,imax + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do j = 1,jmax + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + boundary(5)%uphasesingle(ipatch,jpatch) = boundary(5)%uphasesingle(ipatch,jpatch) + & + (-rhobh(kmax)*wp(i+1,j+1,kmax)*dzf(kmax-1)/sign(max(abs(rhobh(kmax)*w0(i+1,j+1,kmax)-rhobh(kmax-1)*w0(i+1,j+1,kmax-1)),real(1e-10, field_r)), & + & rhobh(kmax)*w0(i+1,j+1,kmax)-rhobh(kmax-1)*w0(i+1,j+1,kmax-1)))*dx*dy/(dxint*dyint) + end do + end do + ! Integrate over processes + call D_MPI_ALLREDUCE(boundary(5)%uphasesingle,boundary(5)%uphase,nxpatch*nypatch, & + MPI_SUM, comm3d,mpierr) + end select + end do + end subroutine openboundary_phasevelocity + + subroutine openboundary_turb + ! Subroutine that calls the synthetic turbulence routine for the generation + ! of synthetic turbulence at the dirichlet inflow boundaries. + use modglobal, only : rk3step + implicit none + if(rk3step == 1) call synturb() + end subroutine openboundary_turb + + subroutine applyboundaryf(a,sx,ex,sy,ey,sz,ez,ih,jh,ib,val,nx1,nx2,lmax0,turb,profile) + ! Routine fills ghost cells based on robin (inflow) or + ! homogeneous neumann (outflow) boundary conditions. Adds turbulent + ! pertubations to inflow condition if lsynturb=true. + use modglobal, only : dzh,dx,dy,imax,jmax,kmax,rtimee,rdt,i2,j2,k1,i1,j1,tauh,pbc + use modfields, only : u0,v0,w0,e120 + use modmpi, only : myid + implicit none + integer, intent(in) :: sx,ex,sy,ey,sz,ez,ih,jh,ib,nx1,nx2,lmax0 + real(field_r), intent(in), dimension(nx1,nx2,ntboundary) :: val + real(field_r), intent(in), dimension(nx1,nx2) :: turb + real(field_r), intent(in), dimension(k1), optional :: profile ! optional for top boundary to take gradient into account + real(field_r), intent(inout), dimension(sx-ih:ex+ih,sy-jh:ey+jh,sz:ez) :: a + integer :: i,j,k,itp,itm,kav=5,itpn,itmn + real :: coefdir,coefneu,tp,tm,fp,fm,fpn,fmn,ddz,valtarget,un,e + + ! Get interpolation coefficients for boundary input + itm=1 + if(ntboundary>1) then + do while(rtimee-rdt>tboundary(itm)) + itm=itm+1 + end do + if (rtimee-rdt>tboundary(1)) then + itm=itm-1 + end if + itp = itm+1 + tm = tboundary(itm) + tp = tboundary(itp) + fm = (tp-rtimee+rdt)/(tp-tm) + fp = (rtimee-rdt-tm)/(tp-tm) + else + itp = 1 + fp = 0. + fm = 1. + endif + select case(ib) ! Select domain boundary + case(1) ! West + do k = 1,nx2 + do j = 1,nx1 + un = u0(sx,min(j+1,j1),min(k,kmax)) + if(un<=0) then ! Homogeneous Neumann outflow + a(sx-1,j+1,k)=a(sx,j+1,k) + else ! Robin inflow conditions + e = e120(sx,min(j+1,j1),min(k,kmax)) + coefdir = abs(un)**pbc + coefneu = -tauh*un*(abs(un)**pbc+e**pbc) + valtarget = (fp*val(j,k,itp)+fm*val(j,k,itm)+turb(j,k))*coefdir + a(sx-1,j+1,k) = ( 2.*dx*valtarget - & + a(sx,j+1,k)*(coefdir*dx+2.*coefneu) ) / (coefdir*dx-2.*coefneu) + if(lmax0==1) a(sx-1,j+1,k) = max(0.,a(sx-1,j+1,k)) + endif + end do + end do + case(2) ! East + do k = 1,nx2 + do j = 1,nx1 + un = u0(ex+1,min(j+1,j1),min(k,kmax)) + if(un>=0) then ! Homogeneous Neumann outflow + a(ex+1,j+1,k)=a(ex,j+1,k) + else ! Robin inflow conditions + e = e120(ex,min(j+1,j1),min(k,kmax)) + coefdir = abs(un)**pbc + coefneu = -tauh*un*(abs(un)**pbc+e**pbc) + valtarget = (fp*val(j,k,itp)+fm*val(j,k,itm)+turb(j,k))*coefdir + a(ex+1,j+1,k) = ( 2.*dx*valtarget - & + a(ex,j+1,k)*(coefdir*dx-2.*coefneu) ) / (coefdir*dx+2.*coefneu) + if(lmax0==1) a(ex+1,j+1,k) = max(a(ex+1,j+1,k),0.) + endif + end do + end do + case(3) ! South + do k = 1,nx2 + do i = 1,nx1 + un = v0(min(i+1,i1),sy,min(k,kmax)) + if(un<=0) then ! Homogeneous Neumann outflow + a(i+1,sy-1,k)=a(i+1,sy,k) + else ! Robin inflow conditions + e = e120(min(i+1,i1),sy,min(k,kmax)) + coefdir = abs(un)**pbc + coefneu = -tauh*un*(abs(un)**pbc+e**pbc) + valtarget = (fp*val(i,k,itp)+fm*val(i,k,itm)+turb(i,k))*coefdir + a(i+1,sy-1,k) = ( 2.*dy*valtarget - & + a(i+1,sy,k)*(coefdir*dy+2.*coefneu) ) / (coefdir*dy-2.*coefneu) + if(lmax0==1) a(i+1,sy-1,k) = max(a(i+1,sy-1,k),0.) + endif + end do + end do + case(4) ! North + do k = 1,nx2 + do i = 1,nx1 + un = v0(min(i+1,i1),ey+1,min(k,kmax)) + if(un>=0) then ! Homogeneous Neumann outflow + a(i+1,ey+1,k)=a(i+1,ey,k) + else ! Robin inflow conditions + e = e120(min(i+1,i1),ey,min(k,kmax)) + coefdir = abs(un)**pbc + coefneu = -tauh*un*(abs(un)**pbc+e**pbc) + valtarget = (fp*val(i,k,itp)+fm*val(i,k,itm)+turb(i,k))*coefdir + a(i+1,ey+1,k) = ( 2.*dy*valtarget - & + a(i+1,ey,k)*(coefdir*dy-2.*coefneu) ) / (coefdir*dy+2.*coefneu) + if(lmax0==1) a(i+1,ey+1,k) = max(a(i+1,ey+1,k),0.) + endif + end do + end do + case(5) ! Top + ! Obtain verticle gradient if slab averaged profile is given + if(present(profile)) then + ddz = sum((profile(kmax-kav+1:kmax)-profile(kmax-kav:kmax-1))/ & + dzh(kmax-kav+1:kmax))/kav + else + ddz = 0. + endif + do i = 1,nx1 + do j = 1,nx2 + un = w0(min(i+1,i1),min(j+1,j1),ez) + if(un>=0) then ! Neumann outflow + a(i+1,j+1,ez)=ddz*dzh(ez)+a(i+1,j+1,ez-1) + else ! Robin inflow conditions + e = e120(min(i+1,i1),min(j+1,j1),ez-1) + coefdir = abs(un)**pbc + coefneu = -tauh*un*(abs(un)**pbc+e**pbc) + valtarget = (fp*val(i,j,itp)+fm*val(i,j,itm)+turb(i,j))*coefdir+ddz*coefneu + a(i+1,j+1,ez) = ( 2.*dzh(ez)*valtarget - & + a(i+1,j+1,ez-1)*(coefdir*dzh(ez)-2.*coefneu) ) / (coefdir*dzh(ez)+2.*coefneu) + if(lmax0==1) a(i+1,j+1,ez) = max(a(i+1,j+1,ez),0.) + endif + end do + end do + end select + end subroutine applyboundaryf + + subroutine applyboundaryh(ib,nx1,nx2,turb) + ! Subroutine that applies the radiation and dirichlet boundary conditions + ! for the boundary-normal velocity components. Adds turbulence to + ! the inflow dirichlet boundaries if lsynturb=.true. + use modmpi, only : myidx,myidy,myid + use modglobal, only : dx,dy,dzf,dxi,dyi,rdt,i2,j2,k1,i1,j1,kmax,rtimee,rdt,itot,jtot,imax,jmax,grav,taum + use modfields, only : um,u0,up,vm,v0,vp,wm,w0,wp,rhobf,rhobh,thvh,thv0h + implicit none + integer, intent(in) :: nx1,nx2,ib + real(field_r), intent(in), dimension(nx1,nx2) :: turb + integer :: i,j,k,itmc,itmn,itpc,itpn,ipatch,jpatch,kpatch + real :: tm,tp,fpc,fmc,fpn,fmn,unext,uwallcurrent,ipos,jpos,tau + itmc=1 + itmn=1 + if(ntboundary>1) then + do while(rtimee-rdt>tboundary(itmc)) + itmc=itmc+1 + end do + if (rtimee-rdt>tboundary(1)) then + itmc=itmc-1 + end if + do while(tboundary(itmn)tboundary(1)) then + itmn=itmn-1 + end if + itpc = itmc+1 + itpn = itmn+1 + tm = tboundary(itmc) + tp = tboundary(itpc) + fmc = (tp-rtimee+rdt)/(tp-tm) + fpc = (rtimee-rdt-tm)/(tp-tm) + tm = tboundary(itmn) + tp = tboundary(itpn) + fmn = (tp-rtimee)/(tp-tm) + fpn = (rtimee-tm)/(tp-tm) + else + itpc = 1 + itpn = 1 + fpc = 0. + fmc = 1. + fpn = 0. + fmn = 1. + endif + ! Apply domain boundaries + select case(ib) ! Select boundary + case(1) ! West + tau = max(taum,rdt) + do j = 1,nx1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,nx2 + kpatch = k + uwallcurrent = fpc*boundary(1)%u(j,k,itpc)+fmc*boundary(1)%u(j,k,itmc) + if(uwallcurrent<=0.) then ! Outflow (Radiation) + up(2,j+1,k) = -max(min(boundary(1)%uphase(jpatch,kpatch),uwallcurrent),-dx/rdt) * & + (u0(3,j+1,k)-u0(2,j+1,k))*dxi + else ! Inflow nudging + unext = fpn*boundary(1)%u(j,k,itpn)+fmn*boundary(1)%u(j,k,itmn) + up(2,j+1,k) = ((unext+turb(j,k)) - u0(2,j+1,k))/tau + endif + end do + end do + case(2) ! East + tau = max(taum,rdt) + do j = 1,nx1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,nx2 + kpatch = k + uwallcurrent = fpc*boundary(2)%u(j,k,itpc)+fmc*boundary(2)%u(j,k,itmc) + if(uwallcurrent>=0.) then ! Outflow (Radiation) + up(i2,j+1,k) = -min(max(boundary(2)%uphase(jpatch,kpatch),uwallcurrent),dx/rdt) * & + (u0(i2,j+1,k)-u0(i1,j+1,k))*dxi + else ! Inflow (Dirichlet) + unext = fpn*boundary(2)%u(j,k,itpn)+fmn*boundary(2)%u(j,k,itmn) + up(i2,j+1,k) = ((unext+turb(j,k)) - u0(i2,j+1,k))/tau + endif + end do + end do + case(3) ! South + tau = max(taum,rdt) + do i = 1,nx1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,nx2 + kpatch = k + uwallcurrent = fpc*boundary(3)%v(i,k,itpc)+fmc*boundary(3)%v(i,k,itmc) + if(uwallcurrent<=0.) then ! Outflow (Radiation) + vp(i+1,2,k) = -max(min(boundary(3)%uphase(ipatch,kpatch),uwallcurrent),-dy/rdt) * & + (v0(i+1,3,k)-v0(i+1,2,k))*dyi + else ! Inflow (Dirichlet) + unext = fpn*boundary(3)%v(i,k,itpn)+fmn*boundary(3)%v(i,k,itmn) + vp(i+1,2,k) = ((unext+turb(i,k)) - v0(i+1,2,k))/tau + endif + end do + end do + case(4) ! North + tau = max(taum,rdt) + do i = 1,nx1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,nx2 + kpatch = k + uwallcurrent = fpc*boundary(4)%v(i,k,itpc)+fmc*boundary(4)%v(i,k,itmc) + if(uwallcurrent>=0.) then ! Outflow (Radiation) + vp(i+1,j2,k) = -min(max(boundary(4)%uphase(ipatch,kpatch),uwallcurrent),dy/rdt) * & + (v0(i+1,j2,k)-v0(i+1,j1,k))*dyi + else ! Inflow (Dirichlet) + unext = fpn*boundary(4)%v(i,k,itpn)+fmn*boundary(4)%v(i,k,itmn) + vp(i+1,j2,k) = ((unext+turb(i,k)) - v0(i+1,j2,k))/tau + endif + end do + end do + case(5) ! Top + tau = max(taum,rdt) + do i = 1,nx1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do j = 1,nx2 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + uwallcurrent = fpc*boundary(5)%w(i,j,itpc)+fmc*boundary(5)%w(i,j,itmc) + if(uwallcurrent>=0.) then ! Outflow (Radiation) + wp(i+1,j+1,k1) = -min(max(boundary(5)%uphase(ipatch,jpatch),uwallcurrent),dzf(kmax)/rdt) * & + (rhobh(k1)*w0(i+1,j+1,k1)-rhobh(kmax)*w0(i+1,j+1,kmax))/(dzf(kmax)*rhobh(k1)) + & + grav*(thv0h(i+1,j+1,k1)-thvh(k1))/thvh(k1) + else ! Inflow (Dirichlet) + unext = fpn*boundary(5)%w(i,j,itpn)+fmn*boundary(5)%w(i,j,itmn) + wp(i+1,j+1,k1) = ((unext+turb(i,j)) - w0(i+1,j+1,k1))/tau + endif + end do + end do + end select + end subroutine applyboundaryh + + subroutine radcorrection(ib) + ! Calculates the integrated mass correction term for the boundary normal + ! velocity components + use modmpi, only : comm3d,commrow,commcol,myidx,myidy,mpierr, D_MPI_ALLREDUCE, MPI_SUM + use modglobal, only : jmax,imax,kmax,i1,j1,dx,dy,dzf,i2,j2,k1,dxi,dyi,rtimee,rdt + use modfields, only : rhobf, up, vp, wp + implicit none + integer, intent(in) :: ib + integer :: ipos,jpos,kpos,ipatch,jpatch,kpatch,i,j,k,itp,itm + real :: sum,tp,tm,idtb,dubdt + + itm = 1 + if(ntboundary>1) then + do while(rtimee-rdt>tboundary(itm)) + itm=itm+1 + end do + if (rtimee-rdt>tboundary(1)) then + itm=itm-1 + end if + itp = itm+1 + else + itp = 1 + endif + tm = tboundary(itm) + tp = tboundary(itp) + idtb = 1./max(1e-6,tp-tm) + select case(ib) ! Select boundary + case(1) ! West + ! Calculate correction term for each patch + boundary(1)%radcorrsingle = 0. + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + dubdt = (boundary(1)%u(j-1,k,itp)-boundary(1)%u(j-1,k,itm))*idtb + boundary(1)%radcorrsingle(jpatch,kpatch) = boundary(1)%radcorrsingle(jpatch,kpatch) + & + rhobf(k)*(-up(2,j,k)+dubdt)*dzf(k)*dy*rhointi(kpatch)/dyint + end do + end do + ! Communicate integration between processes + call D_MPI_ALLREDUCE(boundary(1)%radcorrsingle,boundary(1)%radcorr,nypatch*nzpatch, & + MPI_SUM, commcol,mpierr) + ! Apply correction term + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + up(2,j,k) = up(2,j,k) + boundary(1)%radcorr(jpatch,kpatch) + end do + end do + case(2) ! East + ! Calculate correction term for each patch + boundary(2)%radcorrsingle = 0. + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + dubdt = (boundary(2)%u(j-1,k,itp)-boundary(2)%u(j-1,k,itm))*idtb + boundary(2)%radcorrsingle(jpatch,kpatch) = boundary(2)%radcorrsingle(jpatch,kpatch) + & + rhobf(k)*(-up(i2,j,k)+dubdt)*dzf(k)*dy*rhointi(kpatch)/dyint + end do + end do + ! Communicate integration between processes + call D_MPI_ALLREDUCE(boundary(2)%radcorrsingle,boundary(2)%radcorr,nypatch*nzpatch, & + MPI_SUM, commcol,mpierr) + ! Apply correction term + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + do k = 1,kmax + kpatch = k + up(i2,j,k) = up(i2,j,k) + boundary(2)%radcorr(jpatch,kpatch) + end do + end do + case(3) ! South + ! Calculate correction term for each patch + boundary(3)%radcorrsingle= 0. + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + dubdt = (boundary(3)%v(i-1,k,itp)-boundary(3)%v(i-1,k,itm))*idtb + boundary(3)%radcorrsingle(ipatch,kpatch) = boundary(3)%radcorrsingle(ipatch,kpatch) + & + rhobf(k)*(-vp(i,2,k)+dubdt)*dzf(k)*dx*rhointi(kpatch)/dxint + end do + end do + ! Communicate integration between processes + call D_MPI_ALLREDUCE(boundary(3)%radcorrsingle,boundary(3)%radcorr,nxpatch*nzpatch, & + MPI_SUM, commrow,mpierr) + ! Apply correction term + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + vp(i,2,k) = vp(i,2,k) + boundary(3)%radcorr(ipatch,kpatch) + end do + end do + case(4) ! North + ! Calculate correction term for each patch + boundary(4)%radcorrsingle = 0. + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + dubdt = (boundary(4)%v(i-1,k,itp)-boundary(4)%v(i-1,k,itm))*idtb + boundary(4)%radcorrsingle(ipatch,kpatch) = boundary(4)%radcorrsingle(ipatch,kpatch) + & + rhobf(k)*(-vp(i,j2,k)+dubdt)*dzf(k)*dx*rhointi(kpatch)/dxint + end do + end do + ! Communicate integration between processes + call D_MPI_ALLREDUCE(boundary(4)%radcorrsingle,boundary(4)%radcorr,nxpatch*nzpatch, & + MPI_SUM, commrow,mpierr) + ! Apply correction term + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do k = 1,kmax + kpatch = k + vp(i,j2,k) = vp(i,j2,k) + boundary(4)%radcorr(ipatch,kpatch) + end do + end do + case(5) ! Top + ! Calculate correction term for each patch + boundary(5)%radcorrsingle = 0. + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + dubdt = (boundary(5)%w(i-1,j-1,itp)-boundary(5)%w(i-1,j-1,itm))*idtb + boundary(5)%radcorrsingle(ipatch,jpatch) = boundary(5)%radcorrsingle(ipatch,jpatch) + & + (-wp(i,j,k1)+dubdt)*dx*dy/(dxint*dyint) + end do + end do + ! Communicate integration between processes + call D_MPI_ALLREDUCE(boundary(5)%radcorrsingle,boundary(5)%radcorr,nxpatch*nypatch, & + MPI_SUM, comm3d,mpierr) + ! Apply correction term + do i = 2,i1 + ipos = i + (myidx * imax) - 1 + ipatch = int((ipos-0.5)*dx/dxint)+1 + do j = 2,j1 + jpos = j + (myidy * jmax) - 1 + jpatch = int((jpos-0.5)*dy/dyint)+1 + wp(i,j,k1) = wp(i,j,k1) + boundary(5)%radcorr(ipatch,jpatch) + end do + end do + end select + end subroutine radcorrection + + subroutine take_prof(field0,fieldm,prof) + use modglobal, only : i1,j1,k1,ih,jh,kmax + implicit none + real(field_r), intent(inout), dimension(2-ih:i1+ih,2-jh:j1+jh,k1) :: field0,fieldm + real(field_r), intent(in), dimension(k1) :: prof + integer :: i,j,k + do k=1,kmax + do j=2,j1 + do i=2,i1 + field0(i,j,k) = prof(k) + fieldm(i,j,k) = prof(k) + end do + end do + end do + end subroutine take_prof + + subroutine handle_err(errcode) + + implicit none + + integer errcode + + write(6,*) 'Error: ', nf90_strerror(errcode) + stop 2 + + end subroutine handle_err + +end module modopenboundary diff --git a/src/modpois.f90 b/src/modpois.f90 index 8e9afa84..c2f915bc 100644 --- a/src/modpois.f90 +++ b/src/modpois.f90 @@ -46,10 +46,10 @@ module modpois contains subroutine initpois - use modglobal, only : solver_id !,i1,j1,ih,jh,kmax + use modglobal, only : solver_id,i1,j1,ih,jh,kmax,solver_id,maxiter,tolerance,precond_id,n_pre,n_post,psolver,maxiter_precond use modfft2d, only : fft2dinit use modfftw, only : fftwinit - use modhypre, only : inithypre + use modhypre, only : inithypre_grid, inithypre_solver implicit none @@ -63,19 +63,20 @@ subroutine initpois ! HYPRE based solver ! using FFT based solver as fallback - call fft2dinit(p, Fp, d, xyrt, ps,pe,qs,qe) + !call fft2dinit(p, Fp, d, xyrt, ps,pe,qs,qe) !NOTE: If you don't want to do that, you will need the line below !allocate(p(2-ih:i1+ih,2-jh:j1+jh,kmax)) - call inithypre + call inithypre_grid + call inithypre_solver(psolver,solver_id,maxiter,tolerance,precond_id,n_pre,n_post,maxiter_precond) endif end subroutine initpois subroutine exitpois - use modglobal, only : solver_id + use modglobal, only : solver_id,psolver use modfft2d, only : fft2dexit - use modhypre, only : exithypre + use modhypre, only : exithypre_grid, exithypre_solver use modfftw, only : fftwexit implicit none @@ -88,22 +89,27 @@ subroutine exitpois call fftwexit(p,Fp,d,xyrt) else ! HYPRE based solver - call fft2dexit(p,Fp,d,xyrt) - call exithypre + !call fft2dexit(p,Fp,d,xyrt) + deallocate(p) + call exithypre_grid + call exithypre_solver(psolver) endif end subroutine exitpois subroutine poisson - use modglobal, only : solver_id + use modglobal, only : solver_id,psolver use modmpi, only : myid - use modhypre, only : solve_hypre, set_initial_guess + use modhypre, only : solve_hypre, set_zero_guess use modfftw, only : fftwf, fftwb use modfft2d, only : fft2df, fft2db + use mpi implicit none - + !real wtime logical converged + !wtime = MPI_Wtime() + call fillps if (solver_id == 0) then @@ -123,27 +129,33 @@ subroutine poisson ! Backward FFT call fftwb(p, Fp) else - call solve_hypre(p, converged) + call solve_hypre(psolver, p, converged) if (.not. converged) then if (myid == 0) then - write (*,*) 'Falling back to fft2d solver.' + write (*,*) 'Not converged' endif - call fillps + !call set_zero_guess() + !call solve_hypre(psolver, p, converged) + !call fillps ! Forward FFT - call fft2df(p, Fp) + !call fft2df(p, Fp) - call solmpj + !call solmpj ! Backward FFT - call fft2db(p, Fp) + !call fft2db(p, Fp) ! Re-use our current solution as the next initial guess - call set_initial_guess(p) + !call set_initial_guess(p) endif endif call tderive + !if (myid == 0) then + ! wtime = MPI_Wtime() - wtime + ! write (*,*) 'Poisson, time spent:', wtime, 's' + !end if end subroutine poisson subroutine fillps @@ -157,8 +169,10 @@ subroutine fillps ! Adapted fillps for RK3 time loop use modfields, only : up, vp, wp, um, vm, wm, rhobf,rhobh - use modglobal, only : rk3step,i1,j1,kmax,k1,dx,dy,dzf,rdt,ih,jh + use modglobal, only : rk3step,i1,j1,kmax,k1,dx,dy,dzf,rdt,ih,jh, & + lboundary,lopenbc,i2,j2,lperiodic use modmpi, only : excjs + use modopenboundary, only : openboundary_excjs implicit none real(pois_r),allocatable :: pup(:,:,:), pvp(:,:,:), pwp(:,:,:) integer i,j,k @@ -170,9 +184,18 @@ subroutine fillps rk3coef = rdt / (4. - dble(rk3step)) + if(lopenbc) then + ! do exchanges here, because we have boundary conditions for up, vp but not for pup,pvp below + ! note: needs um, vm to be exchanged also + call openboundary_excjs(up , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(vp , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + endif + do k=1,kmax - do j=2,j1 - do i=2,i1 + do j=2,j2 ! openbc needs these to i2,j2. Periodic bc needs them to i1,j1 + do i=2,i2 pup(i,j,k) = up(i,j,k) + um(i,j,k) / rk3coef pvp(i,j,k) = vp(i,j,k) + vm(i,j,k) / rk3coef pwp(i,j,k) = wp(i,j,k) + wm(i,j,k) / rk3coef @@ -189,17 +212,22 @@ subroutine fillps !************************************************************** - do j=2,j1 do i=2,i1 pwp(i,j,1) = 0. - pwp(i,j,k1) = 0. + pwp(i,j,k1) = wp(i,j,k1) + wm(i,j,k1) / rk3coef + !pwp(i,j,k1) = 0. end do end do - call excjs(pup,2,i1,2,j1,1,kmax,ih,jh) - call excjs(pvp,2,i1,2,j1,1,kmax,ih,jh) - + ! Already implemented in loop for pup and periodicity check for up and vp for openbc + !call excjs(pup,2,i1,2,j1,1,kmax,ih,jh) + !call excjs(pvp,2,i1,2,j1,1,kmax,ih,jh) + if(.not. lopenbc) then + call excjs( pup , 2,i1,2,j1,1,kmax,ih,jh) + call excjs( pvp , 2,i1,2,j1,1,kmax,ih,jh) + endif + do k=1,kmax do j=2,j1 do i=2,i1 @@ -237,14 +265,23 @@ subroutine tderive !-----------------------------------------------------------------| use modfields, only : up, vp, wp - use modglobal, only : i1,j1,kmax,dx,dy,dzh,ih,jh + use modglobal, only : i1,j1,kmax,dx,dy,dzh,ih,jh,lopenbc,lboundary,i2,j2,k1,lperiodic use modmpi, only : excjs + use modopenboundary, only : openboundary_excjs implicit none integer i,j,k ! ** Cyclic boundary conditions ************** ! ** set by the commcart communication in excj - call excjs( p, 2,i1,2,j1,1,kmax,ih,jh) + if(lopenbc) then ! If openboundaries are used only cyclic conditions for non-domain boundaries, hom. Neumann otherwise. + call openboundary_excjs(p , 2,i1,2,j1,1,kmax,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + if(lboundary(1).and. .not. lperiodic(1)) p(1,:,:) = p(2,:,:) + if(lboundary(2).and. .not. lperiodic(2)) p(i2,:,:) = p(i1,:,:) + if(lboundary(3).and. .not. lperiodic(3)) p(:,1,:) = p(:,2,:) + if(lboundary(4).and. .not. lperiodic(4)) p(:,j2,:) = p(:,j1,:) + else + call excjs( p, 2,i1,2,j1,1,kmax,ih,jh) + endif !***************************************************************** ! ** Calculate time-derivative for the velocities with known **** diff --git a/src/modstartup.f90 b/src/modstartup.f90 index 2eb79fd2..0fde6ca2 100644 --- a/src/modstartup.f90 +++ b/src/modstartup.f90 @@ -70,15 +70,18 @@ subroutine startup(path) lmoist,lcoriol,lpressgrad,igrw_damp,geodamptime,lmomsubs,cu, cv,ifnamopt,fname_options,llsadv,& ibas_prf,lambda_crit,iadv_mom,iadv_tke,iadv_thl,iadv_qt,iadv_sv,courant,peclet,ladaptive,author,& lnoclouds,lfast_thermo,lrigidlid,unudge,ntimedep,& - solver_id, maxiter, tolerance, n_pre, n_post, precond, checknamelisterror, loutdirs, output_prefix + solver_id, maxiter, maxiter_precond, tolerance, n_pre, n_post, precond_id, checknamelisterror, & + loutdirs, output_prefix, & + lopenbc,linithetero,lperiodic,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,nmodes,tau,lambda,lambdas,lambdas_x,lambdas_y,lambdas_z,iturb, & + hypre_logging,rdt,rk3step,i1,j1,k1,ih,jh,lboundary,lconstexner use modforces, only : lforce_user use modsurfdata, only : z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,isurf use modsurface, only : initsurface use moddatetime, only : initdatetime use modemission, only : initemission use modlsm, only : initlsm, kmax_soil - use modfields, only : initfields - use modpois, only : initpois + use modfields, only : initfields,um,vm,wm,u0,v0,w0,up,vp,wp + use modpois, only : initpois,poisson use modradiation, only : initradiation use modraddata, only : irad,iradiation,& rad_ls,rad_longw,rad_shortw,rad_smoke,useMcICA,& @@ -90,14 +93,18 @@ subroutine startup(path) use modthermodynamics, only : initthermodynamics,lqlnr, chi_half use modmicrophysics, only : initmicrophysics use modsubgrid, only : initsubgrid - use modmpi, only : initmpi,commwrld,myid,myidx,cmyidy,nprocx,nprocy,mpierr & + use modmpi, only : initmpi,commwrld,myid,myidx,cmyidy,nprocx,nprocy,mpierr,periods & , D_MPI_BCAST use modchem, only : initchem use modversion, only : git_version + use modopenboundary, only : initopenboundary,openboundary_divcorr,openboundary_excjs + use modchecksim, only : chkdiv implicit none integer :: ierr + logical,dimension(2) :: lper = .false. character(256), optional, intent(in) :: path + real rk3coef !declare namelists namelist/RUN/ & @@ -112,12 +119,16 @@ subroutine startup(path) namelist/PHYSICS/ & !cstep z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,chi_half,lmoist,isurf,lneutraldrag,& z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,lmoist,isurf,chi_half,& - lcoriol,lpressgrad,igrw_damp,geodamptime,lmomsubs,ltimedep,ltimedepuv,ltimedepsv,ntimedep,irad,timerad,iradiation,rad_ls,rad_longw,rad_shortw,rad_smoke,useMcICA,& - rka,dlwtop,dlwbot,sw0,gc,reff,isvsmoke,lforce_user,lcloudshading,lrigidlid,unudge,lfast_thermo + lcoriol,lpressgrad,igrw_damp,geodamptime,lmomsubs,ltimedep,ltimedepuv,ltimedepsv,& + ntimedep,irad,timerad,iradiation,rad_ls,rad_longw,rad_shortw,rad_smoke,useMcICA,& + rka,dlwtop,dlwbot,sw0,gc,reff,isvsmoke,lforce_user,lcloudshading,lrigidlid,unudge,& + lfast_thermo,lconstexner namelist/DYNAMICS/ & llsadv, lqlnr, lambda_crit, cu, cv, ibas_prf, iadv_mom, iadv_tke, iadv_thl, iadv_qt, iadv_sv, lnoclouds namelist/SOLVER/ & - solver_id, maxiter, tolerance, n_pre, n_post, precond + solver_id, maxiter, tolerance, n_pre, n_post, precond_id, maxiter_precond, hypre_logging + namelist/OPENBC/ & + lopenbc,linithetero,lper,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,iturb,tau,lambda,nmodes,lambdas,lambdas_x,lambdas_y,lambdas_z ! get myid @@ -159,12 +170,24 @@ subroutine startup(path) read (ifnamopt,SOLVER,iostat=ierr) call checknamelisterror(ierr, ifnamopt, 'SOLVER') write(6 ,SOLVER) + rewind(ifnamopt) + read (ifnamopt,OPENBC,iostat=ierr) + call checknamelisterror(ierr, ifnamopt, 'OPENBC') + write(6 ,OPENBC) + close(ifnamopt) + if(lopenbc) then + ! Check if grid needs to be periodic + periods = (/lper(1),lper(2)/) + lperiodic(1:2) = lper(1) + lperiodic(3:4) = lper(2) + endif close(ifnamopt) end if ! these must be shared before initmpi sets up the cartesian grid ! commwrld is already set up + call D_MPI_BCAST(periods,2,0,commwrld,mpierr) call D_MPI_BCAST(nprocx ,1,0,commwrld,mpierr) call D_MPI_BCAST(nprocy ,1,0,commwrld,mpierr) @@ -272,7 +295,28 @@ subroutine startup(path) call D_MPI_BCAST(n_pre,1,0,commwrld,mpierr) call D_MPI_BCAST(n_post,1,0,commwrld,mpierr) call D_MPI_BCAST(tolerance,1,0,commwrld,mpierr) - call D_MPI_BCAST(precond,1,0,commwrld,mpierr) + call D_MPI_BCAST(precond_id,1,0,commwrld,mpierr) + call D_MPI_BCAST(maxiter_precond,1,0,commwrld,mpierr) + + ! Broadcast openboundaries Variables + call D_MPI_BCAST(lopenbc, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(linithetero,1, 0,commwrld,mpierr) + call D_MPI_BCAST(lperiodic, 5, 0,commwrld,mpierr) + call D_MPI_BCAST(dxint, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(dyint, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(dzint, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(taum, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(tauh, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(pbc, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lsynturb, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(iturb, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lambda, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(tau, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(nmodes, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lambdas, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lambdas_x, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lambdas_y, 1, 0,commwrld,mpierr) + call D_MPI_BCAST(lambdas_z, 1, 0,commwrld,mpierr) call testwctime ! Allocate and initialize core modules @@ -280,7 +324,11 @@ subroutine startup(path) call initfields call inittestbed !reads initial profiles from scm_in.nc, to be used in readinitfiles - call initboundary + if(.not.lopenbc) then + call initboundary + else + call initopenboundary + endif call initthermodynamics call initradiation call initchem @@ -303,6 +351,28 @@ subroutine startup(path) call readinitfiles ! moved to obtain the correct btime for the timedependent forcings in case of a warmstart call inittimedep !depends on modglobal,modfields, modmpi, modsurf, modradiation call initpois ! hypre solver needs grid and baseprofiles + if(lopenbc) then ! Correct boundaries and initial field for divergence + call chkdiv + call openboundary_divcorr ! Remove divergence from large scale input + ! Use poisson solver to get rid of divergence in initial field, needs to + ! be here to avoid cross dependencies between modopenbondaries and modpois + if(myid==0) print *, 'Start divergence correction initial field' + call chkdiv + up = 0.; vp = 0.; wp = 0. ! Set tendencies to zero + call poisson + rk3coef = rdt / (4. - dble(rk3step)) + um = um + rk3coef * up + vm = vm + rk3coef * vp + wm = wm + rk3coef * wp + call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + u0 = um; v0 = vm; w0 = wm + call chkdiv + ! Reset tendencies + up = 0.; vp = 0.; wp = 0. + if(myid==0) print *, 'Finished divergence correction initial field' + endif call checkinitvalues @@ -411,7 +481,7 @@ subroutine readinitfiles rtimee,timee,ntrun,btime,dt_lim,nsv,& zf,dzf,dzh,rv,rd,cp,rlv,pref0,om23_gs,& ijtot,cu,cv,e12min,dzh,cexpnr,ifinput,lwarmstart,ltotruntime,itrestart,& - trestart, ladaptive,llsadv,tnextrestart,longint + trestart, ladaptive,llsadv,tnextrestart,longint,lopenbc,linithetero use modsubgrid, only : ekm,ekh use modsurfdata, only : wsvsurf, & thls,tskin,tskinm,tsoil,tsoilm,phiw,phiwm,Wl,Wlm,thvs,qts,isurf,svs,obl,oblav,& @@ -425,6 +495,7 @@ subroutine readinitfiles use modtestbed, only : ltestbed,tb_ps,tb_thl,tb_qt,tb_u,tb_v,tb_w,tb_ug,tb_vg,& tb_dqtdxls,tb_dqtdyls,tb_qtadv,tb_thladv + use modopenboundary, only : openboundary_ghost,openboundary_readboundary,openboundary_initfields integer i,j,k,n,ierr logical negval !switch to allow or not negative values in randomnization @@ -473,7 +544,6 @@ subroutine readinitfiles !thls !wtsurf !wqsurf - else open (ifinput,file='prof.inp.'//cexpnr,status='old',iostat=ierr) if (ierr /= 0) then @@ -524,50 +594,6 @@ subroutine readinitfiles call D_MPI_BCAST(uprof ,kmax,0,comm3d,mpierr) call D_MPI_BCAST(vprof ,kmax,0,comm3d,mpierr) call D_MPI_BCAST(e12prof,kmax,0,comm3d,mpierr) - do k=1,kmax - do j=1,j2 - do i=1,i2 - thl0(i,j,k) = thlprof(k) - thlm(i,j,k) = thlprof(k) - qt0 (i,j,k) = qtprof (k) - qtm (i,j,k) = qtprof (k) - u0 (i,j,k) = uprof (k) - cu - um (i,j,k) = uprof (k) - cu - v0 (i,j,k) = vprof (k) - cv - vm (i,j,k) = vprof (k) - cv - w0 (i,j,k) = 0.0 - wm (i,j,k) = 0.0 - e120(i,j,k) = e12prof(k) - e12m(i,j,k) = e12prof(k) - ekm (i,j,k) = 0.0 - ekh (i,j,k) = 0.0 - end do - end do - end do - !--------------------------------------------------------------- - ! 1.2 randomnize fields - !--------------------------------------------------------------- - - krand = min(krand,kmax) - negval = .False. ! No negative perturbations for qt (negative moisture is non physical) - do k = 1,krand - call randomnize(qtm ,k,randqt ,irandom,ih,jh,negval) - call randomnize(qt0 ,k,randqt ,irandom,ih,jh,negval) - end do - negval = .True. ! negative perturbations allowed - do k = 1,krand - call randomnize(thlm,k,randthl,irandom,ih,jh,negval) - call randomnize(thl0,k,randthl,irandom,ih,jh,negval) - end do - - do k=krandumin,krandumax - call randomnize(um ,k,randu ,irandom,ih,jh,negval) - call randomnize(u0 ,k,randu ,irandom,ih,jh,negval) - call randomnize(vm ,k,randu ,irandom,ih,jh,negval) - call randomnize(v0 ,k,randu ,irandom,ih,jh,negval) - call randomnize(wm ,k,randu ,irandom,ih,jh,negval) - call randomnize(w0 ,k,randu ,irandom,ih,jh,negval) - end do svprof = 0. if(myid==0)then @@ -596,17 +622,81 @@ subroutine readinitfiles end if ! end if myid==0 call D_MPI_BCAST(wsvsurf,nsv ,0,comm3d,mpierr) - call D_MPI_BCAST(svprof ,k1*nsv,0,comm3d,mpierr) - do k=1,kmax + ! Initialize fields + if(lopenbc .and. linithetero) then! Openboundaries with heterogeneous initialisation + call openboundary_initfields() + do j = 1,j2 + do i = 1,i2 + wm(i,j,1) = 0. + w0(i,j,1) = 0. + end do + end do + do k = 1,kmax + do j = 1,j2 + do i = 1,i2 + ekm(i,j,k) = 0.0 + ekh(i,j,k) = 0.0 + end do + end do + end do + else + do k=1,kmax do j=1,j2 - do i=1,i2 - do n=1,nsv - sv0(i,j,k,n) = svprof(k,n) - svm(i,j,k,n) = svprof(k,n) + do i=1,i2 + thl0(i,j,k) = thlprof(k) + thlm(i,j,k) = thlprof(k) + qt0 (i,j,k) = qtprof (k) + qtm (i,j,k) = qtprof (k) + u0 (i,j,k) = uprof (k) - cu + um (i,j,k) = uprof (k) - cu + v0 (i,j,k) = vprof (k) - cv + vm (i,j,k) = vprof (k) - cv + w0 (i,j,k) = 0.0 + wm (i,j,k) = 0.0 + e120(i,j,k) = e12prof(k) + e12m(i,j,k) = e12prof(k) + ekm (i,j,k) = 0.0 + ekh (i,j,k) = 0.0 + end do + end do + end do + if(nsv>0) then + do k=1,kmax + do j=1,j2 + do i=1,i2 + do n=1,nsv + sv0(i,j,k,n) = svprof(k,n) + svm(i,j,k,n) = svprof(k,n) + end do + end do end do end do - end do + endif + endif + !--------------------------------------------------------------- + ! 1.2 randomnize fields + !--------------------------------------------------------------- + + krand = min(krand,kmax) + negval = .False. ! No negative perturbations for qt (negative moisture is non physical) + do k = 1,krand + call randomnize(qtm ,k,randqt ,irandom,ih,jh,negval) + call randomnize(qt0 ,k,randqt ,irandom,ih,jh,negval) + end do + negval = .True. ! negative perturbations allowed + do k = 1,krand + call randomnize(thlm,k,randthl,irandom,ih,jh,negval) + call randomnize(thl0,k,randthl,irandom,ih,jh,negval) + end do + + do k=krandumin,krandumax + call randomnize(um ,k,randu ,irandom,ih,jh,negval) + call randomnize(u0 ,k,randu ,irandom,ih,jh,negval) + call randomnize(vm ,k,randu ,irandom,ih,jh,negval) + call randomnize(v0 ,k,randu ,irandom,ih,jh,negval) + call randomnize(wm ,k,randu ,irandom,ih,jh,negval) + call randomnize(w0 ,k,randu ,irandom,ih,jh,negval) end do !----------------------------------------------------------------- @@ -636,6 +726,10 @@ subroutine readinitfiles case(10) call initsurf_user end select + if(lopenbc) then + call openboundary_readboundary + call openboundary_ghost + endif ! Set initial Obukhov length to -0.1 for iteration obl = -0.1 @@ -652,7 +746,11 @@ subroutine readinitfiles svs = svprof(1,:) call baseprofs ! call baseprofs before thermodynamics - call boundary + if(lopenbc) then + call openboundary_ghost() + else + call boundary + endif call thermodynamics call surface @@ -663,7 +761,11 @@ subroutine readinitfiles ! dsv(n) = (svprof(kmax,n)-svprof(kmax-1,n)) / dzh(kmax) ! end do - call boundary + if(lopenbc) then + call openboundary_ghost + else + call boundary + endif call thermodynamics ! save initial pressure profiles @@ -685,8 +787,8 @@ subroutine readinitfiles svm = sv0 e12m = e120 call calc_halflev - exnf = (presf/pref0)**(rd/cp) - exnh = (presh/pref0)**(rd/cp) + exnf = (initial_presf/pref0)**(rd/cp) + exnh = (initial_presh/pref0)**(rd/cp) do j=2,j1 do i=2,i1 @@ -747,6 +849,10 @@ subroutine readinitfiles ! CvH - only do this for fixed timestepping. In adaptive dt comes from restartfile if(ladaptive .eqv. .false.) rdt=dtmax + if(lopenbc) then + call openboundary_readboundary + endif + end if ! end if (.not. warmstart) !----------------------------------------------------------------- @@ -1249,7 +1355,7 @@ end subroutine testwctime subroutine exitmodules use modfields, only : exitfields - use modglobal, only : exitglobal + use modglobal, only : exitglobal,lopenbc use modmpi, only : exitmpi use modboundary, only : exitboundary use modmicrophysics, only : exitmicrophysics @@ -1261,6 +1367,7 @@ subroutine exitmodules use modlsm, only : exitlsm use modthermodynamics, only : exitthermodynamics use modemission, only : exitemission + use modopenboundary, only : exitopenboundary call exittimedep call exitthermodynamics @@ -1271,7 +1378,11 @@ subroutine exitmodules call exitpois call exitmicrophysics call exitemission - call exitboundary + if(lopenbc) then + call exitopenboundary + else + call exitboundary + endif call exitfields call exitglobal call exitmpi diff --git a/src/modsurface.f90 b/src/modsurface.f90 index 10d81e9a..e862337b 100644 --- a/src/modsurface.f90 +++ b/src/modsurface.f90 @@ -165,9 +165,9 @@ subroutine initsurface call D_MPI_BCAST(phiwp , 1, 0, comm3d, mpierr) call D_MPI_BCAST(R10 , 1, 0, comm3d, mpierr) call D_MPI_BCAST(lsplitleaf , 1, 0, comm3d, mpierr) - + call D_MPI_BCAST(land_use(1:mpatch,1:mpatch),mpatch*mpatch, 0, comm3d, mpierr) - + call D_MPI_BCAST(i_expemis , 1, 0, comm3d, mpierr) call D_MPI_BCAST(expemis0 , 1, 0, comm3d, mpierr) call D_MPI_BCAST(expemis1 , 1, 0, comm3d, mpierr) @@ -708,11 +708,12 @@ end subroutine initsurface !> Calculates the interaction with the soil, the surface temperature and humidity, and finally the surface fluxes. subroutine surface - use modglobal, only : i1,j1,i2,j2,fkar,zf,cu,cv,nsv,ijtot,rd,rv,rtimee + use modglobal, only : i1,j1,i2,j2,fkar,zf,cu,cv,nsv,ijtot,rd,rv,rtimee,lopenbc,lboundary,lperiodic use modfields, only : thl0, qt0, u0, v0, u0av, v0av use modmpi, only : mpierr, comm3d, mpi_sum, excjs & , D_MPI_ALLREDUCE, D_MPI_BCAST use moduser, only : surf_user + use modopenboundary, only : openboundary_excjs implicit none integer :: i, j, n, patchx, patchy @@ -1050,7 +1051,15 @@ subroutine surface ! Transfer ustar to neighbouring cells, do this like a 3D field ustar_3D(1:i2,1:j2,1:1) => ustar - call excjs(ustar_3D,2,i1,2,j1,1,1,1,1) + + if(lopenbc) then ! Only use periodicity for non-domain boundaries when openboundaries are used + call openboundary_excjs(ustar_3D, 2,i1,2,j1,1,1,1,1, & + & (.not.lboundary(1:4)).or.lperiodic(1:4)) + else + call excjs(ustar_3D,2,i1,2,j1,1,1,1,1) + !call excjs(ustar, 2,i1,2,j1,1,1,1,1) + endif + return end subroutine surface !> Calculate the surface humidity assuming saturation. diff --git a/src/modsynturb.f90 b/src/modsynturb.f90 new file mode 100644 index 00000000..3d022cfe --- /dev/null +++ b/src/modsynturb.f90 @@ -0,0 +1,757 @@ +!> \file modopenboundary.f90 +!! Creates synthetic turbulence for the open boundary implementation +!> +!! Creates synthetic turbulence for the open boundary implementation +!> +!! \author Frans Liqui Lung +! This file is part of DALES. +! To do: +! +! DALES is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! DALES is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License +! along with this program. If not, see . +! +! Copyright 1993-2009 Delft University of Technology, Wageningen University, Utrecht University, KNMI +! +module modsynturb +use netcdf +use modglobal,only: lsynturb,iturb,lboundary,lperiodic,boundary,nmodes,lambda,tau,dxint,dyint,itot,jtot,dx,dy,kmax +use RandomNumbers, only : getRandomReal,randomNumberSequence,new_RandomNumberSequence +use modprecision, only: field_r +implicit none +real, allocatable, dimension(:,:) :: kn,p,q,vturb,wturb,k_thl,k_qt +real, allocatable, dimension(:) :: omega,omega_thl,omega_qt,p_thl,p_qt,q_thl,q_qt +real, allocatable, dimension(:) :: xf,xh,yf,yh +real :: nisqrt,ctot,nisqrt2 +real, dimension(3) :: lambdasxyz +integer :: nxpatch,nypatch,nzpatch +integer, parameter :: isepsim_mom = 10,isepsim_all=11, isynturb_mom = 0, isynturb_all = 1 +integer :: ntturb,itimestep=1 +real, allocatable, dimension(:) :: tturb +real, allocatable, dimension(:,:,:) :: uturbin,vturbin,wturbin,thlturbin,qtturbin +character(len = nf90_max_name) :: RecordDimName +integer :: VARID,STATUS,NCID,mpierr,timeID +! ! Uncommend for netcdf output turbulent pertubations west boundary +! character (80) :: fname = 'turbOut.xxx.xxx.nc' +! integer :: ncid +! integer, parameter :: NDIMS = 3 +! character (len = *), parameter :: Z_NAME = "z" +! character (len = *), parameter :: Y_NAME = "y" +! character (len = *), parameter :: t_NAME = "time" +! integer :: z_dimid, y_dimid, t_dimid +! integer :: z_varid, y_varid, t_varid +! integer :: uturb_varid, vturb_varid, wturb_varid, thlturb_varid, qtturb_varid +! integer :: dimids(NDIMS) +! integer :: start(NDIMS), count(NDIMS) + +type(randomNumberSequence) :: noise +contains + subroutine initsynturb + use netcdf + use modglobal, only : dx,dy,imax,jmax,i1,j1,zf,lambdas,lambdas_x,lambdas_y,lambdas_z,kmax,k1,cexpnr,lmoist + use modmpi, only : myidx, myidy + implicit none + integer :: i,j,ib + if(.not.lsynturb) return + if(any(lboundary.and..not.(lperiodic))) then + if(iturb == isynturb_all .or. iturb == isynturb_mom) then + ! Constants + nisqrt = sqrt(2./nmodes) + nisqrt2 = sqrt(1./nmodes) + ctot = lambda/tau + noise = new_RandomNumberSequence(seed = 100) + nxpatch = int(dx/dxint*real(itot)); + nypatch = int(dy/dyint*real(jtot)); + nzpatch = kmax + lambdas = merge(lambda,lambdas,lambdas==-1.) + lambdasxyz = (/merge(lambdas,lambdas_x,lambdas_x==-1.), & + & merge(lambdas,lambdas_y,lambdas_y==-1.), & + & merge(lambdas,lambdas_z,lambdas_z==-1.)/) + ! Allocate variables + allocate(kn(nmodes,3),q(nmodes,3),p(nmodes,3),omega(nmodes),& + xf(imax),xh(i1),yf(jmax),yh(j1),k_thl(nmodes,3),k_qt(nmodes,3), & + omega_thl(nmodes),omega_qt(nmodes), & + p_thl(nmodes),p_qt(nmodes),q_thl(nmodes),q_qt(nmodes)) + ! allocate(vturb(jmax,kmax),wturb(jmax,kmax)) + ! Calculate coordinates + xf = (/((i-0.5)*dx,i=1,imax,1)/)+imax*myidx*dx + xh = (/(i*dx,i=1,i1,1)/)+imax*myidx*dx + yf = (/((i-0.5)*dy,i=1,jmax,1)/)+jmax*myidy*dy + yh = (/(i*dy,i=1,j1,1)/)+jmax*myidy*dy + ! Fill random distributed variables + do i = 1,3 + do j = 1,nmodes + kn(j,i) = gaussrand(0.,0.5) + k_thl(j,i) = gaussrand(0.,0.5) + k_qt(j,i) = gaussrand(0.,0.5) + if(i==1) then + omega(j) = gaussrand(0.,1.) + omega_thl(j) = gaussrand(0.,1.) + omega_qt(j) = gaussrand(0.,1.) + p_thl(j) = gaussrand(0.,1.) + q_thl(j) = gaussrand(0.,1.) + p_qt(j) = gaussrand(0.,1.) + q_qt(j) = gaussrand(0.,1.) + endif + end do + end do + ! Obtain p and q with cross product + do i = 1,nmodes + p(i,:) = cross((/gaussrand(0.,1.),gaussrand(0.,1.),gaussrand(0.,1.)/),kn(i,:)) + q(i,:) = cross((/gaussrand(0.,1.),gaussrand(0.,1.),gaussrand(0.,1.)/),kn(i,:)) + end do + do ib = 1,5 + if(.not.lboundary(ib)) cycle + allocate(boundary(ib)%eigvec(boundary(ib)%nx1patch,boundary(ib)%nx2patch,3,3), & + & boundary(ib)%ci(boundary(ib)%nx1patch,boundary(ib)%nx2patch,3))!, & + ! & boundary(ib)%randthl(boundary(ib)%nx1,boundary(ib)%nx2), & + ! & boundary(ib)%randqt(boundary(ib)%nx1,boundary(ib)%nx2)) + end do + elseif((iturb == isepsim_mom .or. iturb == isepsim_all) .and. lboundary(1)) then + !--- open nc file --- + STATUS = NF90_OPEN('turb.inp.'//cexpnr//'.nc', nf90_nowrite, NCID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + !--- get time dimensions + status = nf90_inq_dimid(ncid, "time", timeID) + if (status /= nf90_noerr) call handle_err(status) + status = nf90_inquire_dimension(NCID, timeID, len=ntturb, name=RecordDimName) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + !--- read time + allocate(tturb(ntturb)) + STATUS = NF90_INQ_VARID(NCID, 'time', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, tturb, start=(/1/), count=(/ntturb/) ) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + ! --- allocate fields + allocate(uturbin(jmax,kmax,ntturb)) + allocate(vturbin(j1,kmax,ntturb)) + allocate(wturbin(jmax,k1,ntturb)) + ! --- read fields + ! Read u + STATUS = NF90_INQ_VARID(NCID,'uturbwest', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, uturbin, start=(/myidy*jmax+1,1,1/), & + & count=(/jmax,kmax,ntturb/)) + ! Read v + STATUS = NF90_INQ_VARID(NCID,'vturbwest', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, vturbin, start=(/myidy*jmax+1,1,1/), & + & count=(/j1,kmax,ntturb/)) + ! Read w + STATUS = NF90_INQ_VARID(NCID,'wturbwest', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, wturbin, start=(/myidy*jmax+1,1,1/), & + & count=(/jmax,k1,ntturb/)) + if(iturb==isepsim_all) then + allocate(thlturbin(jmax,kmax,ntturb)) + ! Read thl + STATUS = NF90_INQ_VARID(NCID,'thlturbwest', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, thlturbin, start=(/myidy*jmax+1,1,1/), & + & count=(/jmax,kmax,ntturb/)) + if(lmoist) then + allocate(qtturbin(jmax,kmax,ntturb)) + ! qt + STATUS = NF90_INQ_VARID(NCID,'wturbwest', VARID) + if (STATUS .ne. nf90_noerr) call handle_err(STATUS) + STATUS = NF90_GET_VAR (NCID, VARID, wturbin, start=(/myidy*jmax+1,1,1/), & + & count=(/jmax,k1,ntturb/)) + endif + endif + endif + ! ! Uncommend for netcdf output turbulent pertubations west boundary + ! if(lboundary(1)) then + ! write(fname,'(A,i3.3,A,i3.3,A)') 'turbOut.', myidx, '.', myidy, '.nc' + ! call check( nf90_create(fname, nf90_clobber, ncid) ) + ! call check( nf90_def_dim(ncid, z_NAME, kmax, z_dimid) ) + ! call check( nf90_def_dim(ncid, y_NAME, jmax, y_dimid) ) + ! call check( nf90_def_dim(ncid, t_NAME, NF90_UNLIMITED, t_dimid) ) + ! call check( nf90_def_var(ncid, z_NAME, NF90_REAL, z_dimid, z_varid) ) + ! call check( nf90_def_var(ncid, y_NAME, NF90_REAL, y_dimid, y_varid) ) + ! call check( nf90_def_var(ncid, t_NAME, NF90_REAL, t_dimid, t_varid) ) + ! dimids = (/y_dimid, z_dimid, t_dimid/) + ! call check( nf90_def_var(ncid, 'u', NF90_REAL, dimids, uturb_varid) ) + ! call check( nf90_def_var(ncid, 'v', NF90_REAL, dimids, vturb_varid) ) + ! call check( nf90_def_var(ncid, 'w', NF90_REAL, dimids, wturb_varid) ) + ! call check( nf90_def_var(ncid, 'thl', NF90_REAL, dimids, thlturb_varid) ) + ! call check( nf90_def_var(ncid, 'qt', NF90_REAL, dimids, qtturb_varid) ) + ! call check( nf90_enddef(ncid) ) + ! call check( nf90_put_var(ncid, z_varid, zf(1:kmax)) ) + ! call check( nf90_put_var(ncid, y_varid, yf) ) + ! count = (/ jmax, kmax, 1 /) + ! start = (/ 1, 1, 1 /) + ! endif + endif + end subroutine initsynturb + + subroutine handle_err(errcode) + + implicit none + + integer errcode + + write(6,*) 'Error: ', nf90_strerror(errcode) + stop 2 + + end subroutine handle_err + + subroutine exitsynturb + use modglobal, only : lmoist + implicit none + integer :: ib + if(.not.lsynturb) return + ! ! Uncommend for netcdf output turbulent pertubations + ! if(lboundary(1)) call check( nf90_close(ncid) ) + if(any(lboundary.and..not.(lperiodic))) then + if(iturb == isynturb_mom .or. iturb == isynturb_all) then + do ib = 1,5 + if(.not.lboundary(ib)) cycle + deallocate(boundary(ib)%u2,boundary(ib)%v2,boundary(ib)%w2,& + & boundary(ib)%uv,boundary(ib)%uw,boundary(ib)%vw,& + & boundary(ib)%thl2,boundary(ib)%qt2,boundary(ib)%wthl, & + & boundary(ib)%wqt,boundary(ib)%eigvec,boundary(ib)%ci) + end do + !deallocate(vturb,wturb) + deallocate(kn,q,p,omega,xf,xh,yf,yh,k_thl,k_qt,omega_thl,omega_qt,p_thl,p_qt,q_thl,q_qt) + elseif(iturb == isepsim_mom .and. lboundary(1)) then + deallocate(uturbin,vturbin,wturbin) + elseif(iturb == isepsim_all .and. lboundary(1)) then + deallocate(uturbin,vturbin,wturbin,thlturbin) + if(lmoist) deallocate(qtturbin) + endif + endif + end subroutine exitsynturb + + ! ! Uncommend for netcdf output turbulent pertubations + ! subroutine check(status) + ! integer, intent ( in) :: status + ! + ! if(status /= nf90_noerr) then + ! print *, trim(nf90_strerror(status)) + ! stop "Stopped" + ! end if + ! end subroutine check + + subroutine synturb() + implicit none + if(.not.lsynturb) return + select case(iturb) + case(isynturb_mom) + call synturb_mom() + case(isynturb_all) + call synturb_all() + case(isepsim_mom, isepsim_all) + call sepsim + end select + end subroutine synturb + + subroutine synturb_mom() + use modglobal, only : dx,dy,itot,jtot,zh,zf,imax,jmax,i1,j1,kmax,k1 + implicit none + integer :: ib + do ib = 1,5 + if(.not.lboundary(ib).or.lperiodic(ib)) cycle + call calc_eigdec(ib) + select case(ib) + case(1) + call calc_pert(ib,(/0./),yf,zf,1,jmax,kmax,boundary(ib)%uturb,1) + call calc_pert(ib,(/0./),yh,zf,1,j1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,(/0./),yf,zh,1,jmax,k1,boundary(ib)%wturb,3) + case(2) + call calc_pert(ib,(/(itot+1)*dx/),yf,zf,1,jmax,kmax,boundary(ib)%uturb,1) + call calc_pert(ib,(/(itot+1)*dx/),yh,zf,1,j1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,(/(itot+1)*dx/),yf,zh,1,jmax,k1,boundary(ib)%wturb,3) + case(3) + call calc_pert(ib,xh,(/0./),zf,i1,1,kmax,boundary(ib)%uturb,1) + call calc_pert(ib,xf,(/0./),zf,imax,1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,xh,(/0./),zh,imax,1,k1,boundary(ib)%wturb,1) + case(4) + call calc_pert(ib,xh,(/(jtot+1)*dy/),zf,i1,1,kmax,boundary(ib)%uturb,1) + call calc_pert(ib,xf,(/(jtot+1)*dy/),zf,imax,1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,xh,(/(jtot+1)*dy/),zh,imax,1,k1,boundary(ib)%wturb,1) + case(5) + call calc_pert(ib,xh,yf,(/zh(k1)/),i1,jmax,1,boundary(ib)%uturb,1) + call calc_pert(ib,xf,yh,(/zh(k1)/),imax,j1,1,boundary(ib)%vturb,2) + call calc_pert(ib,xf,yf,(/zh(k1)/),imax,jmax,1,boundary(ib)%wturb,3) + end select + end do + end subroutine synturb_mom + + subroutine synturb_all() + use modglobal, only : dx,dy,itot,jtot,zh,zf,imax,jmax,i1,j1,kmax,k1 + implicit none + integer :: ib + do ib = 1,5 + if(.not.lboundary(ib).or.lperiodic(ib)) cycle + call calc_eigdec(ib) + select case(ib) + case(1) + call calc_pert2(ib,(/0./),yf,zf,1,jmax,kmax,boundary(ib)%uturb,1, & + & boundary(ib)%thlturb,boundary(ib)%qtturb) + call calc_pert(ib,(/0./),yh,zf,1,j1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,(/0./),yf,zh,1,jmax,k1,boundary(ib)%wturb,3) + case(2) + call calc_pert2(ib,(/(itot+1)*dx/),yf,zf,1,jmax,kmax,boundary(ib)%uturb,1, & + & boundary(ib)%thlturb,boundary(ib)%qtturb) + call calc_pert(ib,(/(itot+1)*dx/),yh,zf,1,j1,kmax,boundary(ib)%vturb,2) + call calc_pert(ib,(/(itot+1)*dx/),yf,zh,1,jmax,k1,boundary(ib)%wturb,3) + case(3) + call calc_pert(ib,xh,(/0./),zf,i1,1,kmax,boundary(ib)%uturb,1) + call calc_pert2(ib,xf,(/0./),zf,imax,1,kmax,boundary(ib)%vturb,2, & + & boundary(ib)%thlturb,boundary(ib)%qtturb) + call calc_pert(ib,xh,(/0./),zh,imax,1,k1,boundary(ib)%wturb,1) + case(4) + call calc_pert(ib,xh,(/(jtot+1)*dy/),zf,i1,1,kmax,boundary(ib)%uturb,1) + call calc_pert2(ib,xf,(/(jtot+1)*dy/),zf,imax,1,kmax,boundary(ib)%vturb,2, & + & boundary(ib)%thlturb,boundary(ib)%qtturb) + call calc_pert(ib,xh,(/(jtot+1)*dy/),zh,imax,1,k1,boundary(ib)%wturb,1) + case(5) + call calc_pert(ib,xh,yf,(/zh(k1)/),i1,jmax,1,boundary(ib)%uturb,1) + call calc_pert(ib,xf,yh,(/zh(k1)/),imax,j1,1,boundary(ib)%vturb,2) + call calc_pert2(ib,xf,yf,(/zh(k1)/),imax,jmax,1,boundary(ib)%wturb,3, & + boundary(ib)%thlturb,boundary(ib)%qtturb) + end select + end do + end subroutine synturb_all + + subroutine sepsim() + use modglobal, only : rtimee,lmoist + use modmpi, only : myid + implicit none + integer :: ib + do ib = 1,5 + if(.not.lboundary(ib).or.lperiodic(ib)) cycle + select case(ib) + case(1) + do while(tturb(itimestep)0.01 .and. real(rtimee)>5.) then + print *, 'mistake in time', itimestep,tturb(itimestep),real(rtimee) + stop + endif + boundary(ib)%uturb = uturbin(:,:,itimestep) + boundary(ib)%vturb = vturbin(:,:,itimestep) + boundary(ib)%wturb = wturbin(:,:,itimestep) + if(iturb==11) then + boundary(ib)%thlturb = thlturbin(:,:,itimestep) + if(lmoist) boundary(ib)%qtturb = qtturbin(:,:,itimestep) + endif + case(2) + ! Do nothing (needs to be added) + case(3) + ! Do nothing (needs to be added) + case(4) + ! Do nothing (needs to be added) + case(5) + ! Do nothing (needs to be added) + end select + end do + end subroutine sepsim + + subroutine calc_eigdec(ib) + use modglobal, only : rtimee,tboundary,ntboundary + implicit none + integer, intent(in) :: ib + real*8,dimension(3,3) :: r,eigvec + real*8,dimension(3) :: eigval + integer :: i,j,itp,itm,ii + real :: fm,fp + ! Interpolate covariance to current time + itm = 1 + if(ntboundary>1) then + do while(rtimee>tboundary(itm)) + itm = itm+1 + end do + if (rtimee>tboundary(1)) then + itm = itm-1 + end if + itp = itm+1 + fm = (tboundary(itp)-rtimee)/(tboundary(itp)-tboundary(itm)) + fp = (rtimee-tboundary(itm))/(tboundary(itp)-tboundary(itm)) + else + itm = 1; itp = 1; fp = 1.; fm = 1. + endif + ! Calculate eigenvalues and eigenvectors for each patch + do i = 1,boundary(ib)%nx1patch + do j = 1,boundary(ib)%nx2patch + r(1,1) = fp*boundary(ib)%u2(i,j,itp)+fm*boundary(ib)%u2(i,j,itm) + r(2,2) = fp*boundary(ib)%v2(i,j,itp)+fm*boundary(ib)%v2(i,j,itm) + r(3,3) = fp*boundary(ib)%w2(i,j,itp)+fm*boundary(ib)%w2(i,j,itm) + r(1,2) = fp*boundary(ib)%uv(i,j,itp)+fm*boundary(ib)%uv(i,j,itm) + r(1,3) = fp*boundary(ib)%uw(i,j,itp)+fm*boundary(ib)%uw(i,j,itp) + r(2,3) = fp*boundary(ib)%vw(i,j,itp)+fm*boundary(ib)%vw(i,j,itm) + r(2,1) = r(1,2); r(3,1) = r(1,3); r(3,2) = r(2,3) + call DSYEVJ3(r,eigvec,eigval) + do ii = 1,3 + !if(eigval(ii)<-1.e-8) print *,"warning negative eigenvalue ",eigval(ii), " value set to 1e-8" + eigval(ii) = max(eigval(ii),1.e-8) + end do + boundary(ib)%eigvec(i,j,:,:) = real(eigvec) + boundary(ib)%ci(i,j,:) = real(sqrt(eigval)) + end do + end do + end subroutine calc_eigdec + + subroutine calc_pert(ib,x,y,z,nx,ny,nz,uturb,iuturb) + use modglobal, only : rtimee,dxint,dyint + use modmpi, only : myidx,myidy + implicit none + real, dimension(:), intent(in) :: x,y,z + integer, intent(in) :: nx,ny,nz,iuturb,ib + real(field_r), dimension(:,:), intent(out) :: uturb + integer,target :: i,j,k,ipatch,jpatch,kpatch + integer :: ii + integer, pointer :: pi1, pi2,pi1patch,pi2patch + real, dimension(3) :: xx,ci + real, dimension(3,3) :: eigvec + real :: t,utemp,vtemp,wtemp + t = rtimee + select case(ib) + case(1,2) + pi1 => j; pi2 => k; pi1patch => jpatch; pi2patch => kpatch + case(3,4) + pi1 => i; pi2 => k; pi1patch => ipatch; pi2patch => kpatch + case(5) + pi1 => i; pi2 => j; pi1patch => ipatch; pi2patch => jpatch + end select + do i = 1,nx + xx(1) = x(i); ipatch = min(int(x(i)/dxint)+1,nxpatch) + do j = 1,ny + xx(2) = y(j); jpatch = min(int(y(j)/dyint)+1,nypatch) + do k = 1,nz + xx(3) = z(k); kpatch = min(k,nzpatch) + ci = boundary(ib)%ci(pi1patch,pi2patch,:) + eigvec = boundary(ib)%eigvec(pi1patch,pi2patch,:,:) + utemp = 0.; vtemp = 0.; wtemp = 0. + do ii = 1,nmodes + utemp = utemp + p(ii,1)*cos(dot(kn(ii,:)/ci(1)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,1)*sin(dot(kn(ii,:)/ci(1)*ctot,xx/lambda)+omega(ii)*t/tau) + vtemp = vtemp + p(ii,2)*cos(dot(kn(ii,:)/ci(2)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,2)*sin(dot(kn(ii,:)/ci(2)*ctot,xx/lambda)+omega(ii)*t/tau) + wtemp = wtemp + p(ii,3)*cos(dot(kn(ii,:)/ci(3)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,3)*sin(dot(kn(ii,:)/ci(3)*ctot,xx/lambda)+omega(ii)*t/tau) + end do + ! Scale velocity fields + utemp = utemp*ci(1) + vtemp = vtemp*ci(2) + wtemp = wtemp*ci(3) + ! Reproject to cartesian velocity pertubations + uturb(pi1,pi2) = nisqrt*dot(eigvec(iuturb,:),(/utemp,vtemp,wtemp/)) + end do + end do + end do + nullify(pi1,pi2,pi1patch,pi2patch) + end subroutine calc_pert + + subroutine calc_pert2(ib,x,y,z,nx,ny,nz,uturb,iuturb,thlturb,qtturb) + use modglobal, only : rtimee,tboundary,ntboundary + use modmpi, only : myidx,myidy + implicit none + real, dimension(:), intent(in) :: x,y,z + integer, intent(in) :: nx,ny,nz,iuturb,ib + real(field_r), dimension(:,:), intent(out) :: uturb,thlturb,qtturb + integer,target :: i,j,k,ipatch,jpatch,kpatch + integer :: ii,itp,itm + integer, pointer :: pi1, pi2,pi1patch,pi2patch + real, dimension(3) :: xx,ci + real, dimension(3,3) :: eigvec + real :: wthl,wqt,w2,thl2,qt2,utemp,vtemp,wtemp,wturbf,thltemp,qttemp + real :: t,fp,fm,rho + t = rtimee + itm = 1 + ! Interpolate covariance to current time + if(ntboundary>1) then + do while(t>tboundary(itm)) + itm = itm+1 + end do + if(t>tboundary(1)) then + itm = itm-1 + end if + itp = itm+1 + fm = (tboundary(itp)-t)/(tboundary(itp)-tboundary(itm)) + fp = (t-tboundary(itm))/(tboundary(itp)-tboundary(itm)) + else + itm = 1; itp = 1; fp = 1.; fm = 1. + endif + select case(ib) + case(1,2) + pi1 => j; pi2 => k; pi1patch => jpatch; pi2patch => kpatch + case(3,4) + pi1 => i; pi2 => k; pi1patch => ipatch; pi2patch => kpatch + case(5) + pi1 => i; pi2 => j; pi1patch => ipatch; pi2patch => jpatch + end select + do i = 1,nx + xx(1) = x(i); ipatch = min(int(x(i)/dxint)+1,nxpatch) + do j = 1,ny + xx(2) = y(j); jpatch = min(int(y(j)/dyint)+1,nypatch) + do k = 1,nz + xx(3) = z(k); kpatch = min(k,nzpatch) + ci = boundary(ib)%ci(pi1patch,pi2patch,:) + eigvec = boundary(ib)%eigvec(pi1patch,pi2patch,:,:) + utemp = 0.; vtemp = 0.; wtemp = 0.; thltemp = 0.; qttemp = 0. + do ii = 1,nmodes + utemp = utemp + p(ii,1)*cos(dot(kn(ii,:)/ci(1)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,1)*sin(dot(kn(ii,:)/ci(1)*ctot,xx/lambda)+omega(ii)*t/tau) + vtemp = vtemp + p(ii,2)*cos(dot(kn(ii,:)/ci(2)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,2)*sin(dot(kn(ii,:)/ci(2)*ctot,xx/lambda)+omega(ii)*t/tau) + wtemp = wtemp + p(ii,3)*cos(dot(kn(ii,:)/ci(3)*ctot,xx/lambda)+omega(ii)*t/tau) + & + & q(ii,3)*sin(dot(kn(ii,:)/ci(3)*ctot,xx/lambda)+omega(ii)*t/tau) + thltemp = thltemp + p_thl(ii)*cos(dot(k_thl(ii,:),xx/lambdasxyz)+omega_thl(ii)*t/tau) + & + & q_thl(ii)*sin(dot(k_thl(ii,:),xx/lambdasxyz)+omega_thl(ii)*t/tau) + qttemp = qttemp + p_qt(ii)*cos(dot(k_qt(ii,:),xx/lambdasxyz)+omega_qt(ii)*t/tau) + & + & q_qt(ii)*sin(dot(k_qt(ii,:),xx/lambdasxyz)+omega_qt(ii)*t/tau) + end do + ! Scale velocity fields + utemp = utemp*ci(1) + vtemp = vtemp*ci(2) + wtemp = wtemp*ci(3) + ! Reproject to cartesian velocity pertubations + uturb(pi1,pi2) = nisqrt*dot(eigvec(iuturb,:),(/utemp,vtemp,wtemp/)) + ! Calculate thlturb and qtturb + thltemp = thltemp * nisqrt2 + qttemp = qttemp * nisqrt2 + wturbf = nisqrt*dot(eigvec(3,:),(/utemp,vtemp,wtemp/)) + wthl = fp*boundary(ib)%wthl(pi1patch,pi2patch,itp) + fm*boundary(ib)%wthl(pi1patch,pi2patch,itm) + wqt = fp*boundary(ib)%wqt(pi1patch,pi2patch,itp) + fm*boundary(ib)%wqt(pi1patch,pi2patch,itm) + w2 = fp*boundary(ib)%w2(pi1patch,pi2patch,itp) + fm*boundary(ib)%w2(pi1patch,pi2patch,itm) + thl2 = fp*boundary(ib)%thl2(pi1patch,pi2patch,itp) + fm*boundary(ib)%thl2(pi1patch,pi2patch,itm) + qt2 = fp*boundary(ib)%qt2(pi1patch,pi2patch,itp) + fm*boundary(ib)%qt2(pi1patch,pi2patch,itm) + if(thl2==0. .or. w2==0.) then + thlturb(pi1,pi2) = thltemp*sqrt(thl2) + else + rho = min(max(wthl/sqrt(thl2*w2),-1.),1.) + thlturb(pi1,pi2) = (rho*wturbf/sqrt(w2) + sqrt(1-rho**2)*thltemp)*sqrt(thl2) + endif + if(qt2==0. .or. w2==0.) then + qtturb(pi1,pi2) = qttemp*sqrt(qt2) + else + rho = min(max(wqt/sqrt(qt2*w2),-1.),1.) + qtturb(pi1,pi2) = (rho*wturbf/sqrt(w2) + sqrt(1-rho**2)*qttemp)*sqrt(qt2) + endif + !if(ib==1) then + ! vturb(pi1,pi2) = nisqrt*dot(eigvec(2,:),(/utemp,vtemp,wtemp/)) + ! wturb(pi1,pi2) = nisqrt*dot(eigvec(3,:),(/utemp,vtemp,wtemp/)) + !endif + end do + end do + end do + ! ! Uncommend for netcdf output turbulent pertubations + ! if(ib==1) then + ! call check( nf90_put_var(ncid, uturb_varid, uturb, start = start, & + ! count = count) ) + ! call check( nf90_put_var(ncid, vturb_varid, vturb, start = start, & + ! count = count) ) + ! call check( nf90_put_var(ncid, wturb_varid, wturb, start = start, & + ! count = count) ) + ! call check( nf90_put_var(ncid, thlturb_varid, thlturb, start = start, & + ! count = count) ) + ! call check( nf90_put_var(ncid, qtturb_varid, qtturb, start = start, & + ! count = count) ) + ! call check( nf90_put_var(ncid, t_varid, (/t/), start = (/start(3)/), & + ! count = (/count(3)/)) ) + ! start(3) = start(3)+1 + ! endif + nullify(pi1,pi2,pi1patch,pi2patch) + end subroutine calc_pert2 + + function gaussrand(mu,sigma) + use modglobal, only : pi,eps1 + implicit none + real :: gaussrand + real, intent(in) :: mu, sigma + real :: temp1,temp2 + temp1 = 0. + do while(temp1