Add to develop branch readme, scripts and code to create bnu soil texture data

util/bnu_soil_gen/Makefile

@@ -0,0 +1,10 @@
+#
+
+NETCDF_INC = -I${NETCDF}/include
+NETCDF_LIB = -L${NETCDF}/lib -lnetcdf -lnetcdff
+
+F90 = ifort
+
+soil_fill: soil_fill.f90
+	$(F90) $(FFLAGS) -o $(@) $(NETCDF_INC) soil_fill.f90 $(NETCDF_LIB)
+

util/bnu_soil_gen/README

@@ -0,0 +1,157 @@
+
+Steps to create soil texture class dataset from BNU sand/silt/clay data source
+
+All processing scripts and code are run on NOAA hera HPC
+
+=======================================
+
+1. download original sand,silt,clay data from here:
+	http://globalchange.bnu.edu.cn/research/soilw
+
+6 files needed:
+
+CLAY1.nc  SAND1.nc  SILT1.nc
+CLAY2.nc  SAND2.nc  SILT2.nc
+
+documentation: Shangguan, W., Dai, Y., Duan, Q., Liu, B. and Yuan, H., 2014. A Global Soil Data Set for Earth System Modeling. Journal of Advances in Modeling Earth Systems, 6: 249-263 .
+
+data notes: 
+- horizontal resolution is ~global 30", 84N -> 56S
+- vertical resolution: 8 layers to the depth of 2.3 m:
+    0     - 0.045 m
+    0.045 - 0.091 m
+    0.091 - 0.166 m
+    0.166 - 0.289 m
+    0.289 - 0.493 m
+    0.493 - 0.829 m
+    0.829 - 1.383 m
+    1.383 - 2.296 m
+
+store 6 files in a directory named original/
+
+=======================================
+
+2. convert to texture using NCL; based on USDA soil texture triangle
+
+NCL does not handle the large size of this data so the process is split into 30deg x 30deg tiles
+
+The code produces a texture class for layers 0-1m and 1-2m.
+
+create a directory name tiles/
+
+since there is no data at either pole, run three separate scripts:
+
+convert_to_texture.ncl       : create tiles from 30S to 60N
+convert_to_texture_Spole.ncl : create tile  from 30S to 60S
+convert_to_texture_Npole.ncl : create tile  from 60N to 90N
+
+Reduced lat organization in the BNU dataset:
+
+lat(beg) = 83.99578
+lat(end) = -55.99583
+
+That is, centroids of 84N - 56S, i.e., 90N-60N and 30S-60S are only partially covered.
+
+=======================================
+
+3. stitch tiles, fill missing and make consistent with viirs vegetation
+
+fill in missing value with these rules based on VIIRS land:
+
+where land is snow/ice, soil texture = 16
+where water           , soil texture = 14
+
+If missing, search an 11x11 region and use dominant of 12 categories.
+
+If still missing, search a 51x51 region and use dominant of 12 categories.
+
+If still missing, search a 201x201 region and use dominant of 12 categories.
+
+If still missing, set to loam.
+
+fortran code (soil_fill.f90) run on hera with:
+
+module load intel/2022.2.0 netcdf/4.7.0
+make
+./soil_fill
+
+output notes:
+
+ Num missing before fill:      2536964
+ Num missing after fill #1:      61684
+ Num missing after fill #2:      22021
+ Num missing after fill #3:      19103
+ Num missing after filling:          0
+
+The file dataset only uses the 0-1m texture class.
+
+output file will be named bnu_soil.nc
+
+=======================================
+
+4. final formatting to use in ufs_utils
+
+a. create 30s soil file template from viirs vegetation file
+
+cp VEG_LOCATION/vegetation_type.viirs.igbp.30s.nc soil_type.bnu.30s.nc
+
+b. run a bunch of nco commands to get the metadata and naming correct for soil type
+
+ncrename -O -h -v vegetation_type,soil_type soil_type.bnu.30s.nc 
+ncatted -h -a landice_category,soil_type,m,i,16 soil_type.bnu.30s.nc
+ncatted -h -a class_01,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_02,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_03,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_04,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_05,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_06,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_07,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_08,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_09,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_10,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_11,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_12,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_13,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_14,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_15,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_16,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_17,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_18,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_19,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a class_20,soil_type,d,,, soil_type.bnu.30s.nc
+ncatted -h -a source,global,m,c,"BNU SOIL TYPE" soil_type.bnu.30s.nc
+ncatted -h -a description,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a data_source,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a primary_developer,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a developer,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a nesdis_poc,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a contributor,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a primary_documentation,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a additional_documentation,global,d,,, soil_type.bnu.30s.nc
+ncatted -h -a units,lat,c,c,"degrees_north" soil_type.bnu.30s.nc
+ncatted -h -a units,lon,c,c,"degrees_east" soil_type.bnu.30s.nc
+
+ncatted -h -a class_01,soil_type,c,c,"sand" soil_type.bnu.30s.nc
+ncatted -h -a class_02,soil_type,c,c,"loamy sand" soil_type.bnu.30s.nc
+ncatted -h -a class_03,soil_type,c,c,"sandy loam" soil_type.bnu.30s.nc
+ncatted -h -a class_04,soil_type,c,c,"silt loam" soil_type.bnu.30s.nc
+ncatted -h -a class_05,soil_type,c,c,"silt" soil_type.bnu.30s.nc
+ncatted -h -a class_06,soil_type,c,c,"loam" soil_type.bnu.30s.nc
+ncatted -h -a class_07,soil_type,c,c,"sandy clay loam" soil_type.bnu.30s.nc
+ncatted -h -a class_08,soil_type,c,c,"silty clay loam" soil_type.bnu.30s.nc
+ncatted -h -a class_09,soil_type,c,c,"clay loam" soil_type.bnu.30s.nc
+ncatted -h -a class_10,soil_type,c,c,"sandy clay" soil_type.bnu.30s.nc
+ncatted -h -a class_11,soil_type,c,c,"silty clay" soil_type.bnu.30s.nc
+ncatted -h -a class_12,soil_type,c,c,"clay" soil_type.bnu.30s.nc
+
+c. replace soil type data from processed file into template file
+
+ncks -A -h -v soil_type bnu_soil.nc soil_type.bnu.30s.nc
+
+d. to save 90+% space compress
+
+ncks -O -h -4 -L 1 soil_type.bnu.30s.nc soil_type.bnu.30s.nc
+
+e. dataset is stored here
+
+hera:/scratch2/NCEPDEV/land/data/input_data/soil_type

util/bnu_soil_gen/convert_to_texture.ncl

@@ -0,0 +1,164 @@
+load "$NCARG_ROOT/lib/ncarg/nclex/gsun/gsn_code.ncl"
+load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl"
+load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl"  
+
+begin
+
+latnames = (/"+60","+30","+00","-30"/)
+lonnames = (/"-180","-150","-120","-090","-060","-030","+000","+030","+060","+090","+120","+150","+180"/)
+wtshal = (/0.045,0.046,0.075,0.123,0.204,0.336,0.171/)
+wtdeep = (/0.383,0.617/)
+
+do ilat = 0,2
+do ilon = 0,11
+
+  latbeg = ilat*30*120 + 24*120
+  latend = ilat*30*120 + 24*120 + 30*120 - 1
+  lonbeg = ilon*30*120
+  lonend = ilon*30*120 + 30*120 - 1
+
+  filename = "lat_"+latnames(ilat)+"_"+latnames(ilat+1)+"_lon_"+lonnames(ilon)+"_"+lonnames(ilon+1)
+  print("Starting: "+filename)
+
+  infile = addfile("original/CLAY1.nc","r")
+    inclay1 = infile->CLAY(0,latbeg:latend,lonbeg:lonend)
+    inclay2 = infile->CLAY(1,latbeg:latend,lonbeg:lonend)
+    inclay3 = infile->CLAY(2,latbeg:latend,lonbeg:lonend)
+    inclay4 = infile->CLAY(3,latbeg:latend,lonbeg:lonend)
+  infile = addfile("original/CLAY2.nc","r")
+    inclay5 = infile->CLAY(0,latbeg:latend,lonbeg:lonend)
+    inclay6 = infile->CLAY(1,latbeg:latend,lonbeg:lonend)
+    inclay7 = infile->CLAY(2,latbeg:latend,lonbeg:lonend)
+    inclay8 = infile->CLAY(3,latbeg:latend,lonbeg:lonend)
+  infile = addfile("original/SAND1.nc","r")
+    insand1 = infile->SAND(0,latbeg:latend,lonbeg:lonend)
+    insand2 = infile->SAND(1,latbeg:latend,lonbeg:lonend)
+    insand3 = infile->SAND(2,latbeg:latend,lonbeg:lonend)
+    insand4 = infile->SAND(3,latbeg:latend,lonbeg:lonend)
+  infile = addfile("original/SAND2.nc","r")
+    insand5 = infile->SAND(0,latbeg:latend,lonbeg:lonend)
+    insand6 = infile->SAND(1,latbeg:latend,lonbeg:lonend)
+    insand7 = infile->SAND(2,latbeg:latend,lonbeg:lonend)
+    insand8 = infile->SAND(3,latbeg:latend,lonbeg:lonend)
+  infile = addfile("original/SILT1.nc","r")
+    insilt1 = infile->SILT(0,latbeg:latend,lonbeg:lonend)
+    insilt2 = infile->SILT(1,latbeg:latend,lonbeg:lonend)
+    insilt3 = infile->SILT(2,latbeg:latend,lonbeg:lonend)
+    insilt4 = infile->SILT(3,latbeg:latend,lonbeg:lonend)
+  infile = addfile("original/SILT2.nc","r")
+    insilt5 = infile->SILT(0,latbeg:latend,lonbeg:lonend)
+    insilt6 = infile->SILT(1,latbeg:latend,lonbeg:lonend)
+    insilt7 = infile->SILT(2,latbeg:latend,lonbeg:lonend)
+    insilt8 = infile->SILT(3,latbeg:latend,lonbeg:lonend)
+
+sand_top = new((/3600,3600/),float)
+silt_top = new((/3600,3600/),float)
+clay_top = new((/3600,3600/),float)
+sand_bot = new((/3600,3600/),float)
+silt_bot = new((/3600,3600/),float)
+clay_bot = new((/3600,3600/),float)
+
+sand_top = wtshal(0) * insand1 + wtshal(1) * insand2 + wtshal(2) * insand3 + wtshal(3) * insand4 + wtshal(4) * insand5 + wtshal(5) * insand6 + wtshal(6) * insand7
+silt_top = wtshal(0) * insilt1 + wtshal(1) * insilt2 + wtshal(2) * insilt3 + wtshal(3) * insilt4 + wtshal(4) * insilt5 + wtshal(5) * insilt6 + wtshal(6) * insilt7
+clay_top = wtshal(0) * inclay1 + wtshal(1) * inclay2 + wtshal(2) * inclay3 + wtshal(3) * inclay4 + wtshal(4) * inclay5 + wtshal(5) * inclay6 + wtshal(6) * inclay7
+
+sand_top = mask(sand_top,sand_top.ge.0.and.sand_top.le.100,True)
+silt_top = mask(silt_top,silt_top.ge.0.and.silt_top.le.100,True)
+clay_top = mask(clay_top,clay_top.ge.0.and.clay_top.le.100,True)
+
+sand_bot = wtdeep(0) * insand7 + wtdeep(1) * insand8
+silt_bot = wtdeep(0) * insilt7 + wtdeep(1) * insilt8
+clay_bot = wtdeep(0) * inclay7 + wtdeep(1) * inclay8
+
+sand_bot = mask(sand_bot,sand_bot.ge.0.and.sand_bot.le.100,True)
+silt_bot = mask(silt_bot,silt_bot.ge.0.and.silt_bot.le.100,True)
+clay_bot = mask(clay_bot,clay_bot.ge.0.and.clay_bot.le.100,True)
+
+soil_texture_top = new((/3600,3600/),integer)
+  soil_texture_top@_FillValue = -999
+soil_texture_bot = new((/3600,3600/),integer)
+  soil_texture_bot@_FillValue = -999
+
+; SOILPARM:1  sand
+
+soil_texture_top = where((silt_top + 1.5*clay_top) .lt. 15, 1, soil_texture_top) 
+soil_texture_bot = where((silt_bot + 1.5*clay_bot) .lt. 15, 1, soil_texture_bot) 
+
+; SOILPARM:2  loamy sand
+
+soil_texture_top = where((silt_top + 1.5*clay_top .ge. 15) .and. (silt_top + 2*clay_top .lt. 30), 2, soil_texture_top)
+soil_texture_bot = where((silt_bot + 1.5*clay_bot .ge. 15) .and. (silt_bot + 2*clay_bot .lt. 30), 2, soil_texture_bot)
+
+; SOILPARM:3  sandy loam
+
+soil_texture_top = where((clay_top .ge. 7 .and. clay_top .lt. 20) .and. (sand_top .gt. 52) .and. ((silt_top + 2*clay_top) .ge. 30) .or. (clay_top .lt. 7 .and. silt_top .lt. 50 .and. (silt_top+2*clay_top).ge.30), 3, soil_texture_top)
+soil_texture_bot = where((clay_bot .ge. 7 .and. clay_bot .lt. 20) .and. (sand_bot .gt. 52) .and. ((silt_bot + 2*clay_bot) .ge. 30) .or. (clay_bot .lt. 7 .and. silt_bot .lt. 50 .and. (silt_bot+2*clay_bot).ge.30), 3, soil_texture_bot)
+
+; SOILPARM:6  loam
+
+soil_texture_top = where((clay_top .ge. 7 .and. clay_top .lt. 27) .and. (silt_top .ge. 28 .and. silt_top .lt. 50) .and. (sand_top .le. 52), 6, soil_texture_top)
+soil_texture_bot = where((clay_bot .ge. 7 .and. clay_bot .lt. 27) .and. (silt_bot .ge. 28 .and. silt_bot .lt. 50) .and. (sand_bot .le. 52), 6, soil_texture_bot)
+
+; SOILPARM:4  silt loam
+
+soil_texture_top = where((silt_top .ge. 50 .and. (clay_top .ge. 12 .and. clay_top .lt. 27)) .or. ((silt_top .ge. 50 .and. silt_top .lt. 80) .and. clay_top .lt. 12), 4, soil_texture_top)
+soil_texture_bot = where((silt_bot .ge. 50 .and. (clay_bot .ge. 12 .and. clay_bot .lt. 27)) .or. ((silt_bot .ge. 50 .and. silt_bot .lt. 80) .and. clay_bot .lt. 12), 4, soil_texture_bot)
+
+; SOILPARM:5  silt
+
+soil_texture_top = where(silt_top .ge. 80 .and. clay_top .lt. 12, 5, soil_texture_top)
+soil_texture_bot = where(silt_bot .ge. 80 .and. clay_bot .lt. 12, 5, soil_texture_bot)
+
+; SOILPARM:7  sandy clay loam
+
+soil_texture_top = where((clay_top .ge. 20 .and. clay_top .lt. 35) .and. (silt_top .lt. 28) .and. (sand_top .gt. 45), 7, soil_texture_top) 
+soil_texture_bot = where((clay_bot .ge. 20 .and. clay_bot .lt. 35) .and. (silt_bot .lt. 28) .and. (sand_bot .gt. 45), 7, soil_texture_bot) 
+
+; SOILPARM:9  clay loam
+
+soil_texture_top = where((clay_top .ge. 27 .and. clay_top .lt. 40) .and. (sand_top .gt. 20 .and. sand_top .le. 45), 9, soil_texture_top)
+soil_texture_bot = where((clay_bot .ge. 27 .and. clay_bot .lt. 40) .and. (sand_bot .gt. 20 .and. sand_bot .le. 45), 9, soil_texture_bot)
+
+; SOILPARM:8  silty clay loam
+
+soil_texture_top = where((clay_top .ge. 27 .and. clay_top .lt. 40) .and. (sand_top  .le. 20), 8, soil_texture_top)
+soil_texture_bot = where((clay_bot .ge. 27 .and. clay_bot .lt. 40) .and. (sand_bot  .le. 20), 8, soil_texture_bot)
+
+; SOILPARM:10 sandy clay
+
+soil_texture_top = where(clay_top .ge. 35 .and. sand_top .gt. 45, 10, soil_texture_top)
+soil_texture_bot = where(clay_bot .ge. 35 .and. sand_bot .gt. 45, 10, soil_texture_bot)
+
+; SOILPARM:11 silty clay
+
+soil_texture_top = where(clay_top .ge. 40 .and. silt_top .ge. 40, 11, soil_texture_top)
+soil_texture_bot = where(clay_bot .ge. 40 .and. silt_bot .ge. 40, 11, soil_texture_bot)
+
+; SOILPARM:12 clay
+
+soil_texture_top = where(clay_top .ge. 40 .and. sand_top .le. 45 .and. silt_top .lt. 40, 12, soil_texture_top)
+soil_texture_bot = where(clay_bot .ge. 40 .and. sand_bot .le. 45 .and. silt_bot .lt. 40, 12, soil_texture_bot)
+
+soil_texture_top = mask(soil_texture_top,soil_texture_top.ge.0,True)
+soil_texture_bot = mask(soil_texture_bot,soil_texture_bot.ge.0,True)
+
+outname = "tiles/"+filename+".nc"
+ system("if [ -e "+outname+" ]; then rm -f "+outname+ ";fi")
+outfile = addfile(outname,"c")
+outfile->soil_texture_top = soil_texture_top(::-1,:)
+outfile->soil_texture_bot = soil_texture_bot(::-1,:)
+
+delete(sand_top)
+delete(silt_top)
+delete(clay_top)
+delete(sand_bot)
+delete(silt_bot)
+delete(clay_bot)
+delete(soil_texture_top)
+delete(soil_texture_bot)
+
+end do
+end do
+
+end
+