From d7ae00c9fee31603fb673eeed17d7133dcb43ad1 Mon Sep 17 00:00:00 2001 From: braczka Date: Mon, 15 May 2023 18:26:55 -0600 Subject: [PATCH 01/25] Generating new files for fluxnetfull obs converter Creating new observation converter for site level eddy covariance flux tower data (NEE,GPP,RECO,LH,SH). The existing Ameriflux_L4 converter is outdated and new database has new flux options and time aggregates --- .../Ameriflux/fluxnetfull_to_obs.f90 | 854 ++++++++++++++++++ .../Ameriflux/fluxnetfull_to_obs.nml | 13 + .../Ameriflux/fluxnetfull_to_obs.rst | 177 ++++ .../obs_converters/Ameriflux/work/input.nml | 15 + .../Ameriflux/work/quickbuild.sh | 1 + 5 files changed, 1060 insertions(+) create mode 100644 observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 create mode 100644 observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml create mode 100644 observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 new file mode 100644 index 0000000000..2eee5ce67c --- /dev/null +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -0,0 +1,854 @@ +! DART software - Copyright UCAR. This open source software is provided +! by UCAR, "as is", without charge, subject to all terms of use at +! http://www.image.ucar.edu/DAReS/DART/DART_download +! +! $Id$ + +program level4_to_obs + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! level4_to_obs - a program that only needs minor customization to read +! in a text-based dataset - either white-space separated values or +! fixed-width column data. +! +! created 3 May 2012 Tim Hoar NCAR/IMAGe +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +use types_mod, only : r8, MISSING_R8 + +use utilities_mod, only : initialize_utilities, finalize_utilities, & + register_module, error_handler, E_MSG, E_ERR, & + open_file, close_file, do_nml_file, do_nml_term, & + check_namelist_read, find_namelist_in_file, & + nmlfileunit, logfileunit + +use time_manager_mod, only : time_type, set_calendar_type, GREGORIAN, & + set_date, set_time, get_time, print_time, & + print_date, operator(-), operator(+), operator(>), & + operator(<), operator(==), operator(<=), operator(>=) + +use location_mod, only : VERTISHEIGHT + +use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, & + static_init_obs_sequence, init_obs, write_obs_seq, & + init_obs_sequence, get_num_obs, & + set_copy_meta_data, set_qc_meta_data + +use obs_kind_mod, only : TOWER_SENSIBLE_HEAT_FLUX, & + TOWER_NETC_ECO_EXCHANGE, & + TOWER_LATENT_HEAT_FLUX + +implicit none + +! version controlled file description for error handling, do not edit +character(len=256), parameter :: source = & + "$URL$" +character(len=32 ), parameter :: revision = "$Revision$" +character(len=128), parameter :: revdate = "$Date$" + +!----------------------------------------------------------------------- +! Namelist with default values +!----------------------------------------------------------------------- + +character(len=256) :: text_input_file = 'textdata.input' +character(len=256) :: obs_out_file = 'obs_seq.out' +integer :: year = -1 +real(r8) :: timezoneoffset = -1.0_r8 +real(r8) :: latitude = -1.0_r8 +real(r8) :: longitude = -1.0_r8 +real(r8) :: elevation = -1.0_r8 +real(r8) :: flux_height = -1.0_r8 +real(r8) :: maxgoodqc = 3.0_r8 +logical :: verbose = .false. + +namelist /level4_to_obs_nml/ text_input_file, obs_out_file, year, & + timezoneoffset, latitude, longitude, elevation, & + flux_height, maxgoodqc, verbose + +!----------------------------------------------------------------------- +! globally-scoped variables +!----------------------------------------------------------------------- + +character(len=300) :: input_line, bigline +character(len=512) :: string1, string2, string3 +integer :: iline, nlines, nwords +logical :: first_obs +integer :: oday, osec, rcio, iunit +integer :: num_copies, num_qc, max_obs +real(r8) :: oerr, qc +type(obs_sequence_type) :: obs_seq +type(obs_type) :: obs, prev_obs +type(time_type) :: prev_time, offset +real(r8), parameter :: umol_to_gC = (1.0_r8/1000000.0_r8) * 12.0_r8 + +type towerdata + type(time_type) :: time_obs + character(len=20) :: monthstring = 'Month' + character(len=20) :: daystring = 'Day' + character(len=20) :: hourstring = 'Hour' + character(len=20) :: doystring = 'DoY' + character(len=20) :: neestring = 'NEE_or_fMDS' + character(len=20) :: neeQCstring = 'NEE_or_fMDSqc' + character(len=20) :: lestring = 'LE_f' + character(len=20) :: leQCstring = 'LE_fqc' + character(len=20) :: hstring = 'H_f' + character(len=20) :: hQCstring = 'H_fqc' + integer :: monthindex + integer :: dayindex + integer :: hourindex + integer :: doyindex + integer :: neeindex + integer :: neeQCindex + integer :: leindex + integer :: leQCindex + integer :: hindex + integer :: hQCindex + integer :: month + integer :: day + real(r8) :: hour + real(r8) :: doy + real(r8) :: nee + integer :: neeQC + real(r8) :: le + integer :: leQC + real(r8) :: h + integer :: hQC +end type towerdata + +type(towerdata) :: tower + +!----------------------------------------------------------------------- +! start of executable code +!----------------------------------------------------------------------- + +call initialize_utilities('level4_to_obs') + +! Print module information to log file and stdout. +call register_module(source, revision, revdate) + +! Read the namelist entry +call find_namelist_in_file("input.nml", "level4_to_obs_nml", iunit) +read(iunit, nml = level4_to_obs_nml, iostat = rcio) +call check_namelist_read(iunit, rcio, "level4_to_obs_nml") + +! Record the namelist values used for the run ... +if (do_nml_file()) write(nmlfileunit, nml=level4_to_obs_nml) +if (do_nml_term()) write( * , nml=level4_to_obs_nml) + +! time setup +call set_calendar_type(GREGORIAN) +offset = set_time(nint(abs(timezoneoffset)*3600.0_r8),0) +prev_time = set_time(0, 0) + +write(string1, *) 'tower located at lat, lon, elev =', latitude, longitude, elevation +write(string2, *) 'flux observations taken at =', flux_height,'m' +if (verbose) call error_handler(E_MSG,'level4_to_obs',string1,text2=string2) + +! check the lat/lon values to see if they are ok +if (longitude < 0.0_r8) longitude = longitude + 360.0_r8 + +if (( latitude > 90.0_r8 .or. latitude < -90.0_r8 ) .or. & + (longitude < 0.0_r8 .or. longitude > 360.0_r8 )) then + + write (string2,*)'latitude should be [-90, 90] but is ',latitude + write (string3,*)'longitude should be [ 0,360] but is ',longitude + + string1 ='tower location error in input.nml&level4_to_obs_nml' + call error_handler(E_ERR,'level4_to_obs', string1, source, revision, & + revdate, text2=string2,text3=string3) + +endif + +! We need to know the maximum number of observations in the input file. +! Each line has info for the 3 observations we want. +! The max possible number of obs needs to be specified but it +! will only write out the actual number created. +! Each observation in this series will have a single +! observation value and a quality control flag. +! Initialize two empty observations - one to track location +! in observation sequence - the other is for the new observation. + +iunit = open_file(text_input_file, 'formatted', 'read') +if (verbose) call error_handler(E_MSG,'level4_to_obs','opened input file '//trim(text_input_file)) + +nlines = count_file_lines(iunit) +max_obs = 3*nlines +num_copies = 1 +num_qc = 1 +first_obs = .true. + +call static_init_obs_sequence() +call init_obs(obs, num_copies, num_qc) +call init_obs(prev_obs, num_copies, num_qc) +call init_obs_sequence(obs_seq, num_copies, num_qc, max_obs) + +! the first one needs to contain the string 'observation' and the +! second needs the string 'QC'. +call set_copy_meta_data(obs_seq, 1, 'observation') +call set_qc_meta_data( obs_seq, 1, 'Ameriflux QC') + +! The first line describes all the fields ... column headers, if you will + +rewind(iunit) +call decode_header(iunit, nwords) + +obsloop: do iline = 2,nlines + + ! read in entire text line into a buffer + read(iunit,'(A)',iostat=rcio) bigline + if (rcio < 0) exit obsloop + if (rcio > 0) then + write (string1,'(''Cannot read (error '',i3,'') line '',i8,'' in '',A)') & + rcio, iline, trim(text_input_file) + call error_handler(E_ERR,'main', string1, source, revision, revdate) + endif + + input_line = adjustl(bigline) + + ! parse the line into the tower structure (including the observation time) + call stringparse(input_line, nwords, iline) + + if (iline <= 2) then + write(*,*)'' + call print_date(tower%time_obs, ' first observation date (local time) is') + call print_time(tower%time_obs, ' first observation time (local time) is') + write(*,*)'first observation raw values: (column,string,value) timezone not applied' + write(*,*)tower%monthindex, tower%monthstring , tower%month + write(*,*)tower%dayindex , tower%daystring , tower%day + write(*,*)tower%hourindex , tower%hourstring , tower%hour + write(*,*)tower%doyindex , tower%doystring , tower%doy + write(*,*)tower%hindex , tower%hstring , tower%h + write(*,*)tower%hQCindex , tower%hQCstring , tower%hQC + write(*,*)tower%leindex , tower%lestring , tower%le + write(*,*)tower%leQCindex , tower%leQCstring , tower%leQC + write(*,*)tower%neeindex , tower%neestring , tower%nee + write(*,*)tower%neeQCindex, tower%neeQCstring , tower%neeQC + write(*,*)'' + + write(logfileunit,*)'' + call print_date(tower%time_obs, ' first observation date (local time) is',logfileunit) + call print_time(tower%time_obs, ' first observation time (local time) is',logfileunit) + write(logfileunit,*)'first observation raw values: (column,string,value) timezone not applied' + write(logfileunit,*)tower%monthindex, tower%monthstring , tower%month + write(logfileunit,*)tower%dayindex , tower%daystring , tower%day + write(logfileunit,*)tower%hourindex , tower%hourstring , tower%hour + write(logfileunit,*)tower%doyindex , tower%doystring , tower%doy + write(logfileunit,*)tower%hindex , tower%hstring , tower%h + write(logfileunit,*)tower%hQCindex , tower%hQCstring , tower%hQC + write(logfileunit,*)tower%leindex , tower%lestring , tower%le + write(logfileunit,*)tower%leQCindex , tower%leQCstring , tower%leQC + write(logfileunit,*)tower%neeindex , tower%neestring , tower%nee + write(logfileunit,*)tower%neeQCindex, tower%neeQCstring , tower%neeQC + write(logfileunit,*)'' + end if + + call get_time(tower%time_obs, osec, oday) + + if (verbose) then + write(string1,*)'obs time is (seconds,days) ',osec, oday,' obs date is ' + call print_date(tower%time_obs, trim(string1)) + call print_date(tower%time_obs, trim(string1),logfileunit) + endif + + ! make an obs derived type, and then add it to the sequence + ! If the QC value is good, use the observation. + ! Increasingly larger QC values are more questionable quality data. + ! The observation errors are from page 183, Table 7.1(A) in + ! Chapter 7 of a book by A.D. Richardson et al. via Andy Fox. + + if (tower%hQC <= maxgoodqc) then ! Sensible Heat Flux [W m-2] + oerr = 10.0_r8 + abs(tower%h)*0.22_r8 + qc = real(tower%hQC,r8) + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%h, & + TOWER_SENSIBLE_HEAT_FLUX, oerr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + endif + + if (tower%leQC <= maxgoodqc) then ! Latent Heat Flux [W m-2] + oerr = 10.0_r8 + abs(tower%le)*0.32_r8 + qc = real(tower%leQC,r8) + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%le, & + TOWER_LATENT_HEAT_FLUX, oerr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + endif + + if (tower%neeQC <= maxgoodqc) then ! Net Ecosystem Exchange [umol m-2 s-1] + if (tower%nee <= 0) then + oerr = (2.0_r8 + abs(tower%nee)*0.1_r8) * umol_to_gC + else + oerr = (2.0_r8 + abs(tower%nee)*0.4_r8) * umol_to_gC + endif + tower%nee = -tower%nee * umol_to_gC ! to match convention in CLM [gC m-2 s-1] + qc = real(tower%neeQC,r8) + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%nee, & + TOWER_NETC_ECO_EXCHANGE, oerr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + endif + +end do obsloop + +! if we added any obs to the sequence, write it out to a file now. +if ( get_num_obs(obs_seq) > 0 ) then + write(string1,*)'writing obs_seq, obs_count = ', get_num_obs(obs_seq) + if (verbose) call error_handler(E_MSG,'level4_to_obs',string1) + call write_obs_seq(obs_seq, obs_out_file) +endif + +! end of main program +call finalize_utilities() + +contains + + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! create_3d_obs - subroutine that is used to create an observation +! type from observation data. +! +! NOTE: assumes the code is using the threed_sphere locations module, +! that the observation has a single data value and a single +! qc value, and that this obs type has no additional required +! data (e.g. gps and radar obs need additional data per obs) +! +! lat - latitude of observation +! lon - longitude of observation +! vval - vertical coordinate +! vkind - kind of vertical coordinate (pressure, level, etc) +! obsv - observation value +! okind - observation kind +! oerr - observation error (in units of standard deviation) +! day - gregorian day +! sec - gregorian second +! qc - quality control value +! obs - observation type +! +! created Oct. 2007 Ryan Torn, NCAR/MMM +! adapted for more generic use 11 Mar 2010, nancy collins, ncar/image +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +subroutine create_3d_obs(lat, lon, vval, vkind, obsv, okind, oerr, day, sec, qc, obs) +use obs_def_mod, only : obs_def_type, set_obs_def_time, set_obs_def_type_of_obs, & + set_obs_def_error_variance, set_obs_def_location +use obs_sequence_mod, only : obs_type, set_obs_values, set_qc, set_obs_def +use time_manager_mod, only : time_type, set_time +use location_mod, only : set_location + + integer, intent(in) :: okind, vkind, day, sec + real(r8), intent(in) :: lat, lon, vval, obsv, oerr, qc + type(obs_type), intent(inout) :: obs + +real(r8) :: obs_val(1), qc_val(1) +type(obs_def_type) :: obs_def + +call set_obs_def_location(obs_def, set_location(lon, lat, vval, vkind)) +call set_obs_def_type_of_obs(obs_def, okind) +call set_obs_def_time(obs_def, set_time(sec, day)) +call set_obs_def_error_variance(obs_def, oerr * oerr) +call set_obs_def(obs, obs_def) + +obs_val(1) = obsv +call set_obs_values(obs, obs_val) +qc_val(1) = qc +call set_qc(obs, qc_val) + +end subroutine create_3d_obs + + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! add_obs_to_seq -- adds an observation to a sequence. inserts if first +! obs, inserts with a prev obs to save searching if that's possible. +! +! seq - observation sequence to add obs to +! obs - observation, already filled in, ready to add +! obs_time - time of this observation, in dart time_type format +! prev_obs - the previous observation that was added to this sequence +! (will be updated by this routine) +! prev_time - the time of the previously added observation (will also +! be updated by this routine) +! first_obs - should be initialized to be .true., and then will be +! updated by this routine to be .false. after the first obs +! has been added to this sequence. +! +! created Mar 8, 2010 nancy collins, ncar/image +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +subroutine add_obs_to_seq(seq, obs, obs_time, prev_obs, prev_time, first_obs) + +use obs_sequence_mod, only : obs_sequence_type, obs_type, insert_obs_in_seq +use time_manager_mod, only : time_type, operator(>=) + +type(obs_sequence_type), intent(inout) :: seq +type(obs_type), intent(inout) :: obs, prev_obs +type(time_type), intent(in) :: obs_time +type(time_type), intent(inout) :: prev_time +logical, intent(inout) :: first_obs + +! insert(seq,obs) always works (i.e. it inserts the obs in +! proper time format) but it can be slow with a long file. +! supplying a previous observation that is older (or the same +! time) as the new one speeds up the searching a lot. + +if(first_obs) then ! for the first observation, no prev_obs + call insert_obs_in_seq(seq, obs) + first_obs = .false. +else + if(obs_time >= prev_time) then ! same time or later than previous obs + call insert_obs_in_seq(seq, obs, prev_obs) + else ! earlier, search from start of seq + call insert_obs_in_seq(seq, obs) + endif +endif + +! update for next time +prev_obs = obs +prev_time = obs_time + +end subroutine add_obs_to_seq + + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! count_file_lines -- +! count the lines in a text file. +! rewinds the unit after counting. +! +! iunit - handle to the already-open text file +! +! created May 2, 2012 Tim Hoar, NCAR/IMAGe +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +function count_file_lines(iunit) + +integer, intent(in) :: iunit +integer :: count_file_lines + +integer :: i +character(len=128) :: oneline + +integer, parameter :: tenmillion = 10000000 +rewind(iunit) + +count_file_lines = 0 +countloop : do i = 1,tenmillion + + read(iunit,'(A)',iostat=rcio) oneline + + if (rcio < 0) exit countloop ! end of file + if (rcio > 0) then + write (string1,'('' read around line '',i8)')i + call error_handler(E_ERR,'count_file_lines', string1, & + source, revision, revdate) + endif + count_file_lines = count_file_lines + 1 + +enddo countloop +rewind(iunit) + +if (count_file_lines >= tenmillion) then + write (string1,'('' suspiciously large number of lines '',i8)')count_file_lines + call error_handler(E_MSG,'count_file_lines', string1, & + source, revision, revdate) +endif + +end function count_file_lines + + + + +subroutine decode_header(iunit,ncolumns) +! Reads the first line of the header and parses the information. +! And by parse, I mean determine which columns are the columns +! of interest. + +integer, intent(in) :: iunit +integer, intent(out) :: ncolumns + +integer, parameter :: maxwordlength = 30 +integer :: i,charcount,columncount,wordlength +character(len=maxwordlength), dimension(:), allocatable :: columns +integer, dimension(10) :: qc = 0 + +! Read the line and strip off any leading whitespace. + +read(iunit,'(A)',iostat=rcio) bigline +if (rcio /= 0) then + write(string1,*)'Cannot parse header. Begins <',trim(bigline(1:40)),'>' + call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) +endif + +input_line = adjustl(bigline) + +! Count how many commas are in the line - use this to determine how many columns + +charcount = CountChar(input_line,',') +ncolumns = charcount + 1 +allocate(columns(ncolumns)) + +columncount = 0 ! track the number of columns +wordlength = 0 ! number of characters in the column descriptor +charcount = 0 ! the position of the (last) comma +do i = 1,len_trim(input_line) + if (input_line(i:i) == ',') then + columncount = columncount + 1 + if (wordlength > maxwordlength) then + write(string1,*)'unexpected long word ... starts <',& + input_line((i-wordlength):(i-1)),'>' + call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) + endif + columns(columncount) = input_line((i-wordlength):(i-1)) + write(string1,*) 'word(',columncount,') is ',columns(columncount) + if (verbose) call error_handler(E_MSG,'decode_header',string1) + wordlength = 0 + charcount = i + else + wordlength = wordlength + 1 + endif +enddo + +! There is one more column after the last comma + +if ((columncount+1) /= ncolumns) then + write(string1,*)'parsed wrong number of words ...' + write(string2,*)'expected ',ncolumns,' got ',columncount+1 + call error_handler(E_ERR,'decode_header',string1,source,revision,revdate, & + text2=trim(string2), text3=trim(input_line)) +endif + +columns(ncolumns) = input_line((charcount+1):len_trim(input_line)) + +write(string1,*)'word(',ncolumns,') is ',columns(ncolumns) +if (verbose) call error_handler(E_MSG,'decode_header',string1) + +! Finally get to the task at hand + +tower%monthindex = Match(columns, tower%monthstring) +tower%dayindex = Match(columns, tower%daystring) +tower%hourindex = Match(columns, tower%hourstring) +tower%doyindex = Match(columns, tower%doystring) +tower%hindex = Match(columns, tower%hstring) +tower%hQCindex = Match(columns, tower%hQCstring) +tower%leindex = Match(columns, tower%lestring) +tower%leQCindex = Match(columns, tower%leQCstring) +tower%neeindex = Match(columns, tower%neestring) +tower%neeQCindex = Match(columns, tower%neeQCstring) + +! FIXME ... find a column marked 'year' or 'error_var' and use if possible. + +! Check to make sure we got all the indices we need + +qc( 1) = CheckIndex( tower%monthindex, tower%monthstring) +qc( 2) = CheckIndex( tower%dayindex , tower%daystring) +qc( 3) = CheckIndex( tower%hourindex , tower%hourstring) +qc( 4) = CheckIndex( tower%doyindex , tower%doystring) +qc( 5) = CheckIndex( tower%hindex , tower%hstring) +qc( 6) = CheckIndex( tower%hQCindex , tower%hQCstring) +qc( 7) = CheckIndex( tower%leindex , tower%lestring) +qc( 8) = CheckIndex( tower%leQCindex , tower%leQCstring) +qc( 9) = CheckIndex( tower%neeindex , tower%neestring) +qc(10) = CheckIndex( tower%neeQCindex, tower%neeQCstring) + +if (any(qc /= 0) ) then + write(string1,*)'Did not find all the required column indices.' + call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) +endif + +! Summarize if desired + +if (verbose) then +110 format('index for ',A20,' is ',i3) + write(*,110)tower%monthstring, tower%monthindex + write(*,110)tower%daystring , tower%dayindex + write(*,110)tower%hourstring , tower%hourindex + write(*,110)tower%doystring , tower%doyindex + write(*,110)tower%hstring , tower%hindex + write(*,110)tower%hQCstring , tower%hQCindex + write(*,110)tower%lestring , tower%leindex + write(*,110)tower%leQCstring , tower%leQCindex + write(*,110)tower%neestring , tower%neeindex + write(*,110)tower%neeQCstring, tower%neeQCindex +endif + +deallocate(columns) + +end subroutine decode_header + + + +function CountChar(str1,solo) +! Count the number of instances of the single character in a character string. +! useful when parsing a comma-separated list, for example. +! Count the commas and add 1 to get the number of items in the list. + +integer :: CountChar +character(len=*), intent(in) :: str1 +character, intent(in) :: solo + +integer :: i + +CountChar = 0 +do i = 1,len_trim(str1) + if (str1(i:i) == solo) CountChar = CountChar + 1 +enddo + +end function CountChar + + + +function Match(sentence,word) +! Determine the first occurrence of the 'word' in a sentence. +! In this context, a sentence is a character array, the dimension +! of the array is the number of words in the sentence. +! This is a case-sensitive match. Trailing blanks are removed. + +integer :: Match +character(len=*), dimension(:), intent(in) :: sentence +character(len=*), intent(in) :: word + +integer :: i + +Match = 0 +WordLoop : do i = 1,size(sentence) + if (trim(sentence(i)) == trim(word)) then + Match = i + return + endif +enddo WordLoop + +end function Match + + + +function CheckIndex( myindex, context ) +! Routine to issue a warning if the index was not found. +! Returns an error code ... 0 means the index WAS found +! a negative number means the index was NOT found - an error condition. +! I want to check ALL the indexes before fatally ending. + +integer :: CheckIndex +integer, intent(in) :: myindex +character(len=*), intent(in) :: context + +if (myindex == 0) then + write(string1,*)'Did not find column header matching ',trim(context) + call error_handler(E_MSG,'decode_header:CheckIndex',string1, source, revision, revdate) + CheckIndex = -1 ! not a good thing +else + CheckIndex = 0 ! Good to go +endif + +end function CheckIndex + + + +subroutine stringparse(str1, nwords, linenum) +! just declare everything as reals and chunk it + +character(len=*), intent(in) :: str1 +integer , intent(in) :: nwords +integer , intent(in) :: linenum + +real(r8), allocatable, dimension(:) :: values +integer :: iday, ihour, imin, isec, seconds +type(time_type) :: time0, time1, time2 + +allocate(values(nwords)) + +values = MISSING_R8 + +read(str1,*,iostat=rcio) values +if (rcio /= 0) then + write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' + call error_handler(E_ERR,'stringparse',string1, source, revision, revdate) +endif + +! Stuff what we want into the tower structure +! +! Convert to 'CLM-friendly' units AFTER we determine observation error variance. +! That happens in the main routine. +! +! NEE_or_fMDS has units [umolCO2 m-2 s-1] +! H_f has units [W m-2] +! LE_f has units [W m-2] +! +! (CLM) NEE has units [gC m-2 s-1] + +tower%month = nint(values(tower%monthindex)) +tower%day = nint(values(tower%dayindex )) +tower%hour = values(tower%hourindex ) +tower%doy = values(tower%doyindex ) +tower%nee = values(tower%neeindex ) +tower%neeQC = nint(values(tower%neeQCindex)) +tower%le = values(tower%leindex ) +tower%leQC = nint(values(tower%leQCindex )) +tower%h = values(tower%hindex ) +tower%hQC = nint(values(tower%hQCindex )) + +deallocate(values) + +! decode the time pieces ... two times ... +! The LAST line of these files is knackered ... and we have to check that +! if the doy is greater than the ymd ... +! 12,31,23.5,366.979 N-1 +! 1, 1, 0.0,367.000 N + +iday = int(tower%doy) +ihour = int(tower%hour) +seconds = nint((tower%hour - real(ihour,r8))*3600) +imin = seconds / 60 +isec = seconds - imin * 60 +time0 = set_date(year, tower%month, tower%day, ihour, imin, isec) + +isec = ihour*3600 + imin*60 + isec +time1 = set_date(year,1,1,0,0,0) + set_time(isec, (iday-1)) +time2 = time0 - time1 + +call get_time(time2, isec, iday) + +if ( iday > 0 ) then + ! FIXME we need to change the day ... + ! This blows up if you try to use a non-leap year with leapyear ... + + tower%time_obs = time1 + + if (verbose) then + write(string1,*)'converting ',tower%month,tower%day,tower%hour,tower%doy + write(string2,*)'the day-of-year indicates we should amend the month/day values.' + call error_handler(E_MSG,'stringparse', string1, source, revision, & + revdate, text2=string2) + + call print_date(time0, 'stringparse: using ymd date is') + call print_date(time1, 'stringparse: using doy date is') + call print_time(time0, 'stringparse: using ymd time is') + call print_time(time1, 'stringparse: using doy time is') + call print_time(time2, 'stringparse: difference is') + + call print_date(time0, 'stringparse: using ymd date is',logfileunit) + call print_date(time1, 'stringparse: using doy date is',logfileunit) + call print_time(time0, 'stringparse: using ymd time is',logfileunit) + call print_time(time1, 'stringparse: using doy time is',logfileunit) + call print_time(time2, 'stringparse: difference is',logfileunit) + endif +else + + tower%time_obs = time0 + +endif + +! 8AM East Coast is 1PM Greenwich +if (timezoneoffset < 0.0_r8) then + tower%time_obs = tower%time_obs + offset +else + tower%time_obs = tower%time_obs - offset +endif + +! The QC values can be 'missing' ... in which case the values are too + +if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 +if (tower%leQC < 0) tower%leQC = maxgoodqc + 1000 +if (tower%hQC < 0) tower%hQC = maxgoodqc + 1000 + +! if (tower%neeQC < maxgoodqc) then +! write(*,*)'nee umol m-2 s-1 ',tower%nee +! write(*,*)'nee gC m-2 s-1 ',tower%nee*umol_to_gC +! endif + +end subroutine stringparse + + + +end program level4_to_obs + + +! LEVEL 4 VARIABLE DESCRIPTION +! +! Variables description: +! Level 4 data are obtained from the level 3 products, data are ustar filtered, +! gap-filled using different methods and partitioned. +! Datasets are also aggregated from daily to monthly. +! Flags with information regarding quality of the original and gapfilled data are added. +! +! Half hourly dataset variables description: +! +! - Month : from 1 to 12 +! - Day : day of the month +! - Hour : from 0 to 23.5, indicates the end of the half hour of measurement +! - DoY : decimal day of the year +! - Rg_f : global radiation filled [W m-2] +! - Rg_fqc : global radiation quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - Ta_f : air temperature filled [deg C] +! - Ta_fqc : air temperature quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - VPD_f : vapour pressure deficit [hPa] +! - VPD_fqc : vapour pressure deficit quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - Ts_f : soil temperature filled [deg C] +! - Ts_fqc : soil temperature quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - Precip : precipitation [mm] +! - SWC : soil water content [%vol] +! - H_f : sensible heat flux filled [W m-2] +! - H_fqc : sensible heat flux quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - LE_f : latent heat flux filled [W m-2] +! - LE_fqc : latent heat flux quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - qf_NEE_st : fluxes quality flags as defined in the Level3 product +! - qf_NEE_or : fluxes quality flags as defined in the Level3 product +! - Reco_st : Estimated ecosystem respiration according to the short-term temperature +! response of night-time fluxes based on NEE_st +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! [umolCO2 m-2 s-1] +! - Reco_or : Estimated ecosystem respiration according to the short-term temperature +! response of night-time fluxes based on NEE_or +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! [umolCO2 m-2 s-1] +! - NEE_st_fMDS : NEE_st filled using the Marginal Distribution Sampling method +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! [umolCO2 m-2 s-1] +! - NEE_st_fMDSqc : NEE_st_fMDS quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - GPP_st_MDS : Gross Primary Production calculated as GPP_st_MDS = Reco_st - NEE_st_MDS +! [umolCO2 m-2 s-1] +! - NEE_or_fMDS : NEE_or filled using the Marginal Distribution Sampling method +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! [umolCO2 m-2 s-1] +! - NEE_or_fMDSqc : NEE_or_fMDS quality flags: +! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). +! (Refer to Reichstein et al. 2005 Global Change Biology ) +! - GPP_or_MDS : Gross Primary Production calculated as GPP_or_MDS = Reco_or - NEE_or_MDS +! [umolCO2 m-2 s-1] +! - NEE_st_fANN : NEE_st filled using the Artificial Neural Network method +! (Refer to Papale et al. 2003 Global Change Biology and to the Other Information section in this document) +! [umolCO2 m-2 s-1] +! - NEE_st_fANNqc : NEE_st_fANN quality flags: +! 0 = original, 1 = filled using original meteorological inputs or filled with qc=1, +! 2 = filled using filled meteorological inputs with qc=2 or 3, +! 3 = not filled using ANN due to one or more input missed but filled with the MDS method +! - GPP_st_ANN : Gross Primary Production calculated as GPP_st_ ANN = Reco_st - NEE_st_ ANN +! [umolCO2 m-2 s-1] +! - NEE_or_f ANN : NEE_or filled using the Artificial Neural Network method +! (Refer to Papale et al. 2003 Global Change Biology and to the Other Information section in this document) +! [umolCO2 m-2 s-1] +! - NEE_or_f ANNqc : NEE_or_fANN quality flags: +! 0 = original, 1 = filled using original meteorological inputs or filled with qc=1, +! 2 = filled using filled meteorological inputs with qc=2 or 3, +! 3 = not filled using ANN due to one or more input missed but filled with the MDS method +! - GPP_or_ ANN : Gross Primary Production calculated as GPP_or_ ANN = Reco_or - NEE_or_ ANN +! [umolCO2 m-2 s-1] + +! +! $URL$ +! $Id$ +! $Revision$ +! $Date$ diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml new file mode 100644 index 0000000000..4bf70b0e1c --- /dev/null +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -0,0 +1,13 @@ +&fluxnetfull_to_obs_nml + text_input_file = 'textdata.input', + obs_out_file = 'obs_seq.out', + year = -1, + timezoneoffset = -1, + latitude = -1.0, + longitude = -1.0, + elevation = -1.0, + flux_height = -1.0, + maxgoodqc = 3, + verbose = .false. + / + diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst new file mode 100644 index 0000000000..d72ff74253 --- /dev/null +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -0,0 +1,177 @@ +PROGRAM ``fluxnetfull_to_obs`` +========================= + +Overview +-------- + +FLUXNET FULLSET data to DART observation sequence converter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| This routine is designed to convert the flux tower Level 4 data from the `AmeriFlux `__ + network of observations from micrometeorological tower sites. AmeriFlux is part of + `FLUXNET `__ and the converter is hoped to be a suitable starting point for the conversion of + observations from FLUXNET. As of May 2012, I have not yet tried to work with any other observations from FLUXNET. +| The AmeriFlux Level 4 products are recorded using the local time. DART observation sequence files use GMT. For more + information about AmeriFlux data products, go to http://ameriflux.lbl.gov. + +.. warning:: + + There was a pretty severe bug in the converter that swapped latent heat flux and sensible heat flux. The bug was + present through revision 7200. It was corrected on 30 Dec 2016. + +The workflow is usually: + +#. download the Level 4 data for the towers and years in question (see DATA SOURCES below) +#. record the TIME ZONE, latitude, longitude, and elevation for each tower +#. build the DART executables with support for the tower observations. This is done by running ``preprocess`` with + ``obs_def_tower_mod.f90`` in the list of ``input_files`` for ``preprocess_nml``. +#. provide basic tower information via the ``fluxnetfull_to_obs_nml`` namelist since this information is not contained in the + Level 4 data file +#. convert each Level 4 data file individually using ``fluxnetfull_to_obs`` +#. combine all output files for the region and timeframe of interest into one file using + :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` + +For some models (CLM, for example), it is required to reorganize the observation sequence files into a series of files +that contains ONLY the observations for each assimilation. This can be achieved with the `makedaily.sh `__ +script. + +Namelist +-------- + +This namelist is read from the file ``input.nml``. Namelists start with an ampersand '&' and terminate with a slash '/'. +Character strings that contain a '/' must be enclosed in quotes to prevent them from prematurely terminating the +namelist. + +:: + + &fluxnetfull_to_obs_nml + text_input_file = 'textdata.input', + obs_out_file = 'obs_seq.out', + year = -1, + timezoneoffset = -1, + latitude = -1.0, + longitude = -1.0, + elevation = -1.0, + flux_height = -1.0, + maxgoodqc = 3, + verbose = .false. + / + +.. container:: + + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | Contents | Type | Description | + +=================+====================+=============================================================================+ + | text_input_file | character(len=128) | Name of the Level 4 ASCII file of comma-separated values. This may be a | + | | | relative or absolute filename. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | obs_out_file | character(len=128) | Name of the output observation sequence file. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | year | integer | The year of the observations in the Level 4 text file. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | timezoneoffset | real | the time zone offset (in hours) of the station. The tower observation times | + | | | are local time, we need to convert them to GMT. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | latitude | real | Latitude (in degrees N) of the tower. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | longitude | real | Longitude (in degrees E) of the tower. For internal consistency, DART uses | + | | | longitudes in the range [0,360]. An input value of -90 will be converted to | + | | | 270, for example. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | elevation | real | surface elevation (in meters) of the tower. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | flux_height | real | height (in meters) of the flux instrument on the tower. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | maxgoodqc | real | maximum value of any observation quality control flag to pass through to | + | | | the output observation sequence. Keep in mind that ``filter`` has the | + | | | ability to discriminate on the value, so there is really little to be | + | | | gained by rejecting them during the conversion. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | verbose | logical | Print extra information during the ``level4_to_obs`` execution. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + +Data sources +------------ + +| The data was acquired from https://ameriflux.lbl.gov/data/download-data/ +| and choosing the data product ``Ameriflux FLUXNET`` and variable set ``FULLSET`` +| The filenames are organized by site and time (HH,HR,WW,DD,MM,YY) and look like: +| ``AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv,`` +| ``AMF_US-xBR_FLUXNET_FULLSET_MM_2017-2021_3-5.csv`` +| The Level 4 products in question are ASCII files of comma-separated values taken every 30 minutes for an entire year. + The first line is a comma-separated list of column descriptors, all subsequent lines are comma-separated numerical + values. The converter presently searches for the columns pertaining to *NEE_or_fMDS*, *H_f*, *LE_f*, their + corresponding quality control fields, and those columns pertaining to the time of the observation. These values are + mapped as follows: + ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| Level 4 units | Level 4 variable | description | DART type | DART kind | DART units | ++===============+==================+==========================+==========================+===========================+============+ +| W/m^2 | LE_f | Latent Heat Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| [0-3] | LE_fqc | QC for LE_f | N/A | N/A | same | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| W/m^2 | H_f | Sensible Heat Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| [0-3] | H_fqc | QC for H_f | N/A | N/A | same | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| umolCO2/m^2/s | NEE_or_fMDS | Net Ecosystem Production | TOWER_NETC_ECO_EXCHANGE | QTY_NET_CARBON_PRODUCTION | gC/m^2/s | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ +| [0-3] | NEE_or_fMDSqc | QC for NEE_or_fMDS | N/A | N/A | same | ++---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ + + + + +The ``LE_fqc``, ``H_fqc``, and ``NEE_or_fMDSqc`` variables use the following convention: + + 0 = original, 1 = category A (most reliable), 2 = category B (medium), 3 = category C (least reliable). (Refer to + Reichstein et al. 2005 Global Change Biology for more information) + + +I am repeating the AmeriFlux `Data Fair-Use Policy `__ because +I believe it is important to be a good scientific citizen: + + "The AmeriFlux data provided on this site are freely available and were furnished by individual AmeriFlux scientists + who encourage their use. + Please kindly inform in writing (or e-mail) the appropriate AmeriFlux scientist(s) of how you intend to use the data + and of any publication plans. It is also important to contact the AmeriFlux investigator to assure you are + downloading the latest revision of the data and to prevent potential misuse or misinterpretation of the data. + Please acknowledge the data source as a citation or in the acknowledgments if no citation is available. If the + AmeriFlux Principal Investigators (PIs) feel that they should be acknowledged or offered participation as authors, + they will let you know and we assume that an agreement on such matters will be reached before publishing and/or use + of the data for publication. + If your work directly competes with the PI's analysis they may ask that they have the opportunity to submit a + manuscript before you submit one that uses unpublished data. In addition, when publishing please acknowledge the + agency that supported the research. + Lastly, we kindly request that those publishing papers using AmeriFlux data provide reprints to the PIs providing the + data and to the AmeriFlux archive via ameriflux.lbl.gov." + +Programs +-------- + +The ``fluxnetfull_to_obs.f90`` file is the source for the main converter program. Look at the source code where it reads the +example data file. You will almost certainly need to change the "read" statement to match your data format. The example +code reads each text line into a character buffer and then reads from that buffer to parse up the data items. + +To compile and test, go into the work subdirectory and run the ``quickbuild.sh`` script to build the converter and a +couple of general purpose utilities. ``advance_time`` helps with calendar and time computations, and the +``obs_sequence_tool`` manipulates DART observation files once they have been created. + +To change the observation types, look in the ``DART/obs_def`` directory. If you can find an obs_def_XXX_mod.f90 file +with an appropriate set of observation types, change the 'use' lines in the converter source to include those types. +Then add that filename in the ``input.nml`` namelist file to the &preprocess_nml namelist, the 'input_files' variable. +Multiple files can be listed. Then run quickbuild.sh again. It remakes the table of supported observation types before +trying to recompile the source code. + +An example script for converting batches of files is in the ``shell_scripts`` directory. A tiny example data file is in +the ``data`` directory. These are *NOT* intended to be turnkey scripts; they will certainly need to be customized for +your use. There are comments at the top of the script saying what options they include, and should be commented enough +to indicate where changes will be likely to need to be made. + +Decisions you might need to make +-------------------------------- + +See the discussion in the :doc:`../../../guide/creating-obs-seq-real` page about what options are available +for the things you need to specify. These include setting a time, specifying an expected error, setting a location, and +an observation type. diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index fe965b1083..b9b78ef034 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -36,6 +36,21 @@ verbose = .TRUE. / +&fluxnetfull_to_obs_nml + text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv' + obs_out_file = 'obs_seq.out' + year = 2003 + timezoneoffset = -6 + latitude = 42.5378 + longitude = -72.1715 + elevation = 353 + flux_height = 29 + maxgoodqc = 3 + verbose = .TRUE. + / + + + # This is appropriate for a days worth of flux tower observations # the obs in the file end 1 second before the time in the name. # If you are using these obs with CLM, ending 1 second before is appropriate. diff --git a/observations/obs_converters/Ameriflux/work/quickbuild.sh b/observations/obs_converters/Ameriflux/work/quickbuild.sh index 8ee2705cef..329028c0d8 100755 --- a/observations/obs_converters/Ameriflux/work/quickbuild.sh +++ b/observations/obs_converters/Ameriflux/work/quickbuild.sh @@ -14,6 +14,7 @@ LOCATION=threed_sphere programs=( +fluxnetfull_to_obs level4_to_obs obs_sequence_tool advance_time From 48d163b31d2fd2b6413011a18c875b717d8c654e Mon Sep 17 00:00:00 2001 From: braczka Date: Tue, 16 May 2023 17:24:00 -0600 Subject: [PATCH 02/25] Adding new flux variables to converter --- .../Ameriflux/fluxnetfull_to_obs.f90 | 244 +++++++----------- .../Ameriflux/fluxnetfull_to_obs.nml | 2 +- .../Ameriflux/fluxnetfull_to_obs.rst | 10 +- .../obs_converters/Ameriflux/work/input.nml | 2 +- 4 files changed, 100 insertions(+), 158 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 2eee5ce67c..3129565c7d 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -1,20 +1,15 @@ ! DART software - Copyright UCAR. This open source software is provided ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -! -! $Id$ -program level4_to_obs +program Fluxnetfull_to_obs -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! -! level4_to_obs - a program that only needs minor customization to read -! in a text-based dataset - either white-space separated values or -! fixed-width column data. +!------------------------------------------------------------------------ ! -! created 3 May 2012 Tim Hoar NCAR/IMAGe -! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Fluxnetfull_to_obs - a program that converts Ameriflux/Fluxnet FULLSET eddy +! covariance tower data of carbon, water and energy into +! DART obs_seq formatted files +! ## Fixme Add more description here once completed use types_mod, only : r8, MISSING_R8 @@ -38,15 +33,14 @@ program level4_to_obs use obs_kind_mod, only : TOWER_SENSIBLE_HEAT_FLUX, & TOWER_NETC_ECO_EXCHANGE, & - TOWER_LATENT_HEAT_FLUX + TOWER_LATENT_HEAT_FLUX, & + TOWER_ER_FLUX, & + TOWER_GPP_FLUX implicit none -! version controlled file description for error handling, do not edit -character(len=256), parameter :: source = & - "$URL$" -character(len=32 ), parameter :: revision = "$Revision$" -character(len=128), parameter :: revdate = "$Date$" +character(len=*), parameter :: source = 'Fluxnetfull_to_obs.f90' +character(len=512), parameter :: string1,string2,string3 !----------------------------------------------------------------------- ! Namelist with default values @@ -54,18 +48,19 @@ program level4_to_obs character(len=256) :: text_input_file = 'textdata.input' character(len=256) :: obs_out_file = 'obs_seq.out' -integer :: year = -1 real(r8) :: timezoneoffset = -1.0_r8 real(r8) :: latitude = -1.0_r8 real(r8) :: longitude = -1.0_r8 real(r8) :: elevation = -1.0_r8 real(r8) :: flux_height = -1.0_r8 real(r8) :: maxgoodqc = 3.0_r8 +! Latent,sensible heat and NEE have this option only +logical :: gap_filled = .false. logical :: verbose = .false. -namelist /level4_to_obs_nml/ text_input_file, obs_out_file, year, & +namelist /Fluxnetfull_to_obs_nml/ text_input_file, obs_out_file, & timezoneoffset, latitude, longitude, elevation, & - flux_height, maxgoodqc, verbose + flux_height, maxgoodqc, gap_filled, verbose !----------------------------------------------------------------------- ! globally-scoped variables @@ -85,36 +80,78 @@ program level4_to_obs type towerdata type(time_type) :: time_obs - character(len=20) :: monthstring = 'Month' - character(len=20) :: daystring = 'Day' - character(len=20) :: hourstring = 'Hour' - character(len=20) :: doystring = 'DoY' - character(len=20) :: neestring = 'NEE_or_fMDS' - character(len=20) :: neeQCstring = 'NEE_or_fMDSqc' - character(len=20) :: lestring = 'LE_f' - character(len=20) :: leQCstring = 'LE_fqc' - character(len=20) :: hstring = 'H_f' - character(len=20) :: hQCstring = 'H_fqc' - integer :: monthindex - integer :: dayindex - integer :: hourindex - integer :: doyindex + character(len=20) :: startstring = 'TIMESTAMP_START' + character(len=20) :: endstring = 'TIMESTAMP_END' + character(len=20) :: neestring = 'NEE_VUT_REF' + character(len=20) :: neeUNCstring = 'NEE_VUT_REF_JOINTUNC' + character(len=20) :: neeQCstring = 'NEE_VUT_REF_QC' + character(len=20) :: lestring = 'LE_F_MDS' + character(len=20) :: leUNCstring = 'LE_RANDUNC' + character(len=20) :: leQCstring = 'LE_F_MDS_QC' + character(len=20) :: hstring = 'H_F_MDS' + character(len=20) :: hUNCstring = 'H_RANDUNC' + character(len=20) :: hQCstring = 'H_F_MDS_QC' + character(len=20) :: gppDTstring = 'GPP_DT_VUT_REF' + character(len=20) :: gppNTstring = 'GPP_NT_VUT_REF' + character(len=20) :: recoDTstring = 'RECO_DT_VUT_REF' + character(len=20) :: recoNTstring = 'RECO_NT_VUT_REF' + character(len=20) :: gppUNCNT16string = 'GPP_NT_VUT_16' + character(len=20) :: gppUNCNT84string = 'GPP_NT_VUT_84' + character(len=20) :: gppUNCDT16string = 'GPP_DT_VUT_16' + character(len=20) :: gppUNCDT84string = 'GPP_DT_VUT_84' + character(len=20) :: recoUNCNT16string = 'RECO_NT_VUT_16' + character(len=20) :: recoUNCNT84string = 'RECO_NT_VUT_84' + character(len=20) :: recoUNCDT16string = 'RECO_DT_VUT_16' + character(len=20) :: recoUNCDT84string = 'RECO_DT_VUT_84' + integer :: startindex + integer :: endindex integer :: neeindex + integer :: neeUNCindex integer :: neeQCindex integer :: leindex + integer :: leUNCindex integer :: leQCindex integer :: hindex + integer :: hUNCindex integer :: hQCindex + integer :: gppDTindex + integer :: gppNTindex + integer :: recoDTindex + integer :: recoNTindex + integer :: gppUNCDT16index + integer :: gppUNCDT84index + integer :: gppUNCNT16index + integer :: gppUNCNT84index + integer :: recoUNCDT16index + integer :: recoUNCDT84index + integer :: recoUNCNT16index + integer :: recoUNCNT84index + + integer :: year integer :: month integer :: day + integer :: minute real(r8) :: hour real(r8) :: doy real(r8) :: nee + real(r8) :: neeUNC integer :: neeQC real(r8) :: le + real(r8) :: leUNC integer :: leQC real(r8) :: h + real(r8) :: hUNC integer :: hQC + real(r8) :: gppNT + real(r8) :: gppDT + real(r8) :: gppUNCNT84 + real(r8) :: gppUNCNT16 + real(r8) :: gppUNCDT84 + real(r8) :: gppUNCDT16 + real(r8) :: recoUNCNT84 + real(r8) :: recoUNCNT16 + real(r8) :: recoUNCDT84 + real(r8) :: recoUNCDT16 end type towerdata type(towerdata) :: tower @@ -123,19 +160,16 @@ program level4_to_obs ! start of executable code !----------------------------------------------------------------------- -call initialize_utilities('level4_to_obs') - -! Print module information to log file and stdout. -call register_module(source, revision, revdate) +call initialize_utilities('Fluxnetfull_to_obs') ! Read the namelist entry -call find_namelist_in_file("input.nml", "level4_to_obs_nml", iunit) -read(iunit, nml = level4_to_obs_nml, iostat = rcio) -call check_namelist_read(iunit, rcio, "level4_to_obs_nml") +call find_namelist_in_file("input.nml", "Fluxnetfull_to_obs_nml", iunit) +read(iunit, nml = Fluxnetfull_to_obs_nml, iostat = rcio) +call check_namelist_read(iunit, rcio, "Fluxnetfull_to_obs_nml") -! Record the namelist values used for the run ... -if (do_nml_file()) write(nmlfileunit, nml=level4_to_obs_nml) -if (do_nml_term()) write( * , nml=level4_to_obs_nml) +! Record the namelist values used for the run +if (do_nml_file()) write(nmlfileunit, nml=Fluxnetfull_to_obs_nml) +if (do_nml_term()) write( * , nml=Fluxnetfull_to_obs_nml) ! time setup call set_calendar_type(GREGORIAN) @@ -144,7 +178,7 @@ program level4_to_obs write(string1, *) 'tower located at lat, lon, elev =', latitude, longitude, elevation write(string2, *) 'flux observations taken at =', flux_height,'m' -if (verbose) call error_handler(E_MSG,'level4_to_obs',string1,text2=string2) +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) ! check the lat/lon values to see if they are ok if (longitude < 0.0_r8) longitude = longitude + 360.0_r8 @@ -155,9 +189,9 @@ program level4_to_obs write (string2,*)'latitude should be [-90, 90] but is ',latitude write (string3,*)'longitude should be [ 0,360] but is ',longitude - string1 ='tower location error in input.nml&level4_to_obs_nml' - call error_handler(E_ERR,'level4_to_obs', string1, source, revision, & - revdate, text2=string2,text3=string3) + string1 ='tower location error in input.nml&Fluxnetfull_to_obs_nml' + call error_handler(E_ERR,'Fluxnetfull_to_obs', source, string1, & + text2=string2,text3=string3) endif @@ -202,7 +236,7 @@ program level4_to_obs if (rcio > 0) then write (string1,'(''Cannot read (error '',i3,'') line '',i8,'' in '',A)') & rcio, iline, trim(text_input_file) - call error_handler(E_ERR,'main', string1, source, revision, revdate) + call error_handler(E_ERR,'main', string1, source) endif input_line = adjustl(bigline) @@ -439,8 +473,7 @@ function count_file_lines(iunit) if (rcio < 0) exit countloop ! end of file if (rcio > 0) then write (string1,'('' read around line '',i8)')i - call error_handler(E_ERR,'count_file_lines', string1, & - source, revision, revdate) + call error_handler(E_ERR,'count_file_lines', string1, source) endif count_file_lines = count_file_lines + 1 @@ -449,8 +482,7 @@ function count_file_lines(iunit) if (count_file_lines >= tenmillion) then write (string1,'('' suspiciously large number of lines '',i8)')count_file_lines - call error_handler(E_MSG,'count_file_lines', string1, & - source, revision, revdate) + call error_handler(E_MSG,'count_file_lines', string1, source) endif end function count_file_lines @@ -476,7 +508,7 @@ subroutine decode_header(iunit,ncolumns) read(iunit,'(A)',iostat=rcio) bigline if (rcio /= 0) then write(string1,*)'Cannot parse header. Begins <',trim(bigline(1:40)),'>' - call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) + call error_handler(E_ERR,'decode_header',string1, source) endif input_line = adjustl(bigline) @@ -496,7 +528,7 @@ subroutine decode_header(iunit,ncolumns) if (wordlength > maxwordlength) then write(string1,*)'unexpected long word ... starts <',& input_line((i-wordlength):(i-1)),'>' - call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) + call error_handler(E_ERR,'decode_header',string1, source) endif columns(columncount) = input_line((i-wordlength):(i-1)) write(string1,*) 'word(',columncount,') is ',columns(columncount) @@ -513,7 +545,7 @@ subroutine decode_header(iunit,ncolumns) if ((columncount+1) /= ncolumns) then write(string1,*)'parsed wrong number of words ...' write(string2,*)'expected ',ncolumns,' got ',columncount+1 - call error_handler(E_ERR,'decode_header',string1,source,revision,revdate, & + call error_handler(E_ERR,'decode_header',string1,source, & text2=trim(string2), text3=trim(input_line)) endif @@ -552,7 +584,7 @@ subroutine decode_header(iunit,ncolumns) if (any(qc /= 0) ) then write(string1,*)'Did not find all the required column indices.' - call error_handler(E_ERR,'decode_header',string1, source, revision, revdate) + call error_handler(E_ERR,'decode_header',string1, source) endif ! Summarize if desired @@ -633,7 +665,7 @@ function CheckIndex( myindex, context ) if (myindex == 0) then write(string1,*)'Did not find column header matching ',trim(context) - call error_handler(E_MSG,'decode_header:CheckIndex',string1, source, revision, revdate) + call error_handler(E_MSG,'decode_header:CheckIndex',string1, source) CheckIndex = -1 ! not a good thing else CheckIndex = 0 ! Good to go @@ -661,7 +693,7 @@ subroutine stringparse(str1, nwords, linenum) read(str1,*,iostat=rcio) values if (rcio /= 0) then write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' - call error_handler(E_ERR,'stringparse',string1, source, revision, revdate) + call error_handler(E_ERR,'stringparse',string1, source) endif ! Stuff what we want into the tower structure @@ -716,8 +748,8 @@ subroutine stringparse(str1, nwords, linenum) if (verbose) then write(string1,*)'converting ',tower%month,tower%day,tower%hour,tower%doy write(string2,*)'the day-of-year indicates we should amend the month/day values.' - call error_handler(E_MSG,'stringparse', string1, source, revision, & - revdate, text2=string2) + call error_handler(E_MSG,'stringparse', string1, source, & + text2=string2) call print_date(time0, 'stringparse: using ymd date is') call print_date(time1, 'stringparse: using doy date is') @@ -762,93 +794,3 @@ end subroutine stringparse end program level4_to_obs -! LEVEL 4 VARIABLE DESCRIPTION -! -! Variables description: -! Level 4 data are obtained from the level 3 products, data are ustar filtered, -! gap-filled using different methods and partitioned. -! Datasets are also aggregated from daily to monthly. -! Flags with information regarding quality of the original and gapfilled data are added. -! -! Half hourly dataset variables description: -! -! - Month : from 1 to 12 -! - Day : day of the month -! - Hour : from 0 to 23.5, indicates the end of the half hour of measurement -! - DoY : decimal day of the year -! - Rg_f : global radiation filled [W m-2] -! - Rg_fqc : global radiation quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - Ta_f : air temperature filled [deg C] -! - Ta_fqc : air temperature quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - VPD_f : vapour pressure deficit [hPa] -! - VPD_fqc : vapour pressure deficit quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - Ts_f : soil temperature filled [deg C] -! - Ts_fqc : soil temperature quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - Precip : precipitation [mm] -! - SWC : soil water content [%vol] -! - H_f : sensible heat flux filled [W m-2] -! - H_fqc : sensible heat flux quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - LE_f : latent heat flux filled [W m-2] -! - LE_fqc : latent heat flux quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - qf_NEE_st : fluxes quality flags as defined in the Level3 product -! - qf_NEE_or : fluxes quality flags as defined in the Level3 product -! - Reco_st : Estimated ecosystem respiration according to the short-term temperature -! response of night-time fluxes based on NEE_st -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! [umolCO2 m-2 s-1] -! - Reco_or : Estimated ecosystem respiration according to the short-term temperature -! response of night-time fluxes based on NEE_or -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! [umolCO2 m-2 s-1] -! - NEE_st_fMDS : NEE_st filled using the Marginal Distribution Sampling method -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! [umolCO2 m-2 s-1] -! - NEE_st_fMDSqc : NEE_st_fMDS quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - GPP_st_MDS : Gross Primary Production calculated as GPP_st_MDS = Reco_st - NEE_st_MDS -! [umolCO2 m-2 s-1] -! - NEE_or_fMDS : NEE_or filled using the Marginal Distribution Sampling method -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! [umolCO2 m-2 s-1] -! - NEE_or_fMDSqc : NEE_or_fMDS quality flags: -! 0 = original, 1 = A (most reliable), 2 = B (medium), 3 = C (least reliable). -! (Refer to Reichstein et al. 2005 Global Change Biology ) -! - GPP_or_MDS : Gross Primary Production calculated as GPP_or_MDS = Reco_or - NEE_or_MDS -! [umolCO2 m-2 s-1] -! - NEE_st_fANN : NEE_st filled using the Artificial Neural Network method -! (Refer to Papale et al. 2003 Global Change Biology and to the Other Information section in this document) -! [umolCO2 m-2 s-1] -! - NEE_st_fANNqc : NEE_st_fANN quality flags: -! 0 = original, 1 = filled using original meteorological inputs or filled with qc=1, -! 2 = filled using filled meteorological inputs with qc=2 or 3, -! 3 = not filled using ANN due to one or more input missed but filled with the MDS method -! - GPP_st_ANN : Gross Primary Production calculated as GPP_st_ ANN = Reco_st - NEE_st_ ANN -! [umolCO2 m-2 s-1] -! - NEE_or_f ANN : NEE_or filled using the Artificial Neural Network method -! (Refer to Papale et al. 2003 Global Change Biology and to the Other Information section in this document) -! [umolCO2 m-2 s-1] -! - NEE_or_f ANNqc : NEE_or_fANN quality flags: -! 0 = original, 1 = filled using original meteorological inputs or filled with qc=1, -! 2 = filled using filled meteorological inputs with qc=2 or 3, -! 3 = not filled using ANN due to one or more input missed but filled with the MDS method -! - GPP_or_ ANN : Gross Primary Production calculated as GPP_or_ ANN = Reco_or - NEE_or_ ANN -! [umolCO2 m-2 s-1] - -! -! $URL$ -! $Id$ -! $Revision$ -! $Date$ diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index 4bf70b0e1c..7cd3caf184 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -1,4 +1,4 @@ -&fluxnetfull_to_obs_nml +&Fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', year = -1, diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index d72ff74253..b316ace71e 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -1,4 +1,4 @@ -PROGRAM ``fluxnetfull_to_obs`` +PROGRAM ``Fluxnetfull_to_obs`` ========================= Overview @@ -25,9 +25,9 @@ The workflow is usually: #. record the TIME ZONE, latitude, longitude, and elevation for each tower #. build the DART executables with support for the tower observations. This is done by running ``preprocess`` with ``obs_def_tower_mod.f90`` in the list of ``input_files`` for ``preprocess_nml``. -#. provide basic tower information via the ``fluxnetfull_to_obs_nml`` namelist since this information is not contained in the +#. provide basic tower information via the ``Fluxnetfull_to_obs_nml`` namelist since this information is not contained in the Level 4 data file -#. convert each Level 4 data file individually using ``fluxnetfull_to_obs`` +#. convert each Level 4 data file individually using ``Fluxnetfull_to_obs`` #. combine all output files for the region and timeframe of interest into one file using :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` @@ -44,7 +44,7 @@ namelist. :: - &fluxnetfull_to_obs_nml + &Fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', year = -1, @@ -150,7 +150,7 @@ I believe it is important to be a good scientific citizen: Programs -------- -The ``fluxnetfull_to_obs.f90`` file is the source for the main converter program. Look at the source code where it reads the +The ``Fluxnetfull_to_obs.f90`` file is the source for the main converter program. Look at the source code where it reads the example data file. You will almost certainly need to change the "read" statement to match your data format. The example code reads each text line into a character buffer and then reads from that buffer to parse up the data items. diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index b9b78ef034..8da69eff7e 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -36,7 +36,7 @@ verbose = .TRUE. / -&fluxnetfull_to_obs_nml +&Fluxnetfull_to_obs_nml text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv' obs_out_file = 'obs_seq.out' year = 2003 From 9ef0a40efb09ab88be84dd3049ee6c0462152082 Mon Sep 17 00:00:00 2001 From: braczka Date: Fri, 19 May 2023 18:17:32 -0600 Subject: [PATCH 03/25] Updating decode_header in Flux obs converter --- .../Ameriflux/fluxnetfull_to_obs.f90 | 139 +++++++++++------- .../Ameriflux/fluxnetfull_to_obs.nml | 1 + 2 files changed, 89 insertions(+), 51 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 3129565c7d..fb0fc69fbf 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -195,20 +195,19 @@ program Fluxnetfull_to_obs endif -! We need to know the maximum number of observations in the input file. -! Each line has info for the 3 observations we want. -! The max possible number of obs needs to be specified but it -! will only write out the actual number created. +! Specify the maximum number of observations in the input file, +! but only the actual number created will be written out. +! Each line has 5 flux observations available. ! Each observation in this series will have a single ! observation value and a quality control flag. ! Initialize two empty observations - one to track location ! in observation sequence - the other is for the new observation. iunit = open_file(text_input_file, 'formatted', 'read') -if (verbose) call error_handler(E_MSG,'level4_to_obs','opened input file '//trim(text_input_file)) +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs','opened input file '//trim(text_input_file)) nlines = count_file_lines(iunit) -max_obs = 3*nlines +max_obs = 5*nlines num_copies = 1 num_qc = 1 first_obs = .true. @@ -223,8 +222,8 @@ program Fluxnetfull_to_obs call set_copy_meta_data(obs_seq, 1, 'observation') call set_qc_meta_data( obs_seq, 1, 'Ameriflux QC') -! The first line describes all the fields ... column headers, if you will - +! Subroutine decode_header reads obs file header and identifies the columns +! where flux variables of interest are located rewind(iunit) call decode_header(iunit, nwords) @@ -241,7 +240,10 @@ program Fluxnetfull_to_obs input_line = adjustl(bigline) - ! parse the line into the tower structure (including the observation time) + ! Parse the header line into the tower structure (including the observation time) + + !! FIXME --- Start editing here again + call stringparse(input_line, nwords, iline) if (iline <= 2) then @@ -491,9 +493,8 @@ end function count_file_lines subroutine decode_header(iunit,ncolumns) -! Reads the first line of the header and parses the information. -! And by parse, I mean determine which columns are the columns -! of interest. +! Reads the first line of the obs header and identifies whic columns +! the flux variable of interest is located integer, intent(in) :: iunit integer, intent(out) :: ncolumns @@ -554,53 +555,89 @@ subroutine decode_header(iunit,ncolumns) write(string1,*)'word(',ncolumns,') is ',columns(ncolumns) if (verbose) call error_handler(E_MSG,'decode_header',string1) -! Finally get to the task at hand - -tower%monthindex = Match(columns, tower%monthstring) -tower%dayindex = Match(columns, tower%daystring) -tower%hourindex = Match(columns, tower%hourstring) -tower%doyindex = Match(columns, tower%doystring) -tower%hindex = Match(columns, tower%hstring) -tower%hQCindex = Match(columns, tower%hQCstring) -tower%leindex = Match(columns, tower%lestring) -tower%leQCindex = Match(columns, tower%leQCstring) -tower%neeindex = Match(columns, tower%neestring) -tower%neeQCindex = Match(columns, tower%neeQCstring) - -! FIXME ... find a column marked 'year' or 'error_var' and use if possible. - -! Check to make sure we got all the indices we need - -qc( 1) = CheckIndex( tower%monthindex, tower%monthstring) -qc( 2) = CheckIndex( tower%dayindex , tower%daystring) -qc( 3) = CheckIndex( tower%hourindex , tower%hourstring) -qc( 4) = CheckIndex( tower%doyindex , tower%doystring) -qc( 5) = CheckIndex( tower%hindex , tower%hstring) -qc( 6) = CheckIndex( tower%hQCindex , tower%hQCstring) -qc( 7) = CheckIndex( tower%leindex , tower%lestring) -qc( 8) = CheckIndex( tower%leQCindex , tower%leQCstring) -qc( 9) = CheckIndex( tower%neeindex , tower%neestring) -qc(10) = CheckIndex( tower%neeQCindex, tower%neeQCstring) +! Finally, identify column index based on string name +tower%startindex = Match(columns, tower%startstring) +tower%endindex = Match(columns, tower%endstring) +tower%neeindex = Match(columns, tower%neestring) +tower%neeUNCindex = Match(columns, tower%neeUNCstring) +tower%neeQCindex = Match(columns, tower%neeQCstring) +tower%neeUNCQCindex = Match(columns, tower%neeUNCstring) +tower%leindex = Match(columns, tower%lestring) +tower%leQCindex = Match(columns, tower%leQCstring) +tower%leUNCindex = Match(columns, tower%leUNCstring) +tower%hindex = Match(columns, tower%hstring) +tower%hQCindex = Match(columns, tower%hQCstring) +tower%hUNCindex = Match(columns, tower%hUNCstring) +tower%gppDTindex = Match(columns, tower%gppDTstring) +tower%gppNTindex = Match(columns, tower%gppNTstring) +tower%recoDTindex = Match(columns, tower%recoDTstring) +tower%recoNTindex = Match(columns, tower%recoNTstring) +tower%gppUNCDT16index = Match(columns, tower%gppUNCDT16string) +tower%gppUNCDT84index = Match(columns, tower%gppUNCDT84string) +tower%gppUNCNT16index = Match(columns, tower%gppUNCNT16string) +tower%gppUNCNT84index = Match(columns, tower%gppUNCNT84string) +tower%recoUNCDT16index = Match(columns, tower%recoUNCDT16string) +tower%recoUNCDT84index = Match(columns, tower%recoUNCDT84string) +tower%recoUNCNT16index = Match(columns, tower%recoUNCNT16string) +tower%recoUNCNT84index = Match(columns, tower%recoUNCNT84string) + +! Confirm indices were found successfully +qc( 1) = CheckIndex( tower%startindex , tower%startstring) +qc( 2) = CheckIndex( tower%endindex , tower%endstring) +qc( 3) = CheckIndex( tower%neeindex , tower%neestring) +qc( 4) = CheckIndex( tower%neeUNCindex , tower%neeUNCstring) +qc( 5) = CheckIndex( tower%neeQCindex , tower%neeQCstring) +qc( 6) = CheckIndex( tower%neeUNCQCindex , tower%neeUNCstring) +qc( 7) = CheckIndex( tower%leindex , tower%lestring) +qc( 8) = CheckIndex( tower%leQCindex , tower%leQCstring) +qc( 9) = CheckIndex( tower%leUNCindex , tower%leUNCstring) +qc(10) = CheckIndex( tower%hindex , tower%hstring) +qc(11) = CheckIndex( tower%hQCindex , tower%hQCstring) +qc(12) = CheckIndex( tower%hUNCindex , tower%hUNCstring) +qc(13) = CheckIndex( tower%gppDTindex , tower%gppDTstring) +qc(14) = CheckIndex( tower%gppNTindex , tower%gppNTstring) +qc(15) = CheckIndex( tower%recoDTindex , tower%recoDTstring) +qc(16) = CheckIndex( tower%recoNTindex , tower%recoNTstring) +qc(17) = CheckIndex( tower%gppUNCDT16index , tower%gppUNCDT16string) +qc(18) = CheckIndex( tower%gppUNCDT84index , tower%gppUNCDT84string) +qc(19) = CheckIndex( tower%gppUNCNT16index , tower%gppUNCNT16string) +qc(20) = CheckIndex( tower%gppUNCNT84index , tower%gppUNCNT84string) +qc(21) = CheckIndex( tower%recoUNCDT16index, tower%recoUNCDT16string) +qc(22) = CheckIndex( tower%recoUNCDT84index, tower%recoUNCDT84string) +qc(23) = CheckIndex( tower%recoUNCNT16index, tower%recoUNCNT16string) +qc(24) = CheckIndex( tower%recoUNCNT84index, tower%recoUNCNT84string) if (any(qc /= 0) ) then write(string1,*)'Did not find all the required column indices.' call error_handler(E_ERR,'decode_header',string1, source) endif -! Summarize if desired - if (verbose) then 110 format('index for ',A20,' is ',i3) - write(*,110)tower%monthstring, tower%monthindex - write(*,110)tower%daystring , tower%dayindex - write(*,110)tower%hourstring , tower%hourindex - write(*,110)tower%doystring , tower%doyindex - write(*,110)tower%hstring , tower%hindex - write(*,110)tower%hQCstring , tower%hQCindex - write(*,110)tower%lestring , tower%leindex - write(*,110)tower%leQCstring , tower%leQCindex - write(*,110)tower%neestring , tower%neeindex - write(*,110)tower%neeQCstring, tower%neeQCindex + write(*,110)tower%startstring ,tower%startindex + write(*,110)tower%endstring , tower%endindex + write(*,110)tower%neestring , tower%neeindex + write(*,110)tower%neeUNCstring , tower%neeUNCindex + write(*,110)tower%neeQCstring , tower%neeQCindex + write(*,110)tower%neeUNCstring , tower%neeUNCindex + write(*,110)tower%lestring , tower%leindex + write(*,110)tower%leQCstring , tower%leQCindex + write(*,110)tower%leUNCstring , tower%leUNCindex + write(*,110)tower%hstring , tower%hindex + write(*,110)tower%hQCstring , tower%hQCindex + write(*,110)tower%hUNCstring , tower%hUNCindex + write(*,110)tower%gppDTstring , tower%gppDTindex + write(*,110)tower%gppNTstring , tower%gppNTindex + write(*,110)tower%recoDTstring , tower%recoDTindex + write(*,110)tower%recoNTstring , tower%recoNTindex + write(*,110)tower%gppUNCDT16string , tower%gppUNCDT16index + write(*,110)tower%gppUNCDT84string , tower%gppUNCDT84index + write(*,110)tower%gppUNCNT16string , tower%gppUNCNT16index + write(*,110)tower%gppUNCNT84string , tower%gppUNCNT84index + write(*,110)tower%recoUNCDT16string, tower%recoUNCDT16index + write(*,110)tower%recoUNCDT84string, tower%recoUNCDT84index + write(*,110)tower%recoUNCNT16string, tower%recoUNCNT16index + write(*,110)tower%recoUNCNT84string, tower%recoUNCNT84index endif deallocate(columns) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index 7cd3caf184..da089f9cbe 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -8,6 +8,7 @@ elevation = -1.0, flux_height = -1.0, maxgoodqc = 3, + gap_filled = .false. verbose = .false. / From 794c8509ecc1687d82cea85fa9d2bf63179842fa Mon Sep 17 00:00:00 2001 From: braczka Date: Mon, 22 May 2023 14:06:06 -0600 Subject: [PATCH 04/25] Modify stringparse for Fluxnet obs --- .../Ameriflux/fluxnetfull_to_obs.f90 | 159 +++++++++--------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index fb0fc69fbf..8edc68afc7 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -78,6 +78,9 @@ program Fluxnetfull_to_obs type(time_type) :: prev_time, offset real(r8), parameter :: umol_to_gC = (1.0_r8/1000000.0_r8) * 12.0_r8 +! Fixme: These are for high resolution format HH or HR +! Fixme: If aggregrated (DD,WW,MM) need to edit + type towerdata type(time_type) :: time_obs character(len=20) :: startstring = 'TIMESTAMP_START' @@ -133,6 +136,8 @@ program Fluxnetfull_to_obs integer :: minute real(r8) :: hour real(r8) :: doy + character(len=12) :: start + character(len=12) :: end real(r8) :: nee real(r8) :: neeUNC integer :: neeQC @@ -144,10 +149,16 @@ program Fluxnetfull_to_obs integer :: hQC real(r8) :: gppNT real(r8) :: gppDT + real(r8) :: gppNTQC + real(r8) :: gppDTQC real(r8) :: gppUNCNT84 real(r8) :: gppUNCNT16 real(r8) :: gppUNCDT84 real(r8) :: gppUNCDT16 + real(r8) :: recoNT + real(r8) :: recoDT + real(r8) :: recoNTQC + real(r8) :: recoDTQC real(r8) :: recoUNCNT84 real(r8) :: recoUNCNT16 real(r8) :: recoUNCDT84 @@ -220,7 +231,7 @@ program Fluxnetfull_to_obs ! the first one needs to contain the string 'observation' and the ! second needs the string 'QC'. call set_copy_meta_data(obs_seq, 1, 'observation') -call set_qc_meta_data( obs_seq, 1, 'Ameriflux QC') +call set_qc_meta_data( obs_seq, 1, 'Fluxnet QC') ! Subroutine decode_header reads obs file header and identifies the columns ! where flux variables of interest are located @@ -493,7 +504,7 @@ end function count_file_lines subroutine decode_header(iunit,ncolumns) -! Reads the first line of the obs header and identifies whic columns +! Reads the first line of the obs header and identifies which columns ! the flux variable of interest is located integer, intent(in) :: iunit @@ -514,7 +525,7 @@ subroutine decode_header(iunit,ncolumns) input_line = adjustl(bigline) -! Count how many commas are in the line - use this to determine how many columns +! Comma separated file, thus count commas to determine the number of columns charcount = CountChar(input_line,',') ncolumns = charcount + 1 @@ -720,8 +731,9 @@ subroutine stringparse(str1, nwords, linenum) integer , intent(in) :: linenum real(r8), allocatable, dimension(:) :: values -integer :: iday, ihour, imin, isec, seconds -type(time_type) :: time0, time1, time2 +integer :: iyear0, imonth0, iday0, ihour0, imin0 +integer :: iyear1, imonth1, iday1, ihour1, imin1 +type(time_type) :: time_start, time_end allocate(values(nwords)) @@ -733,101 +745,96 @@ subroutine stringparse(str1, nwords, linenum) call error_handler(E_ERR,'stringparse',string1, source) endif -! Stuff what we want into the tower structure +! Assign flux observations, uncertainty and QC to tower structure ! +! Fixme: These are for high resolution format HH or HR only +! Fixme: If aggregrated (DD,WW,MM) need to edit, expand this + +! start,end format YYYYMMDDHHMM +! nee units [umolCO2 m-2 s-1], VUT_REF +! neeUNC units [umolCO2 m-2 s-1], joint +! neeQC no dim 0=measured,gap_filled: 1=good,2=medium,3=poor +! le units [W m-2] +! leUNC units [W m-2], random +! leQC no dim 0=measured,gap_filled: 1=good gf,2=medium,3=poor +! h units [W m-2] +! hUNC units [W m-2], random +! hQC no dim 0=measured,gap_filled: 1=good gf,2=medium,3=poor +! gppDT units [umolCO2 m-2 s-1] VUT_REF daytime partition +! gppNT units [umolCO2 m-2 s-1] VUT_REF nighttime partition +! gppUNCDT[xx] units [umolCO2 m-2 s-1] 16,84 percentile +! gppUNCNT[xx] units [umolCO2 m-2 s-1] 16,84 percentile +! recoDT units [umolCO2 m-2 s-1] VUT_REF daytime partition +! recoNT units [umolCO2 m-2 s-1] VUT_REF nighttime partition +! recoUNCDT[xx] units [umolCO2 m-2 s-1] 16,84 percentile +! recoUNCNT[xx] units [umolCO2 m-2 s-1] 16,84 percentile + ! Convert to 'CLM-friendly' units AFTER we determine observation error variance. ! That happens in the main routine. -! -! NEE_or_fMDS has units [umolCO2 m-2 s-1] -! H_f has units [W m-2] -! LE_f has units [W m-2] -! -! (CLM) NEE has units [gC m-2 s-1] - -tower%month = nint(values(tower%monthindex)) -tower%day = nint(values(tower%dayindex )) -tower%hour = values(tower%hourindex ) -tower%doy = values(tower%doyindex ) -tower%nee = values(tower%neeindex ) -tower%neeQC = nint(values(tower%neeQCindex)) -tower%le = values(tower%leindex ) -tower%leQC = nint(values(tower%leQCindex )) -tower%h = values(tower%hindex ) -tower%hQC = nint(values(tower%hQCindex )) +! Fixme: Double check these defs and units +! (CLM) NEE,GPP,ER units [gC m-2 s-1] +! (CLM) LE,SH units [W m-2] + +tower%start = values(tower%startindex ) +tower%end = values(tower%endindex ) +tower%nee = values(tower%neeindex ) +tower%nee = values(tower%neeUNCindex ) +tower%neeQC = nint(values(tower%neeQCindex )) +tower%le = values(tower%leindex ) +tower%leUNC = values(tower%leUNCindex ) +tower%leQC = nint(values(tower%leQCindex )) +tower%h = values(tower%hindex ) +tower%hUNC = values(tower%hUNCindex ) +tower%hQC = nint(values(tower%hQCindex )) +tower%gppDT = values(tower%gppDTindex ) +tower%gppNT = values(tower%gppNTindex ) +tower%recoDT = values(tower%recoDTindex ) +tower%recoNT = values(tower%recoNTindex ) +tower%gppUNCDT16 = values(tower%gppUNCDT16index ) +tower%gppUNCDT84 = values(tower%gppUNCDT84index ) +tower%gppUNCNT16 = values(tower%gppUNCNT16index ) +tower%gppUNCNT84 = values(tower%gppUNCNT84index ) +tower%recoUNCDT16 = values(tower%recoUNCDT16index) +tower%recoUNCDT84 = values(tower%recoUNCDT84index) +tower%recoUNCNT16 = values(tower%recoUNCNT16index) +tower%recoUNCNT84 = values(tower%recoUNCNT84index) deallocate(values) -! decode the time pieces ... two times ... -! The LAST line of these files is knackered ... and we have to check that -! if the doy is greater than the ymd ... -! 12,31,23.5,366.979 N-1 -! 1, 1, 0.0,367.000 N - -iday = int(tower%doy) -ihour = int(tower%hour) -seconds = nint((tower%hour - real(ihour,r8))*3600) -imin = seconds / 60 -isec = seconds - imin * 60 -time0 = set_date(year, tower%month, tower%day, ihour, imin, isec) - -isec = ihour*3600 + imin*60 + isec -time1 = set_date(year,1,1,0,0,0) + set_time(isec, (iday-1)) -time2 = time0 - time1 - -call get_time(time2, isec, iday) +! The observation time must be assigned through the flux timestamp string -if ( iday > 0 ) then - ! FIXME we need to change the day ... - ! This blows up if you try to use a non-leap year with leapyear ... +write(tower%start,'(i4, 4i2)') iyear0,imonth0,iday0,ihour0,imin0 +write(tower%end,'(i4, 4i2)') iyear1,imonth1,iday1,ihour1,imin1 - tower%time_obs = time1 +time_start= set_date(iyear0,imonth0,iday0,ihour0,imin0,0) +time_end= set_date(iyear1,imonth1,iday1,ihour1,imin0,0) - if (verbose) then - write(string1,*)'converting ',tower%month,tower%day,tower%hour,tower%doy - write(string2,*)'the day-of-year indicates we should amend the month/day values.' - call error_handler(E_MSG,'stringparse', string1, source, & - text2=string2) - - call print_date(time0, 'stringparse: using ymd date is') - call print_date(time1, 'stringparse: using doy date is') - call print_time(time0, 'stringparse: using ymd time is') - call print_time(time1, 'stringparse: using doy time is') - call print_time(time2, 'stringparse: difference is') - - call print_date(time0, 'stringparse: using ymd date is',logfileunit) - call print_date(time1, 'stringparse: using doy date is',logfileunit) - call print_time(time0, 'stringparse: using ymd time is',logfileunit) - call print_time(time1, 'stringparse: using doy time is',logfileunit) - call print_time(time2, 'stringparse: difference is',logfileunit) - endif -else - - tower%time_obs = time0 +! Assign average of flux time window to obs_seq (DART time) +tower%time_obs = (time_start+time_end) / 2 -endif - -! 8AM East Coast is 1PM Greenwich +! Covert from Fluxnet provided LTC to UTC, UTC is standard for DART and CLM +! For example, EST = UTC-5 if (timezoneoffset < 0.0_r8) then tower%time_obs = tower%time_obs + offset else tower%time_obs = tower%time_obs - offset endif -! The QC values can be 'missing' ... in which case the values are too - +! If missing value (-9999) manually assign poor QC value +! such that value is excluded in obs_seq if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 if (tower%leQC < 0) tower%leQC = maxgoodqc + 1000 if (tower%hQC < 0) tower%hQC = maxgoodqc + 1000 -! if (tower%neeQC < maxgoodqc) then -! write(*,*)'nee umol m-2 s-1 ',tower%nee -! write(*,*)'nee gC m-2 s-1 ',tower%nee*umol_to_gC -! endif +if (tower%gppNT < 0) tower%gppNTQC = maxgoodqc + 1000 +if (tower%gppDT < 0) tower%gppDTQC = maxgoodqc + 1000 +if (tower%recoNT < 0) tower%recoNTQC = maxgoodqc + 1000 +if (tower%recoDT < 0) tower%recoDTQC = maxgoodqc + 1000 end subroutine stringparse -end program level4_to_obs +end program Fluxnetfull_to_obs From a746d46aae5b39b82d0e6b56268d84cd2c40f2a7 Mon Sep 17 00:00:00 2001 From: braczka Date: Mon, 22 May 2023 18:31:32 -0600 Subject: [PATCH 05/25] Added GPP and ER obs to Fluxnet converter --- .../Ameriflux/fluxnetfull_to_obs.f90 | 163 +++++++++++------- 1 file changed, 105 insertions(+), 58 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 8edc68afc7..42f50082cc 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -73,6 +73,7 @@ program Fluxnetfull_to_obs integer :: oday, osec, rcio, iunit integer :: num_copies, num_qc, max_obs real(r8) :: oerr, qc +real(r8) :: sig_gppdt, sig_gppnt, sig_recodt, sig_recont type(obs_sequence_type) :: obs_seq type(obs_type) :: obs, prev_obs type(time_type) :: prev_time, offset @@ -130,14 +131,8 @@ program Fluxnetfull_to_obs integer :: recoUNCNT16index integer :: recoUNCNT84index - integer :: year - integer :: month - integer :: day - integer :: minute - real(r8) :: hour - real(r8) :: doy - character(len=12) :: start - character(len=12) :: end + character(len=12) :: start_time + character(len=12) :: end_time real(r8) :: nee real(r8) :: neeUNC integer :: neeQC @@ -149,6 +144,7 @@ program Fluxnetfull_to_obs integer :: hQC real(r8) :: gppNT real(r8) :: gppDT + real(r8) :: gpp real(r8) :: gppNTQC real(r8) :: gppDTQC real(r8) :: gppUNCNT84 @@ -157,6 +153,7 @@ program Fluxnetfull_to_obs real(r8) :: gppUNCDT16 real(r8) :: recoNT real(r8) :: recoDT + real(r8) :: reco real(r8) :: recoNTQC real(r8) :: recoDTQC real(r8) :: recoUNCNT84 @@ -252,9 +249,6 @@ program Fluxnetfull_to_obs input_line = adjustl(bigline) ! Parse the header line into the tower structure (including the observation time) - - !! FIXME --- Start editing here again - call stringparse(input_line, nwords, iline) if (iline <= 2) then @@ -262,32 +256,60 @@ program Fluxnetfull_to_obs call print_date(tower%time_obs, ' first observation date (local time) is') call print_time(tower%time_obs, ' first observation time (local time) is') write(*,*)'first observation raw values: (column,string,value) timezone not applied' - write(*,*)tower%monthindex, tower%monthstring , tower%month - write(*,*)tower%dayindex , tower%daystring , tower%day - write(*,*)tower%hourindex , tower%hourstring , tower%hour - write(*,*)tower%doyindex , tower%doystring , tower%doy - write(*,*)tower%hindex , tower%hstring , tower%h - write(*,*)tower%hQCindex , tower%hQCstring , tower%hQC - write(*,*)tower%leindex , tower%lestring , tower%le - write(*,*)tower%leQCindex , tower%leQCstring , tower%leQC - write(*,*)tower%neeindex , tower%neestring , tower%nee - write(*,*)tower%neeQCindex, tower%neeQCstring , tower%neeQC + write(*,*)tower%hindex , tower%hstring , tower%h + write(*,*)tower%hUNCindex , tower%hUNCstring , tower%hUNC + write(*,*)tower%hQCindex , tower%hQCstring , tower%hQC + write(*,*)tower%leindex , tower%lestring , tower%le + write(*,*)tower%leUNCindex , tower%leUNCstring , tower%leUNC + write(*,*)tower%leQCindex , tower%leQCstring , tower%leQC + write(*,*)tower%neeindex , tower%neestring , tower%nee + write(*,*)tower%neeUNCindex , tower%neeUNCstring , tower%UNCnee + write(*,*)tower%neeQCindex , tower%neeQCstring , tower%neeQC + write(*,*)tower%gppDTindex , tower%gppDTstring , tower%gppDT + write(*,*)tower%gppDTUNC16index , tower%gppDTUNC16string , tower%gppDTUNC16 + write(*,*)tower%gppDTUNC84index , tower%gppDTUNC84string , tower%gppDTUNC84 + write(*,*)tower%gppDTQC + write(*,*)tower%gppNTindex , tower%gppNTstring , tower%gppNT + write(*,*)tower%gppNTUNC16index , tower%gppNTUNC16string , tower%gppNTUNC16 + write(*,*)tower%gppNTUNC84index , tower%gppNTUNC84string , tower%gppNTUNC84 + write(*,*)tower%gppNTQC + write(*,*)tower%recoDTindex , tower%recoDTstring , tower%recoDT + write(*,*)tower%recoDTUNC16index, tower%recoDTUNC16string, tower%recoDTUNC16 + write(*,*)tower%recoDTUNC84index, tower%recoDTUNC84string, tower%recoDTUNC84 + write(*,*)tower%recoDTQC + write(*,*)tower%recoNTindex , tower%recoNTstring , tower%recoNT + write(*,*)tower%recoNTUNC16index, tower%recoNTUNC16string, tower%recoNTUNC16 + write(*,*)tower%recoNTUNC84index, tower%recoNTUNC84string, tower%recoNTUNC84 + write(*,*)tower%recoNTQC write(*,*)'' write(logfileunit,*)'' call print_date(tower%time_obs, ' first observation date (local time) is',logfileunit) call print_time(tower%time_obs, ' first observation time (local time) is',logfileunit) write(logfileunit,*)'first observation raw values: (column,string,value) timezone not applied' - write(logfileunit,*)tower%monthindex, tower%monthstring , tower%month - write(logfileunit,*)tower%dayindex , tower%daystring , tower%day - write(logfileunit,*)tower%hourindex , tower%hourstring , tower%hour - write(logfileunit,*)tower%doyindex , tower%doystring , tower%doy - write(logfileunit,*)tower%hindex , tower%hstring , tower%h - write(logfileunit,*)tower%hQCindex , tower%hQCstring , tower%hQC - write(logfileunit,*)tower%leindex , tower%lestring , tower%le - write(logfileunit,*)tower%leQCindex , tower%leQCstring , tower%leQC - write(logfileunit,*)tower%neeindex , tower%neestring , tower%nee - write(logfileunit,*)tower%neeQCindex, tower%neeQCstring , tower%neeQC + write(logfileunit,*)tower%hindex , tower%hstring , tower%h + write(logfileunit,*)tower%hUNCindex , tower%hUNCstring , tower%hUNC + write(logfileunit,*)tower%hQCindex , tower%hQCstring , tower%hQC + write(logfileunit,*)tower%leindex , tower%lestring , tower%le + write(logfileunit,*)tower%leUNCindex , tower%leUNCstring , tower%leUNC + write(logfileunit,*)tower%leQCindex , tower%leQCstring , tower%leQC + write(logfileunit,*)tower%neeindex , tower%neestring , tower%nee + write(logfileunit,*)tower%neeQCindex , tower%neeQCstring , tower%neeQC + write(logfileunit,*)tower%gppDTUNC16index , tower%gppDTUNC16string , tower%gppDTUNC16 + write(logfileunit,*)tower%gppDTUNC84index , tower%gppDTUNC84string , tower%gppDTUNC84 + write(logfileunit,*)tower%gppDTQC + write(logfileunit,*)tower%gppNTindex , tower%gppNTstring , tower%gppNT + write(logfileunit,*)tower%gppNTUNC16index , tower%gppNTUNC16string , tower%gppNTUNC16 + write(logfileunit,*)tower%gppNTUNC84index , tower%gppNTUNC84string , tower%gppNTUNC84 + write(logifleunit,*)tower%gppNTQC + write(logfileunit,*)tower%recoDTindex , tower%recoDTstring , tower%recoDT + write(logfileunit,*)tower%recoDTUNC16index, tower%recoDTUNC16string, tower%recoDTUNC16 + write(logfileunit,*)tower%recoDTUNC84index, tower%recoDTUNC84string, tower%recoDTUNC84 + write(logfileunit,*)tower%recoDTQC + write(logfileunit,*)tower%recoNTindex , tower%recoNTstring , tower%recoNT + write(logfileunit,*)tower%recoNTUNC16index, tower%recoNTUNC16string, tower%recoNTUNC16 + write(logfileunit,*)tower%recoNTUNC84index, tower%recoNTUNC84string, tower%recoNTUNC84 + write(logfileunit,*)tower%recoNTQC write(logfileunit,*)'' end if @@ -299,14 +321,10 @@ program Fluxnetfull_to_obs call print_date(tower%time_obs, trim(string1),logfileunit) endif - ! make an obs derived type, and then add it to the sequence - ! If the QC value is good, use the observation. - ! Increasingly larger QC values are more questionable quality data. - ! The observation errors are from page 183, Table 7.1(A) in - ! Chapter 7 of a book by A.D. Richardson et al. via Andy Fox. - + ! Create and add observation and uncertainty (1 SD) to obs_seq file + ! Assign the observation the appropriate obs type if (tower%hQC <= maxgoodqc) then ! Sensible Heat Flux [W m-2] - oerr = 10.0_r8 + abs(tower%h)*0.22_r8 + oerr = tower%hUNC qc = real(tower%hQC,r8) call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%h, & TOWER_SENSIBLE_HEAT_FLUX, oerr, oday, osec, qc, obs) @@ -314,32 +332,56 @@ program Fluxnetfull_to_obs endif if (tower%leQC <= maxgoodqc) then ! Latent Heat Flux [W m-2] - oerr = 10.0_r8 + abs(tower%le)*0.32_r8 + oerr = tower%leUNC qc = real(tower%leQC,r8) call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%le, & TOWER_LATENT_HEAT_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) endif - if (tower%neeQC <= maxgoodqc) then ! Net Ecosystem Exchange [umol m-2 s-1] - if (tower%nee <= 0) then - oerr = (2.0_r8 + abs(tower%nee)*0.1_r8) * umol_to_gC - else - oerr = (2.0_r8 + abs(tower%nee)*0.4_r8) * umol_to_gC - endif - tower%nee = -tower%nee * umol_to_gC ! to match convention in CLM [gC m-2 s-1] + if (tower%neeQC <= maxgoodqc) then ! Net Ecosystem Exchange [umol m-2 s-1] + oerr = tower%neeUNC * umol_to_gC + tower%nee = -tower%nee * umol_to_gC ! Matches units in CLM [gC m-2 s-1] qc = real(tower%neeQC,r8) call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%nee, & TOWER_NETC_ECO_EXCHANGE, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) endif + if (tower%gppDTQC .and. tower%gppNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] + sig_gppdt = (((tower%gppDTUNC84-tower%gppDTUNC16 / 2))^2)^0.5 ! Ustar unc contribution + sig_gppnt = (((tower%gppNTUNC84-tower%gppNTUNC16 / 2))^2)^0.5 + + oerr = (0.25 * (sig_gppdt)^2 + 0.25 * (sig_gppnt)^2)^0.5 ! Combine Ustar and partitioning unc + oerr = oerr * umol_to_gC + ! Take average of night and day partitioning methods + tower%gpp = ((tower%gppDT + tower%gppNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] + qc = maxval(real(tower%gppDTQC,r8),real(tower%gppNTQC,r8)) + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%gpp, & + TOWER_GPP_FLUX, oerr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + endif + + if (tower%recoDTQC .and. tower%recoNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] + sig_recodt = (((tower%recoDTUNC84-tower%recoDTUNC16 / 2))^2)^0.5 ! Ustar unc contribution + sig_recont = (((tower%recoNTUNC84-tower%recoNTUNC16 / 2))^2)^0.5 + + oerr = (0.25 * (sig_recodt)^2 + 0.25 * (sig_recont)^2)^0.5 ! Combine Ustar and partitioning unc + oerr = oerr * umol_to_gC + ! Take average of night and day partitioning methods + tower%reco = ((tower%recoDT + tower%recoNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] + qc = maxval(real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)) + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%reco, & + TOWER_ER_FLUX, oerr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + endif + end do obsloop -! if we added any obs to the sequence, write it out to a file now. +! If obs added to the sequence, write it out to a file now. if ( get_num_obs(obs_seq) > 0 ) then write(string1,*)'writing obs_seq, obs_count = ', get_num_obs(obs_seq) - if (verbose) call error_handler(E_MSG,'level4_to_obs',string1) + if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) call write_obs_seq(obs_seq, obs_out_file) endif @@ -705,7 +747,7 @@ function CheckIndex( myindex, context ) ! Routine to issue a warning if the index was not found. ! Returns an error code ... 0 means the index WAS found ! a negative number means the index was NOT found - an error condition. -! I want to check ALL the indexes before fatally ending. +! ALL indices checked before fatally ending. integer :: CheckIndex integer, intent(in) :: myindex @@ -733,7 +775,7 @@ subroutine stringparse(str1, nwords, linenum) real(r8), allocatable, dimension(:) :: values integer :: iyear0, imonth0, iday0, ihour0, imin0 integer :: iyear1, imonth1, iday1, ihour1, imin1 -type(time_type) :: time_start, time_end +type(time_type) :: date_start, date_end allocate(values(nwords)) @@ -750,7 +792,7 @@ subroutine stringparse(str1, nwords, linenum) ! Fixme: These are for high resolution format HH or HR only ! Fixme: If aggregrated (DD,WW,MM) need to edit, expand this -! start,end format YYYYMMDDHHMM +! start_time,end_time format YYYYMMDDHHMM ! nee units [umolCO2 m-2 s-1], VUT_REF ! neeUNC units [umolCO2 m-2 s-1], joint ! neeQC no dim 0=measured,gap_filled: 1=good,2=medium,3=poor @@ -776,8 +818,8 @@ subroutine stringparse(str1, nwords, linenum) ! (CLM) NEE,GPP,ER units [gC m-2 s-1] ! (CLM) LE,SH units [W m-2] -tower%start = values(tower%startindex ) -tower%end = values(tower%endindex ) +tower%start_time = values(tower%startindex ) +tower%end_end = values(tower%endindex ) tower%nee = values(tower%neeindex ) tower%nee = values(tower%neeUNCindex ) tower%neeQC = nint(values(tower%neeQCindex )) @@ -803,14 +845,14 @@ subroutine stringparse(str1, nwords, linenum) ! The observation time must be assigned through the flux timestamp string -write(tower%start,'(i4, 4i2)') iyear0,imonth0,iday0,ihour0,imin0 -write(tower%end,'(i4, 4i2)') iyear1,imonth1,iday1,ihour1,imin1 +write(tower%start_time,'(i4, 4i2)') iyear0,imonth0,iday0,ihour0,imin0 +write(tower%end_time, '(i4, 4i2)') iyear1,imonth1,iday1,ihour1,imin1 -time_start= set_date(iyear0,imonth0,iday0,ihour0,imin0,0) -time_end= set_date(iyear1,imonth1,iday1,ihour1,imin0,0) +date_start= set_date(iyear0,imonth0,iday0,ihour0,imin0,0) +date_end= set_date(iyear1,imonth1,iday1,ihour1,imin0,0) ! Assign average of flux time window to obs_seq (DART time) -tower%time_obs = (time_start+time_end) / 2 +tower%time_obs = (date_start+date_end) / 2 ! Covert from Fluxnet provided LTC to UTC, UTC is standard for DART and CLM ! For example, EST = UTC-5 @@ -826,6 +868,11 @@ subroutine stringparse(str1, nwords, linenum) if (tower%leQC < 0) tower%leQC = maxgoodqc + 1000 if (tower%hQC < 0) tower%hQC = maxgoodqc + 1000 +tower%gppNTQC = 1 +tower%gppDTQC = 1 +tower%recoNTQC = 1 +tower%recoDTQC = 1 + if (tower%gppNT < 0) tower%gppNTQC = maxgoodqc + 1000 if (tower%gppDT < 0) tower%gppDTQC = maxgoodqc + 1000 if (tower%recoNT < 0) tower%recoNTQC = maxgoodqc + 1000 From e383f80fdac220bbd42c63359e1b58872ab9cbf0 Mon Sep 17 00:00:00 2001 From: braczka Date: Tue, 23 May 2023 09:45:33 -0600 Subject: [PATCH 06/25] Update deprecated documentation for level4_to_obs --- .../Ameriflux/fluxnetfull_to_obs.f90 | 6 ++++-- .../Ameriflux/fluxnetfull_to_obs.rst | 11 ++++------- .../obs_converters/Ameriflux/level4_to_obs.rst | 14 ++++++++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 42f50082cc..e807dec6d2 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -53,9 +53,11 @@ program Fluxnetfull_to_obs real(r8) :: longitude = -1.0_r8 real(r8) :: elevation = -1.0_r8 real(r8) :: flux_height = -1.0_r8 +! A maxgooqc=3 allows for good,medium and poor quality gap-filled data real(r8) :: maxgoodqc = 3.0_r8 -! Latent,sensible heat and NEE have this option only -logical :: gap_filled = .false. +! Always 'true' except for latent,sensible heat and NEE for hourly time periods +! Fixme This option must be worked into the code later on +logical :: gap_filled = .true. logical :: verbose = .false. namelist /Fluxnetfull_to_obs_nml/ text_input_file, obs_out_file, & diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index b316ace71e..c21213be9d 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -6,7 +6,8 @@ Overview FLUXNET FULLSET data to DART observation sequence converter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - +! Fixme*** These need to be edited. +*********************************** | This routine is designed to convert the flux tower Level 4 data from the `AmeriFlux `__ network of observations from micrometeorological tower sites. AmeriFlux is part of `FLUXNET `__ and the converter is hoped to be a suitable starting point for the conversion of @@ -14,11 +15,6 @@ FLUXNET FULLSET data to DART observation sequence converter | The AmeriFlux Level 4 products are recorded using the local time. DART observation sequence files use GMT. For more information about AmeriFlux data products, go to http://ameriflux.lbl.gov. -.. warning:: - - There was a pretty severe bug in the converter that swapped latent heat flux and sensible heat flux. The bug was - present through revision 7200. It was corrected on 30 Dec 2016. - The workflow is usually: #. download the Level 4 data for the towers and years in question (see DATA SOURCES below) @@ -32,7 +28,7 @@ The workflow is usually: :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` For some models (CLM, for example), it is required to reorganize the observation sequence files into a series of files -that contains ONLY the observations for each assimilation. This can be achieved with the `makedaily.sh `__ +that contains ONLY the observations for each assimilation time step. This can be achieved with the `makedaily.sh `__ script. Namelist @@ -54,6 +50,7 @@ namelist. elevation = -1.0, flux_height = -1.0, maxgoodqc = 3, + gap_filled = .true. verbose = .false. / diff --git a/observations/obs_converters/Ameriflux/level4_to_obs.rst b/observations/obs_converters/Ameriflux/level4_to_obs.rst index 4b2c8b2663..b0f87d3d4a 100644 --- a/observations/obs_converters/Ameriflux/level4_to_obs.rst +++ b/observations/obs_converters/Ameriflux/level4_to_obs.rst @@ -7,12 +7,14 @@ Overview AmeriFlux level 4 data to DART observation sequence converter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -| This routine is designed to convert the flux tower Level 4 data from the `AmeriFlux `__ - network of observations from micrometeorological tower sites. AmeriFlux is part of - `FLUXNET `__ and the converter is hoped to be a suitable starting point for the conversion of - observations from FLUXNET. As of May 2012, I have not yet tried to work with any other observations from FLUXNET. -| The AmeriFlux Level 4 products are recorded using the local time. DART observation sequence files use GMT. For more - information about AmeriFlux data products, go to http://ameriflux.lbl.gov. +| This routine was designed to convert the flux tower Level 4 data from the `AmeriFlux `__ + network of observations from micrometeorological tower sites. The download link and flux data format for this code + *has been discontinued* (e.g. ``USBar2004_L4_h.txt``). Thus if you are using flux obs converters for the first time + *PLEASE USE* the updated ``Fluxnetfull_to_obs.f90`` converter and follow the documentation there :doc:`./Fluxnetfull_to_obs` + We have kept this code available if you still use the older Ameriflux data format. Also this code uses a general approach + to calculating sensible, latent and net ecosystem exchange uncertainty, that may be helpful. +| The AmeriFlux Level 4 data products are provided in the local time of the flux tower location. DART observation sequence + files are provided in UTC, thus this routine includes a time conversion. .. warning:: From 97cf892f4513eb6ea13a5228109a40bab5c1c8de Mon Sep 17 00:00:00 2001 From: braczka Date: Tue, 23 May 2023 18:58:01 -0600 Subject: [PATCH 07/25] Syntax changes to to fluxnetfull to compile Necessary changes in syntax to allow fluxnetfull_to_obs.f90 to compile correctly. Also providing proper changes to support files input.nml and quickbuild.sh within work directory --- .../Ameriflux/fluxnetfull_to_obs.f90 | 175 +++++++++--------- .../Ameriflux/fluxnetfull_to_obs.nml | 1 - .../obs_converters/Ameriflux/work/input.nml | 8 +- .../Ameriflux/work/quickbuild.sh | 2 +- 4 files changed, 92 insertions(+), 94 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index e807dec6d2..17d37fbbdc 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -22,7 +22,8 @@ program Fluxnetfull_to_obs use time_manager_mod, only : time_type, set_calendar_type, GREGORIAN, & set_date, set_time, get_time, print_time, & print_date, operator(-), operator(+), operator(>), & - operator(<), operator(==), operator(<=), operator(>=) + operator(<), operator(==), operator(<=), operator(>=), & + operator(/) use location_mod, only : VERTISHEIGHT @@ -40,7 +41,6 @@ program Fluxnetfull_to_obs implicit none character(len=*), parameter :: source = 'Fluxnetfull_to_obs.f90' -character(len=512), parameter :: string1,string2,string3 !----------------------------------------------------------------------- ! Namelist with default values @@ -101,14 +101,14 @@ program Fluxnetfull_to_obs character(len=20) :: gppNTstring = 'GPP_NT_VUT_REF' character(len=20) :: recoDTstring = 'RECO_DT_VUT_REF' character(len=20) :: recoNTstring = 'RECO_NT_VUT_REF' - character(len=20) :: gppUNCNT16string = 'GPP_NT_VUT_16' - character(len=20) :: gppUNCNT84string = 'GPP_NT_VUT_84' - character(len=20) :: gppUNCDT16string = 'GPP_DT_VUT_16' - character(len=20) :: gppUNCDT84string = 'GPP_DT_VUT_84' - character(len=20) :: recoUNCNT16string = 'RECO_NT_VUT_16' - character(len=20) :: recoUNCNT84string = 'RECO_NT_VUT_84' - character(len=20) :: recoUNCDT16string = 'RECO_DT_VUT_16' - character(len=20) :: recoUNCDT84string = 'RECO_DT_VUT_84' + character(len=20) :: gppNTUNC16string = 'GPP_NT_VUT_16' + character(len=20) :: gppNTUNC84string = 'GPP_NT_VUT_84' + character(len=20) :: gppDTUNC16string = 'GPP_DT_VUT_16' + character(len=20) :: gppDTUNC84string = 'GPP_DT_VUT_84' + character(len=20) :: recoNTUNC16string = 'RECO_NT_VUT_16' + character(len=20) :: recoNTUNC84string = 'RECO_NT_VUT_84' + character(len=20) :: recoDTUNC16string = 'RECO_DT_VUT_16' + character(len=20) :: recoDTUNC84string = 'RECO_DT_VUT_84' integer :: startindex integer :: endindex integer :: neeindex @@ -124,17 +124,17 @@ program Fluxnetfull_to_obs integer :: gppNTindex integer :: recoDTindex integer :: recoNTindex - integer :: gppUNCDT16index - integer :: gppUNCDT84index - integer :: gppUNCNT16index - integer :: gppUNCNT84index - integer :: recoUNCDT16index - integer :: recoUNCDT84index - integer :: recoUNCNT16index - integer :: recoUNCNT84index - - character(len=12) :: start_time - character(len=12) :: end_time + integer :: gppDTUNC16index + integer :: gppDTUNC84index + integer :: gppNTUNC16index + integer :: gppNTUNC84index + integer :: recoDTUNC16index + integer :: recoDTUNC84index + integer :: recoNTUNC16index + integer :: recoNTUNC84index + + integer :: start_time + integer :: end_time real(r8) :: nee real(r8) :: neeUNC integer :: neeQC @@ -149,19 +149,19 @@ program Fluxnetfull_to_obs real(r8) :: gpp real(r8) :: gppNTQC real(r8) :: gppDTQC - real(r8) :: gppUNCNT84 - real(r8) :: gppUNCNT16 - real(r8) :: gppUNCDT84 - real(r8) :: gppUNCDT16 + real(r8) :: gppNTUNC84 + real(r8) :: gppNTUNC16 + real(r8) :: gppDTUNC84 + real(r8) :: gppDTUNC16 real(r8) :: recoNT real(r8) :: recoDT real(r8) :: reco real(r8) :: recoNTQC real(r8) :: recoDTQC - real(r8) :: recoUNCNT84 - real(r8) :: recoUNCNT16 - real(r8) :: recoUNCDT84 - real(r8) :: recoUNCDT16 + real(r8) :: recoNTUNC84 + real(r8) :: recoNTUNC16 + real(r8) :: recoDTUNC84 + real(r8) :: recoDTUNC16 end type towerdata type(towerdata) :: tower @@ -265,7 +265,7 @@ program Fluxnetfull_to_obs write(*,*)tower%leUNCindex , tower%leUNCstring , tower%leUNC write(*,*)tower%leQCindex , tower%leQCstring , tower%leQC write(*,*)tower%neeindex , tower%neestring , tower%nee - write(*,*)tower%neeUNCindex , tower%neeUNCstring , tower%UNCnee + write(*,*)tower%neeUNCindex , tower%neeUNCstring , tower%neeUNC write(*,*)tower%neeQCindex , tower%neeQCstring , tower%neeQC write(*,*)tower%gppDTindex , tower%gppDTstring , tower%gppDT write(*,*)tower%gppDTUNC16index , tower%gppDTUNC16string , tower%gppDTUNC16 @@ -303,7 +303,7 @@ program Fluxnetfull_to_obs write(logfileunit,*)tower%gppNTindex , tower%gppNTstring , tower%gppNT write(logfileunit,*)tower%gppNTUNC16index , tower%gppNTUNC16string , tower%gppNTUNC16 write(logfileunit,*)tower%gppNTUNC84index , tower%gppNTUNC84string , tower%gppNTUNC84 - write(logifleunit,*)tower%gppNTQC + write(logfileunit,*)tower%gppNTQC write(logfileunit,*)tower%recoDTindex , tower%recoDTstring , tower%recoDT write(logfileunit,*)tower%recoDTUNC16index, tower%recoDTUNC16string, tower%recoDTUNC16 write(logfileunit,*)tower%recoDTUNC84index, tower%recoDTUNC84string, tower%recoDTUNC84 @@ -350,29 +350,29 @@ program Fluxnetfull_to_obs call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) endif - if (tower%gppDTQC .and. tower%gppNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] - sig_gppdt = (((tower%gppDTUNC84-tower%gppDTUNC16 / 2))^2)^0.5 ! Ustar unc contribution - sig_gppnt = (((tower%gppNTUNC84-tower%gppNTUNC16 / 2))^2)^0.5 + if (tower%gppDTQC <=maxgoodqc .and. tower%gppNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] + sig_gppdt = (((tower%gppDTUNC84-tower%gppDTUNC16) / 2)**2)**0.5 ! Ustar unc contribution + sig_gppnt = (((tower%gppNTUNC84-tower%gppNTUNC16) / 2)**2)**0.5 - oerr = (0.25 * (sig_gppdt)^2 + 0.25 * (sig_gppnt)^2)^0.5 ! Combine Ustar and partitioning unc + oerr = (0.25 * (sig_gppdt)**2 + 0.25 * (sig_gppnt)**2)**0.5 ! Combine Ustar and partitioning unc oerr = oerr * umol_to_gC ! Take average of night and day partitioning methods tower%gpp = ((tower%gppDT + tower%gppNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] - qc = maxval(real(tower%gppDTQC,r8),real(tower%gppNTQC,r8)) + qc = maxval((/real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)/)) call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%gpp, & TOWER_GPP_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) endif - if (tower%recoDTQC .and. tower%recoNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] - sig_recodt = (((tower%recoDTUNC84-tower%recoDTUNC16 / 2))^2)^0.5 ! Ustar unc contribution - sig_recont = (((tower%recoNTUNC84-tower%recoNTUNC16 / 2))^2)^0.5 + if (tower%recoDTQC <= maxgoodqc .and. tower%recoNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] + sig_recodt = (((tower%recoDTUNC84-tower%recoDTUNC16) / 2)**2)**0.5 ! Ustar unc contribution + sig_recont = (((tower%recoNTUNC84-tower%recoNTUNC16) / 2)**2)**0.5 - oerr = (0.25 * (sig_recodt)^2 + 0.25 * (sig_recont)^2)^0.5 ! Combine Ustar and partitioning unc + oerr = (0.25 * (sig_recodt)**2 + 0.25 * (sig_recont)**2)**0.5 ! Combine Ustar and partitioning unc oerr = oerr * umol_to_gC ! Take average of night and day partitioning methods tower%reco = ((tower%recoDT + tower%recoNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] - qc = maxval(real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)) + qc = maxval((/real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)/)) call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%reco, & TOWER_ER_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) @@ -557,7 +557,7 @@ subroutine decode_header(iunit,ncolumns) integer, parameter :: maxwordlength = 30 integer :: i,charcount,columncount,wordlength character(len=maxwordlength), dimension(:), allocatable :: columns -integer, dimension(10) :: qc = 0 +integer, dimension(23) :: qc = 0 ! Read the line and strip off any leading whitespace. @@ -616,7 +616,7 @@ subroutine decode_header(iunit,ncolumns) tower%neeindex = Match(columns, tower%neestring) tower%neeUNCindex = Match(columns, tower%neeUNCstring) tower%neeQCindex = Match(columns, tower%neeQCstring) -tower%neeUNCQCindex = Match(columns, tower%neeUNCstring) +tower%neeUNCindex = Match(columns, tower%neeUNCstring) tower%leindex = Match(columns, tower%lestring) tower%leQCindex = Match(columns, tower%leQCstring) tower%leUNCindex = Match(columns, tower%leUNCstring) @@ -627,14 +627,14 @@ subroutine decode_header(iunit,ncolumns) tower%gppNTindex = Match(columns, tower%gppNTstring) tower%recoDTindex = Match(columns, tower%recoDTstring) tower%recoNTindex = Match(columns, tower%recoNTstring) -tower%gppUNCDT16index = Match(columns, tower%gppUNCDT16string) -tower%gppUNCDT84index = Match(columns, tower%gppUNCDT84string) -tower%gppUNCNT16index = Match(columns, tower%gppUNCNT16string) -tower%gppUNCNT84index = Match(columns, tower%gppUNCNT84string) -tower%recoUNCDT16index = Match(columns, tower%recoUNCDT16string) -tower%recoUNCDT84index = Match(columns, tower%recoUNCDT84string) -tower%recoUNCNT16index = Match(columns, tower%recoUNCNT16string) -tower%recoUNCNT84index = Match(columns, tower%recoUNCNT84string) +tower%gppDTUNC16index = Match(columns, tower%gppDTUNC16string) +tower%gppDTUNC84index = Match(columns, tower%gppDTUNC84string) +tower%gppNTUNC16index = Match(columns, tower%gppNTUNC16string) +tower%gppNTUNC84index = Match(columns, tower%gppNTUNC84string) +tower%recoDTUNC16index = Match(columns, tower%recoDTUNC16string) +tower%recoDTUNC84index = Match(columns, tower%recoDTUNC84string) +tower%recoNTUNC16index = Match(columns, tower%recoNTUNC16string) +tower%recoNTUNC84index = Match(columns, tower%recoNTUNC84string) ! Confirm indices were found successfully qc( 1) = CheckIndex( tower%startindex , tower%startstring) @@ -642,25 +642,24 @@ subroutine decode_header(iunit,ncolumns) qc( 3) = CheckIndex( tower%neeindex , tower%neestring) qc( 4) = CheckIndex( tower%neeUNCindex , tower%neeUNCstring) qc( 5) = CheckIndex( tower%neeQCindex , tower%neeQCstring) -qc( 6) = CheckIndex( tower%neeUNCQCindex , tower%neeUNCstring) -qc( 7) = CheckIndex( tower%leindex , tower%lestring) -qc( 8) = CheckIndex( tower%leQCindex , tower%leQCstring) -qc( 9) = CheckIndex( tower%leUNCindex , tower%leUNCstring) -qc(10) = CheckIndex( tower%hindex , tower%hstring) -qc(11) = CheckIndex( tower%hQCindex , tower%hQCstring) -qc(12) = CheckIndex( tower%hUNCindex , tower%hUNCstring) -qc(13) = CheckIndex( tower%gppDTindex , tower%gppDTstring) -qc(14) = CheckIndex( tower%gppNTindex , tower%gppNTstring) -qc(15) = CheckIndex( tower%recoDTindex , tower%recoDTstring) -qc(16) = CheckIndex( tower%recoNTindex , tower%recoNTstring) -qc(17) = CheckIndex( tower%gppUNCDT16index , tower%gppUNCDT16string) -qc(18) = CheckIndex( tower%gppUNCDT84index , tower%gppUNCDT84string) -qc(19) = CheckIndex( tower%gppUNCNT16index , tower%gppUNCNT16string) -qc(20) = CheckIndex( tower%gppUNCNT84index , tower%gppUNCNT84string) -qc(21) = CheckIndex( tower%recoUNCDT16index, tower%recoUNCDT16string) -qc(22) = CheckIndex( tower%recoUNCDT84index, tower%recoUNCDT84string) -qc(23) = CheckIndex( tower%recoUNCNT16index, tower%recoUNCNT16string) -qc(24) = CheckIndex( tower%recoUNCNT84index, tower%recoUNCNT84string) +qc( 6) = CheckIndex( tower%leindex , tower%lestring) +qc( 7) = CheckIndex( tower%leQCindex , tower%leQCstring) +qc( 8) = CheckIndex( tower%leUNCindex , tower%leUNCstring) +qc( 9) = CheckIndex( tower%hindex , tower%hstring) +qc(10) = CheckIndex( tower%hQCindex , tower%hQCstring) +qc(11) = CheckIndex( tower%hUNCindex , tower%hUNCstring) +qc(12) = CheckIndex( tower%gppDTindex , tower%gppDTstring) +qc(13) = CheckIndex( tower%gppNTindex , tower%gppNTstring) +qc(14) = CheckIndex( tower%recoDTindex , tower%recoDTstring) +qc(15) = CheckIndex( tower%recoNTindex , tower%recoNTstring) +qc(16) = CheckIndex( tower%gppDTUNC16index , tower%gppDTUNC16string) +qc(17) = CheckIndex( tower%gppDTUNC84index , tower%gppDTUNC84string) +qc(18) = CheckIndex( tower%gppNTUNC16index , tower%gppNTUNC16string) +qc(19) = CheckIndex( tower%gppNTUNC84index , tower%gppNTUNC84string) +qc(20) = CheckIndex( tower%recoDTUNC16index, tower%recoDTUNC16string) +qc(21) = CheckIndex( tower%recoDTUNC84index, tower%recoDTUNC84string) +qc(22) = CheckIndex( tower%recoNTUNC16index, tower%recoNTUNC16string) +qc(23) = CheckIndex( tower%recoNTUNC84index, tower%recoNTUNC84string) if (any(qc /= 0) ) then write(string1,*)'Did not find all the required column indices.' @@ -685,14 +684,14 @@ subroutine decode_header(iunit,ncolumns) write(*,110)tower%gppNTstring , tower%gppNTindex write(*,110)tower%recoDTstring , tower%recoDTindex write(*,110)tower%recoNTstring , tower%recoNTindex - write(*,110)tower%gppUNCDT16string , tower%gppUNCDT16index - write(*,110)tower%gppUNCDT84string , tower%gppUNCDT84index - write(*,110)tower%gppUNCNT16string , tower%gppUNCNT16index - write(*,110)tower%gppUNCNT84string , tower%gppUNCNT84index - write(*,110)tower%recoUNCDT16string, tower%recoUNCDT16index - write(*,110)tower%recoUNCDT84string, tower%recoUNCDT84index - write(*,110)tower%recoUNCNT16string, tower%recoUNCNT16index - write(*,110)tower%recoUNCNT84string, tower%recoUNCNT84index + write(*,110)tower%gppDTUNC16string , tower%gppDTUNC16index + write(*,110)tower%gppDTUNC84string , tower%gppDTUNC84index + write(*,110)tower%gppNTUNC16string , tower%gppNTUNC16index + write(*,110)tower%gppNTUNC84string , tower%gppNTUNC84index + write(*,110)tower%recoDTUNC16string, tower%recoDTUNC16index + write(*,110)tower%recoDTUNC84string, tower%recoDTUNC84index + write(*,110)tower%recoNTUNC16string, tower%recoNTUNC16index + write(*,110)tower%recoNTUNC84string, tower%recoNTUNC84index endif deallocate(columns) @@ -794,7 +793,7 @@ subroutine stringparse(str1, nwords, linenum) ! Fixme: These are for high resolution format HH or HR only ! Fixme: If aggregrated (DD,WW,MM) need to edit, expand this -! start_time,end_time format YYYYMMDDHHMM +! start_time,end_time format YYYYMMDDHHMM ! nee units [umolCO2 m-2 s-1], VUT_REF ! neeUNC units [umolCO2 m-2 s-1], joint ! neeQC no dim 0=measured,gap_filled: 1=good,2=medium,3=poor @@ -821,7 +820,7 @@ subroutine stringparse(str1, nwords, linenum) ! (CLM) LE,SH units [W m-2] tower%start_time = values(tower%startindex ) -tower%end_end = values(tower%endindex ) +tower%end_time = values(tower%endindex ) tower%nee = values(tower%neeindex ) tower%nee = values(tower%neeUNCindex ) tower%neeQC = nint(values(tower%neeQCindex )) @@ -835,14 +834,14 @@ subroutine stringparse(str1, nwords, linenum) tower%gppNT = values(tower%gppNTindex ) tower%recoDT = values(tower%recoDTindex ) tower%recoNT = values(tower%recoNTindex ) -tower%gppUNCDT16 = values(tower%gppUNCDT16index ) -tower%gppUNCDT84 = values(tower%gppUNCDT84index ) -tower%gppUNCNT16 = values(tower%gppUNCNT16index ) -tower%gppUNCNT84 = values(tower%gppUNCNT84index ) -tower%recoUNCDT16 = values(tower%recoUNCDT16index) -tower%recoUNCDT84 = values(tower%recoUNCDT84index) -tower%recoUNCNT16 = values(tower%recoUNCNT16index) -tower%recoUNCNT84 = values(tower%recoUNCNT84index) +tower%gppDTUNC16 = values(tower%gppDTUNC16index ) +tower%gppDTUNC84 = values(tower%gppDTUNC84index ) +tower%gppNTUNC16 = values(tower%gppNTUNC16index ) +tower%gppNTUNC84 = values(tower%gppNTUNC84index ) +tower%recoDTUNC16 = values(tower%recoDTUNC16index) +tower%recoDTUNC84 = values(tower%recoDTUNC84index) +tower%recoNTUNC16 = values(tower%recoNTUNC16index) +tower%recoNTUNC84 = values(tower%recoNTUNC84index) deallocate(values) ! The observation time must be assigned through the flux timestamp string diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index da089f9cbe..528b1ca277 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -1,7 +1,6 @@ &Fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', - year = -1, timezoneoffset = -1, latitude = -1.0, longitude = -1.0, diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index 8da69eff7e..08e0d53667 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -27,7 +27,7 @@ text_input_file = '../data/USHa12003_L4_h.txt' obs_out_file = 'obs_seq.out' year = 2003 - timezoneoffset = -6 + timezoneoffset = -5 latitude = 42.5378 longitude = -72.1715 elevation = 353 @@ -39,14 +39,14 @@ &Fluxnetfull_to_obs_nml text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv' obs_out_file = 'obs_seq.out' - year = 2003 - timezoneoffset = -6 + timezoneoffset = -5 latitude = 42.5378 longitude = -72.1715 elevation = 353 flux_height = 29 maxgoodqc = 3 - verbose = .TRUE. + gap_filled = .false. + verbose = .true. / diff --git a/observations/obs_converters/Ameriflux/work/quickbuild.sh b/observations/obs_converters/Ameriflux/work/quickbuild.sh index 329028c0d8..07f4f33c3b 100755 --- a/observations/obs_converters/Ameriflux/work/quickbuild.sh +++ b/observations/obs_converters/Ameriflux/work/quickbuild.sh @@ -14,7 +14,7 @@ LOCATION=threed_sphere programs=( -fluxnetfull_to_obs +Fluxnetfull_to_obs level4_to_obs obs_sequence_tool advance_time From 3228690b4312d9a91d081f823cb4070fb8289962 Mon Sep 17 00:00:00 2001 From: braczka Date: Wed, 24 May 2023 12:33:35 -0600 Subject: [PATCH 08/25] Edit flux converter to compile and run fluxnetfull_to_obs converter compiles and runs testing on hourly flux fullset data. Header now includes timestamp which is treated differently than real values --- .../Ameriflux/fluxnetfull_to_obs.f90 | 113 ++++++++++++------ 1 file changed, 75 insertions(+), 38 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 17d37fbbdc..4ce2e1bef0 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -68,7 +68,7 @@ program Fluxnetfull_to_obs ! globally-scoped variables !----------------------------------------------------------------------- -character(len=300) :: input_line, bigline +character(len=3100) :: input_line, bigline character(len=512) :: string1, string2, string3 integer :: iline, nlines, nwords logical :: first_obs @@ -132,9 +132,9 @@ program Fluxnetfull_to_obs integer :: recoDTUNC84index integer :: recoNTUNC16index integer :: recoNTUNC84index - - integer :: start_time - integer :: end_time + + character(len=12) :: start_time + character(len=12) :: end_time real(r8) :: nee real(r8) :: neeUNC integer :: neeQC @@ -318,7 +318,7 @@ program Fluxnetfull_to_obs call get_time(tower%time_obs, osec, oday) if (verbose) then - write(string1,*)'obs time is (seconds,days) ',osec, oday,' obs date is ' + write(string1,*)'obs time (UTC) is (seconds,days) ',osec, oday,' obs date (UTC) is ' call print_date(tower%time_obs, trim(string1)) call print_date(tower%time_obs, trim(string1),logfileunit) endif @@ -767,22 +767,26 @@ end function CheckIndex subroutine stringparse(str1, nwords, linenum) -! just declare everything as reals and chunk it character(len=*), intent(in) :: str1 integer , intent(in) :: nwords integer , intent(in) :: linenum real(r8), allocatable, dimension(:) :: values -integer :: iyear0, imonth0, iday0, ihour0, imin0 -integer :: iyear1, imonth1, iday1, ihour1, imin1 + + +integer :: yeara, yearb +integer :: montha, daya, houra, mina +integer :: monthb, dayb, hourb, minb type(time_type) :: date_start, date_end -allocate(values(nwords)) +allocate(values(nwords-2)) values = MISSING_R8 -read(str1,*,iostat=rcio) values +! First two elements of string read in as character (time stamp) +! remainder of line read in as reals. +read(str1,*,iostat=rcio) tower%start_time,tower%end_time,values if (rcio /= 0) then write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' call error_handler(E_ERR,'stringparse',string1, source) @@ -819,38 +823,54 @@ subroutine stringparse(str1, nwords, linenum) ! (CLM) NEE,GPP,ER units [gC m-2 s-1] ! (CLM) LE,SH units [W m-2] -tower%start_time = values(tower%startindex ) -tower%end_time = values(tower%endindex ) -tower%nee = values(tower%neeindex ) -tower%nee = values(tower%neeUNCindex ) -tower%neeQC = nint(values(tower%neeQCindex )) -tower%le = values(tower%leindex ) -tower%leUNC = values(tower%leUNCindex ) -tower%leQC = nint(values(tower%leQCindex )) -tower%h = values(tower%hindex ) -tower%hUNC = values(tower%hUNCindex ) -tower%hQC = nint(values(tower%hQCindex )) -tower%gppDT = values(tower%gppDTindex ) -tower%gppNT = values(tower%gppNTindex ) -tower%recoDT = values(tower%recoDTindex ) -tower%recoNT = values(tower%recoNTindex ) -tower%gppDTUNC16 = values(tower%gppDTUNC16index ) -tower%gppDTUNC84 = values(tower%gppDTUNC84index ) -tower%gppNTUNC16 = values(tower%gppNTUNC16index ) -tower%gppNTUNC84 = values(tower%gppNTUNC84index ) -tower%recoDTUNC16 = values(tower%recoDTUNC16index) -tower%recoDTUNC84 = values(tower%recoDTUNC84index) -tower%recoNTUNC16 = values(tower%recoNTUNC16index) -tower%recoNTUNC84 = values(tower%recoNTUNC84index) +tower%nee = values(tower%neeindex -2 ) +tower%neeUNC = values(tower%neeUNCindex -2 ) +tower%neeQC = nint(values(tower%neeQCindex -2)) +tower%le = values(tower%leindex -2 ) +tower%leUNC = values(tower%leUNCindex -2 ) +tower%leQC = nint(values(tower%leQCindex -2)) +tower%h = values(tower%hindex -2 ) +tower%hUNC = values(tower%hUNCindex -2 ) +tower%hQC = nint(values(tower%hQCindex -2)) +tower%gppDT = values(tower%gppDTindex -2) +tower%gppNT = values(tower%gppNTindex -2) +tower%recoDT = values(tower%recoDTindex -2) +tower%recoNT = values(tower%recoNTindex -2) +tower%gppDTUNC16 = values(tower%gppDTUNC16index -2) +tower%gppDTUNC84 = values(tower%gppDTUNC84index -2) +tower%gppNTUNC16 = values(tower%gppNTUNC16index -2) +tower%gppNTUNC84 = values(tower%gppNTUNC84index -2) +tower%recoDTUNC16 = values(tower%recoDTUNC16index-2) +tower%recoDTUNC84 = values(tower%recoDTUNC84index-2) +tower%recoNTUNC16 = values(tower%recoNTUNC16index-2) +tower%recoNTUNC84 = values(tower%recoNTUNC84index-2) deallocate(values) -! The observation time must be assigned through the flux timestamp string -write(tower%start_time,'(i4, 4i2)') iyear0,imonth0,iday0,ihour0,imin0 -write(tower%end_time, '(i4, 4i2)') iyear1,imonth1,iday1,ihour1,imin1 -date_start= set_date(iyear0,imonth0,iday0,ihour0,imin0,0) -date_end= set_date(iyear1,imonth1,iday1,ihour1,imin0,0) +write(*,*)'' +write(string1, *) 'Display tower%start_time and tower%end_time (LTC) =', tower%start_time,' ', tower%end_time +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + +write(*,*)'' +write(string1, *) 'Display tower%nee tower%neeQC =', tower%nee, tower%neeQC +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + + +read(tower%start_time(1:12), fmt='(i4, 4i2)') yeara,montha,daya,houra,mina +read(tower%end_time(1:12), fmt='(i4, 4i2)') yearb,monthb,dayb,hourb,minb + +write(*,*)'' +write(string1, *) 'Display tower%start_time,yeara,montha,daya,houra,mina (LTC) =', tower%start_time, yeara, montha, daya, houra, mina +write(string2, *) 'Display tower%end_time,yearb,monthb,dayb,hourb,minb (LTC) =', tower%end_time, yearb, monthb, dayb, hourb, minb +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + + + +date_start= set_date(yeara,montha,daya,houra,mina,0) +date_end= set_date(yearb,monthb,dayb,hourb,minb,0) + + ! Assign average of flux time window to obs_seq (DART time) tower%time_obs = (date_start+date_end) / 2 @@ -869,6 +889,8 @@ subroutine stringparse(str1, nwords, linenum) if (tower%leQC < 0) tower%leQC = maxgoodqc + 1000 if (tower%hQC < 0) tower%hQC = maxgoodqc + 1000 +! No qc values for gpp/reco, thus assign good qc unless +! the gpp/reco value is missing tower%gppNTQC = 1 tower%gppDTQC = 1 tower%recoNTQC = 1 @@ -878,6 +900,21 @@ subroutine stringparse(str1, nwords, linenum) if (tower%gppDT < 0) tower%gppDTQC = maxgoodqc + 1000 if (tower%recoNT < 0) tower%recoNTQC = maxgoodqc + 1000 if (tower%recoDT < 0) tower%recoDTQC = maxgoodqc + 1000 + +! Assign very bad qc to gap_filled data if user requests it + +if (gap_filled .eqv. .false.) then + if (tower%neeQC >0) tower%neeQC = maxgoodqc + 100 + if (tower%leQC >0) tower%leQC = maxgoodqc + 100 + if (tower%hQC >0) tower%hQC = maxgoodqc + 100 + ! GPP and RECO are modeled data and considered gap_filled + tower%gppNTQC = maxgoodqc + 100 + tower%gppDTQC = maxgoodqc + 100 + tower%recoNTQC = maxgoodqc + 100 + tower%recoDTQC = maxgoodqc + 100 +endif + + end subroutine stringparse From ad0be7bf47027bb84fab6b4c57f774479a19439b Mon Sep 17 00:00:00 2001 From: braczka Date: Thu, 25 May 2023 15:33:30 -0600 Subject: [PATCH 09/25] Added option to ingest multiple time resolutions Adds ability to use HH,HR,DD,WW,MM,YY flux data where test file format changes --- .../Ameriflux/fluxnetfull_to_obs.f90 | 118 +++++++++++++++--- .../Ameriflux/fluxnetfull_to_obs.nml | 2 + .../obs_converters/Ameriflux/work/input.nml | 2 + 3 files changed, 103 insertions(+), 19 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 4ce2e1bef0..6b84f92039 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -53,22 +53,26 @@ program Fluxnetfull_to_obs real(r8) :: longitude = -1.0_r8 real(r8) :: elevation = -1.0_r8 real(r8) :: flux_height = -1.0_r8 -! A maxgooqc=3 allows for good,medium and poor quality gap-filled data +! A maxgooqc=3 allows for good=1, medium=2, and poor=3 quality gap-filled data real(r8) :: maxgoodqc = 3.0_r8 -! Always 'true' except for latent,sensible heat and NEE for hourly time periods -! Fixme This option must be worked into the code later on +! Always true except for latent,sensible heat and NEE for hourly time periods logical :: gap_filled = .true. +! Option for energy balance correction for latent and sensible heat +! Recommned to keep false as these values are typically missing +logical :: energy_balance = .false. +character(len=2) :: time_resolution = 'HH' logical :: verbose = .false. namelist /Fluxnetfull_to_obs_nml/ text_input_file, obs_out_file, & timezoneoffset, latitude, longitude, elevation, & - flux_height, maxgoodqc, gap_filled, verbose + flux_height, maxgoodqc, gap_filled, energy_balance, & + time_resolution, verbose !----------------------------------------------------------------------- ! globally-scoped variables !----------------------------------------------------------------------- -character(len=3100) :: input_line, bigline +character(len=3100) :: input_line, bigline character(len=512) :: string1, string2, string3 integer :: iline, nlines, nwords logical :: first_obs @@ -81,8 +85,7 @@ program Fluxnetfull_to_obs type(time_type) :: prev_time, offset real(r8), parameter :: umol_to_gC = (1.0_r8/1000000.0_r8) * 12.0_r8 -! Fixme: These are for high resolution format HH or HR -! Fixme: If aggregrated (DD,WW,MM) need to edit +! Initialize with default tower strings, modify later type towerdata type(time_type) :: time_obs @@ -134,7 +137,7 @@ program Fluxnetfull_to_obs integer :: recoNTUNC84index character(len=12) :: start_time - character(len=12) :: end_time + character(len=12) :: end_time real(r8) :: nee real(r8) :: neeUNC integer :: neeQC @@ -166,6 +169,50 @@ program Fluxnetfull_to_obs type(towerdata) :: tower +! Modify towerdata strings based on user input.nml + +! 1) time_resolution: DD(daily),MM(monthly),YY(yearly) uses 'TIMESTAMP' header + +if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & + time_resolution == 'YY') then + + startstring = 'TIMESTAMP' + endstring = 'TIMESTAMP' + + write(string1, *) 'Time resolution is set to =', time_res + write(string2, *) 'Using TIMESTAMP to set DART observation time' + if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + +elseif (time_resolution == 'HH' .or. time_resolution == 'HR' .or. & + time_resolution == 'WW') then + write(string1, *) 'Time resolution is set to =', time_res + write(string2, *) 'Using TIMESTAMP_START and TIMESTAMP_END to set DART observation time' + if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + +else + write(string1,*) 'time_resolution set incorrectly within input.nml' + write(string2,*) 'time_resolution must be HR,HH,DD,WW,MM, or YY' + call error_handler(E_ERR, source, string1,text2=string2) + +endif + +! 2) energy_balance: .true. changes sensible and latent heat strings +! Note: There are no qc values for energy_balance correction +! The qc values are manually set later in code +if (energy_balance .eqv. .true.) then + + lestring = 'LE_CORR' + leUNCstring = 'LE_CORR_JOINTUNC' + hstring = 'H_CORR' + hUNCstring = 'H_CORR_JOINTUNC' + write(string1,*) 'WARNING! Energy balance correction data turned on for LE and H' + write(string2,*) 'Check to make sure LE_CORR and H_CORR data is not missing' + call error_handler(E_MSG, source, string1,text=string2) +endif + + + + !----------------------------------------------------------------------- ! start of executable code !----------------------------------------------------------------------- @@ -200,11 +247,17 @@ program Fluxnetfull_to_obs write (string3,*)'longitude should be [ 0,360] but is ',longitude string1 ='tower location error in input.nml&Fluxnetfull_to_obs_nml' - call error_handler(E_ERR,'Fluxnetfull_to_obs', source, string1, & + call error_handler(E_ERR, source, string1, & text2=string2,text3=string3) +endif +if (gap_filled .eqv. .false.) then + + string1 ='WARNING!: gap_filled=false which removes all gpp and reco data' + call error_handler(E_MSG, source, string1) endif + ! Specify the maximum number of observations in the input file, ! but only the actual number created will be written out. ! Each line has 5 flux observations available. @@ -819,9 +872,9 @@ subroutine stringparse(str1, nwords, linenum) ! Convert to 'CLM-friendly' units AFTER we determine observation error variance. ! That happens in the main routine. -! Fixme: Double check these defs and units -! (CLM) NEE,GPP,ER units [gC m-2 s-1] -! (CLM) LE,SH units [W m-2] +! CLM history file names and units +! (CLM) NEP,GPP,ER units [gC m-2 s-1] +! (CLM) EFLX_LH_TOT_R,FSH units [W m-2] tower%nee = values(tower%neeindex -2 ) tower%neeUNC = values(tower%neeUNCindex -2 ) @@ -855,6 +908,13 @@ subroutine stringparse(str1, nwords, linenum) write(*,*)'' write(string1, *) 'Display tower%nee tower%neeQC =', tower%nee, tower%neeQC if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) +write(*,*)'' +write(string1, *) 'Display tower%le tower%leQC =', tower%le, tower%leQC +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) +write(*,*)'' +write(string1, *) 'Display tower%h tower%hQC =', tower%h, tower%hQC +if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + read(tower%start_time(1:12), fmt='(i4, 4i2)') yeara,montha,daya,houra,mina @@ -883,14 +943,23 @@ subroutine stringparse(str1, nwords, linenum) tower%time_obs = tower%time_obs - offset endif -! If missing value (-9999) manually assign poor QC value -! such that value is excluded in obs_seq -if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 -if (tower%leQC < 0) tower%leQC = maxgoodqc + 1000 -if (tower%hQC < 0) tower%hQC = maxgoodqc + 1000 + +! Reject NEE data where neeQC is missing +if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 + +! The QC values are typically missing for le and h (-9999) +! Thus manually assign poor QC values in these cases +if (energy_balance .eqv. .false.) then + (tower%leQC < 0) tower%leQC = 3 + (tower%hQC < 0) tower%hQC = 3 +else ! No QC values for energy balance le and h. Assign poor QC. + tower%leQC = 3 + tower%leQC = 3 +endif + ! No qc values for gpp/reco, thus assign good qc unless -! the gpp/reco value is missing +! the gpp/reco value is missing (-9999) tower%gppNTQC = 1 tower%gppDTQC = 1 tower%recoNTQC = 1 @@ -902,7 +971,7 @@ subroutine stringparse(str1, nwords, linenum) if (tower%recoDT < 0) tower%recoDTQC = maxgoodqc + 1000 ! Assign very bad qc to gap_filled data if user requests it - +! such that maxgoodqc threshold does not add gap_filled data to obs_seq file if (gap_filled .eqv. .false.) then if (tower%neeQC >0) tower%neeQC = maxgoodqc + 100 if (tower%leQC >0) tower%leQC = maxgoodqc + 100 @@ -914,6 +983,17 @@ subroutine stringparse(str1, nwords, linenum) tower%recoDTQC = maxgoodqc + 100 endif +if (energy_balance .eqv. .true.) then + + lestring = 'LE_CORR' + leUNCstring = 'LE_CORR_JOINTUNC' + hstring = 'H_CORR' + hUNCstring = 'H_CORR_JOINTUNC' + write(string1,*) 'WARNING! Energy balance correction data turned on for LE and H' + write(string2,*) 'Check to make sure LE_CORR and H_CORR data is not missing' + call error_handler(E_MSG, source, string1,text=string2) +endif + end subroutine stringparse diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index 528b1ca277..07bb79e9b2 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -8,6 +8,8 @@ flux_height = -1.0, maxgoodqc = 3, gap_filled = .false. + energy_balance = .false. + time_resolution = 'HH' verbose = .false. / diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index 08e0d53667..d104b5a4bf 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -46,6 +46,8 @@ flux_height = 29 maxgoodqc = 3 gap_filled = .false. + energy_balance = .false. + time_resolution = 'HH' verbose = .true. / From c02652441fca79e0ede05c9b6aad581bacc3cb46 Mon Sep 17 00:00:00 2001 From: braczka Date: Fri, 26 May 2023 10:21:22 -0600 Subject: [PATCH 10/25] Fluxnet convert ingests multiple time averages Expanded functionality such that fluxnet converter can take in HH,HR,DD,WW,MM and YY time averages --- .../Ameriflux/fluxnetfull_to_obs.f90 | 166 +++++++++--------- 1 file changed, 85 insertions(+), 81 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 6b84f92039..136599d8a3 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -89,7 +89,7 @@ program Fluxnetfull_to_obs type towerdata type(time_type) :: time_obs - character(len=20) :: startstring = 'TIMESTAMP_START' + character(len=20) :: startstring = 'TIMESTAMP_START' character(len=20) :: endstring = 'TIMESTAMP_END' character(len=20) :: neestring = 'NEE_VUT_REF' character(len=20) :: neeUNCstring = 'NEE_VUT_REF_JOINTUNC' @@ -150,8 +150,8 @@ program Fluxnetfull_to_obs real(r8) :: gppNT real(r8) :: gppDT real(r8) :: gpp - real(r8) :: gppNTQC - real(r8) :: gppDTQC + integer :: gppNTQC + integer :: gppDTQC real(r8) :: gppNTUNC84 real(r8) :: gppNTUNC16 real(r8) :: gppDTUNC84 @@ -159,8 +159,8 @@ program Fluxnetfull_to_obs real(r8) :: recoNT real(r8) :: recoDT real(r8) :: reco - real(r8) :: recoNTQC - real(r8) :: recoDTQC + integer :: recoNTQC + integer :: recoDTQC real(r8) :: recoNTUNC84 real(r8) :: recoNTUNC16 real(r8) :: recoDTUNC84 @@ -169,49 +169,6 @@ program Fluxnetfull_to_obs type(towerdata) :: tower -! Modify towerdata strings based on user input.nml - -! 1) time_resolution: DD(daily),MM(monthly),YY(yearly) uses 'TIMESTAMP' header - -if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & - time_resolution == 'YY') then - - startstring = 'TIMESTAMP' - endstring = 'TIMESTAMP' - - write(string1, *) 'Time resolution is set to =', time_res - write(string2, *) 'Using TIMESTAMP to set DART observation time' - if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) - -elseif (time_resolution == 'HH' .or. time_resolution == 'HR' .or. & - time_resolution == 'WW') then - write(string1, *) 'Time resolution is set to =', time_res - write(string2, *) 'Using TIMESTAMP_START and TIMESTAMP_END to set DART observation time' - if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) - -else - write(string1,*) 'time_resolution set incorrectly within input.nml' - write(string2,*) 'time_resolution must be HR,HH,DD,WW,MM, or YY' - call error_handler(E_ERR, source, string1,text2=string2) - -endif - -! 2) energy_balance: .true. changes sensible and latent heat strings -! Note: There are no qc values for energy_balance correction -! The qc values are manually set later in code -if (energy_balance .eqv. .true.) then - - lestring = 'LE_CORR' - leUNCstring = 'LE_CORR_JOINTUNC' - hstring = 'H_CORR' - hUNCstring = 'H_CORR_JOINTUNC' - write(string1,*) 'WARNING! Energy balance correction data turned on for LE and H' - write(string2,*) 'Check to make sure LE_CORR and H_CORR data is not missing' - call error_handler(E_MSG, source, string1,text=string2) -endif - - - !----------------------------------------------------------------------- ! start of executable code @@ -251,9 +208,57 @@ program Fluxnetfull_to_obs text2=string2,text3=string3) endif + +! Modify values to towerdata strings based on user input.nml +! 1) time_resolution: DD(daily),MM(monthly),YY(yearly) uses 'TIMESTAMP' header + +if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & + time_resolution == 'YY') then + + tower%startstring = 'TIMESTAMP' + tower%endstring = 'TIMESTAMP' + + write(string1, *) 'Time resolution is set to =', time_resolution + write(string2, *) 'Using TIMESTAMP to set DART observation time' + if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + +elseif (time_resolution == 'HH' .or. time_resolution == 'HR' .or. & + time_resolution == 'WW') then + + write(string1, *) 'Time resolution is set to =', time_resolution + write(string2, *) 'Using TIMESTAMP_START and TIMESTAMP_END to set: DART observation time' + if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + +else + write(string1,*) 'time_resolution set incorrectly within input.nml' + write(string2,*) 'time_resolution must be HR,HH,DD,WW,MM, or YY' + call error_handler(E_ERR, source, string1,text2=string2) + +endif + +! 2) energy_balance: .true. changes sensible and latent heat strings +! Note: There are no qc values for energy_balance correction +! The qc values are manually set later in code +if (energy_balance .eqv. .true.) then + + tower%lestring = 'LE_CORR' + tower%leUNCstring = 'LE_CORR_JOINTUNC' + tower%hstring = 'H_CORR' + tower%hUNCstring = 'H_CORR_JOINTUNC' + write(string1,*) 'WARNING! Energy balance correction data turned on for LE and H' + write(string2,*) 'Check to make sure LE_CORR and H_CORR data is not missing' + call error_handler(E_MSG, source, string1,text2=string2) +else + + write(string1,*) 'Standard LE (LE_F_MDS) and H (H_F_MDS) data being used' + call error_handler(E_MSG, source, string1) +endif + + if (gap_filled .eqv. .false.) then - string1 ='WARNING!: gap_filled=false which removes all gpp and reco data' + write(string1,*) 'WARNING!: gap_filled = false which removes all gpp and reco data' + write(string2,*) ' ...and also will remove nee, h and le if qc value is missing' call error_handler(E_MSG, source, string1) endif @@ -374,6 +379,33 @@ program Fluxnetfull_to_obs write(string1,*)'obs time (UTC) is (seconds,days) ',osec, oday,' obs date (UTC) is ' call print_date(tower%time_obs, trim(string1)) call print_date(tower%time_obs, trim(string1),logfileunit) + + + write(*,*)'' + write(string1, *) 'Display tower%start_time and tower%end_time (LTC) =', tower%start_time,' ', tower%end_time + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%nee tower%neeQC =', tower%nee, tower%neeQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%le tower%leQC =', tower%le, tower%leQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%h tower%hQC =', tower%h, tower%hQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%gppDT tower%gppDTQC =', tower%gppDT, tower%gppDTQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%gppNT tower%gppNTQC =', tower%gppNT, tower%gppNTQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%recoDT tower%recoDTQC =', tower%recoDT, tower%recoDTQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + write(*,*)'' + write(string1, *) 'Display tower%recoNT tower%recoNTQC =', tower%recoNT, tower%recoNTQC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif ! Create and add observation and uncertainty (1 SD) to obs_seq file @@ -901,22 +933,6 @@ subroutine stringparse(str1, nwords, linenum) -write(*,*)'' -write(string1, *) 'Display tower%start_time and tower%end_time (LTC) =', tower%start_time,' ', tower%end_time -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) - -write(*,*)'' -write(string1, *) 'Display tower%nee tower%neeQC =', tower%nee, tower%neeQC -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) -write(*,*)'' -write(string1, *) 'Display tower%le tower%leQC =', tower%le, tower%leQC -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) -write(*,*)'' -write(string1, *) 'Display tower%h tower%hQC =', tower%h, tower%hQC -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) - - - read(tower%start_time(1:12), fmt='(i4, 4i2)') yeara,montha,daya,houra,mina read(tower%end_time(1:12), fmt='(i4, 4i2)') yearb,monthb,dayb,hourb,minb @@ -950,9 +966,9 @@ subroutine stringparse(str1, nwords, linenum) ! The QC values are typically missing for le and h (-9999) ! Thus manually assign poor QC values in these cases if (energy_balance .eqv. .false.) then - (tower%leQC < 0) tower%leQC = 3 - (tower%hQC < 0) tower%hQC = 3 -else ! No QC values for energy balance le and h. Assign poor QC. + if (tower%leQC < 0) tower%leQC = 3 + if (tower%hQC < 0) tower%hQC = 3 +else ! No QC values for energy balance corrected le and h. Assign poor QC. tower%leQC = 3 tower%leQC = 3 endif @@ -972,6 +988,7 @@ subroutine stringparse(str1, nwords, linenum) ! Assign very bad qc to gap_filled data if user requests it ! such that maxgoodqc threshold does not add gap_filled data to obs_seq file +! If leQC and hQC are missing no way to identify if gap_filled, thus rejected if (gap_filled .eqv. .false.) then if (tower%neeQC >0) tower%neeQC = maxgoodqc + 100 if (tower%leQC >0) tower%leQC = maxgoodqc + 100 @@ -983,19 +1000,6 @@ subroutine stringparse(str1, nwords, linenum) tower%recoDTQC = maxgoodqc + 100 endif -if (energy_balance .eqv. .true.) then - - lestring = 'LE_CORR' - leUNCstring = 'LE_CORR_JOINTUNC' - hstring = 'H_CORR' - hUNCstring = 'H_CORR_JOINTUNC' - write(string1,*) 'WARNING! Energy balance correction data turned on for LE and H' - write(string2,*) 'Check to make sure LE_CORR and H_CORR data is not missing' - call error_handler(E_MSG, source, string1,text=string2) -endif - - - end subroutine stringparse From c0319642ace57dcd05269e57f5d0ef4593f6ff21 Mon Sep 17 00:00:00 2001 From: braczka Date: Mon, 7 Aug 2023 15:49:09 -0600 Subject: [PATCH 11/25] Add edit and warnings for coarse time resolution Added more fixes and warnings to make code compatiable with coarser time resolution. File formatting changes based on time resolution. --- .../Ameriflux/fluxnetfull_to_obs.f90 | 131 ++++++++++++------ .../Ameriflux/fluxnetfull_to_obs.nml | 2 +- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 136599d8a3..d8e38ebb4e 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -8,8 +8,9 @@ program Fluxnetfull_to_obs ! ! Fluxnetfull_to_obs - a program that converts Ameriflux/Fluxnet FULLSET eddy ! covariance tower data of carbon, water and energy into -! DART obs_seq formatted files -! ## Fixme Add more description here once completed +! DART obs_seq formatted files. Works on native time +! resolution (HH, HR) or coarser resolution files (DD,WW,MM). + use types_mod, only : r8, MISSING_R8 @@ -72,7 +73,7 @@ program Fluxnetfull_to_obs ! globally-scoped variables !----------------------------------------------------------------------- -character(len=3100) :: input_line, bigline +character(len=5000) :: input_line, bigline character(len=512) :: string1, string2, string3 integer :: iline, nlines, nwords logical :: first_obs @@ -84,6 +85,7 @@ program Fluxnetfull_to_obs type(obs_type) :: obs, prev_obs type(time_type) :: prev_time, offset real(r8), parameter :: umol_to_gC = (1.0_r8/1000000.0_r8) * 12.0_r8 +logical :: res ! Initialize with default tower strings, modify later @@ -208,6 +210,28 @@ program Fluxnetfull_to_obs text2=string2,text3=string3) endif +! Provide gap-filling warning, and provide error if gap-filling +! turned on with DD or coarser time resolution + +if (gap_filled .eqv. .false.) then + + write(string1,*) 'WARNING!: gap filling is turned off. This is only recommended for' + write(string2,*) 'NEE, H and LE observations at native time resolution (HH, HR)' + write(string3,*) ' ...this approach assigns poor QC values to gap filled obs' + call error_handler(E_MSG, source, string1, & + text2=string2,text3=string3) + + if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & + time_resolution == 'WW') then + write(string1,*) 'ERROR!: gap filling can only be turned off for native time' + write(string2,*) 'resolution data (HH, HR)' + write(string3,*) 'Coarser time resolution (DD, WW, MM) must have gap_filled = .true.' + call error_handler(E_ERR, source, string1, & + text2=string2,text3=string3) + endif + +endif + ! Modify values to towerdata strings based on user input.nml ! 1) time_resolution: DD(daily),MM(monthly),YY(yearly) uses 'TIMESTAMP' header @@ -217,6 +241,7 @@ program Fluxnetfull_to_obs tower%startstring = 'TIMESTAMP' tower%endstring = 'TIMESTAMP' + res = .false. write(string1, *) 'Time resolution is set to =', time_resolution write(string2, *) 'Using TIMESTAMP to set DART observation time' @@ -225,6 +250,7 @@ program Fluxnetfull_to_obs elseif (time_resolution == 'HH' .or. time_resolution == 'HR' .or. & time_resolution == 'WW') then + res = .true. write(string1, *) 'Time resolution is set to =', time_resolution write(string2, *) 'Using TIMESTAMP_START and TIMESTAMP_END to set: DART observation time' if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) @@ -254,15 +280,6 @@ program Fluxnetfull_to_obs call error_handler(E_MSG, source, string1) endif - -if (gap_filled .eqv. .false.) then - - write(string1,*) 'WARNING!: gap_filled = false which removes all gpp and reco data' - write(string2,*) ' ...and also will remove nee, h and le if qc value is missing' - call error_handler(E_MSG, source, string1) -endif - - ! Specify the maximum number of observations in the input file, ! but only the actual number created will be written out. ! Each line has 5 flux observations available. @@ -646,6 +663,7 @@ subroutine decode_header(iunit,ncolumns) ! Read the line and strip off any leading whitespace. + read(iunit,'(A)',iostat=rcio) bigline if (rcio /= 0) then write(string1,*)'Cannot parse header. Begins <',trim(bigline(1:40)),'>' @@ -860,23 +878,43 @@ subroutine stringparse(str1, nwords, linenum) real(r8), allocatable, dimension(:) :: values -integer :: yeara, yearb -integer :: montha, daya, houra, mina -integer :: monthb, dayb, hourb, minb +integer :: yeara, yearb, montha, monthb, daya, dayb +integer :: houra, hourb, mina, minb + +integer :: time_adjust type(time_type) :: date_start, date_end -allocate(values(nwords-2)) +if (res .eqv. .true.) then + time_adjust = 2 +else + time_adjust = 1 +endif + + +allocate(values(nwords-time_adjust)) values = MISSING_R8 -! First two elements of string read in as character (time stamp) +! First two/one element(s) of string read in as character (time stamp) ! remainder of line read in as reals. -read(str1,*,iostat=rcio) tower%start_time,tower%end_time,values -if (rcio /= 0) then - write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' - call error_handler(E_ERR,'stringparse',string1, source) + +if (res .eqv. .true.) then + + read(str1,*,iostat=rcio) tower%start_time,tower%end_time,values + if (rcio /= 0) then + write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' + call error_handler(E_ERR,'stringparse',string1, source) + endif +else + read(str1,*,iostat=rcio) tower%start_time,values + if (rcio /= 0) then + write(string1,*)'Cannot parse line',linenum,'. Begins <',trim(str1(1:40)),'>' + call error_handler(E_ERR,'stringparse',string1, source) + endif + tower%end_time=tower%start_time endif + ! Assign flux observations, uncertainty and QC to tower structure ! ! Fixme: These are for high resolution format HH or HR only @@ -908,27 +946,27 @@ subroutine stringparse(str1, nwords, linenum) ! (CLM) NEP,GPP,ER units [gC m-2 s-1] ! (CLM) EFLX_LH_TOT_R,FSH units [W m-2] -tower%nee = values(tower%neeindex -2 ) -tower%neeUNC = values(tower%neeUNCindex -2 ) -tower%neeQC = nint(values(tower%neeQCindex -2)) -tower%le = values(tower%leindex -2 ) -tower%leUNC = values(tower%leUNCindex -2 ) -tower%leQC = nint(values(tower%leQCindex -2)) -tower%h = values(tower%hindex -2 ) -tower%hUNC = values(tower%hUNCindex -2 ) -tower%hQC = nint(values(tower%hQCindex -2)) -tower%gppDT = values(tower%gppDTindex -2) -tower%gppNT = values(tower%gppNTindex -2) -tower%recoDT = values(tower%recoDTindex -2) -tower%recoNT = values(tower%recoNTindex -2) -tower%gppDTUNC16 = values(tower%gppDTUNC16index -2) -tower%gppDTUNC84 = values(tower%gppDTUNC84index -2) -tower%gppNTUNC16 = values(tower%gppNTUNC16index -2) -tower%gppNTUNC84 = values(tower%gppNTUNC84index -2) -tower%recoDTUNC16 = values(tower%recoDTUNC16index-2) -tower%recoDTUNC84 = values(tower%recoDTUNC84index-2) -tower%recoNTUNC16 = values(tower%recoNTUNC16index-2) -tower%recoNTUNC84 = values(tower%recoNTUNC84index-2) +tower%nee = values(tower%neeindex -time_adjust ) +tower%neeUNC = values(tower%neeUNCindex -time_adjust ) +tower%neeQC = nint(values(tower%neeQCindex -time_adjust)) +tower%le = values(tower%leindex -time_adjust ) +tower%leUNC = values(tower%leUNCindex -time_adjust ) +tower%leQC = nint(values(tower%leQCindex -time_adjust)) +tower%h = values(tower%hindex -time_adjust ) +tower%hUNC = values(tower%hUNCindex -time_adjust ) +tower%hQC = nint(values(tower%hQCindex -time_adjust)) +tower%gppDT = values(tower%gppDTindex -time_adjust) +tower%gppNT = values(tower%gppNTindex -time_adjust) +tower%recoDT = values(tower%recoDTindex -time_adjust) +tower%recoNT = values(tower%recoNTindex -time_adjust) +tower%gppDTUNC16 = values(tower%gppDTUNC16index -time_adjust) +tower%gppDTUNC84 = values(tower%gppDTUNC84index -time_adjust) +tower%gppNTUNC16 = values(tower%gppNTUNC16index -time_adjust) +tower%gppNTUNC84 = values(tower%gppNTUNC84index -time_adjust) +tower%recoDTUNC16 = values(tower%recoDTUNC16index-time_adjust) +tower%recoDTUNC84 = values(tower%recoDTUNC84index-time_adjust) +tower%recoNTUNC16 = values(tower%recoNTUNC16index-time_adjust) +tower%recoNTUNC84 = values(tower%recoNTUNC84index-time_adjust) deallocate(values) @@ -936,6 +974,15 @@ subroutine stringparse(str1, nwords, linenum) read(tower%start_time(1:12), fmt='(i4, 4i2)') yeara,montha,daya,houra,mina read(tower%end_time(1:12), fmt='(i4, 4i2)') yearb,monthb,dayb,hourb,minb + +! Certain time resolutions (MM) does not define days +if (time_resolution == 'MM') then + daya = 1 + dayb = 1 +endif + + + write(*,*)'' write(string1, *) 'Display tower%start_time,yeara,montha,daya,houra,mina (LTC) =', tower%start_time, yeara, montha, daya, houra, mina write(string2, *) 'Display tower%end_time,yearb,monthb,dayb,hourb,minb (LTC) =', tower%end_time, yearb, monthb, dayb, hourb, minb diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index 07bb79e9b2..d0111cf670 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -7,7 +7,7 @@ elevation = -1.0, flux_height = -1.0, maxgoodqc = 3, - gap_filled = .false. + gap_filled = .true. energy_balance = .false. time_resolution = 'HH' verbose = .false. From c8fbef674a35e52ed9667665b7c313bc5b053a5d Mon Sep 17 00:00:00 2001 From: braczka Date: Fri, 11 Aug 2023 16:02:21 -0600 Subject: [PATCH 12/25] Fixes to flux QC and flux uncertainty values QC values change from integers to fraction depending upon time resolution. Added code to convert fraction back to integer. Provided empirical fix when uncertainty values for H,LE and NEE are missing. --- .../Ameriflux/fluxnetfull_to_obs.f90 | 173 +++++++++++++++--- 1 file changed, 147 insertions(+), 26 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index d8e38ebb4e..08a8ed2300 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -59,7 +59,7 @@ program Fluxnetfull_to_obs ! Always true except for latent,sensible heat and NEE for hourly time periods logical :: gap_filled = .true. ! Option for energy balance correction for latent and sensible heat -! Recommned to keep false as these values are typically missing +! Recommend to keep false as these values are typically missing logical :: energy_balance = .false. character(len=2) :: time_resolution = 'HH' logical :: verbose = .false. @@ -143,6 +143,7 @@ program Fluxnetfull_to_obs real(r8) :: nee real(r8) :: neeUNC integer :: neeQC + real(r8) :: neeQCfrac real(r8) :: le real(r8) :: leUNC integer :: leQC @@ -216,8 +217,8 @@ program Fluxnetfull_to_obs if (gap_filled .eqv. .false.) then write(string1,*) 'WARNING!: gap filling is turned off. This is only recommended for' - write(string2,*) 'NEE, H and LE observations at native time resolution (HH, HR)' - write(string3,*) ' ...this approach assigns poor QC values to gap filled obs' + write(string2,*) 'NEE,LE,SH observations 0=measured,gap_filled: 1=good gf,2=medium,3=poor. This approach' + write(string3,*) 'assigns QC values > maxgoodQC thus gap-filled data will not be written to obs_seq' call error_handler(E_MSG, source, string1, & text2=string2,text3=string3) @@ -372,6 +373,7 @@ program Fluxnetfull_to_obs write(logfileunit,*)tower%leQCindex , tower%leQCstring , tower%leQC write(logfileunit,*)tower%neeindex , tower%neestring , tower%nee write(logfileunit,*)tower%neeQCindex , tower%neeQCstring , tower%neeQC + write(logfileunit,*)tower%neeQCindex , tower%neeQCstring , tower%neeQCfrac write(logfileunit,*)tower%gppDTUNC16index , tower%gppDTUNC16string , tower%gppDTUNC16 write(logfileunit,*)tower%gppDTUNC84index , tower%gppDTUNC84string , tower%gppDTUNC84 write(logfileunit,*)tower%gppDTQC @@ -402,7 +404,7 @@ program Fluxnetfull_to_obs write(string1, *) 'Display tower%start_time and tower%end_time (LTC) =', tower%start_time,' ', tower%end_time call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) write(*,*)'' - write(string1, *) 'Display tower%nee tower%neeQC =', tower%nee, tower%neeQC + write(string1, *) 'Display tower%nee tower%neeQC tower%neeQCfrac =', tower%nee, tower%neeQC, tower%neeQCfrac call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%le tower%leQC =', tower%le, tower%leQC @@ -430,28 +432,93 @@ program Fluxnetfull_to_obs if (tower%hQC <= maxgoodqc) then ! Sensible Heat Flux [W m-2] oerr = tower%hUNC qc = real(tower%hQC,r8) + ! Check for missing uncertainty value, if needed assign % error + ! based on empirical examination of data + if (oerr <=0) then + select case(time_resolution) + case ('HR', 'HH') + oerr= tower%h*0.2 + case ('DD', 'WW') + oerr= tower%h*0.1 + case ('MM') + oerr= tower%h*0.05 + case default + write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution + call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + end select + endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%h, & TOWER_SENSIBLE_HEAT_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + + if (verbose) then + write(*,*)'' + write(string1, *) 'Display tower%h, tower%hUNC (1 SD) =', tower%h,' ', tower%hUNC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif + endif if (tower%leQC <= maxgoodqc) then ! Latent Heat Flux [W m-2] oerr = tower%leUNC qc = real(tower%leQC,r8) + if (oerr <=0) then + select case( time_resolution ) + case ('HR', 'HH') + oerr= tower%le*0.2 + case ('DD', 'WW') + oerr= tower%le*0.1 + case ('MM') + oerr= tower%le*0.05 + case default + write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution + call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + end select + endif + call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%le, & TOWER_LATENT_HEAT_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + + if (verbose) then + write(*,*)'' + write(string1, *) 'Display tower%le, tower%leUNC (1 SD) =', tower%le,' ', tower%leUNC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif + endif if (tower%neeQC <= maxgoodqc) then ! Net Ecosystem Exchange [umol m-2 s-1] oerr = tower%neeUNC * umol_to_gC tower%nee = -tower%nee * umol_to_gC ! Matches units in CLM [gC m-2 s-1] qc = real(tower%neeQC,r8) + if (oerr <=0) then + select case( time_resolution ) + case ('HR', 'HH') + oerr= abs(tower%nee)*0.2 + case ('DD', 'WW') + oerr= abs(tower%nee)*0.1 + case ('MM') + oerr= abs(tower%nee)*0.05 + case default + write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution + call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + end select + endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%nee, & TOWER_NETC_ECO_EXCHANGE, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + + if (verbose) then + write(*,*)'' + write(string1, *) 'Display tower%nee, tower%neeUNC (1 SD) =', tower%nee,' ', tower%neeUNC + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif + endif + + if (tower%gppDTQC <=maxgoodqc .and. tower%gppNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] sig_gppdt = (((tower%gppDTUNC84-tower%gppDTUNC16) / 2)**2)**0.5 ! Ustar unc contribution sig_gppnt = (((tower%gppNTUNC84-tower%gppNTUNC16) / 2)**2)**0.5 @@ -461,9 +528,29 @@ program Fluxnetfull_to_obs ! Take average of night and day partitioning methods tower%gpp = ((tower%gppDT + tower%gppNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] qc = maxval((/real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)/)) + if (oerr <=0) then + select case( time_resolution ) + case ('HR', 'HH') + oerr= tower%gpp*0.2 + case ('DD', 'WW') + oerr= tower%gpp*0.1 + case ('MM') + oerr= tower%gpp*0.05 + case default + write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution + call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + end select + endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%gpp, & TOWER_GPP_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + + if (verbose) then + write(*,*)'' + write(string1, *) 'Display tower%gpp, gpp uncertainty (1 SD) =', tower%gpp,' ', oerr + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif + endif if (tower%recoDTQC <= maxgoodqc .and. tower%recoNTQC <= maxgoodqc) then ! Gross Primary Production [umol m-2 s-1] @@ -475,11 +562,34 @@ program Fluxnetfull_to_obs ! Take average of night and day partitioning methods tower%reco = ((tower%recoDT + tower%recoNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] qc = maxval((/real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)/)) + if (oerr <=0) then + select case( time_resolution ) + case ('HR', 'HH') + oerr= tower%reco*0.2 + case ('DD', 'WW') + oerr= tower%reco*0.1 + case ('MM') + oerr= tower%reco*0.05 + case default + write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution + call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + end select + endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%reco, & TOWER_ER_FLUX, oerr, oday, osec, qc, obs) call add_obs_to_seq(obs_seq, obs, tower%time_obs, prev_obs, prev_time, first_obs) + + if (verbose) then + write(*,*)'' + write(string1, *) 'Display tower%reco, reco uncertainty (1 SD) =', tower%reco,' ', oerr + call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + endif + endif + + + end do obsloop ! If obs added to the sequence, write it out to a file now. @@ -916,20 +1026,21 @@ subroutine stringparse(str1, nwords, linenum) ! Assign flux observations, uncertainty and QC to tower structure -! -! Fixme: These are for high resolution format HH or HR only -! Fixme: If aggregrated (DD,WW,MM) need to edit, expand this +! Description of FLUXNET FULLSET variables ! start_time,end_time format YYYYMMDDHHMM ! nee units [umolCO2 m-2 s-1], VUT_REF ! neeUNC units [umolCO2 m-2 s-1], joint -! neeQC no dim 0=measured,gap_filled: 1=good,2=medium,3=poor +! neeQC (HH,HR) no dim 0=measured, gap_filled:1=good,2=medium,3=poor,-9999=missing +! neeQC (DD-MM) no dim fraction between 0-1, % of good quality filled data ! le units [W m-2] ! leUNC units [W m-2], random -! leQC no dim 0=measured,gap_filled: 1=good gf,2=medium,3=poor +! leQC (HH,HR) no dim 0=measured, gap_filled:1=good,2=medium,3=poor,-9999=missing +! leQC (DD-MM) no dim fraction between 0-1, % of good quality filled data ! h units [W m-2] ! hUNC units [W m-2], random -! hQC no dim 0=measured,gap_filled: 1=good gf,2=medium,3=poor +! hQC no dim 0=measured, gap_filled:1=good,2=medium,3=poor,-9999=missing +! hQC (DD-MM) no dim fraction between 0-1, % of good quality filled data ! gppDT units [umolCO2 m-2 s-1] VUT_REF daytime partition ! gppNT units [umolCO2 m-2 s-1] VUT_REF nighttime partition ! gppUNCDT[xx] units [umolCO2 m-2 s-1] 16,84 percentile @@ -947,8 +1058,9 @@ subroutine stringparse(str1, nwords, linenum) ! (CLM) EFLX_LH_TOT_R,FSH units [W m-2] tower%nee = values(tower%neeindex -time_adjust ) -tower%neeUNC = values(tower%neeUNCindex -time_adjust ) +tower%neeUNC = values(tower%neeUNCindex -time_adjust) tower%neeQC = nint(values(tower%neeQCindex -time_adjust)) +tower%neeQCfrac = values(tower%neeQCindex -time_adjust) tower%le = values(tower%leindex -time_adjust ) tower%leUNC = values(tower%leUNCindex -time_adjust ) tower%leQC = nint(values(tower%leQCindex -time_adjust)) @@ -1010,28 +1122,37 @@ subroutine stringparse(str1, nwords, linenum) ! Reject NEE data where neeQC is missing if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 +! If NEE (DD,WW,MM) convert from % filled QC to integer QC good/fair/poor +if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & + time_resolution == 'WW') then + if (tower%neeQCfrac >= 0.90) tower%neeQC = 1 ! >90% fill is good + if (tower%neeQCfrac < 0.90 .and. tower%neeQC >= 0.60) tower%neeQC = 2 ! 60-90% fill is fair + if (tower%neeQCfrac < 0.60 .and. tower%neeQC >= 0.0_r8) tower%neeQC = 3 ! <60% fill is poor +endif + + ! The QC values are typically missing for le and h (-9999) ! Thus manually assign poor QC values in these cases if (energy_balance .eqv. .false.) then - if (tower%leQC < 0) tower%leQC = 3 - if (tower%hQC < 0) tower%hQC = 3 -else ! No QC values for energy balance corrected le and h. Assign poor QC. - tower%leQC = 3 - tower%leQC = 3 + if (tower%leQC < 0.0_r8) tower%leQC = 2 + if (tower%hQC < 0.0_r8) tower%hQC = 2 +else ! No QC values for energy balance corrected le and h. Assign fair QC. + tower%leQC = 2 + tower%leQC = 2 endif -! No qc values for gpp/reco, thus assign good qc unless -! the gpp/reco value is missing (-9999) -tower%gppNTQC = 1 -tower%gppDTQC = 1 -tower%recoNTQC = 1 -tower%recoDTQC = 1 +! No qc values for gpp/reco, thus assign fair qc unless +! the gpp/reco values are negative values, then reject. +tower%gppNTQC = 2 +tower%gppDTQC = 2 +tower%recoNTQC = 2 +tower%recoDTQC = 2 -if (tower%gppNT < 0) tower%gppNTQC = maxgoodqc + 1000 -if (tower%gppDT < 0) tower%gppDTQC = maxgoodqc + 1000 -if (tower%recoNT < 0) tower%recoNTQC = maxgoodqc + 1000 -if (tower%recoDT < 0) tower%recoDTQC = maxgoodqc + 1000 +if (tower%gppNT < 0.0_r8) tower%gppNTQC = maxgoodqc + 1000 +if (tower%gppDT < 0.0_r8) tower%gppDTQC = maxgoodqc + 1000 +if (tower%recoNT < 0.0_r8) tower%recoNTQC = maxgoodqc + 1000 +if (tower%recoDT < 0.0_r8) tower%recoDTQC = maxgoodqc + 1000 ! Assign very bad qc to gap_filled data if user requests it ! such that maxgoodqc threshold does not add gap_filled data to obs_seq file From 5075b569ef8da343c6d3960d2d3fb38367951c84 Mon Sep 17 00:00:00 2001 From: braczka Date: Fri, 11 Aug 2023 16:20:51 -0600 Subject: [PATCH 13/25] Updating work input.nml --- observations/obs_converters/Ameriflux/work/input.nml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index d104b5a4bf..c4da250501 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -37,7 +37,7 @@ / &Fluxnetfull_to_obs_nml - text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv' + text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_MM_1991-2020_3-5.csv' obs_out_file = 'obs_seq.out' timezoneoffset = -5 latitude = 42.5378 @@ -45,9 +45,9 @@ elevation = 353 flux_height = 29 maxgoodqc = 3 - gap_filled = .false. + gap_filled = .true. energy_balance = .false. - time_resolution = 'HH' + time_resolution = 'MM' verbose = .true. / From 034ae1c73ae06c9ca2a58b75fdb1384fb94c2662 Mon Sep 17 00:00:00 2001 From: braczka Date: Mon, 14 Aug 2023 21:16:04 -0600 Subject: [PATCH 14/25] Add documentation for fluxnetfull_to_obs converter --- guide/important-capabilities-dart.rst | 6 +- .../Ameriflux/fluxnetfull_to_obs.f90 | 19 +- .../Ameriflux/fluxnetfull_to_obs.nml | 2 +- .../Ameriflux/fluxnetfull_to_obs.rst | 337 ++++++++++++++---- 4 files changed, 290 insertions(+), 74 deletions(-) diff --git a/guide/important-capabilities-dart.rst b/guide/important-capabilities-dart.rst index 7abb5044e0..520fa61236 100644 --- a/guide/important-capabilities-dart.rst +++ b/guide/important-capabilities-dart.rst @@ -76,7 +76,9 @@ is as follows: +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ | `Aviso `__: satellite derived sea surface height | Aviso | netCDF | +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ -| Level 4 Flux Tower data from `AmeriFlux `__ | Ameriflux | Comma-separated text | +| Level 4 Flux Tower data | Ameriflux | Comma-separated text | ++------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ +| Ameriflux Fullset Flux Tower data from `AmeriFlux `__ | Ameriflux | Comma-separated text | +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ | Level 2 soil moisture from `COSMOS `__ | COSMOS | Fixed-width text | +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ @@ -104,6 +106,8 @@ is as follows: +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ | Sea surface temperature | SST | netCDF | +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ +| Solar-Induced Fluorescence | SIF | netCDF | ++------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ | Special Sensor Ultraviolet Spectrographic Imager `(SSUSI) `__ retrievals | SSUSI | netCDF | +------------------------------------------------------------------------------------------------------+-------------------+-----------------------------------+ | World Ocean Database `(WOD) `__ | WOD | World Ocean Database packed ASCII | diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 08a8ed2300..1f9b3f283f 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -7,9 +7,11 @@ program Fluxnetfull_to_obs !------------------------------------------------------------------------ ! ! Fluxnetfull_to_obs - a program that converts Ameriflux/Fluxnet FULLSET eddy -! covariance tower data of carbon, water and energy into -! DART obs_seq formatted files. Works on native time -! resolution (HH, HR) or coarser resolution files (DD,WW,MM). +! covariance tower data of NEE, GPP, RE, latent heat and sensible +! heat fluxes into DART obs_seq formatted files. Works on native time +! resolution (HH, HR) that ED tower data or collected or at coarser +! aggregated time resolution files (DD,WW,MM) based on ONEFLUX +! methodology (Pastorello et al., 2020) use types_mod, only : r8, MISSING_R8 @@ -57,6 +59,7 @@ program Fluxnetfull_to_obs ! A maxgooqc=3 allows for good=1, medium=2, and poor=3 quality gap-filled data real(r8) :: maxgoodqc = 3.0_r8 ! Always true except for latent,sensible heat and NEE for hourly time periods +! Can only be false of HH or HR time resolution logical :: gap_filled = .true. ! Option for energy balance correction for latent and sensible heat ! Recommend to keep false as these values are typically missing @@ -1120,7 +1123,7 @@ subroutine stringparse(str1, nwords, linenum) ! Reject NEE data where neeQC is missing -if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 1000 +if (tower%neeQC < 0) tower%neeQC = maxgoodqc + 100 ! If NEE (DD,WW,MM) convert from % filled QC to integer QC good/fair/poor if (time_resolution == 'DD' .or. time_resolution == 'MM' .or. & @@ -1149,10 +1152,10 @@ subroutine stringparse(str1, nwords, linenum) tower%recoNTQC = 2 tower%recoDTQC = 2 -if (tower%gppNT < 0.0_r8) tower%gppNTQC = maxgoodqc + 1000 -if (tower%gppDT < 0.0_r8) tower%gppDTQC = maxgoodqc + 1000 -if (tower%recoNT < 0.0_r8) tower%recoNTQC = maxgoodqc + 1000 -if (tower%recoDT < 0.0_r8) tower%recoDTQC = maxgoodqc + 1000 +if (tower%gppNT < 0.0_r8) tower%gppNTQC = maxgoodqc + 100 +if (tower%gppDT < 0.0_r8) tower%gppDTQC = maxgoodqc + 100 +if (tower%recoNT < 0.0_r8) tower%recoNTQC = maxgoodqc + 100 +if (tower%recoDT < 0.0_r8) tower%recoDTQC = maxgoodqc + 100 ! Assign very bad qc to gap_filled data if user requests it ! such that maxgoodqc threshold does not add gap_filled data to obs_seq file diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index d0111cf670..df845865e6 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -9,7 +9,7 @@ maxgoodqc = 3, gap_filled = .true. energy_balance = .false. - time_resolution = 'HH' + time_resolution = 'HR' verbose = .false. / diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index c21213be9d..fbb92b9d73 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -1,30 +1,35 @@ PROGRAM ``Fluxnetfull_to_obs`` -========================= +============================== Overview -------- FLUXNET FULLSET data to DART observation sequence converter -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -! Fixme*** These need to be edited. -*********************************** -| This routine is designed to convert the flux tower Level 4 data from the `AmeriFlux `__ - network of observations from micrometeorological tower sites. AmeriFlux is part of - `FLUXNET `__ and the converter is hoped to be a suitable starting point for the conversion of - observations from FLUXNET. As of May 2012, I have not yet tried to work with any other observations from FLUXNET. -| The AmeriFlux Level 4 products are recorded using the local time. DART observation sequence files use GMT. For more - information about AmeriFlux data products, go to http://ameriflux.lbl.gov. - -The workflow is usually: - -#. download the Level 4 data for the towers and years in question (see DATA SOURCES below) -#. record the TIME ZONE, latitude, longitude, and elevation for each tower -#. build the DART executables with support for the tower observations. This is done by running ``preprocess`` with +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +| This routine is designed to convert Ameriflux FLUXNET data using the FULLSET format into a DART obs_seq file. + The data can be downloaded from the `AmeriFlux server `__ + or `FLUXNET server `__ The AmeriFlux network + is part of `FLUXNET `__ and the converter is suitable for either network given the similarity in + the data format. This code is not compatible with the SUBSET format of Ameriflux data, however, this code could be + modified for that purpose. The FULLSET format contains additional uncertainty information as described in the ONEflux + processing technqiue as described by `Pastorello et al., (2020) `__ + which is required to formally propogate error into aggregated estimates of tower flux data. +| The AmeriFlux/FLUXNET data products use local time, therefore the code converts the time into UTC such that it is compatible + with DART and CLM. More information about the AmeriFlux data product is provided below and also within the `variable + description webpage `__. + +The steps required to prepare Ameriflux data for an assimilation usually include: + +#. Download the Ameriflux or FLUXNET FULLSET data for the towers and years in question (see DATA SOURCES below) +#. Record the TIME ZONE, latitude, longitude, and elevation for each tower. This tower metadata can be found + `here `__ or `here `__. +#. Manually provide tower metadata information via the ``Fluxnetfull_to_obs_nml`` namelist since this information is + not contained in the data file itself. +#. Build the DART executables with support for the tower observations. This is done by running ``preprocess`` with ``obs_def_tower_mod.f90`` in the list of ``input_files`` for ``preprocess_nml``. -#. provide basic tower information via the ``Fluxnetfull_to_obs_nml`` namelist since this information is not contained in the - Level 4 data file -#. convert each Level 4 data file individually using ``Fluxnetfull_to_obs`` -#. combine all output files for the region and timeframe of interest into one file using +#. Convert each Ameriflux data file individually using ``Fluxnetfull_to_obs`` +#. Combine all output files for the region and timeframe of interest into one file using :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` For some models (CLM, for example), it is required to reorganize the observation sequence files into a series of files @@ -43,7 +48,6 @@ namelist. &Fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', - year = -1, timezoneoffset = -1, latitude = -1.0, longitude = -1.0, @@ -51,6 +55,8 @@ namelist. flux_height = -1.0, maxgoodqc = 3, gap_filled = .true. + energy_balance = .false. + time_resolution = 'HR' verbose = .false. / @@ -59,15 +65,13 @@ namelist. +-----------------+--------------------+-----------------------------------------------------------------------------+ | Contents | Type | Description | +=================+====================+=============================================================================+ - | text_input_file | character(len=128) | Name of the Level 4 ASCII file of comma-separated values. This may be a | + | text_input_file | character(len=128) | Name of the Ameriflux ASCII file of comma-separated values. This may be a | | | | relative or absolute filename. | +-----------------+--------------------+-----------------------------------------------------------------------------+ | obs_out_file | character(len=128) | Name of the output observation sequence file. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | year | integer | The year of the observations in the Level 4 text file. | - +-----------------+--------------------+-----------------------------------------------------------------------------+ - | timezoneoffset | real | the time zone offset (in hours) of the station. The tower observation times | - | | | are local time, we need to convert them to GMT. | + | timezoneoffset | real | The local time zone difference (in hours) from UTC of the flux tower site. | + | | | The code will convert from local time to UTC. | +-----------------+--------------------+-----------------------------------------------------------------------------+ | latitude | real | Latitude (in degrees N) of the tower. | +-----------------+--------------------+-----------------------------------------------------------------------------+ @@ -75,57 +79,262 @@ namelist. | | | longitudes in the range [0,360]. An input value of -90 will be converted to | | | | 270, for example. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | elevation | real | surface elevation (in meters) of the tower. | + | elevation | real | Surface elevation (in meters) of the tower. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | flux_height | real | Height (in meters) of the flux instrument on the tower. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | flux_height | real | height (in meters) of the flux instrument on the tower. | + | maxgoodqc | real | Maximum value of the observation (obs) quality control (QC) value to pass | + | | | to the output observation sequence file. Obs that have a QC exceeding this | + | | | value will be rejected from the observation sequence file. The 'filter' step| + | | | is also capable of discriminating obs based on the QC value, therefore it | + | | | is recommended to keep all obs during the conversion unless you | + | | | are certain they are bad/missing. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | maxgoodqc | real | maximum value of any observation quality control flag to pass through to | - | | | the output observation sequence. Keep in mind that ``filter`` has the | - | | | ability to discriminate on the value, so there is really little to be | - | | | gained by rejecting them during the conversion. | + | gap_filled | logical | If set to .false. excludes all observations except for measured NEE, SH and | + | | | and LE. Must be set to .true. for time resolutions greater than HH or HR. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | verbose | logical | Print extra information during the ``level4_to_obs`` execution. | + | energy_balance | logical | If .true. applies energy balance closure to SH and LE fluxes, otherwise | + | | | applies standard approach. | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | time_resolution | logical | Defines the flux time resolution. Possible values include native resolution | + | | | of hourly (HH) or half-hourly (HR), or coarser gap-filled resolutions | + | | | including daily (DD), weekly (WW) or monthly (MM). | + +-----------------+--------------------+-----------------------------------------------------------------------------+ + | verbose | logical | If .true. outputs diagnostic data to help with troubleshooting. | +-----------------+--------------------+-----------------------------------------------------------------------------+ Data sources ------------ | The data was acquired from https://ameriflux.lbl.gov/data/download-data/ -| and choosing the data product ``Ameriflux FLUXNET`` and variable set ``FULLSET`` -| The filenames are organized by site and time (HH,HR,WW,DD,MM,YY) and look like: -| ``AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv,`` + and choosing the data product ``Ameriflux FLUXNET`` and variable set ``FULLSET`` + The filenames are organized by site and time (HH,HR,WW,DD,MM,YY) and look like: +| ``AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv``, | ``AMF_US-xBR_FLUXNET_FULLSET_MM_2017-2021_3-5.csv`` -| The Level 4 products in question are ASCII files of comma-separated values taken every 30 minutes for an entire year. - The first line is a comma-separated list of column descriptors, all subsequent lines are comma-separated numerical - values. The converter presently searches for the columns pertaining to *NEE_or_fMDS*, *H_f*, *LE_f*, their - corresponding quality control fields, and those columns pertaining to the time of the observation. These values are - mapped as follows: - -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| Level 4 units | Level 4 variable | description | DART type | DART kind | DART units | -+===============+==================+==========================+==========================+===========================+============+ -| W/m^2 | LE_f | Latent Heat Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| [0-3] | LE_fqc | QC for LE_f | N/A | N/A | same | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| W/m^2 | H_f | Sensible Heat Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| [0-3] | H_fqc | QC for H_f | N/A | N/A | same | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| umolCO2/m^2/s | NEE_or_fMDS | Net Ecosystem Production | TOWER_NETC_ECO_EXCHANGE | QTY_NET_CARBON_PRODUCTION | gC/m^2/s | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ -| [0-3] | NEE_or_fMDSqc | QC for NEE_or_fMDS | N/A | N/A | same | -+---------------+------------------+--------------------------+--------------------------+---------------------------+------------+ - - - - -The ``LE_fqc``, ``H_fqc``, and ``NEE_or_fMDSqc`` variables use the following convention: - - 0 = original, 1 = category A (most reliable), 2 = category B (medium), 3 = category C (least reliable). (Refer to +| The Ameriflux product in question are ASCII files of comma-separated values taken from all years the data is available. + The first line is a comma-separated list of column descriptors, and all subsequent lines + are comma-separated numerical values. The converter presently searches for the columns pertaining to carbon, water + and energy fluxes, corresponding quality control fields, uncertainty values, and the time of the observation. Data is available + at varying time resolutions incuding: native resolution, hourly (HR) or half-hourly (HH), and aggregated resolution, daily (DD), + weekly (WW), and monthly (MM). The source data does include yearly (YY) time resolution as well, but the coarse nature of yearly flux + observations poorly constrain fast changing ecological process, thus are not supported by this converter. The required column + headers depend upon the namelist definitions, including the ``time_resolution``, ``energy_balance`` and ``gap_filled`` settings. + These variables are defined as follows: + ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| Ameriflux Units | Ameriflux Variable | Description | DART type | DART kind | DART units | ++=================+======================+===============================+==========================+=============================+===============+ +| YYYYMMDDHHMM | TIMESTAMP_START | start of time window | N/A | N/A | Gregorian | +| | TIMESTAMP_END | end of time window | | | | +| | | (HH,HR,WW only) | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| YYYYMMDDHHMM | TIMESTAMP | time (DD and MM only) | N/A | N/A | Gregorian | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | LE_F_MDS | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | LE_RANDUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| [0-3] integer | LE_F_MDS_QC | QC for LE Flux | N/A | N/A | [0-3] integer | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | LE_CORR | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | +| | | energy_balance = .true. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | LE_CORR_JOINTUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | +| | | energy_balance = .true. | | | | +| | | Random and Ustar contributions| | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | H_F_MDS | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | H_RANDUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| [0-3] integer | H_F_MDS_QC | QC for SH Flux | N/A | N/A | [0-3] integer | +| | | energy_balance = .false. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | H_CORR | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | +| | | energy_balance = .true. | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| W/m^2 | H_CORR_JOINTUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | +| | | energy_balance = .true. | | | | +| | | Random and Ustar contributions| | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | NEE_VUT_REF | Net Ecosystem Exchange (NEE) | TOWER_NETC_ECO_EXCHANGE | QTY_NET_CARBON_PRODUCTION | gC/m^2/s | +| | | Variable Ustar, reference | | | | +| | | flux approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | NEE_VUT_REF_JOINTUNC | Uncertainty for NEE Flux | N/A | N/A | gC/m^2/s | +| | | Variable Ustar, reference | | | | +| | | flux approach. Random and | | | | +| | | and Ustar contributions | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| [0-3] integer | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | +| | | Variable Ustar, reference | | | | +| | | flux approach. | | | | +| | | (HH or HR only) | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| [0-1] fraction | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | +| | | Variable Ustar, reference | | | | +| | | flux approach. | | | | +| | | (DD, WW and MM only) | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_DT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_NT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_DT_VUT_16 | 16th percentile uncertainty | | | | +| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_DT_VUT_84 | 84th percentile uncertainty | | | | +| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_NT_VUT_16 | 84th percentile uncertainty | | | | +| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | GPP_NT_VUT_84 | 84th percentile uncertainty | | | | +| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_DT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_NT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_DT_VUT_16 | 16th percentile uncertainty | | | | +| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_DT_VUT_84 | 84th percentile uncertainty | | | | +| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | +| | | Day partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_NT_VUT_16 | 84th percentile uncertainty | | | | +| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ +| umolCO2/m^2/s | RECO_NT_VUT_84 | 84th percentile uncertainty | | | | +| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | +| | | Night partition, Variable | | | | +| | | Ustar, reference approach | | | | ++-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + + +Carbon Fluxes +----------------------- +The flux data files come with several estimates for the carbon fluxes (NEE, GPP and ER). In the conversion code we choose the +'variable Ustar with reference flux' approach (_VUT_REF). This choice was based on guidance from Pastorello et al., (2020) which states: + +"The variable proposed in the SUBSET (or FULLSET) product is NEE_VUT_REF since it maintains the temporal variability (as opposed to the MEAN NEE), +it is representative of the ensemble, and the VUT method is sensitive to possible changes of the canopy (density and height) and site setup, +which can have an impact on the turbulence and consequently on the USTAR threshold. The RECO and GPP products in SUBSET (or FULLSET) are calculated +from the corresponding NEE variables filtered with the VUT method, generating RECO_NT_VUT_REF and RECO_DT_VUT_REF for ER, and +GPP_NT_VUT_REF and GPP_DT_VUT_REF for GPP. It is important to use both daytime (DT) and nighttime (NT) variables, and consider their +difference as uncertainty." + +The reference NEE (_REF) is selected on the basis of model efficiency (MEF) thus is the single NEE data set (out of all ensemble members that sample +the Ustar uncertainty) that is most representative. + +The mean of the night and day partitioning methods for GPP and ER are used as the observation within the converter. + + +Uncertainty +----------- +Multiple methods are used to estimate uncertainty within the conversion code. There are, in general, three separate sources of uncertainty +in flux data. First, random uncertainty, represents the random movement of eddies within the atmosphere where smaller eddies are sampled +more frequently and larger eddies less frequently. Random uncertainty (_RANDC) estimates are based on Hollinger, D. Y. & Richardson, A. D. +Uncertainty in eddy covariance measurements and its application to physiological models. Tree Physiol. 25, 873–885 (2005). +Second, Ustar uncertainty, represents the uncertainty contribution from low turbulence conditions as calculated from the Ustar (friction velocity) +threshold. The ONEflux method uses a bootstrap sampling method to generate a 200 member ensemble from which the flux percentiles are estimated. +The third source of uncertainty, paritioning uncertainty, applies to GPP and ER only. The night (Reichsten et al., 2005) and day (Lasslop et al.,) partitioning methods +estimate both the contributions of photosynthesis (GPP) and ecosystem respiration (ER) as measured from the net carbon exchange (NEE). + +Within the conversion code, the flux uncertainty values (_RANDUNC) account for random uncertainty (SH and LE), where uncertainty denoted with (_JOINTUNC) indicates +combined uncertainty of random and energy balance closure uncertainty (SH and LE). The NEE uncertainty (NEE_VUT_REF_JOINTUNC) accounts for both random and Ustar contributions, +whereas the GPP and ER uncertainty combine both Ustar and partitioning method uncertainty as: + + |`` Day method Ustar uncertainty (sigma_fluxdt) = (((fluxDTUNC84-fluxDTUNC16) / 2)^2)^0.5`` + |``Night method Ustar uncertainty (sigma_fluxnt) = (((fluxNTUNC84-fluxNTUNC16) / 2)^2)^0.5`` + + |``Ustar and partitioning uncertainty (sigma) = (0.25 * (sigma_fluxdt)^2 + 0.25 * (sigma_fluxnt)^2)^0.5`` + +where ``flux`` stands for either GPP or ER. The 84th and 16th percentile estimates are used to generate 1 sigma estimates +for day and night Ustar uncertainty respectively. Then the contributions of the day and night Ustar uncertainty +are propogated together through standard technqiues assuming gaussian uncertainty distributions (Taylor et al, +An Introduction to Error Analysis). + +Quality Control +--------------- +The general QC naming convention uses an integer system (0-3) defined as the following: + + 0 = measured, 1 = good quality gapfill, 2 = medium quality gapfill, 3 = poor quality gapfill. (Refer to Pastorello et al., (2020) or Reichstein et al. 2005 Global Change Biology for more information) +The QC values **do not** follow this convention for NEE, SH and LE fluxes for time resolutions coarser than the native resolution of +HH and HR. For DD,WW,and MM time resolution observations, the QC value is based on a fraction from 0-1 that indicates the fraction of the time period +that consists of measured or good quality gap-filled data. Because it is more straightforward to reject observations in DART +based on an integer value scale, the conversion code converts these fractional QC values (0-1) to integer QC values (0-3) +where QC(integer)=1 when QC(fraction) > 0.90; QC(integer)=2 when 0.90 >= QC(fraction) >= 0.60; and QC(integer)=3 when 0.60 > QC(fraction) >=0. +This conversion system was based on a qualitative assessment of a small sample size of flux data. The user can change these thresholds within +the source code. + +There are times when a QC value is missing or does not exist for an observation. In these cases the converter code does the following: + +1) Missing QC values (-9999) where the associated flux observation looks physically reasonable are assigned a QC = 2. +2) GPP and ER observations do not have a QC. If the observations are physically reasonable a QC = 2 is assigned. +3) There are situations where +100 is added to an existing QC value such that the observations are purposely rejected + during the conversion (assuming maxgoodQC = < 100). These situations are: +| a) When gap_filled = .false., all observations that are not measured (QC = 0) + b) When GPP or RE give non-physical negative values. + c) If NEE QC is missing. This is rare. + + + +Gap-Filling +----------- +Gap-filled data is only available for the native resolution format (HH, HR) for flux observations of +NEE, SH and LE. For all other situations choosing gap-filled data is mandatory, because measurements of fluxes at +the native resolution are frequently violated that cause the eddy covariance method to fail. These include +situations where the friction velocity (Ustar) falls below a certain value that prevents adequate land-atmosphere mixing, or when +their is instrumentation failure. In these cases, gap-filling methods (essentially models) are required to calculate daily and coarser +time resolutions. Because gap-filled data is technically modeled data (e.g. Marginal Distribution Method (MDS) which relies on +met condtions physcially and temporally similar to missing data) a user may desire only real observations. + +In general, we recommend to turn gap-filled data to .true. during the conversion process, and then use the QC value as a way to discriminate +against lower quality observations during the `filter` step. + + + + +Energy-Balance +-------------- + +We provide the option to use LE and SH observations that have been corrected for energy-balance closure. +In these cases the correction is based on comparing the latent and sensible heat flux against other sources +of energy loss/gain including net incoming radiation and energy radiated through the ground. + +Data Policy +----------- I am repeating the AmeriFlux `Data Fair-Use Policy `__ because I believe it is important to be a good scientific citizen: From 674c9c6b63c10f699d3efdb9d2dbb75833bf77e0 Mon Sep 17 00:00:00 2001 From: braczka Date: Tue, 15 Aug 2023 09:04:17 -0600 Subject: [PATCH 15/25] Corrections for fluxnetfull_to_obs documentation --- .../Ameriflux/fluxnetfull_to_obs.rst | 365 +++++++++--------- 1 file changed, 187 insertions(+), 178 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index fbb92b9d73..3d060ecdbc 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -4,37 +4,40 @@ PROGRAM ``Fluxnetfull_to_obs`` Overview -------- -FLUXNET FULLSET data to DART observation sequence converter -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +FLUXNET FULLSET data to DART observation sequence file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | This routine is designed to convert Ameriflux FLUXNET data using the FULLSET format into a DART obs_seq file. - The data can be downloaded from the `AmeriFlux server `__ - or `FLUXNET server `__ The AmeriFlux network + The data can be downloaded from the `AmeriFlux server. `__ + or `FLUXNET server. `__ The AmeriFlux network is part of `FLUXNET `__ and the converter is suitable for either network given the similarity in the data format. This code is not compatible with the SUBSET format of Ameriflux data, however, this code could be modified for that purpose. The FULLSET format contains additional uncertainty information as described in the ONEflux - processing technqiue as described by `Pastorello et al., (2020) `__ + processing technique as described by `Pastorello et al., (2020) `__ which is required to formally propogate error into aggregated estimates of tower flux data. + | The AmeriFlux/FLUXNET data products use local time, therefore the code converts the time into UTC such that it is compatible - with DART and CLM. More information about the AmeriFlux data product is provided below and also within the `variable + with DART and CLM. More information about the AmeriFlux data product is provided below and also within the `flux variable description webpage `__. The steps required to prepare Ameriflux data for an assimilation usually include: #. Download the Ameriflux or FLUXNET FULLSET data for the towers and years in question (see DATA SOURCES below) -#. Record the TIME ZONE, latitude, longitude, and elevation for each tower. This tower metadata can be found +#. Record the TIME ZONE, latitude, longitude, and elevation and tower height at each site. This tower metadata can be found `here `__ or `here `__. -#. Manually provide tower metadata information via the ``Fluxnetfull_to_obs_nml`` namelist since this information is +#. Manually provide tower metadata information via the ``Fluxnetfull_to_obs_nml`` namelist as this information is not contained in the data file itself. #. Build the DART executables with support for the tower observations. This is done by running ``preprocess`` with ``obs_def_tower_mod.f90`` in the list of ``input_files`` for ``preprocess_nml``. #. Convert each Ameriflux data file individually using ``Fluxnetfull_to_obs`` -#. Combine all output files for the region and timeframe of interest into one file using +#. If necessary, combine all output files for the region and timeframe of interest into one file using :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` -For some models (CLM, for example), it is required to reorganize the observation sequence files into a series of files -that contains ONLY the observations for each assimilation time step. This can be achieved with the `makedaily.sh `__ -script. + +.. Note:: + The Ameriflux data typically have all years combined into one file. However, for some models (CLM, for example), + it is required to reorganize the observation sequence files into a series of files that contains ONLY the observations + for each assimilation time step. This can be achieved with the `makedaily.sh `__ script. Namelist -------- @@ -91,14 +94,15 @@ namelist. | | | are certain they are bad/missing. | +-----------------+--------------------+-----------------------------------------------------------------------------+ | gap_filled | logical | If set to .false. excludes all observations except for measured NEE, SH and | - | | | and LE. Must be set to .true. for time resolutions greater than HH or HR. | + | | | and LE. Must be set to .true. for time resolutions greater than hourly | + | | | (HH) or half-hourly (HR). | +-----------------+--------------------+-----------------------------------------------------------------------------+ | energy_balance | logical | If .true. applies energy balance closure to SH and LE fluxes, otherwise | | | | applies standard approach. | +-----------------+--------------------+-----------------------------------------------------------------------------+ - | time_resolution | logical | Defines the flux time resolution. Possible values include native resolution | - | | | of hourly (HH) or half-hourly (HR), or coarser gap-filled resolutions | - | | | including daily (DD), weekly (WW) or monthly (MM). | + | time_resolution | logical | Defines the flux observation time resolution. Possible values are native | + | | | resolution of hourly (HH) or half-hourly (HR), or coarser gap-filled | + | | | resolutions including daily (DD), weekly (WW) or monthly (MM). | +-----------------+--------------------+-----------------------------------------------------------------------------+ | verbose | logical | If .true. outputs diagnostic data to help with troubleshooting. | +-----------------+--------------------+-----------------------------------------------------------------------------+ @@ -107,10 +111,11 @@ Data sources ------------ | The data was acquired from https://ameriflux.lbl.gov/data/download-data/ - and choosing the data product ``Ameriflux FLUXNET`` and variable set ``FULLSET`` + by choosing the data product ``Ameriflux FLUXNET`` and variable set ``FULLSET``. The filenames are organized by site and time (HH,HR,WW,DD,MM,YY) and look like: | ``AMF_US-Ha1_FLUXNET_FULLSET_HR_1991-2020_3-5.csv``, | ``AMF_US-xBR_FLUXNET_FULLSET_MM_2017-2021_3-5.csv`` + | The Ameriflux product in question are ASCII files of comma-separated values taken from all years the data is available. The first line is a comma-separated list of column descriptors, and all subsequent lines are comma-separated numerical values. The converter presently searches for the columns pertaining to carbon, water @@ -121,190 +126,208 @@ Data sources headers depend upon the namelist definitions, including the ``time_resolution``, ``energy_balance`` and ``gap_filled`` settings. These variables are defined as follows: -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| Ameriflux Units | Ameriflux Variable | Description | DART type | DART kind | DART units | -+=================+======================+===============================+==========================+=============================+===============+ -| YYYYMMDDHHMM | TIMESTAMP_START | start of time window | N/A | N/A | Gregorian | -| | TIMESTAMP_END | end of time window | | | | -| | | (HH,HR,WW only) | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| YYYYMMDDHHMM | TIMESTAMP | time (DD and MM only) | N/A | N/A | Gregorian | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | LE_F_MDS | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | LE_RANDUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| [0-3] integer | LE_F_MDS_QC | QC for LE Flux | N/A | N/A | [0-3] integer | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | LE_CORR | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | -| | | energy_balance = .true. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | LE_CORR_JOINTUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | -| | | energy_balance = .true. | | | | -| | | Random and Ustar contributions| | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | H_F_MDS | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | H_RANDUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| [0-3] integer | H_F_MDS_QC | QC for SH Flux | N/A | N/A | [0-3] integer | -| | | energy_balance = .false. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | H_CORR | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | -| | | energy_balance = .true. | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| W/m^2 | H_CORR_JOINTUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | -| | | energy_balance = .true. | | | | -| | | Random and Ustar contributions| | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | NEE_VUT_REF | Net Ecosystem Exchange (NEE) | TOWER_NETC_ECO_EXCHANGE | QTY_NET_CARBON_PRODUCTION | gC/m^2/s | -| | | Variable Ustar, reference | | | | -| | | flux approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | NEE_VUT_REF_JOINTUNC | Uncertainty for NEE Flux | N/A | N/A | gC/m^2/s | -| | | Variable Ustar, reference | | | | -| | | flux approach. Random and | | | | -| | | and Ustar contributions | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| [0-3] integer | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | -| | | Variable Ustar, reference | | | | -| | | flux approach. | | | | -| | | (HH or HR only) | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| [0-1] fraction | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | -| | | Variable Ustar, reference | | | | -| | | flux approach. | | | | -| | | (DD, WW and MM only) | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_DT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_NT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_DT_VUT_16 | 16th percentile uncertainty | | | | -| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_DT_VUT_84 | 84th percentile uncertainty | | | | -| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_NT_VUT_16 | 84th percentile uncertainty | | | | -| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | GPP_NT_VUT_84 | 84th percentile uncertainty | | | | -| | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_DT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_NT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_DT_VUT_16 | 16th percentile uncertainty | | | | -| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_DT_VUT_84 | 84th percentile uncertainty | | | | -| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | -| | | Day partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_NT_VUT_16 | 84th percentile uncertainty | | | | -| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ -| umolCO2/m^2/s | RECO_NT_VUT_84 | 84th percentile uncertainty | | | | -| | | estimate for ER Flux | N/A | N/A | gC/m^2/s | -| | | Night partition, Variable | | | | -| | | Ustar, reference approach | | | | -+-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + + +.. container:: + + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | Ameriflux Units | Ameriflux Variable | Description | DART type | DART kind | DART units | + +=================+======================+===============================+==========================+=============================+===============+ + | YYYYMMDDHHMM | TIMESTAMP_START | start of time window | N/A | N/A | Gregorian | + | | TIMESTAMP_END | end of time window | | | | + | | | (HH,HR,WW only) | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | YYYYMMDDHHMM | TIMESTAMP | time (DD and MM only) | N/A | N/A | Gregorian | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | LE_F_MDS | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | LE_RANDUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | [0-3] integer | LE_F_MDS_QC | QC for LE Flux | N/A | N/A | [0-3] integer | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | LE_CORR | Latent Heat (LE) Flux | TOWER_LATENT_HEAT_FLUX | QTY_LATENT_HEAT_FLUX | W/m^2 | + | | | energy_balance = .true. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | LE_CORR_JOINTUNC | Uncertainty for LE Flux | N/A | N/A | W/m^2 | + | | | energy_balance = .true. | | | | + | | | Random and Ustar contributions| | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | H_F_MDS | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | H_RANDUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | [0-3] integer | H_F_MDS_QC | QC for SH Flux | N/A | N/A | [0-3] integer | + | | | energy_balance = .false. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | H_CORR | Sensible Heat (SH) Flux | TOWER_SENSIBLE_HEAT_FLUX | QTY_SENSIBLE_HEAT_FLUX | W/m^2 | + | | | energy_balance = .true. | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | W/m^2 | H_CORR_JOINTUNC | Uncertainty for SH Flux | N/A | N/A | W/m^2 | + | | | energy_balance = .true. | | | | + | | | Random and Ustar contributions| | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | NEE_VUT_REF | Net Ecosystem Exchange (NEE) | TOWER_NETC_ECO_EXCHANGE | QTY_NET_CARBON_PRODUCTION | gC/m^2/s | + | | | Variable Ustar, reference | | | | + | | | flux approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | NEE_VUT_REF_JOINTUNC | Uncertainty for NEE Flux | N/A | N/A | gC/m^2/s | + | | | Variable Ustar, reference | | | | + | | | flux approach. Random and | | | | + | | | and Ustar contributions | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | [0-3] integer | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | + | | | Variable Ustar, reference | | | | + | | | flux approach. | | | | + | | | (HH or HR only) | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | [0-1] fraction | NEE_VUT_REF_QC | QC for NEE Flux | N/A | N/A | [0-3] integer | + | | | Variable Ustar, reference | | | | + | | | flux approach. | | | | + | | | (DD, WW and MM only) | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_DT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_NT_VUT_REF | Gross Primary Production (GPP)| TOWER_GPP_FLUX | QTY_GROSS_PRIMARY_PROD_FLUX | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_DT_VUT_16 | 16th percentile uncertainty | | | | + | | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_DT_VUT_84 | 84th percentile uncertainty | | | | + | | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_NT_VUT_16 | 16th percentile uncertainty | | | | + | | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | GPP_NT_VUT_84 | 84th percentile uncertainty | | | | + | | | estimate for GPP Flux | N/A | N/A | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_DT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_NT_VUT_REF | Ecosystem Respiration (ER) | TOWER_ER_FLUX | QTY_ER_FLUX | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_DT_VUT_16 | 16th percentile uncertainty | | | | + | | | estimate for ER Flux | N/A | N/A | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_DT_VUT_84 | 84th percentile uncertainty | | | | + | | | estimate for ER Flux | N/A | N/A | gC/m^2/s | + | | | Day partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_NT_VUT_16 | 16th percentile uncertainty | | | | + | | | estimate for ER Flux | N/A | N/A | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ + | umolCO2/m^2/s | RECO_NT_VUT_84 | 84th percentile uncertainty | | | | + | | | estimate for ER Flux | N/A | N/A | gC/m^2/s | + | | | Night partition, Variable | | | | + | | | Ustar, reference approach | | | | + +-----------------+----------------------+-------------------------------+--------------------------+-----------------------------+---------------+ Carbon Fluxes ------------------------ -The flux data files come with several estimates for the carbon fluxes (NEE, GPP and ER). In the conversion code we choose the -'variable Ustar with reference flux' approach (_VUT_REF). This choice was based on guidance from Pastorello et al., (2020) which states: +------------- +The flux data files come with several different approaches to estimate the carbon fluxes (NEE, GPP and ER). In the conversion code we choose the +**variable Ustar with reference flux approach (_VUT_REF)**. This choice was based on guidance from Pastorello et al., (2020) which states: -"The variable proposed in the SUBSET (or FULLSET) product is NEE_VUT_REF since it maintains the temporal variability (as opposed to the MEAN NEE), +`"The variable proposed in the SUBSET (or FULLSET) product is NEE_VUT_REF since it maintains the temporal variability (as opposed to the MEAN NEE), it is representative of the ensemble, and the VUT method is sensitive to possible changes of the canopy (density and height) and site setup, which can have an impact on the turbulence and consequently on the USTAR threshold. The RECO and GPP products in SUBSET (or FULLSET) are calculated from the corresponding NEE variables filtered with the VUT method, generating RECO_NT_VUT_REF and RECO_DT_VUT_REF for ER, and GPP_NT_VUT_REF and GPP_DT_VUT_REF for GPP. It is important to use both daytime (DT) and nighttime (NT) variables, and consider their -difference as uncertainty." +difference as uncertainty."` The reference NEE (_REF) is selected on the basis of model efficiency (MEF) thus is the single NEE data set (out of all ensemble members that sample -the Ustar uncertainty) that is most representative. - -The mean of the night and day partitioning methods for GPP and ER are used as the observation within the converter. +the Ustar uncertainty) that is most representative. The mean of the night and day partitioning methods for GPP and ER are used as the observation within +the converter. Uncertainty ----------- Multiple methods are used to estimate uncertainty within the conversion code. There are, in general, three separate sources of uncertainty -in flux data. First, random uncertainty, represents the random movement of eddies within the atmosphere where smaller eddies are sampled -more frequently and larger eddies less frequently. Random uncertainty (_RANDC) estimates are based on Hollinger, D. Y. & Richardson, A. D. -Uncertainty in eddy covariance measurements and its application to physiological models. Tree Physiol. 25, 873–885 (2005). -Second, Ustar uncertainty, represents the uncertainty contribution from low turbulence conditions as calculated from the Ustar (friction velocity) +in flux data. First, **random uncertainty**, represents the random movement of eddies within the atmosphere where smaller eddies are sampled +more frequently and larger eddies less frequently. Random uncertainty (_RANDC) estimates are based on `Hollinger, D. Y. & Richardson, A. D. +Uncertainty in eddy covariance measurements and its application to physiological models. Tree Physiol. 25, 873–885 (2005)`. +Second, **Ustar uncertainty**, represents the uncertainty contribution from low turbulence conditions as calculated from the Ustar (friction velocity) threshold. The ONEflux method uses a bootstrap sampling method to generate a 200 member ensemble from which the flux percentiles are estimated. -The third source of uncertainty, paritioning uncertainty, applies to GPP and ER only. The night (Reichsten et al., 2005) and day (Lasslop et al.,) partitioning methods +The third source of uncertainty, **partitioning uncertainty**, applies to GPP and ER only. The night (Reichsten et al., 2005) and day (Lasslop et al.,) partitioning methods estimate both the contributions of photosynthesis (GPP) and ecosystem respiration (ER) as measured from the net carbon exchange (NEE). Within the conversion code, the flux uncertainty values (_RANDUNC) account for random uncertainty (SH and LE), where uncertainty denoted with (_JOINTUNC) indicates combined uncertainty of random and energy balance closure uncertainty (SH and LE). The NEE uncertainty (NEE_VUT_REF_JOINTUNC) accounts for both random and Ustar contributions, whereas the GPP and ER uncertainty combine both Ustar and partitioning method uncertainty as: - |`` Day method Ustar uncertainty (sigma_fluxdt) = (((fluxDTUNC84-fluxDTUNC16) / 2)^2)^0.5`` - |``Night method Ustar uncertainty (sigma_fluxnt) = (((fluxNTUNC84-fluxNTUNC16) / 2)^2)^0.5`` + | ``Day method Ustar uncertainty (sigma_fluxdt) = (((fluxDTUNC84-fluxDTUNC16) / 2)^2)^0.5`` + | ``Night method Ustar uncertainty (sigma_fluxnt) = (((fluxNTUNC84-fluxNTUNC16) / 2)^2)^0.5`` - |``Ustar and partitioning uncertainty (sigma) = (0.25 * (sigma_fluxdt)^2 + 0.25 * (sigma_fluxnt)^2)^0.5`` + | ``Ustar and partitioning uncertainty (sigma) = (0.25 * (sigma_fluxdt)^2 + 0.25 * (sigma_fluxnt)^2)^0.5`` where ``flux`` stands for either GPP or ER. The 84th and 16th percentile estimates are used to generate 1 sigma estimates for day and night Ustar uncertainty respectively. Then the contributions of the day and night Ustar uncertainty are propogated together through standard technqiues assuming gaussian uncertainty distributions (Taylor et al, An Introduction to Error Analysis). +.. Note:: + + In practice the relative uncertainty reduces as the flux time resolution increases. This is likely a result of random uncertainty + decreasing with coarser time resolutions as the sample size of measurements increases. This reduced relative + uncertainty will cause a stronger impact of the observations on the prior model state (i.e. larger increments) during the ``filter`` + step. To prevent overconfident observations the ONEflux method attempts to account for as many sources of uncertainty as possible + and that is reflected in this converter code. + + + Quality Control --------------- The general QC naming convention uses an integer system (0-3) defined as the following: - 0 = measured, 1 = good quality gapfill, 2 = medium quality gapfill, 3 = poor quality gapfill. (Refer to Pastorello et al., (2020) or + ``0 = measured``, ``1 = good quality gapfill``, ``2 = medium quality gapfill``, ``3 = poor quality gapfill``. (Refer to Pastorello et al., (2020) or Reichstein et al. 2005 Global Change Biology for more information) The QC values **do not** follow this convention for NEE, SH and LE fluxes for time resolutions coarser than the native resolution of -HH and HR. For DD,WW,and MM time resolution observations, the QC value is based on a fraction from 0-1 that indicates the fraction of the time period +HH and HR. For DD, WW, and MM time resolution observations, the QC value is based on a fraction from 0-1 that indicates the fraction of the time period that consists of measured or good quality gap-filled data. Because it is more straightforward to reject observations in DART -based on an integer value scale, the conversion code converts these fractional QC values (0-1) to integer QC values (0-3) -where QC(integer)=1 when QC(fraction) > 0.90; QC(integer)=2 when 0.90 >= QC(fraction) >= 0.60; and QC(integer)=3 when 0.60 > QC(fraction) >=0. +based on an integer value scale, the conversion code converts these fractional QC values (0-1) to integer QC values (0-3) as follows: + +#. ``QC(integer)=1 when QC(fraction) > 0.90``; +#. ``QC(integer)=2 when 0.90 >= QC(fraction) >= 0.60``; +#. ``QC(integer)=3 when 0.60 > QC(fraction) >=0``. + This conversion system was based on a qualitative assessment of a small sample size of flux data. The user can change these thresholds within the source code. There are times when a QC value is missing or does not exist for an observation. In these cases the converter code does the following: -1) Missing QC values (-9999) where the associated flux observation looks physically reasonable are assigned a QC = 2. -2) GPP and ER observations do not have a QC. If the observations are physically reasonable a QC = 2 is assigned. -3) There are situations where +100 is added to an existing QC value such that the observations are purposely rejected +#. Missing QC values (-9999) where the associated flux observation looks physically reasonable are assigned a QC = 2. +#. GPP and ER observations do not have a QC. If the observations are physically reasonable a QC = 2 is assigned. +#. There are situations where +100 is added to an existing QC value such that the observations are purposely rejected during the conversion (assuming maxgoodQC = < 100). These situations are: -| a) When gap_filled = .false., all observations that are not measured (QC = 0) + + a) When gap_filled = .false., all observations that are not measured (QC = 0) b) When GPP or RE give non-physical negative values. c) If NEE QC is missing. This is rare. @@ -320,8 +343,8 @@ their is instrumentation failure. In these cases, gap-filling methods (essentia time resolutions. Because gap-filled data is technically modeled data (e.g. Marginal Distribution Method (MDS) which relies on met condtions physcially and temporally similar to missing data) a user may desire only real observations. -In general, we recommend to turn gap-filled data to .true. during the conversion process, and then use the QC value as a way to discriminate -against lower quality observations during the `filter` step. +In general, we recommend to turn gap-filled data to ``.true.`` during the conversion process, and then use the QC value as a way to discriminate +against lower quality observations during the ``filter`` step. @@ -335,30 +358,16 @@ of energy loss/gain including net incoming radiation and energy radiated through Data Policy ----------- -I am repeating the AmeriFlux `Data Fair-Use Policy `__ because -I believe it is important to be a good scientific citizen: - - "The AmeriFlux data provided on this site are freely available and were furnished by individual AmeriFlux scientists - who encourage their use. - Please kindly inform in writing (or e-mail) the appropriate AmeriFlux scientist(s) of how you intend to use the data - and of any publication plans. It is also important to contact the AmeriFlux investigator to assure you are - downloading the latest revision of the data and to prevent potential misuse or misinterpretation of the data. - Please acknowledge the data source as a citation or in the acknowledgments if no citation is available. If the - AmeriFlux Principal Investigators (PIs) feel that they should be acknowledged or offered participation as authors, - they will let you know and we assume that an agreement on such matters will be reached before publishing and/or use - of the data for publication. - If your work directly competes with the PI's analysis they may ask that they have the opportunity to submit a - manuscript before you submit one that uses unpublished data. In addition, when publishing please acknowledge the - agency that supported the research. - Lastly, we kindly request that those publishing papers using AmeriFlux data provide reprints to the PIs providing the - data and to the AmeriFlux archive via ameriflux.lbl.gov." +It is important to recognize the flux data providers who have made their research publically available to advance +scientific research. Please see the Ameriflux data policy `here `__ +and the FLUXNET 2015 data policy provided `here `__. + Programs -------- The ``Fluxnetfull_to_obs.f90`` file is the source for the main converter program. Look at the source code where it reads the -example data file. You will almost certainly need to change the "read" statement to match your data format. The example -code reads each text line into a character buffer and then reads from that buffer to parse up the data items. +example data file. The example code reads each text line into a character buffer and then reads from that buffer to parse up the data items. To compile and test, go into the work subdirectory and run the ``quickbuild.sh`` script to build the converter and a couple of general purpose utilities. ``advance_time`` helps with calendar and time computations, and the From 71430448b41c4745286d6e5eb1e284ced47ac823 Mon Sep 17 00:00:00 2001 From: braczka Date: Thu, 17 Aug 2023 08:56:19 -0600 Subject: [PATCH 16/25] Correcting misleading comments in code --- observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 1f9b3f283f..d4487f257f 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -1135,7 +1135,8 @@ subroutine stringparse(str1, nwords, linenum) ! The QC values are typically missing for le and h (-9999) -! Thus manually assign poor QC values in these cases +! Thus manually assign fair QC values in these cases +! When energy balance is turned off if (energy_balance .eqv. .false.) then if (tower%leQC < 0.0_r8) tower%leQC = 2 if (tower%hQC < 0.0_r8) tower%hQC = 2 @@ -1159,7 +1160,7 @@ subroutine stringparse(str1, nwords, linenum) ! Assign very bad qc to gap_filled data if user requests it ! such that maxgoodqc threshold does not add gap_filled data to obs_seq file -! If leQC and hQC are missing no way to identify if gap_filled, thus rejected +! If leQC and hQC are missing values already forced to 2, thus rejected if (gap_filled .eqv. .false.) then if (tower%neeQC >0) tower%neeQC = maxgoodqc + 100 if (tower%leQC >0) tower%leQC = maxgoodqc + 100 From 023b66d0f76687502bc23b267339b17cfa8bbb00 Mon Sep 17 00:00:00 2001 From: braczka Date: Thu, 17 Aug 2023 13:37:37 -0600 Subject: [PATCH 17/25] Correcting typos in flux QC values --- observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index d4487f257f..c5f7cfc507 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -530,7 +530,7 @@ program Fluxnetfull_to_obs oerr = oerr * umol_to_gC ! Take average of night and day partitioning methods tower%gpp = ((tower%gppDT + tower%gppNT) / 2) * umol_to_gC ! Matches units in CLM [gC m-2 s-1] - qc = maxval((/real(tower%recoDTQC,r8),real(tower%recoNTQC,r8)/)) + qc = maxval((/real(tower%gppDTQC,r8),real(tower%gppNTQC,r8)/)) if (oerr <=0) then select case( time_resolution ) case ('HR', 'HH') @@ -1142,7 +1142,7 @@ subroutine stringparse(str1, nwords, linenum) if (tower%hQC < 0.0_r8) tower%hQC = 2 else ! No QC values for energy balance corrected le and h. Assign fair QC. tower%leQC = 2 - tower%leQC = 2 + tower%hQC = 2 endif From 9d68fe59bb772b04da870b823c69249464b63e93 Mon Sep 17 00:00:00 2001 From: braczka Date: Thu, 17 Aug 2023 16:59:39 -0600 Subject: [PATCH 18/25] Fixing typos and adding user documentation Adding better documentation on how user can adjust values that convert fractional QC to integer QC and also empirically derived variables related to uncertainty estimation for missing values --- .../obs_converters/Ameriflux/fluxnetfull_to_obs.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index 3d060ecdbc..4a92224b8c 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -298,6 +298,10 @@ An Introduction to Error Analysis). step. To prevent overconfident observations the ONEflux method attempts to account for as many sources of uncertainty as possible and that is reflected in this converter code. + If an observation has a missing uncertainty value, the code estimates an uncertainty based on an empirically-based relative uncertainty + value that reduces with increasing time resolution. The default relative uncertainty values are 20%, 10% and 5% for HH/HR, DD/WW and MM + time resolution respectively. The user can adjust these values within the source code. + Quality Control @@ -317,8 +321,11 @@ based on an integer value scale, the conversion code converts these fractional Q #. ``QC(integer)=2 when 0.90 >= QC(fraction) >= 0.60``; #. ``QC(integer)=3 when 0.60 > QC(fraction) >=0``. -This conversion system was based on a qualitative assessment of a small sample size of flux data. The user can change these thresholds within -the source code. +.. Note:: + + The fraction QC to integer QC conversion approach was based on a qualitative assessment of flux data from Harvard Forest. + Depending on location and topography not all flux tower data will have a comparably high % of gap filled data.. The user + can change these thresholds within the source code. There are times when a QC value is missing or does not exist for an observation. In these cases the converter code does the following: From 2d8210ba2a3a0c4fb8af8a70fb1493659322970f Mon Sep 17 00:00:00 2001 From: braczka Date: Tue, 29 Aug 2023 14:32:43 -0600 Subject: [PATCH 19/25] changing fluxnetfull naming to lowercase --- .../Ameriflux/fluxnetfull_to_obs.f90 | 72 +++++++++---------- .../Ameriflux/fluxnetfull_to_obs.nml | 2 +- .../Ameriflux/fluxnetfull_to_obs.rst | 10 +-- .../Ameriflux/work/quickbuild.sh | 2 +- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index c5f7cfc507..6aeb3bed01 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -2,11 +2,11 @@ ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -program Fluxnetfull_to_obs +program fluxnetfull_to_obs !------------------------------------------------------------------------ ! -! Fluxnetfull_to_obs - a program that converts Ameriflux/Fluxnet FULLSET eddy +! fluxnetfull_to_obs - a program that converts Ameriflux/Fluxnet FULLSET eddy ! covariance tower data of NEE, GPP, RE, latent heat and sensible ! heat fluxes into DART obs_seq formatted files. Works on native time ! resolution (HH, HR) that ED tower data or collected or at coarser @@ -43,7 +43,7 @@ program Fluxnetfull_to_obs implicit none -character(len=*), parameter :: source = 'Fluxnetfull_to_obs.f90' +character(len=*), parameter :: source = 'fluxnetfull_to_obs.f90' !----------------------------------------------------------------------- ! Namelist with default values @@ -67,7 +67,7 @@ program Fluxnetfull_to_obs character(len=2) :: time_resolution = 'HH' logical :: verbose = .false. -namelist /Fluxnetfull_to_obs_nml/ text_input_file, obs_out_file, & +namelist /fluxnetfull_to_obs_nml/ text_input_file, obs_out_file, & timezoneoffset, latitude, longitude, elevation, & flux_height, maxgoodqc, gap_filled, energy_balance, & time_resolution, verbose @@ -180,16 +180,16 @@ program Fluxnetfull_to_obs ! start of executable code !----------------------------------------------------------------------- -call initialize_utilities('Fluxnetfull_to_obs') +call initialize_utilities('fluxnetfull_to_obs') ! Read the namelist entry -call find_namelist_in_file("input.nml", "Fluxnetfull_to_obs_nml", iunit) -read(iunit, nml = Fluxnetfull_to_obs_nml, iostat = rcio) -call check_namelist_read(iunit, rcio, "Fluxnetfull_to_obs_nml") +call find_namelist_in_file("input.nml", "fluxnetfull_to_obs_nml", iunit) +read(iunit, nml = fluxnetfull_to_obs_nml, iostat = rcio) +call check_namelist_read(iunit, rcio, "fluxnetfull_to_obs_nml") ! Record the namelist values used for the run -if (do_nml_file()) write(nmlfileunit, nml=Fluxnetfull_to_obs_nml) -if (do_nml_term()) write( * , nml=Fluxnetfull_to_obs_nml) +if (do_nml_file()) write(nmlfileunit, nml=fluxnetfull_to_obs_nml) +if (do_nml_term()) write( * , nml=fluxnetfull_to_obs_nml) ! time setup call set_calendar_type(GREGORIAN) @@ -198,7 +198,7 @@ program Fluxnetfull_to_obs write(string1, *) 'tower located at lat, lon, elev =', latitude, longitude, elevation write(string2, *) 'flux observations taken at =', flux_height,'m' -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) +if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs',string1,text2=string2) ! check the lat/lon values to see if they are ok if (longitude < 0.0_r8) longitude = longitude + 360.0_r8 @@ -209,7 +209,7 @@ program Fluxnetfull_to_obs write (string2,*)'latitude should be [-90, 90] but is ',latitude write (string3,*)'longitude should be [ 0,360] but is ',longitude - string1 ='tower location error in input.nml&Fluxnetfull_to_obs_nml' + string1 ='tower location error in input.nml &fluxnetfull_to_obs_nml' call error_handler(E_ERR, source, string1, & text2=string2,text3=string3) endif @@ -249,7 +249,7 @@ program Fluxnetfull_to_obs write(string1, *) 'Time resolution is set to =', time_resolution write(string2, *) 'Using TIMESTAMP to set DART observation time' - if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs',string1,text2=string2) elseif (time_resolution == 'HH' .or. time_resolution == 'HR' .or. & time_resolution == 'WW') then @@ -257,7 +257,7 @@ program Fluxnetfull_to_obs res = .true. write(string1, *) 'Time resolution is set to =', time_resolution write(string2, *) 'Using TIMESTAMP_START and TIMESTAMP_END to set: DART observation time' - if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) + if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs',string1,text2=string2) else write(string1,*) 'time_resolution set incorrectly within input.nml' @@ -293,7 +293,7 @@ program Fluxnetfull_to_obs ! in observation sequence - the other is for the new observation. iunit = open_file(text_input_file, 'formatted', 'read') -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs','opened input file '//trim(text_input_file)) +if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs','opened input file '//trim(text_input_file)) nlines = count_file_lines(iunit) max_obs = 5*nlines @@ -405,28 +405,28 @@ program Fluxnetfull_to_obs write(*,*)'' write(string1, *) 'Display tower%start_time and tower%end_time (LTC) =', tower%start_time,' ', tower%end_time - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%nee tower%neeQC tower%neeQCfrac =', tower%nee, tower%neeQC, tower%neeQCfrac - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%le tower%leQC =', tower%le, tower%leQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%h tower%hQC =', tower%h, tower%hQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%gppDT tower%gppDTQC =', tower%gppDT, tower%gppDTQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%gppNT tower%gppNTQC =', tower%gppNT, tower%gppNTQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%recoDT tower%recoDTQC =', tower%recoDT, tower%recoDTQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) write(*,*)'' write(string1, *) 'Display tower%recoNT tower%recoNTQC =', tower%recoNT, tower%recoNTQC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif @@ -447,7 +447,7 @@ program Fluxnetfull_to_obs oerr= tower%h*0.05 case default write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution - call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + call error_handler(E_ERR,'fluxnetfull_to_obs',string1) end select endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%h, & @@ -457,7 +457,7 @@ program Fluxnetfull_to_obs if (verbose) then write(*,*)'' write(string1, *) 'Display tower%h, tower%hUNC (1 SD) =', tower%h,' ', tower%hUNC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif endif @@ -475,7 +475,7 @@ program Fluxnetfull_to_obs oerr= tower%le*0.05 case default write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution - call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + call error_handler(E_ERR,'fluxnetfull_to_obs',string1) end select endif @@ -486,7 +486,7 @@ program Fluxnetfull_to_obs if (verbose) then write(*,*)'' write(string1, *) 'Display tower%le, tower%leUNC (1 SD) =', tower%le,' ', tower%leUNC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif endif @@ -505,7 +505,7 @@ program Fluxnetfull_to_obs oerr= abs(tower%nee)*0.05 case default write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution - call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + call error_handler(E_ERR,'fluxnetfull_to_obs',string1) end select endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%nee, & @@ -515,7 +515,7 @@ program Fluxnetfull_to_obs if (verbose) then write(*,*)'' write(string1, *) 'Display tower%nee, tower%neeUNC (1 SD) =', tower%nee,' ', tower%neeUNC - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif endif @@ -541,7 +541,7 @@ program Fluxnetfull_to_obs oerr= tower%gpp*0.05 case default write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution - call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + call error_handler(E_ERR,'fluxnetfull_to_obs',string1) end select endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%gpp, & @@ -551,7 +551,7 @@ program Fluxnetfull_to_obs if (verbose) then write(*,*)'' write(string1, *) 'Display tower%gpp, gpp uncertainty (1 SD) =', tower%gpp,' ', oerr - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif endif @@ -575,7 +575,7 @@ program Fluxnetfull_to_obs oerr= tower%reco*0.05 case default write(string1, *) 'ERROR, time_resolution must be HH,HR,DD,WW,MM, value is:', time_resolution - call error_handler(E_ERR,'Fluxnetfull_to_obs',string1) + call error_handler(E_ERR,'fluxnetfull_to_obs',string1) end select endif call create_3d_obs(latitude, longitude, flux_height, VERTISHEIGHT, tower%reco, & @@ -585,7 +585,7 @@ program Fluxnetfull_to_obs if (verbose) then write(*,*)'' write(string1, *) 'Display tower%reco, reco uncertainty (1 SD) =', tower%reco,' ', oerr - call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + call error_handler(E_MSG,'fluxnetfull_to_obs',string1) endif endif @@ -598,7 +598,7 @@ program Fluxnetfull_to_obs ! If obs added to the sequence, write it out to a file now. if ( get_num_obs(obs_seq) > 0 ) then write(string1,*)'writing obs_seq, obs_count = ', get_num_obs(obs_seq) - if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1) + if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs',string1) call write_obs_seq(obs_seq, obs_out_file) endif @@ -1101,7 +1101,7 @@ subroutine stringparse(str1, nwords, linenum) write(*,*)'' write(string1, *) 'Display tower%start_time,yeara,montha,daya,houra,mina (LTC) =', tower%start_time, yeara, montha, daya, houra, mina write(string2, *) 'Display tower%end_time,yearb,monthb,dayb,hourb,minb (LTC) =', tower%end_time, yearb, monthb, dayb, hourb, minb -if (verbose) call error_handler(E_MSG,'Fluxnetfull_to_obs',string1,text2=string2) +if (verbose) call error_handler(E_MSG,'fluxnetfull_to_obs',string1,text2=string2) @@ -1176,6 +1176,6 @@ end subroutine stringparse -end program Fluxnetfull_to_obs +end program fluxnetfull_to_obs diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml index df845865e6..02d958176b 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.nml @@ -1,4 +1,4 @@ -&Fluxnetfull_to_obs_nml +&fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', timezoneoffset = -1, diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst index 4a92224b8c..825b3a27ef 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.rst @@ -1,4 +1,4 @@ -PROGRAM ``Fluxnetfull_to_obs`` +PROGRAM ``fluxnetfull_to_obs`` ============================== Overview @@ -25,11 +25,11 @@ The steps required to prepare Ameriflux data for an assimilation usually include #. Download the Ameriflux or FLUXNET FULLSET data for the towers and years in question (see DATA SOURCES below) #. Record the TIME ZONE, latitude, longitude, and elevation and tower height at each site. This tower metadata can be found `here `__ or `here `__. -#. Manually provide tower metadata information via the ``Fluxnetfull_to_obs_nml`` namelist as this information is +#. Manually provide tower metadata information via the ``fluxnetfull_to_obs_nml`` namelist as this information is not contained in the data file itself. #. Build the DART executables with support for the tower observations. This is done by running ``preprocess`` with ``obs_def_tower_mod.f90`` in the list of ``input_files`` for ``preprocess_nml``. -#. Convert each Ameriflux data file individually using ``Fluxnetfull_to_obs`` +#. Convert each Ameriflux data file individually using ``fluxnetfull_to_obs`` #. If necessary, combine all output files for the region and timeframe of interest into one file using :doc:`../../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool` @@ -48,7 +48,7 @@ namelist. :: - &Fluxnetfull_to_obs_nml + &fluxnetfull_to_obs_nml text_input_file = 'textdata.input', obs_out_file = 'obs_seq.out', timezoneoffset = -1, @@ -373,7 +373,7 @@ and the FLUXNET 2015 data policy provided `here Date: Tue, 29 Aug 2023 17:20:40 -0600 Subject: [PATCH 20/25] Removing duplicate obs_utilities_mod code --- .../Ameriflux/fluxnetfull_to_obs.f90 | 109 +----------------- 1 file changed, 2 insertions(+), 107 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index 6aeb3bed01..e5b19f5004 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -35,6 +35,8 @@ program fluxnetfull_to_obs init_obs_sequence, get_num_obs, & set_copy_meta_data, set_qc_meta_data +use obs_utilities_mod, only : add_obs_to_seq, create_3d_obs + use obs_kind_mod, only : TOWER_SENSIBLE_HEAT_FLUX, & TOWER_NETC_ECO_EXCHANGE, & TOWER_LATENT_HEAT_FLUX, & @@ -608,113 +610,6 @@ program fluxnetfull_to_obs contains -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! -! create_3d_obs - subroutine that is used to create an observation -! type from observation data. -! -! NOTE: assumes the code is using the threed_sphere locations module, -! that the observation has a single data value and a single -! qc value, and that this obs type has no additional required -! data (e.g. gps and radar obs need additional data per obs) -! -! lat - latitude of observation -! lon - longitude of observation -! vval - vertical coordinate -! vkind - kind of vertical coordinate (pressure, level, etc) -! obsv - observation value -! okind - observation kind -! oerr - observation error (in units of standard deviation) -! day - gregorian day -! sec - gregorian second -! qc - quality control value -! obs - observation type -! -! created Oct. 2007 Ryan Torn, NCAR/MMM -! adapted for more generic use 11 Mar 2010, nancy collins, ncar/image -! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -subroutine create_3d_obs(lat, lon, vval, vkind, obsv, okind, oerr, day, sec, qc, obs) -use obs_def_mod, only : obs_def_type, set_obs_def_time, set_obs_def_type_of_obs, & - set_obs_def_error_variance, set_obs_def_location -use obs_sequence_mod, only : obs_type, set_obs_values, set_qc, set_obs_def -use time_manager_mod, only : time_type, set_time -use location_mod, only : set_location - - integer, intent(in) :: okind, vkind, day, sec - real(r8), intent(in) :: lat, lon, vval, obsv, oerr, qc - type(obs_type), intent(inout) :: obs - -real(r8) :: obs_val(1), qc_val(1) -type(obs_def_type) :: obs_def - -call set_obs_def_location(obs_def, set_location(lon, lat, vval, vkind)) -call set_obs_def_type_of_obs(obs_def, okind) -call set_obs_def_time(obs_def, set_time(sec, day)) -call set_obs_def_error_variance(obs_def, oerr * oerr) -call set_obs_def(obs, obs_def) - -obs_val(1) = obsv -call set_obs_values(obs, obs_val) -qc_val(1) = qc -call set_qc(obs, qc_val) - -end subroutine create_3d_obs - - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! -! add_obs_to_seq -- adds an observation to a sequence. inserts if first -! obs, inserts with a prev obs to save searching if that's possible. -! -! seq - observation sequence to add obs to -! obs - observation, already filled in, ready to add -! obs_time - time of this observation, in dart time_type format -! prev_obs - the previous observation that was added to this sequence -! (will be updated by this routine) -! prev_time - the time of the previously added observation (will also -! be updated by this routine) -! first_obs - should be initialized to be .true., and then will be -! updated by this routine to be .false. after the first obs -! has been added to this sequence. -! -! created Mar 8, 2010 nancy collins, ncar/image -! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -subroutine add_obs_to_seq(seq, obs, obs_time, prev_obs, prev_time, first_obs) - -use obs_sequence_mod, only : obs_sequence_type, obs_type, insert_obs_in_seq -use time_manager_mod, only : time_type, operator(>=) - -type(obs_sequence_type), intent(inout) :: seq -type(obs_type), intent(inout) :: obs, prev_obs -type(time_type), intent(in) :: obs_time -type(time_type), intent(inout) :: prev_time -logical, intent(inout) :: first_obs - -! insert(seq,obs) always works (i.e. it inserts the obs in -! proper time format) but it can be slow with a long file. -! supplying a previous observation that is older (or the same -! time) as the new one speeds up the searching a lot. - -if(first_obs) then ! for the first observation, no prev_obs - call insert_obs_in_seq(seq, obs) - first_obs = .false. -else - if(obs_time >= prev_time) then ! same time or later than previous obs - call insert_obs_in_seq(seq, obs, prev_obs) - else ! earlier, search from start of seq - call insert_obs_in_seq(seq, obs) - endif -endif - -! update for next time -prev_obs = obs -prev_time = obs_time - -end subroutine add_obs_to_seq - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! ! count_file_lines -- From 3dcd93e81b30b7de903f332771a5d2c6dba5f690 Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Tue, 5 Sep 2023 06:50:22 -0600 Subject: [PATCH 21/25] fix: remove cloud_overlap integer from list of logicals to query fixes #537 --- observations/forward_operators/obs_def_rttov13_mod.f90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/observations/forward_operators/obs_def_rttov13_mod.f90 b/observations/forward_operators/obs_def_rttov13_mod.f90 index ebcf033d5e..539ea25fb8 100644 --- a/observations/forward_operators/obs_def_rttov13_mod.f90 +++ b/observations/forward_operators/obs_def_rttov13_mod.f90 @@ -4238,8 +4238,6 @@ function get_rttov_option_logical(field_name) result(p) p = USER_CLD_OPT_PARAM case('GRID_BOX_AVG_CLOUD') p = GRID_BOX_AVG_CLOUD - case('CLOUD_OVERLAP') - p = cloud_overlap case('ADDPC') p = ADDPC case('ADDRADREC') From 2e1fecdafa2aba16a20530814939c19b663d59f9 Mon Sep 17 00:00:00 2001 From: ann-norcio Date: Wed, 13 Sep 2023 14:24:19 -0600 Subject: [PATCH 22/25] Added get_channel to obs_def_rttov13_mod.f90 --- observations/forward_operators/obs_def_rttov13_mod.f90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/observations/forward_operators/obs_def_rttov13_mod.f90 b/observations/forward_operators/obs_def_rttov13_mod.f90 index ebcf033d5e..963565cdcf 100644 --- a/observations/forward_operators/obs_def_rttov13_mod.f90 +++ b/observations/forward_operators/obs_def_rttov13_mod.f90 @@ -397,7 +397,8 @@ module obs_def_rttov_mod write_rttov_metadata, & interactive_rttov_metadata, & get_expected_radiance, & - get_rttov_option_logical + get_rttov_option_logical, & + get_channel ! The rttov_test.f90 program uses these, but no one else should. From 6432a869f926f4e0960bff29bdba401cc606c107 Mon Sep 17 00:00:00 2001 From: Helen Kershaw <20047007+hkershaw-brown@users.noreply.github.com> Date: Fri, 15 Sep 2023 13:04:24 -0400 Subject: [PATCH 23/25] last of the capital -> lower case changes for fluxnet --- observations/obs_converters/Ameriflux/work/input.nml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/observations/obs_converters/Ameriflux/work/input.nml b/observations/obs_converters/Ameriflux/work/input.nml index c4da250501..2ad5583cce 100644 --- a/observations/obs_converters/Ameriflux/work/input.nml +++ b/observations/obs_converters/Ameriflux/work/input.nml @@ -36,7 +36,7 @@ verbose = .TRUE. / -&Fluxnetfull_to_obs_nml +&fluxnetfull_to_obs_nml text_input_file = '../data/AMF_US-Ha1_FLUXNET_FULLSET_MM_1991-2020_3-5.csv' obs_out_file = 'obs_seq.out' timezoneoffset = -5 From e6f582a5e6b074a6fef3316b2e85f062ad6baf06 Mon Sep 17 00:00:00 2001 From: braczka Date: Fri, 15 Sep 2023 15:11:33 -0600 Subject: [PATCH 24/25] Removing some unnessary function includes --- .../obs_converters/Ameriflux/fluxnetfull_to_obs.f90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 index e5b19f5004..655eb990d3 100644 --- a/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 +++ b/observations/obs_converters/Ameriflux/fluxnetfull_to_obs.f90 @@ -17,7 +17,7 @@ program fluxnetfull_to_obs use types_mod, only : r8, MISSING_R8 use utilities_mod, only : initialize_utilities, finalize_utilities, & - register_module, error_handler, E_MSG, E_ERR, & + error_handler, E_MSG, E_ERR, & open_file, close_file, do_nml_file, do_nml_term, & check_namelist_read, find_namelist_in_file, & nmlfileunit, logfileunit @@ -30,7 +30,7 @@ program fluxnetfull_to_obs use location_mod, only : VERTISHEIGHT -use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, & +use obs_sequence_mod, only : obs_sequence_type, obs_type, & static_init_obs_sequence, init_obs, write_obs_seq, & init_obs_sequence, get_num_obs, & set_copy_meta_data, set_qc_meta_data @@ -597,6 +597,8 @@ program fluxnetfull_to_obs end do obsloop +call close_file(iunit) + ! If obs added to the sequence, write it out to a file now. if ( get_num_obs(obs_seq) > 0 ) then write(string1,*)'writing obs_seq, obs_count = ', get_num_obs(obs_seq) From 06c13cf28614348d0889741ad4c34511c3f8b079 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Mon, 18 Sep 2023 12:52:48 -0600 Subject: [PATCH 25/25] conf.py bump and CHANGELOG update --- CHANGELOG.rst | 17 +++++++++++++++++ conf.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26b3529ad6..b7ddc8a552 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,23 @@ individual files. The changes are now listed with the most recent at the top. +**September 18 2023 :: Fluxnet observation converter and obs_def_rttov13_mod.f90 bug-fixes. Tag v10.8.4** + +Fluxnet obs converter: + +- Generates a new observation converter (Fluxnetfull_to_obs) for eddy + covariance flux tower data (carbon, water energy fluxes) +- Documentation changes made to the older, deprecated ameriflux + converter (level4_to_obs) and the broken links have been fixed +- New flux tower observation types added to accomodate the forward + operator approach for time aggregated fluxes (daily through monthly) + +obs_def_rttov13_mod.f90 bug-fixes: + +- Added public get_channel to obs_def_rttov13_mod.f90 to compile WRF + successfully with rttov13. +- Removed cloud_overlap (integer) from the function: get_rttov_option_logical + **August 21 2023 :: CAM-FV shell scripts. Tag v10.8.3** Performance improvements for CAM-FV shell scripts: diff --git a/conf.py b/conf.py index 90264588af..7e5f834bae 100644 --- a/conf.py +++ b/conf.py @@ -21,7 +21,7 @@ author = 'Data Assimilation Research Section' # The full version, including alpha/beta/rc tags -release = '10.8.3' +release = '10.8.4' root_doc = 'index' # -- General configuration ---------------------------------------------------