Correct handling of CFTime time dimensions

Project.toml

@@ -1,7 +1,7 @@
 name = "ClimateBase"
 uuid = "35604d93-0fb8-4872-9436-495b01d137e2"
 authors = ["Datseris <datseris.george@gmail.com>", "Philippe Roy <borghor@yahoo.ca>"]
-version = "0.16.1"
+version = "0.16.2"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -17,7 +17,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 [compat]
 DimensionalData = "0.20.1"
 Interpolations = "0.13.2"
-NCDatasets = "0.11"
+NCDatasets = "0.11, 0.12"
 Requires = "1"
 SignalDecomposition = "1"
 StaticArrays = "0.12, 1.0"

docs/src/statistics.md

@@ -8,7 +8,7 @@ timeagg
 monthlyagg
 yearlyagg
 seasonalyagg
-temporalrange
+temporalranges
 maxyearspan
 temporal_sampling
 realtime_days

src/exports.jl

@@ -6,7 +6,7 @@
 
 # Temporal
 export monthday_indices, maxyearspan, daymonth, realtime_days, realtime_milliseconds,
-temporal_sampling, timemean, timeagg, monthlyagg, yearlyagg, temporalrange, seasonalyagg, 
+temporal_sampling, timemean, timeagg, monthlyagg, yearlyagg, temporalranges, seasonalyagg,
 season, DAYS_IN_ORBIT, HOURS_IN_ORBIT, seasonality, sametimespan
 
 # Spatial

src/io/netcdf_write.jl

@@ -66,22 +66,27 @@ function add_dims_to_ncfile!(ds::NCDatasets.AbstractDataset, dimensions::Tuple)
     for (d, dname) ∈ zip(dimensions, dnames)
         dname ∈ dims_in_ds && continue
         println("writing dimension $dname...")
-        v = gnv(d); l = length(v)
-        NCDatasets.defDim(ds, dname, l) # add dimension entry
+        v = gnv(d)
+        NCDatasets.defDim(ds, dname, length(v)) # add dimension entry
         if d isa Coord
             # Define clon/clat variables with this dimension
             lons = getindex.(v, 1); lats = getindex.(v, 2)
             NCDatasets.defVar(ds, "clon", lons, (dname, ); attrib = DEFAULT_ATTRIBS["lon"])
             NCDatasets.defVar(ds, "clat", lats, (dname, ); attrib = DEFAULT_ATTRIBS["lat"])
         else
             # this conversion to DateTime is necessary because CFTime.jl doesn't support Date
-            eltype(v) == Date && (v = DateTime.(v))
+            if eltype(v) == Date; v = DateTime.(v); end
             attrib = DimensionalData.metadata(d)
             if (isnothing(attrib) || attrib == DimensionalData.NoMetadata()) &&
                     haskey(DEFAULT_ATTRIBS, dname)
                 @warn "Dimension $dname has no attributes, adding default attributes."
                 attrib = DEFAULT_ATTRIBS[dname]
             end
+            # Notice that if we have a subtype of CFTime, then
+            # we need different attributes
+            if eltype(v) <: NCDatasets.CFTime.AbstractCFDateTime
+                attrib = Dict("standard_name" => "time")
+            end
             # write dimension values as a variable as well (mandatory)
             NCDatasets.defVar(ds, dname, v, (dname, ); attrib = attrib)
         end

src/io/vector2range.jl

@@ -11,13 +11,15 @@ function vector2range(x::AbstractVector{<:Real})
     return x[1]:dx:x[end]
 end
 
-function vector2range(t::AbstractVector{<:Dates.AbstractTime})
+function vector2range(t::AbstractVector{<:Z}) where {Z<:Dates.AbstractTime}
     tsamp = temporal_sampling(t)
     period = tsamp2period(tsamp)
     isnothing(period) && return t
-    t1 = tsamp == :hourly ? t[1] : Date(t[1])
-    tf = tsamp == :hourly ? t[end] : Date(t[end])
-    r = t1:period:tf
+    special_format = Z <: NCDatasets.CFTime.AbstractCFDateTime
+    use_base_date = (tsamp == :hourly || special_format)
+    ti = use_base_date ? t[1] : Date(t[1])
+    tf = use_base_date ? t[end] : Date(t[end])
+    r = ti:period:tf
     return r == t ? r : t # final safety check to ensure equal values
 end
 

src/physical_dimensions/temporal.jl

@@ -10,11 +10,18 @@ using Dates
 const DAYS_IN_ORBIT = 365.26
 const HOURS_IN_ORBIT = 365.26*24
 
+"""
+    no_hour_datetype(d::TimeType) → D
+Return a type `D` that contains no hour (or less) information, if possible.
+"""
+no_time_datetime(::DateTime) = Date
+no_time_datetime(::T) where {T<:TimeType} = T
+
 "daymonth(t) = day(t), month(t)"
 daymonth(t) = day(t), month(t)
 
 maxyearspan(A::AbstractDimArray, tsamp = temporal_sampling(A)) =
-maxyearspan(dims(A, Time).val, tsamp)
+maxyearspan(gnv(dims(A, Time)), tsamp)
 
 """
     temporal_sampling(x) → symbol
@@ -28,7 +35,7 @@ Possible return values are:
 - `:yearly`, where all dates have the same month and day, but different year.
 - `:other`, which means that `x` doesn't fall to any of the above categories.
 """
-temporal_sampling(A::AbstractDimArray) = temporal_sampling(dims(A, Time).val)
+temporal_sampling(A::AbstractDimArray) = temporal_sampling(gnv(dims(A, Time)))
 temporal_sampling(t::Dimension) = temporal_sampling(t.val)
 
 function temporal_sampling(t::AbstractVector{<:TimeType})
@@ -122,12 +129,12 @@ end
 
 """
     monthday_indices(times, date = times[1])
-Find the indices in `times` (which is a `Vector{Date}`) at which
+Find the indices in `times` at which
 the date in `times` gives the same day and month as `date`.
 """
 function monthday_indices(times, date = times[1])
     d1, m1 = daymonth(date)
-    a = findall(i -> daymonth(times[i]) == (d1, m1), 1:length(times))
+    findall(i -> daymonth(times[i]) == (d1, m1), 1:length(times))
 end
 
 """
@@ -150,7 +157,7 @@ function monthspan(t::TimeType)
     n = mod1(m+1, 12)
     y = year(t)
     u = m == 12 ? y+1 : y
-    d = collect(Date(y, m, 1):Day(1):Date(u, n, 1))[1:end-1]
+    collect(Date(y, m, 1):Day(1):Date(u, n, 1))[1:end-1]
 end
 
 """
@@ -289,6 +296,9 @@ function timeagg(f, A::AbDimArray, w = nothing)
     if tsamp == :other
         return dropagg(f, A, Time, w)
     end
+    # The reason to have three versions of code here is because,
+    # quanti unfortunately, each version needs different weighting
+    # and collection up to full year. Sad sad life.
     r = if tsamp == :daily
         timeagg_daily(f, A, w)
     elseif tsamp == :monthly
@@ -311,7 +321,7 @@ function timeagg_yearly(f, A, w)
 end
 
 function timeagg_monthly(f, A::AbDimArray, w)
-    t = dims(A, Time).val
+    t = gnv(dims(A, Time))
     mys = maxyearspan(t, :monthly)
     tw = daysinmonth.(t)
     W = if isnothing(w)
@@ -333,7 +343,7 @@ function timeagg_monthly(f, A::AbDimArray, w)
 end
 
 function timeagg_daily(f, A::AbDimArray, w)
-    t = dims(A, Time).val
+    t = gnv(dims(A, Time))
     mys = maxyearspan(t)
     _A = view(A, Time(1:mys))
     if w isa AbDimArray
@@ -380,11 +390,12 @@ using the function `f`.
 The dates of the new array always have day number of `mday`.
 """
 function monthlyagg(A::ClimArray, f = mean; mday = 15)
-    t0 = dims(A, Time).val
-    startdate = Date(year(t0[1]), month(t0[1]), mday)
-    finaldate = Date(year(t0[end]), month(t0[end]), mday+1)
+    t0 = gnv(dims(A, Time))
+    DT = no_time_datetime(t0[1])
+    startdate = DT(year(t0[1]), month(t0[1]), mday)
+    finaldate = DT(year(t0[end]), month(t0[end]), mday+1)
     t = startdate:Month(1):finaldate
-    tranges = temporalrange(t0, Dates.month)
+    tranges = temporalranges(t0, Dates.month)
     return timegroup(A, f, t, tranges)
 end
 
@@ -395,11 +406,11 @@ using the function `f`.
 By convention, the dates of the new array always have month and day number of `1`.
 """
 function yearlyagg(A::ClimArray, f = mean)
-    t0 = dims(A, Time).val
+    t0 = gnv(dims(A, Time))
     startdate = Date(year(t0[1]), 1, 1)
     finaldate = Date(year(t0[end]), 2, 1)
     t = startdate:Year(1):finaldate
-    tranges = temporalrange(t0, Dates.year)
+    tranges = temporalranges(t0, Dates.year)
     return timegroup(A, f, t, tranges)
 end
 
@@ -408,22 +419,22 @@ function timegroup(A, f, t, tranges)
     B = ClimArray(zeros(eltype(A), length.(other)..., length(t)),
         (other..., Time(t)); name = A.name)
     for i in 1:length(tranges)
-        B[Time(i)] .= dropagg(f, view(A, Time(tranges[i])), Time)
+        B[Time(i)] = dropagg(f, view(A, Time(tranges[i])), Time)
     end
     return B
 end
 
 """
-    temporalrange(A::ClimArray, f = Dates.month) → r
-    temporalrange(t::AbstractVector{<:TimeType}}, f = Dates.month) → r
+    temporalranges(A::ClimArray, f = Dates.month) → r
+    temporalranges(t::AbstractVector{<:TimeType}}, f = Dates.month) → r
 Return a vector of ranges so that each range of indices are values of `t` that
 belong in either the same month, year, day, or season, depending on `f`.
 `f` can take the values: `Dates.year, Dates.month, Dates.day` or `season`
 (all are functions).
 
 Used in e.g. [`monthlyagg`](@ref), [`yearlyagg`](@ref) or [`seasonalyagg`](@ref).
 """
-function temporalrange(t::AbstractArray{<:TimeType}, f = Dates.month)
+function temporalranges(t::AbstractArray{<:TimeType}, f = Dates.month)
     @assert issorted(t) "Sorted time required."
     L = length(t)
     r = Vector{UnitRange{Int}}()
@@ -437,7 +448,7 @@ function temporalrange(t::AbstractArray{<:TimeType}, f = Dates.month)
     push!(r, i:L) # final range not included in for loop
     return r
 end
-temporalrange(A::AbstractDimArray, f = Dates.month) = temporalrange(dims(A, Time).val, f)
+temporalranges(A::AbstractDimArray, f = Dates.month) = temporalranges(gnv(dims(A, Time)), f)
 
 
 """
@@ -448,11 +459,11 @@ By convention, seasons are represented as Dates spaced 3-months apart, where onl
 months December, March, June and September are used to specify the date, with day 1.
 """
 function seasonalyagg(A::ClimArray, f = mean)
-    t0 = dims(A, Time).val
+    t0 = gnv(dims(A, Time))
     startdate = to_seasonal_date(t0[1])
     finaldate = to_seasonal_date(t0[end])
     t = startdate:Month(3):finaldate
-    tranges = temporalrange(t0, season)
+    tranges = temporalranges(t0, season)
     return timegroup(A, f, t, tranges)
 end
 
@@ -521,5 +532,5 @@ end
 
 function seasonality(A::ClimArray; kwargs...)
     @assert length(dims(A)) == 1
-    return seasonality(dims(A, Time).val, A.data; kwargs...)
+    return seasonality(gnv(dims(A, Time)), A.data; kwargs...)
 end

test/io_tests.jl

@@ -66,4 +66,45 @@ end
     rm("missing_test.nc")
 end
 
+@testset "CFTime dates" begin
+    using NCDatasets.CFTime: DateTime360Day
+    cfdates = collect(DateTime360Day(1900,01,01):Day(1):DateTime360Day(1919,12,30))
+    x = float.(month.(cfdates))
+    X = ClimArray(x, (Tim(cfdates),); name = "x")
+
+    @testset "temporal stats" begin
+        @test temporal_sampling(cfdates) == :daily
+        @test monthday_indices(cfdates) == 1:360:length(cfdates)
+        trange = temporalranges(cfdates)
+        for i in 1:length(trange)
+            @test trange[i] == (1 + (i-1)*30):(i*30)
+        end
+
+
+        Y = monthlyagg(X)
+        @test length(Y) == 20*12
+        ty = gnv(dims(Y, Tim))
+        @test temporal_sampling(ty) == :monthly
+        @test step(ty) == Month(1)
+        for (i, y) in enumerate(Y)
+            @test y == mod1(i, 12)
+        end
+        Z = yearlyagg(X)
+        @test length(Z) == 20
+        # The mean of 1 to 12 is by definition 6.5
+        @test all(isequal(6.5), Z)
+        tz = gnv(dims(Z, Tim))
+        @test temporal_sampling(tz) == :yearly
+        @test step(tz) == Year(1)
+    end
+    @testset "Writing/Reading CFTime" begin
+        ncwrite("cftime_test.nc", X)
+        @test isfile("cftime_test.nc")
+        X2 = ncread("cftime_test.nc", "x")
+        @test eltype(dims(X2, Tim)) == DateTime360Day
+        @test gnv(dims(X2, Tim)) == cfdates
+    end
+
+end
+
 end # NetCDF tests

test/temporal_tests.jl

@@ -26,8 +26,8 @@ end
     thourly = DateTime(2000, 3, 1):Hour(1):DateTime(2001, 4, 15)
     mdates = unique!([(year(d), month(d)) for d in tdaily])
     ydates = unique!([year(d) for d in tdaily])
-    tranges = temporalrange(tdaily, Dates.month)
-    yranges = temporalrange(tdaily, Dates.year)
+    tranges = temporalranges(tdaily, Dates.month)
+    yranges = temporalranges(tdaily, Dates.year)
     @testset "time sampling" begin
         @test length(tranges) == length(mdates)
         @test length(yranges) == length(ydates)