diff --git a/Project.toml b/Project.toml index 0d6ea3a9..c25b0117 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ClimateBase" uuid = "35604d93-0fb8-4872-9436-495b01d137e2" authors = ["Datseris ", "Philippe Roy "] -version = "0.16.1" +version = "0.16.2" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" @@ -17,7 +17,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" [compat] DimensionalData = "0.20.1" Interpolations = "0.13.2" -NCDatasets = "0.11" +NCDatasets = "0.11, 0.12" Requires = "1" SignalDecomposition = "1" StaticArrays = "0.12, 1.0" diff --git a/docs/src/statistics.md b/docs/src/statistics.md index e9bdcb69..22451cfb 100644 --- a/docs/src/statistics.md +++ b/docs/src/statistics.md @@ -8,7 +8,7 @@ timeagg monthlyagg yearlyagg seasonalyagg -temporalrange +temporalranges maxyearspan temporal_sampling realtime_days diff --git a/src/exports.jl b/src/exports.jl index 4d92bc36..cfd1a70c 100644 --- a/src/exports.jl +++ b/src/exports.jl @@ -6,7 +6,7 @@ # Temporal export monthday_indices, maxyearspan, daymonth, realtime_days, realtime_milliseconds, -temporal_sampling, timemean, timeagg, monthlyagg, yearlyagg, temporalrange, seasonalyagg, +temporal_sampling, timemean, timeagg, monthlyagg, yearlyagg, temporalranges, seasonalyagg, season, DAYS_IN_ORBIT, HOURS_IN_ORBIT, seasonality, sametimespan # Spatial diff --git a/src/io/netcdf_write.jl b/src/io/netcdf_write.jl index 94607a60..2395df03 100644 --- a/src/io/netcdf_write.jl +++ b/src/io/netcdf_write.jl @@ -66,8 +66,8 @@ function add_dims_to_ncfile!(ds::NCDatasets.AbstractDataset, dimensions::Tuple) for (d, dname) ∈ zip(dimensions, dnames) dname ∈ dims_in_ds && continue println("writing dimension $dname...") - v = gnv(d); l = length(v) - NCDatasets.defDim(ds, dname, l) # add dimension entry + v = gnv(d) + NCDatasets.defDim(ds, dname, length(v)) # add dimension entry if d isa Coord # Define clon/clat variables with this dimension lons = getindex.(v, 1); lats = getindex.(v, 2) @@ -75,13 +75,18 @@ function add_dims_to_ncfile!(ds::NCDatasets.AbstractDataset, dimensions::Tuple) NCDatasets.defVar(ds, "clat", lats, (dname, ); attrib = DEFAULT_ATTRIBS["lat"]) else # this conversion to DateTime is necessary because CFTime.jl doesn't support Date - eltype(v) == Date && (v = DateTime.(v)) + if eltype(v) == Date; v = DateTime.(v); end attrib = DimensionalData.metadata(d) if (isnothing(attrib) || attrib == DimensionalData.NoMetadata()) && haskey(DEFAULT_ATTRIBS, dname) @warn "Dimension $dname has no attributes, adding default attributes." attrib = DEFAULT_ATTRIBS[dname] end + # Notice that if we have a subtype of CFTime, then + # we need different attributes + if eltype(v) <: NCDatasets.CFTime.AbstractCFDateTime + attrib = Dict("standard_name" => "time") + end # write dimension values as a variable as well (mandatory) NCDatasets.defVar(ds, dname, v, (dname, ); attrib = attrib) end diff --git a/src/io/vector2range.jl b/src/io/vector2range.jl index 9ba60992..1f9c1999 100644 --- a/src/io/vector2range.jl +++ b/src/io/vector2range.jl @@ -11,13 +11,15 @@ function vector2range(x::AbstractVector{<:Real}) return x[1]:dx:x[end] end -function vector2range(t::AbstractVector{<:Dates.AbstractTime}) +function vector2range(t::AbstractVector{<:Z}) where {Z<:Dates.AbstractTime} tsamp = temporal_sampling(t) period = tsamp2period(tsamp) isnothing(period) && return t - t1 = tsamp == :hourly ? t[1] : Date(t[1]) - tf = tsamp == :hourly ? t[end] : Date(t[end]) - r = t1:period:tf + special_format = Z <: NCDatasets.CFTime.AbstractCFDateTime + use_base_date = (tsamp == :hourly || special_format) + ti = use_base_date ? t[1] : Date(t[1]) + tf = use_base_date ? t[end] : Date(t[end]) + r = ti:period:tf return r == t ? r : t # final safety check to ensure equal values end diff --git a/src/physical_dimensions/temporal.jl b/src/physical_dimensions/temporal.jl index e8c34393..b2a70b95 100644 --- a/src/physical_dimensions/temporal.jl +++ b/src/physical_dimensions/temporal.jl @@ -10,11 +10,18 @@ using Dates const DAYS_IN_ORBIT = 365.26 const HOURS_IN_ORBIT = 365.26*24 +""" + no_hour_datetype(d::TimeType) → D +Return a type `D` that contains no hour (or less) information, if possible. +""" +no_time_datetime(::DateTime) = Date +no_time_datetime(::T) where {T<:TimeType} = T + "daymonth(t) = day(t), month(t)" daymonth(t) = day(t), month(t) maxyearspan(A::AbstractDimArray, tsamp = temporal_sampling(A)) = -maxyearspan(dims(A, Time).val, tsamp) +maxyearspan(gnv(dims(A, Time)), tsamp) """ temporal_sampling(x) → symbol @@ -28,7 +35,7 @@ Possible return values are: - `:yearly`, where all dates have the same month and day, but different year. - `:other`, which means that `x` doesn't fall to any of the above categories. """ -temporal_sampling(A::AbstractDimArray) = temporal_sampling(dims(A, Time).val) +temporal_sampling(A::AbstractDimArray) = temporal_sampling(gnv(dims(A, Time))) temporal_sampling(t::Dimension) = temporal_sampling(t.val) function temporal_sampling(t::AbstractVector{<:TimeType}) @@ -122,12 +129,12 @@ end """ monthday_indices(times, date = times[1]) -Find the indices in `times` (which is a `Vector{Date}`) at which +Find the indices in `times` at which the date in `times` gives the same day and month as `date`. """ function monthday_indices(times, date = times[1]) d1, m1 = daymonth(date) - a = findall(i -> daymonth(times[i]) == (d1, m1), 1:length(times)) + findall(i -> daymonth(times[i]) == (d1, m1), 1:length(times)) end """ @@ -150,7 +157,7 @@ function monthspan(t::TimeType) n = mod1(m+1, 12) y = year(t) u = m == 12 ? y+1 : y - d = collect(Date(y, m, 1):Day(1):Date(u, n, 1))[1:end-1] + collect(Date(y, m, 1):Day(1):Date(u, n, 1))[1:end-1] end """ @@ -289,6 +296,9 @@ function timeagg(f, A::AbDimArray, w = nothing) if tsamp == :other return dropagg(f, A, Time, w) end + # The reason to have three versions of code here is because, + # quanti unfortunately, each version needs different weighting + # and collection up to full year. Sad sad life. r = if tsamp == :daily timeagg_daily(f, A, w) elseif tsamp == :monthly @@ -311,7 +321,7 @@ function timeagg_yearly(f, A, w) end function timeagg_monthly(f, A::AbDimArray, w) - t = dims(A, Time).val + t = gnv(dims(A, Time)) mys = maxyearspan(t, :monthly) tw = daysinmonth.(t) W = if isnothing(w) @@ -333,7 +343,7 @@ function timeagg_monthly(f, A::AbDimArray, w) end function timeagg_daily(f, A::AbDimArray, w) - t = dims(A, Time).val + t = gnv(dims(A, Time)) mys = maxyearspan(t) _A = view(A, Time(1:mys)) if w isa AbDimArray @@ -380,11 +390,12 @@ using the function `f`. The dates of the new array always have day number of `mday`. """ function monthlyagg(A::ClimArray, f = mean; mday = 15) - t0 = dims(A, Time).val - startdate = Date(year(t0[1]), month(t0[1]), mday) - finaldate = Date(year(t0[end]), month(t0[end]), mday+1) + t0 = gnv(dims(A, Time)) + DT = no_time_datetime(t0[1]) + startdate = DT(year(t0[1]), month(t0[1]), mday) + finaldate = DT(year(t0[end]), month(t0[end]), mday+1) t = startdate:Month(1):finaldate - tranges = temporalrange(t0, Dates.month) + tranges = temporalranges(t0, Dates.month) return timegroup(A, f, t, tranges) end @@ -395,11 +406,11 @@ using the function `f`. By convention, the dates of the new array always have month and day number of `1`. """ function yearlyagg(A::ClimArray, f = mean) - t0 = dims(A, Time).val + t0 = gnv(dims(A, Time)) startdate = Date(year(t0[1]), 1, 1) finaldate = Date(year(t0[end]), 2, 1) t = startdate:Year(1):finaldate - tranges = temporalrange(t0, Dates.year) + tranges = temporalranges(t0, Dates.year) return timegroup(A, f, t, tranges) end @@ -408,14 +419,14 @@ function timegroup(A, f, t, tranges) B = ClimArray(zeros(eltype(A), length.(other)..., length(t)), (other..., Time(t)); name = A.name) for i in 1:length(tranges) - B[Time(i)] .= dropagg(f, view(A, Time(tranges[i])), Time) + B[Time(i)] = dropagg(f, view(A, Time(tranges[i])), Time) end return B end """ - temporalrange(A::ClimArray, f = Dates.month) → r - temporalrange(t::AbstractVector{<:TimeType}}, f = Dates.month) → r + temporalranges(A::ClimArray, f = Dates.month) → r + temporalranges(t::AbstractVector{<:TimeType}}, f = Dates.month) → r Return a vector of ranges so that each range of indices are values of `t` that belong in either the same month, year, day, or season, depending on `f`. `f` can take the values: `Dates.year, Dates.month, Dates.day` or `season` @@ -423,7 +434,7 @@ belong in either the same month, year, day, or season, depending on `f`. Used in e.g. [`monthlyagg`](@ref), [`yearlyagg`](@ref) or [`seasonalyagg`](@ref). """ -function temporalrange(t::AbstractArray{<:TimeType}, f = Dates.month) +function temporalranges(t::AbstractArray{<:TimeType}, f = Dates.month) @assert issorted(t) "Sorted time required." L = length(t) r = Vector{UnitRange{Int}}() @@ -437,7 +448,7 @@ function temporalrange(t::AbstractArray{<:TimeType}, f = Dates.month) push!(r, i:L) # final range not included in for loop return r end -temporalrange(A::AbstractDimArray, f = Dates.month) = temporalrange(dims(A, Time).val, f) +temporalranges(A::AbstractDimArray, f = Dates.month) = temporalranges(gnv(dims(A, Time)), f) """ @@ -448,11 +459,11 @@ By convention, seasons are represented as Dates spaced 3-months apart, where onl months December, March, June and September are used to specify the date, with day 1. """ function seasonalyagg(A::ClimArray, f = mean) - t0 = dims(A, Time).val + t0 = gnv(dims(A, Time)) startdate = to_seasonal_date(t0[1]) finaldate = to_seasonal_date(t0[end]) t = startdate:Month(3):finaldate - tranges = temporalrange(t0, season) + tranges = temporalranges(t0, season) return timegroup(A, f, t, tranges) end @@ -521,5 +532,5 @@ end function seasonality(A::ClimArray; kwargs...) @assert length(dims(A)) == 1 - return seasonality(dims(A, Time).val, A.data; kwargs...) + return seasonality(gnv(dims(A, Time)), A.data; kwargs...) end diff --git a/test/io_tests.jl b/test/io_tests.jl index 1d72bb6b..b54965de 100644 --- a/test/io_tests.jl +++ b/test/io_tests.jl @@ -66,4 +66,45 @@ end rm("missing_test.nc") end +@testset "CFTime dates" begin + using NCDatasets.CFTime: DateTime360Day + cfdates = collect(DateTime360Day(1900,01,01):Day(1):DateTime360Day(1919,12,30)) + x = float.(month.(cfdates)) + X = ClimArray(x, (Tim(cfdates),); name = "x") + + @testset "temporal stats" begin + @test temporal_sampling(cfdates) == :daily + @test monthday_indices(cfdates) == 1:360:length(cfdates) + trange = temporalranges(cfdates) + for i in 1:length(trange) + @test trange[i] == (1 + (i-1)*30):(i*30) + end + + + Y = monthlyagg(X) + @test length(Y) == 20*12 + ty = gnv(dims(Y, Tim)) + @test temporal_sampling(ty) == :monthly + @test step(ty) == Month(1) + for (i, y) in enumerate(Y) + @test y == mod1(i, 12) + end + Z = yearlyagg(X) + @test length(Z) == 20 + # The mean of 1 to 12 is by definition 6.5 + @test all(isequal(6.5), Z) + tz = gnv(dims(Z, Tim)) + @test temporal_sampling(tz) == :yearly + @test step(tz) == Year(1) + end + @testset "Writing/Reading CFTime" begin + ncwrite("cftime_test.nc", X) + @test isfile("cftime_test.nc") + X2 = ncread("cftime_test.nc", "x") + @test eltype(dims(X2, Tim)) == DateTime360Day + @test gnv(dims(X2, Tim)) == cfdates + end + +end + end # NetCDF tests \ No newline at end of file diff --git a/test/temporal_tests.jl b/test/temporal_tests.jl index ae8babfd..ab62b87a 100644 --- a/test/temporal_tests.jl +++ b/test/temporal_tests.jl @@ -26,8 +26,8 @@ end thourly = DateTime(2000, 3, 1):Hour(1):DateTime(2001, 4, 15) mdates = unique!([(year(d), month(d)) for d in tdaily]) ydates = unique!([year(d) for d in tdaily]) - tranges = temporalrange(tdaily, Dates.month) - yranges = temporalrange(tdaily, Dates.year) + tranges = temporalranges(tdaily, Dates.month) + yranges = temporalranges(tdaily, Dates.year) @testset "time sampling" begin @test length(tranges) == length(mdates) @test length(yranges) == length(ydates)