Add copyto! and similar for Datasets

src/HDF5.jl

@@ -934,6 +934,8 @@ end
 # This infers the Julia type from the HDF5.Datatype. Specific file formats should provide their own read(dset).
 const DatasetOrAttribute = Union{Dataset,Attribute}
 
+include("views.jl")
+
 function Base.read(obj::DatasetOrAttribute)
     dtype = datatype(obj)
     T = get_jl_type(dtype)
@@ -957,6 +959,19 @@ function Base.read(obj::DatasetOrAttribute, ::Type{T}, I...) where T
     return val
 end
 
+"""
+    read!(obj::DatasetOrAttribute, output_buffer::AbstractArray{T} [, I...]) where T
+
+Read [part of] a dataset or attribute into a preallocated output buffer.
+The output buffer must be convertible to a pointer and have a contiguous layout.
+"""
+function Base.read!(obj::DatasetOrAttribute, buf::AbstractArray{T}, I...) where T
+    dtype = datatype(obj)
+    val = generic_read!(buf, obj, dtype, T, I...)
+    close(dtype)
+    return val
+end
+
 # `Type{String}` does not have a definite size, so the generic_read does not accept
 # it even though it will return a `String`. This explicit overload allows that usage.
 function Base.read(obj::DatasetOrAttribute, ::Type{String}, I...)
@@ -968,10 +983,26 @@ function Base.read(obj::DatasetOrAttribute, ::Type{String}, I...)
     return val
 end
 
+"""
+    copyto!(output_buffer::AbstractArray{T}, obj::Union{DatasetOrAttribute}) where T
+    copyto!(output_buffer::AbstractArray{T}, @view(obj::Union{DatasetOrAttribute}, I...))
+
+Copy [part of] a HDF5 dataset or attribute to a preallocated output buffer.
+The output buffer must be convertible to a pointer and have a contiguous layout.
+"""
+function Base.copyto!(output_buffer::AbstractArray{T}, obj::DatasetOrAttribute, I...) where T
+    return Base.read!(obj, output_buffer, I...)
+end
+
 # Special handling for reading OPAQUE datasets and attributes
-function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{Opaque})
+function generic_read!(buf::Matrix{UInt8}, obj::DatasetOrAttribute, filetype::Datatype, ::Type{Opaque})
+    generic_read(obj, filetype, Opaque, buf)
+end
+function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{Opaque}, buf::Union{Matrix{UInt8}, Nothing} = nothing)
     sz  = size(obj)
-    buf = Matrix{UInt8}(undef, sizeof(filetype), prod(sz))
+    if isnothing(buf)
+        buf = Matrix{UInt8}(undef, sizeof(filetype), prod(sz))
+    end
     if obj isa Dataset
         read_dataset(obj, filetype, buf, obj.xfer)
     else
@@ -989,7 +1020,14 @@ function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{Opaque
 end
 
 # generic read function
+function generic_read!(buf::Union{AbstractMatrix{UInt8}, AbstractArray{T}}, obj::DatasetOrAttribute, filetype::Datatype, ::Type{T}, I...) where T
+    return _generic_read(obj, filetype, T, buf, I...)
+end
 function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{T}, I...) where T
+    return _generic_read(obj, filetype, T, nothing, I...)
+end
+function _generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{T},
+    buf::Union{AbstractMatrix{UInt8}, AbstractArray{T}, Nothing}, I...) where T
     !isconcretetype(T) && error("type $T is not concrete")
     !isempty(I) && obj isa Attribute && error("HDF5 attributes do not support hyperslab selections")
 
@@ -1026,13 +1064,18 @@ function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{T}, I.
         end
     end
 
-    if do_normalize(T)
-        # The entire dataset is read into in a buffer matrix where the first dimension at
-        # any stage of normalization is the bytes for a single element of type `T`, and
-        # the second dimension of the matrix runs through all elements.
-        buf = Matrix{UInt8}(undef, sizeof(T), prod(sz))
+    if isnothing(buf)
+        if do_normalize(T)
+            # The entire dataset is read into in a buffer matrix where the first dimension at
+            # any stage of normalization is the bytes for a single element of type `T`, and
+            # the second dimension of the matrix runs through all elements.
+            buf = Matrix{UInt8}(undef, sizeof(T), prod(sz))
+        else
+            buf = Array{T}(undef, sz...)
+        end
     else
-        buf = Array{T}(undef, sz...)
+        sizeof(buf) != prod(sz)*sizeof(T) &&
+            error("Provided array buffer of size, $(size(buf)), and element type, $(eltype(buf)), does not match the dataset of size, $sz, and type, $T")
     end
     memspace = isempty(I) ? dspace : dataspace(sz)
 
@@ -1062,6 +1105,89 @@ function generic_read(obj::DatasetOrAttribute, filetype::Datatype, ::Type{T}, I.
     end
 end
 
+
+"""
+    similar(obj::DatasetOrAttribute, [::Type{T}], [I::Integer...])
+
+Return a `Array{T}` or `Matrix{UInt8}` to that can contain [part of] the dataset.
+"""
+function Base.similar(obj::DatasetOrAttribute, I::Integer...)
+    dtype = datatype(obj)
+    T = get_jl_type(dtype)
+    val = similar(obj, dtype, T, I...)
+    close(dtype)
+    return val
+end
+
+function Base.similar(obj::DatasetOrAttribute, ::Type{T}, I::Integer...) where T
+    dtype = datatype(obj)
+    val = similar(obj, dtype, T, I...)
+    close(dtype)
+    return val
+end
+
+function Base.similar(obj::DatasetOrAttribute, filetype::Datatype, ::Type{Opaque})
+    sz  = size(obj)
+    return Matrix{UInt8}(undef, sizeof(filetype), prod(sz))
+end
+
+# Duplicated from generic_read. TODO: Deduplicate
+function Base.similar(obj::DatasetOrAttribute, filetype::Datatype, ::Type{T}, I::Integer...) where T
+    !isconcretetype(T) && error("type $T is not concrete")
+    !isempty(I) && obj isa Attribute && error("HDF5 attributes do not support hyperslab selections")
+
+    I = Base.OneTo.(I)
+
+    memtype = Datatype(API.h5t_get_native_type(filetype))  # padded layout in memory
+
+    if sizeof(T) != sizeof(memtype)
+        error("""
+              Type size mismatch
+              sizeof($T) = $(sizeof(T))
+              sizeof($memtype) = $(sizeof(memtype))
+              """)
+    end
+
+    dspace = dataspace(obj)
+    stype = API.h5s_get_simple_extent_type(dspace)
+    stype == API.H5S_NULL && return EmptyArray{T}()
+
+    if !isempty(I)
+        indices = Base.to_indices(obj, I)
+        dspace = hyperslab(dspace, indices...)
+    end
+
+    scalar = false
+    if stype == API.H5S_SCALAR
+        sz = (1,)
+        scalar = true
+    elseif isempty(I)
+        sz = size(dspace)
+    else
+        sz = map(length, filter(i -> !isa(i, Int), indices))
+        if isempty(sz)
+            sz = (1,)
+            scalar = true
+        end
+    end
+
+    if do_normalize(T)
+        # The entire dataset is read into in a buffer matrix where the first dimension at
+        # any stage of normalization is the bytes for a single element of type `T`, and
+        # the second dimension of the matrix runs through all elements.
+        buf = Matrix{UInt8}(undef, sizeof(T), prod(sz))
+        buf = reshape(normalize_types(T, buf), sz...)
+    else
+        buf = Array{T}(undef, sz...)
+    end
+
+
+    close(memtype)
+    close(dspace)
+
+    return buf
+end
+
 # Array constructor for datasets
 Array(x::Dataset) = read(x)
 

src/views.jl

@@ -0,0 +1,41 @@
+# Permit Base.view for Dataset and Attribute to normalize Base.copyto! syntax
+
+# Long term consideration:
+# If Dataset and Attribute become an AbstractArray, perhaps these should just be SubArrays ?
+"""
+    DatasetView
+
+This is an experimental data type and may change. Use `view` rather than `DatasetView` directly.
+"""
+struct DatasetView
+    parent::Dataset
+    indices
+end
+
+function Base.view(obj::Dataset, I...)
+    return DatasetView(obj, I)
+end
+
+Base.similar(view::DatasetView) = similar(view.parent, length.(view.indices)...)
+
+"""
+    AttributeView
+
+This is an experimental data type and may change. Use `view` rather than `AttributeView` directly.
+"""
+struct AttributeView
+    parent::Attribute
+    indices
+end
+
+function Base.view(obj::Attribute, I...)
+    return AttributeView(obj, I)
+end
+
+Base.similar(view::AttributeView) = similar(view.parent, length.(view.indices)...)
+
+const DatasetOrAttributeView = Union{DatasetView, AttributeView}
+
+function Base.copyto!(output_buffer::AbstractArray{T}, view::DatasetOrAttributeView) where T
+    return Base.read!(view.parent, output_buffer, view.indices...)
+end

test/runtests.jl

@@ -50,6 +50,8 @@ include("filter.jl")
 include("chunkstorage.jl")
 @debug "fileio"
 include("fileio.jl")
+@debug "views"
+include("views.jl")
 @debug "filter test utils"
 include("filters/FilterTestUtils.jl")
 

test/views.jl

@@ -0,0 +1,39 @@
+using HDF5
+using Test
+
+@testset "views and non-allocating methods" begin
+    fn = tempname()
+
+    @info "view.jl" fn
+    data = rand(UInt16, 16, 16)
+
+    h5open(fn, "w") do h5f
+        h5f["data"] = data
+    end
+
+    h5open(fn, "r") do h5f
+        buffer = similar(h5f["data"])
+        copyto!(buffer, h5f["data"])
+        @test isequal(buffer, data)
+
+        read!(h5f["data"], buffer)
+        @test isequal(buffer, data)
+
+        v = @view(h5f["data"][1:4, 1:4])
+
+        buffer = similar(v)
+        @test size(buffer) == (4,4)
+        copyto!(buffer, v)
+        @test isequal(buffer, @view(data[1:4, 1:4]))
+
+        buffer .= 1
+        read!(h5f["data"], buffer, 1:4, 1:4)
+        @test isequal(buffer, @view(data[1:4, 1:4]))
+
+        @test size(similar(h5f["data"], Int16)) == size(h5f["data"])
+        @test size(similar(h5f["data"], 5,6)) == (5, 6)
+        @test size(similar(h5f["data"], Int16, 8,7)) == (8,7)
+    end
+
+    rm(fn)
+end