fix vararg tuples

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.41
+ - fix ntuple type with typevar length
+ - fix OpaqueData test (HDF5 compat)
+
 ## 0.4.40
  - fix unitialized custom-serialized objects
  - allow serializing pkg-modules by name

Project.toml

@@ -1,6 +1,6 @@
 name = "JLD2"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.40"
+version = "0.4.41"
 
 [deps]
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"

src/data/specialcased_types.jl

@@ -3,8 +3,8 @@ struct OpaqueData{N}
     data::Vector{UInt8}
     OpaqueData(data) = new{length(data)}(data)
 end
-odr_sizeof(x::Type{OpaqueData{N}}) where {N} = UInt32(N)
-function jlconvert(rr::ReadRepresentation{OpaqueData{N}, Vector{UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N
+
+function jlconvert(rr::ReadRepresentation{OpaqueData{N}, NTuple{N,UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N
     data = Vector{UInt8}(undef, N)
     unsafe_copyto!(pointer(data), convert(Ptr{UInt8}, ptr), N)
     OpaqueData(data)
@@ -62,7 +62,7 @@ function jltype(f::JLDFile, dt::BasicDatatype)
                 throw(UnsupportedFeatureException("Encountered an unsupported string type. $dt"))
             end
         elseif dt.class << 4 == DT_OPAQUE  << 4
-            return ReadRepresentation{OpaqueData{Int(dt.size)},Vector{UInt8}}()
+            return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}()
 
         else
             throw(UnsupportedFeatureException("Encountered an unsupported type."))
@@ -77,7 +77,7 @@ function jltype(f::JLDFile, dt::BasicDatatype)
             throw(UnsupportedFeatureException("Encountered an unsupported string type."))
         end
     elseif dt.class << 4 == DT_OPAQUE  << 4
-        return ReadRepresentation{OpaqueData{Int(dt.size)},Vector{UInt8}}()
+        return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}()
     elseif dt.class << 4 == DT_REFERENCE  << 4
         return ReadRepresentation{Any,RelOffset}()
     else

src/data/writing_datatypes.jl

@@ -428,21 +428,28 @@ end
 
 
 # This is a trick to compactly write long NTuple
-# This uses that NTuple{N,T} == Tuple{T,T,T,T,...,T}
-function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{NTuple{N,ET}}, wsession::JLDWriteSession) where {N, ET}
-    if isempty(T.parameters)
+# This uses that NTuple{N,T} === Tuple{T,T,T,T,...,T}
+function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{<: NTuple}, wsession::JLDWriteSession)
+    params = T.parameters
+    N = length(params)
+    if N ≤ 1
         store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "Tuple"), f.datatype_wsession)
-        h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8})
-    else
+        if N == 0
+            h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8})
+        else # N==1
+            # this also catches NTuples with indeterminate length
+            refs = refs_from_types(f, params, wsession)
+            store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession)
+        end     
+    else # actual NTuple with more than one entry
         store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "NTuple"), f.datatype_wsession)
+        ET = params[1] # T === Tuple{ET,ET,ET,...}
         refs = refs_from_types(f, Any[N,ET], wsession)
         store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession)
     end
     nothing
 end
 
-
-
 ## Union Types
 
 const H5TYPE_UNION = CompoundDatatype(

test/loadsave.jl

@@ -701,5 +701,17 @@ end
         o = load_object("test.jld2")
         @test !any(isassigned.(Ref(o), eachindex(o)))
     end
-
 end
+
+@testset "Issue 486 store NTuple type with indeterminate length" begin
+    cd(mktempdir()) do
+        s_type = Tuple{Int, Tuple{Vararg{Int, T}} where T}
+        a = Dict{s_type, Int}()
+        a[(0, (1, 2, 3))] = 4
+        if VERSION < v"1.7.0-A"
+            @test_broken a == (save_object("test.jld2", a); load_object("test.jld2"))
+        else
+            @test a == (save_object("test.jld2", a); load_object("test.jld2"))
+        end
+    end
+end 

test/test_files.jl

@@ -6,120 +6,120 @@ testfiles = artifact"testfiles/JLD2TestFiles-0.1.0/artifacts"
 
 @testset "HDF5 compat test files" begin
     # These are test files copied from the HDF5.jl test suite
-    cd(testfiles) do
-
-        fn = "compound.h5"
-        jldopen(fn) do f 
-            data = f["data"]
-            @test data[1] == data[2]
-            nt = data[1]
-            @test nt.wgt == 1.0
-            @test nt.xyz == [-2.4559041161056125, 0.43236207188504794, -0.5088338908493437]
-            @test nt.uvw == [-0.44966656055677057, 0.6453930541533174, 0.6174688574881305]
-            @test nt.E == 1.1915731810042547
-        end
-
-        # Should return some enum type and load names correctly
-        fn = "h5ex_t_enum.h5"
-        jldopen(fn) do f
-            @test size(f["DS1"]) == (7,4)
-        end 
-
-        fn = "h5ex_t_array.h5"
-        jldopen(fn) do f
-            @test f["DS1"][1] == (0:-1:-4) .* [0,1,2]'
-            @test f["DS1"][2] == hcat(collect(0:4), ones(Int,5), collect(2:-1:-2))
-        end
-
-        fn = "h5ex_t_float.h5"
-        jldopen(fn) do f
-            @test size(f["DS1"]) == (7,4)
-            @test f["DS1"][9] ≈ 5/3
-        end
-
-        # Big Endian Integers are not implemented
-        fn = "h5ex_t_int.h5"
-        jldopen(fn) do f
-            @test f["DS1"] == [0:-1:-6 zeros(Int,7) 0:6 0:2:12]
-        end
-
-        fn = "h5ex_t_objref.h5"
-        jldopen(fn) do f
-            @test f["DS1"][1] === f["G1"]
-            @test f["DS1"][2] === f["DS2"]
-        end
-
-        fn = "h5ex_t_opaque.h5"
-        jldopen(fn) do f
-            @test f["DS1"][4].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x30]
-        end
-
-        fn = "h5ex_t_string.h5"
-        jldopen(fn) do f
-            @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."]
-        end
-
-        fn = "h5ex_t_vlen.h5"
-        jldopen(fn) do f
-            @test f["DS1"][1] == [3, 2, 1]
-            @test f["DS1"][2] == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]
-        end
-
-        fn = "h5ex_t_vlstring.h5"
-        jldopen(fn) do f
-            @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."]
-        end
-
-        fn = "nullterm_ascii.h5"
-        jldopen(fn) do f
-            @test f["test"] == "Hello World"
-        end
-
-        fn = "large_fractal_heap.h5"
-        jldopen(fn) do f
-            @test length(keys(f)) == 200000
-        end
-
-        fn = "netcdf.nc"
-        jldopen(fn) do f
-            @test f["hello"] == ones(5)
-            #@test_broken f["x"]
-            #@test_broken f["z"]
-            @test f["grouped/data"] == 0:9
-            #@test_broken f["grouped/y"]
-        end
-
-        fn = "simple.nc"
-        jldopen(fn) do f
-            @test f["dim1"] == [2, 4, 6]
-            @test f["dim2"] == ["a", "b", "c", "d"]
-            @test f["mydata"] == Matrix(reshape(1:12, 4, 3))
-            JLD2.load_attributes(f, "dim1") # not sure what to test for. just not erroring so far
-            JLD2.load_attributes(f, "dim2")
-            JLD2.load_attributes(f, "mydata")
-        end
-
-        # julia> using JLD
-        # julia> struct A; x::Int; y::Float64; z::String; end
-        # julia> save("jldstruct.jld", "a", A(1,2.0,"3"))
-        fn = "jldstruct.jld"
-        jldopen(fn) do f
-            a = f["a"]
-            @test a.x == 1
-            @test a.y == 2.0
-            @test a.z == "3"
-        end
-
-        fn = "chunking1.h5"
-        jldopen(fn) do f
-            @test f["uncompressed_chunks"] == reshape(1:1000., 25, 40)
-            @test f["compressed_chunks"] == reshape(1:1000., 25, 40)
-            @test f["shuffle_compressed_chunks"] == reshape(1:1000, 25, 40)
-            @test size(f["incomplete_allocation"]) == (50,50,10)
-            @test f["incomplete_allocation"][1:50,1:50, 2] == reshape(1:2500, 50,50)
-            #f["incomplete_allocation"][1,1,1] == 0 
-        end
+
+    fn = joinpath(testfiles,"compound.h5")
+    jldopen(fn) do f 
+        data = f["data"]
+        @test data[1] == data[2]
+        nt = data[1]
+        @test nt.wgt == 1.0
+        @test nt.xyz == [-2.4559041161056125, 0.43236207188504794, -0.5088338908493437]
+        @test nt.uvw == [-0.44966656055677057, 0.6453930541533174, 0.6174688574881305]
+        @test nt.E == 1.1915731810042547
+    end
+
+    # Should return some enum type and load names correctly
+    fn = joinpath(testfiles,"h5ex_t_enum.h5")
+    jldopen(fn) do f
+        @test size(f["DS1"]) == (7,4)
+    end 
+
+    fn = joinpath(testfiles,"h5ex_t_array.h5")
+    jldopen(fn) do f
+        @test f["DS1"][1] == (0:-1:-4) .* [0,1,2]'
+        @test f["DS1"][2] == hcat(collect(0:4), ones(Int,5), collect(2:-1:-2))
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_float.h5")
+    jldopen(fn) do f
+        @test size(f["DS1"]) == (7,4)
+        @test f["DS1"][9] ≈ 5/3
+    end
+
+    # Big Endian Integers are not implemented
+    fn = joinpath(testfiles,"h5ex_t_int.h5")
+    jldopen(fn) do f
+        @test f["DS1"] == [0:-1:-6 zeros(Int,7) 0:6 0:2:12]
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_objref.h5")
+    jldopen(fn) do f
+        @test f["DS1"][1] === f["G1"]
+        @test f["DS1"][2] === f["DS2"]
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_opaque.h5")
+    jldopen(fn) do f
+        @test f["DS1"][1].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x30]
+        @test f["DS1"][2].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x31]
+        @test f["DS1"][3].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x32]
+        @test f["DS1"][4].data == [0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x33]
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_string.h5")
+    jldopen(fn) do f
+        @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."]
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_vlen.h5")
+    jldopen(fn) do f
+        @test f["DS1"][1] == [3, 2, 1]
+        @test f["DS1"][2] == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]
+    end
+
+    fn = joinpath(testfiles,"h5ex_t_vlstring.h5")
+    jldopen(fn) do f
+        @test f["DS1"] == ["Parting", "is such", "sweet", "sorrow."]
+    end
+
+    fn = joinpath(testfiles,"nullterm_ascii.h5")
+    jldopen(fn) do f
+        @test f["test"] == "Hello World"
+    end
+
+    fn = joinpath(testfiles,"large_fractal_heap.h5")
+    jldopen(fn) do f
+        @test length(keys(f)) == 200000
+    end
+
+    fn = joinpath(testfiles,"netcdf.nc")
+    jldopen(fn) do f
+        @test f["hello"] == ones(5)
+        #@test_broken f["x"]
+        #@test_broken f["z"]
+        @test f["grouped/data"] == 0:9
+        #@test_broken f["grouped/y"]
+    end
+
+    fn = joinpath(testfiles,"simple.nc")
+    jldopen(fn) do f
+        @test f["dim1"] == [2, 4, 6]
+        @test f["dim2"] == ["a", "b", "c", "d"]
+        @test f["mydata"] == Matrix(reshape(1:12, 4, 3))
+        JLD2.load_attributes(f, "dim1") # not sure what to test for. just not erroring so far
+        JLD2.load_attributes(f, "dim2")
+        JLD2.load_attributes(f, "mydata")
+    end
+
+    # julia> using JLD
+    # julia> struct A; x::Int; y::Float64; z::String; end
+    # julia> save("jldstruct.jld", "a", A(1,2.0,"3"))
+    fn = joinpath(testfiles,"jldstruct.jld")
+    jldopen(fn) do f
+        a = f["a"]
+        @test a.x == 1
+        @test a.y == 2.0
+        @test a.z == "3"
+    end
 
+    fn = joinpath(testfiles,"chunking1.h5")
+    jldopen(fn) do f
+        @test f["uncompressed_chunks"] == reshape(1:1000., 25, 40)
+        @test f["compressed_chunks"] == reshape(1:1000., 25, 40)
+        @test f["shuffle_compressed_chunks"] == reshape(1:1000, 25, 40)
+        @test size(f["incomplete_allocation"]) == (50,50,10)
+        @test f["incomplete_allocation"][1:50,1:50, 2] == reshape(1:2500, 50,50)
+        #f["incomplete_allocation"][1,1,1] == 0 
     end
 end