From ee0d3e10938b7323ac5c3e81e96ed3645c319582 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 11 Dec 2023 10:22:08 +0100 Subject: [PATCH 01/52] feat: add a method to convert `PyIterable` --- ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl index fed937f..1bfd1ec 100644 --- a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl +++ b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl @@ -32,4 +32,20 @@ function AwkwardArray.convert(array::Py)::AwkwardArray.Content ) end +function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content + form, len, _containers = pyimport("awkward").to_buffers(array) + containers = pyconvert(Dict, _containers) + + julia_buffers = Dict{String,AbstractVector{UInt8}}() + for (key, buffer) in containers + julia_buffers[key] = reinterpret(UInt8, buffer) + end + + AwkwardArray.from_buffers( + pyconvert(String, form.to_json()), + pyconvert(Int, len), + julia_buffers, + ) +end + end # module From e3c09a5803a3121d2d5792db04ee93467f35a9ff Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 12 Dec 2023 15:44:34 +0100 Subject: [PATCH 02/52] feat: add auto conversion rules --- .../AwkwardPythonCallExt.jl | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl index 1bfd1ec..68ca778 100644 --- a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl +++ b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl @@ -48,4 +48,45 @@ function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content ) end + +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringOffsetArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringRegularArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringOffsetArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringRegularArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) + +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringRegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringRegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedOptionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.BitMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnmaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + end # module From 5ab43352bd424998cbfc60c652aafbf0ac95cc97 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 14 Dec 2023 16:59:10 +0100 Subject: [PATCH 03/52] fix: update conversion rules --- ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl index 68ca778..f3c0450 100644 --- a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl +++ b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl @@ -49,17 +49,12 @@ function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content end + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringOffsetArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.StringRegularArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringOffsetArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteStringRegularArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) @@ -74,12 +69,6 @@ PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.StringRegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -# FIXME: PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteStringRegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) From 7af724dc3fdda09dace4545f184881ba91c9a6e9 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 17 Jan 2024 14:14:10 +0100 Subject: [PATCH 04/52] feat: define __init__ function --- Project.toml | 2 +- .../AwkwardPythonCallExt.jl | 57 ++++++++++--------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/Project.toml b/Project.toml index c78d433..6f79a23 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AwkwardArray" uuid = "7d259134-7f60-4bf1-aa00-7452e11bde56" authors = ["Jim Pivarski ", "Jerry Ling ", "and contributors"] -version = "0.1.2" +version = "0.1.3" [deps] JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" diff --git a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl index f3c0450..fe4b91d 100644 --- a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl +++ b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl @@ -49,33 +49,34 @@ function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content end - -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) - -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedOptionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.BitMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnmaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) -PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +function __init__() + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) + PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) + + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedOptionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.BitMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnmaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) +end end # module From 009bcafae0bad9da9a07d466b16f0a0e2dd9ae83 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 17 Jan 2024 14:23:36 +0100 Subject: [PATCH 05/52] fix: move convert out of the __init__ function --- .../AwkwardPythonCallExt.jl | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl index fe4b91d..eab0a4f 100644 --- a/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl +++ b/ext/AwkwardPythonCallExt/AwkwardPythonCallExt.jl @@ -49,21 +49,22 @@ function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content end -function __init__() - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) +PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) + +function __init__() PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) From d6000ab09887874857c000211649492c361a6822 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 17 Jan 2024 14:31:22 +0100 Subject: [PATCH 06/52] test: add a test passing a Python array to a Julia function --- Project.toml | 2 +- test/runpytests.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 6f79a23..c78d433 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AwkwardArray" uuid = "7d259134-7f60-4bf1-aa00-7452e11bde56" authors = ["Jim Pivarski ", "Jerry Ling ", "and contributors"] -version = "0.1.3" +version = "0.1.2" [deps] JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" diff --git a/test/runpytests.jl b/test/runpytests.jl index e0ab499..fd65e29 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -30,3 +30,15 @@ end @test array == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] end + +# Test passing Python array to Julia function +@testset "pass Python array to Julia test" begin + function f1(x) + x + end + + py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) + + array = f1(py_array) + @test array isa AwkwardArray.ListOffsetArray +end From f2b14066271ca5234997f2b05c56aa60442065ba Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 23 Jan 2024 11:17:50 +0100 Subject: [PATCH 07/52] feat: define conversion rules --- Project.toml | 14 +++++--------- src/AwkwardArray.jl | 5 +++-- .../AwkwardPythonCallExt.jl | 0 test/runpytests.jl | 3 ++- 4 files changed, 10 insertions(+), 12 deletions(-) rename {ext/AwkwardPythonCallExt => src}/AwkwardPythonCallExt.jl (100%) diff --git a/Project.toml b/Project.toml index c78d433..b5455c0 100644 --- a/Project.toml +++ b/Project.toml @@ -4,23 +4,19 @@ authors = ["Jim Pivarski ", "Jerry Ling Date: Tue, 23 Jan 2024 12:11:30 +0100 Subject: [PATCH 08/52] fix: remove convertion of an iterable --- src/AwkwardPythonCallExt.jl | 16 ---------------- test/runpytests.jl | 5 ++--- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/AwkwardPythonCallExt.jl b/src/AwkwardPythonCallExt.jl index eab0a4f..040365a 100644 --- a/src/AwkwardPythonCallExt.jl +++ b/src/AwkwardPythonCallExt.jl @@ -32,22 +32,6 @@ function AwkwardArray.convert(array::Py)::AwkwardArray.Content ) end -function AwkwardArray.convert(array::PyIterable{Any})::AwkwardArray.Content - form, len, _containers = pyimport("awkward").to_buffers(array) - containers = pyconvert(Dict, _containers) - - julia_buffers = Dict{String,AbstractVector{UInt8}}() - for (key, buffer) in containers - julia_buffers[key] = reinterpret(UInt8, buffer) - end - - AwkwardArray.from_buffers( - pyconvert(String, form.to_json()), - pyconvert(Int, len), - julia_buffers, - ) -end - PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) diff --git a/test/runpytests.jl b/test/runpytests.jl index d32e935..fcbb298 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -34,12 +34,11 @@ end # Test passing Python array to Julia function @testset "pass Python array to Julia test" begin function f1(x) - print(typeof(x)) - x + convert(x) # FIXME invoke convert when passing Py as an argument? end py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) array = f1(py_array) - @test array isa Py #AwkwardArray.ListOffsetArray + @test array isa AwkwardArray.ListOffsetArray end From 11c7c4681f985d79315fa03ff0e60c79f173a4cc Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 23 Jan 2024 15:07:01 +0100 Subject: [PATCH 09/52] feat: add rules and register --- src/AwkwardPythonCallExt.jl | 114 ++++++++++++++++++++++++++++-------- test/runpytests.jl | 2 + 2 files changed, 90 insertions(+), 26 deletions(-) diff --git a/src/AwkwardPythonCallExt.jl b/src/AwkwardPythonCallExt.jl index 040365a..bbec639 100644 --- a/src/AwkwardPythonCallExt.jl +++ b/src/AwkwardPythonCallExt.jl @@ -32,36 +32,98 @@ function AwkwardArray.convert(array::Py)::AwkwardArray.Content ) end +# rule functions +function pyconvert_rule_awkward_array_primitive(::Type{AwkwardArray.PrimitiveArray}, x::Py) + py_arr = pyconvert(AwkwardArray.PrimitiveArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_empty(::Type{AwkwardArray.EmptyArray}, x::Py) + py_arr = pyconvert(AwkwardArray.EmptyArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_listoffset(::Type{AwkwardArray.ListOffsetArray}, x::Py) + py_arr = pyconvert(AwkwardArray.ListOffsetArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_list(::Type{AwkwardArray.ListArray}, x::Py) + py_arr = pyconvert(AwkwardArray.ListArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end +function pyconvert_rule_awkward_array_regular(::Type{AwkwardArray.RegularArray}, x::Py) + py_arr = pyconvert(AwkwardArray.RegularArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_record(::Type{AwkwardArray.RecordArray}, x::Py) + py_arr = pyconvert(AwkwardArray.RecordArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_tuple(::Type{AwkwardArray.TupleArray}, x::Py) + py_arr = pyconvert(AwkwardArray.TupleArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_indexed(::Type{AwkwardArray.IndexedArray}, x::Py) + py_arr = pyconvert(AwkwardArray.IndexedArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.PrimitiveArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.EmptyArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListOffsetArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ListArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RegularArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.RecordArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.TupleArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.IndexedOptionArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.ByteMaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.BitMaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnmaskedArray}, x::Py) = AwkwardArray.convert(x) -PythonCall.pyconvert_rule_array_nocopy(::Type{AwkwardArray.UnionArray}, x::Py) = AwkwardArray.convert(x) +function pyconvert_rule_awkward_array_indexedoption(::Type{AwkwardArray.IndexedOptionArray}, x::Py) + py_arr = pyconvert(AwkwardArray.IndexedOptionArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_bytemasked(::Type{AwkwardArray.ByteMaskedArray}, x::Py) + py_arr = pyconvert(AwkwardArray.ByteMaskedArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end +function pyconvert_rule_awkward_array_bitmasked(::Type{AwkwardArray.BitMaskedArray}, x::Py) + py_arr = pyconvert(AwkwardArray.BitMaskedArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_unmasked(::Type{AwkwardArray.UnmaskedArray}, x::Py) + py_arr = pyconvert(AwkwardArray.UnmaskedArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end + +function pyconvert_rule_awkward_array_union(::Type{AwkwardArray.UnionArray}, x::Py) + py_arr = pyconvert(AwkwardArray.UnionArray, x) + array = AwkwardArray.convert(py_arr) + return PythonCall.pyconvert_return(array) +end function __init__() - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedOptionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.BitMaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnmaskedArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) - PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnionArray, PythonCall.pyconvert_rule_array_nocopy, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.PrimitiveArray, pyconvert_rule_awkward_array_primitive, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.EmptyArray, pyconvert_rule_awkward_array_empty, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListOffsetArray, pyconvert_rule_awkward_array_listoffset, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ListArray, pyconvert_rule_awkward_array_list, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RegularArray, pyconvert_rule_awkward_array_regular, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.RecordArray, pyconvert_rule_awkward_array_record, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.TupleArray, pyconvert_rule_awkward_array_tuple, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedArray, pyconvert_rule_awkward_array_indexed, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.IndexedOptionArray, pyconvert_rule_awkward_array_indexedoption, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.ByteMaskedArray, pyconvert_rule_awkward_array_bytemasked, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.BitMaskedArray, pyconvert_rule_awkward_array_bitmasked, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnmaskedArray, pyconvert_rule_awkward_array_unmasked, PythonCall.PYCONVERT_PRIORITY_ARRAY) + PythonCall.pyconvert_add_rule("awkward.highlevel:Array", AwkwardArray.UnionArray, pyconvert_rule_awkward_array_union, PythonCall.PYCONVERT_PRIORITY_ARRAY) end end # module diff --git a/test/runpytests.jl b/test/runpytests.jl index fcbb298..04c5429 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -34,6 +34,8 @@ end # Test passing Python array to Julia function @testset "pass Python array to Julia test" begin function f1(x) + println(typeof(x)) + println(typeof(convert(x))) convert(x) # FIXME invoke convert when passing Py as an argument? end From 3b8663e02f7954c44cc75e8ebca164cf72899f5c Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 23 Jan 2024 15:15:55 +0100 Subject: [PATCH 10/52] fix: leave it to a user to invoke pyconvert --- src/AwkwardPythonCallExt.jl | 39 +++++++++++++------------------------ test/runpytests.jl | 2 +- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/AwkwardPythonCallExt.jl b/src/AwkwardPythonCallExt.jl index bbec639..e906efb 100644 --- a/src/AwkwardPythonCallExt.jl +++ b/src/AwkwardPythonCallExt.jl @@ -34,79 +34,66 @@ end # rule functions function pyconvert_rule_awkward_array_primitive(::Type{AwkwardArray.PrimitiveArray}, x::Py) - py_arr = pyconvert(AwkwardArray.PrimitiveArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_empty(::Type{AwkwardArray.EmptyArray}, x::Py) - py_arr = pyconvert(AwkwardArray.EmptyArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_listoffset(::Type{AwkwardArray.ListOffsetArray}, x::Py) - py_arr = pyconvert(AwkwardArray.ListOffsetArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_list(::Type{AwkwardArray.ListArray}, x::Py) - py_arr = pyconvert(AwkwardArray.ListArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_regular(::Type{AwkwardArray.RegularArray}, x::Py) - py_arr = pyconvert(AwkwardArray.RegularArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_record(::Type{AwkwardArray.RecordArray}, x::Py) - py_arr = pyconvert(AwkwardArray.RecordArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_tuple(::Type{AwkwardArray.TupleArray}, x::Py) - py_arr = pyconvert(AwkwardArray.TupleArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_indexed(::Type{AwkwardArray.IndexedArray}, x::Py) - py_arr = pyconvert(AwkwardArray.IndexedArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_indexedoption(::Type{AwkwardArray.IndexedOptionArray}, x::Py) - py_arr = pyconvert(AwkwardArray.IndexedOptionArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_bytemasked(::Type{AwkwardArray.ByteMaskedArray}, x::Py) - py_arr = pyconvert(AwkwardArray.ByteMaskedArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_bitmasked(::Type{AwkwardArray.BitMaskedArray}, x::Py) - py_arr = pyconvert(AwkwardArray.BitMaskedArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_unmasked(::Type{AwkwardArray.UnmaskedArray}, x::Py) - py_arr = pyconvert(AwkwardArray.UnmaskedArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end function pyconvert_rule_awkward_array_union(::Type{AwkwardArray.UnionArray}, x::Py) - py_arr = pyconvert(AwkwardArray.UnionArray, x) - array = AwkwardArray.convert(py_arr) + array = AwkwardArray.convert(x) return PythonCall.pyconvert_return(array) end diff --git a/test/runpytests.jl b/test/runpytests.jl index 04c5429..76b5e44 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -36,7 +36,7 @@ end function f1(x) println(typeof(x)) println(typeof(convert(x))) - convert(x) # FIXME invoke convert when passing Py as an argument? + pyconvert(Any, x) # FIXME invoke convert when passing Py as an argument? end py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) From 9ea8d9c0b18b50086a747e0ade1247fc38e98b7c Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 23 Jan 2024 16:06:13 +0100 Subject: [PATCH 11/52] test: test pyconvert of an awkward array --- test/runpytests.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/runpytests.jl b/test/runpytests.jl index 76b5e44..148a72e 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -31,16 +31,10 @@ end @test array == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] end -# Test passing Python array to Julia function -@testset "pass Python array to Julia test" begin - function f1(x) - println(typeof(x)) - println(typeof(convert(x))) - pyconvert(Any, x) # FIXME invoke convert when passing Py as an argument? - end - +# Test pyconvert Python array to Julia +@testset "convert Python array to Julia test" begin py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) - array = f1(py_array) + array = pyconvert(Any, py_array) @test array isa AwkwardArray.ListOffsetArray end From 34f33a79af903350d67709ae2937564e2ac1e6c6 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 24 Jan 2024 13:28:31 +0100 Subject: [PATCH 12/52] tests: add tests --- test/runpytests.jl | 167 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) diff --git a/test/runpytests.jl b/test/runpytests.jl index 148a72e..f89d556 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -31,10 +31,173 @@ end @test array == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] end -# Test pyconvert Python array to Julia -@testset "convert Python array to Julia test" begin +# Test pyconvert Python Awkwar Array to Julia Awkward Array +# @testset "convert Python array to Julia test" begin +@testset "convert # PrimitiveArray" begin + layout = pyimport("awkward").contents.NumpyArray( + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + ) + py_array = pyimport("awkward").Array(layout) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.PrimitiveArray +end + +@testset "convert # EmptyArray" begin + layout = pyimport("awkward").contents.EmptyArray() + py_array = pyimport("awkward").Array(layout) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.EmptyArray +end + +@testset "convert # ListOffsetArray" begin py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) array = pyconvert(Any, py_array) @test array isa AwkwardArray.ListOffsetArray end + +@testset "convert # ListArray" begin + content = pyimport("awkward").contents.NumpyArray( + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + ) + starts = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6])) + stops = pyimport("awkward").index.Index64(pyimport("numpy").array([3, 3, 5, 6, 9])) + offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 9])) + layout = pyimport("awkward").contents.ListArray(starts, stops, content) + + py_array = pyimport("awkward").Array(layout) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.ListArray +end + +@testset "convert # RegularArray" begin + content = pyimport("awkward").contents.NumpyArray( + pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + ) + offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 10, 10])) + listoffsetarray = pyimport("awkward").contents.ListOffsetArray(offsets, content) + regulararray = pyimport("awkward").contents.RegularArray(listoffsetarray, 2, zeros_length=0) + + py_array = pyimport("awkward").Array(regulararray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.RegularArray +end + +@testset "convert # RecordArray" begin + content1 = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([1, 2, 3, 4, 5], dtype=pyimport("numpy").int64)) + content2 = pyimport("awkward").contents.NumpyArray( + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9], dtype=pyimport("numpy").float64) + ) + offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 9], dtype=pyimport("numpy").int64)) + listoffsetarray = pyimport("awkward").contents.ListOffsetArray(offsets, content2) + recordarray = pyimport("awkward").contents.RecordArray( + [content1, listoffsetarray, content2, content1], + fields=["one", "two", "2", "wonky"], + ) + + py_array = pyimport("awkward").Array(recordarray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.RecordArray +end + +@testset "convert # TupleArray" begin + tuplearray = pyimport("awkward").contents.RecordArray([pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(10))], pybuiltins.None) + + py_array = pyimport("awkward").Array(tuplearray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.TupleArray +end + +@testset "convert # IndexedArray" begin + content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4])) + + ind = pyimport("numpy").array([2, 2, 0, 3, 4], dtype=pyimport("numpy").int32) + index = pyimport("awkward").index.Index32(ind) + indexedarray = pyimport("awkward").contents.IndexedArray(index, content) + + py_array = pyimport("awkward").Array(indexedarray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.IndexedArray +end + +@testset "convert # IndexedOptionArray" begin + content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4])) + index = pyimport("awkward").index.Index64(pyimport("numpy").array([2, 2, 0, -1, 4], dtype=pyimport("numpy").int64)) + indexedoptionarray = pyimport("awkward").contents.IndexedOptionArray(index, content) + + py_array = pyimport("awkward").Array(indexedoptionarray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.IndexedOptionArray +end + +@testset "convert # ByteMaskedArray" begin + layout = pyimport("awkward").contents.ByteMaskedArray( + pyimport("awkward").index.Index8(pyimport("numpy").array([0, 1, 0, 1, 0], dtype=pyimport("numpy").int8)), + pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(5)), + valid_when=pybuiltins.True, + ) + py_array = pyimport("awkward").Array(layout) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.ByteMaskedArray +end + +# @testset "convert # BitMaskedArray" begin +# content = pyimport("awkward").operations.from_iter( +# [[0.0, 1.1, 2.2], [3.3, 4.4], [5.5], [6.6, 7.7, 8.8, 9.9]], highlevel=pybuiltins.False +# ) +# mask = pyimport("awkward").index.IndexU8(pyimport("numpy").array([66], dtype=pyimport("numpy").uint8)) +# maskedarray = pyimport("awkward").contents.BitMaskedArray( +# mask, content, valid_when=pybuiltins.False, length=4, lsb_order=pybuiltins.True +# ) +# py_array = pyimport("awkward").Array(maskedarray) + +# array = pyconvert(Any, py_array) +# @test array isa AwkwardArray.BitMaskedArray +# end + +@testset "convert # UnmaskedArray" begin + unmaskedarray = pyimport("awkward").contents.UnmaskedArray( + pyimport("awkward").contents.NumpyArray( + pyimport("numpy").array([0.0, 1.1, 2.2, 3.3], dtype=pyimport("numpy").float64) + ) + ) + py_array = pyimport("awkward").Array(unmaskedarray) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.UnmaskedArray +end + +@testset "convert # UnionArray" begin + layout = pyimport("awkward").contents.unionarray.UnionArray( + pyimport("awkward").index.Index(pyimport("numpy").array([1, 1, 0, 0, 1, 0, 1], dtype=pyimport("numpy").int8)), + pyimport("awkward").index.Index(pyimport("numpy").array([4, 3, 0, 1, 2, 2, 4, 100])), + [ + pyimport("awkward").contents.recordarray.RecordArray( + [pyimport("awkward").from_iter(["1", "2", "3"], highlevel=pybuiltins.False)], ["nest"] + ), + pyimport("awkward").contents.recordarray.RecordArray( + [ + pyimport("awkward").contents.numpyarray.NumpyArray( + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5]) + ) + ], + ["nest"], + ), + ], + ) + py_array = pyimport("awkward").Array(layout) + + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.UnionArray +end + +#end From c396ae73c0c4aedd04d8593fa18cbd8cd50b9a4c Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 24 Jan 2024 14:14:19 +0100 Subject: [PATCH 13/52] fix: use Vector in convert to avoid problems with BitMasked buffers --- src/AwkwardPythonCallExt.jl | 2 +- test/runpytests.jl | 33 +++++++++++++++------------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/AwkwardPythonCallExt.jl b/src/AwkwardPythonCallExt.jl index e906efb..11fb4f8 100644 --- a/src/AwkwardPythonCallExt.jl +++ b/src/AwkwardPythonCallExt.jl @@ -19,7 +19,7 @@ function AwkwardArray.convert(array::Py)::AwkwardArray.Content form, len, _containers = pyimport("awkward").to_buffers(array) containers = pyconvert(Dict, _containers) - julia_buffers = Dict{String,AbstractVector{UInt8}}() + julia_buffers = Dict{String,Vector{UInt8}}() for (key, buffer) in containers julia_buffers[key] = reinterpret(UInt8, buffer) diff --git a/test/runpytests.jl b/test/runpytests.jl index f89d556..4ab3398 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -32,14 +32,13 @@ end end # Test pyconvert Python Awkwar Array to Julia Awkward Array -# @testset "convert Python array to Julia test" begin @testset "convert # PrimitiveArray" begin layout = pyimport("awkward").contents.NumpyArray( pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) ) py_array = pyimport("awkward").Array(layout) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.PrimitiveArray, py_array) @test array isa AwkwardArray.PrimitiveArray end @@ -47,14 +46,14 @@ end layout = pyimport("awkward").contents.EmptyArray() py_array = pyimport("awkward").Array(layout) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.EmptyArray, py_array) @test array isa AwkwardArray.EmptyArray end @testset "convert # ListOffsetArray" begin py_array = pyimport("awkward").Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]]) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.ListOffsetArray, py_array) @test array isa AwkwardArray.ListOffsetArray end @@ -150,19 +149,19 @@ end @test array isa AwkwardArray.ByteMaskedArray end -# @testset "convert # BitMaskedArray" begin -# content = pyimport("awkward").operations.from_iter( -# [[0.0, 1.1, 2.2], [3.3, 4.4], [5.5], [6.6, 7.7, 8.8, 9.9]], highlevel=pybuiltins.False -# ) -# mask = pyimport("awkward").index.IndexU8(pyimport("numpy").array([66], dtype=pyimport("numpy").uint8)) -# maskedarray = pyimport("awkward").contents.BitMaskedArray( -# mask, content, valid_when=pybuiltins.False, length=4, lsb_order=pybuiltins.True -# ) -# py_array = pyimport("awkward").Array(maskedarray) +@testset "convert # BitMaskedArray" begin + content = pyimport("awkward").operations.from_iter( + [[0.0, 1.1, 2.2], [3.3, 4.4], [5.5], [6.6, 7.7, 8.8, 9.9]], highlevel=pybuiltins.False + ) + mask = pyimport("awkward").index.IndexU8(pyimport("numpy").array([66], dtype=pyimport("numpy").uint8)) + maskedarray = pyimport("awkward").contents.BitMaskedArray( + mask, content, valid_when=pybuiltins.False, length=4, lsb_order=pybuiltins.True + ) + py_array = pyimport("awkward").Array(maskedarray) -# array = pyconvert(Any, py_array) -# @test array isa AwkwardArray.BitMaskedArray -# end + array = pyconvert(Any, py_array) + @test array isa AwkwardArray.BitMaskedArray +end @testset "convert # UnmaskedArray" begin unmaskedarray = pyimport("awkward").contents.UnmaskedArray( @@ -199,5 +198,3 @@ end array = pyconvert(Any, py_array) @test array isa AwkwardArray.UnionArray end - -#end From 1cc71e5c3317d1a9c41c87ef4f5575e48802a0fb Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 24 Jan 2024 14:45:12 +0100 Subject: [PATCH 14/52] feat: convert to an awkward type --- test/runpytests.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/runpytests.jl b/test/runpytests.jl index 4ab3398..a06085c 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -68,7 +68,7 @@ end py_array = pyimport("awkward").Array(layout) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.ListArray, py_array) @test array isa AwkwardArray.ListArray end @@ -82,7 +82,7 @@ end py_array = pyimport("awkward").Array(regulararray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.RegularArray, py_array) @test array isa AwkwardArray.RegularArray end @@ -100,7 +100,7 @@ end py_array = pyimport("awkward").Array(recordarray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.RecordArray, py_array) @test array isa AwkwardArray.RecordArray end @@ -109,7 +109,7 @@ end py_array = pyimport("awkward").Array(tuplearray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.TupleArray, py_array) @test array isa AwkwardArray.TupleArray end @@ -122,7 +122,7 @@ end py_array = pyimport("awkward").Array(indexedarray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.IndexedArray, py_array) @test array isa AwkwardArray.IndexedArray end @@ -133,7 +133,7 @@ end py_array = pyimport("awkward").Array(indexedoptionarray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.IndexedOptionArray, py_array) @test array isa AwkwardArray.IndexedOptionArray end @@ -145,7 +145,7 @@ end ) py_array = pyimport("awkward").Array(layout) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.ByteMaskedArray, py_array) @test array isa AwkwardArray.ByteMaskedArray end @@ -159,7 +159,7 @@ end ) py_array = pyimport("awkward").Array(maskedarray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.BitMaskedArray, py_array) @test array isa AwkwardArray.BitMaskedArray end @@ -171,7 +171,7 @@ end ) py_array = pyimport("awkward").Array(unmaskedarray) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.UnmaskedArray, py_array) @test array isa AwkwardArray.UnmaskedArray end @@ -192,9 +192,9 @@ end ["nest"], ), ], - ) + ) py_array = pyimport("awkward").Array(layout) - array = pyconvert(Any, py_array) + array = pyconvert(AwkwardArray.UnionArray, py_array) @test array isa AwkwardArray.UnionArray end From f193bf4bf31fe10c8108bfc78f7390d340e11b09 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 24 Jan 2024 15:02:07 +0100 Subject: [PATCH 15/52] fix: add dtypes --- test/runpytests.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runpytests.jl b/test/runpytests.jl index a06085c..c2e6561 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -34,7 +34,7 @@ end # Test pyconvert Python Awkwar Array to Julia Awkward Array @testset "convert # PrimitiveArray" begin layout = pyimport("awkward").contents.NumpyArray( - pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9], dtype=pyimport("numpy").float64) ) py_array = pyimport("awkward").Array(layout) @@ -59,11 +59,11 @@ end @testset "convert # ListArray" begin content = pyimport("awkward").contents.NumpyArray( - pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9], dtype=pyimport("numpy").float64) ) - starts = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6])) - stops = pyimport("awkward").index.Index64(pyimport("numpy").array([3, 3, 5, 6, 9])) - offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 9])) + starts = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6], dtype=pyimport("numpy").int64)) + stops = pyimport("awkward").index.Index64(pyimport("numpy").array([3, 3, 5, 6, 9], dtype=pyimport("numpy").int64)) + offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 9], dtype=pyimport("numpy").int64)) layout = pyimport("awkward").contents.ListArray(starts, stops, content) py_array = pyimport("awkward").Array(layout) @@ -74,9 +74,9 @@ end @testset "convert # RegularArray" begin content = pyimport("awkward").contents.NumpyArray( - pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9], dtype=pyimport("numpy").float64) ) - offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 10, 10])) + offsets = pyimport("awkward").index.Index64(pyimport("numpy").array([0, 3, 3, 5, 6, 10, 10], dtype=pyimport("numpy").int64)) listoffsetarray = pyimport("awkward").contents.ListOffsetArray(offsets, content) regulararray = pyimport("awkward").contents.RegularArray(listoffsetarray, 2, zeros_length=0) From 7d2c69664e68215ac993af0d6b5e2de0ead891ba Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 24 Jan 2024 15:11:05 +0100 Subject: [PATCH 16/52] fix: more dtypes --- test/runpytests.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/runpytests.jl b/test/runpytests.jl index c2e6561..bcdbd55 100644 --- a/test/runpytests.jl +++ b/test/runpytests.jl @@ -105,7 +105,7 @@ end end @testset "convert # TupleArray" begin - tuplearray = pyimport("awkward").contents.RecordArray([pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(10))], pybuiltins.None) + tuplearray = pyimport("awkward").contents.RecordArray([pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(10, dtype=pyimport("numpy").int64))], pybuiltins.None) py_array = pyimport("awkward").Array(tuplearray) @@ -114,7 +114,7 @@ end end @testset "convert # IndexedArray" begin - content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4])) + content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4], dtype=pyimport("numpy").float64)) ind = pyimport("numpy").array([2, 2, 0, 3, 4], dtype=pyimport("numpy").int32) index = pyimport("awkward").index.Index32(ind) @@ -127,7 +127,7 @@ end end @testset "convert # IndexedOptionArray" begin - content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4])) + content = pyimport("awkward").contents.NumpyArray(pyimport("numpy").array([0.0, 1.1, 2.2, 3.3, 4.4], dtype=pyimport("numpy").float64)) index = pyimport("awkward").index.Index64(pyimport("numpy").array([2, 2, 0, -1, 4], dtype=pyimport("numpy").int64)) indexedoptionarray = pyimport("awkward").contents.IndexedOptionArray(index, content) @@ -140,7 +140,7 @@ end @testset "convert # ByteMaskedArray" begin layout = pyimport("awkward").contents.ByteMaskedArray( pyimport("awkward").index.Index8(pyimport("numpy").array([0, 1, 0, 1, 0], dtype=pyimport("numpy").int8)), - pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(5)), + pyimport("awkward").contents.NumpyArray(pyimport("numpy").arange(5, dtype=pyimport("numpy").int64)), valid_when=pybuiltins.True, ) py_array = pyimport("awkward").Array(layout) @@ -178,7 +178,7 @@ end @testset "convert # UnionArray" begin layout = pyimport("awkward").contents.unionarray.UnionArray( pyimport("awkward").index.Index(pyimport("numpy").array([1, 1, 0, 0, 1, 0, 1], dtype=pyimport("numpy").int8)), - pyimport("awkward").index.Index(pyimport("numpy").array([4, 3, 0, 1, 2, 2, 4, 100])), + pyimport("awkward").index.Index(pyimport("numpy").array([4, 3, 0, 1, 2, 2, 4, 100], dtype=pyimport("numpy").int64)), [ pyimport("awkward").contents.recordarray.RecordArray( [pyimport("awkward").from_iter(["1", "2", "3"], highlevel=pybuiltins.False)], ["nest"] @@ -186,7 +186,7 @@ end pyimport("awkward").contents.recordarray.RecordArray( [ pyimport("awkward").contents.numpyarray.NumpyArray( - pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5]) + pyimport("numpy").array([1.1, 2.2, 3.3, 4.4, 5.5], dtype=pyimport("numpy").float64) ) ], ["nest"], From 6687140f04a6282ef6fad0941fc15379383b942a Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 30 Jan 2024 23:01:17 +0100 Subject: [PATCH 17/52] feat: add content types --- src/AwkwardArray.jl | 9 +++ src/all_implementations.jl | 1 + test/runtests.jl | 118 ++++++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 5b331af..74fb9ee 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -9,5 +9,14 @@ include("./tables.jl") include("./AwkwardPythonCallExt.jl") using .AwkwardPythonCallExt: convert +Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} +Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS +Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS} +Base.eltype(::IndexedArray{INDEX,CONTENT,BEHAVIOR}) where {INDEX,CONTENT,BEHAVIOR} = CONTENT +Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT +Base.eltype(::ByteMaskedArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT +Base.eltype(::BitMaskedArray{CONTENT}) where {CONTENT} = CONTENT +Base.eltype(::UnmaskedArray{CONTENT}) where {CONTENT} = CONTENT +Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS end # module AwkwardArray diff --git a/src/all_implementations.jl b/src/all_implementations.jl index f5c9a57..a78bcc6 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -126,6 +126,7 @@ function Base.iterate(layout::Content, state) end Base.size(layout::Content) = (length(layout),) +Base.eltype(layout::Content) = (eltype(iterate(layout, 1))) function Base.:(==)(layout1::Content, layout2::Content) if length(layout1) != length(layout2) diff --git a/test/runtests.jl b/test/runtests.jl index 8e80a4f..6956b32 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,6 +26,8 @@ using Tables @inferred layout[2:4] @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, 4.4, 5.5] + + @test eltype(layout) == typeof(layout[1]) end begin @@ -47,6 +49,8 @@ using Tables @test length(layout) == 6 @test layout == AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5, 0.0]) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -60,6 +64,8 @@ using Tables append!(layout, Vector{Int16}([4, 5])) @test layout == AwkwardArray.PrimitiveArray(Vector{Int32}([1, 2, 3, 4, 5])) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -83,6 +89,8 @@ using Tables @test layout == AwkwardArray.PrimitiveArray( Vector{Float32}([1.0, 2.0, 3.0, 4.0, 5.0, 3.14, 2.71]), ) + + @test eltype(layout) == typeof(layout[1]) end end @@ -102,18 +110,24 @@ end @test layout == AwkwardArray.PrimitiveArray(Vector{Float64}()) @test AwkwardArray.to_vector(layout) == [] + + @test eltype(layout) == nothing end begin layout = AwkwardArray.EmptyArray() @test length(layout) == 0 @test layout == AwkwardArray.EmptyArray() + + @test eltype(layout) == nothing end begin layout = AwkwardArray.EmptyArray() append!(layout, []) append!(layout, Vector{Int64}([])) + + @test eltype(layout) == nothing end end @@ -146,6 +160,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -154,6 +170,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -162,6 +180,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -170,6 +190,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -202,6 +224,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -231,6 +255,8 @@ end [0, 3, 3, 5, 5, 9], AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5, 6, 7, 8, 9]), ) + + @test eltype(layout) == Vector{eltype(layout.content)} end end @@ -265,6 +291,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -274,6 +302,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -283,6 +313,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -292,6 +324,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test !AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -326,6 +360,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -353,6 +389,8 @@ end [3, 3, 5, 5, 9], AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5, 6, 7, 8, 9]), ) + + @test eltype(layout) == Vector{eltype(layout.content)} end end @@ -375,6 +413,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6]] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -414,6 +454,8 @@ end ] @test AwkwardArray.to_vector(layout[3:5]) == [[10, 11, 12, 13, 14], [15, 16, 17, 18, 19], [20, 21, 22, 23, 24]] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -437,6 +479,8 @@ end ) @test layout[2:2] == AwkwardArray.RegularArray(AwkwardArray.PrimitiveArray([3.3, 4.4]), 2) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -448,6 +492,8 @@ end @test AwkwardArray.is_valid(layout) @test length(layout) == 5 @test layout.size == 0 + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -477,6 +523,8 @@ end 3, ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -508,6 +556,8 @@ end 2, ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -524,6 +574,8 @@ end 0, zeros_length = 5, ) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -541,6 +593,8 @@ end AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), 3, ) + + @test eltype(layout) == Vector{eltype(layout.content)} end end @@ -557,6 +611,8 @@ end AwkwardArray.is_valid(AwkwardArray.StringRegularArray("onetwo", 3)) AwkwardArray.is_valid(AwkwardArray.StringRegularArray(3)) AwkwardArray.is_valid(AwkwardArray.StringRegularArray()) + + @test eltype(AwkwardArray.StringRegularArray("onetwo", 3)) == Vector{eltype(AwkwardArray.StringRegularArray("onetwo", 3).content)} end begin @@ -566,6 +622,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == ["hey", "there", "\$", "¢", "€", "💰"] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -579,6 +637,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == ["hey", "there", "\$", "¢", "€", "💰"] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -588,6 +648,8 @@ end @inferred layout[1:2] @test AwkwardArray.to_vector(layout) == ["hey", "you"] + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -631,6 +693,8 @@ end AwkwardArray.push_dummy!(layout) @test Vector(layout) == ["hey", "there", "\$", "¢", "€", "💰", ""] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -675,6 +739,8 @@ end AwkwardArray.push_dummy!(layout) @test Vector(layout) == ["hey", "there", "\$", "¢", "€", "💰", ""] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -695,6 +761,8 @@ end AwkwardArray.push_dummy!(layout) @test Vector(layout) == ["one", "two", "\0\0\0"] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -705,6 +773,8 @@ end append!(layout, ["two", "", "three"]) @test layout == AwkwardArray.StringOffsetArray([0, 3, 6, 6, 11], "onetwothree") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -715,6 +785,8 @@ end append!(layout, ["two", "", "three"]) @test layout == AwkwardArray.StringArray([0, 3, 6, 6], [3, 6, 6, 11], "onetwothree") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -725,6 +797,8 @@ end append!(layout, ["two", "333"]) @test layout == AwkwardArray.StringRegularArray("onetwo333", 3) + + @test eltype(layout) == Vector{eltype(layout.content)} end ### ListType with behavior = :bytestring ################################# @@ -755,6 +829,8 @@ end ) AwkwardArray.is_valid(AwkwardArray.ByteStringRegularArray(3)) AwkwardArray.is_valid(AwkwardArray.ByteStringRegularArray()) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -796,6 +872,8 @@ end AwkwardArray.push_dummy!(layout) @test layout[7] == [] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -838,6 +916,8 @@ end AwkwardArray.push_dummy!(layout) @test layout[7] == [] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -858,6 +938,8 @@ end AwkwardArray.push_dummy!(layout) @test Vector(layout) == [[0x6f, 0x6e, 0x65], [0x74, 0x77, 0x6f], [0x00, 0x00, 0x00]] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -871,6 +953,8 @@ end [0, 3, 3, 5], Vector{UInt8}([0, 1, 2, 3, 4]), ) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -885,6 +969,8 @@ end [3, 3, 5], Vector{UInt8}([0, 1, 2, 3, 4]), ) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -898,6 +984,8 @@ end Vector{UInt8}([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, ) + + @test eltype(layout) == typeof(layout[1]) end end @@ -913,6 +1001,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice list" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -925,6 +1015,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice list" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -936,6 +1028,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice list" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -951,6 +1045,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice string" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -967,6 +1063,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice string" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end begin @@ -982,6 +1080,8 @@ end @test AwkwardArray.get_parameter(layout, "__doc__") == "nice list" @test !AwkwardArray.has_parameter(layout, "__list__") + + @test eltype(layout) == Vector{eltype(layout.content)} end end @@ -1012,7 +1112,9 @@ end tmp += x[:b] end @test tmp == 16.5 - end + + @test eltype(layout) == typeof(layout[1]) + end begin layout = AwkwardArray.RecordArray( @@ -1060,6 +1162,8 @@ end NamedTuple{(:a, :b)}((2, [])), NamedTuple{(:a, :b)}((3, [4.4, 5.5])), ] + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1167,6 +1271,8 @@ end 1, ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1191,6 +1297,8 @@ end ) @test layout_2 == AwkwardArray.copy(layout_3, length = 2) + + @test eltype(layout_2) == eltype(layout_3) end begin @@ -1240,6 +1348,8 @@ end ), )), ) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1281,6 +1391,8 @@ end ), )), ) + + @test eltype(layout) == typeof(layout[1]) end end ### TupleArray ########################################################## @@ -1306,6 +1418,8 @@ end tmp += x[2] end @test tmp == 16.5 + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1350,6 +1464,8 @@ end @test AwkwardArray.to_vector(layout) == [(1, [1.1, 2.2, 3.3]), (2, []), (3, [4.4, 5.5])] + + @test eltype(layout) == typeof(layout[1]) end begin From 15b11a8e7e29d23a5264b0e06ef442266d86346f Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 5 Feb 2024 16:27:10 +0100 Subject: [PATCH 18/52] fix: add tuple behaviour --- src/AwkwardArray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 74fb9ee..67bd744 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -11,7 +11,7 @@ using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS -Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS} +Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} Base.eltype(::IndexedArray{INDEX,CONTENT,BEHAVIOR}) where {INDEX,CONTENT,BEHAVIOR} = CONTENT Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT Base.eltype(::ByteMaskedArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT From 1f53fb88eaaf945a53b77f3200c24fb483fef1a9 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 17:05:35 +0100 Subject: [PATCH 19/52] test: separate from_iter tests --- src/AwkwardArray.jl | 2 +- test/runtests.jl | 150 +++++++++++++++++++++++++++----------------- 2 files changed, 92 insertions(+), 60 deletions(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 67bd744..5a162c8 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -10,7 +10,7 @@ include("./AwkwardPythonCallExt.jl") using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} -Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS +#Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} Base.eltype(::IndexedArray{INDEX,CONTENT,BEHAVIOR}) where {INDEX,CONTENT,BEHAVIOR} = CONTENT Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT diff --git a/test/runtests.jl b/test/runtests.jl index ceecaf8..b8cd953 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2556,83 +2556,115 @@ end end ### from_iter ############################################################ -@testset "from_iter" begin - begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, 3])) +@testset "from_iter # 1" begin +# begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, 3])) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3]])) +@testset "from_iter # 2" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3]])) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter(["one", "two", "three"])) +@testset "from_iter # 3" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(["one", "two", "three"])) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 3; 4 5 6]))) +@testset "from_iter # 4" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 3; 4 5 6]))) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([ - NamedTuple{(:a, :b)}((1.1, [1, 2])), - NamedTuple{(:a, :b)}((2.2, [3])), - ]), - ) +@testset "from_iter # 5" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([ + NamedTuple{(:a, :b)}((1.1, [1, 2])), + NamedTuple{(:a, :b)}((2.2, [3])), + ]), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), (3.3, [0])]), - ) +@testset "from_iter # 6" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), (3.3, [0])]), + ) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, missing, 3])) +@testset "from_iter # 7" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, missing, 3])) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3, missing]])) +@testset "from_iter # 8" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3, missing]])) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], missing, [3]])) +@testset "from_iter # 9" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], missing, [3]])) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter(["one", "two", missing, "three"]), - ) +@testset "from_iter # 10" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter(["one", "two", missing, "three"]), + ) +end - @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 missing; 4 5 6]))) +@testset "from_iter # 11" begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 missing; 4 5 6]))) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([ - NamedTuple{(:a, :b)}((1.1, [1, 2])), - missing, - NamedTuple{(:a, :b)}((2.2, [3])), - ]), - ) +@testset "from_iter # 12" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([ + NamedTuple{(:a, :b)}((1.1, [1, 2])), + missing, + NamedTuple{(:a, :b)}((2.2, [3])), + ]), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), missing, (3.3, [0])]), - ) +@testset "from_iter # 13" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), missing, (3.3, [0])]), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), - ), - ) +@testset "from_iter # 14" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), + ), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), - ), - ) +@testset "from_iter # 15" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), + ), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - ), - ) +@testset "from_iter # 16" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + ), + ) +end - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,String,Vector{Int64}}}([ - 1.1, - [1, 2], - "hello", - missing, - [3], - ]), - ), - ) +@testset "from_iter # 17" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,String,Vector{Int64}}}([ + 1.1, + [1, 2], + "hello", + missing, + [3], + ]), + ), + ) - end end +#end ### from_buffers ######################################################### From a69f2b0c7b0fd34e5d287241787d4317caabe2b3 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 17:09:53 +0100 Subject: [PATCH 20/52] fix: eltype of UnionArray --- src/AwkwardArray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 5a162c8..5f3bafb 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -17,6 +17,6 @@ Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT Base.eltype(::ByteMaskedArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT Base.eltype(::BitMaskedArray{CONTENT}) where {CONTENT} = CONTENT Base.eltype(::UnmaskedArray{CONTENT}) where {CONTENT} = CONTENT -Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS +#Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS end # module AwkwardArray From 8825ea49aa614618e9be59c1dd71ce072a69da80 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 17:39:28 +0100 Subject: [PATCH 21/52] fix: cleanup --- src/AwkwardArray.jl | 4 ++-- src/all_implementations.jl | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 5f3bafb..67bd744 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -10,13 +10,13 @@ include("./AwkwardPythonCallExt.jl") using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} -#Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS +Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} Base.eltype(::IndexedArray{INDEX,CONTENT,BEHAVIOR}) where {INDEX,CONTENT,BEHAVIOR} = CONTENT Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT Base.eltype(::ByteMaskedArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT Base.eltype(::BitMaskedArray{CONTENT}) where {CONTENT} = CONTENT Base.eltype(::UnmaskedArray{CONTENT}) where {CONTENT} = CONTENT -#Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS +Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS end # module AwkwardArray diff --git a/src/all_implementations.jl b/src/all_implementations.jl index a78bcc6..f5c9a57 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -126,7 +126,6 @@ function Base.iterate(layout::Content, state) end Base.size(layout::Content) = (length(layout),) -Base.eltype(layout::Content) = (eltype(iterate(layout, 1))) function Base.:(==)(layout1::Content, layout2::Content) if length(layout1) != length(layout2) From 09aa99658584fe6a859bcd41d458bd8db2db41f9 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 17:57:11 +0100 Subject: [PATCH 22/52] test: comment out a test of from_iter on union --- test/runtests.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index b8cd953..04addad 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2634,13 +2634,13 @@ end ) end -@testset "from_iter # 15" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), - ), - ) -end +# @testset "from_iter # 15" begin +# @test AwkwardArray.is_valid( +# AwkwardArray.from_iter( +# Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), +# ), +# ) +# end @testset "from_iter # 16" begin @test AwkwardArray.is_valid( From 0de6fcac84500141ebb5bebae4aac7eed36e0dd1 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 18:01:53 +0100 Subject: [PATCH 23/52] test: comment out another test --- test/runtests.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 04addad..ded72bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2626,13 +2626,13 @@ end ) end -@testset "from_iter # 14" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), - ), - ) -end +# @testset "from_iter # 14" begin +# @test AwkwardArray.is_valid( +# AwkwardArray.from_iter( +# Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), +# ), +# ) +# end # @testset "from_iter # 15" begin # @test AwkwardArray.is_valid( From 9f9b4d2b207826845d02101806c32a1e72e748f7 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 18:06:11 +0100 Subject: [PATCH 24/52] test: add a test --- test/runtests.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index ded72bf..c1b9234 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2626,30 +2626,30 @@ end ) end -# @testset "from_iter # 14" begin +@testset "from_iter # 14" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), + ), + ) +end + +# @testset "from_iter # 15" begin # @test AwkwardArray.is_valid( # AwkwardArray.from_iter( -# Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), +# Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), # ), # ) # end -# @testset "from_iter # 15" begin +# @testset "from_iter # 16" begin # @test AwkwardArray.is_valid( # AwkwardArray.from_iter( -# Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), +# Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), # ), # ) # end -@testset "from_iter # 16" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - ), - ) -end - @testset "from_iter # 17" begin @test AwkwardArray.is_valid( AwkwardArray.from_iter( From fe720aad1ca2b782bba8ab7bf79270a89789533a Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Tue, 6 Feb 2024 18:12:30 +0100 Subject: [PATCH 25/52] test: comment out a test --- test/runtests.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c1b9234..57f9f7a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2650,20 +2650,20 @@ end # ) # end -@testset "from_iter # 17" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,String,Vector{Int64}}}([ - 1.1, - [1, 2], - "hello", - missing, - [3], - ]), - ), - ) +# @testset "from_iter # 17" begin +# @test AwkwardArray.is_valid( +# AwkwardArray.from_iter( +# Vector{Union{Missing,Float64,String,Vector{Int64}}}([ +# 1.1, +# [1, 2], +# "hello", +# missing, +# [3], +# ]), +# ), +# ) -end +# end #end ### from_buffers ######################################################### From fa4d5d127bca5d11500e2be0c3bbdd02447a3ecd Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 7 Feb 2024 08:19:05 +0100 Subject: [PATCH 26/52] fix: tests --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 57f9f7a..dc06ac0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2634,7 +2634,7 @@ end ) end -# @testset "from_iter # 15" begin +# testset "from_iter # 15" begin # @test AwkwardArray.is_valid( # AwkwardArray.from_iter( # Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), @@ -2642,7 +2642,7 @@ end # ) # end -# @testset "from_iter # 16" begin +# testset "from_iter # 16" begin # @test AwkwardArray.is_valid( # AwkwardArray.from_iter( # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), @@ -2650,7 +2650,7 @@ end # ) # end -# @testset "from_iter # 17" begin +# testset "from_iter # 17" begin # @test AwkwardArray.is_valid( # AwkwardArray.from_iter( # Vector{Union{Missing,Float64,String,Vector{Int64}}}([ From 440a2d6ced5aba57adf1a5e8f63e5f507eefd824 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 7 Feb 2024 08:23:10 +0100 Subject: [PATCH 27/52] fix: add a test --- test/runtests.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index dc06ac0..f029196 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2634,13 +2634,13 @@ end ) end -# testset "from_iter # 15" begin -# @test AwkwardArray.is_valid( -# AwkwardArray.from_iter( -# Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), -# ), -# ) -# end +@testset "from_iter # 15" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), + ), + ) +end # testset "from_iter # 16" begin # @test AwkwardArray.is_valid( From fba439ead77da261ca14993dedbce238f0a3bb21 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 7 Feb 2024 11:07:37 +0100 Subject: [PATCH 28/52] fix: add a test --- test/runtests.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f029196..27cef15 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2642,13 +2642,13 @@ end ) end -# testset "from_iter # 16" begin -# @test AwkwardArray.is_valid( -# AwkwardArray.from_iter( -# Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), -# ), -# ) -# end +@testset "from_iter # 16" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + ), + ) +end # testset "from_iter # 17" begin # @test AwkwardArray.is_valid( From a6e99369c117ec55c6e83b3feb12f52aabdb2c88 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 7 Feb 2024 11:35:33 +0100 Subject: [PATCH 29/52] debug: add print statements in the failing test --- src/all_implementations.jl | 5 +++++ test/runtests.jl | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index f5c9a57..290f711 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2284,6 +2284,8 @@ end ### from_iter ############################################################ function layout_for(ItemType) + println("layout_for ", ItemType) + if ItemType <: Number # || ItemType <: Dates.DateTime || ItemType <: Dates.TimePeriod PrimitiveArray{ItemType} @@ -2309,10 +2311,12 @@ function layout_for(ItemType) TupleArray{Base.Tuple{contents...}} elseif Missing <: ItemType + println("Missing ", ItemType) if ItemType == Any error("cannot produce an AwkwardArray layout for $ItemType (too generic)") end OtherTypes = [x for x in Base.uniontypes(ItemType) if x != Missing] + println("OtherTypes ", OtherTypes) if length(OtherTypes) == 0 IndexedOptionArray{Vector{Int64},EmptyArray} else @@ -2327,6 +2331,7 @@ function layout_for(ItemType) else contents = [out] for i = (firstindex(OtherTypes)+1):(lastindex(OtherTypes)) + println(i, " ", OtherTypes[i]) push!(contents, UnmaskedArray{layout_for(OtherTypes[i])}) end UnionArray{Index8,Vector{Int64},Base.Tuple{contents...}} diff --git a/test/runtests.jl b/test/runtests.jl index 27cef15..b974ef6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2643,6 +2643,9 @@ end end @testset "from_iter # 16" begin + my_vec = Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]) + println(my_vec) + @test AwkwardArray.is_valid( AwkwardArray.from_iter( Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), From 504126924818260514ca035651817fb812588859 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 8 Feb 2024 16:29:14 +0100 Subject: [PATCH 30/52] test: add debug statements --- src/all_implementations.jl | 6 ++++++ test/runtests.jl | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 290f711..905d127 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2332,6 +2332,12 @@ function layout_for(ItemType) contents = [out] for i = (firstindex(OtherTypes)+1):(lastindex(OtherTypes)) println(i, " ", OtherTypes[i]) + try + println("Try it first:") + layout_for(OtherTypes[i]) + catch err + println("Failed with ", err) + end push!(contents, UnmaskedArray{layout_for(OtherTypes[i])}) end UnionArray{Index8,Vector{Int64},Base.Tuple{contents...}} diff --git a/test/runtests.jl b/test/runtests.jl index b974ef6..a7bdd1e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2644,13 +2644,18 @@ end @testset "from_iter # 16" begin my_vec = Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]) - println(my_vec) - - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - ), - ) + println(my_vec, "eltype ", eltype(my_vec)) + + # array = AwkwardArray.from_iter( + # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + # ) + # println(array) + + # test AwkwardArray.is_valid( + # AwkwardArray.from_iter( + # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + # ), + # ) end # testset "from_iter # 17" begin From c8891bc524ca602d16630a53b6fc5992a8cc0c06 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 8 Feb 2024 16:49:43 +0100 Subject: [PATCH 31/52] fix: add test --- test/runtests.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index a7bdd1e..e3cda5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2646,10 +2646,10 @@ end my_vec = Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]) println(my_vec, "eltype ", eltype(my_vec)) - # array = AwkwardArray.from_iter( - # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - # ) - # println(array) + array = AwkwardArray.from_iter( + Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + ) + println(array) # test AwkwardArray.is_valid( # AwkwardArray.from_iter( From e69f907d16047c97c58aa7e5fa269b266e77b534 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 8 Feb 2024 17:25:44 +0100 Subject: [PATCH 32/52] fix: test --- test/runtests.jl | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index e3cda5a..6a457b2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2646,10 +2646,10 @@ end my_vec = Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]) println(my_vec, "eltype ", eltype(my_vec)) - array = AwkwardArray.from_iter( - Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - ) - println(array) + # array = AwkwardArray.from_iter( + # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + # ) + # println(array) # test AwkwardArray.is_valid( # AwkwardArray.from_iter( @@ -2658,20 +2658,20 @@ end # ) end -# testset "from_iter # 17" begin -# @test AwkwardArray.is_valid( -# AwkwardArray.from_iter( -# Vector{Union{Missing,Float64,String,Vector{Int64}}}([ -# 1.1, -# [1, 2], -# "hello", -# missing, -# [3], -# ]), -# ), -# ) - -# end +@testset "from_iter # 17" begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,String,Vector{Int64}}}([ + 1.1, + [1, 2], + "hello", + missing, + [3], + ]), + ), + ) + +end #end ### from_buffers ######################################################### From a719729e89e9f4adaa9142c60918094f1e4bf251 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 9 Feb 2024 09:50:44 +0100 Subject: [PATCH 33/52] fix: cleanup debug statements --- src/all_implementations.jl | 10 ---------- test/runtests.jl | 18 +++++------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 905d127..050ba04 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2284,7 +2284,6 @@ end ### from_iter ############################################################ function layout_for(ItemType) - println("layout_for ", ItemType) if ItemType <: Number # || ItemType <: Dates.DateTime || ItemType <: Dates.TimePeriod PrimitiveArray{ItemType} @@ -2311,12 +2310,10 @@ function layout_for(ItemType) TupleArray{Base.Tuple{contents...}} elseif Missing <: ItemType - println("Missing ", ItemType) if ItemType == Any error("cannot produce an AwkwardArray layout for $ItemType (too generic)") end OtherTypes = [x for x in Base.uniontypes(ItemType) if x != Missing] - println("OtherTypes ", OtherTypes) if length(OtherTypes) == 0 IndexedOptionArray{Vector{Int64},EmptyArray} else @@ -2331,13 +2328,6 @@ function layout_for(ItemType) else contents = [out] for i = (firstindex(OtherTypes)+1):(lastindex(OtherTypes)) - println(i, " ", OtherTypes[i]) - try - println("Try it first:") - layout_for(OtherTypes[i]) - catch err - println("Failed with ", err) - end push!(contents, UnmaskedArray{layout_for(OtherTypes[i])}) end UnionArray{Index8,Vector{Int64},Base.Tuple{contents...}} diff --git a/test/runtests.jl b/test/runtests.jl index 6a457b2..b8cd953 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2643,19 +2643,11 @@ end end @testset "from_iter # 16" begin - my_vec = Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]) - println(my_vec, "eltype ", eltype(my_vec)) - - # array = AwkwardArray.from_iter( - # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - # ) - # println(array) - - # test AwkwardArray.is_valid( - # AwkwardArray.from_iter( - # Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - # ), - # ) + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + ), + ) end @testset "from_iter # 17" begin From 01cb941bd52e8d60af6697854f4c61d3886d98dd Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 21 Feb 2024 16:16:41 +0100 Subject: [PATCH 34/52] feat: add eltype --- src/AwkwardArray.jl | 5 ---- src/all_implementations.jl | 5 ++++ test/runtests.jl | 53 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index 67bd744..c704509 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -12,11 +12,6 @@ using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} -Base.eltype(::IndexedArray{INDEX,CONTENT,BEHAVIOR}) where {INDEX,CONTENT,BEHAVIOR} = CONTENT -Base.eltype(::IndexedOptionArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT -Base.eltype(::ByteMaskedArray{INDEX,CONTENT}) where {INDEX,CONTENT} = CONTENT -Base.eltype(::BitMaskedArray{CONTENT}) where {CONTENT} = CONTENT -Base.eltype(::UnmaskedArray{CONTENT}) where {CONTENT} = CONTENT Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS end # module AwkwardArray diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 050ba04..820078a 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1447,6 +1447,7 @@ function is_valid(layout::IndexedArray) return is_valid(layout.content) end +Base.eltype(layout::IndexedArray) = eltype(layout.content) Base.length(layout::IndexedArray) = length(layout.index) Base.firstindex(layout::IndexedArray) = firstindex(layout.index) Base.lastindex(layout::IndexedArray) = lastindex(layout.index) @@ -1585,6 +1586,7 @@ function is_valid(layout::IndexedOptionArray) return is_valid(layout.content) end +Base.eltype(layout::IndexedOptionArray) = eltype(layout.content) Base.length(layout::IndexedOptionArray) = length(layout.index) Base.firstindex(layout::IndexedOptionArray) = firstindex(layout.index) Base.lastindex(layout::IndexedOptionArray) = lastindex(layout.index) @@ -1731,6 +1733,7 @@ function is_valid(layout::ByteMaskedArray) return is_valid(layout.content) end +Base.eltype(layout::ByteMaskedArray) = eltype(layout.content) Base.length(layout::ByteMaskedArray) = length(layout.mask) Base.firstindex(layout::ByteMaskedArray) = firstindex(layout.mask) Base.lastindex(layout::ByteMaskedArray) = lastindex(layout.mask) @@ -1884,6 +1887,7 @@ function is_valid(layout::BitMaskedArray) return is_valid(layout.content) end +Base.eltype(layout::BitMaskedArray) = eltype(layout.content) Base.length(layout::BitMaskedArray) = length(layout.mask) Base.firstindex(layout::BitMaskedArray) = firstindex(layout.mask) Base.lastindex(layout::BitMaskedArray) = lastindex(layout.mask) @@ -2006,6 +2010,7 @@ end is_valid(layout::UnmaskedArray) = is_valid(layout.content) +Base.eltype(layout::UnmaskedArray) = eltype(layout.content) Base.length(layout::UnmaskedArray) = length(layout.content) Base.firstindex(layout::UnmaskedArray) = firstindex(layout.content) Base.lastindex(layout::UnmaskedArray) = lastindex(layout.content) diff --git a/test/runtests.jl b/test/runtests.jl index b8cd953..5a9061d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1558,6 +1558,8 @@ end 1, ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1578,6 +1580,8 @@ end ),) @test layout_2 == AwkwardArray.copy(layout_3, length = 2) + + @test eltype(layout_2) == typeof(layout_2[1]) end begin @@ -1624,6 +1628,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ), ),) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1654,6 +1660,8 @@ end AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), ), ),) + + @test eltype(layout) == typeof(layout[1]) end end @@ -1695,6 +1703,8 @@ end @test AwkwardArray.is_valid(layout) @test AwkwardArray.to_vector(layout) == [5.5, 4.4, 4.4, 1.1, 6.6, 7.7, 0.0] + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1739,6 +1749,8 @@ end @test AwkwardArray.to_vector(layout) == [[4.4, 5.5], [1.1, 2.2, 3.3], [1.1, 2.2, 3.3], [], [6.6, 7.7], []] + + @test eltype(layout) == Vector{Float64} # FIXME: typeof(layout[1]) end begin @@ -1779,6 +1791,8 @@ end @test layout[end][:b] == 0.0 @test layout.index == [3, 4, 0, 0, 1, 2, 5, 6] @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -1823,6 +1837,8 @@ end ), )), ) + + @test eltype(layout) == typeof(layout[1]) end end @@ -1878,7 +1894,9 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [5.5, 4.4, 4.4, nothing, nothing, 1.1, 6.6, nothing, nothing, 7.7, nothing] - end + + @test eltype(layout) == typeof(layout.content[1]) + end begin layout = AwkwardArray.IndexedOptionArray{ @@ -1940,6 +1958,8 @@ end )), ), ) + + @test eltype(layout) == typeof(layout.content[1]) end end @@ -1986,7 +2006,9 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [1.1, nothing, nothing, 4.4, 5.5, 6.6, nothing, nothing] - end + + @test eltype(layout) == typeof(layout[1]) +end begin layout = AwkwardArray.ByteMaskedArray( @@ -2019,6 +2041,8 @@ end @test length(layout) == 6 @test ismissing(layout[6]) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{Float64} end begin @@ -2055,6 +2079,8 @@ end @test length(layout) == 8 @test ismissing(layout[8]) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -2098,6 +2124,8 @@ end ), valid_when = true, ) + + @test eltype(layout) == Vector{Float64} end end @@ -2144,6 +2172,8 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [1.1, nothing, nothing, 4.4, 5.5, 6.6, nothing, nothing] + + @test eltype(layout) == typeof(layout.content[1]) end begin @@ -2177,6 +2207,8 @@ end @test length(layout) == 6 @test ismissing(layout[6]) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == Vector{Float64} end begin @@ -2213,6 +2245,8 @@ end @test length(layout) == 8 @test ismissing(layout[8]) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout[1]) end begin @@ -2255,6 +2289,8 @@ end ), valid_when = true, ) + + @test eltype(layout) == Vector{Float64} end end @@ -2292,6 +2328,8 @@ end @test AwkwardArray.is_valid(layout) @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 0.0] + + @test eltype(layout) == typeof(layout.content[1]) end begin @@ -2323,6 +2361,8 @@ end @inferred layout[1] @inferred layout[1][1] @inferred layout[2:3] + + @test eltype(layout) == Vector{Float64} end begin @@ -2356,6 +2396,8 @@ end @test layout[:a][7] == 0 @test layout[:b][7] == 0.0 @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout.content[1]) end begin @@ -2385,6 +2427,8 @@ end ), valid_when = true, ) + + @test eltype(layout) == Vector{Float64} end end ### UnionArray ########################################################### @@ -2431,6 +2475,8 @@ end @inferred layout[2:4] @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, [4.4, 5.5]] + + @test eltype(layout) == typeof(layout.contents) end begin @@ -2502,6 +2548,8 @@ end ) @test AwkwardArray.is_valid(layout) + + @test eltype(layout) == typeof(layout.contents) end begin @@ -2552,6 +2600,7 @@ end ), ), ) + @test eltype(layout) == typeof(layout.contents) end end ### from_iter ############################################################ From 3343c948c2463a0c2d45aaafdbc3b2f7aa9fe478 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 21 Feb 2024 16:37:36 +0100 Subject: [PATCH 35/52] tests: cleanup --- test/runtests.jl | 182 +++++++++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 5a9061d..3131d8e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2605,115 +2605,115 @@ end end ### from_iter ############################################################ -@testset "from_iter # 1" begin -# begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, 3])) -end +@testset "from_iter" begin + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, 3])) + end -@testset "from_iter # 2" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3]])) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3]])) + end -@testset "from_iter # 3" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter(["one", "two", "three"])) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(["one", "two", "three"])) + end -@testset "from_iter # 4" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 3; 4 5 6]))) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 3; 4 5 6]))) + end -@testset "from_iter # 5" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([ - NamedTuple{(:a, :b)}((1.1, [1, 2])), - NamedTuple{(:a, :b)}((2.2, [3])), - ]), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([ + NamedTuple{(:a, :b)}((1.1, [1, 2])), + NamedTuple{(:a, :b)}((2.2, [3])), + ]), + ) + end -@testset "from_iter # 6" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), (3.3, [0])]), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), (3.3, [0])]), + ) + end -@testset "from_iter # 7" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, missing, 3])) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([1, 2, missing, 3])) + end -@testset "from_iter # 8" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3, missing]])) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], [3, missing]])) + end -@testset "from_iter # 9" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], missing, [3]])) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter([[1, 2], missing, [3]])) + end -@testset "from_iter # 10" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter(["one", "two", missing, "three"]), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter(["one", "two", missing, "three"]), + ) + end -@testset "from_iter # 11" begin - @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 missing; 4 5 6]))) -end + begin + @test AwkwardArray.is_valid(AwkwardArray.from_iter(Array([1 2 missing; 4 5 6]))) + end -@testset "from_iter # 12" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([ - NamedTuple{(:a, :b)}((1.1, [1, 2])), - missing, - NamedTuple{(:a, :b)}((2.2, [3])), - ]), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([ + NamedTuple{(:a, :b)}((1.1, [1, 2])), + missing, + NamedTuple{(:a, :b)}((2.2, [3])), + ]), + ) + end -@testset "from_iter # 13" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), missing, (3.3, [0])]), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter([(1.1, [1, 2]), (2.2, [3]), missing, (3.3, [0])]), + ) + end -@testset "from_iter # 14" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), - ), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,Vector{Int64}}}([1.1, [1, 2], [3]]), + ), + ) + end -@testset "from_iter # 15" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), - ), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Float64,String,Vector{Int64}}}([1.1, [1, 2], "hello", [3]]), + ), + ) + end -@testset "from_iter # 16" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), - ), - ) -end + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,Vector{Int64}}}([1.1, [1, 2], missing, [3]]), + ), + ) + end -@testset "from_iter # 17" begin - @test AwkwardArray.is_valid( - AwkwardArray.from_iter( - Vector{Union{Missing,Float64,String,Vector{Int64}}}([ - 1.1, - [1, 2], - "hello", - missing, - [3], - ]), - ), - ) + begin + @test AwkwardArray.is_valid( + AwkwardArray.from_iter( + Vector{Union{Missing,Float64,String,Vector{Int64}}}([ + 1.1, + [1, 2], + "hello", + missing, + [3], + ]), + ), + ) + end end -#end ### from_buffers ######################################################### From edee010305ea7817ae6addb733cb9aa627d20812 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 21 Feb 2024 16:53:57 +0100 Subject: [PATCH 36/52] feat: set empty eltype as union --- src/all_implementations.jl | 2 +- test/runtests.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 820078a..34208a4 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -307,7 +307,7 @@ Base.length(layout::EmptyArray) = 0 Base.firstindex(layout::EmptyArray) = 1 Base.lastindex(layout::EmptyArray) = 0 -Base.eltype(layout::EmptyArray) = nothing +Base.eltype(layout::EmptyArray) = Union{Float64, Nothing} Base.getindex(layout::EmptyArray, i::Int) = throw(BoundsError(layout, i)) function Base.getindex(layout::EmptyArray, r::UnitRange{Int}) diff --git a/test/runtests.jl b/test/runtests.jl index 3131d8e..3a8fd0f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -111,7 +111,7 @@ end @test AwkwardArray.to_vector(layout) == [] - @test eltype(layout) == nothing + @test eltype(layout) == Union{Nothing, Float64} end begin @@ -119,7 +119,7 @@ end @test length(layout) == 0 @test layout == AwkwardArray.EmptyArray() - @test eltype(layout) == nothing + @test eltype(layout) == Union{Nothing, Float64} end begin @@ -127,7 +127,7 @@ end append!(layout, []) append!(layout, Vector{Int64}([])) - @test eltype(layout) == nothing + @test eltype(layout) == Union{Nothing, Float64} end end From defc61a29e806e305be843c112d694f048427d9b Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 21 Feb 2024 16:56:38 +0100 Subject: [PATCH 37/52] fix: use bottom type --- src/all_implementations.jl | 2 +- test/runtests.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 34208a4..3c84f7b 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -307,7 +307,7 @@ Base.length(layout::EmptyArray) = 0 Base.firstindex(layout::EmptyArray) = 1 Base.lastindex(layout::EmptyArray) = 0 -Base.eltype(layout::EmptyArray) = Union{Float64, Nothing} +Base.eltype(layout::EmptyArray) = Union{} Base.getindex(layout::EmptyArray, i::Int) = throw(BoundsError(layout, i)) function Base.getindex(layout::EmptyArray, r::UnitRange{Int}) diff --git a/test/runtests.jl b/test/runtests.jl index 3a8fd0f..dafd896 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -111,7 +111,7 @@ end @test AwkwardArray.to_vector(layout) == [] - @test eltype(layout) == Union{Nothing, Float64} + @test eltype(layout) == Union{} end begin @@ -119,7 +119,7 @@ end @test length(layout) == 0 @test layout == AwkwardArray.EmptyArray() - @test eltype(layout) == Union{Nothing, Float64} + @test eltype(layout) == Union{} end begin @@ -127,7 +127,7 @@ end append!(layout, []) append!(layout, Vector{Int64}([])) - @test eltype(layout) == Union{Nothing, Float64} + @test eltype(layout) == Union{} end end From 6ef15d58dbfc5638978a801962538847ff03b59f Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Fri, 23 Feb 2024 17:32:53 +0100 Subject: [PATCH 38/52] Update src/all_implementations.jl --- src/all_implementations.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 3c84f7b..9102ccb 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2289,7 +2289,6 @@ end ### from_iter ############################################################ function layout_for(ItemType) - if ItemType <: Number # || ItemType <: Dates.DateTime || ItemType <: Dates.TimePeriod PrimitiveArray{ItemType} From a6946572b22a519a5859d97a7d2aa2c5b2d464ec Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Sat, 24 Feb 2024 07:03:27 +0100 Subject: [PATCH 39/52] Update src/all_implementations.jl Co-authored-by: Jim Pivarski --- src/all_implementations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 9102ccb..9aefb3e 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1586,7 +1586,7 @@ function is_valid(layout::IndexedOptionArray) return is_valid(layout.content) end -Base.eltype(layout::IndexedOptionArray) = eltype(layout.content) +Base.eltype(layout::IndexedOptionArray) = Union{Missing, eltype(layout.content)} Base.length(layout::IndexedOptionArray) = length(layout.index) Base.firstindex(layout::IndexedOptionArray) = firstindex(layout.index) Base.lastindex(layout::IndexedOptionArray) = lastindex(layout.index) From 6a93325c56aa8e4bed7208da2bbcb56b1a00bc8d Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Sat, 24 Feb 2024 07:03:42 +0100 Subject: [PATCH 40/52] Update src/all_implementations.jl Co-authored-by: Jim Pivarski --- src/all_implementations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 9aefb3e..2fd7047 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1887,7 +1887,7 @@ function is_valid(layout::BitMaskedArray) return is_valid(layout.content) end -Base.eltype(layout::BitMaskedArray) = eltype(layout.content) +Base.eltype(layout::BitMaskedArray) = Union{Missing, eltype(layout.content)} Base.length(layout::BitMaskedArray) = length(layout.mask) Base.firstindex(layout::BitMaskedArray) = firstindex(layout.mask) Base.lastindex(layout::BitMaskedArray) = lastindex(layout.mask) From f9475b1b333426a7bf7ef6c3cb1364dafbad941b Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Sat, 24 Feb 2024 07:03:52 +0100 Subject: [PATCH 41/52] Update src/all_implementations.jl Co-authored-by: Jim Pivarski --- src/all_implementations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 2fd7047..809d365 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1733,7 +1733,7 @@ function is_valid(layout::ByteMaskedArray) return is_valid(layout.content) end -Base.eltype(layout::ByteMaskedArray) = eltype(layout.content) +Base.eltype(layout::ByteMaskedArray) = Union{Missing, eltype(layout.content)} Base.length(layout::ByteMaskedArray) = length(layout.mask) Base.firstindex(layout::ByteMaskedArray) = firstindex(layout.mask) Base.lastindex(layout::ByteMaskedArray) = lastindex(layout.mask) From 051a2d9c342e2ea1d8818e082a688de7861ee965 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Sat, 24 Feb 2024 07:04:02 +0100 Subject: [PATCH 42/52] Update src/all_implementations.jl Co-authored-by: Jim Pivarski --- src/all_implementations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 809d365..2b2ba92 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2010,7 +2010,7 @@ end is_valid(layout::UnmaskedArray) = is_valid(layout.content) -Base.eltype(layout::UnmaskedArray) = eltype(layout.content) +Base.eltype(layout::UnmaskedArray) = Union{Missing, eltype(layout.content)} Base.length(layout::UnmaskedArray) = length(layout.content) Base.firstindex(layout::UnmaskedArray) = firstindex(layout.content) Base.lastindex(layout::UnmaskedArray) = lastindex(layout.content) From 6056cb0266f9e3d07c581d92474fa7c833b8db3e Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Sat, 24 Feb 2024 19:42:54 +0100 Subject: [PATCH 43/52] fix: tests --- test/runtests.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index dafd896..72e677b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1895,7 +1895,7 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [5.5, 4.4, 4.4, nothing, nothing, 1.1, 6.6, nothing, nothing, 7.7, nothing] - @test eltype(layout) == typeof(layout.content[1]) + @test eltype(layout) == Union{Missing, Float64} end begin @@ -1959,7 +1959,7 @@ end ), ) - @test eltype(layout) == typeof(layout.content[1]) + @test eltype(layout) == Union{Missing, typeof(layout.content[1])} end end @@ -2007,7 +2007,7 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [1.1, nothing, nothing, 4.4, 5.5, 6.6, nothing, nothing] - @test eltype(layout) == typeof(layout[1]) + @test eltype(layout) == Union{Missing, typeof(layout[1])} end begin @@ -2042,7 +2042,7 @@ end @test ismissing(layout[6]) @test AwkwardArray.is_valid(layout) - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end begin @@ -2080,7 +2080,7 @@ end @test ismissing(layout[8]) @test AwkwardArray.is_valid(layout) - @test eltype(layout) == typeof(layout[1]) + @test eltype(layout) == Union{Missing, typeof(layout[1])} end begin @@ -2125,7 +2125,7 @@ end valid_when = true, ) - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end end @@ -2173,7 +2173,7 @@ end @test AwkwardArray.to_vector(layout, na = nothing) == [1.1, nothing, nothing, 4.4, 5.5, 6.6, nothing, nothing] - @test eltype(layout) == typeof(layout.content[1]) + @test eltype(layout) == Union{Missing, typeof(layout.content[1])} end begin @@ -2208,7 +2208,7 @@ end @test ismissing(layout[6]) @test AwkwardArray.is_valid(layout) - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end begin @@ -2246,7 +2246,7 @@ end @test ismissing(layout[8]) @test AwkwardArray.is_valid(layout) - @test eltype(layout) == typeof(layout[1]) + @test eltype(layout) == Union{Missing, typeof(layout[1])} end begin @@ -2290,7 +2290,7 @@ end valid_when = true, ) - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end end @@ -2329,7 +2329,7 @@ end @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 0.0] - @test eltype(layout) == typeof(layout.content[1]) + @test eltype(layout) == Union{Missing, typeof(layout.content[1])} end begin @@ -2362,7 +2362,7 @@ end @inferred layout[1][1] @inferred layout[2:3] - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end begin @@ -2397,7 +2397,7 @@ end @test layout[:b][7] == 0.0 @test AwkwardArray.is_valid(layout) - @test eltype(layout) == typeof(layout.content[1]) + @test eltype(layout) == Union{Missing, typeof(layout.content[1])} end begin @@ -2428,7 +2428,7 @@ end valid_when = true, ) - @test eltype(layout) == Vector{Float64} + @test eltype(layout) == Union{Missing, Vector{Float64}} end end ### UnionArray ########################################################### From 46391cb9da577ab54239610bbf7cdc67c785ff9c Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 26 Feb 2024 11:05:41 +0100 Subject: [PATCH 44/52] feat: unionarray eltype --- src/AwkwardArray.jl | 2 +- src/all_implementations.jl | 10 ++++++++++ test/runtests.jl | 6 +++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index c704509..f770e88 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -12,6 +12,6 @@ using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} -Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = CONTENTS +Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = Union{CONTENTS...} end # module AwkwardArray diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 2b2ba92..e5a397e 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2170,10 +2170,20 @@ function is_valid(layout::UnionArray) return true end +# import Base.eltype + +# function eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} +# tuple_types = CONTENTS +# return Union{tuple_types...} +# end +Base.eltype(layout::UnionArray) = Union{typeof(layout.contents).parameters...} Base.length(layout::UnionArray) = length(layout.tags) Base.firstindex(layout::UnionArray) = firstindex(layout.tags) Base.lastindex(layout::UnionArray) = lastindex(layout.tags) +# Base.iterate(layout::UnionArray) = start(layout.contents), next(layout.contents, lastindex(layout.contents)) +# Base.iterate(layout::UnionArray, state) = state === nothing ? nothing : (layout.contents[state], state == lastindex(layout.contents) ? nothing : state + 1) + function Base.getindex(layout::UnionArray, i::Int) adjustment = firstindex(layout.tags) - firstindex(layout.index) tag = layout.tags[i] diff --git a/test/runtests.jl b/test/runtests.jl index 72e677b..bcb65f7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2476,7 +2476,7 @@ end @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, [4.4, 5.5]] - @test eltype(layout) == typeof(layout.contents) + @test eltype(layout) == Union{AwkwardArray.ListOffsetArray{Vector{Int64}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}, :default}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}} end begin @@ -2549,7 +2549,7 @@ end @test AwkwardArray.is_valid(layout) - @test eltype(layout) == typeof(layout.contents) + @test eltype(layout) == Union{AwkwardArray.ListOffsetArray{Vector{Int64}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}, :default}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}} end begin @@ -2600,7 +2600,7 @@ end ), ), ) - @test eltype(layout) == typeof(layout.contents) + @test eltype(layout) == Union{AwkwardArray.ListOffsetArray{Vector{Int64}, AwkwardArray.PrimitiveArray{Int64, Vector{Int64}, :default}, :default}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}} end end ### from_iter ############################################################ From 8e611f71d56a5da6a29c68857000daef63413ccf Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 26 Feb 2024 11:52:41 +0100 Subject: [PATCH 45/52] fix: cleanup --- src/AwkwardArray.jl | 1 - src/all_implementations.jl | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/AwkwardArray.jl b/src/AwkwardArray.jl index f770e88..a86984e 100644 --- a/src/AwkwardArray.jl +++ b/src/AwkwardArray.jl @@ -12,6 +12,5 @@ using .AwkwardPythonCallExt: convert Base.eltype(::RecordArray{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = Record{FIELDS,CONTENTS,BEHAVIOR} Base.eltype(::Record{FIELDS,CONTENTS,BEHAVIOR}) where {FIELDS,CONTENTS,BEHAVIOR} = CONTENTS Base.eltype(::TupleArray{CONTENTS,BEHAVIOR}) where {CONTENTS,BEHAVIOR} = Tuple{CONTENTS,BEHAVIOR} -Base.eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} = Union{CONTENTS...} end # module AwkwardArray diff --git a/src/all_implementations.jl b/src/all_implementations.jl index e5a397e..6924e65 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -2170,20 +2170,11 @@ function is_valid(layout::UnionArray) return true end -# import Base.eltype - -# function eltype(::UnionArray{TAGS,INDEX,CONTENTS}) where {TAGS,INDEX,CONTENTS} -# tuple_types = CONTENTS -# return Union{tuple_types...} -# end Base.eltype(layout::UnionArray) = Union{typeof(layout.contents).parameters...} Base.length(layout::UnionArray) = length(layout.tags) Base.firstindex(layout::UnionArray) = firstindex(layout.tags) Base.lastindex(layout::UnionArray) = lastindex(layout.tags) -# Base.iterate(layout::UnionArray) = start(layout.contents), next(layout.contents, lastindex(layout.contents)) -# Base.iterate(layout::UnionArray, state) = state === nothing ? nothing : (layout.contents[state], state == lastindex(layout.contents) ? nothing : state + 1) - function Base.getindex(layout::UnionArray, i::Int) adjustment = firstindex(layout.tags) - firstindex(layout.index) tag = layout.tags[i] From 4d3ae8d05b1c640a2cd609222277eebedc1de307 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 26 Feb 2024 14:37:13 +0100 Subject: [PATCH 46/52] test: add record eltype test --- test/runtests.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index bcb65f7..0cf09db 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1238,6 +1238,8 @@ end 3, ) + @test eltype(layout[3]) == Tuple{AwkwardArray.PrimitiveArray{Int64, Vector{Int64}, :default}, AwkwardArray.ListOffsetArray{Vector{Int64}, AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default}, :default}} + @test layout[3] == AwkwardArray.Record( AwkwardArray.RecordArray( NamedTuple{(:a, :b)}(( From 95ad897d18b2018b4aee727715c6647530c28050 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 26 Feb 2024 16:17:05 +0100 Subject: [PATCH 47/52] test: add more tests --- test/runtests.jl | 68 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 0cf09db..65df4f1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3134,7 +3134,7 @@ end end begin - layout = AwkwardArray.PrimitiveArray([UInt32(1), UInt32(0), UInt32(33), Int32(66)]) + layout = AwkwardArray.PrimitiveArray([UInt32(1), UInt32(0), UInt32(33), UInt32(66)]) form, len, containers = AwkwardArray.to_buffers(layout) @test JSON.parse(form) == JSON.parse( @@ -3148,6 +3148,56 @@ end ) end + begin + layout = AwkwardArray.PrimitiveArray([UInt64(1), UInt64(0), UInt64(33), UInt64(66)]) + form, len, containers = AwkwardArray.to_buffers(layout) + + @test JSON.parse(form) == JSON.parse( + """{"class": "NumpyArray", "primitive": "uint64", "inner_shape": [], "parameters": {}, "form_key": "node0"}""", + ) + @test len == 4 + @test containers == Dict{String,Vector{UInt8}}( + "node0-data" => Vector{UInt8}( + b"\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x21\x00\x00\x00\x00\x00\x00\x00\x42\x00\x00\x00\x00\x00\x00\x00", + ), + ) + end + + begin + layout = AwkwardArray.PrimitiveArray([Int128(1), Int128(2), Int128(3), Int128(4), Int128(5)]) + @test_throws ErrorException AwkwardArray.to_buffers(layout) + end + + begin + layout = AwkwardArray.PrimitiveArray([Float16(1.1), Float16(2.2), Float16(3.3), Float16(4.4), Float16(5.5)]) + form, len, containers = AwkwardArray.to_buffers(layout) + + @test JSON.parse(form) == JSON.parse( + """{"class": "NumpyArray", "primitive": "float16", "inner_shape": [], "parameters": {}, "form_key": "node0"}""", + ) + @test len == 5 + @test containers == Dict{String,Vector{UInt8}}( + "node0-data" => Vector{UInt8}( + b"\x66\x3c\x66\x40\x9a\x42\x66\x44\x80\x45", + ), + ) + end + + begin + layout = AwkwardArray.PrimitiveArray([Float32(1.1), Float32(2.2), Float32(3.3), Float32(4.4), Float32(5.5)]) + form, len, containers = AwkwardArray.to_buffers(layout) + + @test JSON.parse(form) == JSON.parse( + """{"class": "NumpyArray", "primitive": "float32", "inner_shape": [], "parameters": {}, "form_key": "node0"}""", + ) + @test len == 5 + @test containers == Dict{String,Vector{UInt8}}( + "node0-data" => Vector{UInt8}( + b"\xcd\xcc\x8c\x3f\xcd\xcc\x0c\x40\x33\x33\x53\x40\xcd\xcc\x8c\x40\x00\x00\xb0\x40", + ), + ) + end + begin layout = AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]) form, len, containers = AwkwardArray.to_buffers(layout) @@ -3179,7 +3229,21 @@ end ), ) end - + + begin + layout = AwkwardArray.PrimitiveArray([Complex{Float32}(1,1), Complex{Float32}(0,0.2), Complex{Float32}(-3.3 ,0.1), Complex{Float32}(66,0)]) + form, len, containers = AwkwardArray.to_buffers(layout) + @test JSON.parse(form) == JSON.parse( + """{"class": "NumpyArray", "primitive": "complex64", "inner_shape": [], "parameters": {}, "form_key": "node0"}""", + ) + @test len == 4 + @test containers == Dict{String,Vector{UInt8}}( + "node0-data" => Vector{UInt8}( + b"\x00\x00\x80\x3f\x00\x00\x80\x3f\x00\x00\x00\x00\xcd\xcc\x4c\x3e\x33\x33\x53\xc0\xcd\xcc\xcc\x3d\x00\x00\x84\x42\x00\x00\x00\x00", + ), + ) + end + begin layout = AwkwardArray.PrimitiveArray([1+1im, 0+0.2im, -3.3 + 0.1im, 66]) form, len, containers = AwkwardArray.to_buffers(layout) From 44d196c0b4e51df5d51dcfd1f618054523b656ad Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Mon, 26 Feb 2024 18:46:52 +0100 Subject: [PATCH 48/52] test: add more test coverage --- src/all_implementations.jl | 11 ++++-- test/runtests.jl | 74 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 6924e65..5d49a41 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -293,10 +293,17 @@ end ### EmptyArray ########################################################### struct EmptyArray{BEHAVIOR} <: LeafType{BEHAVIOR} - EmptyArray(; behavior::Symbol = :default) = new{behavior}() + behavior::Symbol + + function EmptyArray(; behavior::Symbol = :default) + new{behavior}(behavior) + end end -copy(behavior::Union{Unset,Symbol} = Unset()) = EmptyArray(behavior = behavior) +function copy(behavior::Union{Unset,Symbol} = Unset()) + behavior = behavior isa Unset ? :default : behavior + return EmptyArray(behavior = behavior) +end parameters_of(content::EmptyArray) = Parameters() has_parameter(content::EmptyArray, key::String) = false diff --git a/test/runtests.jl b/test/runtests.jl index 65df4f1..6a53c73 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,7 @@ using Tables @test AwkwardArray.to_vector(layout) == [1.1, 2.2, 3.3, 4.4, 5.5] @test eltype(layout) == typeof(layout[1]) + @test !(layout[1:2] == layout[2:5]) end begin @@ -92,6 +93,15 @@ using Tables @test eltype(layout) == typeof(layout[1]) end + + begin + # Test with default parameters + @test AwkwardArray.PrimitiveArray{Int, Vector{Int}}() == AwkwardArray.PrimitiveArray(Vector{Int}([])) + + # Test with custom parameters and behavior + params = AwkwardArray.Parameters("param1" => 1, "param2" => "abc") + @test AwkwardArray.PrimitiveArray{Float64, Vector{Float64}}(parameters=params, behavior=:custom) == AwkwardArray.PrimitiveArray(Vector{Float64}([]), parameters=params, behavior=:custom) + end end ### EmptyArray ########################################################### @@ -128,6 +138,37 @@ end append!(layout, Vector{Int64}([])) @test eltype(layout) == Union{} + + # Test the EmptyArray constructor + @test AwkwardArray.EmptyArray().behavior == :default + @test AwkwardArray.EmptyArray(behavior=:custom).behavior == :custom + + # Test the copy function + @test AwkwardArray.copy().behavior == :default + @test AwkwardArray.copy(:custom).behavior == :custom + + # Test the parameters_of function + @test AwkwardArray.parameters_of(AwkwardArray.EmptyArray()) == AwkwardArray.Parameters() + @test AwkwardArray.parameters_of(AwkwardArray.EmptyArray(behavior=:custom)) == AwkwardArray.Parameters() + + # Test the has_parameter function + @test !AwkwardArray.has_parameter(AwkwardArray.EmptyArray(), "key") + @test !AwkwardArray.has_parameter(AwkwardArray.EmptyArray(behavior=:custom), "key") + + # Test the get_parameter function + @test AwkwardArray.get_parameter(AwkwardArray.EmptyArray(), "key") == nothing + @test AwkwardArray.get_parameter(AwkwardArray.EmptyArray(behavior=:custom), "key") == nothing + + # Test the getindex function + @test_throws BoundsError getindex(AwkwardArray.EmptyArray(), 1) + @test_throws BoundsError getindex(AwkwardArray.EmptyArray(behavior=:custom), 1) + @test_throws BoundsError getindex(AwkwardArray.EmptyArray(), 1:5) + @test_throws BoundsError getindex(AwkwardArray.EmptyArray(behavior=:custom), 1:5) + + # Test the push! function + @test_throws ErrorException("attempting to fill AwkwardArray.EmptyArray{:default} with data") push!(AwkwardArray.EmptyArray(), 1) + @test_throws ErrorException("attempting to fill AwkwardArray.EmptyArray{:custom} with data") push!(AwkwardArray.EmptyArray(behavior=:custom), 1) + @test_throws ErrorException("attempting to fill AwkwardArray.EmptyArray{:default} with data") push!(AwkwardArray.EmptyArray(), "data") end end @@ -257,6 +298,39 @@ end ) @test eltype(layout) == Vector{eltype(layout.content)} + + # # Define the :some_function symbol + # some_function = :some_function + + # # Define a dictionary with a function as one of the values + # my_dict = Dict( + # :add => (+), + # :subtract => (-), + # :multiply => (*), + # some_function => x -> x^2 # Define a custom function for :some_function + # ) + + # # Test getindex method for the :some_function symbol + # result = layout[my_dict[:some_function]] + + # # Check if the content of the result matches the expected content + # @test result.content == AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5, 6, 7, 8, 9])[:some_function] + end + + begin + # Create a valid ListOffsetArray + content = [1, 2, 3, 4, 5, 6, 7, 8, 9] + layout_valid = AwkwardArray.ListOffsetArray([0, 2, 5, 9], AwkwardArray.PrimitiveArray(content)) + + # Test for a valid ListOffsetArray + @test AwkwardArray.is_valid(layout_valid) == true + + # Create an invalid ListOffsetArray (with negative offset) + offsets_invalid = [-1, 2, 5, 9] + layout_invalid = AwkwardArray.ListOffsetArray([-1, 2, 5, 9], AwkwardArray.PrimitiveArray(content)) + + # Test for an invalid ListOffsetArray + @test AwkwardArray.is_valid(layout_invalid) == false end end From f74515aa58a61d8fb1c620d219a638863c92e906 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 28 Feb 2024 14:45:53 +0100 Subject: [PATCH 49/52] test: more tests and fixes --- Project.toml | 2 +- src/all_implementations.jl | 49 +++++++++++++++++++------ test/runtests.jl | 75 +++++++++++++++++++++++++++++++------- 3 files changed, 100 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index b5455c0..04f2652 100644 --- a/Project.toml +++ b/Project.toml @@ -11,9 +11,9 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] JSON = "0.21.4" +PythonCall = "0.9" Tables = "1.11.1" julia = "1.9" -PythonCall = "0.9" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 5d49a41..c3504fb 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -418,8 +418,11 @@ end Base.getindex(layout::ListOffsetArray, r::UnitRange{Int}) = copy(layout, offsets = layout.offsets[(r.start):(r.stop+1)]) -Base.getindex(layout::ListOffsetArray, f::Symbol) = +# Define the getindex method for ListOffsetArray +function Base.getindex(layout::ListOffsetArray, f::Symbol) + @assert typeof(layout.content) <: RecordArray "content must be of type RecordArray" copy(layout, content = layout.content[f]) +end function end_list!(layout::ListOffsetArray) push!(layout.offsets, length(layout.content)) @@ -540,7 +543,11 @@ function Base.getindex(layout::ListArray, r::UnitRange{Int}) ) end -Base.getindex(layout::ListArray, f::Symbol) = copy(layout, content = layout.content[f]) +# Define the getindex method for ListArray +function Base.getindex(layout::ListArray, f::Symbol) + @assert typeof(layout.content) <: RecordArray "content must be of type RecordArray" + copy(layout, content = layout.content[f]) +end function end_list!(layout::ListArray) if isempty(layout.stops) @@ -686,7 +693,10 @@ function Base.getindex(layout::RegularArray, r::UnitRange{Int}) copy(layout, content = layout.content[start:stop], zeros_length = r.stop - r.start + 1) end -Base.getindex(layout::RegularArray, f::Symbol) = copy(layout, content = layout.content[f]) +function Base.getindex(layout::RegularArray, f::Symbol) + @assert typeof(layout.content) <: RecordArray "content must be of type RecordArray" + copy(layout, content = layout.content[f]) +end function end_list!(layout::RegularArray) if layout.size < 0 && layout.length == 0 @@ -1301,19 +1311,35 @@ Base.length(layout::TupleArray) = layout.length Base.firstindex(layout::TupleArray) = 1 Base.lastindex(layout::TupleArray) = layout.length -Base.getindex( +function Base.getindex( layout::TupleArray{CONTENTS,BEHAVIOR}, i::Int, -) where {CONTENTS<:Base.Tuple{Vararg{Content}},BEHAVIOR} = Tuple(layout, i) +) where {CONTENTS<:Base.Tuple{Vararg{Content}},BEHAVIOR} + println("getindex with Int called") + println("Contents of layout: ", layout.contents) + println("Index: ", i) + println("Length: ", layout.length) + + result = Tuple(layout, i) + + println("Result: ", result) + return result +end Base.getindex( layout::TupleArray{CONTENTS,BEHAVIOR}, r::UnitRange{Int}, -) where {VALUES<:Content,CONTENTS<:Base.Tuple{VALUES},BEHAVIOR} = copy( - layout, - contents = Base.Tuple{VALUES}(x[r] for x in layout.contents), - length = min(r.stop, layout.length) - max(r.start, 1) + 1, # unnecessary min/max -) +) where {VALUES<:Content,CONTENTS<:Base.Tuple{VALUES},BEHAVIOR} = + copy( + layout, + contents = Base.Tuple{VALUES}(x[r] for x in layout.contents), + length = min(r.stop, layout.length) - max(r.start, 1) + 1, # unnecessary min/max + ) +# Base.getindex(layout::TupleArray, r::UnitRange{Int},) = copy( +# layout, +# contents = Base.Tuple(x[r] for x in layout.contents), +# length = min(r.stop, layout.length) - max(r.start, 1) + 1, +# ) function slot( layout::TupleArray{CONTENTS,BEHAVIOR}, @@ -1326,7 +1352,8 @@ end Base.getindex( layout::Tuple{CONTENTS}, f::Int64, -) where {CONTENTS<:Base.Tuple{Vararg{Content}}} = layout.array.contents[f][layout.at] +) where {CONTENTS<:Base.Tuple{Vararg{Content}}} = + layout.array.contents[f][layout.at] function Base.:(==)( layout1::TupleArray{CONTENTS1}, diff --git a/test/runtests.jl b/test/runtests.jl index 6a53c73..fc00ef0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -298,23 +298,21 @@ end ) @test eltype(layout) == Vector{eltype(layout.content)} + end - # # Define the :some_function symbol - # some_function = :some_function - - # # Define a dictionary with a function as one of the values - # my_dict = Dict( - # :add => (+), - # :subtract => (-), - # :multiply => (*), - # some_function => x -> x^2 # Define a custom function for :some_function - # ) + begin + content_layout = AwkwardArray.RecordArray( + ( + a = AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5]), + b = AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), + ) + ) - # # Test getindex method for the :some_function symbol - # result = layout[my_dict[:some_function]] + layout = AwkwardArray.ListOffsetArray([1, 2, 5], content_layout) - # # Check if the content of the result matches the expected content - # @test result.content == AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5, 6, 7, 8, 9])[:some_function] + @test layout[:a] == [[2], [3, 4, 5]] + @test_throws ErrorException getindex(layout, :invalid) + @test_throws AssertionError getindex(layout[:a], :invalid) end begin @@ -368,6 +366,17 @@ end @test eltype(layout) == Vector{eltype(layout.content)} end + + begin + # Create a ListArray with default parameters + list_array = AwkwardArray.ListArray{Vector{Int}, AwkwardArray.PrimitiveArray{Float64}, :default}() + + # Check that the ListArray is created correctly + @test length(list_array.starts) == 0 + @test length(list_array.stops) == 0 + @test length(list_array.content) == 0 + @test typeof(list_array.content) == AwkwardArray.PrimitiveArray{Float64, Vector{Float64}, :default} + end begin layout = AwkwardArray.ListArray( @@ -466,6 +475,22 @@ end @test eltype(layout) == Vector{eltype(layout.content)} end + + begin + content_layout = AwkwardArray.RecordArray( + ( + a = AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5]), + b = AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), + ) + ) + + + layout = AwkwardArray.ListArray([1, 2, 5], [2, 5, 5], content_layout) + + @test layout[:a] == [[2], [3, 4, 5], []] + @test_throws ErrorException getindex(layout, :invalid) + @test_throws AssertionError getindex(layout[:a], :invalid) + end end ### RegularArray ######################################################### @@ -670,6 +695,21 @@ end @test eltype(layout) == Vector{eltype(layout.content)} end + + begin + content_layout = AwkwardArray.RecordArray( + ( + a = AwkwardArray.PrimitiveArray([1, 2, 3, 4, 5]), + b = AwkwardArray.PrimitiveArray([1.1, 2.2, 3.3, 4.4, 5.5]), + ) + ) + + layout = AwkwardArray.RegularArray(content_layout, 2) + + @test layout[:a] == [[1, 2], [3, 4]] + @test_throws ErrorException getindex(layout, :invalid) + @test_throws AssertionError getindex(layout[:a], :invalid) + end end ### ListType with behavior = :string ##################################### @@ -1496,6 +1536,13 @@ end @test tmp == 16.5 @test eltype(layout) == typeof(layout[1]) + + # Test getindex with a unit range + @test layout[1:2] == AwkwardArray.TupleArray(( + AwkwardArray.PrimitiveArray([1, 2]), + AwkwardArray.PrimitiveArray([1.1, 2.2]), + ),) + # FIXME: MethodError: no method matching to_vector(::Vector{Any}) in test AwkwardArray.to_vector(layout[1:2]) == [(1, 1.1), (2, 2.2)] end begin From b6d3a1de3f6448d5bea295e22d2b028e192c3f55 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 28 Feb 2024 14:51:37 +0100 Subject: [PATCH 50/52] cleanup: remove debug printout --- src/all_implementations.jl | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index c3504fb..20e81b5 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1311,20 +1311,10 @@ Base.length(layout::TupleArray) = layout.length Base.firstindex(layout::TupleArray) = 1 Base.lastindex(layout::TupleArray) = layout.length -function Base.getindex( +Base.getindex( layout::TupleArray{CONTENTS,BEHAVIOR}, i::Int, -) where {CONTENTS<:Base.Tuple{Vararg{Content}},BEHAVIOR} - println("getindex with Int called") - println("Contents of layout: ", layout.contents) - println("Index: ", i) - println("Length: ", layout.length) - - result = Tuple(layout, i) - - println("Result: ", result) - return result -end +) where {CONTENTS<:Base.Tuple{Vararg{Content}},BEHAVIOR} = Tuple(layout, i) Base.getindex( layout::TupleArray{CONTENTS,BEHAVIOR}, From a70f26f3eb0b6214781b55e8e832c4463655cdf4 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 28 Feb 2024 18:19:32 +0100 Subject: [PATCH 51/52] test: more tests --- test/runtests.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index fc00ef0..a1b28ac 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -52,6 +52,8 @@ using Tables @test AwkwardArray.is_valid(layout) @test eltype(layout) == typeof(layout[1]) + + @test AwkwardArray.to_vector(layout[1:2]) == [1.1, 2.2] end begin From 6769ac14f931fe79b38bccf5e908c2f2d5104bd4 Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Wed, 28 Feb 2024 18:33:46 +0100 Subject: [PATCH 52/52] cleanup --- src/all_implementations.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/all_implementations.jl b/src/all_implementations.jl index 20e81b5..c0d398b 100644 --- a/src/all_implementations.jl +++ b/src/all_implementations.jl @@ -1325,11 +1325,6 @@ Base.getindex( contents = Base.Tuple{VALUES}(x[r] for x in layout.contents), length = min(r.stop, layout.length) - max(r.start, 1) + 1, # unnecessary min/max ) -# Base.getindex(layout::TupleArray, r::UnitRange{Int},) = copy( -# layout, -# contents = Base.Tuple(x[r] for x in layout.contents), -# length = min(r.stop, layout.length) - max(r.start, 1) + 1, -# ) function slot( layout::TupleArray{CONTENTS,BEHAVIOR},