Convert Arrow <--> Parquet, and hence Awkward <--> Parquet. (#343)

* Convert Arrow <--> Parquet, and hence Awkward <--> Parquet.

* Completely implmented Parquet reading and writing, with tests of eagerness and laziness. Added 'metadata' on ak.Array and ak.Record (for VirtualArray cache).

* Add old OAMap Parquet samples and remember to pytest.importorskip.

* Add an 'explode_records' option to make it easier to write structured data.

src/awkward1/_util.py

@@ -334,16 +334,16 @@ def behaviorof(*arrays):
     return behavior
 
 
-def wrap(content, behavior):
+def wrap(content, behavior, metadata=None):
     import awkward1.highlevel
 
     if isinstance(
         content, (awkward1.layout.Content, awkward1.partition.PartitionedArray)
     ):
-        return awkward1.highlevel.Array(content, behavior=behavior)
+        return awkward1.highlevel.Array(content, behavior=behavior, metadata=metadata)
 
     elif isinstance(content, awkward1.layout.Record):
-        return awkward1.highlevel.Record(content, behavior=behavior)
+        return awkward1.highlevel.Record(content, behavior=behavior, metadata=metadata)
 
     else:
         return content

src/awkward1/highlevel.py

@@ -7,8 +7,10 @@
 
 try:
     from collections.abc import Iterable
+    from collections.abc import MutableMapping
 except ImportError:
     from collections import Iterable
+    from collections import MutableMapping
 
 import numpy
 
@@ -38,6 +40,9 @@ class Array(
         with_name (None or str): Gives tuples and records a name that can be
             used to override their behavior (see below).
         check_valid (bool): If True, verify that the #layout is valid.
+        metadata (None or MutableMapping): Optional metadata, attached to this
+            array, which persists through `__getitem__` but not any other
+            operations.
 
     High-level array that can contain data of any type.
 
@@ -201,7 +206,9 @@ class Array(
     derivatives elementwise on each numeric value in an ak.Array.
     """
 
-    def __init__(self, data, behavior=None, with_name=None, check_valid=False):
+    def __init__(
+        self, data, behavior=None, with_name=None, check_valid=False, metadata=None
+    ):
         if isinstance(
             data, (awkward1.layout.Content, awkward1.partition.PartitionedArray)
         ):
@@ -240,6 +247,8 @@ def __init__(self, data, behavior=None, with_name=None, check_valid=False):
         if check_valid:
             awkward1.operations.describe.validity_error(self, exception=True)
 
+        self.metadata = metadata
+
     @property
     def layout(self):
         """
@@ -320,6 +329,17 @@ def behavior(self, behavior):
         else:
             raise TypeError("behavior must be None or a dict")
 
+    @property
+    def metadata(self):
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, value):
+        if value is None or isinstance(value, MutableMapping):
+            self._metadata = value
+        else:
+            raise TypeError("metadata must be None or a MutableMapping")
+
     class Mask(object):
         def __init__(self, array, valid_when):
             self._array = array
@@ -857,7 +877,9 @@ def __getitem__(self, where):
         acting at the last level, while the higher levels of the indexer all
         have the same dimension as the array being indexed.
         """
-        return awkward1._util.wrap(self._layout[where], self._behavior)
+        return awkward1._util.wrap(
+            self._layout[where], self._behavior, metadata=self._metadata
+        )
 
     def __setitem__(self, where, what):
         """
@@ -1332,6 +1354,9 @@ class Record(awkward1._connect._numpy.NDArrayOperatorsMixin):
         with_name (None or str): Gives the record type a name that can be
             used to override its behavior (see below).
         check_valid (bool): If True, verify that the #layout is valid.
+        metadata (None or MutableMapping): Optional metadata, attached to this
+            array, which persists through `__getitem__` but not any other
+            operations.
 
     High-level record that can contain fields of any type.
 
@@ -1347,7 +1372,9 @@ class Record(awkward1._connect._numpy.NDArrayOperatorsMixin):
     See also #ak.Array and #ak.behavior.
     """
 
-    def __init__(self, data, behavior=None, with_name=None, check_valid=False):
+    def __init__(
+        self, data, behavior=None, with_name=None, check_valid=False, metadata=None
+    ):
         if isinstance(data, awkward1.layout.Record):
             layout = data
         elif isinstance(data, Record):
@@ -1379,6 +1406,8 @@ def __init__(self, data, behavior=None, with_name=None, check_valid=False):
         if check_valid:
             awkward1.operations.describe.validity_error(self, exception=True)
 
+        self.metadata = metadata
+
     @property
     def layout(self):
         """
@@ -1453,6 +1482,17 @@ def behavior(self, behavior):
         else:
             raise TypeError("behavior must be None or a dict")
 
+    @property
+    def metadata(self):
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, value):
+        if value is None or isinstance(value, MutableMapping):
+            self._metadata = value
+        else:
+            raise TypeError("metadata must be None or a MutableMapping")
+
     def tolist(self):
         """
         Converts this Record into Python objects.
@@ -1562,7 +1602,9 @@ def __getitem__(self, where):
             >>> record["y", 1]
             2
         """
-        return awkward1._util.wrap(self._layout[where], self._behavior)
+        return awkward1._util.wrap(
+            self._layout[where], self._behavior, metadata=self._metadata
+        )
 
     def __setitem__(self, where, what):
         """