Merge pull request #400 from scikit-hep/issue-399

Added interpretation of vector<class*>

tests/test_issues.py

@@ -240,3 +240,9 @@ def test_issue390(self):
         t = uproot.open("tests/samples/issue390.root")["E"]
         t.pandas.df("hits.*")
         t.pandas.df("trks.*")
+
+    def test_issue399(self):
+        t = uproot.open("tests/samples/issue399.root")["Event"]
+        a = t["Histos.histograms1D"].array()
+        for i in range(t.numentries):
+            assert [x.title for x in a[i]] == [b"Primary Hits", b"Primary Loss", b"Energy Loss", b"Primary Hits per Element", b"Primary Loss per Element", b"Energy Loss per Element"]

uproot/__init__.py

@@ -142,6 +142,7 @@
 from uproot.interp.objects import STLVector
 from uproot.interp.objects import STLMap
 from uproot.interp.objects import STLString
+from uproot.interp.objects import Pointer
 asdebug = asjagged(asdtype("u1"))
 
 from uproot import pandas
@@ -157,4 +158,4 @@
 # don't expose uproot.uproot; it's ugly
 del uproot
 
-__all__ = ["open", "xrootd", "http", "iterate", "numentries", "lazyarray", "lazyarrays", "daskarray", "daskframe", "create", "recreate", "update", "ZLIB", "LZMA", "LZ4", "newtree", "newbranch", "MemmapSource", "FileSource", "XRootDSource", "HTTPSource", "ArrayCache", "ThreadSafeArrayCache", "interpret", "asdtype", "asarray", "asdouble32", "asstlbitset", "asjagged", "astable", "asobj", "asgenobj", "asstring", "asdebug", "SimpleArray", "STLVector", "STLMap", "STLString", "pandas", "__version__"]
+__all__ = ["open", "xrootd", "http", "iterate", "numentries", "lazyarray", "lazyarrays", "daskarray", "daskframe", "create", "recreate", "update", "ZLIB", "LZMA", "LZ4", "newtree", "newbranch", "MemmapSource", "FileSource", "XRootDSource", "HTTPSource", "ArrayCache", "ThreadSafeArrayCache", "interpret", "asdtype", "asarray", "asdouble32", "asstlbitset", "asjagged", "astable", "asobj", "asgenobj", "asstring", "asdebug", "SimpleArray", "STLVector", "STLMap", "STLString", "Pointer", "pandas", "__version__"]

uproot/_help.py

@@ -1913,7 +1913,7 @@ def _method(x):
     **source_numitems(self, source)**
         calculate the number of "items" given a ``source`` instance.
 
-    **fromroot(self, data, offsets, local_entrystart, local_entrystop)**
+    **fromroot(self, data, offsets, local_entrystart, local_entrystop, keylen)**
         produce a source from one basket ``data`` array (dtype ``numpy.uint8``) and its corresponding ``offsets`` array (dtype **numpy.int32** or ``None`` if not present) that has *n + 1* elements for *n* entries: ``offsets[0] == 0 and offsets[-1] == numentries``. The ``local_entrystart`` and ``local_entrystop`` are entry start (inclusive) and stop (exclusive), in which the first entry in the basket is number zero (hence "local"). The result of this operation may be a zero-copy cast of the basket data.
 
     **destination(self, numitems, numentries)**

uproot/interp/auto.py

@@ -25,6 +25,7 @@
 from uproot.interp.objects import STLVector
 from uproot.interp.objects import STLMap
 from uproot.interp.objects import STLString
+from uproot.interp.objects import Pointer
 
 class _NotNumerical(Exception): pass
 
@@ -380,6 +381,11 @@ def transform(node, tofloat=True):
                     return asjagged(asdtype("f8"), skipbytes=10)
                 elif getattr(branch._streamer, "_fTypeName", None) == b"vector<string>":
                     return asgenobj(STLVector(STLString(awkward)), branch._context, 6)
+                else:
+                    m = interpret._vectorpointer.match(getattr(branch._streamer, "_fTypeName", b""))
+                    if m is not None and m.group(1) in branch._context.streamerinfosmap:
+                        streamer = branch._context.streamerinfosmap[m.group(1)]
+                        return asgenobj(STLVector(Pointer(streamer.pyclass)), branch._context, skipbytes=6)
 
                 if getattr(branch._streamer, "_fTypeName", None) == b"map<string,bool>" or getattr(branch._streamer, "_fTypeName", None) == b"map<string,Bool_t>":
                     return asgenobj(STLMap(STLString(awkward), asdtype(awkward.numpy.bool_)), branch._context, 6)
@@ -550,4 +556,5 @@ def transform(node, tofloat=True):
 interpret._titlehasdims = re.compile(br"^([^\[\]]+)(\[[^\[\]]+\])+")
 interpret._itemdimpattern = re.compile(br"\[([1-9][0-9]*)\]")
 interpret._itemanypattern = re.compile(br"\[(.*)\]")
+interpret._vectorpointer = re.compile(br"vector\<([^<>]*)\*\>")
 interpret._pairsecond = re.compile(br"pair\<[^<>]*,(.*) \>")

uproot/interp/interp.py

@@ -41,7 +41,7 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         raise NotImplementedError
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
         raise NotImplementedError
 
     def destination(self, numitems, numentries):

uproot/interp/jagged.py

@@ -66,15 +66,15 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         return self.content.source_numitems(source.content)
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
         if local_entrystart == local_entrystop:
-            return self.awkward.JaggedArray.fromoffsets([0], self.content.fromroot(data, None, local_entrystart, local_entrystop))
+            return self.awkward.JaggedArray.fromoffsets([0], self.content.fromroot(data, None, local_entrystart, local_entrystop, keylen))
         else:
             if self.skipbytes == 0:
                 offsets = _destructive_divide(byteoffsets, self.content.itemsize, self.awkward)
                 starts  = offsets[local_entrystart     : local_entrystop    ]
                 stops   = offsets[local_entrystart + 1 : local_entrystop + 1]
-                content = self.content.fromroot(data, None, starts[0], stops[-1])
+                content = self.content.fromroot(data, None, starts[0], stops[-1], keylen)
                 return self.awkward.JaggedArray(starts, stops, content)
 
             else:
@@ -87,7 +87,7 @@ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
                 self.awkward.numpy.cumsum(mask, out=mask)
                 data = data[mask.view(self.awkward.numpy.bool_)]
 
-                content = self.content.fromroot(data, None, 0, bytestops[-1])
+                content = self.content.fromroot(data, None, 0, bytestops[-1], keylen)
 
                 itemsize = 1
                 sub = self.content

uproot/interp/numerical.py

@@ -152,7 +152,7 @@ def numitems(self, numbytes, numentries):
         assert remainder == 0
         return quotient
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
         dtype, shape = _dtypeshape(self.fromdtype)
         return data.view(dtype).reshape((-1,) + shape)[local_entrystart:local_entrystop]
 
@@ -269,7 +269,7 @@ def numitems(self, numbytes, numentries):
         assert remainder == 0
         return quotient
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
         # Interpret input data using proper type
         array = data.view(dtype=self.fromdtypeflat)
         # Make sure the interpreted data has correct shape
@@ -359,7 +359,7 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         return int(self.awkward.numpy.prod(source.shape))
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
         return data.view(self.todtype).reshape((-1, self.numbytes + 4))[:, 4:]
 
     def destination(self, numitems, numentries):

uproot/interp/objects.py

@@ -6,7 +6,11 @@
 
 import copy
 import struct
+import numbers
 
+import numpy
+
+import uproot.rootio
 import uproot.interp.interp
 import uproot.interp.numerical
 import uproot.interp.jagged
@@ -129,6 +133,25 @@ def read(self, source, cursor, context, parent):
             numitems = cursor.field(source, self._format2)
         return cursor.array(source, numitems, self.awkward.ObjectArray.CHARTYPE).tostring()
 
+class Pointer(object):
+    def __init__(self, cls):
+        self.cls = cls
+
+    @property
+    def __name__(self):
+        return "Pointer"
+
+    def __repr__(self):
+        if isinstance(self.cls, type):
+            return "Pointer({0})".format(self.cls.__name__)
+        else:
+            return "Pointer({0})".format(repr(self.cls))
+
+    def read(self, source, cursor, context, parent):
+        return uproot.rootio._readobjany(source, cursor, context, parent)
+
+    _format1 = struct.Struct(">II")
+
 class astable(uproot.interp.interp.Interpretation):
     # makes __doc__ attribute mutable before Python 3.3
     __metaclass__ = type.__new__(type, "type", (uproot.interp.interp.Interpretation.__metaclass__,), {})
@@ -180,8 +203,8 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         return self.content.source_numitems(source)
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
-        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
     def destination(self, numitems, numentries):
         return self.content.destination(numitems, numentries)
@@ -233,8 +256,8 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         return self.content.source_numitems(source)
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
-        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
     def destination(self, numitems, numentries):
         return self.content.destination(numitems, numentries)
@@ -286,14 +309,14 @@ def numitems(self, numbytes, numentries):
     def source_numitems(self, source):
         return self.content.source_numitems(source)
 
-    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
-        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+        return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
     def destination(self, numitems, numentries):
         return self.content.destination(numitems, numentries)
 
     def fill(self, source, destination, itemstart, itemstop, entrystart, entrystop):
-        return self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
+        self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
 
     def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
         return self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
@@ -304,17 +327,55 @@ def finalize(self, destination, branch):
             print("reading {0}".format(repr(out)))
         return out
 
-class asgenobj(_variable):
+class _variable_withoffsets(_variable):
+    def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+        out = self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
+        out.byteoffsets = byteoffsets[local_entrystart:local_entrystop] + keylen + self.content.skipbytes
+        return out
+
+    def destination(self, numitems, numentries):
+        out = self.content.destination(numitems, numentries)
+        out.byteoffsets = self.awkward.numpy.empty(numentries, dtype=self.awkward.numpy.int32)
+        return out
+
+    def fill(self, source, destination, itemstart, itemstop, entrystart, entrystop):
+        self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
+        destination.byteoffsets[entrystart:entrystop] = source.byteoffsets
+
+    def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
+        out = self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
+        out.byteoffsets = destination.byteoffsets[entrystart:entrystop]
+        return out
+
+    def finalize(self, destination, branch):
+        out = self.awkward.ObjectArray(JaggedWithByteOffsets(self.content.finalize(destination, branch), destination.byteoffsets), self.generator, *self.args, **self.kwargs)
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
+        return out
+
+class JaggedWithByteOffsets(object):
+    def __init__(self, jagged, byteoffsets):
+        self.jagged = jagged
+        self.byteoffsets = byteoffsets
+
+    def __len__(self):
+        return len(self.jagged)
+
+    def __getitem__(self, where):
+        return self.jagged[where], -self.byteoffsets[where]
+
+class asgenobj(_variable_withoffsets):
     # makes __doc__ attribute mutable before Python 3.3
     __metaclass__ = type.__new__(type, "type", (_variable.__metaclass__,), {})
 
     class _Wrapper(object):
         def __init__(self, cls, context):
             self.cls = cls
             self.context = context
-        def __call__(self, bytes):
+        def __call__(self, arg):
+            bytes, origin = arg
             source = uproot.source.source.Source(bytes)
-            cursor = uproot.source.cursor.Cursor(0)
+            cursor = uproot.source.cursor.Cursor(0, origin=origin)
             return self.cls.read(source, cursor, self.context, None)
         def __repr__(self):
             if isinstance(self.cls, type):

uproot/tree.py

@@ -1194,7 +1194,7 @@ def _basket(self, i, interpretation, local_entrystart, local_entrystop, awkward,
             byteoffsets[-1] = key._fLast
             awkward.numpy.subtract(byteoffsets, key._fKeylen, byteoffsets)
 
-        return interpretation.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+        return interpretation.fromroot(data, byteoffsets, local_entrystart, local_entrystop, key._fKeylen)
 
     def basket(self, i, interpretation=None, entrystart=None, entrystop=None, flatten=False, awkwardlib=None, cache=None, basketcache=None, keycache=None):
         awkward = _normalize_awkwardlib(awkwardlib)

uproot/version.py

@@ -6,7 +6,7 @@
 
 import re
 
-__version__ = "3.10.10"
+__version__ = "3.10.11"
 version = __version__
 version_info = tuple(re.split(r"[-\.]", __version__))