Merge branch 'main' into ioanaif/passing-behaviour

.all-contributorsrc

@@ -293,6 +293,15 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "Ahmad-AlSubaie",
+      "name": "Ahmad-AlSubaie",
+      "avatar_url": "https://avatars.githubusercontent.com/u/32343365?v=4",
+      "profile": "https://github.com/Ahmad-AlSubaie",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

CMakeLists.txt

@@ -16,6 +16,8 @@ project(
 message(STATUS "CMake version ${CMAKE_VERSION}")
 message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 
+include(CMakeDependentOption)
+
 # Defaults for properties in this directory (and below)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -93,10 +95,16 @@ if(BUILD_TESTING)
 endif()
 
 option(PYBUILD "Build Python modules")
+cmake_dependent_option(AWKWARD_EXTERNAL_PYBIND11 "Build against an external pybind11" OFF
+                       "PYBUILD" OFF)
 
 # Third tier: Python modules.
 if(PYBUILD)
-  add_subdirectory(pybind11)
+  if(AWKWARD_EXTERNAL_PYBIND11)
+    find_package(pybind11 CONFIG REQUIRED)
+  else()
+    add_subdirectory(pybind11)
+  endif()
 
   file(GLOB LAYOUT_SOURCES "src/python/*.cpp")
   pybind11_add_module(_ext ${LAYOUT_SOURCES})

README.md

@@ -291,6 +291,7 @@ Thanks especially to the gracious help of Awkward Array contributors (including
     <td align="center"><a href="http://www.matthewfeickert.com/"><img src="https://avatars.githubusercontent.com/u/5142394?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Matthew Feickert</b></sub></a><br /><a href="#maintenance-matthewfeickert" title="Maintenance">🚧</a></td>
     <td align="center"><a href="https://github.com/SantamRC"><img src="https://avatars.githubusercontent.com/u/52635773?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Santam Roy Choudhury</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward-1.0/commits?author=SantamRC" title="Tests">⚠️</a></td>
     <td align="center"><a href="http://jeroen.vangoey.be"><img src="https://avatars.githubusercontent.com/u/59344?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Jeroen Van Goey</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward-1.0/commits?author=BioGeek" title="Documentation">📖</a></td>
+    <td align="center"><a href="https://github.com/Ahmad-AlSubaie"><img src="https://avatars.githubusercontent.com/u/32343365?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ahmad-AlSubaie</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward-1.0/commits?author=Ahmad-AlSubaie" title="Code">💻</a></td>
   </tr>
 </table>
 

src/awkward/_connect/_numba/builder.py

@@ -20,9 +20,7 @@ def globalstring(context, builder, pyvalue):
         buf = dynamic_addrs[pyvalue] = numpy.array(pyvalue.encode("utf-8") + b"\x00")
         context.add_dynamic_addr(builder, buf.ctypes.data, info=f"str({pyvalue!r})")
     ptr = context.get_constant(numba.types.uintp, dynamic_addrs[pyvalue].ctypes.data)
-    return builder.inttoptr(
-        ptr, llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8))
-    )
+    return builder.inttoptr(ptr, llvmlite.ir.PointerType(llvmlite.ir.IntType(8)))
 
 
 class ArrayBuilderType(numba.types.Type):

src/awkward/_v2/_connect/numba/builder.py

@@ -18,9 +18,7 @@ def globalstring(context, builder, pyvalue):
         buf = dynamic_addrs[pyvalue] = numpy.array(pyvalue.encode("utf-8") + b"\x00")
         context.add_dynamic_addr(builder, buf.ctypes.data, info=f"str({repr(pyvalue)})")
     ptr = context.get_constant(numba.types.uintp, dynamic_addrs[pyvalue].ctypes.data)
-    return builder.inttoptr(
-        ptr, llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8))
-    )
+    return builder.inttoptr(ptr, llvmlite.ir.PointerType(llvmlite.ir.IntType(8)))
 
 
 class ArrayBuilderType(numba.types.Type):

src/awkward/_v2/behaviors/string.py

@@ -167,7 +167,7 @@ def _string_numba_lower(
     context, builder, rettype, viewtype, viewval, viewproxy, attype, atval
 ):
     import numba
-    import llvmlite.llvmpy.core
+    import llvmlite.ir
 
     whichpos = ak._v2._connect.numba.layout.posat(
         context, builder, viewproxy.pos, viewtype.type.CONTENT
@@ -213,9 +213,7 @@ def _string_numba_lower(
     )
     rawptr_cast = builder.inttoptr(
         rawptr,
-        llvmlite.llvmpy.core.Type.pointer(
-            llvmlite.llvmpy.core.Type.int(numba.intp.bitwidth // 8)
-        ),
+        llvmlite.ir.PointerType(llvmlite.ir.IntType(numba.intp.bitwidth // 8)),
     )
     strsize = builder.sub(stop, start)
     strsize_cast = ak._v2._connect.numba.layout.castint(

src/awkward/_v2/contents/bitmaskedarray.py

@@ -634,11 +634,12 @@ def _to_list(self, behavior):
             return out
 
         mask = self.mask_as_bool(valid_when=True, nplike=self.nplike)[: self._length]
-        content = self._content._to_list(behavior)
-        out = [None] * self._length
+        out = self._content._getitem_range(slice(0, self._length))._to_list(behavior)
+
         for i, isvalid in enumerate(mask):
-            if isvalid:
-                out[i] = content[i]
+            if not isvalid:
+                out[i] = None
+
         return out
 
     def _to_nplike(self, nplike):

src/awkward/_v2/contents/bytemaskedarray.py

@@ -1035,11 +1035,12 @@ def _to_list(self, behavior):
             return out
 
         mask = self.mask_as_bool(valid_when=True, nplike=self.nplike)
-        content = self._content._to_list(behavior)
-        out = [None] * self._mask.length
+        out = self._content._getitem_range(slice(0, len(mask)))._to_list(behavior)
+
         for i, isvalid in enumerate(mask):
-            if isvalid:
-                out[i] = content[i]
+            if not isvalid:
+                out[i] = None
+
         return out
 
     def _to_nplike(self, nplike):

src/awkward/_v2/contents/indexedarray.py

@@ -1221,11 +1221,8 @@ def _to_list(self, behavior):
             return out
 
         index = self._index.raw(numpy)
-        content = self._content._to_list(behavior)
-        out = [None] * index.length
-        for i, ind in enumerate(index):
-            out[i] = content[ind]
-        return out
+        nextcontent = self._content._carry(ak._v2.index.Index(index), False)
+        return nextcontent._to_list(behavior)
 
     def _to_nplike(self, nplike):
         index = self._index._to_nplike(nplike)

src/awkward/_v2/contents/indexedoptionarray.py

@@ -1653,11 +1653,17 @@ def _to_list(self, behavior):
             return out
 
         index = self._index.raw(numpy)
-        content = self._content._to_list(behavior)
-        out = [None] * len(index)
-        for i, ind in enumerate(index):
-            if ind >= 0:
-                out[i] = content[ind]
+        not_missing = index >= 0
+
+        nextcontent = self._content._carry(
+            ak._v2.index.Index(index[not_missing]), False
+        )
+        out = nextcontent._to_list(behavior)
+
+        for i, isvalid in enumerate(not_missing):
+            if not isvalid:
+                out.insert(i, None)
+
         return out
 
     def _to_nplike(self, nplike):

src/awkward/_v2/contents/listoffsetarray.py

@@ -2062,33 +2062,48 @@ def packed(self):
         )
 
     def _to_list(self, behavior):
+        starts, stops = self.starts, self.stops
+        starts_data = starts.raw(numpy)
+        stops_data = stops.raw(numpy)[: len(starts_data)]
+
+        nonempty = starts_data != stops_data
+        if numpy.count_nonzero(nonempty) == 0:
+            mini, maxi = 0, 0
+        else:
+            mini = starts_data.min()
+            maxi = stops_data.max()
+
+        starts_data = starts_data - mini
+        stops_data = stops_data - mini
+
+        nextcontent = self._content._getitem_range(slice(mini, maxi))
+
         if self.parameter("__array__") == "bytestring":
-            content = ak._v2._util.tobytes(self._content.data)
-            starts, stops = self.starts, self.stops
+            content = ak._v2._util.tobytes(nextcontent.data)
             out = [None] * starts.length
             for i in range(starts.length):
-                out[i] = content[starts[i] : stops[i]]
+                out[i] = content[starts_data[i] : stops_data[i]]
             return out
 
         elif self.parameter("__array__") == "string":
-            content = ak._v2._util.tobytes(self._content.data)
-            starts, stops = self.starts, self.stops
+            content = ak._v2._util.tobytes(nextcontent.data)
             out = [None] * starts.length
             for i in range(starts.length):
-                out[i] = content[starts[i] : stops[i]].decode(errors="surrogateescape")
+                out[i] = content[starts_data[i] : stops_data[i]].decode(
+                    errors="surrogateescape"
+                )
             return out
 
         else:
             out = self._to_list_custom(behavior)
             if out is not None:
                 return out
 
-            content = self._content._to_list(behavior)
-            starts, stops = self.starts, self.stops
+            content = nextcontent._to_list(behavior)
             out = [None] * starts.length
 
             for i in range(starts.length):
-                out[i] = content[starts[i] : stops[i]]
+                out[i] = content[starts_data[i] : stops_data[i]]
             return out
 
     def _to_nplike(self, nplike):

src/awkward/_v2/highlevel.py

@@ -437,6 +437,13 @@ def fields(self):
         """
         return self._layout.fields
 
+    @property
+    def is_tuple(self):
+        """
+        If True, the top-most record structure has no named fields, i.e. it's a tuple.
+        """
+        return self._layout.is_tuple
+
     def _ipython_key_completions_(self):
         return self._layout.fields
 
@@ -533,7 +540,7 @@ def __iter__(self):
                             errors="surrogateescape"
                         )
                     else:
-                        yield x
+                        yield ak._v2._util.wrap(x, self._behavior)
                 elif isinstance(x, (ak._v2.contents.Content, ak._v2.record.Record)):
                     yield ak._v2._util.wrap(x, self._behavior)
                 else:
@@ -1580,6 +1587,13 @@ def fields(self):
         """
         return self._layout.array.fields
 
+    @property
+    def is_tuple(self):
+        """
+        If True, the top-most record structure has no named fields, i.e. it's a tuple.
+        """
+        return self._layout.array.is_tuple
+
     def _ipython_key_completions_(self):
         return self._layout.array.fields
 

src/awkward/behaviors/string.py

@@ -182,7 +182,7 @@ def _string_numba_lower(
     context, builder, rettype, viewtype, viewval, viewproxy, attype, atval
 ):
     import numba
-    import llvmlite.llvmpy.core
+    import llvmlite.ir
 
     whichpos = ak._connect._numba.layout.posat(
         context, builder, viewproxy.pos, viewtype.type.CONTENT
@@ -228,9 +228,7 @@ def _string_numba_lower(
     )
     rawptr_cast = builder.inttoptr(
         rawptr,
-        llvmlite.llvmpy.core.Type.pointer(
-            llvmlite.llvmpy.core.Type.int(numba.intp.bitwidth // 8)
-        ),
+        llvmlite.ir.PointerType(llvmlite.ir.IntType(numba.intp.bitwidth // 8)),
     )
     strsize = builder.sub(stop, start)
     strsize_cast = ak._connect._numba.castint(