Engine can load bytes

src/python/pants/engine/native.py

@@ -327,6 +327,15 @@ def extern_type_to_str(self, context_handle, type_id):
     c = self._ffi.from_handle(context_handle)
     return c.utf8_buf(text_type(c.from_id(type_id.tup_0).__name__))
 
+  @_extern_decl('Buffer', ['ExternContext*', 'Handle*'])
+  def extern_val_to_bytes(self, context_handle, val):
+    """Given a Handle for `obj`, write bytes(obj) and return it."""
+    c = self._ffi.from_handle(context_handle)
+    v = c.from_value(val[0])
+    # Consistently use the empty string to indicate None.
+    v_bytes = b'' if v is None else binary_type(v)
+    return c.buf(v_bytes)
+
   @_extern_decl('Buffer', ['ExternContext*', 'Handle*'])
   def extern_val_to_str(self, context_handle, val):
     """Given a Handle for `obj`, write str(obj) and return it."""
@@ -667,6 +676,7 @@ def init_externs():
                            self.ffi_lib.extern_equals,
                            self.ffi_lib.extern_clone_val,
                            self.ffi_lib.extern_drop_handles,
+                           self.ffi_lib.extern_val_to_bytes,
                            self.ffi_lib.extern_type_to_str,
                            self.ffi_lib.extern_val_to_str,
                            self.ffi_lib.extern_store_tuple,

src/rust/engine/src/externs.rs

@@ -145,6 +145,19 @@ pub fn project_multi(value: &Value, field: &str) -> Vec<Value> {
   })
 }
 
+pub fn project_bytes(value: &Value, field: &str) -> Vec<u8> {
+  let named_val = with_externs(|e| {
+    (e.project_ignoring_type)(
+      e.context,
+      value as &Handle,
+      field.as_ptr(),
+      field.len() as u64,
+    )
+    .into()
+  });
+  val_to_bytes(&named_val)
+}
+
 pub fn project_multi_strs(item: &Value, field: &str) -> Vec<String> {
   project_multi(item, field)
     .iter()
@@ -185,6 +198,10 @@ pub fn val_to_str(val: &Value) -> String {
   })
 }
 
+pub fn val_to_bytes(val: &Value) -> Vec<u8> {
+  with_externs(|e| (e.val_to_bytes)(e.context, val as &Handle).to_bytes())
+}
+
 pub fn create_exception(msg: &str) -> Value {
   with_externs(|e| (e.create_exception)(e.context, msg.as_ptr(), msg.len() as u64).into())
 }
@@ -331,6 +348,7 @@ pub struct Externs {
   pub store_bool: StoreBoolExtern,
   pub project_ignoring_type: ProjectIgnoringTypeExtern,
   pub project_multi: ProjectMultiExtern,
+  pub val_to_bytes: ValToBytesExtern,
   pub type_to_str: TypeToStrExtern,
   pub val_to_str: ValToStrExtern,
   pub create_exception: CreateExceptionExtern,
@@ -659,6 +677,8 @@ impl BufferBuffer {
   }
 }
 
+pub type ValToBytesExtern = extern "C" fn(*const ExternContext, *const Handle) -> Buffer;
+
 pub type TypeToStrExtern = extern "C" fn(*const ExternContext, TypeId) -> Buffer;
 
 pub type ValToStrExtern = extern "C" fn(*const ExternContext, *const Handle) -> Buffer;

src/rust/engine/src/lib.rs

@@ -70,7 +70,7 @@ use crate::externs::{
   EqualsExtern, ExternContext, Externs, GeneratorSendExtern, GetTypeForExtern, HandleBuffer,
   IdentifyExtern, ProjectIgnoringTypeExtern, ProjectMultiExtern, PyResult, RawBuffer,
   StoreBoolExtern, StoreBytesExtern, StoreF64Extern, StoreI64Extern, StoreTupleExtern,
-  StoreUtf8Extern, TypeIdBuffer, TypeToStrExtern, ValToStrExtern,
+  StoreUtf8Extern, TypeIdBuffer, TypeToStrExtern, ValToBytesExtern, ValToStrExtern,
 };
 use crate::handles::Handle;
 use crate::rule_graph::{GraphMaker, RuleGraph};
@@ -118,6 +118,7 @@ pub extern "C" fn externs_set(
   equals: EqualsExtern,
   clone_val: CloneValExtern,
   drop_handles: DropHandlesExtern,
+  val_to_bytes: ValToBytesExtern,
   type_to_str: TypeToStrExtern,
   val_to_str: ValToStrExtern,
   store_tuple: StoreTupleExtern,
@@ -143,6 +144,7 @@ pub extern "C" fn externs_set(
     equals,
     clone_val,
     drop_handles,
+    val_to_bytes,
     type_to_str,
     val_to_str,
     store_tuple,