Skip to content

Commit

Permalink
Add generator @rule integration, with support for Geting products…
Browse files Browse the repository at this point in the history 
… for subjects.
  • Loading branch information
@stuhood
stuhood committed Mar 29, 2018
1 parent 2ecc367 commit 2a0356a
Show file tree
Hide file tree
Showing 9 changed files with 299 additions and 132 deletions.
1 change: 1 addition & 0 deletions src/python/pants/engine/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ python_library(
sources=['native.py'],
dependencies=[
':native_engine_shared_library',
':selectors',
'3rdparty/python:cffi',
'3rdparty/python:setuptools',
'src/python/pants/binaries:binary_util',
Expand Down
125 changes: 87 additions & 38 deletions src/python/pants/engine/native.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import pkg_resources
import six

from pants.engine.selectors import Get, constraint_for
from pants.util.contextutil import temporary_dir
from pants.util.dirutil import safe_mkdir, safe_mkdtemp
from pants.util.memo import memoized_property
Expand Down Expand Up @@ -83,6 +84,12 @@
Value value;
} PyResult;
typedef struct {
uint8_t tag;
ValueBuffer values;
ValueBuffer constraints;
} PyGeneratorResponse;
typedef struct {
int64_t hash_;
Value value;
Expand All @@ -92,25 +99,26 @@
typedef void ExternContext;
// On the rust side the integration is defined in externs.rs
typedef void (*extern_ptr_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
typedef uint8_t extern_log_level;
typedef Ident (*extern_ptr_identify)(ExternContext*, Value*);
typedef _Bool (*extern_ptr_equals)(ExternContext*, Value*, Value*);
typedef Value (*extern_ptr_clone_val)(ExternContext*, Value*);
typedef void (*extern_ptr_drop_handles)(ExternContext*, Handle*, uint64_t);
typedef Buffer (*extern_ptr_type_to_str)(ExternContext*, TypeId);
typedef Buffer (*extern_ptr_val_to_str)(ExternContext*, Value*);
typedef _Bool (*extern_ptr_satisfied_by)(ExternContext*, Value*, Value*);
typedef _Bool (*extern_ptr_satisfied_by_type)(ExternContext*, Value*, TypeId*);
typedef Value (*extern_ptr_store_list)(ExternContext*, Value**, uint64_t, _Bool);
typedef Value (*extern_ptr_store_bytes)(ExternContext*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_store_i32)(ExternContext*, int32_t);
typedef Value (*extern_ptr_project)(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
typedef ValueBuffer (*extern_ptr_project_multi)(ExternContext*, Value*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_project_ignoring_type)(ExternContext*, Value*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_create_exception)(ExternContext*, uint8_t*, uint64_t);
typedef PyResult (*extern_ptr_call)(ExternContext*, Value*, Value*, uint64_t);
typedef PyResult (*extern_ptr_eval)(ExternContext*, uint8_t*, uint64_t);
typedef void (*extern_ptr_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
typedef uint8_t extern_log_level;
typedef Ident (*extern_ptr_identify)(ExternContext*, Value*);
typedef _Bool (*extern_ptr_equals)(ExternContext*, Value*, Value*);
typedef Value (*extern_ptr_clone_val)(ExternContext*, Value*);
typedef void (*extern_ptr_drop_handles)(ExternContext*, Handle*, uint64_t);
typedef Buffer (*extern_ptr_type_to_str)(ExternContext*, TypeId);
typedef Buffer (*extern_ptr_val_to_str)(ExternContext*, Value*);
typedef _Bool (*extern_ptr_satisfied_by)(ExternContext*, Value*, Value*);
typedef _Bool (*extern_ptr_satisfied_by_type)(ExternContext*, Value*, TypeId*);
typedef Value (*extern_ptr_store_list)(ExternContext*, Value**, uint64_t, _Bool);
typedef Value (*extern_ptr_store_bytes)(ExternContext*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_store_i32)(ExternContext*, int32_t);
typedef Value (*extern_ptr_project)(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
typedef ValueBuffer (*extern_ptr_project_multi)(ExternContext*, Value*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_project_ignoring_type)(ExternContext*, Value*, uint8_t*, uint64_t);
typedef Value (*extern_ptr_create_exception)(ExternContext*, uint8_t*, uint64_t);
typedef PyResult (*extern_ptr_call)(ExternContext*, Value*, Value*, uint64_t);
typedef PyGeneratorResponse (*extern_ptr_generator_send)(ExternContext*, Value*, Value*);
typedef PyResult (*extern_ptr_eval)(ExternContext*, uint8_t*, uint64_t);
typedef void Tasks;
typedef void Scheduler;
Expand All @@ -136,6 +144,7 @@
extern_ptr_log,
extern_log_level,
extern_ptr_call,
extern_ptr_generator_send,
extern_ptr_eval,
extern_ptr_identify,
extern_ptr_equals,
Expand Down Expand Up @@ -189,6 +198,7 @@
TypeConstraint,
TypeConstraint,
TypeConstraint,
TypeConstraint,
TypeId,
TypeId,
Buffer,
Expand Down Expand Up @@ -226,24 +236,25 @@

CFFI_EXTERNS = '''
extern "Python" {
void extern_log(ExternContext*, uint8_t, uint8_t*, uint64_t);
PyResult extern_call(ExternContext*, Value*, Value*, uint64_t);
PyResult extern_eval(ExternContext*, uint8_t*, uint64_t);
Ident extern_identify(ExternContext*, Value*);
_Bool extern_equals(ExternContext*, Value*, Value*);
Value extern_clone_val(ExternContext*, Value*);
void extern_drop_handles(ExternContext*, Handle*, uint64_t);
Buffer extern_type_to_str(ExternContext*, TypeId);
Buffer extern_val_to_str(ExternContext*, Value*);
_Bool extern_satisfied_by(ExternContext*, Value*, Value*);
_Bool extern_satisfied_by_type(ExternContext*, Value*, TypeId*);
Value extern_store_list(ExternContext*, Value**, uint64_t, _Bool);
Value extern_store_bytes(ExternContext*, uint8_t*, uint64_t);
Value extern_store_i32(ExternContext*, int32_t);
Value extern_project(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
Value extern_project_ignoring_type(ExternContext*, Value*, uint8_t*, uint64_t);
ValueBuffer extern_project_multi(ExternContext*, Value*, uint8_t*, uint64_t);
Value extern_create_exception(ExternContext*, uint8_t*, uint64_t);
void extern_log(ExternContext*, uint8_t, uint8_t*, uint64_t);
PyResult extern_call(ExternContext*, Value*, Value*, uint64_t);
PyGeneratorResponse extern_generator_send(ExternContext*, Value*, Value*);
PyResult extern_eval(ExternContext*, uint8_t*, uint64_t);
Ident extern_identify(ExternContext*, Value*);
_Bool extern_equals(ExternContext*, Value*, Value*);
Value extern_clone_val(ExternContext*, Value*);
void extern_drop_handles(ExternContext*, Handle*, uint64_t);
Buffer extern_type_to_str(ExternContext*, TypeId);
Buffer extern_val_to_str(ExternContext*, Value*);
_Bool extern_satisfied_by(ExternContext*, Value*, Value*);
_Bool extern_satisfied_by_type(ExternContext*, Value*, TypeId*);
Value extern_store_list(ExternContext*, Value**, uint64_t, _Bool);
Value extern_store_bytes(ExternContext*, uint8_t*, uint64_t);
Value extern_store_i32(ExternContext*, int32_t);
Value extern_project(ExternContext*, Value*, uint8_t*, uint64_t, TypeId*);
Value extern_project_ignoring_type(ExternContext*, Value*, uint8_t*, uint64_t);
ValueBuffer extern_project_multi(ExternContext*, Value*, uint8_t*, uint64_t);
Value extern_create_exception(ExternContext*, uint8_t*, uint64_t);
}
'''

Expand Down Expand Up @@ -459,6 +470,41 @@ def extern_create_exception(context_handle, msg_ptr, msg_len):
msg = to_py_str(msg_ptr, msg_len)
return c.to_value(Exception(msg))

@ffi.def_extern()
def extern_generator_send(context_handle, func, arg):
"""Given a generator, send it the given value and return a response."""
c = ffi.from_handle(context_handle)
try:
res = c.from_value(func).send(c.from_value(arg))
if isinstance(res, Get):
# Get.
values = [res.subject]
constraints = [constraint_for(res.product)]
tag = 2
elif type(res) in (tuple, list):
# GetMulti.
values = [g.subject for g in res]
constraints = [constraint_for(g.product) for g in res]
tag = 3
else:
# Break.
values = [res]
constraints = []
tag = 0
except Exception as e:
# Throw.
val = e
val._formatted_exc = traceback.format_exc()
values = [val]
constraints = []
tag = 1

return (
tag,
c.vals_buf([c.to_value(v) for v in values]),
c.vals_buf([c.to_value(v) for v in constraints])
)

@ffi.def_extern()
def extern_call(context_handle, func, args_ptr, args_len):
"""Given a callable, call it."""
Expand Down Expand Up @@ -632,6 +678,7 @@ def init_externs():
self.ffi_lib.extern_log,
logger.getEffectiveLevel(),
self.ffi_lib.extern_call,
self.ffi_lib.extern_generator_send,
self.ffi_lib.extern_eval,
self.ffi_lib.extern_identify,
self.ffi_lib.extern_equals,
Expand Down Expand Up @@ -705,7 +752,8 @@ def new_scheduler(self,
constraint_file,
constraint_link,
constraint_process_request,
constraint_process_result):
constraint_process_result,
constraint_generator):
"""Create and return an ExternContext and native Scheduler."""

def func(constraint):
Expand Down Expand Up @@ -738,6 +786,7 @@ def tc(constraint):
tc(constraint_link),
tc(constraint_process_request),
tc(constraint_process_result),
tc(constraint_generator),
# Types.
TypeId(self.context.to_id(six.text_type)),
TypeId(self.context.to_id(six.binary_type)),
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/engine/scheduler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import os
import time
from collections import defaultdict
from types import GeneratorType

from pants.base.exceptions import TaskError
from pants.base.project_tree import Dir, File, Link
Expand Down Expand Up @@ -111,6 +112,7 @@ def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index):
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(ExecuteProcessResult),
constraint_for(GeneratorType),
)

def _root_type_ids(self):
Expand Down
21 changes: 20 additions & 1 deletion src/python/pants/engine/selectors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,30 @@ def constraint_for(type_or_constraint):
raise TypeError("Expected a type or constraint: got: {}".format(type_or_constraint))


class Get(datatype('Get', ['product', 'subject'])):
"""TODO: Experimental synchronous generator API.
May be called equivalently as either:
# verbose form: Get(product_type, subject_type, subject)
# shorthand form: Get(product_type, subject_type(subject))
"""

def __new__(cls, *args):
if len(args) == 2:
product, subject = args
elif len(args) == 3:
product, _, subject = args
else:
raise Exception('Expected either two or three arguments to {}; got {}.'.format(
Get.__name__, args))
return super(Get, cls).__new__(cls, product, subject)


class Selector(AbstractClass):
# The type constraint for the product type for this selector.

@property
def type_constraint(self):
"""The type constraint for the product type for this selector."""
return constraint_for(self.product)

@abstractproperty
Expand Down
11 changes: 3 additions & 8 deletions src/rust/engine/fs/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,19 +697,14 @@ pub trait VFS<E: Send + Sync + 'static>: Clone + Send + Sync + 'static {

// If there were any new PathGlobs, continue the expansion.
if expansion.todo.is_empty() {
future::Loop::Break(expansion)
future::Loop::Break(expansion.outputs)
} else {
future::Loop::Continue(expansion)
}
})
}).map(|expansion| {
assert!(
expansion.todo.is_empty(),
"Loop shouldn't have exited with work to do: {:?}",
expansion.todo,
);
}).map(|expansion_outputs| {
// Finally, capture the resulting PathStats from the expansion.
expansion.outputs.into_iter().map(|(k, _)| k).collect()
expansion_outputs.into_iter().map(|(k, _)| k).collect()
})
.to_boxed()
}
Expand Down
Loading

0 comments on commit 2a0356a

Please sign in to comment.