Merge branch 'master' of github.com:pantsbuild/pants into py2-wheels-…

…abi-specified

examples/src/wire/org/pantsbuild/example/element/BUILD

@@ -3,10 +3,12 @@
 
 java_wire_library(
   sources=[
-    'elements.proto', # Order matters here.
+    # NB: Order matters for these two paths, so we set `ordered_sources=True` below.
+    'elements.proto',
     'compound.proto',
   ],
   dependencies=[
     'examples/src/wire/org/pantsbuild/example/temperature',
   ],
+  ordered_sources=True,
 )

src/python/pants/backend/codegen/wire/java/java_wire_library.py

@@ -32,6 +32,7 @@ def __init__(self,
                registry_class=None,
                enum_options=None,
                no_options=None,
+               ordered_sources=None,
                **kwargs):
     """
     :param string service_writer: the name of the class to pass as the --service_writer option to
@@ -43,6 +44,9 @@ def __init__(self,
     doubt, specify com.squareup.wire.SimpleServiceWriter
     :param list enum_options: list of enums to pass to as the --enum-enum_options option, # optional
     :param boolean no_options: boolean that determines if --no_options flag is passed
+    :param boolean ordered_sources: boolean that declares whether the sources argument represents
+      literal ordered sources to be passed directly to the compiler. If false, no ordering is
+      guaranteed for the sources passed to an individual compiler invoke.
     """
 
     if not service_writer and service_writer_options:
@@ -59,6 +63,7 @@ def __init__(self,
       'registry_class': PrimitiveField(registry_class or None),
       'enum_options': PrimitiveField(enum_options or []),
       'no_options': PrimitiveField(no_options or False),
+      'ordered_sources': PrimitiveField(ordered_sources or False),
     })
 
     super(JavaWireLibrary, self).__init__(payload=payload, **kwargs)

src/python/pants/backend/codegen/wire/java/wire_gen.py

@@ -13,10 +13,12 @@
 from pants.backend.jvm.targets.java_library import JavaLibrary
 from pants.backend.jvm.tasks.nailgun_task import NailgunTaskBase
 from pants.base.build_environment import get_buildroot
-from pants.base.exceptions import TaskError
+from pants.base.exceptions import TargetDefinitionException, TaskError
 from pants.base.workunit import WorkUnitLabel
 from pants.java.jar.jar_dependency import JarDependency
+from pants.source.filespec import globs_matches
 from pants.task.simple_codegen_task import SimpleCodegenTask
+from pants.util.dirutil import fast_relpath
 
 
 logger = logging.getLogger(__name__)
@@ -61,24 +63,47 @@ def synthetic_target_extra_dependencies(self, target, target_workdir):
     wire_runtime_deps_spec = self.get_options().javadeps
     return self.resolve_deps([wire_runtime_deps_spec])
 
-  def format_args_for_target(self, target, target_workdir):
-    """Calculate the arguments to pass to the command line for a single target."""
-    sources = OrderedSet(target.sources_relative_to_buildroot())
-
+  def _compute_sources(self, target):
     relative_sources = OrderedSet()
-    source_roots = set()
-    for source in sources:
+    source_roots = OrderedSet()
+
+    def capture_and_relativize_to_source_root(source):
       source_root = self.context.source_roots.find_by_path(source)
       if not source_root:
         source_root = self.context.source_roots.find(target)
       source_roots.add(source_root.path)
-      relative_source = os.path.relpath(source, source_root.path)
-      relative_sources.add(relative_source)
+      return fast_relpath(source, source_root.path)
+
+    if target.payload.get_field_value('ordered_sources'):
+      # Re-match the filespecs against the sources in order to apply them in the literal order
+      # they were specified in.
+      filespec = target.globs_relative_to_buildroot()
+      excludes = filespec.get('excludes', [])
+      for filespec in filespec.get('globs', []):
+        sources = [s for s in target.sources_relative_to_buildroot()
+                   if globs_matches([s], [filespec], excludes)]
+        if len(sources) != 1:
+          raise TargetDefinitionException(
+              target,
+              'With `ordered_sources=True`, expected one match for each file literal, '
+              'but got: {} for literal `{}`.'.format(sources, filespec)
+            )
+        relative_sources.add(capture_and_relativize_to_source_root(sources[0]))
+    else:
+      # Otherwise, use the default (unspecified) snapshot ordering.
+      for source in target.sources_relative_to_buildroot():
+        relative_sources.add(capture_and_relativize_to_source_root(source))
+    return relative_sources, source_roots
+
+  def format_args_for_target(self, target, target_workdir):
+    """Calculate the arguments to pass to the command line for a single target."""
 
     args = ['--java_out={0}'.format(target_workdir)]
 
     # Add all params in payload to args
 
+    relative_sources, source_roots = self._compute_sources(target)
+
     if target.payload.get_field_value('no_options'):
       args.append('--no_options')
 

src/python/pants/backend/graph_info/tasks/cloc.py

@@ -40,7 +40,7 @@ def console_output(self, targets):
     input_snapshots = tuple(
       target.sources_snapshot(scheduler=self.context._scheduler) for target in targets
     )
-    input_files = {f.path for snapshot in input_snapshots for f in snapshot.files}
+    input_files = {f for snapshot in input_snapshots for f in snapshot.files}
 
     # TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
     with temporary_dir() as tmpdir:

src/python/pants/backend/jvm/tasks/jvm_compile/javac/javac_compile.py

@@ -209,8 +209,8 @@ def _execute_hermetic_compile(self, cmd, ctx):
       # Assume no extra .class files to grab. We'll fix up that case soon.
       # Drop the source_root from the file path.
       # Assumes `-d .` has been put in the command.
-      os.path.relpath(f.path.replace('.java', '.class'), ctx.target.target_base)
-      for f in input_snapshot.files if f.path.endswith('.java')
+      os.path.relpath(f.replace('.java', '.class'), ctx.target.target_base)
+      for f in input_snapshot.files if f.endswith('.java')
     )
     exec_process_request = ExecuteProcessRequest(
       argv=tuple(cmd),

src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py

@@ -33,8 +33,7 @@
 from pants.java.jar.jar_dependency import JarDependency
 from pants.reporting.reporting_utils import items_to_report_element
 from pants.util.contextutil import Timer
-from pants.util.dirutil import (fast_relpath, fast_relpath_optional, maybe_read_file,
-                                safe_file_dump, safe_mkdir)
+from pants.util.dirutil import fast_relpath, fast_relpath_optional, safe_mkdir
 from pants.util.memo import memoized_property
 
 
@@ -60,22 +59,6 @@ def stdout_contents(wu):
     return f.read().rstrip()
 
 
-def write_digest(output_dir, digest):
-  safe_file_dump(
-    '{}.digest'.format(output_dir),
-    mode='w',
-    payload='{}:{}'.format(digest.fingerprint, digest.serialized_bytes_length))
-
-
-def load_digest(output_dir):
-  read_file = maybe_read_file('{}.digest'.format(output_dir), binary_mode=False)
-  if read_file:
-    fingerprint, length = read_file.split(':')
-    return Digest(fingerprint, int(length))
-  else:
-    return None
-
-
 def _create_desandboxify_fn(possible_path_patterns):
   # Takes a collection of possible canonical prefixes, and returns a function that
   # if it finds a matching prefix, strips the path prior to the prefix and returns it
@@ -132,7 +115,7 @@ def __init__(self, *args, **kwargs):
 
   @classmethod
   def implementation_version(cls):
-    return super(RscCompile, cls).implementation_version() + [('RscCompile', 171)]
+    return super(RscCompile, cls).implementation_version() + [('RscCompile', 172)]
 
   @classmethod
   def register_options(cls, register):
@@ -201,11 +184,11 @@ def _nailgunnable_combined_classpath(self):
 
   # Overrides the normal zinc compiler classpath, which only contains zinc.
   def get_zinc_compiler_classpath(self):
-    return self.do_for_execution_strategy_variant({
+    return self.execution_strategy_enum.resolve_for_enum_variant({
       self.HERMETIC: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
       self.SUBPROCESS: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
       self.NAILGUN: lambda: self._nailgunnable_combined_classpath,
-    })
+    })()
 
   def register_extra_products_from_contexts(self, targets, compile_contexts):
     super(RscCompile, self).register_extra_products_from_contexts(targets, compile_contexts)
@@ -218,7 +201,7 @@ def pathglob_for(filename):
     def to_classpath_entries(paths, scheduler):
       # list of path ->
       # list of (path, optional<digest>) ->
-      path_and_digests = [(p, load_digest(os.path.dirname(p))) for p in paths]
+      path_and_digests = [(p, Digest.load(os.path.dirname(p))) for p in paths]
       # partition: list of path, list of tuples
       paths_without_digests = [p for (p, d) in path_and_digests if not d]
       if paths_without_digests:
@@ -825,7 +808,7 @@ def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input
       raise TaskError(res.stderr)
 
     if output_dir:
-      write_digest(output_dir, res.output_directory_digest)
+      res.output_directory_digest.dump(output_dir)
       self.context._scheduler.materialize_directories((
         DirectoryToMaterialize(
           # NB the first element here is the root to materialize into, not the dir to snapshot
@@ -861,15 +844,15 @@ def _runtool_nonhermetic(self, parent_workunit, classpath, main, tool_name, args
   def _runtool(self, main, tool_name, args, distribution,
                tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
     with self.context.new_workunit(tool_name) as wu:
-      return self.do_for_execution_strategy_variant({
+      return self.execution_strategy_enum.resolve_for_enum_variant({
         self.HERMETIC: lambda: self._runtool_hermetic(
           main, tool_name, args, distribution,
           tgt=tgt, input_files=input_files, input_digest=input_digest, output_dir=output_dir),
         self.SUBPROCESS: lambda: self._runtool_nonhermetic(
           wu, self.tool_classpath(tool_name), main, tool_name, args, distribution),
         self.NAILGUN: lambda: self._runtool_nonhermetic(
           wu, self._nailgunnable_combined_classpath, main, tool_name, args, distribution),
-      })
+      })()
 
   def _run_metai_tool(self,
                       distribution,

src/python/pants/backend/jvm/tasks/nailgun_task.py

@@ -15,6 +15,7 @@
 from pants.process.subprocess import Subprocess
 from pants.task.task import Task, TaskBase
 from pants.util.memo import memoized_property
+from pants.util.objects import enum, register_enum_option
 
 
 class NailgunTaskBase(JvmToolTaskMixin, TaskBase):
@@ -24,30 +25,16 @@ class NailgunTaskBase(JvmToolTaskMixin, TaskBase):
   SUBPROCESS = 'subprocess'
   HERMETIC = 'hermetic'
 
-  class InvalidExecutionStrategyMapping(Exception): pass
-
-  _all_execution_strategies = frozenset([NAILGUN, SUBPROCESS, HERMETIC])
-
-  def do_for_execution_strategy_variant(self, mapping):
-    """Invoke the method in `mapping` with the key corresponding to the execution strategy.
-
-    `mapping` is a dict mapping execution strategy -> zero-argument lambda.
-    """
-    variants = frozenset(mapping.keys())
-    if variants != self._all_execution_strategies:
-      raise self.InvalidExecutionStrategyMapping(
-        'Must specify a mapping with exactly the keys {} (was: {})'
-        .format(self._all_execution_strategies, variants))
-    method_for_variant = mapping[self.execution_strategy]
-    # The methods need not return a value, but we pass it along if they do.
-    return method_for_variant()
+  class ExecutionStrategy(enum([NAILGUN, SUBPROCESS, HERMETIC])): pass
 
   @classmethod
   def register_options(cls, register):
     super(NailgunTaskBase, cls).register_options(register)
-    register('--execution-strategy', choices=[cls.NAILGUN, cls.SUBPROCESS, cls.HERMETIC], default=cls.NAILGUN,
-             help='If set to nailgun, nailgun will be enabled and repeated invocations of this '
-                  'task will be quicker. If set to subprocess, then the task will be run without nailgun.')
+    register_enum_option(
+      register, cls.ExecutionStrategy, '--execution-strategy',
+      help='If set to nailgun, nailgun will be enabled and repeated invocations of this '
+           'task will be quicker. If set to subprocess, then the task will be run without nailgun. '
+           'Hermetic execution is an experimental subprocess execution framework.')
     register('--nailgun-timeout-seconds', advanced=True, default=10, type=float,
              help='Timeout (secs) for nailgun startup.')
     register('--nailgun-connect-attempts', advanced=True, default=5, type=int,
@@ -60,6 +47,13 @@ def register_options(cls, register):
                                           rev='0.9.1'),
                           ])
 
+  @memoized_property
+  def execution_strategy_enum(self):
+    # TODO: This .create() call can be removed when the enum interface is more stable as the option
+    # is converted into an instance of self.ExecutionStrategy via the `type` argument through
+    # register_enum_option().
+    return self.ExecutionStrategy.create(self.get_options().execution_strategy)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(NailgunTaskBase, cls).subsystem_dependencies() + (Subprocess.Factory,)
@@ -76,9 +70,10 @@ def __init__(self, *args, **kwargs):
     self._executor_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir,
                                           *id_tuple)
 
-  @memoized_property
+  # TODO: eventually deprecate this when we can move all subclasses to use the enum!
+  @property
   def execution_strategy(self):
-    return self.get_options().execution_strategy
+    return self.execution_strategy_enum.value
 
   def create_java_executor(self, dist=None):
     """Create java executor that uses this task's ng daemon, if allowed.

src/python/pants/backend/native/config/environment.py

@@ -9,37 +9,14 @@
 
 from pants.engine.rules import SingletonRule
 from pants.util.meta import AbstractClass
-from pants.util.objects import datatype
+from pants.util.objects import datatype, enum
 from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
 from pants.util.strutil import create_path_env_var
 
 
-class Platform(datatype(['normalized_os_name'])):
+class Platform(enum('normalized_os_name', all_normalized_os_names())):
 
-  class UnsupportedPlatformError(Exception):
-    """Thrown if pants is running on an unrecognized platform."""
-
-  @classmethod
-  def create(cls):
-    return Platform(get_normalized_os_name())
-
-  _NORMALIZED_OS_NAMES = frozenset(all_normalized_os_names())
-
-  def resolve_platform_specific(self, platform_specific_funs):
-    arg_keys = frozenset(platform_specific_funs.keys())
-    unknown_plats = self._NORMALIZED_OS_NAMES - arg_keys
-    if unknown_plats:
-      raise self.UnsupportedPlatformError(
-        "platform_specific_funs {} must support platforms {}"
-        .format(platform_specific_funs, list(unknown_plats)))
-    extra_plats = arg_keys - self._NORMALIZED_OS_NAMES
-    if extra_plats:
-      raise self.UnsupportedPlatformError(
-        "platform_specific_funs {} has unrecognized platforms {}"
-        .format(platform_specific_funs, list(extra_plats)))
-
-    fun_for_platform = platform_specific_funs[self.normalized_os_name]
-    return fun_for_platform()
+  default_value = get_normalized_os_name()
 
 
 class Executable(AbstractClass):
@@ -89,9 +66,9 @@ def as_invocation_environment_dict(self):
 
     :rtype: dict of string -> string
     """
-    lib_env_var = self._platform.resolve_platform_specific({
-      'darwin': lambda: 'DYLD_LIBRARY_PATH',
-      'linux': lambda: 'LD_LIBRARY_PATH',
+    lib_env_var = self._platform.resolve_for_enum_variant({
+      'darwin': 'DYLD_LIBRARY_PATH',
+      'linux': 'LD_LIBRARY_PATH',
     })
     return {
       'PATH': create_path_env_var(self.path_entries),

src/python/pants/backend/native/subsystems/binaries/gcc.py

@@ -44,9 +44,9 @@ def path_entries(self):
 
   @memoized_method
   def _common_lib_dirs(self, platform):
-    lib64_tuples = platform.resolve_platform_specific({
-      'darwin': lambda: [],
-      'linux': lambda: [('lib64',)],
+    lib64_tuples = platform.resolve_for_enum_variant({
+      'darwin': [],
+      'linux': [('lib64',)],
     })
     return self._filemap(lib64_tuples + [
       ('lib',),

src/python/pants/backend/native/subsystems/binaries/llvm.py

@@ -80,16 +80,13 @@ def _filemap(self, all_components_list):
   def path_entries(self):
     return self._filemap([('bin',)])
 
-  _PLATFORM_SPECIFIC_LINKER_NAME = {
-    'darwin': lambda: 'ld64.lld',
-    'linux': lambda: 'lld',
-  }
-
   def linker(self, platform):
     return Linker(
       path_entries=self.path_entries,
-      exe_filename=platform.resolve_platform_specific(
-        self._PLATFORM_SPECIFIC_LINKER_NAME),
+      exe_filename=platform.resolve_for_enum_variant({
+        'darwin': 'ld64.lld',
+        'linux': 'lld',
+      }),
       library_dirs=[],
       linking_library_dirs=[],
       extra_args=[],