Zinc 1.0.0 upgrade: Python portion

build-support/ivy/ivysettings.xml

@@ -10,7 +10,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
   <property name="m2.repo.relpath" value="[organisation]/[module]/[revision]"/>
   <property name="m2.repo.pom" value="${m2.repo.relpath}/[module]-[revision].pom"/>
   <property name="m2.repo.artifact"
-            value="${m2.repo.relpath}/[artifact](-[classifier])-[revision].[ext]"/>
+            value="${m2.repo.relpath}/[artifact]-[revision](-[classifier]).[ext]"/>
   <property name="m2.repo.dir" value="${user.home}/.m2/repository" override="false"/>
   <property name="kythe.artifact" value="[organisation]/[artifact]/[revision]/[artifact]-[revision].[ext]"/>
   <!-- for retrieving jacoco snapshot, remove when a version containing cli is released -->

contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen_integration.py

@@ -20,7 +20,7 @@ def run_pants(self, command, config=None, stdin_data=None, extra_env=None, **kwa
         'pythonpath': ["%(buildroot)s/contrib/scrooge/src/python"],
         'backend_packages': ["pants.backend.codegen", "pants.backend.jvm", "pants.contrib.scrooge"]
       },
-      'scala-platform': { 'version': '2.11' },
+      'scala': { 'version': '2.11' },
       'gen.scrooge': {
         'service_deps': {
             'java': [

examples/src/scala/org/pantsbuild/example/README.md

@@ -56,11 +56,11 @@ Scala Version
 -------------
 
 You can override the default version of the entire Scala toolchain with the single
-`--scala-platform-version` option. You can set that option to one of the supported Scala versions
+`--scala-version` option. You can set that option to one of the supported Scala versions
 (currently "2.10" or "2.11"), or to the special value "custom".
 
-If you choose a custom version, you must use the `--scala-platform-runtime-spec`,
-`--scala-platform-repl-spec` and `--scala-platform-suffix-version` options to provide
+If you choose a custom version, you must use the `--scala-runtime-spec`,
+`--scala-repl-spec` and `--scala-suffix-version` options to provide
 information about your custom Scala version.  The first two of these default to the target
 addresses `//:scala-library` and `//:scala-repl` respectively, so you can simply define those
 targets (in the root `BUILD.tools` file by convention) to point to the relevant JARs.

pants.ini

@@ -180,8 +180,6 @@ exclude_patterns: [
 [compile.zinc]
 jvm_options: [
     '-Xmx4g', '-XX:+UseConcMarkSweepGC', '-XX:ParallelGCThreads=4',
-    # bigger cache size for our big projects (default is just 5)
-    '-Dzinc.analysis.cache.limit=1000',
   ]
 
 args: [
@@ -280,7 +278,7 @@ skip: True
 [pycheck-context-manager]
 skip: True
 
-[scala-platform]
+[scala]
 version: 2.11
 
 

src/python/pants/backend/jvm/register.py

@@ -30,6 +30,7 @@
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
 from pants.backend.jvm.targets.unpacked_jars import UnpackedJars
+from pants.backend.jvm.tasks.analysis_extraction import AnalysisExtraction
 from pants.backend.jvm.tasks.benchmark_run import BenchmarkRun
 from pants.backend.jvm.tasks.binary_create import BinaryCreate
 from pants.backend.jvm.tasks.bootstrap_jvm_tools import BootstrapJvmTools
@@ -161,6 +162,9 @@ def register_goals():
   task(name='zinc', action=ZincCompile).install('compile')
   task(name='javac', action=JavacCompile).install('compile')
 
+  # Analysis extraction.
+  task(name='zinc', action=AnalysisExtraction).install('analysis')
+
   # Dependency resolution.
   task(name='ivy', action=IvyResolve).install('resolve', first=True)
   task(name='coursier', action=CoursierResolve).install('resolve')
@@ -173,7 +177,6 @@ def register_goals():
   task(name='services', action=PrepareServices).install('resources')
 
   task(name='export-classpath', action=RuntimeClasspathPublisher).install()
-  task(name='jvm-dep-check', action=JvmDependencyCheck).install('compile')
 
   task(name='jvm', action=JvmDependencyUsage).install('dep-usage')
 
@@ -210,6 +213,7 @@ def register_goals():
   task(name='scalafmt', action=ScalaFmtCheckFormat, serialize=False).install('lint')
   task(name='scalastyle', action=Scalastyle, serialize=False).install('lint')
   task(name='checkstyle', action=Checkstyle, serialize=False).install('lint')
+  task(name='jvm-dep-check', action=JvmDependencyCheck, serialize=False).install('lint')
 
   # Formatting.
   task(name='scalafmt', action=ScalaFmtFormat, serialize=False).install('fmt')

src/python/pants/backend/jvm/subsystems/BUILD

@@ -1,11 +1,25 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+python_library(
+  name = 'dependency_context',
+  sources = ['dependency_context.py'],
+  dependencies = [
+    ':java',
+    ':scala_platform',
+    'src/python/pants/backend/codegen/thrift/java',
+    'src/python/pants/base:fingerprint_strategy',
+    'src/python/pants/build_graph',
+    'src/python/pants/java/jar',
+  ],
+)
+
 python_library(
   name = 'jvm_tool_mixin',
   sources = ['jvm_tool_mixin.py'],
   dependencies = [
     'src/python/pants/base:exceptions',
+    'src/python/pants/java/distribution',
     'src/python/pants/option',
   ],
 )
@@ -117,7 +131,9 @@ python_library(
   name = 'zinc',
   sources = ['zinc.py'],
   dependencies = [
+    ':dependency_context',
     ':jvm_tool_mixin',
+    ':shader',
     'src/python/pants/subsystem',
   ]
 )

src/python/pants/backend/jvm/subsystems/dependency_context.py

@@ -0,0 +1,118 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import hashlib
+
+from pants.backend.codegen.protobuf.java.java_protobuf_library import JavaProtobufLibrary
+from pants.backend.codegen.thrift.java.java_thrift_library import JavaThriftLibrary
+from pants.backend.jvm.subsystems.java import Java
+from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform
+from pants.backend.jvm.targets.annotation_processor import AnnotationProcessor
+from pants.backend.jvm.targets.jar_library import JarLibrary
+from pants.backend.jvm.targets.javac_plugin import JavacPlugin
+from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
+from pants.base.fingerprint_strategy import FingerprintStrategy
+from pants.build_graph.aliased_target import AliasTarget
+from pants.build_graph.resources import Resources
+from pants.build_graph.target import Target
+from pants.build_graph.target_scopes import Scopes
+from pants.subsystem.subsystem import Subsystem
+
+
+class SyntheticTargetNotFound(Exception):
+  """Exports were resolved for a thrift target which hasn't had a synthetic target generated yet."""
+
+
+class DependencyContext(Subsystem):
+  """Implements calculating `exports` and exception (compiler-plugin) aware dependencies.
+
+  This is a subsystem because in future the compiler plugin types should be injected
+  via subsystem or option dependencies rather than declared statically.
+  """
+
+  options_scope = 'jvm-dependency-context'
+
+  target_closure_kwargs = dict(include_scopes=Scopes.JVM_COMPILE_SCOPES, respect_intransitive=True)
+  compiler_plugin_types = (AnnotationProcessor, JavacPlugin, ScalacPlugin)
+  alias_types = (AliasTarget, Target)
+  codegen_types = (JavaThriftLibrary, JavaProtobufLibrary)
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(DependencyContext, cls).subsystem_dependencies() + (Java, ScalaPlatform)
+
+  def all_dependencies(self, target):
+    """All transitive dependencies of the context's target."""
+    for dep in target.closure(bfs=True, **self.target_closure_kwargs):
+      yield dep
+
+  def create_fingerprint_strategy(self, classpath_products):
+    return ResolvedJarAwareFingerprintStrategy(classpath_products, self)
+
+  def defaulted_property(self, target, selector):
+    """Computes a language property setting for the given JvmTarget.
+
+    :param selector A function that takes a target or platform and returns the boolean value of the
+                    property for that target or platform, or None if that target or platform does
+                    not directly define the property.
+
+    If the target does not override the language property, returns true iff the property
+    is true for any of the matched languages for the target.
+    """
+    if selector(target) is not None:
+      return selector(target)
+
+    prop = False
+    if target.has_sources('.java'):
+      prop |= selector(Java.global_instance())
+    if target.has_sources('.scala'):
+      prop |= selector(ScalaPlatform.global_instance())
+    return prop
+
+
+class ResolvedJarAwareFingerprintStrategy(FingerprintStrategy):
+  """Task fingerprint strategy that also includes the resolved coordinates of dependent jars."""
+
+  def __init__(self, classpath_products, dep_context):
+    super(ResolvedJarAwareFingerprintStrategy, self).__init__()
+    self._classpath_products = classpath_products
+    self._dep_context = dep_context
+
+  def compute_fingerprint(self, target):
+    if isinstance(target, Resources):
+      # Just do nothing, this kind of dependency shouldn't affect result's hash.
+      return None
+
+    hasher = hashlib.sha1()
+    hasher.update(target.payload.fingerprint())
+    if isinstance(target, JarLibrary):
+      # NB: Collects only the jars for the current jar_library, and hashes them to ensure that both
+      # the resolved coordinates, and the requested coordinates are used. This ensures that if a
+      # source file depends on a library with source compatible but binary incompatible signature
+      # changes between versions, that you won't get runtime errors due to using an artifact built
+      # against a binary incompatible version resolved for a previous compile.
+      classpath_entries = self._classpath_products.get_artifact_classpath_entries_for_targets(
+        [target])
+      for _, entry in classpath_entries:
+        hasher.update(str(entry.coordinate))
+    return hasher.hexdigest()
+
+  def direct(self, target):
+    return self._dep_context.defaulted_property(target, lambda x: x.strict_deps)
+
+  def dependencies(self, target):
+    if self.direct(target):
+      return target.strict_dependencies(self._dep_context)
+    return super(ResolvedJarAwareFingerprintStrategy, self).dependencies(target)
+
+  def __hash__(self):
+    # NB: FingerprintStrategy requires a useful override of eq/hash.
+    return hash(type(self))
+
+  def __eq__(self, other):
+    # NB: See __hash__.
+    return type(self) == type(other)

src/python/pants/backend/jvm/subsystems/java.py

@@ -39,6 +39,14 @@ def register_options(cls, register):
                           help='Java compiler to use.  If unspecified, we use the compiler '
                                'embedded in the Java distribution we run on.')
 
+    register('--javac-plugins', advanced=True, type=list, fingerprint=True,
+            help='Use these javac plugins.')
+    register('--javac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
+            help='Map from javac plugin name to list of arguments for that plugin.')
+    cls.register_jvm_tool(register, 'javac-plugin-dep', classpath=[],
+                        help='Search for javac plugins here, as well as in any '
+                                'explicit dependencies.')
+
   def injectables(self, build_graph):
     tools_jar_address = Address.parse(self._tools_jar_spec)
     if not build_graph.contains_address(tools_jar_address):

src/python/pants/backend/jvm/subsystems/jvm_platform.py

@@ -57,7 +57,7 @@ def register_options(cls, register):
              help='Compile settings that can be referred to by name in jvm_targets.')
     register('--default-platform', advanced=True, type=str, default=None, fingerprint=True,
              help='Name of the default platform to use if none are specified.')
-    register('--compiler', choices=['zinc', 'javac'], default='zinc',
+    register('--compiler', advanced=True, choices=['zinc', 'javac'], default='zinc', fingerprint=True,
              help='Java compiler implementation to use.')
 
   @classmethod

src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py

@@ -9,6 +9,7 @@
 from textwrap import dedent
 
 from pants.base.exceptions import TaskError
+from pants.java.distribution.distribution import DistributionLocator
 from pants.option.custom_types import target_option
 
 
@@ -51,6 +52,10 @@ def is_default(self, options):
 
   _jvm_tools = []  # List of JvmTool objects.
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(JvmToolMixin, cls).subsystem_dependencies() + (DistributionLocator,)
+
   @classmethod
   def get_jvm_options_default(cls, bootstrap_option_values):
     """Subclasses may override to provide different defaults for their JVM options.
@@ -158,6 +163,20 @@ def reset_registered_tools():
     """Needed only for test isolation."""
     JvmToolMixin._jvm_tools = []
 
+  def set_distribution(self, minimum_version=None, maximum_version=None, jdk=False):
+    try:
+      self._dist = DistributionLocator.cached(minimum_version=minimum_version,
+                                              maximum_version=maximum_version, jdk=jdk)
+    except DistributionLocator.Error as e:
+      raise TaskError(e)
+
+  @property
+  def dist(self):
+    if getattr(self, '_dist', None) is None:
+      # Use default until told otherwise.
+      self.set_distribution()
+    return self._dist
+
   @classmethod
   def tool_jar_from_products(cls, products, key, scope):
     """Get the jar for the tool previously registered under key in the given scope.

src/python/pants/backend/jvm/subsystems/scala_platform.py

@@ -39,7 +39,10 @@ class ScalaPlatform(JvmToolMixin, ZincLanguageMixin, InjectablesMixin, Subsystem
 
   :API: public
   """
-  options_scope = 'scala-platform'
+  options_scope = 'scala'
+
+  deprecated_options_scope = 'scala-platform'
+  deprecated_options_scope_removal_version = '1.9.0.dev0'
 
   @classmethod
   def _create_jardep(cls, name, version):
@@ -88,6 +91,15 @@ def register_style_tool(version):
                             classpath=[scala_style_jar])
 
     super(ScalaPlatform, cls).register_options(register)
+
+    register('--scalac-plugins', advanced=True, type=list, fingerprint=True,
+            help='Use these scalac plugins.')
+    register('--scalac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
+            help='Map from scalac plugin name to list of arguments for that plugin.')
+    cls.register_jvm_tool(register, 'scalac-plugin-dep', classpath=[],
+                        help='Search for scalac plugins here, as well as in any '
+                                'explicit dependencies.')
+
     register('--version', advanced=True, default='2.12',
              choices=['2.10', '2.11', '2.12', 'custom'], fingerprint=True,
              help='The scala platform version. If --version=custom, the targets '
@@ -153,7 +165,7 @@ def suffix_version(self, name):
         raise RuntimeError('Suffix version must be specified if using a custom scala version. '
                            'Suffix version is used for bootstrapping jars.  If a custom '
                            'scala version is not specified, then the version specified in '
-                           '--scala-platform-suffix-version is used.  For example for Scala '
+                           '--scala-suffix-version is used.  For example for Scala '
                            '2.10.7 you would use the suffix version "2.10".')
 
     elif name.endswith(self.version):