Engine looks up default sources when parsing

Rather than re-implementing default source look-up.

This pushes sources parsing of default sources through the engine, in parallel,
rather than being later synchronous python calls.

This also works for plugin types, and doesn't change any existing APIs.

It updates the Go patterns to match those that the engine currently performs,
rather than ones which aren't actually used by any code.

contrib/go/src/python/pants/contrib/go/targets/go_binary.py

@@ -11,6 +11,8 @@
 class GoBinary(GoLocalSource):
   """A local Go main package."""
 
+  default_sources_globs = '*'
+
   @classmethod
   def alias(cls):
     return 'go_binary'

contrib/go/src/python/pants/contrib/go/targets/go_library.py

@@ -11,7 +11,7 @@
 class GoLibrary(GoLocalSource):
   """A local Go package."""
 
-  default_sources_globs = '*.go'
+  default_sources_globs = '*'
 
   @classmethod
   def alias(cls):

src/python/pants/build_graph/target.py

@@ -868,6 +868,12 @@ def create_sources_field(self, sources, sources_rel_path, key_arg=None):
       # This is so that tests don't need to set up the subsystem when creating targets that
       # legitimately do not require sources.
       if (key_arg is None or key_arg == 'sources') and self.supports_default_sources():
+        logger.warning(
+          'Default sources should always be piped through the engine. This code should be '
+          'unreachable. If you see this message, please contact pants-dev. '
+          'Class which caused this message: %s',
+          self.__class__.__name__
+        )
         sources = self.default_sources(sources_rel_path)
       else:
         sources = FilesetWithSpec.empty(sources_rel_path)

src/python/pants/engine/legacy/structs.py

@@ -11,6 +11,7 @@
 
 from six import string_types
 
+from pants.backend.python.targets.python_library import PythonLibrary
 from pants.base.deprecated import deprecated_conditional
 from pants.build_graph.target import Target
 from pants.engine.addressable import addressable_list
@@ -141,35 +142,6 @@ def __str__(self):
       type(self).__name__, self.address, self.base_globs, self.arg, self.filespecs)
 
 
-class JavaLibraryAdaptor(TargetAdaptor):
-  @property
-  def default_sources_globs(self):
-    return ('*.java',)
-
-  @property
-  def default_sources_exclude_globs(self):
-    return JunitTestsAdaptor.java_test_globs
-
-
-class ScalaLibraryAdaptor(TargetAdaptor):
-  @property
-  def default_sources_globs(self):
-    return ('*.scala',)
-
-  @property
-  def default_sources_exclude_globs(self):
-    return JunitTestsAdaptor.scala_test_globs
-
-
-class JunitTestsAdaptor(TargetAdaptor):
-  java_test_globs = ('*Test.java',)
-  scala_test_globs = ('*Test.scala', '*Spec.scala')
-
-  @property
-  def default_sources_globs(self):
-    return self.java_test_globs + self.scala_test_globs
-
-
 class JvmBinaryAdaptor(TargetAdaptor):
   def validate_sources(self, sources):
     if len(sources.files) > 1:
@@ -294,16 +266,6 @@ def validate_sources(self, sources):
       )
 
 
-class PythonLibraryAdaptor(PythonTargetAdaptor):
-  @property
-  def default_sources_globs(self):
-    return ('*.py',)
-
-  @property
-  def default_sources_exclude_globs(self):
-    return PythonTestsAdaptor.python_test_globs
-
-
 class PythonTestsAdaptor(PythonTargetAdaptor):
   python_test_globs = ('test_*.py', '*_test.py')
 
@@ -329,7 +291,7 @@ def default_sources_exclude_globs(self):
     return ('BUILD', 'BUILD.*')
 
 
-class PantsPluginAdaptor(TargetAdaptor):
+class PantsPluginAdaptor(PythonTargetAdaptor):
   def get_sources(self):
     return ['register.py']
 

src/python/pants/init/engine_initializer.py

@@ -7,8 +7,16 @@
 
 import logging
 
+from pants.backend.docgen.targets.doc import Page
+from pants.backend.jvm.targets.jvm_app import JvmApp
+from pants.backend.jvm.targets.jvm_binary import JvmBinary
+from pants.backend.python.targets.python_app import PythonApp
+from pants.backend.python.targets.python_binary import PythonBinary
+from pants.backend.python.targets.python_library import PythonLibrary
+from pants.backend.python.targets.python_tests import PythonTests
 from pants.base.build_environment import get_buildroot
 from pants.base.file_system_project_tree import FileSystemProjectTree
+from pants.build_graph.remote_sources import RemoteSources
 from pants.engine.build_files import create_graph_rules
 from pants.engine.fs import create_fs_rules
 from pants.engine.isolated_process import create_process_rules
@@ -17,11 +25,10 @@
                                        create_legacy_graph_tasks)
 from pants.engine.legacy.options_parsing import create_options_parsing_rules
 from pants.engine.legacy.parser import LegacyPythonCallbacksParser
-from pants.engine.legacy.structs import (AppAdaptor, GoTargetAdaptor, JavaLibraryAdaptor,
-                                         JunitTestsAdaptor, JvmBinaryAdaptor, PageAdaptor,
+from pants.engine.legacy.structs import (AppAdaptor, JvmBinaryAdaptor, PageAdaptor,
                                          PantsPluginAdaptor, PythonBinaryAdaptor,
-                                         PythonLibraryAdaptor, PythonTestsAdaptor,
-                                         RemoteSourcesAdaptor, ScalaLibraryAdaptor, TargetAdaptor)
+                                         PythonTargetAdaptor, PythonTestsAdaptor,
+                                         RemoteSourcesAdaptor, TargetAdaptor)
 from pants.engine.mapper import AddressMapper
 from pants.engine.native import Native
 from pants.engine.parser import SymbolTable
@@ -46,29 +53,29 @@ def __init__(self, build_file_aliases):
     :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
     """
     self._build_file_aliases = build_file_aliases
-    self._table = {alias: TargetAdaptor for alias in build_file_aliases.target_types}
+    self._table = {
+      alias: self._make_target_adaptor(TargetAdaptor, target_type)
+      for alias, target_type in build_file_aliases.target_types.items()
+    }
 
     # TODO: The alias replacement here is to avoid elevating "TargetAdaptors" into the public
     # API until after https://github.com/pantsbuild/pants/issues/3560 has been completed.
     # These should likely move onto Target subclasses as the engine gets deeper into beta
     # territory.
-    for alias in ['java_library', 'java_agent', 'javac_plugin']:
-      self._table[alias] = JavaLibraryAdaptor
-    for alias in ['scala_library', 'scalac_plugin']:
-      self._table[alias] = ScalaLibraryAdaptor
-    for alias in ['python_library', 'pants_plugin']:
-      self._table[alias] = PythonLibraryAdaptor
-    for alias in ['go_library', 'go_binary']:
-      self._table[alias] = GoTargetAdaptor
-
-    self._table['junit_tests'] = JunitTestsAdaptor
-    self._table['jvm_app'] = AppAdaptor
-    self._table['jvm_binary'] = JvmBinaryAdaptor
-    self._table['python_app'] = AppAdaptor
-    self._table['python_tests'] = PythonTestsAdaptor
-    self._table['python_binary'] = PythonBinaryAdaptor
-    self._table['remote_sources'] = RemoteSourcesAdaptor
-    self._table['page'] = PageAdaptor
+    self._table['python_library'] = self._make_target_adaptor(PythonTargetAdaptor, PythonLibrary)
+
+    self._table['jvm_app'] = self._make_target_adaptor(AppAdaptor, JvmApp)
+    self._table['jvm_binary'] = self._make_target_adaptor(JvmBinaryAdaptor, JvmBinary)
+    self._table['python_app'] = self._make_target_adaptor(AppAdaptor, PythonApp)
+    self._table['python_tests'] = self._make_target_adaptor(PythonTestsAdaptor, PythonTests)
+    self._table['python_binary'] = self._make_target_adaptor(PythonBinaryAdaptor, PythonBinary)
+    self._table['remote_sources'] = self._make_target_adaptor(RemoteSourcesAdaptor, RemoteSources)
+    self._table['page'] = self._make_target_adaptor(PageAdaptor, Page)
+
+    # Note that these don't call _make_target_adaptor because we don't have a handy reference to the
+    # types being constructed. They don't have any default_sources behavior, so this should be ok,
+    # but if we end up doing more things in _make_target_adaptor, we should make sure they're
+    # applied here too.
     self._table['pants_plugin'] = PantsPluginAdaptor
     self._table['contrib_plugin'] = PantsPluginAdaptor
 
@@ -78,6 +85,44 @@ def aliases(self):
   def table(self):
     return self._table
 
+  @classmethod
+  def _make_target_adaptor(cls, base_class, target_type):
+    """
+    Look up the default source globs for the type, and apply them to parsing through the engine.
+    """
+    if not target_type.supports_default_sources() or target_type.default_sources_globs is None:
+      return base_class
+
+    globs = _tuplify(target_type.default_sources_globs)
+    excludes = _tuplify(target_type.default_sources_exclude_globs)
+
+    class GlobsHandlingTargetAdaptor(base_class):
+      @property
+      def default_sources_globs(self):
+        if globs is None:
+          return super(GlobsHandlingTargetAdaptor, self).default_sources_globs
+        else:
+          return globs
+
+      @property
+      def default_sources_exclude_globs(self):
+        if excludes is None:
+          return super(GlobsHandlingTargetAdaptor, self).default_sources_exclude_globs
+        else:
+          return excludes
+
+    return GlobsHandlingTargetAdaptor
+
+
+def _tuplify(v):
+  if v is None:
+    return None
+  if isinstance(v, tuple):
+    return v
+  if isinstance(v, (list, set)):
+    return tuple(v)
+  return (v,)
+
 
 class LegacyGraphScheduler(datatype(['scheduler', 'symbol_table'])):
   """A thin wrapper around a Scheduler configured with @rules for a symbol table."""