Skip to content

Commit

Permalink
don't try to generalize the merged jar for nailgunnability
Browse files Browse the repository at this point in the history
  • Loading branch information
@cosmicexplorer
cosmicexplorer committed Jan 17, 2019
1 parent d44123f commit 70977ef
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 115 deletions.
63 changes: 0 additions & 63 deletions src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,9 @@
from collections import namedtuple
from textwrap import dedent

from future.utils import text_type

from pants.base.exceptions import TaskError
from pants.java.distribution.distribution import DistributionLocator
from pants.option.custom_types import target_option
from pants.util.memo import memoized_method
from pants.util.objects import Exactly, datatype


class JvmToolMixin(object):
Expand Down Expand Up @@ -149,65 +145,6 @@ def formulate_help():
jvm_tool = cls.JvmTool(register.scope, key, classpath, main, custom_rules)
JvmToolMixin._jvm_tools.append(jvm_tool)

# TODO: Deprecate .register_jvm_tool()?
class JvmToolDeclaration(datatype([
('tool_name', text_type),
# If the tool's `main` class name is supplied the tool classpath will be shaded!
('main', Exactly(text_type, type(None))),
('classpath', tuple),
('custom_rules', tuple),
])):
"""A mostly-typed specification for a JVM tool.
This allows a specification for a JVM tool to be passed around verbatim across JvmToolMixin
subclasses.
"""

def __new__(cls, tool_name, main=None, classpath=None, custom_rules=None):
return super(JvmToolMixin.JvmToolDeclaration, cls).__new__(
cls, text_type(tool_name), main, tuple(classpath or []), tuple(custom_rules or []))

class JvmToolDeclError(Exception): pass

@classmethod
def register_jvm_tool_decl(cls, register, decl, **kwargs):
"""Register a `JvmToolDeclaration` with `.register_jvm_tool()`."""
if not isinstance(decl, cls.JvmToolDeclaration):
raise cls.JvmToolDeclError(
'decl {} must be an instance of {}.{}'
.format(decl, cls.__name__, cls.JvmToolDeclaration.__name__))
kwargs = dict(
classpath=list(decl.classpath),
custom_rules=list(decl.custom_rules)
)
if decl.main:
kwargs['main'] = decl.main
cls.register_jvm_tool(register,
decl.tool_name,
**kwargs)

class CombinedJvmToolsError(JvmToolDeclError): pass

@memoized_method
def ensure_combined_jvm_tool_classpath(self, tool_name, combined_jvm_tool_names):
"""Get a single classpath for all tools returned by `combined_jvm_tool_names`.
Also check to ensure `tool_name` is a member of `combined_jvm_tool_names`.
This allows tools to be invoked one at a time, but with the combined classpath of all of
them. This allows creating nailgun instances for the tools which have the same fingerprint, and
allows a task to invoke multiple different JVM tools from the same nailgun instances. See #7089.
"""
if tool_name not in combined_jvm_tool_names:
raise self.CombinedJvmToolsError(
"tool with name '{}' must be added to `combined_jvm_tool_names` "
"(which was: {})"
.format(tool_name, combined_jvm_tool_names))
cp = []
for component_tool_name in combined_jvm_tool_names:
cp.extend(self.tool_classpath(component_tool_name))
return cp

@classmethod
def prepare_tools(cls, round_manager):
"""Subclasses must call this method to ensure jvm tool products are available."""
Expand Down
48 changes: 21 additions & 27 deletions src/python/pants/backend/jvm/subsystems/zinc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,6 @@
from pants.util.memo import memoized_method, memoized_property


_ZINC_SHADER_RULES = [
# The compiler-interface and compiler-bridge tool jars carry xsbt and
# xsbti interfaces that are used across the shaded tool jar boundary so
# we preserve these root packages wholesale along with the core scala
# APIs.
Shader.exclude_package('scala', recursive=True),
Shader.exclude_package('xsbt', recursive=True),
Shader.exclude_package('xsbti', recursive=True),
# Unfortunately, is loaded reflectively by the compiler.
Shader.exclude_package('org.apache.logging.log4j', recursive=True),
]


class Zinc(object):
"""Configuration for Pants' zinc wrapper tool."""

Expand Down Expand Up @@ -74,17 +61,34 @@ def register_options(cls, register):

zinc_rev = '1.0.3'

shader_rules = [
# The compiler-interface and compiler-bridge tool jars carry xsbt and
# xsbti interfaces that are used across the shaded tool jar boundary so
# we preserve these root packages wholesale along with the core scala
# APIs.
Shader.exclude_package('scala', recursive=True),
Shader.exclude_package('xsbt', recursive=True),
Shader.exclude_package('xsbti', recursive=True),
# Unfortunately, is loaded reflectively by the compiler.
Shader.exclude_package('org.apache.logging.log4j', recursive=True),
]

cls.register_jvm_tool(register,
Zinc.ZINC_BOOTSTRAPPER_TOOL_NAME,
classpath=[
JarDependency('org.pantsbuild', 'zinc-bootstrapper_2.11', '0.0.4'),
],
main=Zinc.ZINC_BOOTSTRAPER_MAIN,
custom_rules=_ZINC_SHADER_RULES,
custom_rules=shader_rules,
)

# Defined at the bottom of this file so it can be consumed elsewhere.
cls.register_jvm_tool_decl(register, ZINC_COMPILER_DECL)
cls.register_jvm_tool(register,
Zinc.ZINC_COMPILER_TOOL_NAME,
classpath=[
JarDependency('org.pantsbuild', 'zinc-compiler_2.11', '0.0.9'),
],
main=Zinc.ZINC_COMPILE_MAIN,
custom_rules=shader_rules)

cls.register_jvm_tool(register,
'compiler-bridge',
Expand All @@ -107,7 +111,7 @@ def register_options(cls, register):
# broken up into multiple jars, but zinc does not yet support a sequence
# of jars for the interface.
main='no.such.main.Main',
custom_rules=_ZINC_SHADER_RULES)
custom_rules=shader_rules)

cls.register_jvm_tool(register,
Zinc.ZINC_EXTRACTOR_TOOL_NAME,
Expand Down Expand Up @@ -431,13 +435,3 @@ def compile_classpath(self, classpath_product_key, target, extra_cp_entries=None
"Classpath entry does not start with buildroot: {}".format(entry)

return classpath_entries


ZINC_COMPILER_DECL = JvmToolMixin.JvmToolDeclaration(
Zinc.ZINC_COMPILER_TOOL_NAME,
main=Zinc.ZINC_COMPILE_MAIN,
classpath=[
JarDependency('org.pantsbuild', 'zinc-compiler_2.11', '0.0.9'),
],
custom_rules=_ZINC_SHADER_RULES,
)
45 changes: 20 additions & 25 deletions src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@

from pants.backend.jvm.subsystems.dependency_context import DependencyContext # noqa
from pants.backend.jvm.subsystems.jvm_platform import JvmPlatform
from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin
from pants.backend.jvm.subsystems.shader import Shader
from pants.backend.jvm.subsystems.zinc import ZINC_COMPILER_DECL
from pants.backend.jvm.targets.jar_library import JarLibrary
from pants.backend.jvm.targets.java_library import JavaLibrary
from pants.backend.jvm.targets.jvm_target import JvmTarget
Expand All @@ -37,6 +35,7 @@
from pants.util.contextutil import Timer
from pants.util.dirutil import (fast_relpath, fast_relpath_optional, maybe_read_file,
safe_file_dump, safe_mkdir)
from pants.util.memo import memoized_property


#
Expand Down Expand Up @@ -140,7 +139,8 @@ def register_options(cls, register):
rsc_toolchain_version = '0.0.0-446-c64e6937'
scalameta_toolchain_version = '4.0.0'

rsc_decl = JvmToolMixin.JvmToolDeclaration(
cls.register_jvm_tool(
register,
'rsc',
classpath=[
JarDependency(
Expand All @@ -153,7 +153,8 @@ def register_options(cls, register):
Shader.exclude_package('rsc', recursive=True),
]
)
metacp_decl = JvmToolMixin.JvmToolDeclaration(
cls.register_jvm_tool(
register,
'metacp',
classpath=[
JarDependency(
Expand All @@ -166,7 +167,8 @@ def register_options(cls, register):
Shader.exclude_package('scala', recursive=True),
]
)
metai_decl = JvmToolMixin.JvmToolDeclaration(
cls.register_jvm_tool(
register,
'metai',
classpath=[
JarDependency(
Expand All @@ -180,33 +182,27 @@ def register_options(cls, register):
]
)

# Register all of these as "combined" JVM tools so that we can invoke their combined classpath
# in a single nailgun instance. We still invoke their classpaths separately when not using
# nailgun, however. Note that we have to register zinc again as in ZincCompile it is being
# accessed through the Zinc subsystem, but we want to invoke all of these as a combined tool
# through *this task's* implementation of JvmToolMixin.
# Note that registering as a "combined" JVM tool is only done by overriding
# combined_jvm_tool_names.
for decl in [rsc_decl, metacp_decl, metai_decl, ZINC_COMPILER_DECL]:
cls.register_jvm_tool_decl(register, decl)

def get_nailgunnable_combined_classpath(self, tool_name):
# TODO: allow @memoized_method to convert lists into tuples so they can be hashed!
@memoized_property
def _nailgunnable_combined_classpath(self):
"""Register all of the component tools of the rsc compile task as a "combined" jvm tool.
This allows us to invoke their combined classpath in a single nailgun instance. We still invoke
their classpaths separately when not using nailgun, however.
This allows us to invoke their combined classpath in a single nailgun instance (see #7089 and
#7092). We still invoke their classpaths separately when not using nailgun, however.
"""
return self.ensure_combined_jvm_tool_classpath(
tool_name,
# TODO: allow @memoized_method to convert lists into tuples so they can be hashed!
('rsc', 'metai', 'metacp', ZINC_COMPILER_DECL.tool_name))
cp = []
for component_tool_name in ['rsc', 'metai', 'metacp']:
cp.extend(self.tool_classpath(component_tool_name))
# Add zinc's classpath so that it can be invoked from the same nailgun instance.
cp.extend(super(RscCompile, self).get_zinc_compiler_classpath())
return cp

# Overrides the normal zinc compiler classpath, which only contains zinc.
def get_zinc_compiler_classpath(self):
return self.do_for_execution_strategy_variant({
self.HERMETIC: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.SUBPROCESS: lambda: super(RscCompile, self).get_zinc_compiler_classpath(),
self.NAILGUN: lambda: self.get_nailgunnable_combined_classpath(ZINC_COMPILER_DECL.tool_name),
self.NAILGUN: lambda: self._nailgunnable_combined_classpath,
})

def register_extra_products_from_contexts(self, targets, compile_contexts):
Expand Down Expand Up @@ -870,8 +866,7 @@ def _runtool(self, main, tool_name, args, distribution,
self.SUBPROCESS: lambda: self._runtool_nonhermetic(
wu, self.tool_classpath(tool_name), main, tool_name, args, distribution),
self.NAILGUN: lambda: self._runtool_nonhermetic(
wu, self.get_nailgunnable_combined_classpath(tool_name),
main, tool_name, args, distribution),
wu, self._nailgunnable_combined_classpath, main, tool_name, args, distribution),
})

def _run_metai_tool(self,
Expand Down

0 comments on commit 70977ef

Please sign in to comment.