From bbeeddd9d79b8a545a2e699f74fa221777b681d9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 15 Nov 2019 18:23:00 -0800 Subject: [PATCH 1/6] Partially type check more targets --- .../pants/backend/codegen/thrift/lib/BUILD | 1 + src/python/pants/backend/jvm/subsystems/BUILD | 14 +++++++++++-- .../backend/jvm/subsystems/jvm_tool_mixin.py | 12 +++++------ src/python/pants/backend/jvm/targets/BUILD | 3 +++ .../backend/jvm/targets/jvm_prep_command.py | 10 +++++---- src/python/pants/backend/jvm/tasks/BUILD | 4 ++++ src/python/pants/backend/native/config/BUILD | 1 + .../backend/native/config/environment.py | 2 +- .../backend/native/subsystems/binaries/BUILD | 2 ++ .../native/subsystems/binaries/binutils.py | 16 +++++++------- .../backend/native/subsystems/binaries/gcc.py | 11 +++++----- .../native/subsystems/binaries/llvm.py | 21 ++++++++++--------- .../native/subsystems/native_build_step.py | 2 +- .../backend/native/subsystems/utils/BUILD | 3 ++- src/python/pants/backend/native/targets/BUILD | 1 + src/python/pants/backend/python/BUILD | 9 ++++---- src/python/pants/backend/python/targets/BUILD | 1 + src/python/pants/console/BUILD | 3 ++- src/python/pants/engine/BUILD | 7 +++++-- src/python/pants/engine/addressable.py | 7 ++++--- src/python/pants/invalidation/BUILD | 1 + src/python/pants/scm/subsystems/BUILD | 1 + src/python/pants/task/BUILD | 1 + src/python/pants/task/goal_options_mixin.py | 4 +++- src/python/pants/task/mutex_task_mixin.py | 3 ++- 25 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/python/pants/backend/codegen/thrift/lib/BUILD b/src/python/pants/backend/codegen/thrift/lib/BUILD index ffa278a2fe9..aa5828007c9 100644 --- a/src/python/pants/backend/codegen/thrift/lib/BUILD +++ b/src/python/pants/backend/codegen/thrift/lib/BUILD @@ -11,4 +11,5 @@ python_library( 'src/python/pants/task', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/jvm/subsystems/BUILD b/src/python/pants/backend/jvm/subsystems/BUILD index d7c13c1ee5f..2e1c19d7ce4 100644 --- a/src/python/pants/backend/jvm/subsystems/BUILD +++ b/src/python/pants/backend/jvm/subsystems/BUILD @@ -22,6 +22,7 @@ python_library( 'src/python/pants/java/distribution', 'src/python/pants/option', ], + tags = {"partially_type_checked"}, ) python_library( @@ -34,6 +35,7 @@ python_library( 'src/python/pants/option', 'src/python/pants/subsystem', ], + tags = {"partially_type_checked"}, ) python_library( @@ -46,6 +48,7 @@ python_library( 'src/python/pants/option', 'src/python/pants/subsystem', ], + tags = {"partially_type_checked"}, ) python_library( @@ -59,6 +62,7 @@ python_library( 'src/python/pants/build_graph', 'src/python/pants/subsystem', ], + tags = {"partially_type_checked"}, ) python_library( @@ -69,6 +73,7 @@ python_library( 'src/python/pants/subsystem', 'src/python/pants/util:strutil', ], + tags = {"partially_type_checked"}, ) python_library( @@ -82,6 +87,7 @@ python_library( 'src/python/pants/util:memo', 'src/python/pants/java/distribution', ], + tags = {"partially_type_checked"}, ) python_library( @@ -109,6 +115,7 @@ python_library( 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) python_library( @@ -120,6 +127,7 @@ python_library( 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', ], + tags = {"partially_type_checked"}, ) python_library( @@ -133,7 +141,8 @@ python_library( 'src/python/pants/java/jar', 'src/python/pants/subsystem', 'src/python/pants/util:contextutil', - ] + ], + tags = {"partially_type_checked"}, ) python_library( @@ -154,7 +163,8 @@ python_library( dependencies = [ 'src/python/pants/build_graph', 'src/python/pants/subsystem', - ] + ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py b/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py index db7fe7c18eb..4d153edfe16 100644 --- a/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py +++ b/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py @@ -3,6 +3,7 @@ from collections import namedtuple from textwrap import dedent +from typing import List from pants.base.exceptions import TaskError from pants.java.distribution.distribution import DistributionLocator @@ -46,7 +47,7 @@ def is_default(self, options): """ return options.for_scope(self.scope).is_default(self.key.replace('-', '_')) - _jvm_tools = [] # List of JvmTool objects. + _jvm_tools: List[JvmTool] = [] @classmethod def subsystem_dependencies(cls): @@ -147,15 +148,12 @@ def prepare_tools(cls, round_manager): round_manager.require_data('jvm_build_tools_classpath_callbacks') @staticmethod - def get_registered_tools(): - """Returns all registered jvm tools. - - :rtype: list of :class:`JvmToolMixin.JvmTool` - """ + def get_registered_tools() -> List["JvmTool"]: + """Returns all registered jvm tools.""" return JvmToolMixin._jvm_tools @staticmethod - def reset_registered_tools(): + def reset_registered_tools() -> None: """Needed only for test isolation.""" JvmToolMixin._jvm_tools = [] diff --git a/src/python/pants/backend/jvm/targets/BUILD b/src/python/pants/backend/jvm/targets/BUILD index 827c88f99a6..3ccbf34fbe8 100644 --- a/src/python/pants/backend/jvm/targets/BUILD +++ b/src/python/pants/backend/jvm/targets/BUILD @@ -46,6 +46,7 @@ python_library( 'src/python/pants/util:netrc', 'src/python/pants/util:objects', ], + tags = {"partially_type_checked"}, ) python_library( @@ -64,6 +65,7 @@ python_library( 'src/python/pants/base:exceptions', 'src/python/pants/build_graph', ], + tags = {"partially_type_checked"}, ) python_library( @@ -85,4 +87,5 @@ python_library( 'src/python/pants/build_graph', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/jvm/targets/jvm_prep_command.py b/src/python/pants/backend/jvm/targets/jvm_prep_command.py index bba3f61a659..f514e850f47 100644 --- a/src/python/pants/backend/jvm/targets/jvm_prep_command.py +++ b/src/python/pants/backend/jvm/targets/jvm_prep_command.py @@ -1,6 +1,8 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from typing import FrozenSet + from pants.backend.jvm.targets.jvm_target import JvmTarget from pants.base.exceptions import TargetDefinitionException from pants.base.payload import Payload @@ -34,7 +36,7 @@ class JvmPrepCommand(JvmTarget): :API: public """ - _goals=frozenset() + _goals: FrozenSet[str] = frozenset() @staticmethod def add_goal(goal): @@ -42,12 +44,12 @@ def add_goal(goal): JvmPrepCommand._goals = frozenset(list(JvmPrepCommand._goals) + [goal]) @classmethod - def reset(cls): + def reset(cls) -> None: """Used for testing purposes to reset state.""" - cls._goals=frozenset() + cls._goals = frozenset() @staticmethod - def goals(): + def goals() -> FrozenSet[str]: return JvmPrepCommand._goals def __init__(self, payload=None, mainclass=None, args=None, jvm_options=None, goal=None, diff --git a/src/python/pants/backend/jvm/tasks/BUILD b/src/python/pants/backend/jvm/tasks/BUILD index 060e9b1c8a4..edebd0b243c 100644 --- a/src/python/pants/backend/jvm/tasks/BUILD +++ b/src/python/pants/backend/jvm/tasks/BUILD @@ -165,6 +165,7 @@ python_library( python_library( name = 'classpath_entry', sources = ['classpath_entry.py'], + tags = {"partially_type_checked"}, ) python_library( @@ -177,6 +178,7 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:strutil', ], + tags = {"partially_type_checked"}, ) python_library( @@ -517,6 +519,7 @@ python_library( 'src/python/pants/base:exceptions', 'src/python/pants/task', ], + tags = {"partially_type_checked"}, ) python_library( @@ -547,6 +550,7 @@ python_library( 'src/python/pants/process', 'src/python/pants/task', ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/backend/native/config/BUILD b/src/python/pants/backend/native/config/BUILD index 3957af780dd..ec98e81fccf 100644 --- a/src/python/pants/backend/native/config/BUILD +++ b/src/python/pants/backend/native/config/BUILD @@ -7,4 +7,5 @@ python_library( 'src/python/pants/engine:platform', 'src/python/pants/util:osutil', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/native/config/environment.py b/src/python/pants/backend/native/config/environment.py index 08470a85b24..811eeaf2911 100644 --- a/src/python/pants/backend/native/config/environment.py +++ b/src/python/pants/backend/native/config/environment.py @@ -187,7 +187,7 @@ class Assembler(_AlgebraicDataclass, _Executable): path_entries: Tuple[str, ...] exe_filename: str runtime_library_dirs: Tuple[str, ...] - extra_args: Tuple[str] + extra_args: Tuple[str, ...] class _LinkerMixin(_Executable): diff --git a/src/python/pants/backend/native/subsystems/binaries/BUILD b/src/python/pants/backend/native/subsystems/binaries/BUILD index ea2d832efe7..9435c223cf0 100644 --- a/src/python/pants/backend/native/subsystems/binaries/BUILD +++ b/src/python/pants/backend/native/subsystems/binaries/BUILD @@ -8,7 +8,9 @@ python_library( 'src/python/pants/backend/native/subsystems/utils', 'src/python/pants/binaries', 'src/python/pants/engine:rules', + 'src/python/pants/engine:platform', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/native/subsystems/binaries/binutils.py b/src/python/pants/backend/native/subsystems/binaries/binutils.py index 3e39016eae0..3e7e0d910b7 100644 --- a/src/python/pants/backend/native/subsystems/binaries/binutils.py +++ b/src/python/pants/backend/native/subsystems/binaries/binutils.py @@ -16,21 +16,21 @@ class Binutils(NativeTool): def path_entries(self): return [os.path.join(self.select(), 'bin')] - def assembler(self): + def assembler(self) -> Assembler: return Assembler( path_entries=self.path_entries(), exe_filename='as', - runtime_library_dirs=[], - extra_args=[]) + runtime_library_dirs=(), + extra_args=()) - def linker(self): + def linker(self) -> Linker: return Linker( path_entries=self.path_entries(), exe_filename='ld', - runtime_library_dirs=[], - linking_library_dirs=[], - extra_args=[], - extra_object_files=[], + runtime_library_dirs=(), + linking_library_dirs=(), + extra_args=(), + extra_object_files=(), ) diff --git a/src/python/pants/backend/native/subsystems/binaries/gcc.py b/src/python/pants/backend/native/subsystems/binaries/gcc.py index f45ac7330d9..a016d267f16 100644 --- a/src/python/pants/backend/native/subsystems/binaries/gcc.py +++ b/src/python/pants/backend/native/subsystems/binaries/gcc.py @@ -3,9 +3,10 @@ import os -from pants.backend.native.config.environment import CCompiler, CppCompiler, Platform +from pants.backend.native.config.environment import CCompiler, CppCompiler from pants.backend.native.subsystems.utils.archive_file_mapper import ArchiveFileMapper from pants.binaries.binary_tool import NativeTool +from pants.engine.platform import Platform from pants.engine.rules import rule from pants.util.memo import memoized_method, memoized_property @@ -57,13 +58,13 @@ def _common_include_dirs(self): ('lib/gcc/*', self.version(), 'include'), ]) - def c_compiler(self, platform): + def c_compiler(self, platform: Platform) -> CCompiler: return CCompiler( path_entries=self.path_entries, exe_filename='gcc', runtime_library_dirs=self._common_lib_dirs(platform), include_dirs=self._common_include_dirs, - extra_args=[]) + extra_args=()) @memoized_property def _cpp_include_dirs(self): @@ -83,13 +84,13 @@ def _cpp_include_dirs(self): return most_cpp_include_dirs + [plat_cpp_header_dir] - def cpp_compiler(self, platform): + def cpp_compiler(self, platform: Platform) -> CppCompiler: return CppCompiler( path_entries=self.path_entries, exe_filename='g++', runtime_library_dirs=self._common_lib_dirs(platform), include_dirs=(self._common_include_dirs + self._cpp_include_dirs), - extra_args=[]) + extra_args=()) @rule diff --git a/src/python/pants/backend/native/subsystems/binaries/llvm.py b/src/python/pants/backend/native/subsystems/binaries/llvm.py index 8b572b7b77e..fd60c1bcc62 100644 --- a/src/python/pants/backend/native/subsystems/binaries/llvm.py +++ b/src/python/pants/backend/native/subsystems/binaries/llvm.py @@ -3,10 +3,11 @@ import os -from pants.backend.native.config.environment import CCompiler, CppCompiler, Linker, Platform +from pants.backend.native.config.environment import CCompiler, CppCompiler, Linker from pants.backend.native.subsystems.utils.archive_file_mapper import ArchiveFileMapper from pants.binaries.binary_tool import NativeTool from pants.binaries.binary_util import BinaryToolUrlGenerator +from pants.engine.platform import Platform from pants.engine.rules import RootRule, rule from pants.util.dirutil import is_readable_dir from pants.util.memo import memoized_method, memoized_property @@ -76,17 +77,17 @@ def _filemap(self, all_components_list): def path_entries(self): return self._filemap([('bin',)]) - def linker(self, platform): + def linker(self, platform: Platform) -> Linker: return Linker( path_entries=self.path_entries, exe_filename=platform.match({ Platform.darwin: "ld64.lld", Platform.linux: "lld", }), - library_dirs=[], - linking_library_dirs=[], - extra_args=[], - extra_object_files=[], + runtime_library_dirs=(), + linking_library_dirs=(), + extra_args=(), + extra_object_files=(), ) @memoized_property @@ -97,25 +98,25 @@ def _common_include_dirs(self): def _common_lib_dirs(self): return self._filemap([('lib',)]) - def c_compiler(self): + def c_compiler(self) -> CCompiler: return CCompiler( path_entries=self.path_entries, exe_filename='clang', runtime_library_dirs=self._common_lib_dirs, include_dirs=self._common_include_dirs, - extra_args=[]) + extra_args=()) @memoized_property def _cpp_include_dirs(self): return self._filemap([('include/c++/v1',)]) - def cpp_compiler(self): + def cpp_compiler(self) -> CppCompiler: return CppCompiler( path_entries=self.path_entries, exe_filename='clang++', runtime_library_dirs=self._common_lib_dirs, include_dirs=(self._cpp_include_dirs + self._common_include_dirs), - extra_args=[]) + extra_args=()) # TODO(#5663): use this over the XCode linker! diff --git a/src/python/pants/backend/native/subsystems/native_build_step.py b/src/python/pants/backend/native/subsystems/native_build_step.py index 43173b5734b..21f55936b27 100644 --- a/src/python/pants/backend/native/subsystems/native_build_step.py +++ b/src/python/pants/backend/native/subsystems/native_build_step.py @@ -3,8 +3,8 @@ from abc import abstractmethod -from pants.backend.native.config.environment import Platform from pants.build_graph.mirrored_target_option_mixin import MirroredTargetOptionMixin +from pants.engine.platform import Platform from pants.option.compiler_option_sets_mixin import CompilerOptionSetsMixin from pants.subsystem.subsystem import Subsystem from pants.util.collections import Enum diff --git a/src/python/pants/backend/native/subsystems/utils/BUILD b/src/python/pants/backend/native/subsystems/utils/BUILD index 56a71876c22..5be9c27146f 100644 --- a/src/python/pants/backend/native/subsystems/utils/BUILD +++ b/src/python/pants/backend/native/subsystems/utils/BUILD @@ -5,5 +5,6 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', 'src/python/pants/util:strutil', - ] + ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/native/targets/BUILD b/src/python/pants/backend/native/targets/BUILD index d2f37f855d1..0af864bc789 100644 --- a/src/python/pants/backend/native/targets/BUILD +++ b/src/python/pants/backend/native/targets/BUILD @@ -13,4 +13,5 @@ python_library( 'src/python/pants/engine:platform', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index 11197339097..fa6092a0df6 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -60,7 +60,7 @@ python_library( 'src/python/pants/base:exceptions', 'src/python/pants/build_graph', 'src/python/pants/util:meta', - ] + ], ) python_library( @@ -69,19 +69,18 @@ python_library( dependencies = [ 'src/python/pants/base:payload_field', ], + tags = {"partially_type_checked"}, ) python_library( name = 'python_requirement', sources = ['python_requirement.py'], - dependencies = [ - ] + tags = {"partially_type_checked"}, ) python_library( name = 'python_requirements', sources = ['python_requirements.py'], - dependencies = [ - ] + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/python/targets/BUILD b/src/python/pants/backend/python/targets/BUILD index 9a311a21ac3..0896d6ed001 100644 --- a/src/python/pants/backend/python/targets/BUILD +++ b/src/python/pants/backend/python/targets/BUILD @@ -12,4 +12,5 @@ python_library( 'src/python/pants/base:payload_field', 'src/python/pants/build_graph', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/console/BUILD b/src/python/pants/console/BUILD index 464f606cb1e..d4192a43932 100644 --- a/src/python/pants/console/BUILD +++ b/src/python/pants/console/BUILD @@ -6,5 +6,6 @@ python_library( sources = ['stty_utils.py'], dependencies = [ 'src/python/pants/util:process_handler', - ] + ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index cb22e3242a4..827b36aa10a 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -29,6 +29,7 @@ python_library( 'src/python/pants/util:memo', 'src/python/pants/util:objects', ], + tags = {"partially_type_checked"}, ) python_library( @@ -38,7 +39,8 @@ python_library( ':addressable', ':objects', 'src/python/pants/util:objects', - ] + ], + tags = {"partially_type_checked"}, ) python_library( @@ -156,7 +158,8 @@ python_library( ':objects', 'src/python/pants/build_graph', 'src/python/pants/util:memo', - ] + ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/engine/addressable.py b/src/python/pants/engine/addressable.py index 34e4485b0b7..835c6e927ea 100644 --- a/src/python/pants/engine/addressable.py +++ b/src/python/pants/engine/addressable.py @@ -5,6 +5,7 @@ from collections.abc import MutableMapping, MutableSequence from dataclasses import dataclass from functools import update_wrapper +from typing import Any, Set, Tuple, Type from pants.base.specs import Spec from pants.build_graph.address import Address, BuildFileAddress @@ -96,14 +97,14 @@ class AddressableDescriptor: Here the `Thing.parent` property is re-assigned with a type-constrained addressable descriptor after the class is defined so the class can be referred to in the type constraint. """ - _descriptors = set() + _descriptors: Set[Tuple[Type, str]] = set() @classmethod - def is_addressable(cls, obj, key): + def is_addressable(cls, obj: Any, key: str) -> bool: """Return `True` if the given attribute of `obj` is an addressable attribute. :param obj: The object to inspect. - :param string key: The name of the property on `obj` to check. + :param key: The name of the property on `obj` to check. """ return (type(obj), key) in cls._descriptors diff --git a/src/python/pants/invalidation/BUILD b/src/python/pants/invalidation/BUILD index 823708e9ed8..68f6c4a3709 100644 --- a/src/python/pants/invalidation/BUILD +++ b/src/python/pants/invalidation/BUILD @@ -11,4 +11,5 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/scm/subsystems/BUILD b/src/python/pants/scm/subsystems/BUILD index 355c613c83e..8a9b7496baa 100644 --- a/src/python/pants/scm/subsystems/BUILD +++ b/src/python/pants/scm/subsystems/BUILD @@ -11,4 +11,5 @@ python_library( 'src/python/pants/goal:workspace', 'src/python/pants/subsystem', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/task/BUILD b/src/python/pants/task/BUILD index 0443b3fd339..1fae464d523 100644 --- a/src/python/pants/task/BUILD +++ b/src/python/pants/task/BUILD @@ -27,4 +27,5 @@ python_library( 'src/python/pants/util:memo', 'src/python/pants/util:meta', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/task/goal_options_mixin.py b/src/python/pants/task/goal_options_mixin.py index fb93dccf7e4..4ca12a9ba20 100644 --- a/src/python/pants/task/goal_options_mixin.py +++ b/src/python/pants/task/goal_options_mixin.py @@ -1,6 +1,8 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from typing import Optional, Type + from pants.option.optionable import Optionable from pants.option.scope import ScopeInfo @@ -28,4 +30,4 @@ def registrar_for_scope(cls, goal): class GoalOptionsMixin: """A mixin for tasks that inherit options registered at the goal level.""" # Subclasses must set this to the appropriate subclass of GoalOptionsRegistrar. - goal_options_registrar_cls = None + goal_options_registrar_cls: Optional[Type[GoalOptionsRegistrar]] = None diff --git a/src/python/pants/task/mutex_task_mixin.py b/src/python/pants/task/mutex_task_mixin.py index 72726842f0a..82ad4b0edad 100644 --- a/src/python/pants/task/mutex_task_mixin.py +++ b/src/python/pants/task/mutex_task_mixin.py @@ -3,6 +3,7 @@ from abc import abstractmethod from collections import defaultdict +from typing import DefaultDict from pants.base.exceptions import TaskError from pants.build_graph.target import Target @@ -40,7 +41,7 @@ class NoActivationsError(TaskError): class IncompatibleActivationsError(TaskError): """Indicates a mutexed task group had more than one task eligible to run.""" - _implementations = defaultdict(set) + _implementations: DefaultDict = defaultdict(set) @classmethod def reset_implementations(cls): From c6e53ba1387d689ae4afd76c7891c43ab2853047 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 15 Nov 2019 19:49:20 -0800 Subject: [PATCH 2/6] Round 2 of typing (plane) --- .../pants/backend/codegen/thrift/java/BUILD | 1 + .../thrift/lib/apache_thrift_gen_base.py | 3 ++- .../graph_info/subsystems/cloc_binary.py | 1 - src/python/pants/backend/jvm/BUILD | 9 ++++--- src/python/pants/backend/jvm/ivy_utils.py | 2 +- src/python/pants/backend/jvm/tasks/BUILD | 1 + .../pants/backend/native/subsystems/BUILD | 1 + .../native/subsystems/native_toolchain.py | 2 +- .../native/subsystems/xcode_cli_tools.py | 24 +++++++++---------- src/python/pants/backend/python/BUILD | 4 +++- .../python/subsystems/python_tool_base.py | 8 ++++--- src/python/pants/base/BUILD | 1 + src/python/pants/engine/BUILD | 8 ++++--- src/python/pants/engine/fs.py | 2 +- src/python/pants/engine/legacy/BUILD | 1 + src/python/pants/help/BUILD | 3 ++- src/python/pants/ivy/BUILD | 3 ++- src/python/pants/ivy/ivy.py | 3 +-- src/python/pants/task/simple_codegen_task.py | 6 +++-- 19 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/python/pants/backend/codegen/thrift/java/BUILD b/src/python/pants/backend/codegen/thrift/java/BUILD index 894fe0b7f43..da1a590d418 100644 --- a/src/python/pants/backend/codegen/thrift/java/BUILD +++ b/src/python/pants/backend/codegen/thrift/java/BUILD @@ -13,4 +13,5 @@ python_library( 'src/python/pants/subsystem', 'src/python/pants/task', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py b/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py index 4008643e8f1..0a5b399a7fd 100644 --- a/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py +++ b/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py @@ -5,6 +5,7 @@ import re import shutil import subprocess +from typing import Optional from twitter.common.collections import OrderedSet @@ -20,7 +21,7 @@ class ApacheThriftGenBase(SimpleCodegenTask): # The name of the thrift generator to use. Subclasses must set. # E.g., java, py (see `thrift -help` for all available generators). - thrift_generator = None + thrift_generator: Optional[str] = None # Subclasses may set their own default generator options. default_gen_options_map = None diff --git a/src/python/pants/backend/graph_info/subsystems/cloc_binary.py b/src/python/pants/backend/graph_info/subsystems/cloc_binary.py index 01c244df86d..213ecbc8270 100644 --- a/src/python/pants/backend/graph_info/subsystems/cloc_binary.py +++ b/src/python/pants/backend/graph_info/subsystems/cloc_binary.py @@ -12,4 +12,3 @@ class ClocBinary(Script): replaces_scope = 'cloc' replaces_name = 'version' - diff --git a/src/python/pants/backend/jvm/BUILD b/src/python/pants/backend/jvm/BUILD index cbcff150dd7..0810a7f1040 100644 --- a/src/python/pants/backend/jvm/BUILD +++ b/src/python/pants/backend/jvm/BUILD @@ -7,7 +7,8 @@ python_library( dependencies=[ 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', - ] + ], + tags = {"partially_type_checked"}, ) python_library( @@ -18,6 +19,7 @@ python_library( 'src/python/pants/base:hash_utils', 'src/python/pants/base:payload_field', ], + tags = {"partially_type_checked"}, ) python_library( @@ -59,6 +61,7 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:fileutil', ], + tags = {"partially_type_checked"}, ) resources( @@ -69,8 +72,7 @@ resources( python_library( name='repository', sources=['repository.py'], - dependencies = [ - ] + tags = {"partially_type_checked"}, ) python_library( @@ -90,4 +92,5 @@ python_library( ':artifact', 'src/python/pants/base:validation', ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/jvm/ivy_utils.py b/src/python/pants/backend/jvm/ivy_utils.py index ae511a2d6d2..17551320a2c 100644 --- a/src/python/pants/backend/jvm/ivy_utils.py +++ b/src/python/pants/backend/jvm/ivy_utils.py @@ -796,7 +796,7 @@ def _exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: - with ivy.resolution_lock: + with ivy.resolution_lock(): result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: diff --git a/src/python/pants/backend/jvm/tasks/BUILD b/src/python/pants/backend/jvm/tasks/BUILD index edebd0b243c..e506f086441 100644 --- a/src/python/pants/backend/jvm/tasks/BUILD +++ b/src/python/pants/backend/jvm/tasks/BUILD @@ -195,6 +195,7 @@ python_library( 'src/python/pants/goal:products', 'src/python/pants/util:dirutil', ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/backend/native/subsystems/BUILD b/src/python/pants/backend/native/subsystems/BUILD index 6bc59130702..072a2d088df 100644 --- a/src/python/pants/backend/native/subsystems/BUILD +++ b/src/python/pants/backend/native/subsystems/BUILD @@ -9,6 +9,7 @@ python_library( 'src/python/pants/backend/native/subsystems/utils', 'src/python/pants/build_graph', 'src/python/pants/engine:rules', + 'src/python/pants/engine:platform', 'src/python/pants/engine:selectors', 'src/python/pants/subsystem', 'src/python/pants/util:collections', diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py index 41a2ab1a095..0046db0cb3a 100644 --- a/src/python/pants/backend/native/subsystems/native_toolchain.py +++ b/src/python/pants/backend/native/subsystems/native_toolchain.py @@ -11,7 +11,6 @@ CppToolchain, CToolchain, Linker, - Platform, ) from pants.backend.native.subsystems.binaries.binutils import Binutils from pants.backend.native.subsystems.binaries.gcc import GCC @@ -19,6 +18,7 @@ from pants.backend.native.subsystems.libc_dev import LibcDev from pants.backend.native.subsystems.native_build_step import ToolchainVariant from pants.backend.native.subsystems.xcode_cli_tools import XCodeCLITools +from pants.engine.platform import Platform from pants.engine.rules import RootRule, rule from pants.engine.selectors import Get from pants.subsystem.subsystem import Subsystem diff --git a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py index 0b2dd1b0c75..e5b55650de3 100644 --- a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py +++ b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py @@ -124,41 +124,41 @@ def include_dirs(self, include_cpp_inc=False): return self._get_existing_subdirs('include') @memoized_method - def assembler(self): + def assembler(self) -> Assembler: return Assembler( path_entries=self.path_entries(), exe_filename='as', - runtime_library_dirs=[], - extra_args=[]) + runtime_library_dirs=(), + extra_args=()) @memoized_method - def linker(self): + def linker(self) -> Linker: return Linker( path_entries=self.path_entries(), exe_filename='ld', - runtime_library_dirs=[], - linking_library_dirs=[], - extra_args=[MIN_OSX_VERSION_ARG], - extra_object_files=[], + runtime_library_dirs=(), + linking_library_dirs=(), + extra_args=(MIN_OSX_VERSION_ARG,), + extra_object_files=(), ) @memoized_method - def c_compiler(self): + def c_compiler(self) -> CCompiler: return CCompiler( path_entries=self.path_entries(), exe_filename='clang', runtime_library_dirs=self.lib_dirs(), include_dirs=self.include_dirs(), - extra_args=[MIN_OSX_VERSION_ARG]) + extra_args=(MIN_OSX_VERSION_ARG,)) @memoized_method - def cpp_compiler(self): + def cpp_compiler(self) -> CppCompiler: return CppCompiler( path_entries=self.path_entries(), exe_filename='clang++', runtime_library_dirs=self.lib_dirs(), include_dirs=self.include_dirs(include_cpp_inc=True), - extra_args=[MIN_OSX_VERSION_ARG]) + extra_args=(MIN_OSX_VERSION_ARG,)) @rule diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index fa6092a0df6..3f9610db7b9 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -47,7 +47,8 @@ python_library( 'src/python/pants/subsystem', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', - ] + ], + tags = {"partially_type_checked"}, ) python_library( @@ -61,6 +62,7 @@ python_library( 'src/python/pants/build_graph', 'src/python/pants/util:meta', ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index 80c879c9427..b1a56c12eb6 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -1,6 +1,8 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from typing import Optional, Sequence + from pants.subsystem.subsystem import Subsystem @@ -8,10 +10,10 @@ class PythonToolBase(Subsystem): """Base class for subsystems that configure a python tool to be invoked out-of-process.""" # Subclasses must set. - default_requirements = None - default_entry_point = None + default_requirements: Optional[Sequence[str]] = None + default_entry_point: Optional[str] = None # Subclasses need not override. - default_interpreter_constraints = [] + default_interpreter_constraints: Sequence[str] = [] @classmethod def register_options(cls, register): diff --git a/src/python/pants/base/BUILD b/src/python/pants/base/BUILD index 42785b8fa7f..ef72b81e0b9 100644 --- a/src/python/pants/base/BUILD +++ b/src/python/pants/base/BUILD @@ -182,6 +182,7 @@ python_library( ':specs', 'src/python/pants/build_graph', ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index 827b36aa10a..941c2de6f94 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -88,7 +88,7 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:filtering', 'src/python/pants/util:objects', - ] + ], ) python_library( @@ -101,7 +101,8 @@ python_library( 'src/python/pants/build_graph', 'src/python/pants/util:memo', 'src/python/pants/util:meta', - ] + ], + tags = {"partially_type_checked"}, ) python_library( @@ -140,7 +141,8 @@ python_library( ':struct', 'src/python/pants/base:project_tree', 'src/python/pants/build_graph', - ] + ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/engine/fs.py b/src/python/pants/engine/fs.py index 2bc79d43f15..84eb27e3a59 100644 --- a/src/python/pants/engine/fs.py +++ b/src/python/pants/engine/fs.py @@ -7,7 +7,7 @@ from pants.engine.objects import Collection from pants.engine.rules import RootRule -from pants.option.custom_types import GlobExpansionConjunction +from pants.option.custom_types import GlobExpansionConjunction as GlobExpansionConjunction from pants.option.global_options import GlobMatchErrorBehavior from pants.util.dirutil import maybe_read_file, safe_delete, safe_file_dump from pants.util.meta import frozen_after_init diff --git a/src/python/pants/engine/legacy/BUILD b/src/python/pants/engine/legacy/BUILD index f7a5a11eaa8..d5f8ead1438 100644 --- a/src/python/pants/engine/legacy/BUILD +++ b/src/python/pants/engine/legacy/BUILD @@ -59,6 +59,7 @@ python_library( 'src/python/pants/util:meta', 'src/python/pants/util:objects', ], + tags = {"partially_type_checked"}, ) python_library( diff --git a/src/python/pants/help/BUILD b/src/python/pants/help/BUILD index db94049126a..18a0958a35d 100644 --- a/src/python/pants/help/BUILD +++ b/src/python/pants/help/BUILD @@ -10,7 +10,8 @@ python_library( 'src/python/pants/option', 'src/python/pants/subsystem', 'src/python/pants/util:memo', - ] + ], + tags = {"partially_type_checked"}, ) python_tests( diff --git a/src/python/pants/ivy/BUILD b/src/python/pants/ivy/BUILD index 50ba6ad8384..8aa42142c19 100644 --- a/src/python/pants/ivy/BUILD +++ b/src/python/pants/ivy/BUILD @@ -13,5 +13,6 @@ python_library( 'src/python/pants/subsystem', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', - ] + ], + tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/ivy/ivy.py b/src/python/pants/ivy/ivy.py index 5511ffcec60..d5a79cef10e 100644 --- a/src/python/pants/ivy/ivy.py +++ b/src/python/pants/ivy/ivy.py @@ -59,7 +59,6 @@ def ivy_resolution_cache_dir(self): """Returns the ivy cache dir used by this `Ivy` instance.""" return self._ivy_resolution_cache_dir - @property @contextmanager def resolution_lock(self): safe_mkdir(self._ivy_resolution_cache_dir) @@ -80,7 +79,7 @@ def execute(self, jvm_options=None, args=None, executor=None, executor = executor or SubprocessExecutor(DistributionLocator.cached()) runner = self.runner(jvm_options=jvm_options, args=args, executor=executor) try: - with self.resolution_lock: + with self.resolution_lock(): result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels) if result != 0: raise self.Error('Ivy command failed with exit code {}{}'.format( diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py index 65a447c4971..273fc50cee1 100644 --- a/src/python/pants/task/simple_codegen_task.py +++ b/src/python/pants/task/simple_codegen_task.py @@ -5,6 +5,7 @@ import os from abc import abstractmethod from collections import OrderedDict +from typing import Optional, Tuple, Type from twitter.common.collections import OrderedSet @@ -13,6 +14,7 @@ from pants.base.workunit import WorkUnitLabel from pants.build_graph.address import Address from pants.build_graph.address_lookup_error import AddressLookupError +from pants.build_graph.target import Target from pants.engine.fs import Digest, PathGlobs, PathGlobsAndRoot from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper from pants.task.task import Task @@ -29,12 +31,12 @@ class SimpleCodegenTask(Task): """ # Subclasses may override to provide the type of gen targets the target acts on. # E.g., JavaThriftLibrary. If not provided, the subclass must implement is_gentarget. - gentarget_type = None + gentarget_type: Optional[Type[Target]] = None # Subclasses may override to provide a list of glob patterns matching the generated sources, # relative to the target's workdir. # These must be a tuple of strings, e.g. ('**/*.java',). - sources_globs = None + sources_globs: Optional[Tuple[str, ...]] = None # Tuple of glob patterns to exclude from the above matches. sources_exclude_globs = () From 7f0aa7717d2ded1d5469e9514649d30972b1d138 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 17 Nov 2019 15:11:12 -0700 Subject: [PATCH 3/6] Refactor super() calls from #8637 --- .../pants/backend/jvm/tasks/coursier/coursier_subsystem.py | 2 +- src/python/pants/ivy/ivy_subsystem.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py index 287ce044440..8f77985d23d 100644 --- a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py +++ b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py @@ -68,7 +68,7 @@ def get_external_url_generator(self): class CoursierUrlGenerator(BinaryToolUrlGenerator): def __init__(self, template_urls): - super(CoursierUrlGenerator, self).__init__() + super().__init__() self._template_urls = template_urls def generate_urls(self, version, host_platform): diff --git a/src/python/pants/ivy/ivy_subsystem.py b/src/python/pants/ivy/ivy_subsystem.py index 37d34f5d619..c610210a61c 100644 --- a/src/python/pants/ivy/ivy_subsystem.py +++ b/src/python/pants/ivy/ivy_subsystem.py @@ -126,7 +126,7 @@ def repository_cache_dir(self): class IvyUrlGenerator(BinaryToolUrlGenerator): def __init__(self, template_urls): - super(IvyUrlGenerator, self).__init__() + super().__init__() self._template_urls = template_urls def generate_urls(self, version, host_platform): From b8c862a710fddea4e25313cfb104d2523e8f59f1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 17 Nov 2019 21:58:11 -0700 Subject: [PATCH 4/6] Apply Danny's fix. Thank you! --- src/python/pants/backend/native/config/environment.py | 8 ++++---- .../pants/backend/native/tasks/link_shared_libraries.py | 2 +- src/python/pants/backend/native/tasks/native_compile.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/python/pants/backend/native/config/environment.py b/src/python/pants/backend/native/config/environment.py index 811eeaf2911..5b0d04aa213 100644 --- a/src/python/pants/backend/native/config/environment.py +++ b/src/python/pants/backend/native/config/environment.py @@ -47,9 +47,9 @@ def _single_list_field_operation(self, field_name, list_value, prepend=True): cur_value = getattr(self, field_name) if prepend: - new_value = list_value + cur_value + new_value = tuple(list_value) + tuple(cur_value) else: - new_value = cur_value + list_value + new_value = tuple(cur_value) + tuple(list_value) arg_dict = {field_name: new_value} return self.copy(**arg_dict) @@ -93,7 +93,7 @@ def sequence(self, other, exclude_list_fields=None): for list_field_name in shared_list_fields: lhs_value = getattr(self, list_field_name) rhs_value = getattr(other, list_field_name) - overwrite_kwargs[list_field_name] = lhs_value + rhs_value + overwrite_kwargs[list_field_name] = tuple(lhs_value) + tuple(rhs_value) return self.copy(**overwrite_kwargs) @@ -212,7 +212,7 @@ def extra_object_files(self): def invocation_environment_dict(self): ret = super(_LinkerMixin, self).invocation_environment_dict.copy() - full_library_path_dirs = self.linking_library_dirs + [ + full_library_path_dirs = list(self.linking_library_dirs) + [ os.path.dirname(f) for f in self.extra_object_files ] diff --git a/src/python/pants/backend/native/tasks/link_shared_libraries.py b/src/python/pants/backend/native/tasks/link_shared_libraries.py index 72a32d4fa60..85a16c3680d 100644 --- a/src/python/pants/backend/native/tasks/link_shared_libraries.py +++ b/src/python/pants/backend/native/tasks/link_shared_libraries.py @@ -169,7 +169,7 @@ def _execute_link_request(self, link_request): self.platform.match( {Platform.darwin: ['-Wl,-dylib'], Platform.linux: ['-shared']} ) + - linker.extra_args + + list(linker.extra_args) + ['-o', os.path.abspath(resulting_shared_lib_path)] + ['-L{}'.format(lib_dir) for lib_dir in link_request.external_lib_dirs] + ['-l{}'.format(lib_name) for lib_name in link_request.external_lib_names] + diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py index 13abecae666..1a5665845fc 100644 --- a/src/python/pants/backend/native/tasks/native_compile.py +++ b/src/python/pants/backend/native/tasks/native_compile.py @@ -208,10 +208,10 @@ def _make_compile_argv(self, compile_request): # TODO: add -v to every compiler and linker invocation! argv = ( [compiler.exe_filename] + - compiler.extra_args + + list(compiler.extra_args) + # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). ['-c', '-fPIC'] + - compiler_options + + list(compiler_options) + [ '-I{}'.format(os.path.join(buildroot, inc_dir)) for inc_dir in compile_request.include_dirs From 54b96187bebb834355f8c5b176e5606cb339aefd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Sun, 17 Nov 2019 22:01:53 -0700 Subject: [PATCH 5/6] Add TODO Danny mentioned --- src/python/pants/backend/native/subsystems/binaries/llvm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/backend/native/subsystems/binaries/llvm.py b/src/python/pants/backend/native/subsystems/binaries/llvm.py index fd60c1bcc62..8cb2e525fde 100644 --- a/src/python/pants/backend/native/subsystems/binaries/llvm.py +++ b/src/python/pants/backend/native/subsystems/binaries/llvm.py @@ -77,6 +77,7 @@ def _filemap(self, all_components_list): def path_entries(self): return self._filemap([('bin',)]) + # TODO(#5663): this is currently dead code. def linker(self, platform: Platform) -> Linker: return Linker( path_entries=self.path_entries, From 50c2990a127cebd79b27067f4c51b8eb0cfc61fd Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 18 Nov 2019 07:05:12 -0700 Subject: [PATCH 6/6] Danny's diff to fix failing unit test. Thanks! --- .../backend/native/subsystems/test_native_toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py index 758d4c6a377..a1308a12fbd 100644 --- a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py +++ b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py @@ -121,7 +121,7 @@ def _hello_world_source_environment(self, toolchain_type, file_name, contents): yield toolchain def _invoke_compiler(self, compiler, args): - cmd = [compiler.exe_filename] + compiler.extra_args + args + cmd = [compiler.exe_filename] + list(compiler.extra_args) + args env = compiler.invocation_environment_dict # TODO: add an `extra_args`-like field to `Executable`s which allows for overriding env vars # like this, but declaratively! @@ -129,7 +129,7 @@ def _invoke_compiler(self, compiler, args): return self._invoke_capturing_output(cmd, env) def _invoke_linker(self, linker, args): - cmd = [linker.exe_filename] + linker.extra_args + args + cmd = [linker.exe_filename] + list(linker.extra_args) + args return self._invoke_capturing_output( cmd, linker.invocation_environment_dict)