diff --git a/build-support/bin/_release_helper.py b/build-support/bin/_release_helper.py index da3acabb7e1..cb5cfa4584d 100644 --- a/build-support/bin/_release_helper.py +++ b/build-support/bin/_release_helper.py @@ -73,6 +73,13 @@ _expected_maintainers = {"EricArellano", "gshuflin", "illicitonion", "wisechengyi"} +# Disable the Pants repository-internal internal_plugins.test_lockfile_fixtures plugin because +# otherwise inclusion of that plugin will fail due to its `pytest` import not being included in the pex. +DISABLED_BACKENDS_CONFIG = { + "PANTS_BACKEND_PACKAGES": '-["internal_plugins.test_lockfile_fixtures"]', +} + + class PackageAccessValidator: @classmethod def validate_all(cls): @@ -298,6 +305,10 @@ def run_venv_pants(args: list[str]) -> str: ], check=True, stdout=subprocess.PIPE, + env={ + **os.environ, + **DISABLED_BACKENDS_CONFIG, + }, ) .stdout.decode() .strip() @@ -852,8 +863,9 @@ def build_pex(fetch: bool) -> None: shutil.copyfile(dest, validated_pex_path) validated_pex_path.chmod(0o777) Path(tmpdir, "BUILD_ROOT").touch() - # We also need to filter out Pants options like `PANTS_CONFIG_FILES`. + # We also need to filter out Pants options like `PANTS_CONFIG_FILES` and disable certain internal backends. env = {k: v for k, v in env.items() if not k.startswith("PANTS_")} + env.update(DISABLED_BACKENDS_CONFIG) subprocess.run([validated_pex_path, "--version"], env=env, check=True, cwd=dest.parent) green(f"Validated {dest}") diff --git a/pants-plugins/internal_plugins/test_lockfile_fixtures/BUILD b/pants-plugins/internal_plugins/test_lockfile_fixtures/BUILD new file mode 100644 index 00000000000..95c6150585e --- /dev/null +++ b/pants-plugins/internal_plugins/test_lockfile_fixtures/BUILD @@ -0,0 +1,4 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources() diff --git a/pants-plugins/internal_plugins/test_lockfile_fixtures/__init__.py b/pants-plugins/internal_plugins/test_lockfile_fixtures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pants-plugins/internal_plugins/test_lockfile_fixtures/lockfile_fixture.py b/pants-plugins/internal_plugins/test_lockfile_fixtures/lockfile_fixture.py new file mode 100644 index 00000000000..8fa72368eef --- /dev/null +++ b/pants-plugins/internal_plugins/test_lockfile_fixtures/lockfile_fixture.py @@ -0,0 +1,87 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from pants.jvm.resolve.common import ArtifactRequirement, ArtifactRequirements, Coordinate +from pants.jvm.resolve.coursier_fetch import CoursierResolvedLockfile +from pants.jvm.resolve.lockfile_metadata import LockfileContext +from pants.util.docutil import bin_name + + +@dataclass(frozen=True) +class JVMLockfileFixtureDefinition: + lockfile_rel_path: Path + coordinates: tuple[Coordinate, ...] + + @classmethod + def from_kwargs(cls, kwargs) -> JVMLockfileFixtureDefinition: + lockfile_rel_path = kwargs["path"] + if not lockfile_rel_path: + raise ValueError("`path` must be specified as a relative path to a lockfile") + + requirements = kwargs["requirements"] or [] + coordinates: list[Coordinate] = [] + for requirement in requirements: + if isinstance(requirement, Coordinate): + coordinates.append(requirement) + elif isinstance(requirement, str): + coordinate = Coordinate.from_coord_str(requirement) + coordinates.append(coordinate) + else: + raise ValueError( + f"Unsupported type `{type(requirement)}` for JVM coordinate. Expected `Coordinate` or `str`." + ) + + return cls( + lockfile_rel_path=Path(lockfile_rel_path), + coordinates=tuple(coordinates), + ) + + +@dataclass(frozen=True) +class JVMLockfileFixture: + lockfile: CoursierResolvedLockfile + serialized_lockfile: str + requirements: ArtifactRequirements + + +class JvmLockfilePlugin: + def pytest_configure(self, config): + config.addinivalue_line( + "markers", + "jvm_lockfile(path, requirements): mark test to configure a `jvm_lockfile` fixture", + ) + + @pytest.fixture + def jvm_lockfile(self, request) -> JVMLockfileFixture: + mark = request.node.get_closest_marker("jvm_lockfile") + + definition = JVMLockfileFixtureDefinition.from_kwargs(mark.kwargs) + + # Load the lockfile. + lockfile_path = request.node.path.parent / definition.lockfile_rel_path + lockfile_contents = lockfile_path.read_bytes() + lockfile = CoursierResolvedLockfile.from_serialized(lockfile_contents) + + # Check the lockfile's requirements against the requirements in the lockfile. + # Fail the test if the lockfile needs to be regenerated. + artifact_reqs = ArtifactRequirements( + [ArtifactRequirement(coordinate) for coordinate in definition.coordinates] + ) + if not lockfile.metadata: + raise ValueError( + f"Expected JVM lockfile {definition.lockfile_rel_path} to have metadata." + ) + if not lockfile.metadata.is_valid_for(artifact_reqs, LockfileContext.TOOL): + raise ValueError( + f"Lockfile fixture {definition.lockfile_rel_path} is not valid. " + "Please re-generate it using: " + f"{bin_name()} internal-generate-test-lockfile-fixtures ::" + ) + + return JVMLockfileFixture(lockfile, lockfile_contents.decode(), artifact_reqs) diff --git a/pants-plugins/internal_plugins/test_lockfile_fixtures/register.py b/pants-plugins/internal_plugins/test_lockfile_fixtures/register.py new file mode 100644 index 00000000000..b5f39c3d99e --- /dev/null +++ b/pants-plugins/internal_plugins/test_lockfile_fixtures/register.py @@ -0,0 +1,26 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +from internal_plugins.test_lockfile_fixtures.rules import rules as test_lockfile_fixtures_rules +from pants.backend.python.register import rules as python_rules +from pants.backend.python.register import target_types as python_target_types +from pants.core.goals.test import rules as core_test_rules +from pants.core.util_rules import config_files, source_files +from pants.jvm.resolve import coursier_fetch, coursier_setup + + +def target_types(): + return python_target_types() + + +def rules(): + return ( + *test_lockfile_fixtures_rules(), + *python_rules(), # python backend + *core_test_rules(), + *config_files.rules(), + *coursier_fetch.rules(), + *coursier_setup.rules(), + *source_files.rules(), + ) diff --git a/pants-plugins/internal_plugins/test_lockfile_fixtures/rules.py b/pants-plugins/internal_plugins/test_lockfile_fixtures/rules.py new file mode 100644 index 00000000000..ce4fa76e271 --- /dev/null +++ b/pants-plugins/internal_plugins/test_lockfile_fixtures/rules.py @@ -0,0 +1,271 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import PurePath + +from internal_plugins.test_lockfile_fixtures.lockfile_fixture import JVMLockfileFixtureDefinition +from pants.backend.python.subsystems.pytest import PyTest +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess +from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest +from pants.backend.python.util_rules.python_sources import ( + PythonSourceFiles, + PythonSourceFilesRequest, +) +from pants.core.goals.tailor import group_by_dir +from pants.core.goals.test import TestExtraEnv +from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest +from pants.engine.collection import DeduplicatedCollection +from pants.engine.console import Console +from pants.engine.fs import CreateDigest, DigestContents, FileContent, Workspace +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.internals.native_engine import Digest, MergeDigests, Snapshot +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.process import Process, ProcessCacheScope, ProcessResult +from pants.engine.rules import collect_rules, goal_rule, rule +from pants.engine.target import Targets, TransitiveTargets, TransitiveTargetsRequest +from pants.jvm.resolve.common import ArtifactRequirement, ArtifactRequirements +from pants.jvm.resolve.coursier_fetch import CoursierResolvedLockfile +from pants.jvm.resolve.lockfile_metadata import JVMLockfileMetadata +from pants.util.docutil import bin_name +from pants.util.logging import LogLevel + +COLLECTION_SCRIPT = r"""\ +from pathlib import Path +import json +import sys + +import pytest + +class CollectionPlugin: + def __init__(self): + self.collected = [] + + def pytest_collection_modifyitems(self, items): + for item in items: + self.collected.append(item) + + +collection_plugin = CollectionPlugin() +pytest.main(["--collect-only", *sys.argv[1:]], plugins=[collection_plugin]) +output = [] +cwd = Path.cwd() +for item in collection_plugin.collected: + mark = item.get_closest_marker("jvm_lockfile") + if not mark: + continue + + path = Path(item.path).relative_to(cwd) + + output.append({ + "kwargs": mark.kwargs, + "test_file_path": str(path), + }) + +with open("tests.json", "w") as f: + f.write(json.dumps(output)) +""" + + +@dataclass(frozen=True) +class JVMLockfileFixtureConfig: + definition: JVMLockfileFixtureDefinition + test_file_path: str + + +class CollectedJVMLockfileFixtureConfigs(DeduplicatedCollection[JVMLockfileFixtureConfig]): + pass + + +@dataclass(frozen=True) +class RenderedJVMLockfileFixture: + content: bytes + path: str + + +class RenderedJVMLockfileFixtures(DeduplicatedCollection[RenderedJVMLockfileFixture]): + pass + + +@dataclass(frozen=True) +class CollectFixtureConfigsRequest: + pass + + +# TODO: This rule was mostly copied from the rule `setup_pytest_for_target` in +# `src/python/pants/backend/python/goals/pytest_runner.py`. Some refactoring should be done. +@rule +async def collect_fixture_configs( + _request: CollectFixtureConfigsRequest, + pytest: PyTest, + python_setup: PythonSetup, + test_extra_env: TestExtraEnv, + targets: Targets, +) -> CollectedJVMLockfileFixtureConfigs: + addresses = [tgt.address for tgt in targets] + transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) + all_targets = transitive_targets.closure + + interpreter_constraints = InterpreterConstraints.create_from_targets(all_targets, python_setup) + + pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet( + Get( + Pex, + PexRequest( + output_filename="pytest.pex", + requirements=pytest.pex_requirements(), + interpreter_constraints=interpreter_constraints, + internal_only=True, + ), + ), + Get(Pex, RequirementsPexRequest(addresses)), + Get( + PythonSourceFiles, + PythonSourceFilesRequest(all_targets, include_files=True, include_resources=True), + ), + Get( + PythonSourceFiles, + PythonSourceFilesRequest(targets), + ), + ) + + script_content = FileContent( + path="collect-fixtures.py", content=COLLECTION_SCRIPT.encode(), is_executable=True + ) + script_digest = await Get(Digest, CreateDigest([script_content])) + + pytest_runner_pex_get = Get( + VenvPex, + PexRequest( + output_filename="pytest_runner.pex", + interpreter_constraints=interpreter_constraints, + main=EntryPoint(PurePath(script_content.path).stem), + sources=script_digest, + internal_only=True, + pex_path=[ + pytest_pex, + requirements_pex, + ], + ), + ) + config_file_dirs = list(group_by_dir(prepared_sources.source_files.files).keys()) + config_files_get = Get( + ConfigFiles, + ConfigFilesRequest, + pytest.config_request(config_file_dirs), + ) + pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) + + pytest_config_digest = config_files.snapshot.digest + + input_digest = await Get( + Digest, + MergeDigests( + ( + prepared_sources.source_files.snapshot.digest, + pytest_config_digest, + ) + ), + ) + + extra_env = { + "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), + **test_extra_env.env, + } + + process = await Get( + Process, + VenvPexProcess( + pytest_runner_pex, + argv=[name for name in root_sources.source_files.files if name.endswith(".py")], + extra_env=extra_env, + input_digest=input_digest, + output_files=("tests.json",), + description="Collect test lockfile requirements from all tests.", + level=LogLevel.DEBUG, + cache_scope=ProcessCacheScope.PER_SESSION, + ), + ) + + result = await Get(ProcessResult, Process, process) + digest_contents = await Get(DigestContents, Digest, result.output_digest) + assert len(digest_contents) == 1 + assert digest_contents[0].path == "tests.json" + raw_config_data = json.loads(digest_contents[0].content) + + configs = [] + for item in raw_config_data: + config = JVMLockfileFixtureConfig( + definition=JVMLockfileFixtureDefinition.from_kwargs(item["kwargs"]), + test_file_path=item["test_file_path"], + ) + configs.append(config) + + return CollectedJVMLockfileFixtureConfigs(configs) + + +@rule +async def gather_lockfile_fixtures() -> RenderedJVMLockfileFixtures: + configs = await Get(CollectedJVMLockfileFixtureConfigs, CollectFixtureConfigsRequest()) + rendered_fixtures = [] + for config in configs: + artifact_reqs = ArtifactRequirements( + [ArtifactRequirement(coordinate) for coordinate in config.definition.coordinates] + ) + lockfile = await Get(CoursierResolvedLockfile, ArtifactRequirements, artifact_reqs) + serialized_lockfile = JVMLockfileMetadata.new(artifact_reqs).add_header_to_lockfile( + lockfile.to_serialized(), + regenerate_command=f"{bin_name()} {InternalGenerateTestLockfileFixturesSubsystem.name} ::", + delimeter="#", + ) + + lockfile_path = os.path.join( + os.path.dirname(config.test_file_path), config.definition.lockfile_rel_path + ) + rendered_fixtures.append( + RenderedJVMLockfileFixture( + content=serialized_lockfile, + path=lockfile_path, + ) + ) + + return RenderedJVMLockfileFixtures(rendered_fixtures) + + +class InternalGenerateTestLockfileFixturesSubsystem(GoalSubsystem): + name = "internal-generate-test-lockfile-fixtures" + help = "[Internal] Generate test lockfile fixtures for Pants tests." + + +class InternalGenerateTestLockfileFixturesGoal(Goal): + subsystem_cls = InternalGenerateTestLockfileFixturesSubsystem + + +@goal_rule +async def internal_render_test_lockfile_fixtures( + rendered_fixtures: RenderedJVMLockfileFixtures, + workspace: Workspace, + console: Console, +) -> InternalGenerateTestLockfileFixturesGoal: + if not rendered_fixtures: + console.write_stdout("No test lockfile fixtures found.\n") + return InternalGenerateTestLockfileFixturesGoal(exit_code=0) + + digest_contents = [ + FileContent(rendered_fixture.path, rendered_fixture.content) + for rendered_fixture in rendered_fixtures + ] + snapshot = await Get(Snapshot, CreateDigest(digest_contents)) + console.write_stdout(f"Writing test lockfile fixtures: {snapshot.files}\n") + workspace.write_digest(snapshot.digest) + return InternalGenerateTestLockfileFixturesGoal(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/pants.toml b/pants.toml index c148ab59625..fac82355a10 100644 --- a/pants.toml +++ b/pants.toml @@ -27,6 +27,7 @@ backend_packages.add = [ "pants.backend.experimental.scala.lint.scalafmt", "pants.backend.experimental.scala.debug_goals", "internal_plugins.releases", + "internal_plugins.test_lockfile_fixtures", ] plugins = [ "hdrhistogram", # For use with `--stats-log`. diff --git a/src/python/pants/backend/scala/BUILD b/src/python/pants/backend/scala/BUILD index 760486c9dd3..1aa4bfc2162 100644 --- a/src/python/pants/backend/scala/BUILD +++ b/src/python/pants/backend/scala/BUILD @@ -2,3 +2,5 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources() + +python_test_utils(name="test_utils") diff --git a/src/python/pants/backend/scala/compile/BUILD b/src/python/pants/backend/scala/compile/BUILD index 7fa863eee2d..f5ddbfb9bbe 100644 --- a/src/python/pants/backend/scala/compile/BUILD +++ b/src/python/pants/backend/scala/compile/BUILD @@ -2,4 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources() -python_tests(name="tests", timeout=240) + +python_tests(name="tests", timeout=240, dependencies=[":test_resources"]) +resources(name="test_resources", sources=["*.test.lock"]) \ No newline at end of file diff --git a/src/python/pants/backend/scala/compile/scala-library.test.lock b/src/python/pants/backend/scala/compile/scala-library.test.lock new file mode 100644 index 00000000000..a1e79051a63 --- /dev/null +++ b/src/python/pants/backend/scala/compile/scala-library.test.lock @@ -0,0 +1,26 @@ +# This lockfile was autogenerated by Pants. To regenerate, run: +# +# ./pants internal-generate-test-lockfile-fixtures +# +# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- +# { +# "version": 1, +# "generated_with_requirements": [ +# "org.scala-lang:scala-library:2.13.6,url=not_provided,jar=not_provided" +# ] +# } +# --- END PANTS LOCKFILE METADATA --- + +[[entries]] +directDependencies = [] +dependencies = [] +file_name = "org.scala-lang_scala-library_2.13.6.jar" + +[entries.coord] +group = "org.scala-lang" +artifact = "scala-library" +version = "2.13.6" +packaging = "jar" +[entries.file_digest] +fingerprint = "f19ed732e150d3537794fd3fe42ee18470a3f707efd499ecd05a99e727ff6c8a" +serialized_bytes_length = 5955737 diff --git a/src/python/pants/backend/scala/compile/scalac_test.py b/src/python/pants/backend/scala/compile/scalac_test.py index 2113faf33a0..2f6d08813c6 100644 --- a/src/python/pants/backend/scala/compile/scalac_test.py +++ b/src/python/pants/backend/scala/compile/scalac_test.py @@ -9,6 +9,7 @@ import pytest +from internal_plugins.test_lockfile_fixtures.lockfile_fixture import JVMLockfileFixture from pants.backend.scala.compile.scalac import CompileScalaSourceRequest from pants.backend.scala.compile.scalac import rules as scalac_rules from pants.backend.scala.dependency_inference.rules import rules as scala_dep_inf_rules @@ -65,6 +66,10 @@ def rule_runner() -> RuleRunner: return rule_runner +LOCKFILE_REQUIREMENTS = pytest.mark.jvm_lockfile( + path="scala-library.test.lock", requirements=["org.scala-lang:scala-library:2.13.6"] +) + DEFAULT_LOCKFILE = TestCoursierWrapper( CoursierResolvedLockfile( ( @@ -142,7 +147,8 @@ def main(args: Array[String]): Unit = { @maybe_skip_jdk_test -def test_compile_no_deps(rule_runner: RuleRunner) -> None: +@LOCKFILE_REQUIREMENTS +def test_compile_no_deps(rule_runner: RuleRunner, jvm_lockfile: JVMLockfileFixture) -> None: rule_runner.write_files( { "BUILD": dedent( @@ -153,7 +159,7 @@ def test_compile_no_deps(rule_runner: RuleRunner) -> None: """ ), "3rdparty/jvm/BUILD": DEFAULT_SCALA_LIBRARY_TARGET, - "3rdparty/jvm/default.lock": DEFAULT_LOCKFILE, + "3rdparty/jvm/default.lock": jvm_lockfile.serialized_lockfile, "ExampleLib.scala": SCALA_LIB_SOURCE, } ) diff --git a/src/python/pants/backend/scala/conftest.py b/src/python/pants/backend/scala/conftest.py new file mode 100644 index 00000000000..00f14708f3c --- /dev/null +++ b/src/python/pants/backend/scala/conftest.py @@ -0,0 +1,7 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from internal_plugins.test_lockfile_fixtures.lockfile_fixture import JvmLockfilePlugin + + +def pytest_configure(config): + config.pluginmanager.register(JvmLockfilePlugin())