diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b19d3c0dce..728a95fb90 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 diff --git a/pants-plugins/README.md b/pants-plugins/README.md index d7b5729758..8f99aa3d5e 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". +These StackStorm-specific plugins might be useful in other StackStorm-related repos. +- `pack_metadata` + These StackStorm-specific plugins are probably only useful for the st2 repo. - `api_spec` - `sample_conf` @@ -26,6 +29,26 @@ This plugin also wires up pants so that the `lint` goal runs additional api spec validation on `st2common/st2common/openapi.yaml` with something like `./pants lint st2common/st2common/openapi.yaml`. +### `pack_metadata` plugin + +This plugin adds two new targets to pants: +- `pack_metadata` +- `pack_metadata_in_git_submodule` + +These targets include all StackStorm pack metadata files in a pack. +Pack metadata includes top-level files (`pack.yaml`, `.yaml.example`, +`config.schema.yaml`, and `icon.png`) and metadata (`*.yaml`, `*.yml`) +for actions, action-aliases, policies, rules, and sensors. + +This plugin also wires up the `tailor` goal, so that it will add a +`pack_metadata(name="metadata")` target wherever it finds a `pack.yaml` file. + +One of the packs in this repo is in a git submodule to test our handling +of git submodules (`st2tests/st2tests/fixtures/packs/test_content_version`). +If it is not checked out, then some of the tests will fail. +If it is not checked out, `pack_metadata_in_git_submodule` handles providing +a helpful, instructive error message as early as possible. + ### `sample_conf` plugin This plugin wires up pants to make sure `conf/st2.conf.sample` gets diff --git a/pants-plugins/pack_metadata/BUILD b/pants-plugins/pack_metadata/BUILD new file mode 100644 index 0000000000..0eea8b1cf1 --- /dev/null +++ b/pants-plugins/pack_metadata/BUILD @@ -0,0 +1,5 @@ +python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/pack_metadata/__init__.py b/pants-plugins/pack_metadata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py new file mode 100644 index 0000000000..34a6aa85cc --- /dev/null +++ b/pants-plugins/pack_metadata/register.py @@ -0,0 +1,23 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pack_metadata import tailor +from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule + + +def rules(): + return tailor.rules() + + +def target_types(): + return [PackMetadata, PackMetadataInGitSubmodule] diff --git a/pants-plugins/pack_metadata/tailor.py b/pants-plugins/pack_metadata/tailor.py new file mode 100644 index 0000000000..88f714ce36 --- /dev/null +++ b/pants-plugins/pack_metadata/tailor.py @@ -0,0 +1,62 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from dataclasses import dataclass + +from pants.core.goals.tailor import ( + AllOwnedSources, + PutativeTarget, + PutativeTargets, + PutativeTargetsRequest, +) +from pants.engine.fs import PathGlobs, Paths +from pants.engine.rules import collect_rules, Get, rule, UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.target_types import PackMetadata + + +@dataclass(frozen=True) +class PutativePackMetadataTargetsRequest(PutativeTargetsRequest): + pass + + +@rule( + desc="Find pack (config, action, alias, sensor, icon, etc) metadata files.", + level=LogLevel.DEBUG, +) +async def find_putative_targets( + _: PutativePackMetadataTargetsRequest, all_owned_sources: AllOwnedSources +) -> PutativeTargets: + all_pack_yaml_files = await Get(Paths, PathGlobs(["**/pack.yaml"])) + + unowned_pack_yaml_files = set(all_pack_yaml_files.files) - set(all_owned_sources) + unowned_pack_dirs = [os.path.dirname(p) for p in unowned_pack_yaml_files] + + name = "metadata" + return PutativeTargets( + [ + PutativeTarget.for_target_type( + PackMetadata, dirname, name, ("pack.yaml",), kwargs={"name": name} + ) + for dirname in unowned_pack_dirs + ] + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PutativeTargetsRequest, PutativePackMetadataTargetsRequest), + ] diff --git a/pants-plugins/pack_metadata/tailor_test.py b/pants-plugins/pack_metadata/tailor_test.py new file mode 100644 index 0000000000..26974fbb83 --- /dev/null +++ b/pants-plugins/pack_metadata/tailor_test.py @@ -0,0 +1,107 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.core.goals.tailor import ( + AllOwnedSources, + PutativeTarget, + PutativeTargets, +) +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .tailor import ( + PutativePackMetadataTargetsRequest, + rules as pack_metadata_rules, +) +from .target_types import PackMetadata, PackMetadataInGitSubmodule + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *pack_metadata_rules(), + QueryRule( + PutativeTargets, (PutativePackMetadataTargetsRequest, AllOwnedSources) + ), + ], + target_types=[PackMetadata, PackMetadataInGitSubmodule], + ) + + +def test_find_putative_targets(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/already_owned/pack.yaml": "---\nname: already_owned\n", + "packs/already_owned/actions/action.yaml": "---\nname: action\n", + "packs/foo/pack.yaml": "---\nname: foo\n", + "packs/foo/actions/action.yaml": "---\nname: action\n", + "packs/bar/pack.yaml": "---\nname: bar\n", + "packs/bar/sensors/sensor.yaml": "---\nname: sensor\n", + "other/deep/baz/pack.yaml": "---\nname: baz\n", + } + ) + pts = rule_runner.request( + PutativeTargets, + [ + PutativePackMetadataTargetsRequest( + ( + "packs", + "packs/already_owned", + "packs/already_owned/actions", + "packs/foo", + "packs/foo/actions", + "packs/bar", + "packs/bar/sensors", + "other/deep/baz", + ) + ), + AllOwnedSources( + [ + "packs/already_owned/pack.yaml", + "packs/already_owned/actions/action.yaml", + ] + ), + ], + ) + assert ( + PutativeTargets( + [ + PutativeTarget.for_target_type( + PackMetadata, + path="packs/foo", + name="metadata", + triggering_sources=["pack.yaml"], + kwargs={"name": "metadata"}, + ), + PutativeTarget.for_target_type( + PackMetadata, + path="packs/bar", + name="metadata", + triggering_sources=["pack.yaml"], + kwargs={"name": "metadata"}, + ), + PutativeTarget.for_target_type( + PackMetadata, + path="other/deep/baz", + name="metadata", + triggering_sources=["pack.yaml"], + kwargs={"name": "metadata"}, + ), + ] + ) + == pts + ) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py new file mode 100644 index 0000000000..cfadbaab01 --- /dev/null +++ b/pants-plugins/pack_metadata/target_types.py @@ -0,0 +1,79 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Sequence + +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from pants.core.target_types import ( + ResourcesGeneratingSourcesField, + ResourcesGeneratorTarget, +) + + +class UnmatchedGlobsError(Exception): + """Error thrown when a required set of globs didn't match.""" + + +class PackMetadataSourcesField(ResourcesGeneratingSourcesField): + required = False + default = ( + # metadata does not include any python, shell, or other sources. + "pack.yaml", + "config.schema.yaml", + "*.yaml.example", + "**/*.yaml", + "**/*.yml", + "icon.png", # used in st2web ui + # "requirements*.txt", # including this causes target conflicts + # "README.md", + # "HISTORY.md", + ) + + +class PackMetadataInGitSubmoduleSources(PackMetadataSourcesField): + required = True + + def validate_resolved_files(self, files: Sequence[str]) -> None: + if not files: + raise UnmatchedGlobsError( + # see: st2tests.fixturesloader.GIT_SUBMODULES_NOT_CHECKED_OUT_ERROR + "One or more git submodules is not checked out. Make sure to run " + '"git submodule update --init --recursive"' + "in the repository root directory to check out all the submodules." + ) + super().validate_resolved_files(files) + + +class PackMetadata(ResourcesGeneratorTarget): + alias = "pack_metadata" + core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) + help = ( + "Loose pack metadata files.\n\n" + "Pack metadata includes top-level files (pack.yaml, .yaml.example, " + "config.schema.yaml, and icon.png) and metadata for actions, " + "action-aliases, policies, rules, and sensors." + ) + + +class PackMetadataInGitSubmodule(PackMetadata): + alias = "pack_metadata_in_git_submodule" + core_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + PackMetadataInGitSubmoduleSources, + ) + help = PackMetadata.help + ( + "\npack_metadata_in_git_submodule variant errors if the sources field " + "has unmatched globs. It prints instructions on how to checkout git " + "submodules." + ) diff --git a/pants-plugins/pack_metadata/target_types_test.py b/pants-plugins/pack_metadata/target_types_test.py new file mode 100644 index 0000000000..93a8a23292 --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_test.py @@ -0,0 +1,68 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import RuleRunner + +from .target_types import ( + PackMetadata, + # PackMetadataSourcesField, + PackMetadataInGitSubmodule, + # PackMetadataInGitSubmoduleSources, + UnmatchedGlobsError, +) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[], + target_types=[PackMetadata, PackMetadataInGitSubmodule], + ) + + +GIT_SUBMODULE_BUILD_FILE = """ +pack_metadata_in_git_submodule( + name="metadata", + sources=["./submodule_dir/pack.yaml"], +) +""" + + +def test_git_submodule_sources_missing(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, + } + ) + with pytest.raises(ExecutionError) as e: + _ = rule_runner.get_target(Address("packs", target_name="metadata")) + exc = e.value.wrapped_exceptions[0] + assert isinstance(exc, UnmatchedGlobsError) + assert "One or more git submodules is not checked out" in str(exc) + + +def test_git_submodule_sources_present(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "packs/BUILD": GIT_SUBMODULE_BUILD_FILE, + "packs/submodule_dir/pack.yaml": "---\nname: foobar\n", + } + ) + # basically: this asserts that it does not raise UnmatchedGlobsError + _ = rule_runner.get_target(Address("packs", target_name="metadata")) diff --git a/pants.toml b/pants.toml index 15a057cf02..e9bf57cb95 100644 --- a/pants.toml +++ b/pants.toml @@ -25,6 +25,7 @@ backend_packages = [ # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", + #"pack_metadata", "sample_conf", "schemas", ] diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 407369573e..4813fd914d 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -1,7 +1,9 @@ # The files in test_content_version* targets are in ./test_content_version -# which is a git submodule. +# which is a git submodule. pack_metadata_in_git_submodule will error with +# instructions on how to checkout the submodules if they are not checked out. # The test_content_version* targets are dependencies of ./test_content_version_fixture +# pack_metadata_in_git_submodule( resources( name="test_content_version_metadata", sources=[