diff --git a/.gitignore b/.gitignore index 0a61f55..c303a99 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,4 @@ htmlcov *-stubs coverage*.xml tags -*-output +*-out* diff --git a/.pylintrc b/.pylintrc index 0f7a7b9..49acf47 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,2 +1,2 @@ [DESIGN] -max-attributes=8 +max-attributes=10 diff --git a/README.md b/README.md index 48aeaf8..927dbd5 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ ===================================== generator=datazen version=3.1.2 - hash=4e253e82b70bf6a2a6479e0620da720b + hash=d6cd0e307dd25044b7eb3a7adaf13b33 ===================================== --> -# rcmpy ([1.0.0](https://pypi.org/project/rcmpy/)) +# rcmpy ([1.1.0](https://pypi.org/project/rcmpy/)) [![python](https://img.shields.io/pypi/pyversions/rcmpy.svg)](https://pypi.org/project/rcmpy/) ![Build Status](https://github.com/vkottler/rcmpy/workflows/Python%20Package/badge.svg) @@ -149,10 +149,12 @@ commands: ``` $ ./venv3.8/bin/rcmpy apply -h -usage: rcmpy apply [-h] +usage: rcmpy apply [-h] [-f] [-d] optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit + -f, --force whether or not to forcibly render all outputs + -d, --dry-run whether or not to update output files ``` diff --git a/local/configs/package.yaml b/local/configs/package.yaml index 8accbb5..89b575c 100644 --- a/local/configs/package.yaml +++ b/local/configs/package.yaml @@ -5,7 +5,7 @@ description: A configuration-file management system. entry: {{entry}} requirements: - datazen - - vcorelib>=1.5.2 + - vcorelib>=1.6.0 dev_requirements: - setuptools-wrapper - types-setuptools diff --git a/local/variables/package.yaml b/local/variables/package.yaml index 5b1b583..753387c 100644 --- a/local/variables/package.yaml +++ b/local/variables/package.yaml @@ -1,5 +1,5 @@ --- major: 1 -minor: 0 +minor: 1 patch: 0 entry: rcmpy diff --git a/pyproject.toml b/pyproject.toml index 963d7b3..141579e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "rcmpy" -version = "1.0.0" +version = "1.1.0" description = "A configuration-file management system." readme = "README.md" requires-python = ">=3.7" diff --git a/rcmpy/__init__.py b/rcmpy/__init__.py index b3d21be..8fd1da4 100644 --- a/rcmpy/__init__.py +++ b/rcmpy/__init__.py @@ -1,7 +1,7 @@ # ===================================== # generator=datazen # version=3.1.2 -# hash=de6d52cb7b3c576d64df05d995c59d25 +# hash=db6b75d059437ccfb73620d52efc9103 # ===================================== """ @@ -10,4 +10,4 @@ DESCRIPTION = "A configuration-file management system." PKG_NAME = "rcmpy" -VERSION = "1.0.0" +VERSION = "1.1.0" diff --git a/rcmpy/commands/apply.py b/rcmpy/commands/apply.py index ef11674..7169a0a 100644 --- a/rcmpy/commands/apply.py +++ b/rcmpy/commands/apply.py @@ -16,7 +16,7 @@ from rcmpy.environment import Environment, load_environment -def apply_env(env: Environment) -> int: +def apply_env(args: _Namespace, env: Environment) -> int: """Apply pending changes from the environment.""" result = 0 @@ -31,7 +31,7 @@ def apply_env(env: Environment) -> int: is_new = env.state.is_new() # Check if this file has any updated templates. - if is_new or env.is_updated(file): + if args.force or is_new or env.is_updated(file): template = env.templates_by_name[file.template] # If a template doesn't require rendering, use it as-is. @@ -44,13 +44,14 @@ def apply_env(env: Environment) -> int: path_fd.write(template.template.render(env.state.configs)) env.logger.info("Rendered '%s'.", rel(source)) - # Update the output file. - file.update(source, env.logger) + # Update the output file. + if not args.dry_run: + file.update(source, env.logger) return result -def apply_cmd(_: _Namespace) -> int: +def apply_cmd(args: _Namespace) -> int: """Execute the apply command.""" result = 1 @@ -58,12 +59,25 @@ def apply_cmd(_: _Namespace) -> int: with log_time(getLogger(__name__), "Command"): with load_environment() as env: if env.config_loaded: - result = apply_env(env) + result = apply_env(args, env) return result -def add_apply_cmd(_: _ArgumentParser) -> _CommandFunction: +def add_apply_cmd(parser: _ArgumentParser) -> _CommandFunction: """Add apply-command arguments to its parser.""" + parser.add_argument( + "-f", + "--force", + action="store_true", + help="whether or not to forcibly render all outputs", + ) + parser.add_argument( + "-d", + "--dry-run", + action="store_true", + help="whether or not to update output files", + ) + return apply_cmd diff --git a/rcmpy/config/__init__.py b/rcmpy/config/__init__.py index cd3cf67..591e17d 100644 --- a/rcmpy/config/__init__.py +++ b/rcmpy/config/__init__.py @@ -3,74 +3,22 @@ """ # built-in -from contextlib import suppress -from dataclasses import dataclass from os.path import expandvars from pathlib import Path -from shutil import copyfile from typing import Any, Dict, List, Set, cast # third-party from vcorelib.dict.codec import BasicDictCodec as _BasicDictCodec from vcorelib.io.types import JsonObject as _JsonObject -from vcorelib.logging import LoggerType -from vcorelib.paths import rel # internal +from rcmpy.config.file import ManagedFile from rcmpy.schemas import RcmpyDictCodec as _RcmpyDictCodec - -@dataclass -class ManagedFile: - """ - A data structure for managed files specified in the configuration data. - """ - - template: str - extra_templates: Set[str] - - directory: Path - name: str - - link: bool - - @property - def output(self) -> Path: - """Get the full output path.""" - return self.directory.joinpath(self.name) - - def update_root(self, root: Path) -> None: - """ - If the output directory is a relative path, update it to be an - absolute one based on the provided root directory. - """ - - if not self.directory.is_absolute(): - self.directory = root.joinpath(self.directory) - assert self.directory.is_absolute(), self.directory - - def update(self, source: Path, logger: LoggerType) -> None: - """Update this managed file based on the provided source file.""" - - output = self.output - - # At some point this could be skipped for existing symlinks. - with suppress(FileNotFoundError): - output.unlink() - - # Ensure the output directory exists. - self.directory.mkdir(parents=True, exist_ok=True) - - if self.link: - output.symlink_to(source) - else: - copyfile(source, output) - - logger.info("'%s' -> '%s'.", rel(source), rel(output)) - - FilesConfig = List[Dict[str, Any]] +__all__ = ["FilesConfig", "Config", "ManagedFile"] + class Config(_RcmpyDictCodec, _BasicDictCodec): """The top-level configuration object for the package.""" diff --git a/rcmpy/config/file.py b/rcmpy/config/file.py new file mode 100644 index 0000000..ddd21cc --- /dev/null +++ b/rcmpy/config/file.py @@ -0,0 +1,63 @@ +""" +A module implementing an interface for package-managed files. +""" + +# built-in +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from shutil import copyfile +from typing import Set + +# third-party +from vcorelib.logging import LoggerType +from vcorelib.paths import rel + + +@dataclass +class ManagedFile: + """ + A data structure for managed files specified in the configuration data. + """ + + template: str + extra_templates: Set[str] + + directory: Path + name: str + + link: bool + + @property + def output(self) -> Path: + """Get the full output path.""" + return self.directory.joinpath(self.name) + + def update_root(self, root: Path) -> None: + """ + If the output directory is a relative path, update it to be an + absolute one based on the provided root directory. + """ + + if not self.directory.is_absolute(): + self.directory = root.joinpath(self.directory) + assert self.directory.is_absolute(), self.directory + + def update(self, source: Path, logger: LoggerType) -> None: + """Update this managed file based on the provided source file.""" + + output = self.output + + # At some point this could be skipped for existing symlinks. + with suppress(FileNotFoundError): + output.unlink() + + # Ensure the output directory exists. + self.directory.mkdir(parents=True, exist_ok=True) + + if self.link: + output.symlink_to(source) + else: + copyfile(source, output) + + logger.info("'%s' -> '%s'.", rel(source), rel(output)) diff --git a/rcmpy/data/schemas/ManagedFile.yaml b/rcmpy/data/schemas/ManagedFile.yaml index 490665b..d207055 100644 --- a/rcmpy/data/schemas/ManagedFile.yaml +++ b/rcmpy/data/schemas/ManagedFile.yaml @@ -10,7 +10,7 @@ properties: # Relative to the data-repository root. directory: type: string - default: rcmpy-output + default: rcmpy-out # Whether or not to create a symbolic link instead of a copy. link: diff --git a/rcmpy/data/schemas/State.yaml b/rcmpy/data/schemas/State.yaml index 9fec655..6fa8ecd 100644 --- a/rcmpy/data/schemas/State.yaml +++ b/rcmpy/data/schemas/State.yaml @@ -10,6 +10,9 @@ properties: type: string default: "default" + manifest: + $ref: package://rcmpy/schemas/Config.yaml + previous: type: object additionalProperties: false diff --git a/rcmpy/environment/base.py b/rcmpy/environment/base.py index b8a851a..9900375 100644 --- a/rcmpy/environment/base.py +++ b/rcmpy/environment/base.py @@ -4,12 +4,14 @@ # built-in from contextlib import ExitStack +from pathlib import Path from typing import Optional # third-party -from vcorelib.io.types import FileExtension -from vcorelib.logging import LoggerMixin -from vcorelib.paths import rel +from vcorelib.io import ARBITER +from vcorelib.io.types import FileExtension, LoadResult +from vcorelib.logging import LoggerMixin, LoggerType +from vcorelib.paths import Pathlike, normalize, rel # internal from rcmpy import PKG_NAME @@ -18,6 +20,60 @@ from rcmpy.state import State +def load_if_single_candidate( + path: Pathlike, logger: LoggerType = None +) -> Optional[LoadResult]: + """ + Attempt to load a configuration file if a candidate exists at the given + path. + """ + + result = None + path = normalize(path) + + config_candidates = list( + FileExtension.data_candidates(path, exists_only=True) + ) + + if not config_candidates: + if logger is not None: + logger.error( + "No config found: %s.", + list(str(x) for x in FileExtension.data_candidates(path)), + ) + elif len(config_candidates) == 1: + result = ARBITER.decode( + config_candidates[0], + includes_key="includes", + expect_overwrite=True, + ) + if logger is not None: + logger.info("Loaded config '%s'.", rel(config_candidates[0])) + + return result + + +def load_manifest( + root: Path, variant: str, logger: LoggerType +) -> Optional[LoadResult]: + """Load the top-level data repository configuration.""" + + # Attempt to load the base manifest. + config_data = load_if_single_candidate( + root.joinpath(PKG_NAME), logger=logger + ) + + if config_data is not None: + # Attempt to load variant-specific manifest data. + variant_data = load_if_single_candidate( + root.joinpath("includes", variant) + ) + if variant_data is not None: + config_data.merge(variant_data, expect_overwrite=True) + + return config_data + + class BaseEnvironment(LoggerMixin): """A class implementing this package's base runtime environment.""" @@ -33,30 +89,19 @@ def __init__(self, state: State, stack: ExitStack) -> None: self.build = state.directory.joinpath("build") self.build.mkdir(exist_ok=True) - config_base = state.directory.joinpath(PKG_NAME) - - config_candidates = list( - FileExtension.data_candidates(config_base, exists_only=True) + config_data = load_manifest( + state.directory, state.variant, self.logger ) - - if not config_candidates: - self.logger.error( - "No config found: %s.", - list( - str(x) for x in FileExtension.data_candidates(config_base) - ), - ) - else: - assert len(config_candidates) == 1, config_candidates - self._config = Config.decode( - config_candidates[0], includes_key="includes" - ) - self.logger.info("Loaded config '%s'.", rel(config_candidates[0])) + if config_data is not None: + self._config = Config(data=config_data.data) # Ensure that any relative paths called out are relative to the # root directory of the data repository. self._config.update_root(state.directory) + # Treat manifest changes as criteria for updating outputs. + self.state.update_manifest(self._config.data) + # Consider the config not loaded if initialization fails. # # **Add this back in if initialization can actually fail.** diff --git a/rcmpy/requirements.txt b/rcmpy/requirements.txt index bca969a..d10c36c 100644 --- a/rcmpy/requirements.txt +++ b/rcmpy/requirements.txt @@ -1,2 +1,2 @@ datazen -vcorelib>=1.5.2 +vcorelib>=1.6.0 diff --git a/rcmpy/state/__init__.py b/rcmpy/state/__init__.py index f3104f7..2c7af25 100644 --- a/rcmpy/state/__init__.py +++ b/rcmpy/state/__init__.py @@ -33,6 +33,8 @@ class State(_RcmpyDictCodec): variant: str variables_new: bool configs_new: bool + manifest: Dict[str, Any] + manifest_new: bool def init(self, data: _JsonObject) -> None: """Perform implementation-specific initialization.""" @@ -64,14 +66,26 @@ def init(self, data: _JsonObject) -> None: self.previous.setdefault("configs", {}) self._load_configs() + # Manifest configuration. + self.manifest: Dict[str, Any] = cast( + Dict[str, Any], data.get("manifest", {}) + ) + self.manifest_new: bool = False + def is_new(self) -> bool: """Determine if state has changed.""" return ( self.variant != self.previous["variant"] or self.variables_new or self.configs_new + or self.manifest_new ) + def update_manifest(self, data: Dict[str, Any]) -> None: + """Set new manifest data.""" + self.manifest_new = self.manifest != data + self.manifest = data + def root_directories(self, subdir: str) -> List[Path]: """ Get up to a pair of directories from some sub-directory of the current @@ -107,6 +121,7 @@ def preprocessor(stream: DataStream) -> DataStream: require_success=True, recurse=True, includes_key="includes", + expect_overwrite=True, preprocessor=preprocessor, ).data, logger=self.logger, @@ -130,6 +145,7 @@ def _load_variables(self) -> None: require_success=True, recurse=True, includes_key="includes", + expect_overwrite=True, ).data, logger=self.logger, ) @@ -171,6 +187,7 @@ def asdict(self) -> _JsonObject: "directory": str(self.directory), "variant": self.variant, "previous": self.previous, + "manifest": self.manifest, } diff --git a/tests/data/valid/scenarios/simple/includes/default.yaml b/tests/data/valid/scenarios/simple/includes/default.yaml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/tests/data/valid/scenarios/simple/includes/default.yaml @@ -0,0 +1 @@ +--- diff --git a/tests/data/valid/scenarios/simple/includes/test.yaml b/tests/data/valid/scenarios/simple/includes/test.yaml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/tests/data/valid/scenarios/simple/includes/test.yaml @@ -0,0 +1 @@ +---