From 2ea65cdb5bcbc2d3768505673b48cd622713a75c Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 10 Aug 2023 13:19:02 +0300 Subject: [PATCH 01/14] Added clear and apply commands to ocean CLI --- integrations/pagerduty/main.py | 6 +-- port_ocean/cli/commands/__init__.py | 4 ++ port_ocean/cli/commands/apply_defaults.py | 41 +++++++++++++++++ port_ocean/cli/commands/clear_defaults.py | 46 +++++++++++++++++++ port_ocean/clients/port/mixins/entities.py | 18 ++++++++ port_ocean/config/base.py | 1 - port_ocean/port_defaults.py | 51 ++++++++++++++++++++-- port_ocean/run.py | 8 ++-- 8 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 port_ocean/cli/commands/apply_defaults.py create mode 100644 port_ocean/cli/commands/clear_defaults.py diff --git a/integrations/pagerduty/main.py b/integrations/pagerduty/main.py index 59a56072c0..f65ff668c2 100644 --- a/integrations/pagerduty/main.py +++ b/integrations/pagerduty/main.py @@ -12,9 +12,9 @@ class ObjectKind: pager_duty_client = PagerDutyClient( - ocean.integration_config["token"], - ocean.integration_config["api_url"], - ocean.integration_config["app_host"], + ocean.integration_config.get("token", ""), + ocean.integration_config.get("api_url", ""), + ocean.integration_config.get("app_host", ""), ) diff --git a/port_ocean/cli/commands/__init__.py b/port_ocean/cli/commands/__init__.py index 2e46a59335..80ab3f8c1a 100644 --- a/port_ocean/cli/commands/__init__.py +++ b/port_ocean/cli/commands/__init__.py @@ -4,6 +4,8 @@ from .pull import pull from .sail import sail from .version import version +from .apply_defaults import apply_defaults +from .clear_defaults import clear_defaults __all__ = [ "cli_start", @@ -12,4 +14,6 @@ "pull", "sail", "version", + "apply_defaults", + "clear_defaults" ] diff --git a/port_ocean/cli/commands/apply_defaults.py b/port_ocean/cli/commands/apply_defaults.py new file mode 100644 index 0000000000..ca72e1b847 --- /dev/null +++ b/port_ocean/cli/commands/apply_defaults.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +from inspect import getmembers +import os +import click +from cookiecutter.main import cookiecutter # type: ignore + +from port_ocean import __version__ +from port_ocean.cli.commands.main import cli_start, print_logo, console +from port_ocean.cli.utils import cli_root_path +from port_ocean.ocean import Ocean +from port_ocean.port_defaults import initialize_defaults +from port_ocean.run import _create_default_app, _load_module + + +@cli_start.command() +@click.argument("path", default=".", type=click.Path(exists=True)) +def apply_defaults(path: str) -> None: + """ + Apply defaults of the integration from the .port/resources PATH. + + PATH: Path to the integration. If not provided, the current directory will be used. + """ + print_logo() + + console.print("Applying default blueprints and configurations! ⚓️") + + default_app = _create_default_app(path, False) + + main_path = f"{path}/main.py" if path else "main.py" + + app_module = _load_module(main_path) + app: Ocean = {name: item for name, item in getmembers(app_module)}.get( + "app", + default_app, + ) + + initialize_defaults( + app.integration.AppConfigHandlerClass.CONFIG_CLASS, + app.config, + ) diff --git a/port_ocean/cli/commands/clear_defaults.py b/port_ocean/cli/commands/clear_defaults.py new file mode 100644 index 0000000000..ce99b6451a --- /dev/null +++ b/port_ocean/cli/commands/clear_defaults.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +from inspect import getmembers +import os +import click +from cookiecutter.main import cookiecutter # type: ignore + +from port_ocean import __version__ +from port_ocean.cli.commands.main import cli_start, print_logo, console +from port_ocean.cli.utils import cli_root_path +from port_ocean.ocean import Ocean +from port_ocean.port_defaults import clear_defaults as clear +from port_ocean.run import _create_default_app, _load_module + + +@cli_start.command() +@click.argument("path", default=".", type=click.Path(exists=True)) +@click.option( + "-f", + "--force", + "force", + type=bool, + default=False, + help="Delete all the entities of the Blueprint as well as the blueprint itself.", +) +def clear_defaults(path: str, force: bool) -> None: + """ + Apply defaults of the integration from the .port/resources PATH. + + PATH: Path to the integration. If not provided, the current directory will be used. + """ + print_logo() + + console.print("Applying default blueprints and configurations! ⚓️") + + default_app = _create_default_app(path, False) + + main_path = f"{path}/main.py" if path else "main.py" + + app_module = _load_module(main_path) + app: Ocean = {name: item for name, item in getmembers(app_module)}.get( + "app", + default_app, + ) + + clear(app.integration.AppConfigHandlerClass.CONFIG_CLASS, force) diff --git a/port_ocean/clients/port/mixins/entities.py b/port_ocean/clients/port/mixins/entities.py index e14cc9024f..d0ef58bb24 100644 --- a/port_ocean/clients/port/mixins/entities.py +++ b/port_ocean/clients/port/mixins/entities.py @@ -155,3 +155,21 @@ async def validate_entity_payload( "validation_only": True, }, ) + + async def delete_all_entities( + self, + blueprint_identifier: str, + should_raise: bool, + ) -> None: + logger.info(f"Delete all entities of blueprint: {blueprint_identifier}") + response = await self.client.delete( + f"{self.auth.api_url}/blueprints/{blueprint_identifier}/all-entities", + headers=await self.auth.headers(), + ) + + if response.is_error: + logger.error( + f"Error deleting all entities " f"blueprint: {blueprint_identifier}" + ) + + handle_status_code(response, should_raise) diff --git a/port_ocean/config/base.py b/port_ocean/config/base.py index 262d90e417..a8799595cc 100644 --- a/port_ocean/config/base.py +++ b/port_ocean/config/base.py @@ -18,7 +18,6 @@ def read_yaml_config_settings_source( yaml_file = getattr(settings.__config__, "yaml_file", "") assert yaml_file, "Settings.yaml_file not properly configured" - path = Path(base_path, yaml_file) if not path.exists(): diff --git a/port_ocean/port_defaults.py b/port_ocean/port_defaults.py index 05c35cf5b4..01de549ea9 100644 --- a/port_ocean/port_defaults.py +++ b/port_ocean/port_defaults.py @@ -152,11 +152,9 @@ async def _initialize_defaults( config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration ) -> None: port_client = ocean.port_client - is_exists = await _is_integration_exists(port_client) - if is_exists: - return None defaults = get_port_integration_defaults(config_class) if not defaults: + logger.warning("No defaults found. Skipping...") return None try: @@ -177,6 +175,41 @@ async def _initialize_defaults( raise ExceptionGroup(str(e), e.errors) +async def _clear_defaults( + config_class: Type[PortAppConfig], + force, +) -> None: + port_client = ocean.port_client + is_exists = await _is_integration_exists(port_client) + if not is_exists: + return None + defaults = get_port_integration_defaults(config_class) + if not defaults: + return None + + try: + if force: + await asyncio.gather( + *( + port_client.delete_all_entities( + should_raise=False, blueprint_identifier=blueprint["identifier"] + ) + for blueprint in defaults.blueprints + ) + ) + await asyncio.gather( + *( + port_client.delete_blueprint( + blueprint["identifier"], should_raise=False + ) + for blueprint in defaults.blueprints + ) + ) + except httpx.HTTPStatusError as e: + logger.error(f"Failed to delete blueprints: {e.response.text}.") + raise e + + def initialize_defaults( config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration ) -> None: @@ -188,6 +221,18 @@ def initialize_defaults( logger.debug(f"Failed to initialize defaults, skipping... Error: {e}") +def clear_defaults( + config_class: Type[PortAppConfig], + force: bool, +) -> None: + try: + asyncio.new_event_loop().run_until_complete( + _clear_defaults(config_class, force) + ) + except Exception as e: + logger.debug(f"Failed to clear defaults, skipping... Error: {e}") + + def get_port_integration_defaults( port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".") ) -> Defaults | None: diff --git a/port_ocean/run.py b/port_ocean/run.py index fd1f43e15b..f4d1ea0d38 100644 --- a/port_ocean/run.py +++ b/port_ocean/run.py @@ -31,7 +31,7 @@ def _get_base_integration_class_from_module( def _load_module(file_path: str) -> ModuleType: - spec = spec_from_file_location("module_name", file_path) + spec = spec_from_file_location("module.name", file_path) if spec is None or spec.loader is None: raise Exception(f"Failed to load integration from path: {file_path}") @@ -41,7 +41,9 @@ def _load_module(file_path: str) -> ModuleType: return module -def _create_default_app(path: str | None = None) -> Ocean: +def _create_default_app( + path: str | None = None, use_default_config_factory: bool = False +) -> Ocean: sys.path.append(".") try: integration_path = f"{path}/integration.py" if path else "integration.py" @@ -52,7 +54,7 @@ def _create_default_app(path: str | None = None) -> Ocean: spec = get_spec_file() config_factory = None - if spec is not None: + if spec is not None and use_default_config_factory: config_factory = default_config_factory(spec.get("configurations", [])) return Ocean(integration_class=integration_class, config_factory=config_factory) From a9e987640608047dc2a1e31dd1e0745fa5516ac8 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 10 Aug 2023 13:19:36 +0300 Subject: [PATCH 02/14] bumped version of ocean framework --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29b5415556..2bd11be8e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.2.1" +version = "0.2.2" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io" From b461cace854af639b2a787ae3e3ac7203439688b Mon Sep 17 00:00:00 2001 From: danielsinai Date: Sun, 13 Aug 2023 14:56:44 +0300 Subject: [PATCH 03/14] Added group of defaults --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 136b6169f2..f8263e198a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.2.0 (2023-08-10) +================== + +No significant changes. + + 0.2.1 (2023-08-09) ================== From 09a9b32a0ac76f08dee3609ea139ab76f61154d4 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Sun, 13 Aug 2023 15:47:06 +0300 Subject: [PATCH 04/14] Added migration ids --- port_ocean/cli/commands/__init__.py | 8 +-- port_ocean/cli/commands/clear_defaults.py | 46 --------------- port_ocean/cli/commands/defaults/__init___.py | 7 +++ port_ocean/cli/commands/defaults/clean.py | 58 +++++++++++++++++++ .../{apply_defaults.py => defaults/dock.py} | 16 +++-- port_ocean/cli/commands/defaults/group.py | 6 ++ .../{ => cli/defaults}/port_defaults.py | 24 +++----- port_ocean/clients/port/mixins/blueprints.py | 29 +++++++--- port_ocean/clients/port/mixins/entities.py | 18 ------ port_ocean/run.py | 2 +- 10 files changed, 112 insertions(+), 102 deletions(-) delete mode 100644 port_ocean/cli/commands/clear_defaults.py create mode 100644 port_ocean/cli/commands/defaults/__init___.py create mode 100644 port_ocean/cli/commands/defaults/clean.py rename port_ocean/cli/commands/{apply_defaults.py => defaults/dock.py} (72%) create mode 100644 port_ocean/cli/commands/defaults/group.py rename port_ocean/{ => cli/defaults}/port_defaults.py (93%) diff --git a/port_ocean/cli/commands/__init__.py b/port_ocean/cli/commands/__init__.py index 80ab3f8c1a..d49ce36451 100644 --- a/port_ocean/cli/commands/__init__.py +++ b/port_ocean/cli/commands/__init__.py @@ -4,8 +4,8 @@ from .pull import pull from .sail import sail from .version import version -from .apply_defaults import apply_defaults -from .clear_defaults import clear_defaults +from .defaults.dock import dock +from .defaults.clean import clean __all__ = [ "cli_start", @@ -14,6 +14,6 @@ "pull", "sail", "version", - "apply_defaults", - "clear_defaults" + "dock", + "clean", ] diff --git a/port_ocean/cli/commands/clear_defaults.py b/port_ocean/cli/commands/clear_defaults.py deleted file mode 100644 index ce99b6451a..0000000000 --- a/port_ocean/cli/commands/clear_defaults.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -from inspect import getmembers -import os -import click -from cookiecutter.main import cookiecutter # type: ignore - -from port_ocean import __version__ -from port_ocean.cli.commands.main import cli_start, print_logo, console -from port_ocean.cli.utils import cli_root_path -from port_ocean.ocean import Ocean -from port_ocean.port_defaults import clear_defaults as clear -from port_ocean.run import _create_default_app, _load_module - - -@cli_start.command() -@click.argument("path", default=".", type=click.Path(exists=True)) -@click.option( - "-f", - "--force", - "force", - type=bool, - default=False, - help="Delete all the entities of the Blueprint as well as the blueprint itself.", -) -def clear_defaults(path: str, force: bool) -> None: - """ - Apply defaults of the integration from the .port/resources PATH. - - PATH: Path to the integration. If not provided, the current directory will be used. - """ - print_logo() - - console.print("Applying default blueprints and configurations! ⚓️") - - default_app = _create_default_app(path, False) - - main_path = f"{path}/main.py" if path else "main.py" - - app_module = _load_module(main_path) - app: Ocean = {name: item for name, item in getmembers(app_module)}.get( - "app", - default_app, - ) - - clear(app.integration.AppConfigHandlerClass.CONFIG_CLASS, force) diff --git a/port_ocean/cli/commands/defaults/__init___.py b/port_ocean/cli/commands/defaults/__init___.py new file mode 100644 index 0000000000..335c55a42b --- /dev/null +++ b/port_ocean/cli/commands/defaults/__init___.py @@ -0,0 +1,7 @@ +from .dock import dock +from .clean import clean + +__all__ = [ + "dock", + "clean", +] diff --git a/port_ocean/cli/commands/defaults/clean.py b/port_ocean/cli/commands/defaults/clean.py new file mode 100644 index 0000000000..e68502348d --- /dev/null +++ b/port_ocean/cli/commands/defaults/clean.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +import click + +from inspect import getmembers +from .group import defaults +from port_ocean import __version__ +from port_ocean.cli.commands.main import print_logo, console +from port_ocean.cli.utils import cli_root_path +from port_ocean.ocean import Ocean +from port_ocean.cli.defaults.port_defaults import clean_defaults +from port_ocean.run import _create_default_app, _load_module + + +@defaults.command() +@click.argument("path", default=".", type=click.Path(exists=True)) +@click.option( + "-f", + "--force", + "force", + type=bool, + default=False, + help="Delete all the entities of the Blueprint as well as the blueprint itself.", +) +def clean(path: str, force: bool) -> None: + """ + Clean defaults of the integration from the .port/resources PATH. + + PATH: Path to the integration. If not provided, the current directory will be used. + """ + print_logo() + + console.print("Cleaning blueprints and configurations! ⚓️") + + if force: + console.print( + "Deleting entities forcefully I sure hope you know what you are doing 🚨 🚨 🚨 ", + ) + default_app = _create_default_app(path, False) + + main_path = f"{path}/main.py" if path else "main.py" + + app_module = _load_module(main_path) + app: Ocean = {name: item for name, item in getmembers(app_module)}.get( + "app", + default_app, + ) + + migration_ids = clean_defaults( + app.integration.AppConfigHandlerClass.CONFIG_CLASS, force + ) + + if ( + migration_ids + and len([migration_id for migration_id in migration_ids if migration_id]) > 0 + ): + console.print( + f"The clean migration has started, you can track the migration process using the following migration ids {migration_ids} ⚓️" + ) diff --git a/port_ocean/cli/commands/apply_defaults.py b/port_ocean/cli/commands/defaults/dock.py similarity index 72% rename from port_ocean/cli/commands/apply_defaults.py rename to port_ocean/cli/commands/defaults/dock.py index ca72e1b847..1603e4b515 100644 --- a/port_ocean/cli/commands/apply_defaults.py +++ b/port_ocean/cli/commands/defaults/dock.py @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- - -from inspect import getmembers -import os import click -from cookiecutter.main import cookiecutter # type: ignore +from inspect import getmembers +from .group import defaults from port_ocean import __version__ -from port_ocean.cli.commands.main import cli_start, print_logo, console +from port_ocean.cli.commands.main import print_logo, console from port_ocean.cli.utils import cli_root_path from port_ocean.ocean import Ocean -from port_ocean.port_defaults import initialize_defaults +from port_ocean.cli.defaults.port_defaults import initialize_defaults from port_ocean.run import _create_default_app, _load_module -@cli_start.command() +@defaults.command() @click.argument("path", default=".", type=click.Path(exists=True)) -def apply_defaults(path: str) -> None: +def dock(path: str) -> None: """ Apply defaults of the integration from the .port/resources PATH. @@ -23,7 +21,7 @@ def apply_defaults(path: str) -> None: """ print_logo() - console.print("Applying default blueprints and configurations! ⚓️") + console.print("Unloading cargo at the dock... 📦🚢") default_app = _create_default_app(path, False) diff --git a/port_ocean/cli/commands/defaults/group.py b/port_ocean/cli/commands/defaults/group.py new file mode 100644 index 0000000000..4155c2a7e3 --- /dev/null +++ b/port_ocean/cli/commands/defaults/group.py @@ -0,0 +1,6 @@ +from port_ocean.cli.commands.main import cli_start + + +@cli_start.group("defaults") +def defaults(): + pass diff --git a/port_ocean/port_defaults.py b/port_ocean/cli/defaults/port_defaults.py similarity index 93% rename from port_ocean/port_defaults.py rename to port_ocean/cli/defaults/port_defaults.py index 01de549ea9..f96cd5292f 100644 --- a/port_ocean/port_defaults.py +++ b/port_ocean/cli/defaults/port_defaults.py @@ -175,7 +175,7 @@ async def _initialize_defaults( raise ExceptionGroup(str(e), e.errors) -async def _clear_defaults( +async def _clean_defaults( config_class: Type[PortAppConfig], force, ) -> None: @@ -188,19 +188,10 @@ async def _clear_defaults( return None try: - if force: - await asyncio.gather( - *( - port_client.delete_all_entities( - should_raise=False, blueprint_identifier=blueprint["identifier"] - ) - for blueprint in defaults.blueprints - ) - ) - await asyncio.gather( + return await asyncio.gather( *( port_client.delete_blueprint( - blueprint["identifier"], should_raise=False + blueprint["identifier"], should_raise=True, delete_entities=force ) for blueprint in defaults.blueprints ) @@ -221,14 +212,15 @@ def initialize_defaults( logger.debug(f"Failed to initialize defaults, skipping... Error: {e}") -def clear_defaults( +def clean_defaults( config_class: Type[PortAppConfig], force: bool, -) -> None: +) -> list[str] | None: try: - asyncio.new_event_loop().run_until_complete( - _clear_defaults(config_class, force) + migration_ids = asyncio.new_event_loop().run_until_complete( + _clean_defaults(config_class, force) ) + return migration_ids except Exception as e: logger.debug(f"Failed to clear defaults, skipping... Error: {e}") diff --git a/port_ocean/clients/port/mixins/blueprints.py b/port_ocean/clients/port/mixins/blueprints.py index b57b1bd361..4718ba934f 100644 --- a/port_ocean/clients/port/mixins/blueprints.py +++ b/port_ocean/clients/port/mixins/blueprints.py @@ -45,15 +45,28 @@ async def patch_blueprint( handle_status_code(response) async def delete_blueprint( - self, identifier: str, should_raise: bool = False - ) -> None: - logger.info(f"Deleting blueprint with id: {identifier}") - headers = await self.auth.headers() - response = await self.client.delete( - f"{self.auth.api_url}/blueprints/{identifier}", - headers=headers, + self, identifier: str, should_raise: bool = False, delete_entities: bool = False + ) -> None or str: + logger.info( + f"Deleting blueprint with id: {identifier} with all entities: {delete_entities}" ) - handle_status_code(response, should_raise) + headers = await self.auth.headers() + response = None + if not delete_entities: + response = await self.client.delete( + f"{self.auth.api_url}/blueprints/{identifier}", + headers=headers, + ) + handle_status_code(response, should_raise) + return None + else: + response = await self.client.delete( + f"{self.auth.api_url}/blueprints/{identifier}/all-entities?delete_blueprint=true", + headers=await self.auth.headers(), + ) + + handle_status_code(response, should_raise) + return response.json().get("migrationId", "") async def create_action( self, blueprint_identifier: str, action: dict[str, Any] diff --git a/port_ocean/clients/port/mixins/entities.py b/port_ocean/clients/port/mixins/entities.py index d0ef58bb24..e14cc9024f 100644 --- a/port_ocean/clients/port/mixins/entities.py +++ b/port_ocean/clients/port/mixins/entities.py @@ -155,21 +155,3 @@ async def validate_entity_payload( "validation_only": True, }, ) - - async def delete_all_entities( - self, - blueprint_identifier: str, - should_raise: bool, - ) -> None: - logger.info(f"Delete all entities of blueprint: {blueprint_identifier}") - response = await self.client.delete( - f"{self.auth.api_url}/blueprints/{blueprint_identifier}/all-entities", - headers=await self.auth.headers(), - ) - - if response.is_error: - logger.error( - f"Error deleting all entities " f"blueprint: {blueprint_identifier}" - ) - - handle_status_code(response, should_raise) diff --git a/port_ocean/run.py b/port_ocean/run.py index f4d1ea0d38..d4a04da29e 100644 --- a/port_ocean/run.py +++ b/port_ocean/run.py @@ -11,7 +11,7 @@ from port_ocean.core.integrations.base import BaseIntegration from port_ocean.logger_setup import setup_logger from port_ocean.ocean import Ocean -from port_ocean.port_defaults import initialize_defaults +from port_ocean.cli.defaults.port_defaults import initialize_defaults from port_ocean.utils import get_spec_file From a4e9066acaa04c3c928893fdf58c8ffd1d371039 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Sun, 13 Aug 2023 15:48:42 +0300 Subject: [PATCH 05/14] Reverted change log --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 136b6169f2..dae6fee016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.2.2 (2023-08-11) +================== + +### Bug Fixes + +- Fixed an issue causing the config yaml providers to not be parsed + + 0.2.1 (2023-08-09) ================== @@ -123,4 +131,4 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Added `ocean version` to get the framework version. - Added `make new` to scaffold in the Ocean repository. - (PORT-4307) + (PORT-4307) \ No newline at end of file From 3e47c8659154e277c1a6f7a02004973a36278a20 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Sun, 13 Aug 2023 16:50:54 +0300 Subject: [PATCH 06/14] Added an option to wait for a migration to finish --- port_ocean/cli/commands/defaults/clean.py | 20 +-- port_ocean/cli/commands/defaults/dock.py | 2 +- port_ocean/cli/defaults/__init__.py | 7 ++ port_ocean/cli/defaults/clean.py | 73 +++++++++++ port_ocean/cli/defaults/common.py | 119 ++++++++++++++++++ .../{port_defaults.py => initialize.py} | 116 +---------------- port_ocean/clients/port/client.py | 10 +- port_ocean/clients/port/mixins/migrations.py | 52 ++++++++ port_ocean/core/models.py | 8 ++ port_ocean/run.py | 2 +- 10 files changed, 284 insertions(+), 125 deletions(-) create mode 100644 port_ocean/cli/defaults/__init__.py create mode 100644 port_ocean/cli/defaults/clean.py create mode 100644 port_ocean/cli/defaults/common.py rename port_ocean/cli/defaults/{port_defaults.py => initialize.py} (57%) create mode 100644 port_ocean/clients/port/mixins/migrations.py diff --git a/port_ocean/cli/commands/defaults/clean.py b/port_ocean/cli/commands/defaults/clean.py index e68502348d..28976c71a6 100644 --- a/port_ocean/cli/commands/defaults/clean.py +++ b/port_ocean/cli/commands/defaults/clean.py @@ -5,9 +5,8 @@ from .group import defaults from port_ocean import __version__ from port_ocean.cli.commands.main import print_logo, console -from port_ocean.cli.utils import cli_root_path from port_ocean.ocean import Ocean -from port_ocean.cli.defaults.port_defaults import clean_defaults +from port_ocean.cli.defaults import clean_defaults from port_ocean.run import _create_default_app, _load_module @@ -21,7 +20,15 @@ default=False, help="Delete all the entities of the Blueprint as well as the blueprint itself.", ) -def clean(path: str, force: bool) -> None: +@click.option( + "-w", + "--wait", + "wait", + type=bool, + default=False, + help="Wait for the migration to finish. when force is set to true.", +) +def clean(path: str, force: bool, wait: bool) -> None: """ Clean defaults of the integration from the .port/resources PATH. @@ -46,13 +53,10 @@ def clean(path: str, force: bool) -> None: ) migration_ids = clean_defaults( - app.integration.AppConfigHandlerClass.CONFIG_CLASS, force + app.integration.AppConfigHandlerClass.CONFIG_CLASS, force, wait ) - if ( - migration_ids - and len([migration_id for migration_id in migration_ids if migration_id]) > 0 - ): + if migration_ids and len(migration_ids) > 0 and not wait: console.print( f"The clean migration has started, you can track the migration process using the following migration ids {migration_ids} ⚓️" ) diff --git a/port_ocean/cli/commands/defaults/dock.py b/port_ocean/cli/commands/defaults/dock.py index 1603e4b515..bb8133797b 100644 --- a/port_ocean/cli/commands/defaults/dock.py +++ b/port_ocean/cli/commands/defaults/dock.py @@ -7,7 +7,7 @@ from port_ocean.cli.commands.main import print_logo, console from port_ocean.cli.utils import cli_root_path from port_ocean.ocean import Ocean -from port_ocean.cli.defaults.port_defaults import initialize_defaults +from port_ocean.cli.defaults.initialize import initialize_defaults from port_ocean.run import _create_default_app, _load_module diff --git a/port_ocean/cli/defaults/__init__.py b/port_ocean/cli/defaults/__init__.py new file mode 100644 index 0000000000..9beaf06ce5 --- /dev/null +++ b/port_ocean/cli/defaults/__init__.py @@ -0,0 +1,7 @@ +from .clean import clean_defaults +from .initialize import initialize_defaults + +__all__ = [ + "clean_defaults", + "initialize_defaults", +] diff --git a/port_ocean/cli/defaults/clean.py b/port_ocean/cli/defaults/clean.py new file mode 100644 index 0000000000..bb7f94417f --- /dev/null +++ b/port_ocean/cli/defaults/clean.py @@ -0,0 +1,73 @@ +import asyncio +from typing import Type + +import httpx +from loguru import logger + +from port_ocean.context.ocean import ocean +from port_ocean.core.handlers.port_app_config.models import PortAppConfig + + +from port_ocean.cli.defaults.common import ( + get_port_integration_defaults, + is_integration_exists, +) + + +def clean_defaults( + config_class: Type[PortAppConfig], + force: bool, + wait: bool, +) -> list[str] | None: + try: + asyncio.new_event_loop().run_until_complete( + _clean_defaults(config_class, force, wait) + ) + + except Exception as e: + logger.debug(f"Failed to clear defaults, skipping... Error: {e}") + + +async def _clean_defaults( + config_class: Type[PortAppConfig], force: bool, wait: bool +) -> None: + port_client = ocean.port_client + is_exists = await is_integration_exists(port_client) + if not is_exists: + return None + defaults = get_port_integration_defaults(config_class) + if not defaults: + return None + + try: + migration_ids = await asyncio.gather( + *( + port_client.delete_blueprint( + blueprint["identifier"], should_raise=True, delete_entities=force + ) + for blueprint in defaults.blueprints + ) + ) + + if not force: + logger.info( + "Finished deleting blueprints and configurations! ⚓️", + ) + return None + + migration_ids = [migration_id for migration_id in migration_ids if migration_id] + + if migration_ids and len(migration_ids) > 0 and not wait: + logger.info( + f"Migration started. To check the status of the migration, track these ids using /migrations/:id route {migration_ids}", + ) + elif migration_ids and len(migration_ids) > 0 and wait: + await asyncio.gather( + *( + ocean.port_client.wait_for_migration_to_complete(migration_id) + for migration_id in migration_ids + ) + ) + except httpx.HTTPStatusError as e: + logger.error(f"Failed to delete blueprints: {e.response.text}.") + raise e diff --git a/port_ocean/cli/defaults/common.py b/port_ocean/cli/defaults/common.py new file mode 100644 index 0000000000..909cf9d501 --- /dev/null +++ b/port_ocean/cli/defaults/common.py @@ -0,0 +1,119 @@ +import asyncio +import json +from pathlib import Path +from typing import Type, Any, TypedDict, Optional + +import httpx +import yaml +from loguru import logger +from pydantic import BaseModel, Field +from starlette import status + +from port_ocean.clients.port.client import PortClient +from port_ocean.config.settings import IntegrationConfiguration +from port_ocean.context.ocean import ocean +from port_ocean.core.handlers.port_app_config.models import PortAppConfig +from port_ocean.exceptions.port_defaults import ( + AbortDefaultCreationError, + UnsupportedDefaultFileType, +) + +YAML_EXTENSIONS = [".yaml", ".yml"] +ALLOWED_FILE_TYPES = [".json", *YAML_EXTENSIONS] + + +class Preset(TypedDict): + blueprint: str + data: list[dict[str, Any]] + + +class Defaults(BaseModel): + blueprints: list[dict[str, Any]] = [] + actions: list[Preset] = [] + scorecards: list[Preset] = [] + port_app_config: Optional[PortAppConfig] = Field( + default=None, alias="port-app-config" + ) + + class Config: + allow_population_by_field_name = True + + +async def is_integration_exists(port_client: PortClient) -> bool: + try: + await port_client.get_current_integration(should_log=False) + return True + except httpx.HTTPStatusError as e: + if e.response.status_code != status.HTTP_404_NOT_FOUND: + raise e + + return False + + +def deconstruct_blueprints_to_creation_steps( + raw_blueprints: list[dict[str, Any]] +) -> tuple[list[dict[str, Any]], ...]: + """ + Deconstructing the blueprint into stages so the api wont fail to create a blueprint if there is a conflict + example: Preventing the failure of creating a blueprint with a relation to another blueprint + """ + ( + bare_blueprint, + with_relations, + full_blueprint, + ) = ([], [], []) + + for blueprint in raw_blueprints.copy(): + full_blueprint.append(blueprint.copy()) + + blueprint.pop("calculationProperties", {}) + blueprint.pop("mirrorProperties", {}) + with_relations.append(blueprint.copy()) + + blueprint.pop("teamInheritance", {}) + blueprint.pop("relations", {}) + bare_blueprint.append(blueprint) + + return ( + bare_blueprint, + with_relations, + full_blueprint, + ) + + +def get_port_integration_defaults( + port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".") +) -> Defaults | None: + defaults_dir = base_path / ".port/resources" + if not defaults_dir.exists(): + return None + + if not defaults_dir.is_dir(): + raise UnsupportedDefaultFileType( + f"Defaults directory is not a directory: {defaults_dir}" + ) + + default_jsons = {} + allowed_file_names = [ + field_model.alias for _, field_model in Defaults.__fields__.items() + ] + for path in defaults_dir.iterdir(): + if path.stem in allowed_file_names: + if not path.is_file() or path.suffix not in ALLOWED_FILE_TYPES: + raise UnsupportedDefaultFileType( + f"Defaults directory should contain only one of the next types: {ALLOWED_FILE_TYPES}. Found: {path}" + ) + + if path.suffix in YAML_EXTENSIONS: + default_jsons[path.stem] = yaml.safe_load(path.read_text()) + else: + default_jsons[path.stem] = json.loads(path.read_text()) + + return Defaults( + blueprints=default_jsons.get("blueprints", []), + actions=default_jsons.get("actions", []), + scorecards=default_jsons.get("scorecards", []), + port_app_config=port_app_config_class( + **default_jsons.get("port-app-config", {}) + ), + ) diff --git a/port_ocean/cli/defaults/port_defaults.py b/port_ocean/cli/defaults/initialize.py similarity index 57% rename from port_ocean/cli/defaults/port_defaults.py rename to port_ocean/cli/defaults/initialize.py index f96cd5292f..be792c27f4 100644 --- a/port_ocean/cli/defaults/port_defaults.py +++ b/port_ocean/cli/defaults/initialize.py @@ -1,13 +1,8 @@ import asyncio -import json -from pathlib import Path -from typing import Type, Any, TypedDict, Optional +from typing import Type, Any import httpx -import yaml from loguru import logger -from pydantic import BaseModel, Field -from starlette import status from port_ocean.clients.port.client import PortClient from port_ocean.config.settings import IntegrationConfiguration @@ -15,39 +10,9 @@ from port_ocean.core.handlers.port_app_config.models import PortAppConfig from port_ocean.exceptions.port_defaults import ( AbortDefaultCreationError, - UnsupportedDefaultFileType, ) -YAML_EXTENSIONS = [".yaml", ".yml"] -ALLOWED_FILE_TYPES = [".json", *YAML_EXTENSIONS] - - -class Preset(TypedDict): - blueprint: str - data: list[dict[str, Any]] - - -class Defaults(BaseModel): - blueprints: list[dict[str, Any]] = [] - actions: list[Preset] = [] - scorecards: list[Preset] = [] - port_app_config: Optional[PortAppConfig] = Field( - default=None, alias="port-app-config" - ) - - class Config: - allow_population_by_field_name = True - - -async def _is_integration_exists(port_client: PortClient) -> bool: - try: - await port_client.get_current_integration(should_log=False) - return True - except httpx.HTTPStatusError as e: - if e.response.status_code != status.HTTP_404_NOT_FOUND: - raise e - - return False +from port_ocean.cli.defaults.common import Defaults, get_port_integration_defaults def deconstruct_blueprints_to_creation_steps( @@ -175,32 +140,6 @@ async def _initialize_defaults( raise ExceptionGroup(str(e), e.errors) -async def _clean_defaults( - config_class: Type[PortAppConfig], - force, -) -> None: - port_client = ocean.port_client - is_exists = await _is_integration_exists(port_client) - if not is_exists: - return None - defaults = get_port_integration_defaults(config_class) - if not defaults: - return None - - try: - return await asyncio.gather( - *( - port_client.delete_blueprint( - blueprint["identifier"], should_raise=True, delete_entities=force - ) - for blueprint in defaults.blueprints - ) - ) - except httpx.HTTPStatusError as e: - logger.error(f"Failed to delete blueprints: {e.response.text}.") - raise e - - def initialize_defaults( config_class: Type[PortAppConfig], integration_config: IntegrationConfiguration ) -> None: @@ -210,54 +149,3 @@ def initialize_defaults( ) except Exception as e: logger.debug(f"Failed to initialize defaults, skipping... Error: {e}") - - -def clean_defaults( - config_class: Type[PortAppConfig], - force: bool, -) -> list[str] | None: - try: - migration_ids = asyncio.new_event_loop().run_until_complete( - _clean_defaults(config_class, force) - ) - return migration_ids - except Exception as e: - logger.debug(f"Failed to clear defaults, skipping... Error: {e}") - - -def get_port_integration_defaults( - port_app_config_class: Type[PortAppConfig], base_path: Path = Path(".") -) -> Defaults | None: - defaults_dir = base_path / ".port/resources" - if not defaults_dir.exists(): - return None - - if not defaults_dir.is_dir(): - raise UnsupportedDefaultFileType( - f"Defaults directory is not a directory: {defaults_dir}" - ) - - default_jsons = {} - allowed_file_names = [ - field_model.alias for _, field_model in Defaults.__fields__.items() - ] - for path in defaults_dir.iterdir(): - if path.stem in allowed_file_names: - if not path.is_file() or path.suffix not in ALLOWED_FILE_TYPES: - raise UnsupportedDefaultFileType( - f"Defaults directory should contain only one of the next types: {ALLOWED_FILE_TYPES}. Found: {path}" - ) - - if path.suffix in YAML_EXTENSIONS: - default_jsons[path.stem] = yaml.safe_load(path.read_text()) - else: - default_jsons[path.stem] = json.loads(path.read_text()) - - return Defaults( - blueprints=default_jsons.get("blueprints", []), - actions=default_jsons.get("actions", []), - scorecards=default_jsons.get("scorecards", []), - port_app_config=port_app_config_class( - **default_jsons.get("port-app-config", {}) - ), - ) diff --git a/port_ocean/clients/port/client.py b/port_ocean/clients/port/client.py index d3560feb2e..a431e545f2 100644 --- a/port_ocean/clients/port/client.py +++ b/port_ocean/clients/port/client.py @@ -4,6 +4,8 @@ from port_ocean.clients.port.mixins.blueprints import BlueprintClientMixin from port_ocean.clients.port.mixins.entities import EntityClientMixin from port_ocean.clients.port.mixins.integrations import IntegrationClientMixin +from port_ocean.clients.port.mixins.migrations import MigrationClientMixin + from port_ocean.clients.port.types import ( KafkaCreds, ) @@ -11,7 +13,12 @@ from port_ocean.exceptions.clients import KafkaCredentialsNotFound -class PortClient(EntityClientMixin, IntegrationClientMixin, BlueprintClientMixin): +class PortClient( + EntityClientMixin, + IntegrationClientMixin, + BlueprintClientMixin, + MigrationClientMixin, +): def __init__( self, base_url: str, @@ -35,6 +42,7 @@ def __init__( self, integration_identifier, self.auth, self.client ) BlueprintClientMixin.__init__(self, self.auth, self.client) + MigrationClientMixin.__init__(self, self.auth, self.client) async def get_kafka_creds(self) -> KafkaCreds: logger.info("Fetching organization kafka credentials") diff --git a/port_ocean/clients/port/mixins/migrations.py b/port_ocean/clients/port/mixins/migrations.py new file mode 100644 index 0000000000..cc2f04e5e0 --- /dev/null +++ b/port_ocean/clients/port/mixins/migrations.py @@ -0,0 +1,52 @@ +import asyncio +from urllib.parse import quote_plus + +import httpx +from loguru import logger + +from port_ocean.clients.port.authentication import PortAuthentication +from port_ocean.clients.port.utils import handle_status_code +from port_ocean.core.models import Migration + + +class MigrationClientMixin: + def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient): + self.auth = auth + self.client = client + + async def wait_for_migration_to_complete( + self, + migration_id: list[str], + interval: int = 5, + ) -> Migration: + logger.info( + f"Waiting for migration with id: {migration_id} to complete", + ) + + headers = await self.auth.headers() + response = await self.client.get( + f"{self.auth.api_url}/migrations/{migration_id}", + headers=headers, + ) + + if response.is_error: + logger.error( + f"Could not get migration status with id: {migration_id}. Error: {response.text}" + ) + + handle_status_code(response, should_raise=True) + + migration_status = response.json().get("migration", {}).get("status", None) + if ( + migration_status == "RUNNING" + or migration_status == "INITIALIZING" + or migration_status == "PENDING" + ): + await asyncio.sleep(interval) + await self.wait_for_migration_to_complete(migration_id, interval) + + logger.info( + f"Migration with id: {migration_id} completed successfully", + ) + + return Migration.parse_obj(response.json()["migration"]) diff --git a/port_ocean/core/models.py b/port_ocean/core/models.py index 191a26eb9f..5d7bd88331 100644 --- a/port_ocean/core/models.py +++ b/port_ocean/core/models.py @@ -28,3 +28,11 @@ class Blueprint(BaseModel): team: str | None properties_schema: dict[str, Any] = Field(alias="schema") relations: dict[str, BlueprintRelation] + + +class Migration(BaseModel): + id: str + actor: str + sourceBlueprint: str + mapping: dict[str, Any] + status: str diff --git a/port_ocean/run.py b/port_ocean/run.py index 1d067a8425..1fc4758221 100644 --- a/port_ocean/run.py +++ b/port_ocean/run.py @@ -11,7 +11,7 @@ from port_ocean.core.integrations.base import BaseIntegration from port_ocean.logger_setup import setup_logger from port_ocean.ocean import Ocean -from port_ocean.cli.defaults.port_defaults import initialize_defaults +from port_ocean.cli.defaults.initialize import initialize_defaults from port_ocean.utils import get_spec_file From 73df5cadc9de36e92d9fd31dd2aa19fc3e33bec8 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Sun, 13 Aug 2023 17:02:11 +0300 Subject: [PATCH 07/14] Fixed mypy + ruff errors --- port_ocean/cli/commands/defaults/clean.py | 10 +--------- port_ocean/cli/commands/defaults/dock.py | 2 -- port_ocean/cli/commands/defaults/group.py | 2 +- port_ocean/cli/defaults/clean.py | 2 +- port_ocean/cli/defaults/common.py | 5 ----- port_ocean/clients/port/mixins/blueprints.py | 3 ++- port_ocean/clients/port/mixins/migrations.py | 1 - 7 files changed, 5 insertions(+), 20 deletions(-) diff --git a/port_ocean/cli/commands/defaults/clean.py b/port_ocean/cli/commands/defaults/clean.py index 28976c71a6..d24cbe7a8a 100644 --- a/port_ocean/cli/commands/defaults/clean.py +++ b/port_ocean/cli/commands/defaults/clean.py @@ -3,7 +3,6 @@ from inspect import getmembers from .group import defaults -from port_ocean import __version__ from port_ocean.cli.commands.main import print_logo, console from port_ocean.ocean import Ocean from port_ocean.cli.defaults import clean_defaults @@ -52,11 +51,4 @@ def clean(path: str, force: bool, wait: bool) -> None: default_app, ) - migration_ids = clean_defaults( - app.integration.AppConfigHandlerClass.CONFIG_CLASS, force, wait - ) - - if migration_ids and len(migration_ids) > 0 and not wait: - console.print( - f"The clean migration has started, you can track the migration process using the following migration ids {migration_ids} ⚓️" - ) + clean_defaults(app.integration.AppConfigHandlerClass.CONFIG_CLASS, force, wait) diff --git a/port_ocean/cli/commands/defaults/dock.py b/port_ocean/cli/commands/defaults/dock.py index bb8133797b..5cce9ced12 100644 --- a/port_ocean/cli/commands/defaults/dock.py +++ b/port_ocean/cli/commands/defaults/dock.py @@ -3,9 +3,7 @@ from inspect import getmembers from .group import defaults -from port_ocean import __version__ from port_ocean.cli.commands.main import print_logo, console -from port_ocean.cli.utils import cli_root_path from port_ocean.ocean import Ocean from port_ocean.cli.defaults.initialize import initialize_defaults from port_ocean.run import _create_default_app, _load_module diff --git a/port_ocean/cli/commands/defaults/group.py b/port_ocean/cli/commands/defaults/group.py index 4155c2a7e3..2d9243c58f 100644 --- a/port_ocean/cli/commands/defaults/group.py +++ b/port_ocean/cli/commands/defaults/group.py @@ -2,5 +2,5 @@ @cli_start.group("defaults") -def defaults(): +def defaults() -> None: pass diff --git a/port_ocean/cli/defaults/clean.py b/port_ocean/cli/defaults/clean.py index bb7f94417f..db3b29f1c2 100644 --- a/port_ocean/cli/defaults/clean.py +++ b/port_ocean/cli/defaults/clean.py @@ -18,7 +18,7 @@ def clean_defaults( config_class: Type[PortAppConfig], force: bool, wait: bool, -) -> list[str] | None: +) -> None: try: asyncio.new_event_loop().run_until_complete( _clean_defaults(config_class, force, wait) diff --git a/port_ocean/cli/defaults/common.py b/port_ocean/cli/defaults/common.py index 909cf9d501..f9e208b0c5 100644 --- a/port_ocean/cli/defaults/common.py +++ b/port_ocean/cli/defaults/common.py @@ -1,20 +1,15 @@ -import asyncio import json from pathlib import Path from typing import Type, Any, TypedDict, Optional import httpx import yaml -from loguru import logger from pydantic import BaseModel, Field from starlette import status from port_ocean.clients.port.client import PortClient -from port_ocean.config.settings import IntegrationConfiguration -from port_ocean.context.ocean import ocean from port_ocean.core.handlers.port_app_config.models import PortAppConfig from port_ocean.exceptions.port_defaults import ( - AbortDefaultCreationError, UnsupportedDefaultFileType, ) diff --git a/port_ocean/clients/port/mixins/blueprints.py b/port_ocean/clients/port/mixins/blueprints.py index 4718ba934f..ba5e5babfb 100644 --- a/port_ocean/clients/port/mixins/blueprints.py +++ b/port_ocean/clients/port/mixins/blueprints.py @@ -46,12 +46,13 @@ async def patch_blueprint( async def delete_blueprint( self, identifier: str, should_raise: bool = False, delete_entities: bool = False - ) -> None or str: + ) -> None | str: logger.info( f"Deleting blueprint with id: {identifier} with all entities: {delete_entities}" ) headers = await self.auth.headers() response = None + if not delete_entities: response = await self.client.delete( f"{self.auth.api_url}/blueprints/{identifier}", diff --git a/port_ocean/clients/port/mixins/migrations.py b/port_ocean/clients/port/mixins/migrations.py index cc2f04e5e0..f02e4cee43 100644 --- a/port_ocean/clients/port/mixins/migrations.py +++ b/port_ocean/clients/port/mixins/migrations.py @@ -1,5 +1,4 @@ import asyncio -from urllib.parse import quote_plus import httpx from loguru import logger From 2ff9f5efdeabd7da17b2caec451b12ba6bf4a298 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Mon, 14 Aug 2023 19:16:46 +0300 Subject: [PATCH 08/14] Fixed CR comments --- port_ocean/cli/defaults/clean.py | 2 +- port_ocean/clients/port/mixins/migrations.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/port_ocean/cli/defaults/clean.py b/port_ocean/cli/defaults/clean.py index db3b29f1c2..c5a2a7d872 100644 --- a/port_ocean/cli/defaults/clean.py +++ b/port_ocean/cli/defaults/clean.py @@ -25,7 +25,7 @@ def clean_defaults( ) except Exception as e: - logger.debug(f"Failed to clear defaults, skipping... Error: {e}") + logger.error(f"Failed to clear defaults, skipping... Error: {e}") async def _clean_defaults( diff --git a/port_ocean/clients/port/mixins/migrations.py b/port_ocean/clients/port/mixins/migrations.py index f02e4cee43..07d48d7f7d 100644 --- a/port_ocean/clients/port/mixins/migrations.py +++ b/port_ocean/clients/port/mixins/migrations.py @@ -28,11 +28,6 @@ async def wait_for_migration_to_complete( headers=headers, ) - if response.is_error: - logger.error( - f"Could not get migration status with id: {migration_id}. Error: {response.text}" - ) - handle_status_code(response, should_raise=True) migration_status = response.json().get("migration", {}).get("status", None) @@ -43,9 +38,9 @@ async def wait_for_migration_to_complete( ): await asyncio.sleep(interval) await self.wait_for_migration_to_complete(migration_id, interval) - - logger.info( - f"Migration with id: {migration_id} completed successfully", - ) + else: + logger.info( + f"Migration with id: {migration_id} finished with status {migration_status}", + ) return Migration.parse_obj(response.json()["migration"]) From eba493e6743d86112681990e25f16005b65b1098 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 13:58:42 +0300 Subject: [PATCH 09/14] Fixed CR comments --- port_ocean/bootstrap.py | 39 ++++++++++++ port_ocean/cli/commands/defaults/clean.py | 16 ++--- port_ocean/cli/commands/defaults/dock.py | 10 +-- port_ocean/{cli => core}/defaults/__init__.py | 0 port_ocean/{cli => core}/defaults/clean.py | 2 +- port_ocean/{cli => core}/defaults/common.py | 0 .../{cli => core}/defaults/initialize.py | 2 +- port_ocean/run.py | 62 ++++--------------- port_ocean/utils.py | 13 ++++ 9 files changed, 81 insertions(+), 63 deletions(-) create mode 100644 port_ocean/bootstrap.py rename port_ocean/{cli => core}/defaults/__init__.py (100%) rename port_ocean/{cli => core}/defaults/clean.py (97%) rename port_ocean/{cli => core}/defaults/common.py (100%) rename port_ocean/{cli => core}/defaults/initialize.py (98%) diff --git a/port_ocean/bootstrap.py b/port_ocean/bootstrap.py new file mode 100644 index 0000000000..820fcdbe0e --- /dev/null +++ b/port_ocean/bootstrap.py @@ -0,0 +1,39 @@ +import sys +from inspect import getmembers, isclass +from types import ModuleType +from typing import Type + +from pydantic import BaseModel + +from port_ocean.core.integrations.base import BaseIntegration +from port_ocean.ocean import Ocean +from port_ocean.utils import load_module + + +def _get_base_integration_class_from_module( + module: ModuleType, +) -> Type[BaseIntegration]: + for name, obj in getmembers(module): + if ( + isclass(obj) + and type(obj) == type + and issubclass(obj, BaseIntegration) + and obj != BaseIntegration + ): + return obj + + raise Exception(f"Failed to load integration from module: {module.__name__}") + + +def create_default_app( + path: str | None = None, config_factory: Type[BaseModel] | None = None +) -> Ocean: + sys.path.append(".") + try: + integration_path = f"{path}/integration.py" if path else "integration.py" + module = load_module(integration_path) + integration_class = _get_base_integration_class_from_module(module) + except Exception: + integration_class = None + + return Ocean(integration_class=integration_class, config_factory=config_factory) diff --git a/port_ocean/cli/commands/defaults/clean.py b/port_ocean/cli/commands/defaults/clean.py index d24cbe7a8a..d1d8cacb53 100644 --- a/port_ocean/cli/commands/defaults/clean.py +++ b/port_ocean/cli/commands/defaults/clean.py @@ -2,11 +2,13 @@ import click from inspect import getmembers + +from port_ocean.utils import load_module from .group import defaults from port_ocean.cli.commands.main import print_logo, console from port_ocean.ocean import Ocean -from port_ocean.cli.defaults import clean_defaults -from port_ocean.run import _create_default_app, _load_module +from port_ocean.core.defaults import clean_defaults +from port_ocean.bootstrap import create_default_app @defaults.command() @@ -15,16 +17,14 @@ "-f", "--force", "force", - type=bool, - default=False, + is_flag=True, help="Delete all the entities of the Blueprint as well as the blueprint itself.", ) @click.option( "-w", "--wait", "wait", - type=bool, - default=False, + is_flag=True, help="Wait for the migration to finish. when force is set to true.", ) def clean(path: str, force: bool, wait: bool) -> None: @@ -41,11 +41,11 @@ def clean(path: str, force: bool, wait: bool) -> None: console.print( "Deleting entities forcefully I sure hope you know what you are doing 🚨 🚨 🚨 ", ) - default_app = _create_default_app(path, False) + default_app = create_default_app(path) main_path = f"{path}/main.py" if path else "main.py" - app_module = _load_module(main_path) + app_module = load_module(main_path) app: Ocean = {name: item for name, item in getmembers(app_module)}.get( "app", default_app, diff --git a/port_ocean/cli/commands/defaults/dock.py b/port_ocean/cli/commands/defaults/dock.py index 5cce9ced12..32c881bc3a 100644 --- a/port_ocean/cli/commands/defaults/dock.py +++ b/port_ocean/cli/commands/defaults/dock.py @@ -2,11 +2,13 @@ import click from inspect import getmembers + +from port_ocean.utils import load_module from .group import defaults from port_ocean.cli.commands.main import print_logo, console from port_ocean.ocean import Ocean -from port_ocean.cli.defaults.initialize import initialize_defaults -from port_ocean.run import _create_default_app, _load_module +from port_ocean.core.defaults.initialize import initialize_defaults +from port_ocean.bootstrap import create_default_app @defaults.command() @@ -21,11 +23,11 @@ def dock(path: str) -> None: console.print("Unloading cargo at the dock... 📦🚢") - default_app = _create_default_app(path, False) + default_app = create_default_app(path) main_path = f"{path}/main.py" if path else "main.py" - app_module = _load_module(main_path) + app_module = load_module(main_path) app: Ocean = {name: item for name, item in getmembers(app_module)}.get( "app", default_app, diff --git a/port_ocean/cli/defaults/__init__.py b/port_ocean/core/defaults/__init__.py similarity index 100% rename from port_ocean/cli/defaults/__init__.py rename to port_ocean/core/defaults/__init__.py diff --git a/port_ocean/cli/defaults/clean.py b/port_ocean/core/defaults/clean.py similarity index 97% rename from port_ocean/cli/defaults/clean.py rename to port_ocean/core/defaults/clean.py index c5a2a7d872..4f8cada9a8 100644 --- a/port_ocean/cli/defaults/clean.py +++ b/port_ocean/core/defaults/clean.py @@ -8,7 +8,7 @@ from port_ocean.core.handlers.port_app_config.models import PortAppConfig -from port_ocean.cli.defaults.common import ( +from port_ocean.core.defaults.common import ( get_port_integration_defaults, is_integration_exists, ) diff --git a/port_ocean/cli/defaults/common.py b/port_ocean/core/defaults/common.py similarity index 100% rename from port_ocean/cli/defaults/common.py rename to port_ocean/core/defaults/common.py diff --git a/port_ocean/cli/defaults/initialize.py b/port_ocean/core/defaults/initialize.py similarity index 98% rename from port_ocean/cli/defaults/initialize.py rename to port_ocean/core/defaults/initialize.py index be792c27f4..403d0fa397 100644 --- a/port_ocean/cli/defaults/initialize.py +++ b/port_ocean/core/defaults/initialize.py @@ -12,7 +12,7 @@ AbortDefaultCreationError, ) -from port_ocean.cli.defaults.common import Defaults, get_port_integration_defaults +from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults def deconstruct_blueprints_to_creation_steps( diff --git a/port_ocean/run.py b/port_ocean/run.py index 1fc4758221..c63cb561b7 100644 --- a/port_ocean/run.py +++ b/port_ocean/run.py @@ -1,62 +1,25 @@ -import sys -from importlib.util import spec_from_file_location, module_from_spec -from inspect import getmembers, isclass -from types import ModuleType +from inspect import getmembers from typing import Type +from pydantic import BaseModel import uvicorn +from port_ocean.bootstrap import create_default_app from port_ocean.config.dynamic import default_config_factory -from port_ocean.config.settings import LogLevelType, ApplicationSettings -from port_ocean.core.integrations.base import BaseIntegration +from port_ocean.config.settings import ApplicationSettings, LogLevelType +from port_ocean.core.defaults.initialize import initialize_defaults from port_ocean.logger_setup import setup_logger from port_ocean.ocean import Ocean -from port_ocean.cli.defaults.initialize import initialize_defaults -from port_ocean.utils import get_spec_file +from port_ocean.utils import get_spec_file, load_module -def _get_base_integration_class_from_module( - module: ModuleType, -) -> Type[BaseIntegration]: - for name, obj in getmembers(module): - if ( - isclass(obj) - and type(obj) == type - and issubclass(obj, BaseIntegration) - and obj != BaseIntegration - ): - return obj - - raise Exception(f"Failed to load integration from module: {module.__name__}") - - -def _load_module(file_path: str) -> ModuleType: - spec = spec_from_file_location("module.name", file_path) - if spec is None or spec.loader is None: - raise Exception(f"Failed to load integration from path: {file_path}") - - module = module_from_spec(spec) - spec.loader.exec_module(module) - - return module - - -def _create_default_app( - path: str | None = None, use_default_config_factory: bool = False -) -> Ocean: - sys.path.append(".") - try: - integration_path = f"{path}/integration.py" if path else "integration.py" - module = _load_module(integration_path) - integration_class = _get_base_integration_class_from_module(module) - except Exception: - integration_class = None - +def _get_default_config_factory() -> None | Type[BaseModel]: spec = get_spec_file() config_factory = None - if spec is not None and use_default_config_factory: + if spec is not None: config_factory = default_config_factory(spec.get("configurations", [])) - return Ocean(integration_class=integration_class, config_factory=config_factory) + + return config_factory def run( @@ -68,10 +31,11 @@ def run( application_settings = ApplicationSettings(log_level=log_level, port=port) setup_logger(application_settings.log_level) - default_app = _create_default_app(path) + config_factory = _get_default_config_factory() + default_app = create_default_app(path, config_factory) main_path = f"{path}/main.py" if path else "main.py" - app_module = _load_module(main_path) + app_module = load_module(main_path) app: Ocean = {name: item for name, item in getmembers(app_module)}.get( "app", default_app ) diff --git a/port_ocean/utils.py b/port_ocean/utils.py index 6e51cc28e4..2a71b79062 100644 --- a/port_ocean/utils.py +++ b/port_ocean/utils.py @@ -1,6 +1,8 @@ +from importlib.util import module_from_spec, spec_from_file_location import inspect from pathlib import Path from time import time +from types import ModuleType from typing import Callable, Any from uuid import uuid4 @@ -31,3 +33,14 @@ def get_spec_file(path: Path = Path(".")) -> dict[str, Any] | None: return yaml.safe_load((path / ".port/spec.yaml").read_text()) except FileNotFoundError: return None + + +def load_module(file_path: str) -> ModuleType: + spec = spec_from_file_location("module.name", file_path) + if spec is None or spec.loader is None: + raise Exception(f"Failed to load integration from path: {file_path}") + + module = module_from_spec(spec) + spec.loader.exec_module(module) + + return module From 109354fd94c28721c4c0be28b450b6925b541698 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 14:01:07 +0300 Subject: [PATCH 10/14] Added towncrier changelog --- changelog/dock-clean-defaults.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/dock-clean-defaults.feature.md diff --git a/changelog/dock-clean-defaults.feature.md b/changelog/dock-clean-defaults.feature.md new file mode 100644 index 0000000000..f5b80a9542 --- /dev/null +++ b/changelog/dock-clean-defaults.feature.md @@ -0,0 +1 @@ +Added the abillity to clean and create the defaults of an integration \ No newline at end of file From a75f8b14872c3221e00cd3003dff46edc27a124a Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 14:13:12 +0300 Subject: [PATCH 11/14] Fixed Pager Duty integration to use client inside the functions --- integrations/pagerduty/main.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/integrations/pagerduty/main.py b/integrations/pagerduty/main.py index f65ff668c2..711ed8a025 100644 --- a/integrations/pagerduty/main.py +++ b/integrations/pagerduty/main.py @@ -11,16 +11,14 @@ class ObjectKind: INCIDENTS = "incidents" -pager_duty_client = PagerDutyClient( - ocean.integration_config.get("token", ""), - ocean.integration_config.get("api_url", ""), - ocean.integration_config.get("app_host", ""), -) - - @ocean.on_resync(ObjectKind.INCIDENTS) async def on_incidents_resync(kind: str) -> list[dict[str, Any]]: logger.info(f"Listing Pagerduty resource: {kind}") + pager_duty_client = PagerDutyClient( + ocean.integration_config["token"], + ocean.integration_config["api_url"], + ocean.integration_config["app_host"], + ) return await pager_duty_client.paginate_request_to_pager_duty( data_key=ObjectKind.INCIDENTS @@ -30,6 +28,11 @@ async def on_incidents_resync(kind: str) -> list[dict[str, Any]]: @ocean.on_resync(ObjectKind.SERVICES) async def on_services_resync(kind: str) -> list[dict[str, Any]]: logger.info(f"Listing Pagerduty resource: {kind}") + pager_duty_client = PagerDutyClient( + ocean.integration_config["token"], + ocean.integration_config["api_url"], + ocean.integration_config["app_host"], + ) services = await pager_duty_client.paginate_request_to_pager_duty( data_key=ObjectKind.SERVICES @@ -39,6 +42,11 @@ async def on_services_resync(kind: str) -> list[dict[str, Any]]: @ocean.router.post("/webhook") async def upsert_incident_webhook_handler(data: dict[str, Any]) -> None: + pager_duty_client = PagerDutyClient( + ocean.integration_config["token"], + ocean.integration_config["api_url"], + ocean.integration_config["app_host"], + ) event_type = data["event"]["event_type"] logger.info(f"Processing Pagerduty webhook for event type: {event_type}") if event_type in pager_duty_client.service_delete_events: @@ -63,5 +71,10 @@ async def upsert_incident_webhook_handler(data: dict[str, Any]) -> None: @ocean.on_start() async def on_start() -> None: + pager_duty_client = PagerDutyClient( + ocean.integration_config["token"], + ocean.integration_config["api_url"], + ocean.integration_config["app_host"], + ) logger.info("Subscribing to Pagerduty webhooks") await pager_duty_client.create_webhooks_if_not_exists() From 95074f2e2e282e8e92f6e288a9dc6234fecee9f8 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 14:16:57 +0300 Subject: [PATCH 12/14] Fixed changelog fragment --- changelog/dock-clean-defaults.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/dock-clean-defaults.feature.md b/changelog/dock-clean-defaults.feature.md index f5b80a9542..e3d3df0082 100644 --- a/changelog/dock-clean-defaults.feature.md +++ b/changelog/dock-clean-defaults.feature.md @@ -1 +1 @@ -Added the abillity to clean and create the defaults of an integration \ No newline at end of file +Added the ability to clean and create the defaults of an integration using CLI commands of `ocean defaults dock` and `ocean defaults clean` \ No newline at end of file From 71038dfc2a465523a4d5b61f9d611b789ca56f8a Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 14:17:38 +0300 Subject: [PATCH 13/14] Fixed changelog fragment --- changelog/dock-clean-defaults.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/dock-clean-defaults.feature.md b/changelog/dock-clean-defaults.feature.md index e3d3df0082..41979745d6 100644 --- a/changelog/dock-clean-defaults.feature.md +++ b/changelog/dock-clean-defaults.feature.md @@ -1 +1 @@ -Added the ability to clean and create the defaults of an integration using CLI commands of `ocean defaults dock` and `ocean defaults clean` \ No newline at end of file +Added the ability to clean and create the defaults of integration using the following CLI commands: `ocean defaults dock` and `ocean defaults clean` \ No newline at end of file From 6f696de0267a27ab72b952dd8f057e2a177adfa2 Mon Sep 17 00:00:00 2001 From: danielsinai Date: Thu, 17 Aug 2023 14:19:12 +0300 Subject: [PATCH 14/14] Fixed changelog fragment grammer --- changelog/dock-clean-defaults.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/dock-clean-defaults.feature.md b/changelog/dock-clean-defaults.feature.md index 41979745d6..9e64f66c10 100644 --- a/changelog/dock-clean-defaults.feature.md +++ b/changelog/dock-clean-defaults.feature.md @@ -1 +1 @@ -Added the ability to clean and create the defaults of integration using the following CLI commands: `ocean defaults dock` and `ocean defaults clean` \ No newline at end of file +Added the ability to create and clean the defaults of an integration using the following CLI commands: `ocean defaults dock` and `ocean defaults clean` \ No newline at end of file