Add application config path

.github/workflows/release-integrations.yml

@@ -4,6 +4,11 @@ on:
     branches:
       - main
   workflow_dispatch:
+    inputs:
+      LATEST:
+        type: boolean
+        default: true
+        description: "If true, build the Docker image with both 'latest' and version tags. If false, build the Docker image with only the version tag."
 
 jobs:
   release-all:
@@ -41,7 +46,13 @@ jobs:
               echo "Image already exists in $repository: port-ocean-$type:$version"
             else
               echo "Building and pushing new image: port-ocean-$type:$version"
-              docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.."
+              if [[ "${{ inputs.LATEST }}" == "true" ]]; then
+                # If true, build the Docker image with both "latest" and version tags
+                docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.."
+              else
+                  # If false, build the Docker image with only the version tag
+                  docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" "$folder/.."
+              fi
               docker push "ghcr.io/port-labs/port-ocean-$type" --all-tags
             fi
           done

integrations/gitlab/.port/spec.yaml

@@ -1,4 +1,4 @@
-version: v0.1.3
+version: v0.1.4.dev2
 type: gitlab
 description: Gitlab integration for Port Ocean
 icon: GitLab

integrations/gitlab/gitlab_integration/bootstrap.py

@@ -27,10 +27,10 @@ def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> Non
 
 def setup_application() -> None:
     logic_settings = ocean.integration_config
-    for token, group_mapping in logic_settings["tokenMapping"].items():
-        gitlab_client = Gitlab(logic_settings["gitlabHost"], token)
+    for token, group_mapping in logic_settings["token_mapping"].items():
+        gitlab_client = Gitlab(logic_settings["gitlab_host"], token)
         gitlab_service = GitlabService(
-            gitlab_client, logic_settings["appHost"], group_mapping
+            gitlab_client, logic_settings["app_host"], group_mapping
         )
         webhook_ids = gitlab_service.create_webhooks()
         for webhook_id in webhook_ids:

integrations/gitlab/gitlab_integration/utils.py

@@ -12,12 +12,12 @@ def get_all_services() -> List[GitlabService]:
     all_tokens_services = []
 
     logger.info(
-        f"Creating gitlab clients for {len(logic_settings['tokenMapping'])} tokens"
+        f"Creating gitlab clients for {len(logic_settings['token_mapping'])} tokens"
     )
-    for token, group_mapping in logic_settings["tokenMapping"].items():
-        gitlab_client = Gitlab(logic_settings["gitlabHost"], token)
+    for token, group_mapping in logic_settings["token_mapping"].items():
+        gitlab_client = Gitlab(logic_settings["gitlab_host"], token)
         gitlab_service = GitlabService(
-            gitlab_client, logic_settings["appHost"], group_mapping
+            gitlab_client, logic_settings["app_host"], group_mapping
         )
         all_tokens_services.append(gitlab_service)
 

integrations/gitlab/pyproject.toml

@@ -10,7 +10,7 @@ aiofiles = "^0.6.0"
 python-gitlab = "^3.14.0"
 pathlib = "^1.0.1"
 jsonschema = "^4.17.3"
-port-ocean = {version = "0.1.1", extras = ["cli"]}
+port-ocean = {version = "0.1.2rc1", extras = ["cli"]}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2"

port_ocean/config/base.py

@@ -4,7 +4,9 @@
 from typing import Any
 
 import yaml
+from humps import decamelize
 from pydantic import BaseSettings
+from pydantic.env_settings import EnvSettingsSource
 
 PROVIDER_WRAPPER_PATTERN = r"\{\{ from (.*) \}\}"
 PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$"
@@ -47,6 +49,15 @@ def load_from_config_provider(provider_type: str, value: str) -> Any:
         raise ValueError(f"Invalid provider type: {provider_type}")
 
 
+def decamelize_object(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        return {decamelize(k): decamelize_object(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [decamelize_object(v) for v in obj]
+    else:
+        return obj
+
+
 def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, Any]:
     yaml_content = read_yaml_config_settings_source(settings, base_path)
     matches = re.finditer(PROVIDER_WRAPPER_PATTERN, yaml_content)
@@ -56,7 +67,7 @@ def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, A
         # Replace the provider wrapper with the actual value
         yaml_content = re.sub(re.escape(match.group()), data, yaml_content, count=1)
 
-    return yaml.safe_load(yaml_content)
+    return decamelize_object(yaml.safe_load(yaml_content))
 
 
 class BaseOceanSettings(BaseSettings):
@@ -66,8 +77,9 @@ class Config:
         yaml_file = "./config.yaml"
 
         @classmethod
-        def customise_sources(cls, init_settings, *_, **__):  # type: ignore
+        def customise_sources(cls, init_settings, env_settings: EnvSettingsSource, *_, **__):  # type: ignore
             return (
+                env_settings,
                 init_settings,
                 lambda s: load_providers(s, init_settings.init_kwargs["base_path"]),
             )

port_ocean/config/dynamic.py

@@ -1,5 +1,6 @@
 from typing import Type, Any, Optional
 
+from humps import decamelize
 from pydantic import BaseModel, AnyUrl, create_model, Extra, parse_obj_as
 
 
@@ -34,7 +35,7 @@ def default_config_factory(configurations: Any) -> Type[BaseModel]:
         default = ... if config.required else None
         if config.default is not None:
             default = parse_obj_as(field_type, config.default)
-        fields[config.name] = (
+        fields[decamelize(config.name)] = (
             field_type,
             default,
         )

port_ocean/config/integration.py

@@ -1,32 +1,34 @@
 from typing import Any, Literal
 
-from pydantic import Field, BaseSettings
+from pydantic import BaseSettings, BaseModel
 
 from port_ocean.config.base import BaseOceanSettings
 from port_ocean.core.event_listener import EventListenerSettingsType
 
 
-class PortSettings(BaseSettings):
-    client_id: str = Field(alias="clientId")
-    client_secret: str = Field(alias="clientSecret")
-    base_url: str = Field(alias="baseUrl", default="https://api.getport.io")
+class PortSettings(BaseModel):
+    client_id: str
+    client_secret: str
+    base_url: str = "https://api.getport.io"
 
 
-class IntegrationSettings(BaseSettings):
+class IntegrationSettings(BaseModel):
     identifier: str
     type: str
     config: dict[str, Any]
 
 
 class IntegrationConfiguration(BaseOceanSettings):
     port: PortSettings
-    event_listener: EventListenerSettingsType = Field(alias="eventListener")
-    batch_work_size: int = Field(alias="batchWorkSize", default=20)
-    initialize_port_resources: bool = Field(
-        alias="initializePortResources", default=False
-    )
+    event_listener: EventListenerSettingsType
+    batch_work_size: int = 20
+    initialize_port_resources: bool = False
     integration: IntegrationSettings
 
+    class Config:
+        env_prefix = "OCEAN__"
+        env_nested_delimiter = "__"
+
 
 LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
 
@@ -36,4 +38,8 @@ class ApplicationSettings(BaseSettings):
     port: int = 8000
 
     class Config:
-        env_prefix = "APPLICATION_"
+        env_prefix = "APPLICATION__"
+
+        @classmethod
+        def customise_sources(cls, init_settings, env_settings, *_, **__):  # type: ignore
+            return env_settings, init_settings

port_ocean/core/event_listener/base.py

@@ -1,7 +1,7 @@
 from abc import abstractmethod
 from typing import TypedDict, Callable, Any, Awaitable
 
-from pydantic import BaseSettings
+from pydantic import BaseModel
 
 
 class EventListenerEvents(TypedDict):
@@ -20,7 +20,7 @@ async def start(self) -> None:
         pass
 
 
-class EventListenerSettings(BaseSettings):
+class EventListenerSettings(BaseModel):
     type: str
 
     def to_request(self) -> dict[str, Any]:

port_ocean/core/event_listener/http/event_listener.py

@@ -2,7 +2,7 @@
 
 from fastapi import APIRouter
 from loguru import logger
-from pydantic import AnyHttpUrl, Field
+from pydantic import AnyHttpUrl
 
 from port_ocean.context.ocean import ocean
 from port_ocean.core.event_listener.base import (
@@ -14,7 +14,7 @@
 
 class HttpEventListenerSettings(EventListenerSettings):
     type: Literal["WEBHOOK"]
-    app_host: AnyHttpUrl = Field(alias="appHost")
+    app_host: AnyHttpUrl
 
     def to_request(self) -> dict[str, Any]:
         return {

port_ocean/core/event_listener/kafka/event_listener.py

@@ -2,7 +2,6 @@
 from typing import Any, Callable, Literal
 
 from loguru import logger
-from pydantic import Field
 
 from port_ocean.consumers.kafka_consumer import KafkaConsumer, KafkaConsumerConfig
 from port_ocean.context.ocean import (
@@ -20,12 +19,10 @@
 class KafkaEventListenerSettings(EventListenerSettings):
     type: Literal["KAFKA"]
     brokers: str = ""
-    security_protocol: str = Field(alias="securityProtocol", default="SASL_SSL")
-    authentication_mechanism: str = Field(
-        alias="authenticationMechanism", default="SCRAM-SHA-512"
-    )
-    kafka_security_enabled: bool = Field(alias="kafkaSecurityEnabled", default=True)
-    consumer_poll_timeout: int = Field(alias="consumerPollTimeout", default=1)
+    security_protocol: str = "SASL_SSL"
+    authentication_mechanism: str = "SCRAM-SHA-512"
+    kafka_security_enabled: bool = True
+    consumer_poll_timeout: int = 1
 
 
 class KafkaEventListener(BaseEventListener):

port_ocean/core/integrations/mixins/sync.py

@@ -317,7 +317,7 @@ async def sync_raw_all(
                         resource, user_agent_type, ocean.config.batch_work_size
                     )
                 )
-            flat_created_entities, errors = zip_and_sum(creation_results)
+            flat_created_entities, errors = zip_and_sum(creation_results) or [[], []]
 
             if errors:
                 message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.1.1"
+version = "0.1.2rc1"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"
@@ -47,6 +47,7 @@ httpx = "^0.24.1"
 pyjq = "^2.6.0"
 urllib3 = "^1.26.16"
 six = "^1.16.0"
+pyhumps = "^3.8.0"
 
 # CLI
 click = { version = "^8.1.3", optional = true }