From 3a87d97b9110059376f68b71a1151cdf48fc6d45 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 15 Aug 2024 17:25:43 +0000 Subject: [PATCH 01/27] Add OpenID Connect integration --- charmcraft.yaml | 4 + lib/charms/hydra/v0/oauth.py | 807 ++++++++++++++++++++++++++++++++ penpot_rock/rockcraft.yaml | 24 + requirements.txt | 2 + src/charm.py | 108 ++++- tests/integration/test_charm.py | 102 +++- tests/unit/test_charm.py | 4 +- tox.ini | 4 + 8 files changed, 1026 insertions(+), 29 deletions(-) create mode 100644 lib/charms/hydra/v0/oauth.py diff --git a/charmcraft.yaml b/charmcraft.yaml index e4b49a1..4d3f57b 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -60,6 +60,10 @@ requires: interface: smtp limit: 1 optional: true + oauth: + interface: oauth + limit: 1 + optional: true resources: penpot-image: diff --git a/lib/charms/hydra/v0/oauth.py b/lib/charms/hydra/v0/oauth.py new file mode 100644 index 0000000..8d36e96 --- /dev/null +++ b/lib/charms/hydra/v0/oauth.py @@ -0,0 +1,807 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""# Oauth Library. + +This library is designed to enable applications to register OAuth2/OIDC +clients with an OIDC Provider through the `oauth` interface. + +## Getting started + +To get started using this library you just need to fetch the library using `charmcraft`. **Note +that you also need to add `jsonschema` to your charm's `requirements.txt`.** + +```shell +cd some-charm +charmcraft fetch-lib charms.hydra.v0.oauth +EOF +``` + +Then, to initialize the library: +```python +# ... +from charms.hydra.v0.oauth import ClientConfig, OAuthRequirer + +OAUTH = "oauth" +OAUTH_SCOPES = "openid email" +OAUTH_GRANT_TYPES = ["authorization_code"] + +class SomeCharm(CharmBase): + def __init__(self, *args): + # ... + self.oauth = OAuthRequirer(self, client_config, relation_name=OAUTH) + + self.framework.observe(self.oauth.on.oauth_info_changed, self._configure_application) + # ... + + def _on_ingress_ready(self, event): + self.external_url = "https://example.com" + self._set_client_config() + + def _set_client_config(self): + client_config = ClientConfig( + urljoin(self.external_url, "/oauth/callback"), + OAUTH_SCOPES, + OAUTH_GRANT_TYPES, + ) + self.oauth.update_client_config(client_config) +``` +""" + +import json +import logging +import re +from dataclasses import asdict, dataclass, field, fields +from typing import Dict, List, Mapping, Optional + +import jsonschema +from ops.charm import CharmBase, RelationBrokenEvent, RelationChangedEvent, RelationCreatedEvent +from ops.framework import EventBase, EventSource, Handle, Object, ObjectEvents +from ops.model import Relation, Secret, SecretNotFoundError, TooManyRelatedAppsError + +# The unique Charmhub library identifier, never change it +LIBID = "a3a301e325e34aac80a2d633ef61fe97" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 10 + +PYDEPS = ["jsonschema"] + + +logger = logging.getLogger(__name__) + +DEFAULT_RELATION_NAME = "oauth" +ALLOWED_GRANT_TYPES = [ + "authorization_code", + "refresh_token", + "client_credentials", + "urn:ietf:params:oauth:grant-type:device_code", +] +ALLOWED_CLIENT_AUTHN_METHODS = ["client_secret_basic", "client_secret_post"] +CLIENT_SECRET_FIELD = "secret" + +url_regex = re.compile( + r"(^http://)|(^https://)" # http:// or https:// + r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|" + r"[A-Z0-9-]{2,}\.?)|" # domain... + r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip + r"(?::\d+)?" # optional port + r"(?:/?|[/?]\S+)$", + re.IGNORECASE, +) + +OAUTH_PROVIDER_JSON_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://canonical.github.io/charm-relation-interfaces/interfaces/oauth/schemas/provider.json", + "type": "object", + "properties": { + "issuer_url": { + "type": "string", + }, + "authorization_endpoint": { + "type": "string", + }, + "token_endpoint": { + "type": "string", + }, + "introspection_endpoint": { + "type": "string", + }, + "userinfo_endpoint": { + "type": "string", + }, + "jwks_endpoint": { + "type": "string", + }, + "scope": { + "type": "string", + }, + "client_id": { + "type": "string", + }, + "client_secret_id": { + "type": "string", + }, + "groups": {"type": "string", "default": None}, + "ca_chain": {"type": "array", "items": {"type": "string"}, "default": []}, + "jwt_access_token": {"type": "string", "default": "False"}, + }, + "required": [ + "issuer_url", + "authorization_endpoint", + "token_endpoint", + "introspection_endpoint", + "userinfo_endpoint", + "jwks_endpoint", + "scope", + ], +} +OAUTH_REQUIRER_JSON_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://canonical.github.io/charm-relation-interfaces/interfaces/oauth/schemas/requirer.json", + "type": "object", + "properties": { + "redirect_uri": { + "type": "string", + "default": None, + }, + "audience": {"type": "array", "default": [], "items": {"type": "string"}}, + "scope": {"type": "string", "default": None}, + "grant_types": { + "type": "array", + "default": None, + "items": { + "enum": ALLOWED_GRANT_TYPES, + "type": "string", + }, + }, + "token_endpoint_auth_method": { + "type": "string", + "enum": ALLOWED_CLIENT_AUTHN_METHODS, + "default": "client_secret_basic", + }, + }, + "required": ["redirect_uri", "audience", "scope", "grant_types", "token_endpoint_auth_method"], +} + + +class ClientConfigError(Exception): + """Emitted when invalid client config is provided.""" + + +class DataValidationError(RuntimeError): + """Raised when data validation fails on relation data.""" + + +def _load_data(data: Mapping, schema: Optional[Dict] = None) -> Dict: + """Parses nested fields and checks whether `data` matches `schema`.""" + ret = {} + for k, v in data.items(): + try: + ret[k] = json.loads(v) + except json.JSONDecodeError: + ret[k] = v + + if schema: + _validate_data(ret, schema) + return ret + + +def _dump_data(data: Dict, schema: Optional[Dict] = None) -> Dict: + if schema: + _validate_data(data, schema) + + ret = {} + for k, v in data.items(): + if isinstance(v, (list, dict)): + try: + ret[k] = json.dumps(v) + except json.JSONDecodeError as e: + raise DataValidationError(f"Failed to encode relation json: {e}") + elif isinstance(v, bool): + ret[k] = str(v) + else: + ret[k] = v + return ret + + +def strtobool(val: str) -> bool: + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + if not isinstance(val, str): + raise ValueError(f"invalid value type {type(val)}") + + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError(f"invalid truth value {val}") + + +class OAuthRelation(Object): + """A class containing helper methods for oauth relation.""" + + def _pop_relation_data(self, relation_id: Relation) -> None: + if not self.model.unit.is_leader(): + return + + if len(self.model.relations) == 0: + return + + relation = self.model.get_relation(self._relation_name, relation_id=relation_id) + if not relation or not relation.app: + return + + try: + for data in list(relation.data[self.model.app]): + relation.data[self.model.app].pop(data, "") + except Exception as e: + logger.info(f"Failed to pop the relation data: {e}") + + +def _validate_data(data: Dict, schema: Dict) -> None: + """Checks whether `data` matches `schema`. + + Will raise DataValidationError if the data is not valid, else return None. + """ + try: + jsonschema.validate(instance=data, schema=schema) + except jsonschema.ValidationError as e: + raise DataValidationError(data, schema) from e + + +@dataclass +class ClientConfig: + """Helper class containing a client's configuration.""" + + redirect_uri: str + scope: str + grant_types: List[str] + audience: List[str] = field(default_factory=lambda: []) + token_endpoint_auth_method: str = "client_secret_basic" + client_id: Optional[str] = None + + def validate(self) -> None: + """Validate the client configuration.""" + # Validate redirect_uri + if not re.match(url_regex, self.redirect_uri): + raise ClientConfigError(f"Invalid URL {self.redirect_uri}") + + if self.redirect_uri.startswith("http://"): + logger.warning("Provided Redirect URL uses http scheme. Don't do this in production") + + # Validate grant_types + for grant_type in self.grant_types: + if grant_type not in ALLOWED_GRANT_TYPES: + raise ClientConfigError( + f"Invalid grant_type {grant_type}, must be one " f"of {ALLOWED_GRANT_TYPES}" + ) + + # Validate client authentication methods + if self.token_endpoint_auth_method not in ALLOWED_CLIENT_AUTHN_METHODS: + raise ClientConfigError( + f"Invalid client auth method {self.token_endpoint_auth_method}, " + f"must be one of {ALLOWED_CLIENT_AUTHN_METHODS}" + ) + + def to_dict(self) -> Dict: + """Convert object to dict.""" + return {k: v for k, v in asdict(self).items() if v is not None} + + +@dataclass +class OauthProviderConfig: + """Helper class containing provider's configuration.""" + + issuer_url: str + authorization_endpoint: str + token_endpoint: str + introspection_endpoint: str + userinfo_endpoint: str + jwks_endpoint: str + scope: str + client_id: Optional[str] = None + client_secret: Optional[str] = None + groups: Optional[str] = None + ca_chain: Optional[str] = None + jwt_access_token: Optional[bool] = False + + @classmethod + def from_dict(cls, dic: Dict) -> "OauthProviderConfig": + """Generate OauthProviderConfig instance from dict.""" + jwt_access_token = False + if "jwt_access_token" in dic: + jwt_access_token = strtobool(dic["jwt_access_token"]) + return cls( + jwt_access_token=jwt_access_token, + **{ + k: v + for k, v in dic.items() + if k in [f.name for f in fields(cls)] and k != "jwt_access_token" + }, + ) + + +class OAuthInfoChangedEvent(EventBase): + """Event to notify the charm that the information in the databag changed.""" + + def __init__(self, handle: Handle, client_id: str, client_secret_id: str): + super().__init__(handle) + self.client_id = client_id + self.client_secret_id = client_secret_id + + def snapshot(self) -> Dict: + """Save event.""" + return { + "client_id": self.client_id, + "client_secret_id": self.client_secret_id, + } + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + super().restore(snapshot) + self.client_id = snapshot["client_id"] + self.client_secret_id = snapshot["client_secret_id"] + + +class InvalidClientConfigEvent(EventBase): + """Event to notify the charm that the client configuration is invalid.""" + + def __init__(self, handle: Handle, error: str): + super().__init__(handle) + self.error = error + + def snapshot(self) -> Dict: + """Save event.""" + return { + "error": self.error, + } + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + self.error = snapshot["error"] + + +class OAuthInfoRemovedEvent(EventBase): + """Event to notify the charm that the provider data was removed.""" + + def snapshot(self) -> Dict: + """Save event.""" + return {} + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + pass + + +class OAuthRequirerEvents(ObjectEvents): + """Event descriptor for events raised by `OAuthRequirerEvents`.""" + + oauth_info_changed = EventSource(OAuthInfoChangedEvent) + oauth_info_removed = EventSource(OAuthInfoRemovedEvent) + invalid_client_config = EventSource(InvalidClientConfigEvent) + + +class OAuthRequirer(OAuthRelation): + """Register an oauth client.""" + + on = OAuthRequirerEvents() + + def __init__( + self, + charm: CharmBase, + client_config: Optional[ClientConfig] = None, + relation_name: str = DEFAULT_RELATION_NAME, + ) -> None: + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + self._client_config = client_config + events = self._charm.on[relation_name] + self.framework.observe(events.relation_created, self._on_relation_created_event) + self.framework.observe(events.relation_changed, self._on_relation_changed_event) + self.framework.observe(events.relation_broken, self._on_relation_broken_event) + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + try: + self._update_relation_data(self._client_config, event.relation.id) + except ClientConfigError as e: + self.on.invalid_client_config.emit(e.args[0]) + + def _on_relation_broken_event(self, event: RelationBrokenEvent) -> None: + # Workaround for https://github.com/canonical/operator/issues/888 + self._pop_relation_data(event.relation.id) + if self.is_client_created(): + event.defer() + logger.info("Relation data still available. Deferring the event") + return + + # Notify the requirer that the relation data was removed + self.on.oauth_info_removed.emit() + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + data = event.relation.data[event.app] + if not data: + logger.info("No relation data available.") + return + + data = _load_data(data, OAUTH_PROVIDER_JSON_SCHEMA) + + client_id = data.get("client_id") + client_secret_id = data.get("client_secret_id") + if not client_id or not client_secret_id: + logger.info("OAuth Provider info is available, waiting for client to be registered.") + # The client credentials are not ready yet, so we do nothing + # This could mean that the client credentials were removed from the databag, + # but we don't allow that (for now), so we don't have to check for it. + return + + self.on.oauth_info_changed.emit(client_id, client_secret_id) + + def _update_relation_data( + self, client_config: Optional[ClientConfig], relation_id: Optional[int] = None + ) -> None: + if not self.model.unit.is_leader() or not client_config: + return + + if not isinstance(client_config, ClientConfig): + raise ValueError(f"Unexpected client_config type: {type(client_config)}") + + client_config.validate() + + try: + relation = self.model.get_relation( + relation_name=self._relation_name, relation_id=relation_id + ) + except TooManyRelatedAppsError: + raise RuntimeError("More than one relations are defined. Please provide a relation_id") + + if not relation or not relation.app: + return + + data = _dump_data(client_config.to_dict(), OAUTH_REQUIRER_JSON_SCHEMA) + relation.data[self.model.app].update(data) + + def is_client_created(self, relation_id: Optional[int] = None) -> bool: + """Check if the client has been created.""" + if len(self.model.relations) == 0: + return None + try: + relation = self.model.get_relation(self._relation_name, relation_id=relation_id) + except TooManyRelatedAppsError: + raise RuntimeError("More than one relations are defined. Please provide a relation_id") + + if not relation or not relation.app: + return None + + return ( + "client_id" in relation.data[relation.app] + and "client_secret_id" in relation.data[relation.app] + ) + + def get_provider_info( + self, relation_id: Optional[int] = None + ) -> Optional[OauthProviderConfig]: + """Get the provider information from the databag.""" + if len(self.model.relations) == 0: + return None + try: + relation = self.model.get_relation(self._relation_name, relation_id=relation_id) + except TooManyRelatedAppsError: + raise RuntimeError("More than one relations are defined. Please provide a relation_id") + if not relation or not relation.app: + return None + + data = relation.data[relation.app] + if not data: + logger.info("No relation data available.") + return + + data = _load_data(data, OAUTH_PROVIDER_JSON_SCHEMA) + + client_secret_id = data.get("client_secret_id") + if client_secret_id: + _client_secret = self.get_client_secret(client_secret_id) + client_secret = _client_secret.get_content()[CLIENT_SECRET_FIELD] + data["client_secret"] = client_secret + + oauth_provider = OauthProviderConfig.from_dict(data) + return oauth_provider + + def get_client_secret(self, client_secret_id: str) -> Secret: + """Get the client_secret.""" + client_secret = self.model.get_secret(id=client_secret_id) + return client_secret + + def update_client_config( + self, client_config: ClientConfig, relation_id: Optional[int] = None + ) -> None: + """Update the client config stored in the object.""" + self._client_config = client_config + self._update_relation_data(client_config, relation_id=relation_id) + + +class ClientCreatedEvent(EventBase): + """Event to notify the Provider charm to create a new client.""" + + def __init__( + self, + handle: Handle, + redirect_uri: str, + scope: str, + grant_types: List[str], + audience: List, + token_endpoint_auth_method: str, + relation_id: int, + ) -> None: + super().__init__(handle) + self.redirect_uri = redirect_uri + self.scope = scope + self.grant_types = grant_types + self.audience = audience + self.token_endpoint_auth_method = token_endpoint_auth_method + self.relation_id = relation_id + + def snapshot(self) -> Dict: + """Save event.""" + return { + "redirect_uri": self.redirect_uri, + "scope": self.scope, + "grant_types": self.grant_types, + "audience": self.audience, + "token_endpoint_auth_method": self.token_endpoint_auth_method, + "relation_id": self.relation_id, + } + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + self.redirect_uri = snapshot["redirect_uri"] + self.scope = snapshot["scope"] + self.grant_types = snapshot["grant_types"] + self.audience = snapshot["audience"] + self.token_endpoint_auth_method = snapshot["token_endpoint_auth_method"] + self.relation_id = snapshot["relation_id"] + + def to_client_config(self) -> ClientConfig: + """Convert the event information to a ClientConfig object.""" + return ClientConfig( + self.redirect_uri, + self.scope, + self.grant_types, + self.audience, + self.token_endpoint_auth_method, + ) + + +class ClientChangedEvent(EventBase): + """Event to notify the Provider charm that the client config changed.""" + + def __init__( + self, + handle: Handle, + redirect_uri: str, + scope: str, + grant_types: List, + audience: List, + token_endpoint_auth_method: str, + relation_id: int, + client_id: str, + ) -> None: + super().__init__(handle) + self.redirect_uri = redirect_uri + self.scope = scope + self.grant_types = grant_types + self.audience = audience + self.token_endpoint_auth_method = token_endpoint_auth_method + self.relation_id = relation_id + self.client_id = client_id + + def snapshot(self) -> Dict: + """Save event.""" + return { + "redirect_uri": self.redirect_uri, + "scope": self.scope, + "grant_types": self.grant_types, + "audience": self.audience, + "token_endpoint_auth_method": self.token_endpoint_auth_method, + "relation_id": self.relation_id, + "client_id": self.client_id, + } + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + self.redirect_uri = snapshot["redirect_uri"] + self.scope = snapshot["scope"] + self.grant_types = snapshot["grant_types"] + self.audience = snapshot["audience"] + self.token_endpoint_auth_method = snapshot["token_endpoint_auth_method"] + self.relation_id = snapshot["relation_id"] + self.client_id = snapshot["client_id"] + + def to_client_config(self) -> ClientConfig: + """Convert the event information to a ClientConfig object.""" + return ClientConfig( + self.redirect_uri, + self.scope, + self.grant_types, + self.audience, + self.token_endpoint_auth_method, + self.client_id, + ) + + +class ClientDeletedEvent(EventBase): + """Event to notify the Provider charm that the client was deleted.""" + + def __init__( + self, + handle: Handle, + relation_id: int, + ) -> None: + super().__init__(handle) + self.relation_id = relation_id + + def snapshot(self) -> Dict: + """Save event.""" + return {"relation_id": self.relation_id} + + def restore(self, snapshot: Dict) -> None: + """Restore event.""" + self.relation_id = snapshot["relation_id"] + + +class OAuthProviderEvents(ObjectEvents): + """Event descriptor for events raised by `OAuthProviderEvents`.""" + + client_created = EventSource(ClientCreatedEvent) + client_changed = EventSource(ClientChangedEvent) + client_deleted = EventSource(ClientDeletedEvent) + + +class OAuthProvider(OAuthRelation): + """A provider object for OIDC Providers.""" + + on = OAuthProviderEvents() + + def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME) -> None: + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + + events = self._charm.on[relation_name] + self.framework.observe( + events.relation_changed, + self._get_client_config_from_relation_data, + ) + self.framework.observe( + events.relation_broken, + self._on_relation_broken, + ) + + def _get_client_config_from_relation_data(self, event: RelationChangedEvent) -> None: + if not self.model.unit.is_leader(): + return + + data = event.relation.data[event.app] + if not data: + logger.info("No requirer relation data available.") + return + + client_data = _load_data(data, OAUTH_REQUIRER_JSON_SCHEMA) + redirect_uri = client_data.get("redirect_uri") + scope = client_data.get("scope") + grant_types = client_data.get("grant_types") + audience = client_data.get("audience") + token_endpoint_auth_method = client_data.get("token_endpoint_auth_method") + + data = event.relation.data[self._charm.app] + if not data: + logger.info("No provider relation data available.") + return + provider_data = _load_data(data, OAUTH_PROVIDER_JSON_SCHEMA) + client_id = provider_data.get("client_id") + + relation_id = event.relation.id + + if client_id: + # Modify an existing client + self.on.client_changed.emit( + redirect_uri, + scope, + grant_types, + audience, + token_endpoint_auth_method, + relation_id, + client_id, + ) + else: + # Create a new client + self.on.client_created.emit( + redirect_uri, scope, grant_types, audience, token_endpoint_auth_method, relation_id + ) + + def _get_secret_label(self, relation: Relation) -> str: + return f"client_secret_{relation.id}" + + def _on_relation_broken(self, event: RelationBrokenEvent) -> None: + # Workaround for https://github.com/canonical/operator/issues/888 + self._pop_relation_data(event.relation.id) + + self._delete_juju_secret(event.relation) + self.on.client_deleted.emit(event.relation.id) + + def _create_juju_secret(self, client_secret: str, relation: Relation) -> Secret: + """Create a juju secret and grant it to a relation.""" + secret = {CLIENT_SECRET_FIELD: client_secret} + juju_secret = self.model.app.add_secret(secret, label=self._get_secret_label(relation)) + juju_secret.grant(relation) + return juju_secret + + def _delete_juju_secret(self, relation: Relation) -> None: + try: + secret = self.model.get_secret(label=self._get_secret_label(relation)) + except SecretNotFoundError: + return + else: + secret.remove_all_revisions() + + def set_provider_info_in_relation_data( + self, + issuer_url: str, + authorization_endpoint: str, + token_endpoint: str, + introspection_endpoint: str, + userinfo_endpoint: str, + jwks_endpoint: str, + scope: str, + groups: Optional[str] = None, + ca_chain: Optional[str] = None, + jwt_access_token: Optional[bool] = False, + ) -> None: + """Put the provider information in the databag.""" + if not self.model.unit.is_leader(): + return + + data = { + "issuer_url": issuer_url, + "authorization_endpoint": authorization_endpoint, + "token_endpoint": token_endpoint, + "introspection_endpoint": introspection_endpoint, + "userinfo_endpoint": userinfo_endpoint, + "jwks_endpoint": jwks_endpoint, + "scope": scope, + "jwt_access_token": jwt_access_token, + } + if groups: + data["groups"] = groups + if ca_chain: + data["ca_chain"] = ca_chain + + for relation in self.model.relations[self._relation_name]: + relation.data[self.model.app].update(_dump_data(data)) + + def set_client_credentials_in_relation_data( + self, relation_id: int, client_id: str, client_secret: str + ) -> None: + """Put the client credentials in the databag.""" + if not self.model.unit.is_leader(): + return + + relation = self.model.get_relation(self._relation_name, relation_id) + if not relation or not relation.app: + return + # TODO: What if we are refreshing the client_secret? We need to add a + # new revision for that + secret = self._create_juju_secret(client_secret, relation) + data = {"client_id": client_id, "client_secret_id": secret.id} + relation.data[self.model.app].update(_dump_data(data)) diff --git a/penpot_rock/rockcraft.yaml b/penpot_rock/rockcraft.yaml index db7981c..6f20bc2 100644 --- a/penpot_rock/rockcraft.yaml +++ b/penpot_rock/rockcraft.yaml @@ -33,6 +33,30 @@ parts: override-build: | craftctl default + git apply <<'EOF' + diff --git a/docker/images/files/config.js b/docker/images/files/config.js + index 7bc9ce940..5056b0fa5 100644 + --- a/docker/images/files/config.js + +++ b/docker/images/files/config.js + @@ -1,2 +1,2 @@ + // Frontend configuration + -//var penpotFlags = ""; + +var penpotFlags = ""; + diff --git a/docker/images/files/nginx-entrypoint.sh b/docker/images/files/nginx-entrypoint.sh + index c34245230..f286e3c0d 100644 + --- a/docker/images/files/nginx-entrypoint.sh + +++ b/docker/images/files/nginx-entrypoint.sh + @@ -7,7 +7,7 @@ + update_flags() { + if [ -n "$PENPOT_FLAGS" ]; then + sed -i \ + - -e "s|^//var penpotFlags = .*;|var penpotFlags = \"$PENPOT_FLAGS\";|g" \ + + -e "s|^var penpotFlags = .*;|var penpotFlags = \"$PENPOT_FLAGS\";|g" \ + "$1" + fi + } + EOF + # install clojure curl -L https://github.com/clojure/brew-install/releases/download/1.11.3.1463/linux-install.sh -o install-clojure echo "0c41063a2fefb53a31bc1bc236899955f759c5103dc0495489cdd74bf8f114bb install-clojure" | shasum -c diff --git a/requirements.txt b/requirements.txt index d136cbf..0c22e31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ ops==2.15.0 dnspython==2.6.1 +jsonschema==4.23.0 +rpds-py==0.18.1 diff --git a/src/charm.py b/src/charm.py index b013018..1e4e742 100755 --- a/src/charm.py +++ b/src/charm.py @@ -10,11 +10,13 @@ import logging import secrets import typing +import urllib.parse import dns.resolver import ops from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires from charms.data_platform_libs.v0.s3 import S3Requirer +from charms.hydra.v0.oauth import ClientConfig, OAuthRequirer from charms.redis_k8s.v0.redis import RedisRelationCharmEvents, RedisRequires from charms.smtp_integrator.v0.smtp import SmtpRequires, TransportSecurity from charms.traefik_k8s.v2.ingress import IngressPerAppRequirer @@ -42,6 +44,8 @@ def __init__(self, *args: typing.Any): self.smtp = SmtpRequires(self) self.s3 = S3Requirer(self, relation_name="s3") self.ingress = IngressPerAppRequirer(self, port=8080) + self.oauth: OAuthRequirer | None = None + self.framework.observe(self.on.upgrade_charm, self._reconcile) self.framework.observe(self.on.config_changed, self._reconcile) self.framework.observe(self.on.penpot_peer_relation_created, self._reconcile) self.framework.observe(self.on.penpot_peer_relation_changed, self._reconcile) @@ -59,6 +63,9 @@ def __init__(self, *args: typing.Any): self.framework.observe(self.ingress.on.ready, self._reconcile) self.framework.observe(self.ingress.on.revoked, self._reconcile) self.framework.observe(self.on.penpot_pebble_ready, self._reconcile) + self.framework.observe(self.on.oauth_relation_created, self._reconcile) + self.framework.observe(self.on.oauth_relation_changed, self._reconcile) + self.framework.observe(self.on.oauth_relation_broken, self._reconcile) self.framework.observe(self.on.create_profile_action, self._on_create_profile_action) self.framework.observe(self.on.delete_profile_action, self._on_delete_profile_action) @@ -119,6 +126,9 @@ def _on_delete_profile_action(self, event: ops.ActionEvent) -> None: def _reconcile(self, _: ops.EventBase) -> None: """Reconcile penpot services.""" + oauth = self._get_oauth() + if oauth: + oauth.update_client_config(self._get_oauth_client_config()) if not self._check_ready(): if self.container.can_connect() and self.container.get_services(): self.container.stop("backend") @@ -171,6 +181,7 @@ def _gen_pebble_plan(self) -> ops.pebble.LayerDict: **typing.cast(dict[str, str], self._get_redis_credentials()), **typing.cast(dict[str, str], self._get_smtp_credentials()), **typing.cast(dict[str, str], self._get_s3_credentials()), + **self._get_penpot_oauth_config(), }, }, "exporter": { @@ -213,6 +224,8 @@ def _check_ready(self) -> bool: "ingress": self._get_public_uri(), "penpot container": self.container.can_connect(), } + if self._get_penpot_oauth_config(): + requirements["smtp"] = self._get_smtp_credentials() unfulfilled = sorted([k for k, v in requirements.items() if not v]) if unfulfilled: self.unit.status = ops.BlockedStatus(f"waiting for {', '.join(unfulfilled)}") @@ -346,7 +359,9 @@ def _get_public_uri(self) -> str | None: Returns: Penpot public URI. """ - return self.ingress.url + return ( + None if self.ingress.url is None else self.ingress.url.replace("http://", "https://") + ) def _get_penpot_frontend_options(self) -> list[str]: """Retrieve the penpot options for the penpot frontend. @@ -354,13 +369,14 @@ def _get_penpot_frontend_options(self) -> list[str]: Returns: Penpot frontend options. """ - return sorted( - [ - "enable-login-with-password", - "disable-registration", - "disable-onboarding-questions", - ] - ) + options = [ + "disable-onboarding-questions", + ] + if self._get_penpot_oauth_config(): + options.extend(["enable-login-with-oidc", "disable-login-with-password"]) + else: + options.extend(["disable-registration", "enable-login-with-password"]) + return sorted(options) def _get_penpot_backend_options(self) -> list[str]: """Retrieve the penpot options for the penpot backend. @@ -368,17 +384,18 @@ def _get_penpot_backend_options(self) -> list[str]: Returns: Penpot backend options. """ - return sorted( - [ - "enable-login-with-password", - "enable-prepl-server", - "disable-registration", - "disable-telemetry", - "disable-onboarding-questions", - "disable-log-emails", - ("enable" if self._get_smtp_credentials() else "disable") + "-smtp", - ] - ) + options = [ + "enable-prepl-server", + "disable-telemetry", + "disable-onboarding-questions", + "disable-log-emails", + ("enable" if self._get_smtp_credentials() else "disable") + "-smtp", + ] + if self._get_penpot_oauth_config(): + options.extend(["enable-login-with-oidc", "disable-login-with-password"]) + else: + options.extend(["disable-registration", "enable-login-with-password"]) + return sorted(options) def _get_local_resolver(self) -> str: """Retrieve the current nameserver address being used. @@ -428,6 +445,59 @@ def _get_kubernetes_cluster_domain(self) -> str: return "cluster.local" return answers.qname.to_text().removeprefix("kubernetes.default.svc").strip(".") + def _get_oauth(self) -> OAuthRequirer | None: + """Retrieve the OAuthRequirer object if available. + + Returns: + OAuthRequirer object. + """ + if self.oauth: + return self.oauth + client_config = self._get_oauth_client_config() + if not client_config: + return None + self.oauth = OAuthRequirer(self, client_config=client_config) + return self.oauth + + def _get_oauth_client_config(self) -> ClientConfig | None: + """Retrieve the oauth ClientConfig object for the oauth charm library if available. + + Returns: + ClientConfig object. + """ + public_uri = self._get_public_uri() + if not public_uri: + return None + return ClientConfig( + urllib.parse.urljoin(public_uri, "/api/auth/oauth/oidc/callback"), + scope="openid profile email", + grant_types=["authorization_code"], + # this is not a secret + token_endpoint_auth_method="client_secret_post", # nosec + ) + + def _get_penpot_oauth_config(self) -> dict[str, str]: + """Retrieve oauth-related configurations for penpot. + + Returns: + Oauth-related penpot configurations. + """ + oauth = self._get_oauth() + if not oauth: + return {} + oauth_provider = oauth.get_provider_info() + if not oauth_provider: + return {} + return { + "PENPOT_OIDC_CLIENT_ID": oauth_provider.client_id, + "PENPOT_OIDC_BASE_URI": oauth_provider.issuer_url, + "PENPOT_OIDC_CLIENT_SECRET": oauth_provider.client_secret, + "PENPOT_OIDC_AUTH_URI": oauth_provider.authorization_endpoint, + "PENPOT_OIDC_TOKEN_URI": oauth_provider.token_endpoint, + "PENPOT_OIDC_USER_URI": oauth_provider.userinfo_endpoint, + "PENPOT_OIDC_SCOPES": "openid profile email", + } + if __name__ == "__main__": # pragma: nocover ops.main.main(PenpotCharm) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 74bbcea..e4777f2 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,37 +4,57 @@ # See LICENSE file for licensing details. """Integration tests.""" - +import asyncio import logging +import re import time import juju.action import pytest import requests +from oauth_tools.oauth_helpers import ( + access_application_login_page, + click_on_sign_in_button_by_text, + complete_auth_code_login, + deploy_identity_bundle, +) +from playwright.async_api import expect from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) +pytest_plugins = ["oauth_tools.fixtures"] + @pytest.mark.abort_on_fail -async def test_build_and_deploy( - ops_test: OpsTest, pytestconfig: pytest.Config, minio, mailcatcher +async def test_build_and_deploy( # pylint: disable=too-many-locals + ops_test: OpsTest, pytestconfig: pytest.Config, minio, mailcatcher, ext_idp_service ): """ arrange: set up the test Juju model. act: build and deploy the Penpot charm with required services. assert: the Penpot charm becomes active. """ + await deploy_identity_bundle( + ops_test=ops_test, bundle_channel="latest/edge", ext_idp_service=ext_idp_service + ) + await ops_test.juju("refresh", "identity-platform-login-ui-operator", "--revision", "105") + await ops_test.juju( + "integrate", + "identity-platform-login-ui-operator:receive-ca-cert", + "self-signed-certificates", + ) charm = pytestconfig.getoption("--charm-file") penpot_image = pytestconfig.getoption("--penpot-image") assert penpot_image if not charm: charm = await ops_test.build_charm(".") assert ops_test.model + logger.info("deploying penpot charm") + num_units = 2 penpot = await ops_test.model.deploy( - f"./{charm}", resources={"penpot-image": penpot_image}, num_units=2 + f"./{charm}", resources={"penpot-image": penpot_image}, num_units=num_units ) - postgresql_k8s = await ops_test.model.deploy("postgresql-k8s", channel="14/stable", trust=True) redis_k8s = await ops_test.model.deploy("redis-k8s", channel="edge") smtp_integrator = await ops_test.model.deploy( "smtp-integrator", @@ -55,7 +75,9 @@ async def test_build_and_deploy( trust=True, revision=109, ) - await ops_test.model.wait_for_idle(timeout=900) + await ops_test.model.wait_for_idle( + timeout=900, apps=[s3_integrator.name, "self-signed-certificates"] + ) action = await s3_integrator.units[0].run_action( "sync-s3-credentials", **{ @@ -64,12 +86,61 @@ async def test_build_and_deploy( }, ) await action.wait() - await ops_test.model.add_relation(penpot.name, postgresql_k8s.name) + await ops_test.model.add_relation(penpot.name, "postgresql-k8s") await ops_test.model.add_relation(penpot.name, redis_k8s.name) await ops_test.model.add_relation(penpot.name, s3_integrator.name) await ops_test.model.add_relation(penpot.name, f"{smtp_integrator.name}:smtp") await ops_test.model.add_relation(penpot.name, nginx_ingress_integrator.name) - await ops_test.model.wait_for_idle(timeout=900, status="active") + await ops_test.model.wait_for_idle(timeout=900, status="active", raise_on_error=False) + logger.info( + "test user account: (%s, %s)", ext_idp_service.user_email, ext_idp_service.user_password + ) + action = ( + await ops_test.model.applications["self-signed-certificates"] + .units[0] + .run_action("get-ca-certificate") + ) + await action.wait() + ca_cert: str = action.results["ca-certificate"] + for unit in range(num_units): + logger.info("copying oauth ca cert into penpot/%s", unit) + await ops_test.juju( + "ssh", + "--container", + "penpot", + f"penpot/{unit}", + "cp", + "/dev/stdin", + "/oauth.crt", + stdin=ca_cert.encode("ascii"), + ) + logger.info("installing oauth ca cert into penpot/%s java trust", unit) + await ops_test.juju( + "ssh", + "--container", + "penpot", + f"penpot/{unit}", + "/usr/lib/jvm/java-21-openjdk-amd64/bin/keytool", + "-import", + "-trustcacerts", + "-file", + "/oauth.crt", + "-keystore", + "/usr/lib/jvm/java-21-openjdk-amd64/lib/security/cacerts", + "-storepass", + "changeit", + "-noprompt", + ) + logger.info("restart penpot backend in penpot/%s", unit) + await ops_test.juju( + "ssh", + "--container", + "penpot", + f"penpot/{unit}", + "pebble", + "restart", + "backend", + ) async def test_create_profile(ops_test: OpsTest, ingress_address): @@ -127,3 +198,18 @@ async def test_create_profile(ops_test: OpsTest, ingress_address): else: raise TimeoutError("timed out waiting for login response") assert response.status_code == 400 + + +async def test_oauth(ops_test, page, ext_idp_service): + """ + arrange: integrate the penpot charm with an oauth provider. + act: login penpot using openid connect. + assert: login success. + """ + await ops_test.model.add_relation("penpot:oauth", "hydra") + await ops_test.model.wait_for_idle(timeout=900, status="active") + await asyncio.sleep(120) + await access_application_login_page(page=page, url="https://penpot.local/#/auth/login") + await click_on_sign_in_button_by_text(page=page, text="OpenID") + await complete_auth_code_login(page=page, ops_test=ops_test, ext_idp_service=ext_idp_service) + await expect(page).to_have_url(re.compile("^https://penpot\\.local/#/auth/register.*")) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 11966c1..9e46385 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -152,7 +152,7 @@ def test_public_uri(harness): harness.begin_with_initial_hooks() assert harness.charm._get_public_uri() is None harness.setup_ingress_integration() - assert harness.charm._get_public_uri() == "http://penpot.local/" + assert harness.charm._get_public_uri() == "https://penpot.local/" def test_penpot_pebble_layer(harness): @@ -210,7 +210,7 @@ def test_penpot_pebble_layer(harness): "enable-prepl-server " "enable-smtp" ), - "PENPOT_PUBLIC_URI": "http://penpot.local/", + "PENPOT_PUBLIC_URI": "https://penpot.local/", "PENPOT_REDIS_URI": "redis://redis-hostname:6379", "PENPOT_SMTP_DEFAULT_FROM": "no-reply@example.com", "PENPOT_SMTP_DEFAULT_REPLY_TO": "no-reply@example.com", diff --git a/tox.ini b/tox.ini index f6e6175..cc8b3c0 100644 --- a/tox.ini +++ b/tox.ini @@ -54,9 +54,11 @@ deps = pytest pytest-asyncio pytest-operator + pytest-playwright requests types-PyYAML types-requests + git+https://github.com/weiiwang01/iam-bundle@fix-dex-manifest#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pydocstyle {[vars]src_path} @@ -106,6 +108,8 @@ deps = pytest pytest-asyncio pytest-operator + pytest-playwright + git+https://github.com/canonical/iam-bundle@main#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From be95c5aae7ed59d39ddf97f46eacfef8d3dd59cd Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 04:49:37 +0000 Subject: [PATCH 02/27] Add check --- requirements.txt | 1 + src/charm.py | 24 +++++++++++++++++++++++- tests/integration/test_charm.py | 1 - tests/unit/conftest.py | 6 +++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c22e31..8af9a36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ ops==2.15.0 dnspython==2.6.1 jsonschema==4.23.0 +requests==2.32.3 rpds-py==0.18.1 diff --git a/src/charm.py b/src/charm.py index 1e4e742..49c7d48 100755 --- a/src/charm.py +++ b/src/charm.py @@ -9,11 +9,14 @@ import logging import secrets +import time import typing import urllib.parse import dns.resolver import ops +import requests + from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires from charms.data_platform_libs.v0.s3 import S3Requirer from charms.hydra.v0.oauth import ClientConfig, OAuthRequirer @@ -143,7 +146,26 @@ def _reconcile(self, _: ops.EventBase) -> None: self.container.start("exporter") else: self.container.stop("exporter") - self.unit.status = ops.ActiveStatus() + deadline = time.time() + 300 + while time.time() < deadline: + if self._check_penpot_backend_ready(): + self.unit.status = ops.ActiveStatus() + return + else: + time.sleep(3) + self.unit.status = ops.WaitingStatus("waiting for penpot services") + raise TimeoutError("Timeout waiting for penpot services") + + def _check_penpot_backend_ready(self) -> bool: # pragma: nocover + """Check penpot backend is ready. + + Returns: + True if the penpot backend is ready, False otherwise. + """ + try: + return requests.get("http://localhost:6060/readyz", timeout=1).text == "OK" + except (requests.exceptions.RequestException, TimeoutError): + return False def _gen_pebble_plan(self) -> ops.pebble.LayerDict: """Generate penpot pebble plan. diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index e4777f2..34251e0 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -208,7 +208,6 @@ async def test_oauth(ops_test, page, ext_idp_service): """ await ops_test.model.add_relation("penpot:oauth", "hydra") await ops_test.model.wait_for_idle(timeout=900, status="active") - await asyncio.sleep(120) await access_application_login_page(page=page, url="https://penpot.local/#/auth/login") await click_on_sign_in_button_by_text(page=page, text="OpenID") await complete_auth_code_login(page=page, ops_test=ops_test, ext_idp_service=ext_idp_service) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 51a364a..6dd14c5 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -3,6 +3,7 @@ """Unit tests fixtures.""" import json +import unittest.mock import ops.testing import pytest @@ -128,4 +129,7 @@ def __getattr__(self, attr): def harness_fixture(monkeypatch): """Harness fixture.""" monkeypatch.setenv("JUJU_VERSION", "3.5.0") - return Harness(ops.testing.Harness(PenpotCharm)) + with unittest.mock.patch.object( + PenpotCharm, "_check_penpot_backend_ready", unittest.mock.MagicMock(return_value=True) + ): + yield Harness(ops.testing.Harness(PenpotCharm)) From b1ee149d3e9f76eb6646c0915420635db80b59d2 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 05:48:44 +0000 Subject: [PATCH 03/27] Fix linitng --- src/charm.py | 12 +++++++----- tests/integration/test_charm.py | 1 - tox.ini | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/charm.py b/src/charm.py index 49c7d48..c2db25e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -16,7 +16,6 @@ import dns.resolver import ops import requests - from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires from charms.data_platform_libs.v0.s3 import S3Requirer from charms.hydra.v0.oauth import ClientConfig, OAuthRequirer @@ -128,7 +127,11 @@ def _on_delete_profile_action(self, event: ops.ActionEvent) -> None: event.set_results({"email": email}) def _reconcile(self, _: ops.EventBase) -> None: - """Reconcile penpot services.""" + """Reconcile penpot services. + + Raises: + TimeoutError: failed to start penpot service in timeout. + """ oauth = self._get_oauth() if oauth: oauth.update_client_config(self._get_oauth_client_config()) @@ -151,9 +154,8 @@ def _reconcile(self, _: ops.EventBase) -> None: if self._check_penpot_backend_ready(): self.unit.status = ops.ActiveStatus() return - else: - time.sleep(3) - self.unit.status = ops.WaitingStatus("waiting for penpot services") + self.unit.status = ops.WaitingStatus("waiting for penpot services") + time.sleep(3) raise TimeoutError("Timeout waiting for penpot services") def _check_penpot_backend_ready(self) -> bool: # pragma: nocover diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 34251e0..25ed0b7 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,7 +4,6 @@ # See LICENSE file for licensing details. """Integration tests.""" -import asyncio import logging import re import time diff --git a/tox.ini b/tox.ini index cc8b3c0..5dca505 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,7 @@ deps = requests types-PyYAML types-requests - git+https://github.com/weiiwang01/iam-bundle@fix-dex-manifest#egg=oauth_tools + git+https://github.com/canonical/iam-bundle@main#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pydocstyle {[vars]src_path} @@ -109,7 +109,7 @@ deps = pytest-asyncio pytest-operator pytest-playwright - git+https://github.com/canonical/iam-bundle@main#egg=oauth_tools + git+https://github.com/weiiwang01/iam-bundle@fix-dex-manifest -r{toxinidir}/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From c3396933304290ea16a8a041916c644530e13157 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 06:44:02 +0000 Subject: [PATCH 04/27] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5dca505..1fab937 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ deps = pytest-asyncio pytest-operator pytest-playwright - git+https://github.com/weiiwang01/iam-bundle@fix-dex-manifest + git+https://github.com/weiiwang01/iam-bundle@tmp-fix-race-condition#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From 934de84965100b7d015f6d131c82185cda71b948 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 06:58:13 +0000 Subject: [PATCH 05/27] Add pre-run-script --- .github/workflows/test.yaml | 1 + src/charm.py | 2 +- tests/integration/prepare.sh | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100755 tests/integration/prepare.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 302905c..79540d7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,3 +20,4 @@ jobs: juju-channel: 3.4/stable self-hosted-runner: true self-hosted-runner-label: "edge" + pre-run-script: tests/integration/prepare.sh diff --git a/src/charm.py b/src/charm.py index c2db25e..3792fad 100755 --- a/src/charm.py +++ b/src/charm.py @@ -150,11 +150,11 @@ def _reconcile(self, _: ops.EventBase) -> None: else: self.container.stop("exporter") deadline = time.time() + 300 + self.unit.status = ops.WaitingStatus("waiting for penpot services") while time.time() < deadline: if self._check_penpot_backend_ready(): self.unit.status = ops.ActiveStatus() return - self.unit.status = ops.WaitingStatus("waiting for penpot services") time.sleep(3) raise TimeoutError("Timeout waiting for penpot services") diff --git a/tests/integration/prepare.sh b/tests/integration/prepare.sh new file mode 100755 index 0000000..26a35d2 --- /dev/null +++ b/tests/integration/prepare.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -exo pipefail + +sudo bash -c "echo 127.0.0.1 penpot.local >> /etc/hosts" +pip install pytest-playwright +playwright install --with-deps chromium From cf6a63987d3e04d21927dea18c074543151bcf0f Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 07:02:06 +0000 Subject: [PATCH 06/27] Update test_charm.py --- tests/integration/test_charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 25ed0b7..e2e3a0c 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -4,6 +4,7 @@ # See LICENSE file for licensing details. """Integration tests.""" + import logging import re import time From 6a82f50f4aaf4bba5ac325544d14fee90aff83a0 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 07:35:56 +0000 Subject: [PATCH 07/27] Update tests/integration/prepare.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/integration/prepare.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/prepare.sh b/tests/integration/prepare.sh index 26a35d2..c14fd9d 100755 --- a/tests/integration/prepare.sh +++ b/tests/integration/prepare.sh @@ -1,3 +1,6 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + #!/usr/bin/env bash set -exo pipefail From 2c29a6f8c5c1e2a590edd3b4e42434dee16e2952 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 08:18:04 +0000 Subject: [PATCH 08/27] Update test.yaml --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 79540d7..f1cdd77 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,4 +20,5 @@ jobs: juju-channel: 3.4/stable self-hosted-runner: true self-hosted-runner-label: "edge" + microk8s-addons: "dns ingress rbac storage metallb:10.64.140.43-10.64.140.49" pre-run-script: tests/integration/prepare.sh From 0415bf87bd9ee303e7c751c160511263c12d5799 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 16 Aug 2024 08:18:19 +0000 Subject: [PATCH 09/27] Update prepare.sh --- tests/integration/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/prepare.sh b/tests/integration/prepare.sh index c14fd9d..e1b9083 100755 --- a/tests/integration/prepare.sh +++ b/tests/integration/prepare.sh @@ -1,8 +1,8 @@ +#!/usr/bin/env bash + # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -#!/usr/bin/env bash - set -exo pipefail sudo bash -c "echo 127.0.0.1 penpot.local >> /etc/hosts" From 4e62fea7ce2d2c0a60b4914b5d814c389755e49a Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 19 Aug 2024 07:39:24 +0000 Subject: [PATCH 10/27] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1fab937..48591b0 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ deps = pytest-asyncio pytest-operator pytest-playwright - git+https://github.com/weiiwang01/iam-bundle@tmp-fix-race-condition#egg=oauth_tools + git+https://github.com/weiiwang01/iam-bundle@fix-race-condition#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From 6452414fc11168cde5439be34934cd8ed295d330 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 22 Aug 2024 06:09:31 +0000 Subject: [PATCH 11/27] Update --- .trivyignore | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index a3574f5..7ec2549 100644 --- a/.trivyignore +++ b/.trivyignore @@ -11,5 +11,6 @@ CVE-2023-5685 CVE-2021-37714 CVE-2022-1471 CVE-2024-21634 +CVE-2024-22871 # nodejs CVE-2024-37890 diff --git a/tox.ini b/tox.ini index 48591b0..f0ab284 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ deps = pytest-asyncio pytest-operator pytest-playwright - git+https://github.com/weiiwang01/iam-bundle@fix-race-condition#egg=oauth_tools + git+https://github.com/canonical/iam-bundle@main#egg=oauth_tools -r{toxinidir}/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From 0faf2933b82c4353bc0871722be8afc6fbd1f4c9 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 22 Aug 2024 19:11:08 +0000 Subject: [PATCH 12/27] Update charm.py --- src/charm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/charm.py b/src/charm.py index 3792fad..df5e2c0 100755 --- a/src/charm.py +++ b/src/charm.py @@ -127,11 +127,7 @@ def _on_delete_profile_action(self, event: ops.ActionEvent) -> None: event.set_results({"email": email}) def _reconcile(self, _: ops.EventBase) -> None: - """Reconcile penpot services. - - Raises: - TimeoutError: failed to start penpot service in timeout. - """ + """Reconcile penpot services.""" oauth = self._get_oauth() if oauth: oauth.update_client_config(self._get_oauth_client_config()) @@ -156,7 +152,7 @@ def _reconcile(self, _: ops.EventBase) -> None: self.unit.status = ops.ActiveStatus() return time.sleep(3) - raise TimeoutError("Timeout waiting for penpot services") + self.unit.status = ops.BlockedStatus("timeout waiting for penpot services") def _check_penpot_backend_ready(self) -> bool: # pragma: nocover """Check penpot backend is ready. From 796c752fa32059b67a24c29e31be25a375941a62 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 23 Aug 2024 06:19:03 +0000 Subject: [PATCH 13/27] Update charm.py --- src/charm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index df5e2c0..8242b42 100755 --- a/src/charm.py +++ b/src/charm.py @@ -94,7 +94,7 @@ def _on_create_profile_action(self, event: ops.ActionEvent) -> None: combine_stderr=True, ) try: - process.wait() + process.wait_output() except ops.pebble.ExecError as exc: event.fail(typing.cast(str, exc.stdout)) return @@ -120,7 +120,7 @@ def _on_delete_profile_action(self, event: ops.ActionEvent) -> None: combine_stderr=True, ) try: - process.wait() + process.wait_output() except ops.pebble.ExecError as exc: event.fail(typing.cast(str, exc.stdout)) return From c4fcc2b5671e1f962c03abebcb6b20ae6e5c7ebb Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 23 Aug 2024 10:04:48 +0000 Subject: [PATCH 14/27] Update .trivyignore --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index ce7751d..ac5c305 100644 --- a/.trivyignore +++ b/.trivyignore @@ -12,6 +12,7 @@ CVE-2021-37714 CVE-2022-1471 CVE-2024-21634 CVE-2024-22871 +CVE-2024-7885 # nodejs CVE-2024-37890 # clojure From 28f1a14687f0dc3d40ea4eb4b911b47d3e7efeb0 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 26 Aug 2024 16:03:38 +0000 Subject: [PATCH 15/27] Update rockcraft.yaml --- penpot_rock/rockcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/penpot_rock/rockcraft.yaml b/penpot_rock/rockcraft.yaml index 6f20bc2..23e3145 100644 --- a/penpot_rock/rockcraft.yaml +++ b/penpot_rock/rockcraft.yaml @@ -33,6 +33,8 @@ parts: override-build: | craftctl default + # The upstream version of the Nginx entrypoint can only apply changes to config.js once. + # Change it so that it can apply changes multiple times. git apply <<'EOF' diff --git a/docker/images/files/config.js b/docker/images/files/config.js index 7bc9ce940..5056b0fa5 100644 From ecf23fc7ead383f1b9effcbe9bb9c07f0aae3329 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 30 Aug 2024 07:35:46 +0000 Subject: [PATCH 16/27] Update charm.py --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 8242b42..3cd4454 100755 --- a/src/charm.py +++ b/src/charm.py @@ -145,7 +145,7 @@ def _reconcile(self, _: ops.EventBase) -> None: self.container.start("exporter") else: self.container.stop("exporter") - deadline = time.time() + 300 + deadline = time.time() + 120 self.unit.status = ops.WaitingStatus("waiting for penpot services") while time.time() < deadline: if self._check_penpot_backend_ready(): From 13f292c7355db30a2688ee22e6b9906de54e4796 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 2 Sep 2024 15:53:36 +0000 Subject: [PATCH 17/27] Update charm.py --- src/charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/charm.py b/src/charm.py index 3cd4454..9b2b88e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -245,6 +245,7 @@ def _check_ready(self) -> bool: "penpot container": self.container.can_connect(), } if self._get_penpot_oauth_config(): + # SMTP is required for the OpenID Connect-based registration process requirements["smtp"] = self._get_smtp_credentials() unfulfilled = sorted([k for k, v in requirements.items() if not v]) if unfulfilled: From b9837edea730128b2fbbb70da5572db8ccb9a663 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 3 Sep 2024 08:04:11 +0000 Subject: [PATCH 18/27] Require ingress HTTPS --- src/charm.py | 2 ++ tests/integration/test_charm.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/charm.py b/src/charm.py index 9b2b88e..c67d8fc 100755 --- a/src/charm.py +++ b/src/charm.py @@ -243,6 +243,8 @@ def _check_ready(self) -> bool: "s3": self._get_s3_credentials(), "ingress": self._get_public_uri(), "penpot container": self.container.can_connect(), + "https enabled on ingress": not self._get_public_uri() + or self._get_public_uri().startswith("https://"), } if self._get_penpot_oauth_config(): # SMTP is required for the OpenID Connect-based registration process diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index e2e3a0c..0800475 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -86,6 +86,7 @@ async def test_build_and_deploy( # pylint: disable=too-many-locals }, ) await action.wait() + await ops_test.model.add_relation("self-signed-certificates", "nginx-ingress-integrator") await ops_test.model.add_relation(penpot.name, "postgresql-k8s") await ops_test.model.add_relation(penpot.name, redis_k8s.name) await ops_test.model.add_relation(penpot.name, s3_integrator.name) From 0ec7a60693bbf00ab21cd0348c819a906543857f Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 3 Sep 2024 08:11:38 +0000 Subject: [PATCH 19/27] Update charm.py --- src/charm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index c67d8fc..d1ae2a3 100755 --- a/src/charm.py +++ b/src/charm.py @@ -382,9 +382,7 @@ def _get_public_uri(self) -> str | None: Returns: Penpot public URI. """ - return ( - None if self.ingress.url is None else self.ingress.url.replace("http://", "https://") - ) + return self.ingress.url def _get_penpot_frontend_options(self) -> list[str]: """Retrieve the penpot options for the penpot frontend. From 16bf5d6c718eaff8f1a061ab9c47b5a923a99a17 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 4 Sep 2024 08:36:53 +0000 Subject: [PATCH 20/27] Update conftest.py --- tests/unit/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 6dd14c5..e2c2dca 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -74,7 +74,7 @@ def setup_ingress_integration(self): self.harness.add_relation( "ingress", "nginx-ingress-integrator", - app_data={"ingress": '{"url": "http://penpot.local/"}'}, + app_data={"ingress": '{"url": "https://penpot.local/"}'}, ) def setup_smtp_integration(self, use_password: bool = False): From ff4ab7ef7ee99333c555af21ada03ef0154cf735 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 4 Sep 2024 09:12:29 +0000 Subject: [PATCH 21/27] Update charm.py --- src/charm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index d1ae2a3..e893e97 100755 --- a/src/charm.py +++ b/src/charm.py @@ -236,15 +236,15 @@ def _check_ready(self) -> bool: Returns: True if penpot is ready to start. """ + public_uri = self._get_public_uri() requirements = { "peer integration": self._get_penpot_secret_key(), "postgresql": self._get_postgresql_credentials(), "redis": self._get_redis_credentials(), "s3": self._get_s3_credentials(), - "ingress": self._get_public_uri(), + "ingress": public_uri, "penpot container": self.container.can_connect(), - "https enabled on ingress": not self._get_public_uri() - or self._get_public_uri().startswith("https://"), + "https enabled on ingress": not public_uri or public_uri.startswith("https://"), } if self._get_penpot_oauth_config(): # SMTP is required for the OpenID Connect-based registration process From 459173fbc081fc729da0553b12e61da8b0d546b9 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 4 Sep 2024 10:28:57 +0000 Subject: [PATCH 22/27] Update test_charm.py --- tests/integration/test_charm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 0800475..8f30855 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -171,10 +171,11 @@ async def test_create_profile(ops_test: OpsTest, ingress_address): deadline = time.time() + 300 while time.time() < deadline: response = session.post( - f"http://{ingress_address}/api/rpc/command/login-with-password", + f"https://{ingress_address}/api/rpc/command/login-with-password", headers={"Host": "penpot.local"}, json={"~:email": email, "~:password": password}, timeout=10, + verify=False, ) if response.status_code == 200: break @@ -187,10 +188,11 @@ async def test_create_profile(ops_test: OpsTest, ingress_address): deadline = time.time() + 300 while time.time() < deadline: response = session.post( - f"http://{ingress_address}/api/rpc/command/login-with-password", + f"https://{ingress_address}/api/rpc/command/login-with-password", headers={"Host": "penpot.local"}, json={"~:email": email, "~:password": password}, timeout=10, + verify=False, ) if response.status_code == 400: break From 4f048f5d041c3dacf0afbbb1dd3b007aa716b9a6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 9 Sep 2024 07:46:45 +0000 Subject: [PATCH 23/27] Update .trivyignore --- .trivyignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.trivyignore b/.trivyignore index ac5c305..8ad8259 100644 --- a/.trivyignore +++ b/.trivyignore @@ -17,3 +17,5 @@ CVE-2024-7885 CVE-2024-37890 # clojure CVE-2024-22871 +# pebble +CVE-2024-34156 \ No newline at end of file From 407f518b4457963d3583d83fa6e6de3613316d3a Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 18 Sep 2024 06:31:25 +0000 Subject: [PATCH 24/27] Update .trivyignore --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index 8ad8259..f7fefff 100644 --- a/.trivyignore +++ b/.trivyignore @@ -13,6 +13,7 @@ CVE-2022-1471 CVE-2024-21634 CVE-2024-22871 CVE-2024-7885 +CVE-2024-1635 # nodejs CVE-2024-37890 # clojure From df5f50f1b07b07c66e78e546b2c8fe2ecb0963e9 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 20 Sep 2024 15:52:44 +0000 Subject: [PATCH 25/27] Update test_charm.py --- tests/integration/test_charm.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 8f30855..7f94f77 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -27,7 +27,7 @@ @pytest.mark.abort_on_fail -async def test_build_and_deploy( # pylint: disable=too-many-locals +async def test_build_and_deploy( ops_test: OpsTest, pytestconfig: pytest.Config, minio, mailcatcher, ext_idp_service ): """ @@ -51,9 +51,8 @@ async def test_build_and_deploy( # pylint: disable=too-many-locals charm = await ops_test.build_charm(".") assert ops_test.model logger.info("deploying penpot charm") - num_units = 2 penpot = await ops_test.model.deploy( - f"./{charm}", resources={"penpot-image": penpot_image}, num_units=num_units + f"./{charm}", resources={"penpot-image": penpot_image}, num_units=2 ) redis_k8s = await ops_test.model.deploy("redis-k8s", channel="edge") smtp_integrator = await ops_test.model.deploy( @@ -86,7 +85,7 @@ async def test_build_and_deploy( # pylint: disable=too-many-locals }, ) await action.wait() - await ops_test.model.add_relation("self-signed-certificates", "nginx-ingress-integrator") + await ops_test.model.add_relation("self-signed-certificates", nginx_ingress_integrator.name) await ops_test.model.add_relation(penpot.name, "postgresql-k8s") await ops_test.model.add_relation(penpot.name, redis_k8s.name) await ops_test.model.add_relation(penpot.name, s3_integrator.name) @@ -103,24 +102,24 @@ async def test_build_and_deploy( # pylint: disable=too-many-locals ) await action.wait() ca_cert: str = action.results["ca-certificate"] - for unit in range(num_units): - logger.info("copying oauth ca cert into penpot/%s", unit) + for unit in penpot.units: + logger.info("copying oauth ca cert into %s", unit.name) await ops_test.juju( "ssh", "--container", "penpot", - f"penpot/{unit}", + unit.name, "cp", "/dev/stdin", "/oauth.crt", stdin=ca_cert.encode("ascii"), ) - logger.info("installing oauth ca cert into penpot/%s java trust", unit) + logger.info("installing oauth ca cert into penpot/%s java trust", unit.name) await ops_test.juju( "ssh", "--container", "penpot", - f"penpot/{unit}", + unit.name, "/usr/lib/jvm/java-21-openjdk-amd64/bin/keytool", "-import", "-trustcacerts", @@ -132,12 +131,12 @@ async def test_build_and_deploy( # pylint: disable=too-many-locals "changeit", "-noprompt", ) - logger.info("restart penpot backend in penpot/%s", unit) + logger.info("restart penpot backend in penpot/%s", unit.name) await ops_test.juju( "ssh", "--container", "penpot", - f"penpot/{unit}", + unit.name, "pebble", "restart", "backend", From 778f790499abfa4801efb9632eb4fd762706fe1b Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Sun, 22 Sep 2024 15:01:55 +0000 Subject: [PATCH 26/27] Update charm.py --- src/charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/charm.py b/src/charm.py index e893e97..b4127fb 100755 --- a/src/charm.py +++ b/src/charm.py @@ -245,6 +245,7 @@ def _check_ready(self) -> bool: "ingress": public_uri, "penpot container": self.container.can_connect(), "https enabled on ingress": not public_uri or public_uri.startswith("https://"), + "OpenID provider data": not self._get_oauth() or self._get_penpot_oauth_config() } if self._get_penpot_oauth_config(): # SMTP is required for the OpenID Connect-based registration process From 7eb2df9a58184325ef559211f544e6824a1fddf7 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 23 Sep 2024 06:26:18 +0000 Subject: [PATCH 27/27] Update charm.py --- src/charm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index b4127fb..f5cf78c 100755 --- a/src/charm.py +++ b/src/charm.py @@ -245,7 +245,9 @@ def _check_ready(self) -> bool: "ingress": public_uri, "penpot container": self.container.can_connect(), "https enabled on ingress": not public_uri or public_uri.startswith("https://"), - "OpenID provider data": not self._get_oauth() or self._get_penpot_oauth_config() + "OpenID provider data": ( + not self.model.get_relation("oauth") or self._get_penpot_oauth_config() + ), } if self._get_penpot_oauth_config(): # SMTP is required for the OpenID Connect-based registration process