Add support for search identifier (#909)

# Description

What - Add support for search identifier
Why - New feature to allow updating an entity without knowing its
identifier
How - Call upsert entity API with query, and allow it in mapping

## Type of change

Please leave one option from the following and delete the rest:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

<h4> All tests should be run against the port production
environment(using a testing org). </h4>

### Core testing checklist

- [X] Integration able to create all default resources from scratch
- [X] Resync finishes successfully
- [X] Resync able to create entities
- [X] Resync able to update entities
- [X] Resync able to detect and delete entities
- [x] Scheduled resync able to abort existing resync and start a new one
- [x] Tested with at least 2 integrations from scratch
- [x] Tested with Kafka and Polling event listeners


### Integration testing checklist

- [ ] Integration able to create all default resources from scratch
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Resync finishes successfully
- [ ] If new resource kind is added or updated in the integration, add
example raw data, mapping and expected result to the `examples` folder
in the integration directory.
- [ ] If resource kind is updated, run the integration with the example
data and check if the expected result is achieved
- [ ] If new resource kind is added or updated, validate that
live-events for that resource are working as expected
- [ ] Docs PR link [here](#)

### Preflight checklist

- [ ] Handled rate limiting
- [ ] Handled pagination
- [ ] Implemented the code in async
- [ ] Support Multi account

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

Provide links to the API documentation used for this integration.

CHANGELOG.md

@@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.10.7 (2024-08-28)
+
+### Improvements
+
+- Add search identifier support (Allow to run a search query to find the identifier of the entity as part of the mapping)
+
+
 ## 0.10.6 (2024-08-31)
 
 ### Bug Fixes

port_ocean/clients/port/mixins/entities.py

@@ -29,7 +29,7 @@ async def upsert_entity(
         request_options: RequestOptions,
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
-    ) -> Entity:
+    ) -> Entity | None:
         validation_only = request_options["validation_only"]
         async with self.semaphore:
             logger.debug(
@@ -58,9 +58,16 @@ async def upsert_entity(
             )
         handle_status_code(response, should_raise)
         result = response.json()
+
         result_entity = (
             Entity.parse_obj(result["entity"]) if result.get("entity") else entity
         )
+
+        # Happens when upsert fails and search identifier is defined.
+        # We return None to ignore the entity later in the delete process
+        if result_entity.is_using_search_identifier:
+            return None
+
         # In order to save memory we'll keep only the identifier, blueprint and relations of the
         # upserted entity result for later calculations
         reduced_entity = Entity(

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -106,18 +106,27 @@ async def upsert(
                 should_raise=False,
             )
         else:
+            entities_with_search_identifier: list[Entity] = []
+            entities_without_search_identifier: list[Entity] = []
+            for entity in entities:
+                if entity.is_using_search_identifier:
+                    entities_with_search_identifier.append(entity)
+                else:
+                    entities_without_search_identifier.append(entity)
+
             ordered_created_entities = reversed(
-                order_by_entities_dependencies(entities)
+                entities_with_search_identifier
+                + order_by_entities_dependencies(entities_without_search_identifier)
             )
             for entity in ordered_created_entities:
-                modified_entities.append(
-                    await self.context.port_client.upsert_entity(
-                        entity,
-                        event.port_app_config.get_port_request_options(),
-                        user_agent_type,
-                        should_raise=False,
-                    )
+                upsertedEntity = await self.context.port_client.upsert_entity(
+                    entity,
+                    event.port_app_config.get_port_request_options(),
+                    user_agent_type,
+                    should_raise=False,
                 )
+                if upsertedEntity:
+                    modified_entities.append(upsertedEntity)
         return modified_entities
 
     async def delete(

port_ocean/core/handlers/port_app_config/models.py

@@ -13,18 +13,22 @@ class Rule(BaseModel):
     value: str
 
 
-class SearchRelation(BaseModel):
+class IngestSearchQuery(BaseModel):
     combinator: str
-    rules: list[Rule | SearchRelation]
+    rules: list[Rule | IngestSearchQuery]
 
 
 class EntityMapping(BaseModel):
-    identifier: str
+    identifier: str | IngestSearchQuery
     title: str | None
     blueprint: str
     team: str | None
     properties: dict[str, str] = Field(default_factory=dict)
-    relations: dict[str, str | SearchRelation] = Field(default_factory=dict)
+    relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict)
+
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
 
 
 class MappingsConfig(BaseModel):

port_ocean/core/integrations/mixins/sync_raw.py

@@ -154,6 +154,12 @@ async def _unregister_resource_raw(
         results: list[RAW_ITEM],
         user_agent_type: UserAgentType,
     ) -> tuple[list[Entity], list[Exception]]:
+        if resource.port.entity.mappings.is_using_search_identifier:
+            logger.info(
+                f"Skip unregistering resource of kind {resource.kind}, as mapping defined with search identifier"
+            )
+            return [], []
+
         objects_diff = await self._calculate_raw([(resource, results)])
         entities_selector_diff, errors = objects_diff[0]
 
@@ -272,7 +278,8 @@ async def register_raw(
                 [
                     entity
                     for entity in entities_to_delete
-                    if (entity.identifier, entity.blueprint)
+                    if not entity.is_using_search_identifier
+                    and (entity.identifier, entity.blueprint)
                     not in registered_entities_attributes
                 ],
             )

port_ocean/core/models.py

@@ -19,6 +19,10 @@ class Entity(BaseModel):
     properties: dict[str, Any] = {}
     relations: dict[str, Any] = {}
 
+    @property
+    def is_using_search_identifier(self) -> bool:
+        return isinstance(self.identifier, dict)
+
 
 class BlueprintRelation(BaseModel):
     many: bool

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.10.6"
+version = "0.10.7"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"