Feature/connector sync

src/connectors/__init__.py

@@ -10,21 +10,10 @@
 from database.model.project.project import Project
 from database.model.publication.publication import Publication
 from database.model.organisation.organisation import Organisation
-from .abstract.resource_connector import ResourceConnector  # noqa:F401
+from .abstract.resource_connector_by_date import ResourceConnectorByDate  # noqa:F401
 from .example.example_connector import ExampleConnector
 from .example.example_dataset_connector import ExampleDatasetConnector
-from .huggingface.huggingface_dataset_connector import HuggingFaceDatasetConnector
-from .openml.openml_dataset_connector import OpenMlDatasetConnector
-from .zenodo.zenodo_dataset_connector import ZenodoDatasetConnector
 
-dataset_connectors = {
-    c.platform_name: c
-    for c in (
-        OpenMlDatasetConnector(),
-        HuggingFaceDatasetConnector(),
-        ZenodoDatasetConnector(),
-    )
-}
 
 _path_example_resources = pathlib.Path(__file__).parent / "example" / "resources"
 
@@ -57,4 +46,4 @@
     "organisations": ExampleConnector(
         resource_class=Organisation, json_path=_path_example_resources / "organisations.json"
     ),
-}  # type: Dict[str, ResourceConnector]
+}  # type: Dict[str, ResourceConnectorByDate]

src/connectors/abstract/resource_connector.py

@@ -1,9 +1,11 @@
 import abc
-from typing import Generic, TypeVar, Iterator
+
+from typing import Generic, TypeVar
+
 
 from sqlmodel import SQLModel
 
-from connectors.resource_with_relations import ResourceWithRelations
+
 from database.model.platform.platform_names import PlatformName
 
 
@@ -25,10 +27,3 @@ def resource_class(self) -> type[RESOURCE]:
     def platform_name(self) -> PlatformName:
         """The platform of this connector"""
         pass
-
-    @abc.abstractmethod
-    def fetch_all(
-        self, limit: int | None = None
-    ) -> Iterator[SQLModel | ResourceWithRelations[SQLModel]]:
-        """Retrieve information of all resources"""
-        pass

src/connectors/abstract/resource_connector_by_date.py

@@ -0,0 +1,29 @@
+import abc
+from datetime import datetime
+from typing import Generic, Iterator, TypeVar
+from connectors.abstract.resource_connector import ResourceConnector
+from connectors.record_error import RecordError
+
+from sqlmodel import SQLModel
+
+from connectors.resource_with_relations import ResourceWithRelations
+
+RESOURCE = TypeVar("RESOURCE", bound=SQLModel)
+
+
+class ResourceConnectorByDate(ResourceConnector, Generic[RESOURCE]):
+    """
+    For every platform that offers this resource, this ResourceConnector should be implemented.
+    """
+
+    @abc.abstractmethod
+    def retry(self, _id: str) -> SQLModel | ResourceWithRelations[SQLModel] | RecordError:
+        """Retrieve information of the resource identified by id"""
+        pass
+
+    @abc.abstractmethod
+    def fetch(
+        self, from_incl: datetime, to_excl: datetime
+    ) -> Iterator[SQLModel | ResourceWithRelations[SQLModel] | RecordError]:
+        """Retrieve information of all resources"""
+        pass

src/connectors/abstract/resource_connector_by_id.py

@@ -0,0 +1,29 @@
+import abc
+from typing import Generic, Iterator, TypeVar
+
+from sqlmodel import SQLModel
+from connectors.abstract.resource_connector import ResourceConnector
+
+from connectors.resource_with_relations import ResourceWithRelations
+
+from connectors.record_error import RecordError
+
+RESOURCE = TypeVar("RESOURCE", bound=SQLModel)
+
+
+class ResourceConnectorById(ResourceConnector, Generic[RESOURCE]):
+    """
+    For every platform that offers this resource, this ResourceConnector should be implemented.
+    """
+
+    @abc.abstractmethod
+    def retry(self, _id: int) -> SQLModel | ResourceWithRelations[SQLModel] | RecordError:
+        """Retrieve information of the resource identified by id"""
+        pass
+
+    @abc.abstractmethod
+    def fetch(
+        self, from_id: int, to_id: int
+    ) -> Iterator[SQLModel | ResourceWithRelations[SQLModel] | RecordError]:
+        """Retrieve information of all resources"""
+        pass

src/connectors/config.py

@@ -0,0 +1,8 @@
+import pathlib
+import tomllib
+
+with open(pathlib.Path(__file__).parent / "config.toml", "rb") as fh:
+    CONFIG = tomllib.load(fh)
+
+DB_CONFIG = CONFIG.get("database", {})
+KEYCLOAK_CONFIG = CONFIG.get("keycloak", {})

src/connectors/config.toml

@@ -0,0 +1,24 @@
+# Configures the REST API
+# TODO: refactor configuration (https://github.com/aiondemand/AIOD-rest-api/issues/82)
+
+# Information on which database to connect to
+[database]
+host = "sqlserver"
+port = 3306
+database = "aiod"
+username = "root"
+password = "ok"
+
+# Additional options for development
+[dev]
+reload = true
+
+# Authentication and authorization
+[keycloak]
+server_url = "https://test.openml.org/aiod-auth/"
+realm = "dev"
+client_id = "aiod-api"  # a private client, used by the backend
+client_id_swagger = "aiod-api-swagger"  # a public client, used by the Swagger Frontend
+openid_connect_url = "https://test.openml.org/aiod-auth/realms/dev/.well-known/openid-configuration"
+scopes = "openid profile microprofile-jwt"
+role = "edit_aiod_resources"

src/connectors/example/example_connector.py

@@ -1,18 +1,20 @@
+from datetime import datetime
 import json
 import pathlib
 from typing import Iterator, TypeVar
 
 from sqlmodel import SQLModel
+from connectors.abstract.resource_connector_by_date import ResourceConnectorByDate
+
 
-from connectors import ResourceConnector
 from database.model.resource import resource_create
 from database.model.platform.platform_names import PlatformName
 
 
 RESOURCE = TypeVar("RESOURCE", bound=SQLModel)
 
 
-class ExampleConnector(ResourceConnector[RESOURCE]):
+class ExampleConnector(ResourceConnectorByDate[RESOURCE]):
     """
     Creating hardcoded values example values based on json files
     """
@@ -29,9 +31,21 @@ def resource_class(self) -> type[RESOURCE]:
     def platform_name(self) -> PlatformName:
         return PlatformName.example
 
-    def fetch_all(self, limit: int | None = None) -> Iterator[RESOURCE]:
+    def retry(self, _id: str) -> RESOURCE:
+        """Retrieve information of the resource identified by id"""
+        with open(self.json_path) as f:
+            json_data = json.load(f)
+        pydantic_class = resource_create(self.resource_class)
+        for json_item in json_data:
+            if json_item.get("platform_identifier") == _id:
+                return pydantic_class(**json_item)
+        raise ValueError("No resource associated with the id")
+
+    def fetch(
+        self, from_incl: datetime | None = None, to_excl: datetime | None = None
+    ) -> Iterator[RESOURCE]:
         with open(self.json_path) as f:
             json_data = json.load(f)
         pydantic_class = resource_create(self.resource_class)
-        for json_item in json_data[:limit]:
+        for json_item in json_data:
             yield pydantic_class(**json_item)

src/connectors/example/example_dataset_connector.py

@@ -1,14 +1,16 @@
+from datetime import datetime
 import typing  # noqa:F401 (flake8 raises incorrect 'Module imported but unused' error)
 
-from connectors import ResourceConnector
+
+from connectors.abstract.resource_connector_by_date import ResourceConnectorByDate
 from connectors.resource_with_relations import ResourceWithRelations
 from database.model.dataset.dataset import Dataset
 from database.model.publication.publication import Publication
 from database.model.resource import resource_create
 from database.model.platform.platform_names import PlatformName
 
 
-class ExampleDatasetConnector(ResourceConnector[Dataset]):
+class ExampleDatasetConnector(ResourceConnectorByDate[Dataset]):
     @property
     def resource_class(self) -> type[Dataset]:
         return Dataset
@@ -17,12 +19,11 @@ def resource_class(self) -> type[Dataset]:
     def platform_name(self) -> PlatformName:
         return PlatformName.example
 
-    def fetch_all(
-        self, limit: int | None = None
-    ) -> typing.Iterator[ResourceWithRelations[Dataset]]:
+    def retry(self, _id: str) -> ResourceWithRelations[Dataset]:
+        """Retrieve information of the resource identified by id"""
         pydantic_class = resource_create(Dataset)
         pydantic_class_publication = resource_create(Publication)
-        yield from [
+        datasets = [
             ResourceWithRelations[Dataset](
                 resource=pydantic_class(
                     name="Higgs",
@@ -37,6 +38,7 @@ def fetch_all(
                     has_parts=[],
                     keywords=["keyword1", "keyword2"],
                     measured_values=[],
+                    date_published=datetime.now(),
                 ),
                 related_resources={
                     "citations": [
@@ -67,6 +69,28 @@ def fetch_all(
                     has_parts=[],
                     keywords=[],
                     measured_values=[],
+                    date_published=datetime.min,
                 )
             ),
-        ][:limit]
+        ]
+        for dataset in datasets:
+            if dataset.resource.platform_identifier == _id:
+                return dataset
+        raise ValueError("No resource associated with the id")
+
+    def fetch(
+        self, from_incl: datetime | None = None, to_excl: datetime | None = None
+    ) -> typing.Iterator[ResourceWithRelations[Dataset]]:
+        id_list = ["42769", "42742"]
+        for id_ in id_list:
+            dataset = self.retry(id_)
+            if from_incl is None:
+                from_incl = datetime.min
+            if to_excl is None:
+                to_excl = datetime.max
+            if (
+                dataset.resource.date_published is not None
+                and dataset.resource.date_published >= from_incl
+                and dataset.resource.date_published < to_excl
+            ):
+                yield dataset

src/connectors/example/resources/publications.json

@@ -1,7 +1,7 @@
 [
     {
         "platform": "example",
-        "platform_identifier": 1,
+        "platform_identifier": "1",
         "title": "The Art of Fiction",
         "doi": "10.1234/567890",
         "creators": "Jane Smith",

src/connectors/huggingface/huggingface_dataset_connector.py

@@ -7,7 +7,7 @@
 import dateutil.parser
 import requests
 
-from connectors import ResourceConnector
+from connectors.abstract.resource_connector import ResourceConnector
 from connectors.resource_with_relations import ResourceWithRelations
 from database.model.dataset.data_download import DataDownload
 from database.model.dataset.dataset import Dataset
@@ -17,6 +17,11 @@
 
 
 class HuggingFaceDatasetConnector(ResourceConnector[Dataset]):
+    """
+    This must be only runned on the startu due to there is no way to
+    retrieve data from huggingface filtering by time creation
+    """
+
     @property
     def resource_class(self) -> type[Dataset]:
         return Dataset
@@ -29,7 +34,9 @@ def platform_name(self) -> PlatformName:
     def _get(url: str, dataset_id: str) -> typing.List[typing.Dict[str, typing.Any]]:
         """
         Perform a GET request and raise an exception if the response code is not OK.
+        resultaod
         """
+
         response = requests.get(url, params={"dataset": dataset_id})
         response_json = response.json()
         if not response.ok: