Merge pull request #269 from datalad/translator-work

ENH+BUG: Improving translators and catalog generation.

datalad_catalog/translators/bids_dataset_translator.py

@@ -100,7 +100,6 @@ class BIDSTranslator:
     def __init__(self, metadata_record):
         self.metadata_record = metadata_record
         self.extracted_metadata = self.metadata_record["extracted_metadata"]
-        self.graph = self.extracted_metadata["@graph"]
 
     def get_name(self):
         return self.extracted_metadata.get("title", "")

datalad_catalog/translators/metalad_core_translator.py

@@ -100,7 +100,7 @@ class CoreTranslator:
     def __init__(self, metadata_record):
         self.metadata_record = metadata_record
         self.extracted_metadata = self.metadata_record["extracted_metadata"]
-        self.graph = self.extracted_metadata["@graph"]
+        self.graph = self.extracted_metadata.get("@graph", [])
 
     def get_name(self):
         """Return an empty string as name
@@ -110,7 +110,7 @@ def get_name(self):
         """
         return ""
 
-    def get_url(self):
+    def get_dataset_url(self):
         program = (
             '.[]? | select(.["@type"] == "Dataset") | '
             '[.distribution[]? | select(has("url")) | .url]'
@@ -149,15 +149,39 @@ def get_metadata_source(self):
         result = jq.first(program, self.metadata_record)
         return result if len(result) > 0 else None
 
+    def get_file_url(self):
+        program = ".distribution? | .url?"
+        return jq.first(program, self.extracted_metadata)
+
+    def get_file_path(self):
+        return self.metadata_record.get("path", None)
+
+    def get_contentbytesize(self):
+        return self.extracted_metadata.get("contentbytesize", None)
+
     def translate(self):
         translated_record = {
             "type": self.metadata_record["type"],
             "dataset_id": self.metadata_record["dataset_id"],
             "dataset_version": self.metadata_record["dataset_version"],
-            "name": self.get_name(),
-            "url": self.get_url(),
-            "authors": self.get_authors(),
-            "subdatasets": self.get_subdatasets(),
             "metadata_sources": self.get_metadata_source(),
         }
+        if translated_record["type"] == "dataset":
+            translated_record.update(
+                {
+                    "name": self.get_name(),
+                    "url": self.get_dataset_url(),
+                    "authors": self.get_authors(),
+                    "subdatasets": self.get_subdatasets(),
+                }
+            )
+        if translated_record["type"] == "file":
+            translated_record.update(
+                {
+                    "path": self.get_file_path(),
+                    "url": self.get_file_url(),
+                    "contentbytesize": self.get_contentbytesize(),
+                }
+            )
+
         return {k: v for k, v in translated_record.items() if v is not None}

datalad_catalog/webcatalog.py

@@ -156,9 +156,10 @@ def create(self, force=False):
         # Check logo path, if added to config
         if (
             self.catalog_config.get(cnst.LOGO_PATH) is not None
-            and not Path(self.catalog_config[cnst.LOGO_PATH]).exists()
+            and not self.get_logo_path().exists()
+            # and not Path(self.catalog_config[cnst.LOGO_PATH]).exists()
         ):
-            msg = f"Error in config: the specified logo does not exist at path: {self.catalog_config[cnst.LOGO_PATH]}"
+            msg = f"Error in config: the specified logo does not exist at path: {self.get_logo_path()}"
             raise FileNotFoundError(msg)
 
         # Get package-related paths/content
@@ -237,6 +238,21 @@ def get_config_source(
                 # where dataset id and version are available.
                 return None
 
+    def get_logo_path(self):
+        # Returns the absolute path to the logo
+        # If none provided via config -> None
+        if self.catalog_config.get(cnst.LOGO_PATH) is None:
+            return None
+        # If the provided path is absolute, return it
+        # else assume that the provided path us relative to the
+        # parent directory of the config file (within which the
+        # logo path was specified)
+        if Path(self.catalog_config[cnst.LOGO_PATH]).is_absolute():
+            return self.catalog_config[cnst.LOGO_PATH]
+        else:
+            cfg_dir = self.catalog_config_path.parent
+            return cfg_dir / self.catalog_config[cnst.LOGO_PATH]
+
     def get_config(self, config_level: str = "catalog"):
         """"""
         # Read metadata from file
@@ -256,7 +272,8 @@ def write_config(self, force=False):
             and self.catalog_config[cnst.LOGO_PATH]
             != "artwork/catalog_logo.svg"
         ):
-            existing_path = Path(self.catalog_config[cnst.LOGO_PATH])
+            # existing_path = Path(self.catalog_config[cnst.LOGO_PATH])
+            existing_path = self.get_logo_path()
             existing_name = existing_path.name
             new_path = Path(self.location) / "artwork" / existing_name
             copy_overwrite_path(