ENH: Allow DatasetMetadataExtractor.get_required_content() to return a generator

datalad_metalad/extract.py

@@ -458,11 +458,24 @@ def perform_dataset_metadata_extraction(ep: ExtractionArguments,
     }
 
     # Let the extractor get the files it requires
-    if extractor.get_required_content() is False:
-        yield {
-            "status": "impossible",
-            **result_template
-        }
+    # Handle both possibilities of bool return and Generator yield
+    res = extractor.get_required_content()
+
+    if isinstance(res, bool) or res is None:
+        if res is False:
+            yield {
+                "status": "impossible",
+                **result_template
+            }
+            return
+    else:
+        failure_count = 0
+        for r in res:
+            if r['status'] in ["error", "impossible"]:
+                failure_count+=1
+                yield r
+        if failure_count > 0:
+            return
 
     # Process results
     result = extractor.extract(None)

datalad_metalad/extractors/base.py

@@ -12,8 +12,9 @@
 import enum
 from typing import (
     Any,
-    IO,
     Dict,
+    Generator,
+    IO,
     List,
     Optional,
     Union,
@@ -158,17 +159,28 @@ def __init__(self,
         self.ref_commit = ref_commit
         self.parameter = parameter or {}
 
-    def get_required_content(self) -> bool:
-        """
-        Let the extractor get the content that it needs locally.
-        The default implementation is to do nothing.
+    def get_required_content(self) -> Union[bool, Generator]:
+        """Let the extractor get the content that it needs locally.
+
+        The default implementation is to do nothing and return True
+        Extractors that overwrite this function can return a boolean
+        (True/False) value OR yield DataLad result records.
 
         Returns
         -------
-        True if all required content could be fetched, False
-        otherwise. If False is returned, the extractor
-        infrastructure will signal an error and the extractor's
-        extract method will not be called.
+        bool
+          True if all required content could be fetched, False
+          otherwise. If False is returned, the extractor
+          infrastructure will signal an error and the extractor's
+          extract method will not be called.
+
+        Yields
+        ------
+        dict
+          DataLad result records. If a result record is yielded
+          with a failure 'status' (i.e. equal to 'impossible' or
+          'error') the extractor infrastructure will signal an error
+          and the extractor's extract method will not be called.
         """
         return True
 

docs/source/user_guide/writing-extractors.rst

@@ -86,18 +86,55 @@ This function is used in dataset-level extractors only.
 It will be called by MetaLad prior to metadata extraction.
 Its purpose is to allow the extractor to ensure that content that is required for metadata extraction is present
 (relevant, for example, if some of files to be inspected may be annexed).
-The function should return ``True`` if it has obtained the required content, or confirmed its presence.
-If it returns ``False``, metadata extraction will not proceed.
+
+The function should either return a boolean value (``True | False``) or yield a ``Generator`` with 
+`DataLad result records`_. In the case of a boolean value, the function should return ``True`` if
+it has obtained the required content, or confirmed its presence. If it returns ``False``,
+metadata extraction will not proceed. Alternatively, yielding result records provides extractors with
+the capability to signal more expressive messages or errors. If a result record is yielded with a failure
+status (i.e. with ``status`` equal to ``impossible`` or ``error``) metadata extraction will not proceed.
 
 This function can be a place to call ``dataset.get()``.
-It is advisable to disable result rendering (``result_renderer="disabled"``), because during metadata extraction, users will typically want to redirect standard output to a file or another command.
+It is advisable to disable result rendering (``result_renderer="disabled"``), because during metadata
+extraction, users will typically want to redirect standard output to a file or another command.
 
-Example::
+Example 1::
 
   def get_required_content(self) -> bool:
       result = self.dataset.get("CITATION.cff", result_renderer="disabled")
       return result[0]["status"] in ("ok", "notneeded")
 
+Example 2::
+
+  from typing import Generator
+  def get_required_content(self) -> Generator:
+      yield self.dataset.get("CITATION.cff", result_renderer="disabled")
+
+Example 3::
+
+  from typing import Generator
+  def get_required_content(self) -> Generator:
+      result = self.dataset.get("CITATION.cff", result_renderer="disabled")
+      failure_count = 0
+      result_dict = dict(
+          path=self.dataset.path,
+          type='dataset',
+      )
+      for r in res:
+          if r['status'] in ['error', 'impossible']:
+              failure_count+=1
+      if failure_count > 0:
+          result_dict.update({
+              'status': 'error'
+              'message': 'could not retrieve required content'
+          })
+      else:
+          result_dict.update({
+              'status': 'ok'
+              'message': 'required content retrieved'
+          })
+      yield result_dict
+
 ``is_content_required()``
 -------------------------
 
@@ -241,3 +278,5 @@ For example, a list of files with a given extension (including those in subfolde
 
   files = list(self.dataset.repo.call_git_items_(["ls-files", "*.xyz"]))
 
+
+.. _DataLad result records: https://docs.datalad.org/en/stable/design/result_records.html