Fix bugs and documentation test

experimaestro · Feb 27, 2024 · a85192b · a85192b
1 parent 9b35b94
commit a85192b
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 7 deletions.
diff --git a/docs/source/api/conversation.rst b/docs/source/api/conversation.rst
@@ -32,9 +32,7 @@ Contextual query reformulation
 .. autoclass:: ContextualizedRewrittenQuery
     :members:
 
-.. autoxpmconfig:: ContextualizedQueryRewriting
-
-.. autoxpmconfig:: CanardDataset
+.. autoxpmconfig:: datamaestro_text.data.conversation.canard.CanardDataset
     :members: iter
 
 .. autoclass:: OrConvQADatasetAnswer

diff --git a/docs/source/api/ir.rst b/docs/source/api/ir.rst
@@ -21,7 +21,9 @@ Topics
 
 .. autoxpmconfig:: datamaestro_text.data.ir.csv.Topics
 
-.. autoclass:: datamaestro_text.data.ir.Topic
+.. autoxpmconfig:: datamaestro_text.transforms.ir.TopicWrapper
+
+.. autoxpmconfig:: datamaestro_text.data.ir.trec.TrecTopics
 
 Documents
 ---------
@@ -30,7 +32,7 @@ Documents
 .. autoxpmconfig:: datamaestro_text.data.ir.Documents
     :members: iter_documents, iter_ids, documentcount
 .. autoxpmconfig:: datamaestro_text.data.ir.cord19.Documents
-.. autoxpmconfig:: datamaestro_text.data.ir.csv.Documents
+.. autoxpmconfig:: datamaestro_text.data.ir.trec.TipsterCollection
 
 Assessments
 -----------
@@ -85,3 +87,12 @@ Training triplets
 .. autoxpmconfig:: datamaestro_text.data.ir.TrainingTripletsLines
 
 .. autoxpmconfig:: datamaestro_text.data.ir.huggingface.HuggingFacePairwiseSampleDataset
+
+Transforms
+**********
+
+.. autoxpmconfig:: datamaestro_text.transforms.ir.StoreTrainingTripletTopicAdapter
+
+.. autoxpmconfig:: datamaestro_text.transforms.ir.StoreTrainingTripletDocumentAdapter
+
+.. autoxpmconfig:: datamaestro_text.transforms.ir.ShuffledTrainingTripletsLines
diff --git a/src/datamaestro_text/data/ir/__init__.py b/src/datamaestro_text/data/ir/__init__.py
@@ -16,6 +16,7 @@
     # Record items
     IDItem,
     TextItem,
+    InternalIDItem,
     TopicRecord,
     DocumentRecord,
     SimpleTextItem,

diff --git a/src/datamaestro_text/data/ir/base.py b/src/datamaestro_text/data/ir/base.py
@@ -106,9 +106,13 @@ class SimpleTextDocumentRecord(DocumentRecord):
 
 @recordtypes(IDItem, TextItem)
 class GenericDocumentRecord(DocumentRecord):
-    pass
+    @classmethod
+    def create(cls, id: str, text: str, *items: Item):
+        return GenericDocumentRecord(IDItem(id), SimpleTextItem(text), *items)
 
 
 @recordtypes(IDItem, TextItem)
 class GenericTopicRecord(TopicRecord):
-    pass
+    @classmethod
+    def create(cls, id: str, text: str, *items: Item):
+        return GenericDocumentRecord(IDItem(id), SimpleTextItem(text), *items)
diff --git a/src/datamaestro_text/test/test_documented.py b/src/datamaestro_text/test/test_documented.py
@@ -0,0 +1,14 @@
+from pathlib import Path
+from experimaestro.tools.documentation import DocumentationAnalyzer
+
+
+def test_documented():
+    """Test if every configuration is documented"""
+    doc_path = Path(__file__).parents[3] / "docs" / "source" / "index.rst"
+    analyzer = DocumentationAnalyzer(
+        doc_path, set(["datamaestro_text"]), set(["datamaestro_text.test"])
+    )
+
+    analyzer.analyze()
+    analyzer.report()
+    analyzer.assert_valid_documentation()