feat: Add page metadata on PDFMinerLoader

libs/langchain/langchain/document_loaders/parsers/pdf.py

@@ -128,18 +128,29 @@ def _extract_images_from_page(self, page: pypdf._page.PageObject) -> str:
 class PDFMinerParser(BaseBlobParser):
     """Parse `PDF` using `PDFMiner`."""
 
-    def __init__(self, extract_images: bool = False):
+    def __init__(self, extract_images: bool = False, load_per_pages: bool = False):
         self.extract_images = extract_images
+        self.load_per_pages = load_per_pages
 
     def lazy_parse(self, blob: Blob) -> Iterator[Document]:
         """Lazily parse the blob."""
+
         if not self.extract_images:
             from pdfminer.high_level import extract_text
 
             with blob.as_bytes_io() as pdf_file_obj:
-                text = extract_text(pdf_file_obj)
-                metadata = {"source": blob.source}
-                yield Document(page_content=text, metadata=metadata)
+                if self.load_per_pages:
+                    from pdfminer.pdfpage import PDFPage
+
+                    pages = PDFPage.get_pages(pdf_file_obj)
+                    for i, _ in enumerate(pages):
+                        text = extract_text(pdf_file_obj, page_numbers=[i])
+                        metadata = {"source": blob.source, "page": str(i)}
+                        yield Document(page_content=text, metadata=metadata)
+                else:
+                    text = extract_text(pdf_file_obj)
+                    metadata = {"source": blob.source}
+                    yield Document(page_content=text, metadata=metadata)
         else:
             import io
 

libs/langchain/langchain/document_loaders/pdf.py

@@ -251,6 +251,7 @@ def __init__(
         *,
         headers: Optional[Dict] = None,
         extract_images: bool = False,
+        load_per_pages: bool = False,
     ) -> None:
         """Initialize with file path."""
         try:
@@ -262,7 +263,9 @@ def __init__(
             )
 
         super().__init__(file_path, headers=headers)
-        self.parser = PDFMinerParser(extract_images=extract_images)
+        self.parser = PDFMinerParser(
+            extract_images=extract_images, load_per_pages=load_per_pages
+        )
 
     def load(self) -> List[Document]:
         """Eagerly load the content."""

libs/langchain/tests/integration_tests/document_loaders/test_pdf.py

@@ -56,6 +56,19 @@ def test_pdfminer_loader() -> None:
     docs = loader.load()
     assert len(docs) == 1
 
+    """Test PDFMiner loader. (load per pages)"""
+    file_path = Path(__file__).parent.parent / "examples/hello.pdf"
+    loader = PDFMinerLoader(str(file_path), load_per_pages=True)
+    docs = loader.load()
+
+    assert len(docs) == 1
+
+    file_path = Path(__file__).parent.parent / "examples/layout-parser-paper.pdf"
+    loader = PDFMinerLoader(str(file_path), load_per_pages=True)
+
+    docs = loader.load()
+    assert len(docs) == 16
+
 
 def test_pdfminer_pdf_as_html_loader() -> None:
     """Test PDFMinerPDFasHTMLLoader."""