feat: 🎸 add support for pdf2image

✅ Closes: #688
huggingface · Jan 20, 2023 · 47d3297 · 47d3297
1 parent d7d1dd7
commit 47d3297
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 3 deletions.
diff --git a/.github/workflows/_quality-python.yml b/.github/workflows/_quality-python.yml
@@ -36,7 +36,7 @@ jobs:
             ${{ inputs.working-directory }}/poetry.lock
       - name: Install packages for workers that use datasets
         if: ${{ inputs.is-datasets-worker }}
-        run: sudo apt update; sudo apt install -y libicu-dev ffmpeg libavcodec-extra libsndfile1 llvm pkg-config
+        run: sudo apt update; sudo apt install -y libicu-dev ffmpeg libavcodec-extra libsndfile1 llvm pkg-config poppler-utils
       - name: Install dependencies
         # "poetry env use" is required: https://github.com/actions/setup-python/issues/374#issuecomment-1088938718
         run: |

diff --git a/.github/workflows/_unit-tests-python.yml b/.github/workflows/_unit-tests-python.yml
@@ -37,7 +37,7 @@ jobs:
             ${{ inputs.working-directory }}/poetry.lock
       - name: Install packages for workers that use datasets
         if: ${{ inputs.is-datasets-worker }}
-        run: sudo apt update; sudo apt install -y libicu-dev ffmpeg libavcodec-extra libsndfile1 llvm pkg-config
+        run: sudo apt update; sudo apt install -y libicu-dev ffmpeg libavcodec-extra libsndfile1 llvm pkg-config poppler-utils
       - name: Install dependencies
         # "poetry env use" is required: https://github.com/actions/setup-python/issues/374#issuecomment-1088938718
         run: |

diff --git a/workers/datasets_based/Dockerfile b/workers/datasets_based/Dockerfile
@@ -17,6 +17,7 @@ ENV PYTHONFAULTHANDLER=1 \
 RUN apt-get update \
     && apt-get install -y build-essential unzip wget python3-dev make \
     libicu-dev ffmpeg libavcodec-extra libsndfile1 llvm pkg-config \
+    poppler-utils \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip install -U --no-cache-dir pip

diff --git a/workers/datasets_based/poetry.lock b/workers/datasets_based/poetry.lock
diff --git a/workers/datasets_based/pyproject.toml b/workers/datasets_based/pyproject.toml
@@ -23,8 +23,10 @@ lxml = "^4.9.1"
 nlp = "^0.4.0"
 nltk = "^3.6.5"
 openpyxl = "^3.0.9"
+pdf2image = "^1.16.2"
 py7zr = "^0.20.1"
 pydub = "^0.25.1"
+pypdf2 = "^3.0.1"
 python = "3.9.15"
 rarfile = "^4.0"
 scikit-learn = "^1.0"