From Pipenv to Poetry

.github/CODEOWNERS

@@ -1,7 +1,6 @@
 # Package dependency changes should be approved by a member of 'presidio-administrators' team
-*/Pipfile* @microsoft/presidio-administrators
+**/pyproject.toml @microsoft/presidio-administrators
 
 # Version change should be approved by a member of 'presidio-administrators' team
-VERSION @microsoft/presidio-administrators
-VERSION-IMAGE-REDACTOR @microsoft/presidio-administrators
+**/VERSION @microsoft/presidio-administrators
 

.gitignore

@@ -90,7 +90,10 @@ ENV/
 env.bak/
 venv.bak/
 *venv/
-*Pipfile.lock
+Pipfile.lock
+Pipfile
+pdm.lock
+.pdm-python
 
 # Spyder project settings
 .spyderproject

.pipelines/templates/build-analyzer.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps'
@@ -16,9 +15,9 @@ steps:
       workingDirectory: 'presidio-analyzer'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run python -m spacy download en_core_web_lg
-        pipenv run python -m spacy download en_core_web_sm
+        poetry install --all-extras
+        poetry run python -m spacy download en_core_web_lg
+        poetry run python -m spacy download en_core_web_sm
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/build-anonymizer.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps: Anonymizer'
@@ -16,7 +15,7 @@ steps:
       workingDirectory: 'presidio-anonymizer'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
+        poetry install
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/build-cli.yml

@@ -1,22 +1,22 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
+
   - task: Bash@3
     displayName: 'Install deps'
     inputs:
       targetType: 'inline'
       workingDirectory: 'presidio-cli'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run python -m spacy download en_core_web_lg
+        poetry install
+        poetry run python -m spacy download en_core_web_lg
   - template: ./build-python.yml
     parameters:
       SERVICE: 'Cli'

.pipelines/templates/build-image-redactor.yml

@@ -10,24 +10,24 @@ steps:
               sudo apt show tesseract-ocr
               sudo tesseract -v
     - task: Bash@3
-      displayName: 'Setup pipenv'
+      displayName: 'Setup poetry'
       inputs:
           targetType: 'inline'
           script: |
               set -eux  # fail on error
               python -m pip install --upgrade pip
-              python -m pip install pipenv
-              pipenv --python 3
+              python -m pip install poetry
+
     - task: Bash@3
       displayName: 'Install deps'
       inputs:
           targetType: 'inline'
           workingDirectory: 'presidio-image-redactor'
           script: |
               set -eux  # fail on error
-              pipenv install --deploy --dev
-              pipenv run python -m spacy download en_core_web_lg
-              pipenv run pip install -e ../presidio-analyzer/.
+              poetry install
+              poetry run python -m spacy download en_core_web_lg
+              poetry run pip install -e ../presidio-analyzer/.
     - template: ./build-python.yml
       parameters:
           SERVICE: 'Image-Redactor'

.pipelines/templates/build-python.yml

@@ -12,8 +12,8 @@ steps:
           script: |
               set -eux  # fail on error
               # Install pytest and run tests
-              pipenv run pip install pytest pytest-azurepipelines
-              pipenv run pytest -vv
+              poetry run pip install pytest pytest-azurepipelines
+              poetry run pytest -vv
 
     - task: Bash@3
       displayName: 'Package Wheel: ${{ parameters.SERVICE }}'

.pipelines/templates/build-structured.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps'
@@ -16,9 +15,9 @@ steps:
       workingDirectory: 'presidio-structured'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run pip install -e ../presidio-analyzer/. # Use the existing analyzer and not the one in PyPI
-        pipenv run pip install -e ../presidio-anonymizer/. # Use the existing analyzer and not the one in PyPI
+        poetry install
+        poetry run pip install -e ../presidio-analyzer/. # Use the existing analyzer and not the one in PyPI
+        poetry run pip install -e ../presidio-anonymizer/. # Use the existing analyzer and not the one in PyPI
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/e2e-tests.yml

@@ -46,3 +46,10 @@ steps:
           ANONYMIZER_BASE_URL: ${{ parameters.anonymizer_base_url }}
         TEST_SUITE:  ${{ parameters.test_suite }}
       displayName: Run tests
+    - task: DockerCompose@0
+      displayName: Docker Logs
+      inputs:
+        dockerComposeCommand: logs
+        dockerComposeFile: docker-compose.yml
+        buildImages: false
+      condition: always()

docs/development.md

@@ -16,66 +16,53 @@ The project is structured so that:
 - In the project root directory, you will find common code for using, serving and testing Presidio
     as a cluster of services, as well as CI/CD pipelines codebase and documentation.
 
-### Setting up Pipenv
+### Setting up Poetry
 
-[Pipenv](https://pipenv.pypa.io/en/latest/) is a Python workflow manager, handling
-dependencies and environment for Python packages. It is used by each Presidio service
+[PDM](https://daobook.github.io/pdm/) is Python package manager with PEP 582 support. It installs and manages packages
+in a similar way to npm that doesn't need to create a virtualenv at all. It is used by each Presidio service
 as the dependencies manager, to be aligned with the specific requirements versions.
-Follow these steps when starting to work on a Presidio service with Pipenv:
+Follow these steps when starting to work on a Presidio service with pdm:
 
-1. Install Pipenv
+1. Install PDM
 
     - Using Pip
 
         ```sh
-        pip install --user pipenv
+        pip install --user pdm
         ```
 
     - Using Homebrew (in MacOS)
 
         ```
-        brew install pipenv
+        brew install pdm
         ```
 
-    Additional installation instructions for Pipenv: <https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv>
+    Additional installation instructions for PDM: <https://daobook.github.io/pdm/#installation>
 
-2. Have Pipenv create a virtualenv for the project and install all requirements in the Pipfile,
+2. Have PDM create a virtualenv for the project and install all requirements in the pyproject.toml,
     including dev requirements.
 
     For example, in the `presidio-analyzer` folder, run:
 
     ```
-    pipenv install --dev --skip-lock
+    pdm install -G:all
     ```
 
 3. Run all tests:
 
     ```
-    pipenv run pytest
+    pdm run pytest
     ```
 
 4. To run arbitrary scripts within the virtual env, start the command with
-    `pipenv run`. For example:
-    1. `pipenv run ruff check`
-    2. `pipenv run pip freeze`
-    3. `pipenv run python -m spacy download en_core_web_lg`
+    `pdm run`. For example:
+    1. `pdm run ruff check`
+    2. `pdm run pip freeze`
+    3. `pdm run python -m spacy download en_core_web_lg`
 
     Command 3 downloads the default spacy model needed for Presidio Analyzer.`
 
-#### Alternatively, activate the virtual environment and use the commands by starting a pipenv shell
-
-1. Start shell:
-
-    ```
-    pipenv shell
-    ```
-
-2. Run commands in the shell
-
-    ```
-    pytest
-    pip freeze
-    ```
+#### Alternatively, activate the virtual environment and use the commands using [this method](https://pdm-project.org/latest/usage/venv/#activate-a-virtualenv).
 
 ### Development guidelines
 
@@ -107,9 +94,9 @@ use docker-compose ps:
 ```bash
 >docker-compose ps
 CONTAINER ID   IMAGE                       COMMAND                  CREATED         STATUS         PORTS                    NAMES
-6d5a258d19c2   presidio-anonymizer         "/bin/sh -c 'pipenv …"   6 minutes ago   Up 6 minutes   0.0.0.0:5001->5001/tcp   presidio_presidio-anonymizer_1
-9aad2b68f93c   presidio-analyzer           "/bin/sh -c 'pipenv …"   2 days ago      Up 6 minutes   0.0.0.0:5002->5001/tcp   presidio_presidio-analyzer_1
-1448dfb3ec2b   presidio-image-redactor     "/bin/sh -c 'pipenv …"   2 seconds ago   Up 2 seconds   0.0.0.0:5003->5001/tcp   presidio_presidio-image-redactor_1
+6d5a258d19c2   presidio-anonymizer         "/bin/sh -c 'pdm …"   6 minutes ago   Up 6 minutes   0.0.0.0:5001->5001/tcp   presidio_presidio-anonymizer_1
+9aad2b68f93c   presidio-analyzer           "/bin/sh -c 'pdm …"   2 days ago      Up 6 minutes   0.0.0.0:5002->5001/tcp   presidio_presidio-analyzer_1
+1448dfb3ec2b   presidio-image-redactor     "/bin/sh -c 'pdm …"   2 seconds ago   Up 2 seconds   0.0.0.0:5003->5001/tcp   presidio_presidio-image-redactor_1
 ```
 
 Edit docker-compose.yml configuration file to change the default ports.
@@ -152,7 +139,7 @@ Running the tests locally can be done in two ways:
 1. Using cli, from each service directory, run:
 
     ```sh
-    pipenv run pytest
+    pdm run pytest
     ```
 
 2. Using your IDE.
@@ -235,7 +222,7 @@ run.bat
 
 Presidio services are PEP8 compliant and continuously enforced on style guide issues during the build process using `ruff`, in turn running `flake8` and other linters.
 
-Running ruff locally, using `pipenv run ruff check`, you can check for those issues prior to committing a change.
+Running ruff locally, using `pdm run ruff check`, you can check for those issues prior to committing a change.
 
 Ruff runs linters in addition to the basic `flake8` functionality, Presidio uses linters as part as ruff such as:
 

presidio-analyzer/Dockerfile

@@ -4,7 +4,6 @@ ARG NAME
 ARG NLP_CONF_FILE=presidio_analyzer/conf/default.yaml
 ARG ANALYZER_CONF_FILE=presidio_analyzer/conf/default_analyzer.yaml
 ARG RECOGNIZER_REGISTRY_CONF_FILE=presidio_analyzer/conf/default_recognizers.yaml
-ENV PIPENV_VENV_IN_PROJECT=1
 ENV PIP_NO_CACHE_DIR=1
 
 ENV ANALYZER_CONF_FILE=${ANALYZER_CONF_FILE}
@@ -17,19 +16,19 @@ COPY ${NLP_CONF_FILE} /usr/bin/${NAME}/${NLP_CONF_FILE}
 
 WORKDIR /usr/bin/${NAME}
 
-COPY ./Pipfile* /usr/bin/${NAME}/
-
 # Install essential build tools
 RUN apt-get update \
   && apt-get install -y build-essential
 
-RUN pip install pipenv \
-  && pipenv install --deploy
+COPY ./pyproject.toml /usr/bin/${NAME}/
+COPY ./README.md /usr/bin/${NAME}/
+
+RUN pip install poetry && poetry install --no-root --only=main -E server
 # install nlp models specified in NLP_CONF_FILE
 COPY ./install_nlp_models.py /usr/bin/${NAME}/
 
-RUN pipenv run python install_nlp_models.py --conf_file ${NLP_CONF_FILE}
+RUN poetry run python install_nlp_models.py --conf_file ${NLP_CONF_FILE}
 
 COPY . /usr/bin/${NAME}/
 EXPOSE ${PORT}
-CMD pipenv run python app.py --host 0.0.0.0
+CMD poetry run python app.py --host 0.0.0.0

presidio-analyzer/Dockerfile.transformers

@@ -4,7 +4,6 @@ ARG NAME
 ARG NLP_CONF_FILE=presidio_analyzer/conf/transformers.yaml
 ARG ANALYZER_CONF_FILE=presidio_analyzer/conf/default_analyzer.yaml
 ARG RECOGNIZER_REGISTRY_CONF_FILE=presidio_analyzer/conf/default_recognizers.yaml
-ENV PIPENV_VENV_IN_PROJECT=1
 ENV PIP_NO_CACHE_DIR=1
 WORKDIR /usr/bin/${NAME}
 
@@ -16,16 +15,17 @@ COPY ${ANALYZER_CONF_FILE} /usr/bin/${NAME}/${ANALYZER_CONF_FILE}
 COPY ${RECOGNIZER_REGISTRY_CONF_FILE} /usr/bin/${NAME}/${RECOGNIZER_REGISTRY_CONF_FILE}
 COPY ${NLP_CONF_FILE} /usr/bin/${NAME}/${NLP_CONF_FILE}
 
-COPY ./Pipfile* /usr/bin/${NAME}/
-RUN pip install pipenv \
-  && pipenv install --deploy
-RUN pipenv install torch transformers huggingface_hub --skip-lock
+COPY ./pyproject.toml /usr/bin/${NAME}/
+COPY ./README.md /usr/bin/${NAME}/
+RUN pip install poetry && poetry install -E server -E transformers
+
+RUN pdm add torch transformers huggingface_hub --no-lock
 
 # install nlp models specified in NLP_CONF_FILE
 COPY ./install_nlp_models.py /usr/bin/${NAME}/
 
-RUN pipenv run python install_nlp_models.py --conf_file ${NLP_CONF_FILE}
+RUN pdm run python install_nlp_models.py --conf_file ${NLP_CONF_FILE}
 
 COPY . /usr/bin/${NAME}/
 EXPOSE ${PORT}
-CMD pipenv run python app.py --host 0.0.0.0
+CMD pdm run python app.py --host 0.0.0.0