Poetry setup

.github/workflows/python-package-poetry.yml

@@ -0,0 +1,60 @@
+name: Python Package (Poetry)
+
+on:
+  push:
+    branches:
+      - main
+      - prod-deploy
+  pull_request:
+    paths:
+      - "backend/**"
+      - pyproject.toml
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9.16'
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      - name: Run tests
+        run: |
+          source .venv/bin/activate
+          pytest tests

.gitignore

@@ -165,6 +165,9 @@ backend/detection_results.csv
 backend/firebase_helpers/bearer.py
 backend/firebase_helpers/firebase_creds.json
 
+#Poetry
+poetry.lock
+
 # --------------------------------------------------------------------------
 #	Frontend
 # --------------------------------------------------------------------------

Dockerfile

@@ -22,14 +22,14 @@ RUN aws configure set region $AWS_REGION
 RUN aws configure set aws_access_key_id $AWS_DEPLOY_ACCESS_KEY_ID
 RUN aws configure set aws_secret_access_key $AWS_DEPLOY_SECRET_ACCESS_KEY
 
-COPY requirements.txt .
-
-RUN python -m pip install -r requirements.txt
+# Install Poetry and project dependencies
+RUN curl -sSL https://install.python-poetry.org | python -
+COPY pyproject.toml ./
+RUN poetry install --no-interaction --no-ansi --no-root
 
 COPY . .
 
-RUN yarn run secrets:deploy
-
-RUN yarn run build:prod
+RUN poetry run yarn run secrets:deploy
+RUN poetry run yarn run build:prod
 
-CMD python -m backend.driver
+CMD poetry run python -m backend.driver

README.md

@@ -11,24 +11,24 @@ Web Application where people new to Machine Learning can input a dataset and exp
 Have the following installed first:
 
 1. [NodeJS v18](https://nodejs.org/en/download/) (should come with NPM v9, you must install Yarn v1.22 afterwards using NPM)
-1. [Anaconda](https://www.anaconda.com/)
+1. [Poetry](https://python-poetry.org/)
 1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). After installing, type `aws configure` in your terminal and type in the credentials given in [Secrets](https://docs.google.com/spreadsheets/d/1fRndo-7u0MXghiZoMp3uBepDBW9EghcJ9IL4yS0TdD8/edit?usp=sharing)
 
 ## Recommended
 
 1. [GitKraken](https://help.gitkraken.com/gitkraken-client/how-to-install/) for helping with Git commands, especially if you're new to Git
-1. [Node Version Manager](https://www.freecodecamp.org/news/node-version-manager-nvm-install-guide/) for managing NodeJS versions
+1. [pyenv](https://github.com/pyenv/pyenv) for managing python versions
 
 ## To start on localhost (in order):
 
-| Action                                        | Command                                                                                    |
-| --------------------------------------------- | ------------------------------------------------------------------------------------------ |
-| Install (one-time) / Update Frontend Packages | `yarn run installf`                                                                        |
-| Install Backend Packages (one-time)           | `yarn run installb`                                                                        |
-| Update Backend Packages                       | `conda activate dlplayground && cd conda && conda env update -f environment.yml`           |
-| Get secrets                                   | `conda activate dlplayground && python -m backend.aws_helpers.aws_secrets_utils.build_env` |
-| Running the Frontend                          | `yarn run startf`                                                                          |
-| Running the Backend                           | `conda activate dlplayground && python -m backend.driver`                                  |
+| Action                                        | Command                                                                |
+| --------------------------------------------- | ---------------------------------------------------------------------- |
+| Install (one-time) / Update Frontend Packages | `yarn run installf`                                                    |
+| Install Backend Packages (one-time)           | `yarn run installb`                                                    |
+| Update Backend Packages                       | `poetry install`                                                       |
+| Get secrets                                   | `poetry run python -m backend.aws_helpers.aws_secrets_utils.build_env` |
+| Running the Frontend                          | `yarn run startf`                                                      |
+| Running the Backend                           | `poetry run python -m backend.driver`                                  |
 
 ## To run in `production` mode:
 
@@ -58,15 +58,15 @@ If this command works properly, you will be redirected to an auth route in the G
 
 # Further Details: Backend
 
-## Conda Env Setup
+## Poetry Env Setup
 
-- `conda env create -f environment.yml` in the `/conda` directory
+- `poetry install` in the project root directory
 
-- Updating an environment: `conda env update -f environment.yml` in the `/conda` directory
+- Updating dependencies: `poetry update`
 
 ## Backend Infrastructure
 
-`python -m backend.driver` from the `~/Deep-Learning-Playground` directory
+`poetry run python -m backend.driver` from the `~/Deep-Learning-Playground` directory
 
 The backend supports training of a deep learning model and/or a classical ML model
 
@@ -114,14 +114,13 @@ To see how `driver.py` is used, see [`Backend_Examples.md`](./.github/Backend_Ex
 5. Navigate to project directory.
    Type `yarn run installf`.
    If your NodeJS is outdated, follow these [instructions](https://www.hostingadvice.com/how-to/update-node-js-latest-version/).
-   If you're running into Conda issues visit [this page](https://github.com/conda/conda/issues/11919).
 
 6. Enter these commands:
 
 ```
 yarn run installb
-conda activate dlplayground && cd conda && conda env update -f environment.yml
-conda activate dlplayground && python -m backend.aws_helpers.aws_secrets_utils.build_env
+poetry install
+poetry run python -m backend.aws_helpers.aws_secrets_utils.build_env
 ```
 
 ## Mac Installation

TrainingContainer.Dockerfile

@@ -2,11 +2,10 @@ FROM python:3.9-slim
 
 WORKDIR /
 
-COPY requirements.txt .
-RUN apt-get update -y && apt-get install -y gcc && apt-get install -y curl && apt-get install -y unzip
-RUN pip install -r requirements.txt
+COPY pyproject.toml  ./
+RUN pip install poetry && poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root
 COPY . .
-ARG TARGETARCH
+ARG TARGETARCH 
 
 RUN if [ "${TARGETARCH}" = "arm64" ] ; then \
         curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" ; \

package.json

@@ -6,11 +6,11 @@
   "scripts": {
     "build:prod": "cd frontend && yarn install && REACT_APP_MODE=prod yarn run build",
     "installf": "cd frontend && yarn install",
-    "installb": "conda --version && cd conda && if conda env list | grep -q 'dlplayground'; then conda env update -f environment.yml --prune; else conda env create -f environment.yml; fi && ggshield auth login && pre-commit install",
+    "installb": "poetry install && ggshield auth login && pre-commit install",
     "startf": "cd frontend && yarn dev",
-    "secrets": "eval \"$(conda shell.bash hook)\" && conda activate dlplayground && python -m backend.aws_helpers.aws_secrets_utils.build_env",
-    "startb": "eval \"$(conda shell.bash hook)\" && conda activate dlplayground && python -m backend.driver",
-    "secrets:deploy": "python -m backend.aws_helpers.aws_secrets_utils.build_env",
+    "secrets": "poetry run python -m backend.aws_helpers.aws_secrets_utils.build_env",
+    "startb": "poetry run python -m backend.driver",
+    "secrets:deploy": "poetry run python -m backend.aws_helpers.aws_secrets_utils.build_env",
     "docker:build": "docker build -t dlp ."
   },
   "repository": {

pyproject.toml

@@ -0,0 +1,49 @@
+[tool.poetry]
+name = "deep-learning-playground"
+version = "0.1.0"
+description = "Web Application where people new to Deep Learning can input a dataset and toy around with basic Pytorch modules without writing any code"
+authors = ["DSGT-DLP <dlp@datasciencegt.org>"]
+license = "MIT"
+readme = "README.md"
+packages = [{include = "deep_learning_playground"}]
+
+[tool.poetry.dependencies]
+python = ">=3.9, <3.10"
+flask = "2.2.3"
+flask-cors = "*"
+flask-socketio = "*"
+numpy = "*"
+pandas = "*"
+matplotlib = "*"
+seaborn = "*"
+mypy = "*"
+pytest = "*"
+scikit-learn = "*"
+scipy = "1.7.3"
+boto3 = "*"
+pyyaml = "*"
+statsmodels = "*"
+torch = "*"
+transformers = "*"
+pre-commit = "*"
+torchvision = "*"
+timm = "*"
+wwf = "*"
+fastai = "==2.7.4"
+firebase_admin = "*"
+pyrebase4 = "*"
+moto = "*"
+docker = "*"
+ggshield = "*"
+black = "^23.3.0"
+
+[tool.poetry.dev-dependencies]
+python = ">=3.9"
+flake8 = "*"
+jupyter = "*"
+pytest = "*"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"