Testing and automated deployment

.github/actions/action-fly-pr-review/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine
+
+RUN apk add --no-cache curl jq
+
+RUN curl -L https://fly.io/install.sh | FLYCTL_INSTALL=/usr/local sh
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/action-fly-pr-review/entrypoint.sh

@@ -0,0 +1,71 @@
+#!/bin/sh -l
+
+set -ex
+
+if [ -n "$INPUT_PATH" ]; then
+  # Allow user to change directories in which to run Fly commands.
+  cd "$INPUT_PATH" || exit
+fi
+
+PR_NUMBER=$(jq -r .number /github/workflow/event.json)
+if [ -z "$PR_NUMBER" ]; then
+  echo "This action only supports pull_request actions."
+  exit 1
+fi
+
+GITHUB_REPOSITORY_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}
+EVENT_TYPE=$(jq -r .action /github/workflow/event.json)
+
+# Default the Fly app name to pr-{number}-{repo_owner}-{repo_name}
+app="${INPUT_NAME:-pr-$PR_NUMBER-$GITHUB_REPOSITORY_OWNER-$GITHUB_REPOSITORY_NAME}"
+# Change underscores to hyphens.
+app="${app//_/-}"
+region="${INPUT_REGION:-${FLY_REGION:-iad}}"
+org="${INPUT_ORG:-${FLY_ORG:-personal}}"
+image="$INPUT_IMAGE"
+config="${INPUT_CONFIG:-fly.toml}"
+
+if ! echo "$app" | grep "$PR_NUMBER"; then
+  echo "For safety, this action requires the app's name to contain the PR number."
+  exit 1
+fi
+
+# PR was closed - remove the Fly app if one exists and exit.
+if [ "$EVENT_TYPE" = "closed" ]; then
+  flyctl apps destroy "$app" -y || true
+  exit 0
+fi
+
+# Deploy the Fly app, creating it first if needed.
+if ! flyctl status --app "$app"; then
+  # Backup the original config file since 'flyctl launch' messes up the [build.args] section
+  cp "$config" "$config.bak"
+  flyctl launch --no-deploy --copy-config --name "$app" --image "$image" --regions "$region" --org "$org"
+  # Restore the original config file
+  cp "$config.bak" "$config"
+fi
+if [ -n "$INPUT_SECRETS" ]; then
+  echo $INPUT_SECRETS | tr " " "\n" | flyctl secrets import --app "$app"
+fi
+
+# Attach postgres cluster to the app if specified.
+if [ -n "$INPUT_POSTGRES" ]; then
+  flyctl postgres attach "$INPUT_POSTGRES" --app "$app" || true
+fi
+
+# Trigger the deploy of the new version.
+echo "Contents of config $config file: " && cat "$config"
+if [ -n "$INPUT_VM" ]; then
+  flyctl deploy --config "$config" --app "$app" --regions "$region" --image "$image" --strategy immediate --ha=$INPUT_HA --vm-size "$INPUT_VMSIZE"
+else
+  flyctl deploy --config "$config" --app "$app" --regions "$region" --image "$image" --strategy immediate --ha=$INPUT_HA --vm-cpu-kind "$INPUT_CPUKIND" --vm-cpus $INPUT_CPU --vm-memory "$INPUT_MEMORY"
+fi
+
+# Make some info available to the GitHub workflow.
+flyctl status --app "$app" --json >status.json
+hostname=$(jq -r .Hostname status.json)
+appid=$(jq -r .ID status.json)
+echo "hostname=$hostname" >> $GITHUB_OUTPUT
+echo "url=https://$hostname" >> $GITHUB_OUTPUT
+echo "id=$appid" >> $GITHUB_OUTPUT
+echo "name=$app" >> $GITHUB_OUTPUT

.github/workflows/fly-production.yml

@@ -0,0 +1,21 @@
+name: Fly Deploy
+on:
+  push:
+    branches:
+      - master    # change to main if needed
+jobs:
+  deploy:
+    name: Deploy app
+    runs-on: ubuntu-latest
+    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    steps:
+      - uses: actions/checkout@v4
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy ./backend --remote-only
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          POSTGRES_SERVER: ${{ secrets.PROD_PG_SERVER }}
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: ${{ secrets.PROD_PG_DB }}
+          POSTGRES_USER: ${{ secrets.PROD_PG_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.PROD_PG_PASSWORD }}

.github/workflows/fly-review.yml

@@ -0,0 +1,39 @@
+name: Deploy Review App
+on:
+  # Run this workflow on every PR event. Existing review apps will be updated when the PR is updated.
+  pull_request:
+    types: [opened, reopened, synchronize, closed]
+
+env:
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  # Set these to your Fly.io organization and preferred region.
+  FLY_REGION: iad
+  FLY_ORG: personal
+
+jobs:
+  review_app:
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.deploy.outputs.url }}
+    # Only run one deployment at a time per PR.
+    concurrency:
+      group: pr-${{ github.event.number }}
+
+    # Deploying apps with this "review" environment allows the URL for the app to be displayed in the PR UI.
+    # Feel free to change the name of this environment.
+    environment:
+      name: review
+      # The script in the `deploy` sets the URL output for each review app.
+      url: ${{ steps.deploy.outputs.url }}
+
+    steps:
+      - name: Get code
+        uses: actions/checkout@v4
+
+      - name: Deploy PR app to Fly.io
+        id: deploy
+        uses: kj3moraes/fly-pr-review-apps@0.3.2
+        with:
+          path: backend
+          config: fly.staging.toml
+          secrets: PROJECT_NAME=premstats ENVIRONMENT=staging POSTGRES_SERVER=${{ secrets.STAGING_PG_SERVER }} POSTGRES_PORT=5432 POSTGRES_DB=${{ secrets.STAGING_PG_DB }} POSTGRES_USER=${{ secrets.STAGING_PG_USER }} POSTGRES_PASSWORD=${{ secrets.STAGING_PG_PASSWORD }}

README.md

@@ -1,2 +1,76 @@
-# premstats
-A natural language querying system for Premier League stats
+# premstats ⚽️
+
+A natural language querying system for Premier League stats. This project is comprised of 2 parts:
+
+1. An API for all Premier League sports data
+2. A frontend that parses a natural language query and returns the relevant sports statistics
+
+Part 1 is within the `backend/` directory. This also serves Part 2. Part 2 is defined in the `frontend/` directory. 
+
+## Usage
+
+To use 2., you can go to this website for querying - 
+
+To use 1., you can refer to the documentation for the API here - 
+
+## Setup
+
+To run this system, you only need to have Docker installed. The technology stack is as follows:
+
+**Backend**
+
+- FastAPI for the Python backend API.
+  - SQLModel for the Python SQL database interactions (ORM).
+  - Pydantic, used by FastAPI, for the data validation and settings management.
+  - PostgreSQL as the SQL database.
+
+**Frontend**
+
+- Next.js for the frontend.
+  - Using TypeScript, TailwindCSS and Vite
+  - Chakra UI for the frontend components.
+
+## Running
+
+To run this system locally, you only need to make a .env file. The docker compose file handles the rest.
+Your .env file must look like this and must be placed in this directory (at the same level as the docker-compose.yml file)
+
+```.env
+# Domain
+# This would be set to the production domain with an env var on deployment
+DOMAIN=localhost
+
+# Environment: local, staging, production
+ENVIRONMENT=local
+
+PROJECT_NAME=<name-of-your-project>
+
+# Backend
+BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173"
+SECRET_KEY=<key>
+FIRST_SUPERUSER=keanejonathan3@gmail.com
+FIRST_SUPERUSER_PASSWORD=<password>
+
+# Postgres
+POSTGRES_SERVER=localhost
+POSTGRES_PORT=5432
+POSTGRES_DB=app
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+
+# Configure these with your own Docker registry images
+DOCKER_IMAGE_BACKEND=backend
+DOCKER_IMAGE_FRONTEND=frontend
+```
+
+The above .env file will ensure that your application runs locally. To run the system, do the following
+
+```bash
+docker compose up --build
+```
+
+This will do 3 things:
+
+1. Create 3 services (db, backend, frontend) - it makes their images and starts running the containers
+2. Creates a volume called app-db-data where all the database data is stored.
+3. Exposes ports `localhost:5432` for the db, `localhost:8000` for the backend and `localhost:5173` for the frontend

backend/Dockerfile

@@ -16,13 +16,16 @@ RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python
 COPY ./pyproject.toml ./poetry.lock* /app/
 
 # Allow installing dev dependencies to run tests
-ARG INSTALL_DEV=false
-RUN bash -c "if [ $INSTALL_DEV == 'true' ] ; then poetry install --no-root ; else poetry install --no-root --only main ; fi"
+RUN bash -c "poetry install --no-root"
 
 ENV PYTHONPATH=/app
 
+COPY ./scripts/ /app/scripts
+
 COPY ./alembic.ini /app/
 
 COPY ./prestart.sh /app/
 
+COPY ./tests-start.sh /app/
+
 COPY ./app /app/app

backend/README.md

@@ -0,0 +1,54 @@
+# Premstats Backend
+
+This file documents how to work with the backend and continue development on it.
+
+## Setup
+
+To setup your developer environment, you will need
+
+1. [Docker](https://docs.docker.com/desktop/)
+2. [Poetry](https://python-poetry.org/docs/)
+
+You can get them from the relevant links provided. To run the backend you only need docker 
+
+## Running
+
+The easiest way to run this is to run the `docker compose up --build` command in the previous directory. This boots up the database and the backend and exposes the ports necessary. Follow the instructions in the project README.md to get started on this approach.
+
+Another way of running the server is using `uvicorn`. To do so you need to have a `.env` file in the current directory.
+
+
+## Development
+
+To continue developing this backend, you can
+
+### Project Structure
+
+The project structure is explained below
+
+```
+.
+├── Dockerfile
+├── README.md           # this file
+├── alembic.ini         # alembic migrations configuration file
+├── app                 # all the FastAPI code is here
+│   ├── __init__.py
+│   ├── __pycache__
+│   ├── alembic         # migrations in this directory
+│   ├── api             # all the routes are defined here 
+│   ├── core            # essentials like config, db, etc.
+│   ├── main.py         # entrypoint
+│   ├── models.py       # the tables in the database
+│   ├── pre_start.py    # prestart script like checking if the database is online, etc.
+│   └── tests           # testing directory
+├── fly.toml            # the configuration file for fly.io to deploy
+├── poetry.lock         # lock file for Poetry
+├── prestart.sh         # the prestart script (running migrations, calling app/pre_start.py, etc.)
+└── pyproject.toml      # the project configuration file (Poetry dependencies, etc.) 
+```
+
+## Testing
+
+## Deployment
+
+I use fly.io for deploying this server to staging and production.

backend/app/api/routes/match.py

@@ -1,16 +1,24 @@
-from typing import List
+from typing import Annotated, List
 
-from fastapi import APIRouter, Depends, HTTPException, Response, status
-from sqlmodel import Session, select
 from app.core.db import get_session
 from app.models import Match
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import AfterValidator
+from sqlmodel import Session, select
 
 router = APIRouter()
 
 
 # Match CRUD operations
-@router.post("/add", response_model=Match, status_code=status.HTTP_201_CREATED)
-def create_match(match: Match, session: Session = Depends(get_session)):
+@router.post(
+    "/add",
+    response_model=Match,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_match(
+    match: Match,
+    session: Session = Depends(get_session),
+):
     session.add(match)
     session.commit()
     session.refresh(match)
@@ -34,13 +42,16 @@ def read_match(match_id: int, session: Session = Depends(get_session)):
 
 
 @router.put("/update/{match_id}", response_model=Match)
-def update_match(match_id: int, match: Match, session: Session = Depends(get_session)):
+def update_match(
+    match_id: int,
+    match: Annotated[Match, AfterValidator(Match.model_validate)],
+    session: Session = Depends(get_session),
+):
     db_match = session.get(Match, match_id)
     if not db_match:
         raise HTTPException(status_code=404, detail="Match not found")
     match_data = match.model_dump(exclude_unset=True)
-    for key, value in match_data.items():
-        setattr(db_match, key, value)
+    db_match.sqlmodel_update(match_data)
     session.add(db_match)
     session.commit()
     session.refresh(db_match)

backend/app/api/routes/referee.py

@@ -1,8 +1,9 @@
-from typing import List
+from typing import Annotated, List
 
 from app.core.db import get_session
 from app.models import Referee
 from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import AfterValidator
 from sqlalchemy.exc import IntegrityError
 from sqlmodel import Session, select
 
@@ -11,7 +12,11 @@
 
 # Referee CRUD operations
 @router.post("/add", response_model=Referee, status_code=status.HTTP_201_CREATED)
-def create_referee(referee: Referee, session: Session = Depends(get_session)):
+def create_referee(
+    referee: Annotated[Referee, AfterValidator(Referee.model_validate)],
+    session: Session = Depends(get_session),
+):
+    referee = Referee.model_validate(referee)
     try:
         session.add(referee)
         session.commit()
@@ -46,7 +51,9 @@ def read_referee(referee_id: int, session: Session = Depends(get_session)):
 
 @router.put("/update/{referee_id}", response_model=Referee)
 def update_referee(
-    referee_id: int, referee: Referee, session: Session = Depends(get_session)
+    referee_id: int,
+    referee: Annotated[Referee, AfterValidator(Referee.model_validate)],
+    session: Session = Depends(get_session),
 ):
     db_referee = session.get(Referee, referee_id)
     if not db_referee: