Merge pull request #96 from NIAEFEUP/refactor/postgres

refactor: use postgres

.gitignore

@@ -3,14 +3,14 @@ logs
 *.log
 
 # sigarra internal data cannot be committed
-mysql/sql/01_dump_mysql.sql 
+postgres/sql/01_dump_postgres.sql 
 
 # dotenv environment variables file
 .env
 
-# mysql data
-mysql/data/*
-mysql/sql/01_data.sql
+# postgres data
+postgres/data/*
+postgres/sql/01_data.sql
 **__pycache__
 
 # django 

Makefile

@@ -1,6 +1,6 @@
 .PHONY: all clean 
 
-MYSQL_DATA = ./mysql/sql
+POSTGRES_DATA = ./postgres/sql
 
 all: clean_database
 	@echo [EXECUTING] ./scripts/$(EXEC)
@@ -11,10 +11,10 @@ download: clean_fetcher clean_database
 	@-mkdir ./fetcher/data
 	@echo [DOWNLOADING] data from the source... 
 	@docker-compose run fetcher python ./update_data/download.py
-	@echo [REMOVING] data from mysql... 
-	@-rm $(MYSQL_DATA)/01_data.sql
+	@echo [REMOVING] data from postgres... 
+	@-rm $(POSTGRES_DATA)/01_data.sql
 	@echo [MOVING] data from fetcher to sql... 
-	@mv ./fetcher/data/* ./mysql/sql
+	@mv ./fetcher/data/* ./postgres/sql
 
 upload: 
 	@echo [UPLOADING] data...
@@ -27,5 +27,5 @@ clean_fetcher:
 	@-rm -r ./fetcher/data
 
 clean_database:
-	@echo [CLEANING] Removing folder mysql/data...
-	@-rm -r ./mysql/data/
+	@echo [CLEANING] Removing database data...
+	@-docker volume rm tts_postgres_data || true

README.md

@@ -20,9 +20,9 @@ The data is available at the NIAEFEUP drive (Only for NIAEFEUP members):
 
 https://drive.google.com/drive/folders/1hyiwPwwPWhbAPeJm03c0MAo1HTF6s_zK?usp=sharing
 
-- The ```00_schema_mysql.sql``` corresponds to the schema for the most recent data.
+- The ```00_schema_postgres.sql``` corresponds to the schema for the most recent data.
 
-- Copy the ```01_data.sql``` and ```00_schema_mysql.sql``` of year and semester you desire to the ```mysql/sql``` folder.
+- Copy the ```01_data.sql``` and ```00_schema_postgres.sql``` of year and semester you desire to the ```postgres/sql``` folder.
 
 ## Usage 
 
@@ -44,7 +44,7 @@ In case you have __already build the server before and want to repopulate the da
 sudo make clean
 ```
 
-We need to clean the database to repopulate it, since the way the mysql container works is that it only runs the `sql` files present in the `mysql/sql` folder if the database is clean. This is way we need to issue `sudo make clean` in order for the insert sql queries to be run.
+We need to clean the database to repopulate it, since the way the postgres container works is that it only runs the `sql` files present in the `postgres/sql` folder if the database is clean. This is way we need to issue `sudo make clean` in order for the insert sql queries to be run.
 
 #### Running the container
 
@@ -70,8 +70,13 @@ docker compose up
 
 #### Accessing the development database
 
-We are currently using `phpmyadmin` and you can access it
+We are currently using `pgadmin` and you can access it
 
 1. Go to `localhost:4000`
 
-2. On the login screen, both the username and password are `root`. This is fine, since this is only a development environment
+2. On the login screen, both the credentials are as follows:
+
+    - Email: admin@example.com
+    - Password: admin
+
+    This is fine, since this is only a development environment.

django/.env.dev

@@ -1,11 +1,11 @@
 DEBUG=0
 SECRET_KEY=foo
 
-MYSQL_DATABASE=tts
-MYSQL_PASSWORD=root
-MYSQL_USER=root
-MYSQL_HOST=db
-MYSQL_PORT=3306
+POSTGRES_DB=tts
+POSTGRES_USER=root
+POSTGRES_PASSWORD=root
+POSTGRES_HOST=db
+POSTGRES_PORT=5432
 
 TTS_REDIS_HOST=tts_redis
 TTS_REDIS_PORT=6379

django/Dockerfile

@@ -3,22 +3,26 @@ FROM python:3.8-slim-buster
 WORKDIR /usr/src/django/ 
 
 # Get's the output from the django in realtime. 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 # Copy requirements 
 COPY ./requirements.txt ./requirements.txt 
 
-# Dependencies for mysqlclient 
+# Dependencies for building the requirements
 RUN apt-get update
-RUN apt-get -y install build-essential default-libmysqlclient-dev
+RUN apt-get -y install build-essential
 
-# Install mysql command to wait for the database initialization 
-RUN apt -y install default-mysql-client
+# Install postgres dependencies (pgsql client and development files)
+COPY ./etc/pgdg.sh /tmp/pgdg.sh
+RUN /tmp/pgdg.sh
+
+RUN apt -y install libpq-dev postgresql-client-16
+RUN apt -y clean && rm -rf /var/lib/apt/lists/*
 
 # Install the requirements
 RUN pip install -r requirements.txt
 
 EXPOSE 8000
 
 COPY ./entrypoint.sh ./entrypoint.sh  
-ENTRYPOINT ["sh", "/usr/src/django/entrypoint.sh"]
+ENTRYPOINT ["/usr/src/django/entrypoint.sh"]

django/entrypoint.sh

@@ -1,24 +1,22 @@
-#!bin/sh 
+#!/bin/sh 
 
 # WARNING: The script will not work if formated with CRLF. 
 
 # Configure the shell behaviour. 
 set -e
-if [[ ${DEBUG} == 1 ]]
+if [[ "${DEBUG}" == 1 ]]
 then set -x
 fi 
 
 # Get parameters.
-database_host="$1"    # The database host and should be provided the container name. 
-shift
 cmd="$@"
 
-# Waits for mysql initialization. 
-until mysql -h "$database_host" -u ${MYSQL_USER} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} -e 'select 1'; do
-  >&2 echo "MySQL is unavailable - sleeping"
+# Waits for PostgreSQL initialization. 
+until PGPASSWORD="${POSTGRES_PASSWORD}" psql -h "${POSTGRES_HOST}" -U "${POSTGRES_USER}" "${POSTGRES_DB}" -c 'select 1'; do
+  >&2 echo "PostgreSQL is unavailable - sleeping"
   sleep 4
 done
->&2 echo "Mysql is up - executing command" 
+>&2 echo "PostgreSQL is up - executing command" 
 
 # Migrate the Django.
 python manage.py inspectdb > university/models.py

django/etc/pgdg.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Source: https://www.postgresql.org/download/linux/ubuntu/
+
+# Import the repository signing key:
+apt install -y curl ca-certificates postgresql-common lsb-release
+
+install -d /usr/share/postgresql-common/pgdg
+curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc
+
+# Create the repository configuration file:
+sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+# Update the package lists:
+apt update

django/requirements.txt

@@ -5,7 +5,7 @@ django-cors-headers==3.10.1
 djangorestframework==3.11.0
 pytz==2021.3
 sqlparse==0.4.2
-mysqlclient==1.4.6
+psycopg2==2.9.9
 celery==5.2.7
 redis==3.5.3
 python-dotenv==1.0.1

django/tts_be/settings.py

@@ -82,12 +82,12 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': os.getenv('MYSQL_DATABASE'),
-        'USER': os.getenv('MYSQL_USER'),
-        'PASSWORD': os.getenv('MYSQL_PASSWORD'),
-        'HOST': os.getenv('MYSQL_HOST'),
-        'PORT': os.getenv('MYSQL_PORT') ,
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': os.getenv('POSTGRES_DB'),
+        'USER': os.getenv('POSTGRES_USER'),
+        'PASSWORD': os.getenv('POSTGRES_PASSWORD'),
+        'HOST': os.getenv('POSTGRES_HOST'),
+        'PORT': os.getenv('POSTGRES_PORT') ,
     }
 }
 

docker-compose.yaml

@@ -2,24 +2,28 @@ version: "3.7"
 
 services:
   db:
-    #platform: linux/arm64
-    build: ./mysql
+    build: ./postgres
     container_name: tts_db
     volumes:
-      - ./mysql/data:/var/lib/mysql
+      - postgres_data:/var/lib/postgresql/data/
     env_file:
-      - mysql/.env.dev
+      - postgres/.env.dev
     ports:
-      - 3306:3306
+      - 5432:5432
     ulimits:
       nofile:
         soft: 262144
         hard: 262144
 
-  phpmyadmin:
-    image: phpmyadmin:5.1.1-apache
-    container_name: tts_phpmyadmin
+  pgadmin:
+    build:
+      context: ./pgadmin
+    container_name: tts_pgadmin
     restart: always
+    env_file:
+      - pgadmin/.env.dev
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
     ports:
       - 4000:80
 
@@ -28,7 +32,7 @@ services:
     container_name: tts_django
     env_file:
       - django/.env.dev
-    command: ["db", "python", "manage.py", "runserver", "0.0.0.0:8000"]
+    command: ["python", "manage.py", "runserver", "0.0.0.0:8000"]
     volumes:
       - ./django/:/usr/src/django/
     ports:
@@ -46,3 +50,8 @@ services:
     restart: always
     ports:
       - "6379:6379"
+
+volumes:
+  postgres_data:
+    name: tts_postgres_data
+  pgadmin_data: