DRAFT: Satellite dashboard

.github/workflows/tests.yaml

@@ -23,7 +23,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", ] # five is broken on 3.11 - celery update needed "3.11.0-alpha - 3.11"]
+        python-version: [ "3.9", "3.10", "3.11"] # five is broken on 3.11 - celery update needed "3.11.0-alpha - 3.11"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2

.pre-commit-config.yaml

@@ -1,32 +1,36 @@
 repos:
-- repo: https://github.com/asottile/reorder_python_imports
-  rev: v3.9.0
-  hooks:
-  - id: reorder-python-imports
-- repo: https://github.com/ambv/black
-  rev: 22.10.0
-  hooks:
-  - id: black
-    args: [--safe, --quiet, --line-length, "100"]
+
+## PYTHON
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.3.0
+  rev: v4.6.0
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
   - id: check-yaml
     args: [ --allow-multiple-documents ]
   - id: debug-statements
-- repo: https://github.com/pycqa/flake8
-  rev: 6.1.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.4.6
   hooks:
-  - id: flake8
-    args: [--max-line-length, "100", --ignore, "E203, W503"]
-- repo: https://github.com/asottile/pyupgrade
-  rev: v3.2.2
+    - id: ruff
+      args:
+        - '--fix'
+    - id: ruff-format
+
+## TODO: enable
+# - repo: https://github.com/shellcheck-py/shellcheck-py
+#   rev: v0.10.0.1
+#   hooks:
+#     - id: shellcheck
+
+- repo: https://github.com/AleksaC/hadolint-py
+  rev: v2.12.1b3
   hooks:
-  - id: pyupgrade
+    - id: hadolint
+
+## ES
 - repo: https://github.com/pre-commit/mirrors-eslint
-  rev: v8.27.0
+  rev: v9.3.0
   hooks:
   - id: eslint
     additional_dependencies:

.ruff.toml

@@ -0,0 +1,7 @@
+line-length = 100
+
+[lint]
+select = ["F", "I", "UP"]
+
+[format]
+quote-style = "double"

backend/Containerfile

@@ -1,7 +1,9 @@
-FROM  registry.access.redhat.com/ubi8/python-38 as base
+FROM registry.access.redhat.com/ubi8/python-39:1-184.1716989688 as base
+
 USER 0
-RUN mkdir /ibutsu_venv
-RUN chown -R 1001:0 /ibutsu_venv /srv
+RUN mkdir -p /ibutsu_venv && \
+    chown -R 1001:0 /ibutsu_venv /srv
+
 USER 1001
 WORKDIR  /srv
 RUN python -m venv /ibutsu_venv && \

backend/docker/Dockerfile.backend

@@ -1,13 +1,16 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal
+# hadolint global ignore=DL3013,DL3041
+FROM registry.access.redhat.com/ubi8/python-311:1
 
 WORKDIR /app
 
 USER 0
 
-RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel
-COPY . /app
-RUN pip3 install --no-cache-dir gunicorn && pip3 install --no-cache-dir -r requirements.txt
-RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
+COPY . .
+RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \
+    dnf clean all && \
+    pip3 install --no-cache-dir gunicorn && \
+    pip3 install --no-cache-dir -r requirements.txt && \
+    chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
 
 USER 1001
 

backend/docker/Dockerfile.flower

@@ -1,19 +1,23 @@
-FROM registry.access.redhat.com/ubi8/python-38
+# hadolint global ignore=DL3013,DL3041
+FROM registry.access.redhat.com/ubi8/python-311:1
 
 # add application sources with correct perms for OCP
 USER 0
-RUN dnf install -y libpq-devel python38-devel gcc
-ADD . /app
+RUN dnf install -y libpq-devel gcc && \
+    dnf clean all
+
+COPY . /app
 RUN /usr/bin/fix-permissions /app
+
 USER 1001
 
 ENV BROKER_URL=redis://localhost
 
 # Install dependencies
 WORKDIR /app
 RUN pip install --no-cache-dir --upgrade pip wheel && \
-        pip install --no-cache-dir -r requirements.txt && \
-        pip install --no-cache-dir flower
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir flower
 
 # Run application
-CMD celery flower -A ibutsu_server.tasks.queues:app --loglevel=info --broker=$BROKER_URL
+CMD ["celery", "flower", "-A ibutsu_server.tasks.queues:app", "--loglevel=info", "--broker=$BROKER_URL"]

backend/docker/Dockerfile.fuzz_testing

@@ -1,10 +1,14 @@
+# hadolint global ignore=DL3008,DL3013,DL3041
 FROM python:3.9
 
 RUN apt-get update && \
-    apt-get install -y jq && \
-    pip install schemathesis
+    apt-get install -y --no-install-recommends jq && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install --no-cache-dir  schemathesis
 
 
+WORKDIR /
+
 COPY docker/start_fuzz_testing.sh .
 
 CMD ["./start_fuzz_testing.sh"]

backend/docker/Dockerfile.scheduler

@@ -1,13 +1,16 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal
+# hadolint global ignore=DL3013,DL3041
+FROM registry.access.redhat.com/ubi8/python-311:1
 
 WORKDIR /app
 
 USER 0
 
-RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel
+RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \
+    dnf clean all
+
 COPY . /app
-RUN pip3 install --no-cache-dir -r requirements.txt
-RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
+RUN pip3 install --no-cache-dir -r requirements.txt && \
+    chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
 
 USER 1001
 

backend/docker/Dockerfile.worker

@@ -1,13 +1,16 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal
+# hadolint global ignore=DL3013,DL3041
+FROM registry.access.redhat.com/ubi8/python-311:1
 
 WORKDIR /app
 
 USER 0
 
-RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel
+RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \
+    dnf clean all
+
 COPY . /app
-RUN pip3 install --no-cache-dir -r requirements.txt
-RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
+RUN pip3 install --no-cache-dir -r requirements.txt && \
+    chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server
 
 USER 1001
 

backend/ibutsu_server/__init__.py

@@ -1,29 +1,26 @@
 import os
+from http import HTTPStatus
 from importlib import import_module
 from pathlib import Path
-from typing import Any
-from typing import Dict
-from typing import Optional
+from typing import Any, Dict, Optional
 
 import flask
 from connexion import App
-from flask import redirect
-from flask import request
+from flask import redirect, request
 from flask_cors import CORS
 from flask_mail import Mail
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import URL as SQLA_URL
+from yaml import full_load as yaml_load
+
 from ibutsu_server.auth import bcrypt
 from ibutsu_server.db import upgrades
-from ibutsu_server.db.base import db
-from ibutsu_server.db.base import session
-from ibutsu_server.db.models import upgrade_db
-from ibutsu_server.db.models import User
+from ibutsu_server.db.base import db, session
+from ibutsu_server.db.models import User, upgrade_db
 from ibutsu_server.db.util import add_superadmin
 from ibutsu_server.encoder import IbutsuJSONProvider
 from ibutsu_server.tasks import create_celery_app
 from ibutsu_server.util.jwt import decode_token
-from sqlalchemy import create_engine
-from sqlalchemy.engine.url import URL as SQLA_URL
-from yaml import full_load as yaml_load
 
 FRONTEND_PATH = Path("/app/frontend")
 
@@ -75,7 +72,11 @@ def get_app(**extra_config):
     config.from_mapping(os.environ)
     # convert str to bool for USER_LOGIN_ENABLED
     if isinstance(config.get("USER_LOGIN_ENABLED", True), str):
-        config["USER_LOGIN_ENABLED"] = config["USER_LOGIN_ENABLED"].lower()[0] in ["y", "t", "1"]
+        config["USER_LOGIN_ENABLED"] = config["USER_LOGIN_ENABLED"].lower()[0] in [
+            "y",
+            "t",
+            "1",
+        ]
 
     # If you have environment variables, like when running on OpenShift, create the db url
     if "SQLALCHEMY_DATABASE_URI" not in extra_config:
@@ -109,7 +110,10 @@ def get_app(**extra_config):
 
     create_celery_app(app.app)
     app.add_api(
-        "openapi.yaml", arguments={"title": "Ibutsu"}, base_path="/api", pythonic_params=True
+        "openapi.yaml",
+        arguments={"title": "Ibutsu"},
+        base_path="/api",
+        pythonic_params=True,
     )
 
     CORS(app.app, resources={r"/*": {"origins": "*", "send_wildcard": False}})
@@ -118,7 +122,6 @@ def get_app(**extra_config):
     Mail(app.app)
 
     with app.app.app_context():
-
         db.create_all()
         upgrade_db(session, upgrades)
         # add a superadmin user
@@ -135,31 +138,31 @@ def run_task():
         # get params
         params = request.get_json(force=True, silent=True)
         if not params:
-            return "Bad request", 400
+            return HTTPStatus.BAD_REQUEST.phrase, HTTPStatus.BAD_REQUEST
         # get user info
         token = params.get("token")
         if not token:
-            return "Unauthorized", 401
+            return ("Unauthorized",)
         user_id = decode_token(token).get("sub")
         if not user_id:
-            return "Unauthorized", 401
+            return HTTPStatus.UNAUTHORIZED.phrase, HTTPStatus.UNAUTHORIZED
         user = User.query.get(user_id)
         if not user or not user.is_superadmin:
-            return "Forbidden", 403
+            return HTTPStatus.FORBIDDEN.phrase, HTTPStatus.FORBIDDEN
         # get task info
         task_path = params.get("task")
         task_params = params.get("params", {})
         if not task_path:
-            return "Bad request", 400
+            return HTTPStatus.BAD_REQUEST.phrase, HTTPStatus.BAD_REQUEST
         task_module, task_name = task_path.split(".", 2)
         try:
             mod = import_module(f"ibutsu_server.tasks.{task_module}")
         except ImportError:
-            return "Not found", 404
+            return HTTPStatus.NOT_FOUND.phrase, HTTPStatus.NOT_FOUND
         if not hasattr(mod, task_name):
-            return "Not found", 404
+            return HTTPStatus.NOT_FOUND.phrase, HTTPStatus.NOT_FOUND
         task = getattr(mod, task_name)
         task.delay(**task_params)
-        return "Accepted", 202
+        return HTTPStatus.ACCEPTED.phrase, HTTPStatus.ACCEPTED
 
     return app.app

backend/ibutsu_server/constants.py

@@ -1,3 +1,5 @@
+LOCALHOST = "127.0.0.1"
+
 OAUTH_CONFIG = {
     "google": {
         "scope": ["https://www.googleapis.com/auth/userinfo.profile"],
@@ -308,8 +310,16 @@
         "id": "jenkins-job-view",
         "title": "Jenkins Job View",
         "params": [
-            {"name": "filter", "description": "Filters for the Jenkins Jobs", "type": "list"},
-            {"name": "page", "description": "Desired page of builds to return.", "type": "integer"},
+            {
+                "name": "filter",
+                "description": "Filters for the Jenkins Jobs",
+                "type": "list",
+            },
+            {
+                "name": "page",
+                "description": "Desired page of builds to return.",
+                "type": "integer",
+            },
             {
                 "name": "page_size",
                 "description": "Number of builds on each page",
@@ -327,8 +337,16 @@
         "id": "jenkins-analysis-view",
         "title": "Jenkins Job Analysis",
         "params": [
-            {"name": "job_name", "description": "The name of the Jenkins Job", "type": "string"},
-            {"name": "builds", "description": "The number of builds to fetch", "type": "integer"},
+            {
+                "name": "job_name",
+                "description": "The name of the Jenkins Job",
+                "type": "string",
+            },
+            {
+                "name": "builds",
+                "description": "The number of builds to fetch",
+                "type": "integer",
+            },
         ],
         "type": "view",
     },

backend/ibutsu_server/controllers/admin/project_controller.py

@@ -1,15 +1,12 @@
 import connexion
 from flask import abort
+
 from ibutsu_server.db.base import session
-from ibutsu_server.db.models import Group
-from ibutsu_server.db.models import Project
-from ibutsu_server.db.models import User
+from ibutsu_server.db.models import Group, Project, User
 from ibutsu_server.filters import convert_filter
 from ibutsu_server.util.admin import check_user_is_admin
 from ibutsu_server.util.query import get_offset
-from ibutsu_server.util.uuid import convert_objectid_to_uuid
-from ibutsu_server.util.uuid import is_uuid
-from ibutsu_server.util.uuid import validate_uuid
+from ibutsu_server.util.uuid import convert_objectid_to_uuid, is_uuid, validate_uuid
 
 
 def admin_add_project(project=None, token_info=None, user=None):
@@ -60,7 +57,13 @@ def admin_get_project(id_, token_info=None, user=None):
 
 
 def admin_get_project_list(
-    filter_=None, owner_id=None, group_id=None, page=1, page_size=25, token_info=None, user=None
+    filter_=None,
+    owner_id=None,
+    group_id=None,
+    page=1,
+    page_size=25,
+    token_info=None,
+    user=None,
 ):
     """Get a list of projects