diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05f4bc9c..eaa5dd52 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: needs: lint strategy: matrix: - python-version: [ "3.8", "3.9", "3.10", ] # five is broken on 3.11 - celery update needed "3.11.0-alpha - 3.11"] + python-version: [ "3.9", "3.10", "3.11"] # five is broken on 3.11 - celery update needed "3.11.0-alpha - 3.11"] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1448a79b..a65dffed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,32 +1,36 @@ repos: -- repo: https://github.com/asottile/reorder_python_imports - rev: v3.9.0 - hooks: - - id: reorder-python-imports -- repo: https://github.com/ambv/black - rev: 22.10.0 - hooks: - - id: black - args: [--safe, --quiet, --line-length, "100"] + +## PYTHON - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml args: [ --allow-multiple-documents ] - id: debug-statements -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.6 hooks: - - id: flake8 - args: [--max-line-length, "100", --ignore, "E203, W503"] -- repo: https://github.com/asottile/pyupgrade - rev: v3.2.2 + - id: ruff + args: + - '--fix' + - id: ruff-format + +## TODO: enable +# - repo: https://github.com/shellcheck-py/shellcheck-py +# rev: v0.10.0.1 +# hooks: +# - id: shellcheck + +- repo: https://github.com/AleksaC/hadolint-py + rev: v2.12.1b3 hooks: - - id: pyupgrade + - id: hadolint + +## ES - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.27.0 + rev: v9.3.0 hooks: - id: eslint additional_dependencies: diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000..4b16b641 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,7 @@ +line-length = 100 + +[lint] +select = ["F", "I", "UP"] + +[format] +quote-style = "double" diff --git a/backend/Containerfile b/backend/Containerfile index f4a215fa..96171130 100644 --- a/backend/Containerfile +++ b/backend/Containerfile @@ -1,7 +1,9 @@ -FROM registry.access.redhat.com/ubi8/python-38 as base +FROM registry.access.redhat.com/ubi8/python-39:1-184.1716989688 as base + USER 0 -RUN mkdir /ibutsu_venv -RUN chown -R 1001:0 /ibutsu_venv /srv +RUN mkdir -p /ibutsu_venv && \ + chown -R 1001:0 /ibutsu_venv /srv + USER 1001 WORKDIR /srv RUN python -m venv /ibutsu_venv && \ diff --git a/backend/docker/Dockerfile.backend b/backend/docker/Dockerfile.backend index 39dc877f..8fe0a89c 100644 --- a/backend/docker/Dockerfile.backend +++ b/backend/docker/Dockerfile.backend @@ -1,13 +1,16 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal +# hadolint global ignore=DL3013,DL3041 +FROM registry.access.redhat.com/ubi8/python-311:1 WORKDIR /app USER 0 -RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel -COPY . /app -RUN pip3 install --no-cache-dir gunicorn && pip3 install --no-cache-dir -r requirements.txt -RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server +COPY . . +RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \ + dnf clean all && \ + pip3 install --no-cache-dir gunicorn && \ + pip3 install --no-cache-dir -r requirements.txt && \ + chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server USER 1001 diff --git a/backend/docker/Dockerfile.flower b/backend/docker/Dockerfile.flower index ce71e13c..53017ea5 100644 --- a/backend/docker/Dockerfile.flower +++ b/backend/docker/Dockerfile.flower @@ -1,10 +1,14 @@ -FROM registry.access.redhat.com/ubi8/python-38 +# hadolint global ignore=DL3013,DL3041 +FROM registry.access.redhat.com/ubi8/python-311:1 # add application sources with correct perms for OCP USER 0 -RUN dnf install -y libpq-devel python38-devel gcc -ADD . /app +RUN dnf install -y libpq-devel gcc && \ + dnf clean all + +COPY . /app RUN /usr/bin/fix-permissions /app + USER 1001 ENV BROKER_URL=redis://localhost @@ -12,8 +16,8 @@ ENV BROKER_URL=redis://localhost # Install dependencies WORKDIR /app RUN pip install --no-cache-dir --upgrade pip wheel && \ - pip install --no-cache-dir -r requirements.txt && \ - pip install --no-cache-dir flower + pip install --no-cache-dir -r requirements.txt && \ + pip install --no-cache-dir flower # Run application -CMD celery flower -A ibutsu_server.tasks.queues:app --loglevel=info --broker=$BROKER_URL +CMD ["celery", "flower", "-A ibutsu_server.tasks.queues:app", "--loglevel=info", "--broker=$BROKER_URL"] diff --git a/backend/docker/Dockerfile.fuzz_testing b/backend/docker/Dockerfile.fuzz_testing index 822cf94f..ffe0c786 100644 --- a/backend/docker/Dockerfile.fuzz_testing +++ b/backend/docker/Dockerfile.fuzz_testing @@ -1,10 +1,14 @@ +# hadolint global ignore=DL3008,DL3013,DL3041 FROM python:3.9 RUN apt-get update && \ - apt-get install -y jq && \ - pip install schemathesis + apt-get install -y --no-install-recommends jq && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --no-cache-dir schemathesis +WORKDIR / + COPY docker/start_fuzz_testing.sh . CMD ["./start_fuzz_testing.sh"] diff --git a/backend/docker/Dockerfile.scheduler b/backend/docker/Dockerfile.scheduler index 792cb111..a4ed43c9 100644 --- a/backend/docker/Dockerfile.scheduler +++ b/backend/docker/Dockerfile.scheduler @@ -1,13 +1,16 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal +# hadolint global ignore=DL3013,DL3041 +FROM registry.access.redhat.com/ubi8/python-311:1 WORKDIR /app USER 0 -RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel +RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \ + dnf clean all + COPY . /app -RUN pip3 install --no-cache-dir -r requirements.txt -RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server +RUN pip3 install --no-cache-dir -r requirements.txt && \ + chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server USER 1001 diff --git a/backend/docker/Dockerfile.worker b/backend/docker/Dockerfile.worker index b2f88d26..37b3fd8f 100644 --- a/backend/docker/Dockerfile.worker +++ b/backend/docker/Dockerfile.worker @@ -1,13 +1,16 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal +# hadolint global ignore=DL3013,DL3041 +FROM registry.access.redhat.com/ubi8/python-311:1 WORKDIR /app USER 0 -RUN microdnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel python39 python39-devel +RUN dnf install --nodocs -y --disableplugin=subscription-manager gcc libpq-devel && \ + dnf clean all + COPY . /app -RUN pip3 install --no-cache-dir -r requirements.txt -RUN chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server +RUN pip3 install --no-cache-dir -r requirements.txt && \ + chgrp -R 0 ibutsu_server && chmod -R g+rwX ibutsu_server USER 1001 diff --git a/backend/ibutsu_server/__init__.py b/backend/ibutsu_server/__init__.py index 18865a05..4664dd74 100644 --- a/backend/ibutsu_server/__init__.py +++ b/backend/ibutsu_server/__init__.py @@ -1,29 +1,26 @@ import os +from http import HTTPStatus from importlib import import_module from pathlib import Path -from typing import Any -from typing import Dict -from typing import Optional +from typing import Any, Dict, Optional import flask from connexion import App -from flask import redirect -from flask import request +from flask import redirect, request from flask_cors import CORS from flask_mail import Mail +from sqlalchemy import create_engine +from sqlalchemy.engine.url import URL as SQLA_URL +from yaml import full_load as yaml_load + from ibutsu_server.auth import bcrypt from ibutsu_server.db import upgrades -from ibutsu_server.db.base import db -from ibutsu_server.db.base import session -from ibutsu_server.db.models import upgrade_db -from ibutsu_server.db.models import User +from ibutsu_server.db.base import db, session +from ibutsu_server.db.models import User, upgrade_db from ibutsu_server.db.util import add_superadmin from ibutsu_server.encoder import IbutsuJSONProvider from ibutsu_server.tasks import create_celery_app from ibutsu_server.util.jwt import decode_token -from sqlalchemy import create_engine -from sqlalchemy.engine.url import URL as SQLA_URL -from yaml import full_load as yaml_load FRONTEND_PATH = Path("/app/frontend") @@ -75,7 +72,11 @@ def get_app(**extra_config): config.from_mapping(os.environ) # convert str to bool for USER_LOGIN_ENABLED if isinstance(config.get("USER_LOGIN_ENABLED", True), str): - config["USER_LOGIN_ENABLED"] = config["USER_LOGIN_ENABLED"].lower()[0] in ["y", "t", "1"] + config["USER_LOGIN_ENABLED"] = config["USER_LOGIN_ENABLED"].lower()[0] in [ + "y", + "t", + "1", + ] # If you have environment variables, like when running on OpenShift, create the db url if "SQLALCHEMY_DATABASE_URI" not in extra_config: @@ -109,7 +110,10 @@ def get_app(**extra_config): create_celery_app(app.app) app.add_api( - "openapi.yaml", arguments={"title": "Ibutsu"}, base_path="/api", pythonic_params=True + "openapi.yaml", + arguments={"title": "Ibutsu"}, + base_path="/api", + pythonic_params=True, ) CORS(app.app, resources={r"/*": {"origins": "*", "send_wildcard": False}}) @@ -118,7 +122,6 @@ def get_app(**extra_config): Mail(app.app) with app.app.app_context(): - db.create_all() upgrade_db(session, upgrades) # add a superadmin user @@ -135,31 +138,31 @@ def run_task(): # get params params = request.get_json(force=True, silent=True) if not params: - return "Bad request", 400 + return HTTPStatus.BAD_REQUEST.phrase, HTTPStatus.BAD_REQUEST # get user info token = params.get("token") if not token: - return "Unauthorized", 401 + return ("Unauthorized",) user_id = decode_token(token).get("sub") if not user_id: - return "Unauthorized", 401 + return HTTPStatus.UNAUTHORIZED.phrase, HTTPStatus.UNAUTHORIZED user = User.query.get(user_id) if not user or not user.is_superadmin: - return "Forbidden", 403 + return HTTPStatus.FORBIDDEN.phrase, HTTPStatus.FORBIDDEN # get task info task_path = params.get("task") task_params = params.get("params", {}) if not task_path: - return "Bad request", 400 + return HTTPStatus.BAD_REQUEST.phrase, HTTPStatus.BAD_REQUEST task_module, task_name = task_path.split(".", 2) try: mod = import_module(f"ibutsu_server.tasks.{task_module}") except ImportError: - return "Not found", 404 + return HTTPStatus.NOT_FOUND.phrase, HTTPStatus.NOT_FOUND if not hasattr(mod, task_name): - return "Not found", 404 + return HTTPStatus.NOT_FOUND.phrase, HTTPStatus.NOT_FOUND task = getattr(mod, task_name) task.delay(**task_params) - return "Accepted", 202 + return HTTPStatus.ACCEPTED.phrase, HTTPStatus.ACCEPTED return app.app diff --git a/backend/ibutsu_server/constants.py b/backend/ibutsu_server/constants.py index a10d6e63..f33feddc 100644 --- a/backend/ibutsu_server/constants.py +++ b/backend/ibutsu_server/constants.py @@ -1,3 +1,5 @@ +LOCALHOST = "127.0.0.1" + OAUTH_CONFIG = { "google": { "scope": ["https://www.googleapis.com/auth/userinfo.profile"], @@ -308,8 +310,16 @@ "id": "jenkins-job-view", "title": "Jenkins Job View", "params": [ - {"name": "filter", "description": "Filters for the Jenkins Jobs", "type": "list"}, - {"name": "page", "description": "Desired page of builds to return.", "type": "integer"}, + { + "name": "filter", + "description": "Filters for the Jenkins Jobs", + "type": "list", + }, + { + "name": "page", + "description": "Desired page of builds to return.", + "type": "integer", + }, { "name": "page_size", "description": "Number of builds on each page", @@ -327,8 +337,16 @@ "id": "jenkins-analysis-view", "title": "Jenkins Job Analysis", "params": [ - {"name": "job_name", "description": "The name of the Jenkins Job", "type": "string"}, - {"name": "builds", "description": "The number of builds to fetch", "type": "integer"}, + { + "name": "job_name", + "description": "The name of the Jenkins Job", + "type": "string", + }, + { + "name": "builds", + "description": "The number of builds to fetch", + "type": "integer", + }, ], "type": "view", }, diff --git a/backend/ibutsu_server/controllers/admin/project_controller.py b/backend/ibutsu_server/controllers/admin/project_controller.py index e03248ec..6b14349d 100644 --- a/backend/ibutsu_server/controllers/admin/project_controller.py +++ b/backend/ibutsu_server/controllers/admin/project_controller.py @@ -1,15 +1,12 @@ import connexion from flask import abort + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Group -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Group, Project, User from ibutsu_server.filters import convert_filter from ibutsu_server.util.admin import check_user_is_admin from ibutsu_server.util.query import get_offset -from ibutsu_server.util.uuid import convert_objectid_to_uuid -from ibutsu_server.util.uuid import is_uuid -from ibutsu_server.util.uuid import validate_uuid +from ibutsu_server.util.uuid import convert_objectid_to_uuid, is_uuid, validate_uuid def admin_add_project(project=None, token_info=None, user=None): @@ -60,7 +57,13 @@ def admin_get_project(id_, token_info=None, user=None): def admin_get_project_list( - filter_=None, owner_id=None, group_id=None, page=1, page_size=25, token_info=None, user=None + filter_=None, + owner_id=None, + group_id=None, + page=1, + page_size=25, + token_info=None, + user=None, ): """Get a list of projects diff --git a/backend/ibutsu_server/controllers/admin/user_controller.py b/backend/ibutsu_server/controllers/admin/user_controller.py index 143ebb5b..7588e826 100644 --- a/backend/ibutsu_server/controllers/admin/user_controller.py +++ b/backend/ibutsu_server/controllers/admin/user_controller.py @@ -1,14 +1,13 @@ import connexion from flask import abort + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Project, User from ibutsu_server.filters import convert_filter from ibutsu_server.util.admin import check_user_is_admin from ibutsu_server.util.query import get_offset from ibutsu_server.util.uuid import validate_uuid - HIDDEN_FIELDS = ["_password", "password", "activation_code"] diff --git a/backend/ibutsu_server/controllers/artifact_controller.py b/backend/ibutsu_server/controllers/artifact_controller.py index b90f71aa..97f94a94 100644 --- a/backend/ibutsu_server/controllers/artifact_controller.py +++ b/backend/ibutsu_server/controllers/artifact_controller.py @@ -4,15 +4,12 @@ import connexion import magic from flask import make_response + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Artifact -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import User -from ibutsu_server.util.projects import add_user_filter -from ibutsu_server.util.projects import project_has_user +from ibutsu_server.db.models import Artifact, Result, User +from ibutsu_server.util.projects import add_user_filter, project_has_user from ibutsu_server.util.query import get_offset -from ibutsu_server.util.uuid import is_uuid -from ibutsu_server.util.uuid import validate_uuid +from ibutsu_server.util.uuid import is_uuid, validate_uuid def _build_artifact_response(id_): @@ -57,7 +54,7 @@ def download_artifact(id_, token_info=None, user=None): artifact, response = _build_artifact_response(id_) if not project_has_user(artifact.result.project, user): return "Forbidden", 403 - response.headers["Content-Disposition"] = "attachment; filename={}".format(artifact.filename) + response.headers["Content-Disposition"] = f"attachment; filename={artifact.filename}" return response diff --git a/backend/ibutsu_server/controllers/dashboard_controller.py b/backend/ibutsu_server/controllers/dashboard_controller.py index 01db95fb..76c9650b 100644 --- a/backend/ibutsu_server/controllers/dashboard_controller.py +++ b/backend/ibutsu_server/controllers/dashboard_controller.py @@ -1,9 +1,7 @@ import connexion + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Dashboard -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import User -from ibutsu_server.db.models import WidgetConfig +from ibutsu_server.db.models import Dashboard, Project, User, WidgetConfig from ibutsu_server.filters import convert_filter from ibutsu_server.util.projects import project_has_user from ibutsu_server.util.query import get_offset diff --git a/backend/ibutsu_server/controllers/group_controller.py b/backend/ibutsu_server/controllers/group_controller.py index 91112b38..522df794 100644 --- a/backend/ibutsu_server/controllers/group_controller.py +++ b/backend/ibutsu_server/controllers/group_controller.py @@ -1,9 +1,9 @@ import connexion + from ibutsu_server.db.base import session from ibutsu_server.db.models import Group from ibutsu_server.util.query import get_offset -from ibutsu_server.util.uuid import is_uuid -from ibutsu_server.util.uuid import validate_uuid +from ibutsu_server.util.uuid import is_uuid, validate_uuid def add_group(group=None): diff --git a/backend/ibutsu_server/controllers/health_controller.py b/backend/ibutsu_server/controllers/health_controller.py index a26edc5a..41a3b541 100644 --- a/backend/ibutsu_server/controllers/health_controller.py +++ b/backend/ibutsu_server/controllers/health_controller.py @@ -1,6 +1,5 @@ from flask import current_app -from sqlalchemy.exc import InterfaceError -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import InterfaceError, OperationalError try: from ibutsu_server.db.model import Result @@ -27,14 +26,23 @@ def get_database_health(token_info=None, user=None): # Try to connect to the database, and handle various responses try: if not IS_CONNECTED: - response = ({"status": "Error", "message": "Incomplete database configuration"}, 503) + response = ( + {"status": "Error", "message": "Incomplete database configuration"}, + 503, + ) else: Result.query.first() response = ({"status": "OK", "message": "Service is running"}, 200) except OperationalError: - response = ({"status": "Error", "message": "Unable to connect to the database"}, 503) + response = ( + {"status": "Error", "message": "Unable to connect to the database"}, + 503, + ) except InterfaceError: - response = ({"status": "Error", "message": "Incorrect connection configuration"}, 503) + response = ( + {"status": "Error", "message": "Incorrect connection configuration"}, + 503, + ) except Exception as e: response = ({"status": "Error", "message": str(e)}, 500) return response @@ -46,7 +54,7 @@ def get_health_info(token_info=None, user=None): :rtype: HealthInfo """ return { - "frontend": current_app.config.get("FRONTEND_URL", "http://localhost:3000"), - "backend": current_app.config.get("BACKEND_URL", "http://localhost:8080"), - "api_ui": current_app.config.get("BACKEND_URL", "http://localhost:8080") + "/api/ui/", + "frontend": current_app.config.get("FRONTEND_URL", "http://127.0.01:3000"), + "backend": current_app.config.get("BACKEND_URL", "http://127.0.0.1:8080"), + "api_ui": current_app.config.get("BACKEND_URL", "http://127.0.0.1:8080") + "/api/ui/", } diff --git a/backend/ibutsu_server/controllers/import_controller.py b/backend/ibutsu_server/controllers/import_controller.py index 391f1e81..6ab6596f 100644 --- a/backend/ibutsu_server/controllers/import_controller.py +++ b/backend/ibutsu_server/controllers/import_controller.py @@ -2,15 +2,13 @@ from typing import Optional import connexion +from werkzeug.datastructures import FileStorage + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Import -from ibutsu_server.db.models import ImportFile -from ibutsu_server.tasks.importers import run_archive_import -from ibutsu_server.tasks.importers import run_junit_import -from ibutsu_server.util.projects import get_project -from ibutsu_server.util.projects import project_has_user +from ibutsu_server.db.models import Import, ImportFile +from ibutsu_server.tasks.importers import run_archive_import, run_junit_import +from ibutsu_server.util.projects import get_project, project_has_user from ibutsu_server.util.uuid import validate_uuid -from werkzeug.datastructures import FileStorage @validate_uuid @@ -73,7 +71,12 @@ def add_import( if connexion.request.form.get("source"): data["source"] = connexion.request.form["source"] new_import = Import.from_dict( - **{"status": "pending", "filename": import_file.filename, "format": "", "data": data} + **{ + "status": "pending", + "filename": import_file.filename, + "format": "", + "data": data, + } ) session.add(new_import) session.commit() diff --git a/backend/ibutsu_server/controllers/login_controller.py b/backend/ibutsu_server/controllers/login_controller.py index 5453c527..bac98b11 100644 --- a/backend/ibutsu_server/controllers/login_controller.py +++ b/backend/ibutsu_server/controllers/login_controller.py @@ -5,20 +5,17 @@ import connexion import requests -from flask import current_app -from flask import make_response -from flask import redirect +from flask import current_app, make_response, redirect from google.auth.transport.requests import Request from google.oauth2 import id_token + +from ibutsu_server.constants import LOCALHOST from ibutsu_server.db.base import session -from ibutsu_server.db.models import Token -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Token, User from ibutsu_server.util.jwt import generate_token -from ibutsu_server.util.keycloak import get_keycloak_config -from ibutsu_server.util.keycloak import get_user_from_keycloak +from ibutsu_server.util.keycloak import get_keycloak_config, get_user_from_keycloak from ibutsu_server.util.login import validate_activation_code -from ibutsu_server.util.oauth import get_provider_config -from ibutsu_server.util.oauth import get_user_from_provider +from ibutsu_server.util.oauth import get_provider_config, get_user_from_provider from ibutsu_server.util.urls import build_url AUTH_WINDOW = """ @@ -133,15 +130,17 @@ def login(email=None, password=None): session.add(token) session.commit() return {"name": user.name, "email": user.email, "token": login_token} + elif not current_app.config.get("USER_LOGIN_ENABLED", True): + return { + "code": "INVALID", + "message": "Username/password auth is disabled. " + "Please login via one of the links below.", + }, 401 else: - if not current_app.config.get("USER_LOGIN_ENABLED", True): - return { - "code": "INVALID", - "message": "Username/password auth is disabled. " - "Please login via one of the links below.", - }, 401 - else: - return {"code": "INVALID", "message": "Username and/or password are invalid"}, 401 + return { + "code": "INVALID", + "message": "Username and/or password are invalid", + }, 401 def support(): @@ -170,7 +169,7 @@ def auth(provider): return "Bad request", 400 code = connexion.request.args["code"] frontend_url = build_url( - current_app.config.get("FRONTEND_URL", "http://localhost:3000"), "login" + current_app.config.get("FRONTEND_URL", f"http://{LOCALHOST}:3000"), "login" ) provider_config = _get_provider_config(provider) user = _get_user_from_provider(provider, provider_config, code) @@ -212,7 +211,9 @@ def register(email=None, password=None): activation_code = urlsafe_b64encode(str(uuid4()).encode("utf8")).strip(b"=").decode() # Create a user user = User( - email=details["email"], password=details["password"], activation_code=activation_code + email=details["email"], + password=details["password"], + activation_code=activation_code, ) user_exists = User.query.filter_by(email=user.email) if user_exists: @@ -222,7 +223,7 @@ def register(email=None, password=None): # Send an activation e-mail activation_url = build_url( - current_app.config.get("BACKEND_URL", "http://localhost:8080/"), + current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080"), "api", "login", "activate", @@ -292,7 +293,9 @@ def activate(activation_code=None): if result := validate_activation_code(activation_code): return result user = User.query.filter(User.activation_code == activation_code).first() - login_url = build_url(current_app.config.get("FRONTEND_URL", "http://localhost:3000"), "login") + login_url = build_url( + current_app.config.get("FRONTEND_URL", f"http://{LOCALHOST}:3000"), "login" + ) if user: user.is_active = True user.activation_code = None diff --git a/backend/ibutsu_server/controllers/project_controller.py b/backend/ibutsu_server/controllers/project_controller.py index 6bae452e..b95cdae7 100644 --- a/backend/ibutsu_server/controllers/project_controller.py +++ b/backend/ibutsu_server/controllers/project_controller.py @@ -1,15 +1,11 @@ import connexion import flatdict + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import User -from ibutsu_server.util.projects import add_user_filter -from ibutsu_server.util.projects import project_has_user +from ibutsu_server.db.models import Project, Result, User +from ibutsu_server.util.projects import add_user_filter, project_has_user from ibutsu_server.util.query import get_offset -from ibutsu_server.util.uuid import convert_objectid_to_uuid -from ibutsu_server.util.uuid import is_uuid -from ibutsu_server.util.uuid import validate_uuid +from ibutsu_server.util.uuid import convert_objectid_to_uuid, is_uuid, validate_uuid def add_project(project=None, token_info=None, user=None): @@ -57,7 +53,13 @@ def get_project(id_, token_info=None, user=None): def get_project_list( - filter_=None, owner_id=None, group_id=None, page=1, page_size=25, token_info=None, user=None + filter_=None, + owner_id=None, + group_id=None, + page=1, + page_size=25, + token_info=None, + user=None, ): """Get a list of projects diff --git a/backend/ibutsu_server/controllers/report_controller.py b/backend/ibutsu_server/controllers/report_controller.py index d49cc8b9..d657b7d8 100644 --- a/backend/ibutsu_server/controllers/report_controller.py +++ b/backend/ibutsu_server/controllers/report_controller.py @@ -2,9 +2,9 @@ import connexion from flask import make_response + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Report -from ibutsu_server.db.models import ReportFile +from ibutsu_server.db.models import Report, ReportFile from ibutsu_server.tasks.reports import REPORTS from ibutsu_server.util.projects import get_project_id from ibutsu_server.util.query import get_offset @@ -155,5 +155,5 @@ def download_report(id_, filename, token_info=None, user=None): :rtype: file """ report, response = _build_report_response(id_) - response.headers["Content-Disposition"] = "attachment; filename={}".format(report.filename) + response.headers["Content-Disposition"] = f"attachment; filename={report.filename}" return response diff --git a/backend/ibutsu_server/controllers/result_controller.py b/backend/ibutsu_server/controllers/result_controller.py index 66a8c50a..0fde3499 100644 --- a/backend/ibutsu_server/controllers/result_controller.py +++ b/backend/ibutsu_server/controllers/result_controller.py @@ -1,17 +1,14 @@ from datetime import datetime import connexion + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Result, User from ibutsu_server.filters import convert_filter from ibutsu_server.util import merge_dicts from ibutsu_server.util.count import get_count_estimate -from ibutsu_server.util.projects import add_user_filter -from ibutsu_server.util.projects import get_project -from ibutsu_server.util.projects import project_has_user -from ibutsu_server.util.query import get_offset -from ibutsu_server.util.query import query_as_task +from ibutsu_server.util.projects import add_user_filter, get_project, project_has_user +from ibutsu_server.util.query import get_offset, query_as_task from ibutsu_server.util.uuid import validate_uuid diff --git a/backend/ibutsu_server/controllers/run_controller.py b/backend/ibutsu_server/controllers/run_controller.py index 1ff00bab..d7cd7da9 100644 --- a/backend/ibutsu_server/controllers/run_controller.py +++ b/backend/ibutsu_server/controllers/run_controller.py @@ -1,19 +1,20 @@ from datetime import datetime import connexion + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Run -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Run, User from ibutsu_server.filters import convert_filter from ibutsu_server.tasks.runs import update_run as update_run_task from ibutsu_server.util import merge_dicts from ibutsu_server.util.count import get_count_estimate -from ibutsu_server.util.projects import add_user_filter -from ibutsu_server.util.projects import get_project -from ibutsu_server.util.projects import get_project_id -from ibutsu_server.util.projects import project_has_user -from ibutsu_server.util.query import get_offset -from ibutsu_server.util.query import query_as_task +from ibutsu_server.util.projects import ( + add_user_filter, + get_project, + get_project_id, + project_has_user, +) +from ibutsu_server.util.query import get_offset, query_as_task from ibutsu_server.util.uuid import validate_uuid diff --git a/backend/ibutsu_server/controllers/task_controller.py b/backend/ibutsu_server/controllers/task_controller.py index 5e52e803..2170efe8 100644 --- a/backend/ibutsu_server/controllers/task_controller.py +++ b/backend/ibutsu_server/controllers/task_controller.py @@ -1,7 +1,14 @@ from celery.result import AsyncResult + from ibutsu_server.util.uuid import validate_uuid -_STATE_TO_CODE = {"SUCCESS": 200, "PENDING": 206, "STARTED": 206, "RETRY": 206, "FAILURE": 203} +_STATE_TO_CODE = { + "SUCCESS": 200, + "PENDING": 206, + "STARTED": 206, + "RETRY": 206, + "FAILURE": 203, +} @validate_uuid diff --git a/backend/ibutsu_server/controllers/user_controller.py b/backend/ibutsu_server/controllers/user_controller.py index 4ad2baab..88055e04 100644 --- a/backend/ibutsu_server/controllers/user_controller.py +++ b/backend/ibutsu_server/controllers/user_controller.py @@ -1,9 +1,9 @@ from datetime import datetime import connexion + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Token -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Token, User from ibutsu_server.util.jwt import generate_token from ibutsu_server.util.query import get_offset from ibutsu_server.util.uuid import validate_uuid diff --git a/backend/ibutsu_server/controllers/widget_config_controller.py b/backend/ibutsu_server/controllers/widget_config_controller.py index 6b86d4f9..f6d48419 100644 --- a/backend/ibutsu_server/controllers/widget_config_controller.py +++ b/backend/ibutsu_server/controllers/widget_config_controller.py @@ -1,14 +1,13 @@ import connexion -from ibutsu_server.constants import ALLOWED_TRUE_BOOLEANS -from ibutsu_server.constants import WIDGET_TYPES +from sqlalchemy import or_ + +from ibutsu_server.constants import ALLOWED_TRUE_BOOLEANS, WIDGET_TYPES from ibutsu_server.db.base import session from ibutsu_server.db.models import WidgetConfig from ibutsu_server.filters import convert_filter -from ibutsu_server.util.projects import get_project -from ibutsu_server.util.projects import project_has_user +from ibutsu_server.util.projects import get_project, project_has_user from ibutsu_server.util.query import get_offset from ibutsu_server.util.uuid import validate_uuid -from sqlalchemy import or_ def add_widget_config(widget_config=None, token_info=None, user=None): diff --git a/backend/ibutsu_server/controllers/widget_controller.py b/backend/ibutsu_server/controllers/widget_controller.py index 0ef62184..c78cef33 100644 --- a/backend/ibutsu_server/controllers/widget_controller.py +++ b/backend/ibutsu_server/controllers/widget_controller.py @@ -1,16 +1,22 @@ import connexion -from ibutsu_server.constants import ALLOWED_TRUE_BOOLEANS -from ibutsu_server.constants import WIDGET_TYPES -from ibutsu_server.widgets.accessibility_analysis import get_accessibility_analysis_view -from ibutsu_server.widgets.accessibility_analysis import get_accessibility_bar_chart -from ibutsu_server.widgets.accessibility_dashboard_view import get_accessibility_dashboard_view + +from ibutsu_server.constants import ALLOWED_TRUE_BOOLEANS, WIDGET_TYPES +from ibutsu_server.widgets.accessibility_analysis import ( + get_accessibility_analysis_view, + get_accessibility_bar_chart, +) +from ibutsu_server.widgets.accessibility_dashboard_view import ( + get_accessibility_dashboard_view, +) from ibutsu_server.widgets.compare_runs_view import get_comparison_data from ibutsu_server.widgets.filter_heatmap import get_filter_heatmap from ibutsu_server.widgets.importance_component import get_importance_component from ibutsu_server.widgets.jenkins_heatmap import get_jenkins_heatmap -from ibutsu_server.widgets.jenkins_job_analysis import get_jenkins_analysis_data -from ibutsu_server.widgets.jenkins_job_analysis import get_jenkins_bar_chart -from ibutsu_server.widgets.jenkins_job_analysis import get_jenkins_line_chart +from ibutsu_server.widgets.jenkins_job_analysis import ( + get_jenkins_analysis_data, + get_jenkins_bar_chart, + get_jenkins_line_chart, +) from ibutsu_server.widgets.jenkins_job_view import get_jenkins_job_view from ibutsu_server.widgets.result_aggregator import get_recent_result_data from ibutsu_server.widgets.result_summary import get_result_summary @@ -81,7 +87,12 @@ def get_widget_types(type_=None): page_size = len(widget_types) return { "types": list(widget_types), - "pagination": {"page": 1, "pageSize": page_size, "totalItems": page_size, "totalPages": 1}, + "pagination": { + "page": 1, + "pageSize": page_size, + "totalItems": page_size, + "totalPages": 1, + }, } diff --git a/backend/ibutsu_server/db/models.py b/backend/ibutsu_server/db/models.py index bb08a0ed..1b5e400c 100644 --- a/backend/ibutsu_server/db/models.py +++ b/backend/ibutsu_server/db/models.py @@ -1,27 +1,28 @@ from datetime import datetime from uuid import uuid4 -from ibutsu_server.auth import bcrypt -from ibutsu_server.db.base import Boolean -from ibutsu_server.db.base import Column -from ibutsu_server.db.base import DateTime -from ibutsu_server.db.base import Float -from ibutsu_server.db.base import ForeignKey -from ibutsu_server.db.base import inspect -from ibutsu_server.db.base import Integer -from ibutsu_server.db.base import LargeBinary -from ibutsu_server.db.base import Model -from ibutsu_server.db.base import relationship -from ibutsu_server.db.base import Table -from ibutsu_server.db.base import Text -from ibutsu_server.db.types import PortableJSON -from ibutsu_server.db.types import PortableUUID -from sqlalchemy.exc import DBAPIError -from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.exc import DBAPIError, SQLAlchemyError from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import backref from sqlalchemy_json import mutable_json_type +from ibutsu_server.auth import bcrypt +from ibutsu_server.db.base import ( + Boolean, + Column, + DateTime, + Float, + ForeignKey, + Integer, + LargeBinary, + Model, + Table, + Text, + inspect, + relationship, +) +from ibutsu_server.db.types import PortableJSON, PortableUUID + def _gen_uuid(): """Generate a UUID""" diff --git a/backend/ibutsu_server/db/types.py b/backend/ibutsu_server/db/types.py index 65ec51c0..06dce5ef 100644 --- a/backend/ibutsu_server/db/types.py +++ b/backend/ibutsu_server/db/types.py @@ -2,10 +2,7 @@ from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import UUID as PostgresUUID -from sqlalchemy.types import CHAR -from sqlalchemy.types import JSON -from sqlalchemy.types import Text -from sqlalchemy.types import TypeDecorator +from sqlalchemy.types import CHAR, JSON, Text, TypeDecorator class PortableUUID(TypeDecorator): @@ -36,11 +33,10 @@ def process_bind_param(self, value, dialect): return value elif dialect.name == "postgresql": return value + elif isinstance(value, UUID): + return str(value) else: - if isinstance(value, UUID): - return str(value) - else: - return value + return value def process_result_value(self, value, dialect): if value is None: diff --git a/backend/ibutsu_server/db/upgrades.py b/backend/ibutsu_server/db/upgrades.py index 479aa344..e7a6a2c7 100644 --- a/backend/ibutsu_server/db/upgrades.py +++ b/backend/ibutsu_server/db/upgrades.py @@ -1,14 +1,12 @@ from alembic.migration import MigrationContext from alembic.operations import Operations -from ibutsu_server.db.base import Boolean -from ibutsu_server.db.base import Column -from ibutsu_server.db.base import ForeignKey -from ibutsu_server.db.base import Text -from ibutsu_server.db.types import PortableUUID from sqlalchemy import MetaData from sqlalchemy.sql import quoted_name from sqlalchemy.sql.expression import null +from ibutsu_server.db.base import Boolean, Column, ForeignKey, Text +from ibutsu_server.db.types import PortableUUID + __version__ = 5 @@ -37,7 +35,10 @@ def upgrade_1(session): and widget_configs is not None and "dashboard_id" not in [col.name for col in widget_configs.columns] ): - op.add_column("widget_configs", Column("dashboard_id", PortableUUID, server_default=null())) + op.add_column( + "widget_configs", + Column("dashboard_id", PortableUUID, server_default=null()), + ) if engine.url.get_dialect().name != "sqlite": # SQLite doesn't support ALTER TABLE ADD CONSTRAINT op.create_foreign_key( @@ -171,5 +172,6 @@ def upgrade_5(session): and "default_dashboard_id" not in [col.name for col in projects.columns] ): op.add_column( - "projects", Column("default_dashboard_id", PortableUUID(), ForeignKey("dashboards.id")) + "projects", + Column("default_dashboard_id", PortableUUID(), ForeignKey("dashboards.id")), ) diff --git a/backend/ibutsu_server/db/util.py b/backend/ibutsu_server/db/util.py index 4d13b825..4f456432 100644 --- a/backend/ibutsu_server/db/util.py +++ b/backend/ibutsu_server/db/util.py @@ -1,13 +1,13 @@ """ Various utility DB functions """ + from typing import Optional -from ibutsu_server.db import models from sqlalchemy.ext.compiler import compiles -from sqlalchemy.sql.expression import _literal_as_text -from sqlalchemy.sql.expression import ClauseElement -from sqlalchemy.sql.expression import Executable +from sqlalchemy.sql.expression import ClauseElement, Executable, _literal_as_text + +from ibutsu_server.db import models class Explain(Executable, ClauseElement): @@ -52,7 +52,6 @@ def add_superadmin( elif user and not user.is_superadmin: user.is_superadmin = True else: - user = models.User( email=email, name=name, diff --git a/backend/ibutsu_server/encoder.py b/backend/ibutsu_server/encoder.py index d6292893..3c4dca95 100644 --- a/backend/ibutsu_server/encoder.py +++ b/backend/ibutsu_server/encoder.py @@ -1,4 +1,5 @@ from flask.json.provider import DefaultJSONProvider + from ibutsu_server.models.base_model_ import Model diff --git a/backend/ibutsu_server/filters.py b/backend/ibutsu_server/filters.py index eeceeeae..ae25b63b 100644 --- a/backend/ibutsu_server/filters.py +++ b/backend/ibutsu_server/filters.py @@ -1,9 +1,9 @@ import re -from ibutsu_server.constants import ARRAY_FIELDS -from ibutsu_server.constants import NUMERIC_FIELDS from sqlalchemy.dialects.postgresql import array +from ibutsu_server.constants import ARRAY_FIELDS, NUMERIC_FIELDS + OPERATORS = { "=": "$eq", "!": "$ne", diff --git a/backend/ibutsu_server/fixtures/__init__.py b/backend/ibutsu_server/fixtures/__init__.py index 37294b7d..d4600255 100644 --- a/backend/ibutsu_server/fixtures/__init__.py +++ b/backend/ibutsu_server/fixtures/__init__.py @@ -2,6 +2,7 @@ import connexion import pytest + from ibutsu_server.encoder import IbutsuJSONProvider diff --git a/backend/ibutsu_server/models/base_model_.py b/backend/ibutsu_server/models/base_model_.py index 2f438213..4f929ceb 100644 --- a/backend/ibutsu_server/models/base_model_.py +++ b/backend/ibutsu_server/models/base_model_.py @@ -38,9 +38,9 @@ def to_dict(self): elif isinstance(value, dict): result[attr] = dict( map( - lambda item: (item[0], item[1].to_dict()) - if hasattr(item[1], "to_dict") - else item, + lambda item: ( + (item[0], item[1].to_dict()) if hasattr(item[1], "to_dict") else item + ), value.items(), ) ) diff --git a/backend/ibutsu_server/tasks/__init__.py b/backend/ibutsu_server/tasks/__init__.py index 4a0c3b3c..e1613e71 100644 --- a/backend/ibutsu_server/tasks/__init__.py +++ b/backend/ibutsu_server/tasks/__init__.py @@ -1,16 +1,15 @@ import logging from contextlib import contextmanager -from celery import Celery -from celery import signals -from celery import Task +from celery import Celery, Task, signals from celery.schedules import crontab from flask import current_app -from ibutsu_server.db.base import session -from ibutsu_server.db.models import Report from redis import Redis from redis.exceptions import LockError +from ibutsu_server.db.base import session +from ibutsu_server.db.models import Report + LOCK_EXPIRE = 1 task = None _celery_app = None diff --git a/backend/ibutsu_server/tasks/db.py b/backend/ibutsu_server/tasks/db.py index cb51e382..22c06477 100644 --- a/backend/ibutsu_server/tasks/db.py +++ b/backend/ibutsu_server/tasks/db.py @@ -1,13 +1,9 @@ -""" Tasks for DB related things""" -from datetime import datetime -from datetime import timedelta +"""Tasks for DB related things""" + +from datetime import datetime, timedelta from ibutsu_server.db.base import session -from ibutsu_server.db.models import Artifact -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Artifact, Project, Result, Run, User from ibutsu_server.tasks import task DAYS_IN_MONTH = 30 diff --git a/backend/ibutsu_server/tasks/importers.py b/backend/ibutsu_server/tasks/importers.py index 98e0145e..dbb8db3b 100644 --- a/backend/ibutsu_server/tasks/importers.py +++ b/backend/ibutsu_server/tasks/importers.py @@ -7,22 +7,19 @@ from celery.utils.log import get_task_logger from dateutil import parser +from lxml import objectify + from ibutsu_server.db.base import session -from ibutsu_server.db.models import Artifact -from ibutsu_server.db.models import Import -from ibutsu_server.db.models import ImportFile -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run +from ibutsu_server.db.models import Artifact, Import, ImportFile, Result, Run from ibutsu_server.tasks import task from ibutsu_server.tasks.runs import update_run from ibutsu_server.util.projects import get_project_id from ibutsu_server.util.uuid import is_uuid -from lxml import objectify - log = get_task_logger(__name__) uuid_pattern = re.compile( - "([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", re.IGNORECASE + "([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})", + re.IGNORECASE, ) @@ -467,7 +464,6 @@ def run_archive_import(import_): import_record.data["run_id"] = [run.id] # Loop through any artifacts associated with the run and upload them for artifact in run_artifacts: - session.add( Artifact( filename=artifact.name.split("/")[-1], diff --git a/backend/ibutsu_server/tasks/query.py b/backend/ibutsu_server/tasks/query.py index df7adf9d..f2e2b552 100644 --- a/backend/ibutsu_server/tasks/query.py +++ b/backend/ibutsu_server/tasks/query.py @@ -1,10 +1,8 @@ -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run +from ibutsu_server.db.models import Result, Run from ibutsu_server.filters import convert_filter from ibutsu_server.tasks import task from ibutsu_server.util.count import get_count_estimate - TABLENAME_TO_MODEL = {"results": Result, "runs": Run} diff --git a/backend/ibutsu_server/tasks/reports.py b/backend/ibutsu_server/tasks/reports.py index f49a8407..f6f3dd71 100644 --- a/backend/ibutsu_server/tasks/reports.py +++ b/backend/ibutsu_server/tasks/reports.py @@ -6,20 +6,26 @@ from io import StringIO from flask import current_app +from sqlalchemy.exc import OperationalError + +from ibutsu_server.constants import LOCALHOST from ibutsu_server.db.base import session -from ibutsu_server.db.models import Report -from ibutsu_server.db.models import ReportFile -from ibutsu_server.db.models import Result +from ibutsu_server.db.models import Report, ReportFile, Result from ibutsu_server.filters import apply_filters from ibutsu_server.tasks import task from ibutsu_server.templating import render_template from ibutsu_server.util.projects import get_project_id -from sqlalchemy.exc import OperationalError - TREE_ROOT = { "items": {}, - "stats": {"passed": 0, "failed": 0, "skipped": 0, "error": 0, "xpassed": 0, "xfailed": 0}, + "stats": { + "passed": 0, + "failed": 0, + "skipped": 0, + "error": 0, + "xpassed": 0, + "xfailed": 0, + }, "duration": 0.0, } @@ -67,17 +73,17 @@ def _update_report(report): "filename": report_filename, "mimetype": REPORTS[report_type]["mimetype"], "url": "{}/api/report/{}/download/{}".format( - current_app.config.get("BACKEND_URL", "http://localhost:8080"), + current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080"), report["id"], report_filename, ), "download_url": "{}/api/report/{}/download/{}".format( - current_app.config.get("BACKEND_URL", "http://localhost:8080"), + current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080"), report["id"], report_filename, ), "view_url": "{}/api/report/{}/view/{}".format( - current_app.config.get("BACKEND_URL", "http://localhost:8080"), + current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080"), report["id"], report_filename, ), @@ -152,7 +158,7 @@ def _make_result_path(result): fspath = fspath[3:] if fspath.startswith("./"): fspath = fspath[2:] - result_path = "{}::".format(fspath) + result_path = f"{fspath}::" result_path += result["test_id"] return result_path @@ -197,7 +203,7 @@ def _make_row(old, parent_key=None): """ new = {} for key, value in old.items(): - new_key = "{}.{}".format(parent_key, str(key)) if parent_key else str(key) + new_key = f"{parent_key}.{str(key)}" if parent_key else str(key) if isinstance(value, dict): new.update(_make_row(value, new_key)) elif isinstance(value, list): @@ -213,7 +219,8 @@ def _get_files(result): { "filename": report_file.filename, "url": "{}/api/artifact/{}/download".format( - current_app.config.get("BACKEND_URL", "http://localhost:8080"), str(report_file.id) + current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080"), + str(report_file.id), ), } for report_file in ReportFile.query.filter( @@ -396,7 +403,7 @@ def generate_text_report(report): summary[result["result"]] += 1 else: summary["other"] += 1 - text_file.writelines(["{}: {}\n".format(key, value) for key, value in summary.items()]) + text_file.writelines([f"{key}: {value}\n" for key, value in summary.items()]) text_file.write("\n") for result in results: result_path = _make_result_path(result) diff --git a/backend/ibutsu_server/tasks/results.py b/backend/ibutsu_server/tasks/results.py index 373a90c8..dd98d2ed 100644 --- a/backend/ibutsu_server/tasks/results.py +++ b/backend/ibutsu_server/tasks/results.py @@ -1,8 +1,6 @@ from ibutsu_server.db.base import session -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run -from ibutsu_server.tasks import lock -from ibutsu_server.tasks import task +from ibutsu_server.db.models import Result, Run +from ibutsu_server.tasks import lock, task @task diff --git a/backend/ibutsu_server/tasks/runs.py b/backend/ibutsu_server/tasks/runs.py index 0af25165..9e67c891 100644 --- a/backend/ibutsu_server/tasks/runs.py +++ b/backend/ibutsu_server/tasks/runs.py @@ -1,13 +1,9 @@ -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta from ibutsu_server.constants import SYNC_RUN_TIME from ibutsu_server.db.base import session -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run -from ibutsu_server.tasks import lock -from ibutsu_server.tasks import task - +from ibutsu_server.db.models import Result, Run +from ibutsu_server.tasks import lock, task METADATA_TO_COPY = ["jenkins", "tags"] COLUMNS_TO_COPY = ["start_time", "env", "component", "project_id", "source"] diff --git a/backend/ibutsu_server/templating.py b/backend/ibutsu_server/templating.py index fae0efb4..51c5d48b 100644 --- a/backend/ibutsu_server/templating.py +++ b/backend/ibutsu_server/templating.py @@ -1,9 +1,7 @@ from datetime import timedelta from math import ceil -from jinja2 import Environment -from jinja2 import PackageLoader -from jinja2 import select_autoescape +from jinja2 import Environment, PackageLoader, select_autoescape def pretty_duration(duration): diff --git a/backend/ibutsu_server/test/__init__.py b/backend/ibutsu_server/test/__init__.py index 737f1fa4..3b2af7e9 100644 --- a/backend/ibutsu_server/test/__init__.py +++ b/backend/ibutsu_server/test/__init__.py @@ -1,12 +1,12 @@ import logging from inspect import isfunction -import ibutsu_server.tasks from flask_testing import TestCase + +import ibutsu_server.tasks from ibutsu_server import get_app from ibutsu_server.db.base import session -from ibutsu_server.db.models import Token -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Token, User from ibutsu_server.tasks import create_celery_app from ibutsu_server.util import merge_dicts from ibutsu_server.util.jwt import generate_token diff --git a/backend/ibutsu_server/test/test_artifact_controller.py b/backend/ibutsu_server/test/test_artifact_controller.py index 0e252be7..0ea33f26 100644 --- a/backend/ibutsu_server/test/test_artifact_controller.py +++ b/backend/ibutsu_server/test/test_artifact_controller.py @@ -1,12 +1,8 @@ from io import BytesIO from unittest import skip -from unittest.mock import MagicMock -from unittest.mock import patch - -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockArtifact -from ibutsu_server.test import MockResult +from unittest.mock import MagicMock, patch +from ibutsu_server.test import BaseTestCase, MockArtifact, MockResult MOCK_ID = "70202589-4781-4eb9-bcfc-685b1d2c583a" MOCK_RESULT_ID = "23cd86d5-a27e-45a4-83a3-12c74d219709" @@ -64,9 +60,7 @@ def test_delete_artifact(self): Delete an artifact """ headers = {"Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/artifact/{id}".format(id=MOCK_ID), method="DELETE", headers=headers - ) + response = self.client.open(f"/api/artifact/{MOCK_ID}", method="DELETE", headers=headers) self.mock_artifact.query.get.assert_called_once_with(MOCK_ID) self.mock_session.delete.assert_called_once_with(MOCK_ARTIFACT) self.mock_session.commit.assert_called_once() @@ -82,7 +76,7 @@ def test_download_artifact(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/artifact/{id}/download".format(id=MOCK_ID), + f"/api/artifact/{MOCK_ID}/download", method="GET", headers=headers, ) @@ -94,9 +88,12 @@ def test_get_artifact(self): Get a single artifact """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( - "/api/artifact/{id}".format(id=MOCK_ID), + f"/api/artifact/{MOCK_ID}", method="GET", headers=headers, ) @@ -109,7 +106,10 @@ def test_get_artifact_list(self): Get a (filtered) list of artifacts """ query_string = [("resultId", MOCK_RESULT_ID), ("page", 56), ("pageSize", 56)] - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( "/api/artifact", method="GET", headers=headers, query_string=query_string ) diff --git a/backend/ibutsu_server/test/test_group_controller.py b/backend/ibutsu_server/test/test_group_controller.py index f31e54e3..677a7589 100644 --- a/backend/ibutsu_server/test/test_group_controller.py +++ b/backend/ibutsu_server/test/test_group_controller.py @@ -1,9 +1,8 @@ -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockGroup + +from ibutsu_server.test import BaseTestCase, MockGroup MOCK_ID = "c68506e2-202e-4193-a47d-33f1571d4b3e" MOCK_GROUP = MockGroup(id=MOCK_ID, name="Example group", data={}) @@ -58,10 +57,11 @@ def test_get_group(self): Get a group """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/group/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/group/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) self.assert_equal(response.json, MOCK_GROUP_DICT) @@ -71,7 +71,10 @@ def test_get_group_list(self): Get a list of groups """ query_string = [("page", 56), ("pageSize", 56)] - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( "/api/group", method="GET", headers=headers, query_string=query_string ) @@ -80,7 +83,12 @@ def test_get_group_list(self): response.json, { "groups": [MOCK_GROUP_DICT], - "pagination": {"page": 56, "pageSize": 56, "totalItems": 1, "totalPages": 1}, + "pagination": { + "page": 56, + "pageSize": 56, + "totalItems": 1, + "totalPages": 1, + }, }, ) @@ -95,7 +103,7 @@ def test_update_group(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/group/{id}".format(id=MOCK_ID), + f"/api/group/{MOCK_ID}", method="PUT", headers=headers, data=json.dumps({"name": "Changed name"}), diff --git a/backend/ibutsu_server/test/test_importers.py b/backend/ibutsu_server/test/test_importers.py index 069899e0..bee30a87 100644 --- a/backend/ibutsu_server/test/test_importers.py +++ b/backend/ibutsu_server/test/test_importers.py @@ -1,11 +1,8 @@ from pathlib import Path -from unittest.mock import call -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, call, patch from ibutsu_server.test import BaseTestCase - PARENT = Path(__file__).parent XML_FILE = PARENT / "res" / "empty-testsuite.xml" XML_FILE_COMBINED = PARENT / "res" / "combined.xml" @@ -93,7 +90,12 @@ def test_junit_import_non_empty( @patch("ibutsu_server.tasks.importers.session") @patch("ibutsu_server.tasks.importers.get_project_id") def test_junit_import_tests_with_properties( - self, mocked_get_project_id, mocked_session, mocked_update, MockImport, MockImportFile + self, + mocked_get_project_id, + mocked_session, + mocked_update, + MockImport, + MockImportFile, ): """Test the junit importer""" mocked_import = {"id": "12345"} diff --git a/backend/ibutsu_server/test/test_login_controller.py b/backend/ibutsu_server/test/test_login_controller.py index 00a97d07..3a879377 100644 --- a/backend/ibutsu_server/test/test_login_controller.py +++ b/backend/ibutsu_server/test/test_login_controller.py @@ -1,15 +1,20 @@ from unittest.mock import patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockUser + +from ibutsu_server.constants import LOCALHOST +from ibutsu_server.test import BaseTestCase, MockUser from ibutsu_server.util.jwt import generate_token MOCK_ID = "6f7c2d52-54dc-4309-8e2e-c74515d39455" MOCK_EMAIL = "test@example.com" MOCK_PASSWORD = "my super secret password" MOCK_USER = MockUser( - id=MOCK_ID, email=MOCK_EMAIL, password=MOCK_PASSWORD, name="Test User", is_superadmin=False + id=MOCK_ID, + email=MOCK_EMAIL, + password=MOCK_PASSWORD, + name="Test User", + is_superadmin=False, ) @@ -35,7 +40,11 @@ def test_login(self, mocked_generate_token): login_details = {"email": MOCK_EMAIL, "password": MOCK_PASSWORD} expected_token = generate_token(MOCK_ID) mocked_generate_token.return_value = expected_token - expected_response = {"name": "Test User", "email": MOCK_EMAIL, "token": expected_token} + expected_response = { + "name": "Test User", + "email": MOCK_EMAIL, + "token": expected_token, + } headers = {"Accept": "application/json", "Content-Type": "application/json"} response = self.client.open( "/api/login", @@ -76,7 +85,10 @@ def test_login_no_user(self): Log in to the API """ login_details = {"email": "bad@email.com", "password": MOCK_PASSWORD} - expected_response = {"code": "INVALID", "message": "Username and/or password are invalid"} + expected_response = { + "code": "INVALID", + "message": "Username and/or password are invalid", + } headers = {"Accept": "application/json", "Content-Type": "application/json"} self.mock_user.query.filter_by.return_value.first.return_value = None response = self.client.open( @@ -95,7 +107,10 @@ def test_login_bad_password(self): Log in to the API """ login_details = {"email": MOCK_EMAIL, "password": "bad password"} - expected_response = {"code": "INVALID", "message": "Username and/or password are invalid"} + expected_response = { + "code": "INVALID", + "message": "Username and/or password are invalid", + } headers = {"Accept": "application/json", "Content-Type": "application/json"} response = self.client.open( "/api/login", @@ -132,7 +147,7 @@ def test_config_gitlab(self): expected_response = { "authorization_url": "https://gitlab.com/oauth/authorize", "client_id": "dfgfdgh4563453456dsfgdsfg456", - "redirect_uri": "http://localhost:8080/api/login/auth/gitlab", + "redirect_uri": f"http://{LOCALHOST}:8080/api/login/auth/gitlab", "scope": "read_user", } headers = {"Accept": "application/json", "Content-Type": "application/json"} diff --git a/backend/ibutsu_server/test/test_project_controller.py b/backend/ibutsu_server/test/test_project_controller.py index 241c0927..0a18da68 100644 --- a/backend/ibutsu_server/test/test_project_controller.py +++ b/backend/ibutsu_server/test/test_project_controller.py @@ -1,10 +1,8 @@ -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockProject -from ibutsu_server.test import MockUser + +from ibutsu_server.test import BaseTestCase, MockProject, MockUser MOCK_ID = "5ac7d645-45a3-4cbe-acb2-c8d6f7e05468" MOCK_NAME = "my-project" @@ -95,9 +93,7 @@ def test_get_project_by_id(self): "Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}", } - response = self.client.open( - "/api/project/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + response = self.client.open(f"/api/project/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) self.assert_equal(response.json, MOCK_PROJECT_DICT) self.mock_project.query.get.assert_called_once_with(MOCK_ID) @@ -112,9 +108,7 @@ def test_get_project_by_name(self): "Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}", } - response = self.client.open( - "/api/project/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + response = self.client.open(f"/api/project/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) self.assert_equal(response.json, MOCK_PROJECT_DICT) @@ -163,7 +157,7 @@ def test_update_project(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/project/{id}".format(id=MOCK_ID), + f"/api/project/{MOCK_ID}", method="PUT", headers=headers, data=json.dumps(updates), diff --git a/backend/ibutsu_server/test/test_report_controller.py b/backend/ibutsu_server/test/test_report_controller.py index e644d788..9ab9c45a 100644 --- a/backend/ibutsu_server/test/test_report_controller.py +++ b/backend/ibutsu_server/test/test_report_controller.py @@ -1,9 +1,8 @@ -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockReport + +from ibutsu_server.test import BaseTestCase, MockReport MOCK_ID = "751162a7-b0e0-448e-9af3-676d1a48b0ca" MOCK_PARAMS = {"type": "csv", "source": "local"} @@ -65,10 +64,11 @@ def test_get_report(self): Get a report """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/report/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/report/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) assert response.json == MOCK_REPORT_DICT @@ -82,7 +82,10 @@ def test_get_report_list(self): mock_limit.return_value.all.return_value = [MOCK_REPORT] self.mock_report.query.order_by.return_value.offset.return_value.limit = mock_limit query_string = [("page", 56), ("pageSize", 56)] - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } with patch( "ibutsu_server.controllers.report_controller.get_project_id" ) as mocked_get_project_id: @@ -92,7 +95,12 @@ def test_get_report_list(self): ) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) expected_response = { - "pagination": {"page": 56, "pageSize": 56, "totalItems": 1, "totalPages": 1}, + "pagination": { + "page": 56, + "pageSize": 56, + "totalItems": 1, + "totalPages": 1, + }, "reports": [MOCK_REPORT_DICT], } assert response.json == expected_response diff --git a/backend/ibutsu_server/test/test_result_controller.py b/backend/ibutsu_server/test/test_result_controller.py index 93abfe07..1b6a89a5 100644 --- a/backend/ibutsu_server/test/test_result_controller.py +++ b/backend/ibutsu_server/test/test_result_controller.py @@ -1,11 +1,9 @@ from datetime import datetime -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockProject -from ibutsu_server.test import MockResult + +from ibutsu_server.test import BaseTestCase, MockProject, MockResult MOCK_ID = "99fba7d2-4d32-4b9b-b07f-4200c9717661" START_TIME = datetime.utcnow() @@ -123,10 +121,11 @@ def test_get_result(self): Get a single result """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/result/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/result/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) assert response.json == MOCK_RESULT_DICT @@ -140,14 +139,26 @@ def test_get_result_list(self): mock_filter.order_by.return_value.offset.return_value.limit.return_value.all = mock_all mock_filter.count.return_value = 1 self.mock_add_user_filter.return_value.filter.return_value = mock_filter - query_string = [("filter", "metadata.component=frontend"), ("page", 56), ("pageSize", 56)] - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + query_string = [ + ("filter", "metadata.component=frontend"), + ("page", 56), + ("pageSize", 56), + ] + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( "/api/result", method="GET", headers=headers, query_string=query_string ) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) expected_response = { - "pagination": {"page": 56, "pageSize": 56, "totalItems": 1, "totalPages": 1}, + "pagination": { + "page": 56, + "pageSize": 56, + "totalItems": 1, + "totalPages": 1, + }, "results": [MOCK_RESULT_DICT], } assert response.json == expected_response @@ -173,7 +184,7 @@ def test_update_result(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/result/{id}".format(id=MOCK_ID), + f"/api/result/{MOCK_ID}", method="PUT", headers=headers, data=json.dumps(result), diff --git a/backend/ibutsu_server/test/test_run_controller.py b/backend/ibutsu_server/test/test_run_controller.py index 30f4ba33..b7193736 100644 --- a/backend/ibutsu_server/test/test_run_controller.py +++ b/backend/ibutsu_server/test/test_run_controller.py @@ -1,12 +1,11 @@ from datetime import datetime from io import BytesIO from unittest import skip -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockRun + +from ibutsu_server.test import BaseTestCase, MockRun from ibutsu_server.test.test_project_controller import MOCK_PROJECT MOCK_ID = "6b26876f-bcd9-49f3-b5bd-35f895a345d1" @@ -71,7 +70,11 @@ def test_add_run(self): run_dict = { "summary": {"errors": 1, "failures": 3, "skips": 0, "tests": 548}, "duration": 540.05433, - "metadata": {"component": "test-component", "env": "local", "project": MOCK_PROJECT.id}, + "metadata": { + "component": "test-component", + "env": "local", + "project": MOCK_PROJECT.id, + }, "start_time": START_TIME, "created": START_TIME, } @@ -99,10 +102,11 @@ def test_get_run(self): Get a single run by ID """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/run/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/run/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) resp = response.json.copy() resp["project"] = None @@ -114,7 +118,10 @@ def test_get_run_list(self): Get a list of the test runs """ query_string = [("page", 56), ("pageSize", 56)] - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( "/api/run", method="GET", headers=headers, query_string=query_string ) @@ -156,7 +163,7 @@ def test_update_run(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/run/{id}".format(id=MOCK_ID), + f"/api/run/{MOCK_ID}", method="PUT", headers=headers, data=json.dumps(run_dict), diff --git a/backend/ibutsu_server/test/test_tasks.py b/backend/ibutsu_server/test/test_tasks.py index 8250c19a..735e3bff 100644 --- a/backend/ibutsu_server/test/test_tasks.py +++ b/backend/ibutsu_server/test/test_tasks.py @@ -1,8 +1,7 @@ import time import uuid from datetime import datetime -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from ibutsu_server.test import BaseTestCase diff --git a/backend/ibutsu_server/test/test_widget_config_controller.py b/backend/ibutsu_server/test/test_widget_config_controller.py index 4987ca40..225c0ebf 100644 --- a/backend/ibutsu_server/test/test_widget_config_controller.py +++ b/backend/ibutsu_server/test/test_widget_config_controller.py @@ -1,9 +1,8 @@ -from unittest.mock import MagicMock -from unittest.mock import patch +from unittest.mock import MagicMock, patch from flask import json -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockWidgetConfig + +from ibutsu_server.test import BaseTestCase, MockWidgetConfig # from ibutsu_server.test import MockDashboard # from ibutsu_server.test import MockProject @@ -101,10 +100,11 @@ def test_get_widget_config(self): Get a single widget_config """ - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/widget-config/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/widget-config/{MOCK_ID}", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) assert response.json == MOCK_WIDGET_CONFIG_DICT @@ -114,10 +114,11 @@ def test_get_widget_config_404(self): Return a 404 when no widget config is found """ self.mock_widget_config.query.get.return_value = None - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} - response = self.client.open( - "/api/widget-config/{id}".format(id=MOCK_ID), method="GET", headers=headers - ) + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } + response = self.client.open(f"/api/widget-config/{MOCK_ID}", method="GET", headers=headers) self.assert_404(response, "Response body is : " + response.data.decode("utf-8")) def test_get_widget_config_list(self): @@ -129,7 +130,10 @@ def test_get_widget_config_list(self): mock_query = self.mock_widget_config.query mock_query.order_by.return_value.offset.return_value.limit.return_value.all = mock_all mock_query.count.return_value = 1 - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open("/api/widget-config", method="GET", headers=headers) self.assert_200(response, "Response body is : " + response.data.decode("utf-8")) expected_response = { @@ -154,7 +158,7 @@ def test_update_widget_config(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/widget-config/{id}".format(id=MOCK_ID), + f"/api/widget-config/{MOCK_ID}", method="PUT", headers=headers, data=json.dumps(widget_config), @@ -171,7 +175,7 @@ def test_delete_widget_config(self): "Authorization": f"Bearer {self.jwt_token}", } response = self.client.open( - "/api/widget-config/{id}".format(id=MOCK_ID), + f"/api/widget-config/{MOCK_ID}", method="DELETE", headers=headers, ) diff --git a/backend/ibutsu_server/test/test_widget_controller.py b/backend/ibutsu_server/test/test_widget_controller.py index cafb6a38..cb557a64 100644 --- a/backend/ibutsu_server/test/test_widget_controller.py +++ b/backend/ibutsu_server/test/test_widget_controller.py @@ -1,9 +1,7 @@ from datetime import datetime from unittest.mock import patch -from ibutsu_server.test import BaseTestCase -from ibutsu_server.test import MockProject -from ibutsu_server.test import MockResult +from ibutsu_server.test import BaseTestCase, MockProject, MockResult MOCK_ID_1 = "99fba7d2-4d32-4b9b-b07f-4200c9717661" MOCK_ID_2 = "99fba7d2-4d32-4b9b-b07f-4200c9717662" @@ -72,7 +70,10 @@ def test_get_comparison_result_list(self, mocked_query): query_string = { "filters": ["metadata.component=frontend", "metadata.component=frontend"], } - headers = {"Accept": "application/json", "Authorization": f"Bearer {self.jwt_token}"} + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + } response = self.client.open( "/api/widget/compare-runs-view", method="GET", diff --git a/backend/ibutsu_server/test/test_widgets_result_summary.py b/backend/ibutsu_server/test/test_widgets_result_summary.py index 9e82943a..59f53edc 100644 --- a/backend/ibutsu_server/test/test_widgets_result_summary.py +++ b/backend/ibutsu_server/test/test_widgets_result_summary.py @@ -1,6 +1,4 @@ -from unittest.mock import MagicMock -from unittest.mock import Mock -from unittest.mock import patch +from unittest.mock import MagicMock, Mock, patch from ibutsu_server.db.models import Run from ibutsu_server.widgets.result_summary import get_result_summary diff --git a/backend/ibutsu_server/util/admin.py b/backend/ibutsu_server/util/admin.py index 627abbfa..7d024b26 100644 --- a/backend/ibutsu_server/util/admin.py +++ b/backend/ibutsu_server/util/admin.py @@ -1,4 +1,5 @@ from flask import abort + from ibutsu_server.db.models import User diff --git a/backend/ibutsu_server/util/count.py b/backend/ibutsu_server/util/count.py index 0b73dfeb..06049953 100644 --- a/backend/ibutsu_server/util/count.py +++ b/backend/ibutsu_server/util/count.py @@ -1,8 +1,8 @@ -""" Utility functions for counting rows in large tables""" +"""Utility functions for counting rows in large tables""" + from contextlib import contextmanager -from ibutsu_server.constants import COUNT_ESTIMATE_LIMIT -from ibutsu_server.constants import COUNT_TIMEOUT +from ibutsu_server.constants import COUNT_ESTIMATE_LIMIT, COUNT_TIMEOUT from ibutsu_server.db.base import session from ibutsu_server.db.util import Explain diff --git a/backend/ibutsu_server/util/jwt.py b/backend/ibutsu_server/util/jwt.py index c9828af7..f704c561 100644 --- a/backend/ibutsu_server/util/jwt.py +++ b/backend/ibutsu_server/util/jwt.py @@ -1,14 +1,14 @@ import time from flask import current_app -from ibutsu_server.db.models import Token -from ibutsu_server.errors import IbutsuError from jose.constants import ALGORITHMS from jose.exceptions import JWTError from jose.jwt import decode as jwt_decode from jose.jwt import encode as jwt_encode from werkzeug.exceptions import Unauthorized +from ibutsu_server.db.models import Token +from ibutsu_server.errors import IbutsuError JWT_ISSUER = "org.ibutsu-project.server" JWT_SECRET = "" diff --git a/backend/ibutsu_server/util/keycloak.py b/backend/ibutsu_server/util/keycloak.py index d7da5e54..61a46e01 100644 --- a/backend/ibutsu_server/util/keycloak.py +++ b/backend/ibutsu_server/util/keycloak.py @@ -1,5 +1,7 @@ import requests from flask import current_app + +from ibutsu_server.constants import LOCALHOST from ibutsu_server.db.base import session from ibutsu_server.db.models import User from ibutsu_server.util.urls import build_url @@ -11,7 +13,7 @@ def get_keycloak_config(is_private=False): "KEYCLOAK_BASE_URL" ): return {} - backend_url = current_app.config.get("BACKEND_URL", "http://localhost:8080/api") + backend_url = current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080/api") if not backend_url.endswith("/api"): backend_url += "/api" server_url = current_app.config["KEYCLOAK_BASE_URL"] diff --git a/backend/ibutsu_server/util/oauth.py b/backend/ibutsu_server/util/oauth.py index ad49005d..c3740d8a 100644 --- a/backend/ibutsu_server/util/oauth.py +++ b/backend/ibutsu_server/util/oauth.py @@ -1,6 +1,7 @@ import requests from flask import current_app -from ibutsu_server.constants import OAUTH_CONFIG + +from ibutsu_server.constants import LOCALHOST, OAUTH_CONFIG from ibutsu_server.db.base import session from ibutsu_server.db.models import User from ibutsu_server.util.urls import build_url @@ -8,7 +9,7 @@ def get_provider_config(provider, is_private=False): """Return the customised config for a provider""" - backend_url = current_app.config.get("BACKEND_URL", "http://localhost:8080/api") + backend_url = current_app.config.get("BACKEND_URL", f"http://{LOCALHOST}:8080/api") provider_upper = provider.upper() server_url = current_app.config.get(f"{provider_upper}_BASE_URL") provider_config = OAUTH_CONFIG.get(provider, {}) @@ -44,7 +45,11 @@ def get_user_from_provider(provider, auth_data): """Get a user object from the ``provider``, using the ``auth_data``""" provider_config = get_provider_config(provider, is_private=True) if provider == "google": - user_dict = {"id": auth_data["iat"], "email": auth_data["email"], "name": auth_data["name"]} + user_dict = { + "id": auth_data["iat"], + "email": auth_data["email"], + "name": auth_data["name"], + } else: access_token = auth_data.get("accessToken", auth_data.get("access_token")) response = requests.get( @@ -61,7 +66,8 @@ def get_user_from_provider(provider, auth_data): # to make another request to get the e-mail address, see the this answer for more info: # https://stackoverflow.com/a/35387123 response = requests.get( - provider_config["email_url"], headers={"Authorization": f"Bearer {access_token}"} + provider_config["email_url"], + headers={"Authorization": f"Bearer {access_token}"}, ) if response.status_code == 200: emails = response.json() diff --git a/backend/ibutsu_server/util/projects.py b/backend/ibutsu_server/util/projects.py index db20f8bc..742c2920 100644 --- a/backend/ibutsu_server/util/projects.py +++ b/backend/ibutsu_server/util/projects.py @@ -1,5 +1,4 @@ -from ibutsu_server.db.models import Project -from ibutsu_server.db.models import User +from ibutsu_server.db.models import Project, User from ibutsu_server.util.uuid import is_uuid diff --git a/backend/ibutsu_server/util/query.py b/backend/ibutsu_server/util/query.py index 6b2dc633..1c6545fd 100644 --- a/backend/ibutsu_server/util/query.py +++ b/backend/ibutsu_server/util/query.py @@ -1,4 +1,5 @@ -""" Query utilities""" +"""Query utilities""" + import re from ibutsu_server.constants import MAX_PAGE_SIZE diff --git a/backend/ibutsu_server/util/uuid.py b/backend/ibutsu_server/util/uuid.py index f45b88dc..369fac2e 100644 --- a/backend/ibutsu_server/util/uuid.py +++ b/backend/ibutsu_server/util/uuid.py @@ -1,10 +1,8 @@ -from datetime import datetime -from datetime import timezone +from datetime import datetime, timezone from uuid import UUID from bson import ObjectId - UUID_1_EPOCH = datetime(1582, 10, 15, tzinfo=timezone.utc) UUID_TICKS = 10000000 UUID_VARIANT_1 = 0b1000000000000000 @@ -40,13 +38,9 @@ def convert_objectid_to_uuid(object_id): hex_string = str(object_id) counter = int(hex_string[18:], 16) - uuid_time = "1{:015x}".format( - int((unix_time + (unix_time - UUID_1_EPOCH)).timestamp() * UUID_TICKS) - ) - uuid_clock = "{:04x}".format(UUID_VARIANT_1 | (counter & 0x3FFF)) + uuid_time = f"1{int((unix_time + (unix_time - UUID_1_EPOCH)).timestamp() * UUID_TICKS):015x}" + uuid_clock = f"{UUID_VARIANT_1 | (counter & 0x3FFF):04x}" uuid_node = "1" + hex_string[8:18].rjust(11, "0") - string_uuid = "{}-{}-{}-{}-{}".format( - uuid_time[-8:], uuid_time[4:8], uuid_time[:4], uuid_clock, uuid_node - ) + string_uuid = f"{uuid_time[-8:]}-{uuid_time[4:8]}-{uuid_time[:4]}-{uuid_clock}-{uuid_node}" converted_uuid = UUID(string_uuid) return str(converted_uuid) diff --git a/backend/ibutsu_server/widgets/accessibility_analysis.py b/backend/ibutsu_server/widgets/accessibility_analysis.py index dcdb842d..720c887a 100644 --- a/backend/ibutsu_server/widgets/accessibility_analysis.py +++ b/backend/ibutsu_server/widgets/accessibility_analysis.py @@ -1,4 +1,5 @@ import yaml + from ibutsu_server.db.models import Artifact diff --git a/backend/ibutsu_server/widgets/filter_heatmap.py b/backend/ibutsu_server/widgets/filter_heatmap.py index cc76fc9a..dd08cf87 100644 --- a/backend/ibutsu_server/widgets/filter_heatmap.py +++ b/backend/ibutsu_server/widgets/filter_heatmap.py @@ -1,13 +1,9 @@ -from ibutsu_server.constants import HEATMAP_MAX_BUILDS -from ibutsu_server.constants import HEATMAP_RUN_LIMIT -from ibutsu_server.db.base import Float -from ibutsu_server.db.base import session +from sqlalchemy import case, desc, func + +from ibutsu_server.constants import HEATMAP_MAX_BUILDS, HEATMAP_RUN_LIMIT +from ibutsu_server.db.base import Float, session from ibutsu_server.db.models import Run -from ibutsu_server.filters import apply_filters -from ibutsu_server.filters import string_to_column -from sqlalchemy import case -from sqlalchemy import desc -from sqlalchemy import func +from ibutsu_server.filters import apply_filters, string_to_column NO_RUN_TEXT = "None" NO_PASS_RATE_TEXT = "Build failed" diff --git a/backend/ibutsu_server/widgets/importance_component.py b/backend/ibutsu_server/widgets/importance_component.py index 6f7c03b3..1a203c53 100644 --- a/backend/ibutsu_server/widgets/importance_component.py +++ b/backend/ibutsu_server/widgets/importance_component.py @@ -1,18 +1,24 @@ -from ibutsu_server.constants import BARCHART_MAX_BUILDS -from ibutsu_server.constants import JJV_RUN_LIMIT -from ibutsu_server.db.models import Result -from ibutsu_server.db.models import Run +from ibutsu_server.constants import BARCHART_MAX_BUILDS, JJV_RUN_LIMIT +from ibutsu_server.db.models import Result, Run from ibutsu_server.filters import string_to_column from ibutsu_server.widgets.jenkins_job_view import get_jenkins_job_view def get_importance_component( - env="prod", group_field="component", job_name="", builds=5, components="", project=None + env="prod", + group_field="component", + job_name="", + builds=5, + components="", + project=None, ): # taken from get_jenkins_line_chart in jenkins_job_analysis.py run_limit = int((JJV_RUN_LIMIT / BARCHART_MAX_BUILDS) * builds) jobs = get_jenkins_job_view( - filter_=f"job_name={job_name}", page_size=builds, project=project, run_limit=run_limit + filter_=f"job_name={job_name}", + page_size=builds, + project=project, + run_limit=run_limit, ).get("jobs") # A list of job build numbers to filter our runs by @@ -38,10 +44,15 @@ def get_importance_component( mdat = string_to_column("metadata.importance", Result) result_data = ( Result.query.filter( - Result.run_id.in_(run_info.keys()), Result.component.in_(components.split(",")) + Result.run_id.in_(run_info.keys()), + Result.component.in_(components.split(",")), ) .add_columns( - Result.run_id, Result.component, Result.id, Result.result, mdat.label("importance") + Result.run_id, + Result.component, + Result.id, + Result.result, + mdat.label("importance"), ) .all() ) @@ -108,7 +119,10 @@ def get_importance_component( if bnum not in sdatdict[component].keys(): sdatdict[component][bnum] = {} for importance in importances: - sdatdict[component][bnum][importance] = {"percentage": "NA", "result_list": []} + sdatdict[component][bnum][importance] = { + "percentage": "NA", + "result_list": [], + } # Need this broken down more for the table table_data = [] diff --git a/backend/ibutsu_server/widgets/jenkins_heatmap.py b/backend/ibutsu_server/widgets/jenkins_heatmap.py index cc3d3036..f0e5c9ea 100644 --- a/backend/ibutsu_server/widgets/jenkins_heatmap.py +++ b/backend/ibutsu_server/widgets/jenkins_heatmap.py @@ -1,14 +1,9 @@ -from ibutsu_server.constants import HEATMAP_MAX_BUILDS -from ibutsu_server.constants import HEATMAP_RUN_LIMIT -from ibutsu_server.db.base import Float -from ibutsu_server.db.base import Integer -from ibutsu_server.db.base import session +from sqlalchemy import case, desc, func + +from ibutsu_server.constants import HEATMAP_MAX_BUILDS, HEATMAP_RUN_LIMIT +from ibutsu_server.db.base import Float, Integer, session from ibutsu_server.db.models import Run -from ibutsu_server.filters import apply_filters -from ibutsu_server.filters import string_to_column -from sqlalchemy import case -from sqlalchemy import desc -from sqlalchemy import func +from ibutsu_server.filters import apply_filters, string_to_column NO_RUN_TEXT = "None" NO_PASS_RATE_TEXT = "Build failed" @@ -37,7 +32,10 @@ def _calculate_slope(x_data): def _get_builds(job_name, builds, project=None, additional_filters=None): - filters = [f"metadata.jenkins.job_name={job_name}", "metadata.jenkins.build_number@y"] + filters = [ + f"metadata.jenkins.job_name={job_name}", + "metadata.jenkins.build_number@y", + ] if additional_filters: filters.extend(additional_filters.split(",")) if project: @@ -162,7 +160,12 @@ def _get_heatmap(job_name, builds, group_field, count_skips, project=None, addit data = {datum.group_field: [] for datum in query_data} for datum in query_data: data[datum.group_field].append( - [round(datum.pass_percent, 2), datum.run_id, datum.annotations, datum.build_number] + [ + round(datum.pass_percent, 2), + datum.run_id, + datum.annotations, + datum.build_number, + ] ) # compute the slope for each component data_with_slope = data.copy() @@ -198,7 +201,12 @@ def _pad_heatmap(heatmap, builds_in_db): def get_jenkins_heatmap( - job_name, builds, group_field, count_skips=False, project=None, additional_filters=None + job_name, + builds, + group_field, + count_skips=False, + project=None, + additional_filters=None, ): """Generate JSON data for a heatmap of Jenkins runs""" heatmap, builds_in_db = _get_heatmap( diff --git a/backend/ibutsu_server/widgets/jenkins_job_analysis.py b/backend/ibutsu_server/widgets/jenkins_job_analysis.py index 081399a2..a1e116ae 100644 --- a/backend/ibutsu_server/widgets/jenkins_job_analysis.py +++ b/backend/ibutsu_server/widgets/jenkins_job_analysis.py @@ -1,6 +1,8 @@ -from ibutsu_server.constants import BARCHART_MAX_BUILDS -from ibutsu_server.constants import HEATMAP_MAX_BUILDS -from ibutsu_server.constants import JJV_RUN_LIMIT +from ibutsu_server.constants import ( + BARCHART_MAX_BUILDS, + HEATMAP_MAX_BUILDS, + JJV_RUN_LIMIT, +) from ibutsu_server.widgets.jenkins_job_view import get_jenkins_job_view @@ -8,7 +10,10 @@ def get_jenkins_line_chart(job_name, builds, group_field="build_number", project data = {"duration": {}} run_limit = int((JJV_RUN_LIMIT / BARCHART_MAX_BUILDS) * builds) jobs = get_jenkins_job_view( - filter_=f"job_name={job_name}", page_size=builds, project=project, run_limit=run_limit + filter_=f"job_name={job_name}", + page_size=builds, + project=project, + run_limit=run_limit, ).get("jobs") # first determine duration differs from total_execution_time run_had_multiple_components = any( @@ -29,10 +34,20 @@ def get_jenkins_line_chart(job_name, builds, group_field="build_number", project def get_jenkins_bar_chart(job_name, builds, group_field="build_number", project=None): - data = {"passed": {}, "skipped": {}, "error": {}, "failed": {}, "xpassed": {}, "xfailed": {}} + data = { + "passed": {}, + "skipped": {}, + "error": {}, + "failed": {}, + "xpassed": {}, + "xfailed": {}, + } run_limit = int((JJV_RUN_LIMIT / BARCHART_MAX_BUILDS) * builds) jobs = get_jenkins_job_view( - filter_=f"job_name={job_name}", page_size=builds, project=project, run_limit=run_limit + filter_=f"job_name={job_name}", + page_size=builds, + project=project, + run_limit=run_limit, ).get("jobs") for job in jobs: data_id = job.get(group_field) diff --git a/backend/ibutsu_server/widgets/jenkins_job_view.py b/backend/ibutsu_server/widgets/jenkins_job_view.py index 76dc8e47..3ba5ef94 100644 --- a/backend/ibutsu_server/widgets/jenkins_job_view.py +++ b/backend/ibutsu_server/widgets/jenkins_job_view.py @@ -1,12 +1,9 @@ +from sqlalchemy import desc, func + from ibutsu_server.constants import JJV_RUN_LIMIT -from ibutsu_server.db.base import Integer -from ibutsu_server.db.base import session -from ibutsu_server.db.base import Text +from ibutsu_server.db.base import Integer, Text, session from ibutsu_server.db.models import Run -from ibutsu_server.filters import apply_filters -from ibutsu_server.filters import string_to_column -from sqlalchemy import desc -from sqlalchemy import func +from ibutsu_server.filters import apply_filters, string_to_column def _get_jenkins_aggregation(filters=None, project=None, page=1, page_size=25, run_limit=None): diff --git a/backend/ibutsu_server/widgets/result_aggregator.py b/backend/ibutsu_server/widgets/result_aggregator.py index 7bac8b64..5c8ffef4 100644 --- a/backend/ibutsu_server/widgets/result_aggregator.py +++ b/backend/ibutsu_server/widgets/result_aggregator.py @@ -1,13 +1,11 @@ import time -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta + +from sqlalchemy import desc, func from ibutsu_server.db.base import session from ibutsu_server.db.models import Result -from ibutsu_server.filters import apply_filters -from ibutsu_server.filters import string_to_column -from sqlalchemy import desc -from sqlalchemy import func +from ibutsu_server.filters import apply_filters, string_to_column def _get_min_count(days): @@ -56,7 +54,12 @@ def _get_recent_result_data(group_field, days, project=None, run_id=None, additi def get_recent_result_data( - group_field, days=None, project=None, chart_type="pie", run_id=None, additional_filters=None + group_field, + days=None, + project=None, + chart_type="pie", + run_id=None, + additional_filters=None, ): data = _get_recent_result_data( group_field, diff --git a/backend/ibutsu_server/widgets/result_summary.py b/backend/ibutsu_server/widgets/result_summary.py index 05f337ac..c62d327f 100644 --- a/backend/ibutsu_server/widgets/result_summary.py +++ b/backend/ibutsu_server/widgets/result_summary.py @@ -1,8 +1,8 @@ -from ibutsu_server.db.base import Integer -from ibutsu_server.db.base import session +from sqlalchemy import func + +from ibutsu_server.db.base import Integer, session from ibutsu_server.db.models import Run from ibutsu_server.filters import apply_filters -from sqlalchemy import func PAGE_SIZE = 250 diff --git a/backend/ibutsu_server/widgets/run_aggregator.py b/backend/ibutsu_server/widgets/run_aggregator.py index 6bb9de58..6345999a 100644 --- a/backend/ibutsu_server/widgets/run_aggregator.py +++ b/backend/ibutsu_server/widgets/run_aggregator.py @@ -1,24 +1,32 @@ import time -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta -from ibutsu_server.db.base import Float -from ibutsu_server.db.base import session -from ibutsu_server.db.models import Run -from ibutsu_server.filters import apply_filters -from ibutsu_server.filters import string_to_column from sqlalchemy import func +from ibutsu_server.db.base import Float, session +from ibutsu_server.db.models import Run +from ibutsu_server.filters import apply_filters, string_to_column + def _get_recent_run_data(weeks, group_field, project=None, additional_filters=None): """Get all the data from the time period and aggregate the results""" - data = {"passed": {}, "skipped": {}, "error": {}, "failed": {}, "xfailed": {}, "xpassed": {}} + data = { + "passed": {}, + "skipped": {}, + "error": {}, + "failed": {}, + "xfailed": {}, + "xpassed": {}, + } delta = timedelta(weeks=weeks).total_seconds() current_time = time.time() time_period_in_sec = current_time - delta # create filters for start time and that the group_field exists - filters = [f"start_time>{datetime.utcfromtimestamp(time_period_in_sec)}", f"{group_field}@y"] + filters = [ + f"start_time>{datetime.utcfromtimestamp(time_period_in_sec)}", + f"{group_field}@y", + ] if additional_filters: filters.extend(additional_filters.split(",")) if project: diff --git a/backend/main.py b/backend/main.py index bf5b6b64..dfc7b37e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ """This file provides an entry point for the Docker container""" + from ibutsu_server import get_app app = get_app() diff --git a/backend/setup.cfg b/backend/setup.cfg index 1181a9c1..599d872a 100644 --- a/backend/setup.cfg +++ b/backend/setup.cfg @@ -3,8 +3,3 @@ zip_safe = False include_package_data = True packages = find: entry_points = file:entry_points.txt - -[flake8] -ignore = W503,E203 -exclude = .git,__pycache__,build,dist -max-line-length = 100 diff --git a/backend/setup.py b/backend/setup.py index 9e5ac956..176d38bb 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -1,5 +1,4 @@ -from setuptools import find_packages -from setuptools import setup +from setuptools import find_packages, setup NAME = "ibutsu_server" VERSION = "2.5.8" diff --git a/backend/wsgi.py b/backend/wsgi.py index 99bc5222..bf498343 100644 --- a/backend/wsgi.py +++ b/backend/wsgi.py @@ -1,4 +1,5 @@ """This file provides an entry point for the OpenShift container""" + from ibutsu_server import get_app application = get_app() diff --git a/docs/source/conf.py b/docs/source/conf.py index 4ca1727c..4103ba92 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,7 +64,10 @@ "font-stack": "'Red Hat Text', 'Lato', 'proxima-nova', 'Helvetica Neue', Arial, sans-serif", "font-stack-monospace": "'Hack', monospace", }, - "dark_css_variables": {"color-brand-primary": "#3f9c35", "color-brand-content": "#3f9c35"}, + "dark_css_variables": { + "color-brand-primary": "#3f9c35", + "color-brand-content": "#3f9c35", + }, } pygments_style = "gruvbox-light" pygments_dark_style = "gruvbox-dark" diff --git a/frontend/docker/Dockerfile.frontend b/frontend/docker/Dockerfile.frontend index d3ec4eb2..b79ccfef 100644 --- a/frontend/docker/Dockerfile.frontend +++ b/frontend/docker/Dockerfile.frontend @@ -1,32 +1,36 @@ +# hadolint global ignore=DL3041,DL4006 # Build the image first -FROM registry.access.redhat.com/ubi8/ubi-minimal AS builder +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10 AS builder -ARG NODEJS_VERSION=18 +ARG NODEJS_VERSION=16 ENV NODE_ENV=production WORKDIR /app -RUN INSTALL_PKGS="nodejs nodejs-nodemon npm findutils tar" && \ - microdnf module disable nodejs && \ +RUN microdnf module disable nodejs && \ microdnf module enable nodejs:$NODEJS_VERSION && \ - microdnf install --nodocs -y --disableplugin=subscription-manager $INSTALL_PKGS && \ - node -v | grep -qe "^v$NODEJS_VERSION\." && echo "Found VERSION $NODEJS_VERSION" && \ + microdnf install --nodocs -y --disableplugin=subscription-manager \ + nodejs nodejs-nodemon npm findutils tar && \ + node -v | grep -qe "^v$NODEJS_VERSION\." && \ + echo "Found VERSION $NODEJS_VERSION" && \ microdnf clean all && \ rm -rf /mnt/rootfs/var/cache/* /mnt/rootfs/var/log/dnf* /mnt/rootfs/var/log/yum.* + COPY . /app -RUN npm install -RUN npm run build +RUN npm install && \ + npm run build # Deploy it using nginx -FROM registry.access.redhat.com/ubi8/ubi-minimal +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10 ARG NGINX_VERSION=1.20 # rpm -V $INSTALL_PKGS && \ -RUN INSTALL_PKGS="nss_wrapper bind-utils gettext hostname nginx nginx-mod-stream nginx-mod-http-perl" && \ - microdnf -y module enable nginx:$NGINX_VERSION && \ - microdnf install -y --nodocs --disableplugin=subscription-manager $INSTALL_PKGS && \ - nginx -v 2>&1 | grep -qe "nginx/$NGINX_VERSION\." && echo "Found VERSION $NGINX_VERSION" && \ +RUN microdnf -y module enable nginx:$NGINX_VERSION && \ + microdnf install -y --nodocs --disableplugin=subscription-manager \ + nss_wrapper bind-utils gettext hostname nginx nginx-mod-stream nginx-mod-http-perl && \ + nginx -v 2>&1 | grep -qe "nginx/$NGINX_VERSION\." && \ + echo "Found VERSION $NGINX_VERSION" && \ microdnf clean all && \ rm -rf /mnt/rootfs/var/cache/* /mnt/rootfs/var/log/dnf* /mnt/rootfs/var/log/yum.* diff --git a/frontend/public/settings.js b/frontend/public/settings.js index f9d30c70..72756650 100644 --- a/frontend/public/settings.js +++ b/frontend/public/settings.js @@ -1,6 +1,6 @@ // Public settings for the application. If running in a container, this file will be overwritten // at container runtime with ENV variables. window.settings = { - serverUrl: 'http://localhost:8080/api', + serverUrl: 'http://127.0.0.1:8080/api', environment: 'development' } diff --git a/frontend/src/components/ibutsu-header.js b/frontend/src/components/ibutsu-header.js index 7d190ff5..db82c3c3 100644 --- a/frontend/src/components/ibutsu-header.js +++ b/frontend/src/components/ibutsu-header.js @@ -7,7 +7,6 @@ import { Brand, Button, Flex, - FlexItem, PageHeader, Select, SelectOption, @@ -18,7 +17,9 @@ import { Switch, TextContent, TextList, - TextListItem + TextListItem, + FlexItem, + Divider, } from '@patternfly/react-core'; import { UploadIcon, ServerIcon, QuestionCircleIcon, MoonIcon } from '@patternfly/react-icons'; import { css } from '@patternfly/react-styles'; @@ -30,8 +31,7 @@ import { FileUpload, UserDropdown } from '../components'; import { MONITOR_UPLOAD_TIMEOUT } from '../constants'; import { HttpClient } from '../services/http'; import { Settings } from '../settings'; -import { getActiveProject, getTheme, projectToOption, setTheme } from '../utilities'; - +import { getActiveProject, getTheme, projectToOption, portalToOption, setTheme, getActivePortal, clearActivePortal, clearActiveProject } from '../utilities'; export class IbutsuHeader extends React.Component { static propTypes = { @@ -42,22 +42,39 @@ export class IbutsuHeader extends React.Component { constructor(props) { super(props); let project = getActiveProject(); + console.log('active project: '+project); + let portal = getActivePortal(); + console.log('active portal:'+portal); this.eventEmitter = props.eventEmitter; this.state = { uploadFileName: '', importId: '', monitorUploadId: null, isAboutOpen: false, + isSplitMenuOpen: false, + splitMenuActive: "", + splitMenuItems: [], isProjectSelectorOpen: false, selectedProject: projectToOption(project), searchValue: '', projects: [], projectsFilter: '', + portals: [], + selectedPortal: portalToOption(portal), + isPortalSelectorOpen: false, isDarkTheme: getTheme() === 'dark', version: props.version }; } + // This can probably just be a constant + getSplitMenuItems() { + this.setState({splitMenuItems: [ + {key: 'project', label:'Select a Project'}, + {key: 'portal', label:'Select a Dashboard'} + ]}); + } + showNotification(type, title, message, action?, timeout?, key?) { if (!this.eventEmitter) { return; @@ -65,6 +82,13 @@ export class IbutsuHeader extends React.Component { this.eventEmitter.emit('showNotification', type, title, message, action, timeout, key); } + emitPortalChange() { + if (!this.eventEmitter) { + return; + } + this.eventEmitter.emit('portalChange'); + } + emitProjectChange() { if (!this.eventEmitter) { return; @@ -89,6 +113,11 @@ export class IbutsuHeader extends React.Component { .then(data => this.setState({projects: data['projects']})); } + getPortals() { + // TODO: backend record with request, maybe + this.setState({portals: [{id: '1234-abcd', title: 'Insights and Satellite', }]}) + } + onBeforeUpload = (files) => { for (var i = 0; i < files.length; i++) { this.showNotification('info', 'File Uploaded', files[i].name + ' has been uploaded, importing will start momentarily.'); @@ -132,6 +161,54 @@ export class IbutsuHeader extends React.Component { }); } + onSplitMenuToggle = (isOpen) => { + console.log('split menu toggled'); + this.setState({isSplitMenuOpen: isOpen}); + } + + onSplitMenuSelect = (event, value, isPlaceHolder) => { + console.log('split menu selected: ' + event + value + isPlaceHolder); + this.splitMenuActive = value; + this.onProjectClear(); + return; + } + + onPortalToggle = (isOpen) => { + console.log('portal toggled'); + this.setState({isPortalSelectorOpen: isOpen}); + }; + + onPortalSelect = (event, value, isPlaceholder) => { + console.log('portal selected') + + if (isPlaceholder) { + this.onPortalClear(); + return; + } + const activePortal = getActivePortal(); + if (activePortal && activePortal.id === value.portal.id) { + this.setState({isPortalSelectorOpen: false}); + return; + } + // Handle changing portal + const portal = JSON.stringify(value.portal); + localStorage.setItem('portal', portal); + this.setState({ + selectedPortal: value, + isPortalSelectorOpen: false + }); + this.emitPortalChange(); + }; + + onPortalClear = () => { + clearActivePortal(); + this.setState({ + selectedPortal: null, + isPortalSelectorOpen: false + }) + this.emitPortalChange(); + } + onProjectToggle = (isOpen) => { this.setState({isProjectSelectorOpen: isOpen}); }; @@ -141,6 +218,7 @@ export class IbutsuHeader extends React.Component { this.onProjectClear(); return; } + this.onPortalClear(); const activeProject = getActiveProject(); if (activeProject && activeProject.id === value.project.id) { this.setState({isProjectSelectorOpen: false}); @@ -156,7 +234,7 @@ export class IbutsuHeader extends React.Component { }; onProjectClear = () => { - localStorage.removeItem('project'); + clearActiveProject(); this.setState({ selectedProject: null, isProjectSelectorOpen: false @@ -185,6 +263,8 @@ export class IbutsuHeader extends React.Component { componentDidMount() { this.getProjects(); + this.getPortals(); + this.getSplitMenuItems(); } render() { @@ -195,7 +275,32 @@ export class IbutsuHeader extends React.Component { uploadParams['project'] = this.state.selectedProject.project.id; } const topNav = ( - + // Not using this right, try again later + // + // + // + // this.onSplitMenuToggle(isOpen)} + // onSelect={this.onSplitMenuSelect} + // toggle={(toggleRef) => ( + // + // {{this.state.splitMenuActive}} + // + // )} + // > + // {this.state.splitMenuItems.forEach(item => ( + // {item.label} + // ))} + // + // + // + // + + {this.state.portals.map(portal => ( + + ))} + + ); const headerTools = ( diff --git a/frontend/src/utilities.js b/frontend/src/utilities.js index ad9a6f3d..6c2672b8 100644 --- a/frontend/src/utilities.js +++ b/frontend/src/utilities.js @@ -401,8 +401,11 @@ export function getOperationsFromField(field) { return operations; } +// TODO consolidate these into getActive(key) {} functions + export function getActiveProject() { let project = localStorage.getItem('project'); + console.log('local storage project: '+project) if (project) { project = JSON.parse(project); } @@ -413,6 +416,21 @@ export function clearActiveProject() { localStorage.removeItem('project'); } +export function getActivePortal() { + let portal = localStorage.getItem('portal'); + console.log('local storage portal: '+portal) + + if (portal) { + portal = JSON.parse(portal); + } + return portal; +} + +export function clearActivePortal() { + localStorage.removeItem('portal') +} + + export function getActiveDashboard() { let dashboard = localStorage.getItem('dashboard'); if (dashboard) { @@ -425,6 +443,23 @@ export function clearActiveDashboard() { localStorage.removeItem('dashboard'); } +export function portalToOption(portal) { + if (!portal) { + return ''; + } + return { + portal: portal, + toString: function() { + return this.portal.title; + }, + compareTo: function (value) { + if (value.portal) { + return this.portal.id === value.portal.id; + } + } + }; +} + export function projectToOption(project) { if (!project) { return ''; diff --git a/podman_build.sh b/podman_build.sh index feb2f974..d1dd2d50 100755 --- a/podman_build.sh +++ b/podman_build.sh @@ -1,19 +1,19 @@ #!/bin/bash if [[ "$1" == "" ]]; then - IMAGES_TO_BUILD=("backend", "scheduler", "worker", "flower", "frontend") + IMAGES_TO_BUILD=("backend" "scheduler" "worker" "flower" "frontend") else IMAGES_TO_BUILD=("$1") fi # Image info IMAGE_TAG=$(git rev-parse --short=7 HEAD) -IMAGE_PREFIX="quay.io/cloudservices/ibutsu-" +IMAGE_PREFIX="quay.io/ibutsu/" # Build images for IMAGE in "${IMAGES_TO_BUILD[@]}"; do - echo "---- BUILDING: $IMAGE ----" - if [[ "$IMAGE" == "frontend" ]]; then + echo "---- BUILDING: ${IMAGE} ----" + if [[ "${IMAGE}" == "frontend" ]]; then BASE_DIR=frontend else BASE_DIR=backend diff --git a/scripts/ibutsu-pod.sh b/scripts/ibutsu-pod.sh index adb23ead..64b0b6c4 100755 --- a/scripts/ibutsu-pod.sh +++ b/scripts/ibutsu-pod.sh @@ -148,14 +148,14 @@ podman run -d \ $BACKEND_EXTRA_ARGS \ -w /mnt \ -v./backend:/mnt/:Z \ - python:3.8.12 \ - /bin/bash -c 'python -m venv .backend_env && source .backend_env/bin/activate && - pip install -U pip setuptools wheel && - pip install -r requirements.txt && + python:3.11 \ + /bin/bash -c 'python3 -m venv .backend_env && source .backend_env/bin/activate && + pip3 install -U pip setuptools wheel && + pip3 install -r requirements.txt && python -m ibutsu_server --host 0.0.0.0' > /dev/null echo "done." echo -n "Waiting for backend to respond..." -until $(curl --output /dev/null --silent --head --fail http://localhost:8080); do +until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:8080); do echo -n '.' sleep 5 done @@ -176,10 +176,10 @@ podman run -d \ -e CELERY_RESULT_BACKEND=redis://127.0.0.1:6379 \ -w /mnt \ -v./backend:/mnt/:Z \ - python:3.8.12 \ - /bin/bash -c 'python -m venv .worker_env && source .worker_env/bin/activate && - pip install -U pip setuptools wheel && - pip install -r requirements.txt && + python:3.11 \ + /bin/bash -c 'python3 -m venv .worker_env && source .worker_env/bin/activate && + pip3 install -U pip setuptools wheel && + pip3 install -r requirements.txt && ./celery_worker.sh' > /dev/null echo "done." echo -n "Adding frontend to the pod..." @@ -193,7 +193,7 @@ podman run -d \ /bin/bash -c 'npm install && CI=1 npm run devserver' > /dev/null echo "done." echo -n "Waiting for frontend to respond..." -until $(curl --output /dev/null --silent --head --fail http://localhost:3000); do +until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:3000); do printf '.' sleep 5 done @@ -201,8 +201,8 @@ echo "done." if [[ $CREATE_PROJECT = true ]]; then echo -n "Creating default project..." - LOGIN_TOKEN=`curl --no-progress-meter --header "Content-Type: application/json" --request POST --data '{"email": "admin@example.com", "password": "admin12345"}' http://localhost:8080/api/login | grep 'token' | cut -d\" -f 4` - PROJECT_ID=`curl --no-progress-meter --header "Content-Type: application/json" --header "Authorization: Bearer ${LOGIN_TOKEN}" --request POST --data '{"name": "my-project", "title": "My Project"}' http://localhost:8080/api/project | grep '"id"' | cut -d\" -f 4` + LOGIN_TOKEN=`curl --no-progress-meter --header "Content-Type: application/json" --request POST --data '{"email": "admin@example.com", "password": "admin12345"}' http://127.0.0.1:8080/api/login | grep 'token' | cut -d\" -f 4` + PROJECT_ID=`curl --no-progress-meter --header "Content-Type: application/json" --header "Authorization: Bearer ${LOGIN_TOKEN}" --request POST --data '{"name": "my-project", "title": "My Project"}' http://127.0.0.1:8080/api/project | grep '"id"' | cut -d\" -f 4` echo "done." fi echo ""