From 88b888ecf39569e3c08bf7da1079cb1a5485aeb2 Mon Sep 17 00:00:00 2001 From: Amrutha Date: Sat, 29 Oct 2022 23:18:11 +0530 Subject: [PATCH 1/4] #267 replacing all the print calls by calls to log --- .../commands/download_program_contents.py | 15 ++++++++------- catalog/management/commands/download_programs.py | 9 +++++---- catalog/management/parser/load_courses.py | 10 +++++----- documents/logic.py | 3 ++- documents/tasks.py | 3 ++- www/settings.py | 9 +++++---- 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/catalog/management/commands/download_program_contents.py b/catalog/management/commands/download_program_contents.py index b0afbc3a..e1e0d28f 100644 --- a/catalog/management/commands/download_program_contents.py +++ b/catalog/management/commands/download_program_contents.py @@ -5,6 +5,7 @@ import requests from rich import print +import logging from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn @@ -14,7 +15,7 @@ class Command(BaseCommand): def handle(self, *args, **options): with open("programs.json") as f: programs: list[dict] = json.load(f) - print("\n[bold blue]Listing the course content of all programs...[/]\n") + logging.debug("\n[bold blue]Listing the course content of all programs...[/]\n") failed: list = [] program_content: dict[str, dict[str, dict]] = {} @@ -46,19 +47,19 @@ def handle(self, *args, **options): response = requests.get(URL) if not response.ok: if "parent" in progam: - print( + logging.debug( f"[yellow]Skip:[/] [magenta]{progam['slug'].upper()}[/] with bogus parent {progam['parent'].upper()}" ) else: - print( + logging.debug( f"[red]Error:[/] [magenta]{progam['slug'].upper()}[/] failed with {response.status_code}" ) - print(" ", URL) + logging.debug(" ", URL) continue except Exception as e: - print(f"[red]Error:[/] Failed to GET {progam['slug'].upper()}") - print(" URL", URL) + logging.debug(f"[red]Error:[/] Failed to GET {progam['slug'].upper()}") + logging.debug(" URL", URL) progress.console.print_exception() continue @@ -81,7 +82,7 @@ def handle(self, *args, **options): } except Exception as e: failed.append(progam["slug"]) - print(f"Error while listing content of {progam['slug']}") + logging.debug(f"Error while listing content of {progam['slug']}") progress.console.print_exception() with open( diff --git a/catalog/management/commands/download_programs.py b/catalog/management/commands/download_programs.py index e3b32a2f..c76290fc 100644 --- a/catalog/management/commands/download_programs.py +++ b/catalog/management/commands/download_programs.py @@ -6,6 +6,7 @@ import requests from bs4 import BeautifulSoup from rich import print +import logging from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn @@ -20,7 +21,7 @@ def handle(self, *args, **options): programs = [] parent_programs: set[str] = set() - print("[bold blue]Gathering the list of available programs...[/]\n") + logging.debug("[bold blue]Gathering the list of available programs...[/]\n") with Progress( SpinnerColumn(), @@ -107,12 +108,12 @@ def handle(self, *args, **options): progress.update(task1, completed=self.PAGE_SIZE * page) page += 1 - print( + logging.debug( f"Found {len(parent_programs)} programs containing options, ignoring those..." ) - print(parent_programs) + logging.debug(parent_programs) programs = [p for p in programs if p["slug"] not in parent_programs] - print(f"Found {len(programs)} distinct programs, dumping to json...") + logging.debug(f"Found {len(programs)} distinct programs, dumping to json...") with open("programs.json", "w") as f: json.dump(programs, f, indent=4) diff --git a/catalog/management/parser/load_courses.py b/catalog/management/parser/load_courses.py index 7627b276..8fee52cc 100644 --- a/catalog/management/parser/load_courses.py +++ b/catalog/management/parser/load_courses.py @@ -1,6 +1,6 @@ import json import os - +import logging from django.utils.text import slugify from catalog.models import Category, Course @@ -11,21 +11,21 @@ a = 0 ulb, _created = Category.objects.get_or_create(name="ULB", slug="root") for fac_name, fac_info in tree["ULB"].items(): - print(fac_name) + logging.debug(fac_name) fac_obj, _created = Category.objects.get_or_create( name=fac_name, slug=slugify(fac_name), description=fac_info["color"] ) fac_obj.parents.add(ulb) for program_slug, program_info in fac_info["programs"].items(): - print(" ", program_info["name"]) + logging.debug(" ", program_info["name"]) program_obj, _created = Category.objects.get_or_create( name=program_info["name"], slug=program_slug ) program_obj.parents.add(fac_obj) for bloc_name, bloc_info in program_info["blocs"].items(): - print(" ", bloc_name) + logging.debug(" ", bloc_name) bloc_obj, _created = Category.objects.get_or_create( name=f"Bloc {bloc_name}", slug=f"{program_slug}-bloc-{bloc_name}", @@ -34,7 +34,7 @@ for course_mnemo, course_info in bloc_info["courses"].items(): a += 1 - print(str(a).zfill(4), course_info["title"]) + logging.debug(str(a).zfill(4), course_info["title"]) course_obj, _created = Course.objects.get_or_create(slug=course_mnemo) if _created: course_obj.name = course_info["title"] diff --git a/documents/logic.py b/documents/logic.py index a2e2c8f5..7cbdcd60 100644 --- a/documents/logic.py +++ b/documents/logic.py @@ -2,6 +2,7 @@ import mimetypes import uuid +import logging from collections.abc import Iterable from django.core.files import File @@ -16,7 +17,7 @@ import platform if platform.machine() == "arm64" and platform.system() == "Darwin": - print( + logging.debug( "There is a bug with libmagic on Apple Silicon, disabling the feature for now" ) else: diff --git a/documents/tasks.py b/documents/tasks.py index caaefc06..9c31cf62 100644 --- a/documents/tasks.py +++ b/documents/tasks.py @@ -15,6 +15,7 @@ from celery.exceptions import SoftTimeLimitExceeded from PyPDF2 import PdfReader +import logging from documents.models import Document, DocumentError from .exceptions import ( @@ -30,7 +31,7 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): return None doc_id = args[0] - print(f"Document {doc_id} failed.") + logging.debug(f"Document {doc_id} failed.") document = Document.objects.get(id=doc_id) document.state = Document.DocumentState.ERROR diff --git a/www/settings.py b/www/settings.py index 639e5fac..b32ebf44 100644 --- a/www/settings.py +++ b/www/settings.py @@ -1,4 +1,5 @@ import os +import logging from pathlib import Path import environ @@ -158,8 +159,8 @@ AWS_S3_SECRET_ACCESS_KEY = env("STORAGE_SECRET_KEY") AWS_STORAGE_BUCKET_NAME = env("STORAGE_MEDIA_BUCKET_NAME") elif not DEBUG: - print("Warning: no storage configured but DEBUG=False, using local filesystem.") - print("You DO NOT want this in production!") + logging.debug("Warning: no storage configured but DEBUG=False, using local filesystem.") + logging.debug("You DO NOT want this in production!") if DEBUG: INSTALLED_APPS.extend(["django_extensions"]) @@ -209,7 +210,7 @@ if DEBUG: - print( + logging.debug( "Warning: you are running Dochub with DEBUG=True. This is dangerous if your server is publicly accessible." ) - print("You should set DEBUG=False in production.\n\n") + logging.debug("You should set DEBUG=False in production.\n\n") From 8eca03d3cbe76375ad4845799cf860d9f11c61dc Mon Sep 17 00:00:00 2001 From: Amrutha Date: Mon, 31 Oct 2022 21:48:18 +0530 Subject: [PATCH 2/4] #267 fixing the pre-commit issue --- catalog/management/commands/download_program_contents.py | 6 ++++-- catalog/management/commands/download_programs.py | 2 +- catalog/management/parser/load_courses.py | 3 ++- documents/logic.py | 2 +- documents/tasks.py | 2 +- www/settings.py | 6 ++++-- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/catalog/management/commands/download_program_contents.py b/catalog/management/commands/download_program_contents.py index e1e0d28f..fd1447fc 100644 --- a/catalog/management/commands/download_program_contents.py +++ b/catalog/management/commands/download_program_contents.py @@ -1,11 +1,11 @@ import json +import logging from urllib.parse import quote from django.core.management import BaseCommand import requests from rich import print -import logging from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn @@ -58,7 +58,9 @@ def handle(self, *args, **options): continue except Exception as e: - logging.debug(f"[red]Error:[/] Failed to GET {progam['slug'].upper()}") + logging.debug( + f"[red]Error:[/] Failed to GET {progam['slug'].upper()}" + ) logging.debug(" URL", URL) progress.console.print_exception() continue diff --git a/catalog/management/commands/download_programs.py b/catalog/management/commands/download_programs.py index c76290fc..a88dc477 100644 --- a/catalog/management/commands/download_programs.py +++ b/catalog/management/commands/download_programs.py @@ -1,4 +1,5 @@ import json +import logging import re from django.core.management import BaseCommand @@ -6,7 +7,6 @@ import requests from bs4 import BeautifulSoup from rich import print -import logging from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn diff --git a/catalog/management/parser/load_courses.py b/catalog/management/parser/load_courses.py index 8fee52cc..9ca374d6 100644 --- a/catalog/management/parser/load_courses.py +++ b/catalog/management/parser/load_courses.py @@ -1,6 +1,7 @@ import json -import os import logging +import os + from django.utils.text import slugify from catalog.models import Category, Course diff --git a/documents/logic.py b/documents/logic.py index 7cbdcd60..ee5f4402 100644 --- a/documents/logic.py +++ b/documents/logic.py @@ -1,8 +1,8 @@ from typing import Optional +import logging import mimetypes import uuid -import logging from collections.abc import Iterable from django.core.files import File diff --git a/documents/tasks.py b/documents/tasks.py index 9c31cf62..283be631 100644 --- a/documents/tasks.py +++ b/documents/tasks.py @@ -2,6 +2,7 @@ import contextlib import hashlib +import logging import os import re import subprocess @@ -15,7 +16,6 @@ from celery.exceptions import SoftTimeLimitExceeded from PyPDF2 import PdfReader -import logging from documents.models import Document, DocumentError from .exceptions import ( diff --git a/www/settings.py b/www/settings.py index b32ebf44..f4429b8a 100644 --- a/www/settings.py +++ b/www/settings.py @@ -1,5 +1,5 @@ -import os import logging +import os from pathlib import Path import environ @@ -159,7 +159,9 @@ AWS_S3_SECRET_ACCESS_KEY = env("STORAGE_SECRET_KEY") AWS_STORAGE_BUCKET_NAME = env("STORAGE_MEDIA_BUCKET_NAME") elif not DEBUG: - logging.debug("Warning: no storage configured but DEBUG=False, using local filesystem.") + logging.debug( + "Warning: no storage configured but DEBUG=False, using local filesystem." + ) logging.debug("You DO NOT want this in production!") if DEBUG: From 68bc5d742a357a435c8f368ba181bf515424c4bd Mon Sep 17 00:00:00 2001 From: Amrutha Date: Fri, 11 Nov 2022 20:42:09 +0530 Subject: [PATCH 3/4] refactoring code for logging statements --- .../commands/download_program_contents.py | 17 +++++++++-------- .../management/commands/download_programs.py | 11 ++++++----- catalog/management/parser/load_courses.py | 10 +++++----- documents/logic.py | 4 ++-- documents/tasks.py | 4 ++-- www/logger_settings.py | 14 ++++++++++++++ www/settings.py | 11 ++++++----- 7 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 www/logger_settings.py diff --git a/catalog/management/commands/download_program_contents.py b/catalog/management/commands/download_program_contents.py index fd1447fc..5b909fac 100644 --- a/catalog/management/commands/download_program_contents.py +++ b/catalog/management/commands/download_program_contents.py @@ -1,5 +1,4 @@ import json -import logging from urllib.parse import quote from django.core.management import BaseCommand @@ -8,6 +7,8 @@ from rich import print from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn +from www.logger_settings import logger + class Command(BaseCommand): help = "" @@ -15,7 +16,7 @@ class Command(BaseCommand): def handle(self, *args, **options): with open("programs.json") as f: programs: list[dict] = json.load(f) - logging.debug("\n[bold blue]Listing the course content of all programs...[/]\n") + logger.debug("\n[bold blue]Listing the course content of all programs...[/]\n") failed: list = [] program_content: dict[str, dict[str, dict]] = {} @@ -47,21 +48,21 @@ def handle(self, *args, **options): response = requests.get(URL) if not response.ok: if "parent" in progam: - logging.debug( + logger.debug( f"[yellow]Skip:[/] [magenta]{progam['slug'].upper()}[/] with bogus parent {progam['parent'].upper()}" ) else: - logging.debug( + logger.debug( f"[red]Error:[/] [magenta]{progam['slug'].upper()}[/] failed with {response.status_code}" ) - logging.debug(" ", URL) + logger.debug(" ", URL) continue except Exception as e: - logging.debug( + logger.debug( f"[red]Error:[/] Failed to GET {progam['slug'].upper()}" ) - logging.debug(" URL", URL) + logger.debug(" URL", URL) progress.console.print_exception() continue @@ -84,7 +85,7 @@ def handle(self, *args, **options): } except Exception as e: failed.append(progam["slug"]) - logging.debug(f"Error while listing content of {progam['slug']}") + logger.debug(f"Error while listing content of {progam['slug']}") progress.console.print_exception() with open( diff --git a/catalog/management/commands/download_programs.py b/catalog/management/commands/download_programs.py index a88dc477..e20d2986 100644 --- a/catalog/management/commands/download_programs.py +++ b/catalog/management/commands/download_programs.py @@ -1,5 +1,4 @@ import json -import logging import re from django.core.management import BaseCommand @@ -9,6 +8,8 @@ from rich import print from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn +from www.logger_settings import logger + class Command(BaseCommand): help = "" @@ -21,7 +22,7 @@ def handle(self, *args, **options): programs = [] parent_programs: set[str] = set() - logging.debug("[bold blue]Gathering the list of available programs...[/]\n") + logger.debug("[bold blue]Gathering the list of available programs...[/]\n") with Progress( SpinnerColumn(), @@ -108,12 +109,12 @@ def handle(self, *args, **options): progress.update(task1, completed=self.PAGE_SIZE * page) page += 1 - logging.debug( + logger.debug( f"Found {len(parent_programs)} programs containing options, ignoring those..." ) - logging.debug(parent_programs) + logger.debug(parent_programs) programs = [p for p in programs if p["slug"] not in parent_programs] - logging.debug(f"Found {len(programs)} distinct programs, dumping to json...") + logger.debug(f"Found {len(programs)} distinct programs, dumping to json...") with open("programs.json", "w") as f: json.dump(programs, f, indent=4) diff --git a/catalog/management/parser/load_courses.py b/catalog/management/parser/load_courses.py index 9ca374d6..137b2dba 100644 --- a/catalog/management/parser/load_courses.py +++ b/catalog/management/parser/load_courses.py @@ -1,10 +1,10 @@ import json -import logging import os from django.utils.text import slugify from catalog.models import Category, Course +from www.logger_settings import logger with open("catalog/management/parser/data/tree.json") as tree_file: tree = json.load(tree_file) @@ -12,21 +12,21 @@ a = 0 ulb, _created = Category.objects.get_or_create(name="ULB", slug="root") for fac_name, fac_info in tree["ULB"].items(): - logging.debug(fac_name) + logger.debug(fac_name) fac_obj, _created = Category.objects.get_or_create( name=fac_name, slug=slugify(fac_name), description=fac_info["color"] ) fac_obj.parents.add(ulb) for program_slug, program_info in fac_info["programs"].items(): - logging.debug(" ", program_info["name"]) + logger.debug(" ", program_info["name"]) program_obj, _created = Category.objects.get_or_create( name=program_info["name"], slug=program_slug ) program_obj.parents.add(fac_obj) for bloc_name, bloc_info in program_info["blocs"].items(): - logging.debug(" ", bloc_name) + logger.debug(" ", bloc_name) bloc_obj, _created = Category.objects.get_or_create( name=f"Bloc {bloc_name}", slug=f"{program_slug}-bloc-{bloc_name}", @@ -35,7 +35,7 @@ for course_mnemo, course_info in bloc_info["courses"].items(): a += 1 - logging.debug(str(a).zfill(4), course_info["title"]) + logger.debug(str(a).zfill(4), course_info["title"]) course_obj, _created = Course.objects.get_or_create(slug=course_mnemo) if _created: course_obj.name = course_info["title"] diff --git a/documents/logic.py b/documents/logic.py index ee5f4402..58504d7e 100644 --- a/documents/logic.py +++ b/documents/logic.py @@ -1,6 +1,5 @@ from typing import Optional -import logging import mimetypes import uuid from collections.abc import Iterable @@ -10,6 +9,7 @@ from catalog.models import Course from tags.models import Tag from users.models import User +from www.logger_settings import logger try: import magic @@ -17,7 +17,7 @@ import platform if platform.machine() == "arm64" and platform.system() == "Darwin": - logging.debug( + logger.debug( "There is a bug with libmagic on Apple Silicon, disabling the feature for now" ) else: diff --git a/documents/tasks.py b/documents/tasks.py index 283be631..0d68f814 100644 --- a/documents/tasks.py +++ b/documents/tasks.py @@ -2,7 +2,6 @@ import contextlib import hashlib -import logging import os import re import subprocess @@ -17,6 +16,7 @@ from PyPDF2 import PdfReader from documents.models import Document, DocumentError +from www.logger_settings import logger from .exceptions import ( DocumentProcessingError, @@ -31,7 +31,7 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): return None doc_id = args[0] - logging.debug(f"Document {doc_id} failed.") + logger.debug(f"Document {doc_id} failed.") document = Document.objects.get(id=doc_id) document.state = Document.DocumentState.ERROR diff --git a/www/logger_settings.py b/www/logger_settings.py new file mode 100644 index 00000000..746dfa97 --- /dev/null +++ b/www/logger_settings.py @@ -0,0 +1,14 @@ +import logging +import sys + +logger = logging.getLogger() + +# specify here to change log level +log_level=logging.DEBUG +logger.setLevel(log_level) + +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) \ No newline at end of file diff --git a/www/settings.py b/www/settings.py index f4429b8a..839c87eb 100644 --- a/www/settings.py +++ b/www/settings.py @@ -1,4 +1,3 @@ -import logging import os from pathlib import Path @@ -8,6 +7,8 @@ # https://django-environ.readthedocs.io/en/latest/tips.html#docker-style-file-based-variables from sentry_sdk.utils import get_default_release +from www.logger_settings import logger + env = environ.FileAwareEnv() # Set the project base directory @@ -159,10 +160,10 @@ AWS_S3_SECRET_ACCESS_KEY = env("STORAGE_SECRET_KEY") AWS_STORAGE_BUCKET_NAME = env("STORAGE_MEDIA_BUCKET_NAME") elif not DEBUG: - logging.debug( + logger.debug( "Warning: no storage configured but DEBUG=False, using local filesystem." ) - logging.debug("You DO NOT want this in production!") + logger.debug("You DO NOT want this in production!") if DEBUG: INSTALLED_APPS.extend(["django_extensions"]) @@ -212,7 +213,7 @@ if DEBUG: - logging.debug( + logger.debug( "Warning: you are running Dochub with DEBUG=True. This is dangerous if your server is publicly accessible." ) - logging.debug("You should set DEBUG=False in production.\n\n") + logger.debug("You should set DEBUG=False in production.\n\n") From 27b910d1b3ef5ef466f1032f14cea79943a431a0 Mon Sep 17 00:00:00 2001 From: Amrutha Date: Fri, 11 Nov 2022 22:45:38 +0530 Subject: [PATCH 4/4] fixing the pre commit changes for logger settings --- www/logger_settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/www/logger_settings.py b/www/logger_settings.py index 746dfa97..8ee686a1 100644 --- a/www/logger_settings.py +++ b/www/logger_settings.py @@ -3,12 +3,12 @@ logger = logging.getLogger() -# specify here to change log level -log_level=logging.DEBUG +# specify here to change log level +log_level = logging.DEBUG logger.setLevel(log_level) handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) -logger.addHandler(handler) \ No newline at end of file +logger.addHandler(handler)