From 7797bd5b7c0d55a6847097b87c6398c8de300697 Mon Sep 17 00:00:00 2001 From: Mark Hall Date: Thu, 24 Aug 2023 17:30:12 +0100 Subject: [PATCH] Ruff fixes --- museum_map/cli/__init__.py | 3 +-- museum_map/cli/db.py | 22 ++++++++++++++--- museum_map/cli/groups.py | 46 +++++++++++++++++++---------------- museum_map/cli/items.py | 23 ++++++++++-------- museum_map/cli/layout.py | 8 +++--- museum_map/cli/search.py | 2 +- museum_map/cli/server.py | 2 -- museum_map/models/__init__.py | 4 +-- museum_map/models/floor.py | 7 +++--- museum_map/models/group.py | 3 +-- museum_map/models/item.py | 2 +- museum_map/models/room.py | 2 +- museum_map/server/handlers.py | 26 +++++++++----------- 13 files changed, 82 insertions(+), 68 deletions(-) diff --git a/museum_map/cli/__init__.py b/museum_map/cli/__init__.py index 33170da..b6d162a 100644 --- a/museum_map/cli/__init__.py +++ b/museum_map/cli/__init__.py @@ -2,7 +2,6 @@ import logging import os import sys -from typing import Union import click import yaml @@ -298,7 +297,7 @@ def cli(ctx, verbose, config): logger.error(f"Configuration file {config} not found") sys.exit(1) with open(config) as in_f: - config = yaml.load(in_f, Loader=yaml.FullLoader) + config = yaml.safe_load(in_f) ctx.obj["config"] = validate_config(config) diff --git a/museum_map/cli/db.py b/museum_map/cli/db.py index d586f1a..e65d669 100644 --- a/museum_map/cli/db.py +++ b/museum_map/cli/db.py @@ -53,7 +53,7 @@ def load(ctx, source): @click.argument("source") @click.argument("target") @click.pass_context -def load_images(ctx, source, target): +def load_images(ctx, source, target): # noqa: ARG001 """Load and convert images.""" progress = ClickIndeterminate("Loading images") progress.start() @@ -66,10 +66,26 @@ def load_images(ctx, source, target): image_target = os.path.join(target, *image_id, filename) shutil.copy(image_source, image_target) subprocess.run( - ["gm", "convert", image_source, "-resize", "240x240", image_target.replace(".jpg", "-240.jpg")] + [ # noqa: S603 S607 + "gm", + "convert", + image_source, + "-resize", + "240x240", + image_target.replace(".jpg", "-240.jpg"), + ], + check=True, ) subprocess.run( - ["gm", "convert", image_source, "-resize", "320x320", image_target.replace(".jpg", "-320.jpg")] + [ # noqa: S603 S607 + "gm", + "convert", + image_source, + "-resize", + "320x320", + image_target.replace(".jpg", "-320.jpg"), + ], + check=True, ) progress.stop() diff --git a/museum_map/cli/groups.py b/museum_map/cli/groups.py index 7bad0ad..1525461 100644 --- a/museum_map/cli/groups.py +++ b/museum_map/cli/groups.py @@ -29,7 +29,7 @@ async def generate_groups_impl(config): for item in progress: for category in item.attributes["_categories"]: categories.append(category.lower()) - counts = [(cat, count) for cat, count in Counter(categories).most_common() if count >= 15] + counts = [(cat, count) for cat, count in Counter(categories).most_common() if count >= 15] # noqa: PLR2004 counts.sort(key=lambda c: c[1]) max_groups = len(counts) with click.progressbar(length=max_groups, label="Generating groups") as progress: @@ -52,7 +52,9 @@ async def generate_groups_impl(config): for category in item.attributes["_categories"]: categories.append(category.lower()) old_counts = len(counts) - counts = [(cat, count) for cat, count in Counter(categories).most_common() if count >= 15] + counts = [ + (cat, count) for cat, count in Counter(categories).most_common() if count >= 15 # noqa: PLR2004 + ] counts.sort(key=lambda c: c[1]) progress.update(old_counts - len(counts)) await dbsession.commit() @@ -111,7 +113,9 @@ def split_by_attribute(dbsession, group, attr): for item in group.items: if attr in item.attributes and item.attributes[attr]: values.extend(item.attributes[attr]) - categories = [(v, c) for v, c in Counter(values).most_common() if c < len(group.items) * 0.6666 and c >= 15] + categories = [ + (v, c) for v, c in Counter(values).most_common() if c < len(group.items) * 0.6666 and c >= 15 # noqa: PLR2004 + ] if categories: category_values = [v for v, _ in categories] has_values = 0 @@ -123,7 +127,7 @@ def split_by_attribute(dbsession, group, attr): break if found: has_values = has_values + 1 - if has_values / len(group.items) > 0.9: + if has_values / len(group.items) > 0.9: # noqa: PLR2004 categories.reverse() for category in categories: new_group = Group( @@ -151,12 +155,12 @@ def split_by_year(config, dbsession, group): if config["data"]["year_field"] in item.attributes and item.attributes[config["data"]["year_field"]]: years.append(item.attributes[config["data"]["year_field"]]) with_year = with_year + 1 - if with_year / len(group.items) > 0.95: + if with_year / len(group.items) > 0.95: # noqa: PLR2004 common = [(int(v), c) for v, c in Counter(years).most_common()] start_year = min([c for c, _ in common]) end_year = max([c for c, _ in common]) if start_year != end_year: - if (end_year - start_year) <= 100 and (end_year - start_year) > 10: + if (end_year - start_year) <= 100 and (end_year - start_year) > 10: # noqa: PLR2004 start_decade = math.floor(start_year / 10) end_decade = math.floor(end_year / 10) decades = [] @@ -176,7 +180,7 @@ def split_by_year(config, dbsession, group): decades[-1][1] = decades[-1][1] + 1 idx = 0 while idx < len(decades) - 1: - if decades[idx][1] + decades[idx + 1][1] < 100: + if decades[idx][1] + decades[idx + 1][1] < 100: # noqa: PLR2004 decades[idx][0].extend(decades[idx + 1][0]) decades[idx][1] = decades[idx][1] + decades[idx + 1][1] decades.pop(idx + 1) @@ -212,7 +216,7 @@ def split_by_year(config, dbsession, group): for item in list(group.items): item.group = new_group return True - elif (end_year - start_year) > 100: + elif (end_year - start_year) > 100: # noqa: PLR2004 start_century = math.floor(start_year / 100) end_century = math.floor(end_year / 100) centuries = [] @@ -232,7 +236,7 @@ def split_by_year(config, dbsession, group): centuries[-1][1] = centuries[-1][1] + 1 idx = 0 while idx < len(centuries) - 1: - if centuries[idx][1] + centuries[idx + 1][1] < 100: + if centuries[idx][1] + centuries[idx + 1][1] < 100: # noqa: PLR2004 centuries[idx][0].extend(centuries[idx + 1][0]) centuries[idx][1] = centuries[idx][1] + centuries[idx + 1][1] centuries.pop(idx + 1) @@ -252,30 +256,30 @@ def split_by_year(config, dbsession, group): if new_group is None: if len(years) == 1: century = math.floor(years[0] / 100) + 1 - if century % 10 == 1 and century != 11: + if century % 10 == 1 and century != 11: # noqa: PLR2004 label = f"{century}st" - elif century % 10 == 2 and century != 12: + elif century % 10 == 2 and century != 12: # noqa: PLR2004 label = f"{century}nd" - elif century % 10 == 3 and century != 13: + elif century % 10 == 3 and century != 13: # noqa: PLR2004 label = f"{century}rd" else: label = f"{century}th" else: century = math.floor(years[0] / 100) + 1 - if century % 10 == 1 and century != 11: + if century % 10 == 1 and century != 11: # noqa: PLR2004 start_label = f"{century}st" - elif century % 10 == 2 and century != 12: + elif century % 10 == 2 and century != 12: # noqa: PLR2004 start_label = f"{century}nd" - elif century % 10 == 3 and century != 13: + elif century % 10 == 3 and century != 13: # noqa: PLR2004 start_label = f"{century}rd" else: start_label = f"{century}th" century = math.floor(years[-1] / 100) + 1 - if century % 10 == 1 and century != 11: + if century % 10 == 1 and century != 11: # noqa: PLR2004 end_label = f"{century}st" - elif century % 10 == 2 and century != 12: + elif century % 10 == 2 and century != 12: # noqa: PLR2004 end_label = f"{century}nd" - elif century % 10 == 3 and century != 13: + elif century % 10 == 3 and century != 13: # noqa: PLR2004 end_label = f"{century}rd" else: end_label = f"{century}th" @@ -309,13 +313,13 @@ async def split_large_groups_impl(config): result = await dbsession.execute(stmt) for group in result.scalars(): if len(group.children) == 0: - if len(group.items) > 120 and len(group.items) < 300: + if len(group.items) > 120 and len(group.items) < 300: # noqa: PLR2004 if split_by_year(config, dbsession, group): splitting = True else: split_by_similarity(dbsession, group) splitting = True - elif len(group.items) >= 300: + elif len(group.items) >= 300: # noqa: PLR2004 if split_by_attribute(dbsession, group, "concepts"): splitting = True elif split_by_attribute(dbsession, group, "subjects"): @@ -403,7 +407,7 @@ async def add_parent_groups_impl(config): dbsession.add(group) group.parent = parent_group mapped = True - group = parent_group + group = parent_group # noqa: PLW2901 if group.parent_id: break if mapped: diff --git a/museum_map/cli/items.py b/museum_map/cli/items.py index e2533d8..19f7e60 100644 --- a/museum_map/cli/items.py +++ b/museum_map/cli/items.py @@ -134,7 +134,7 @@ def apply_nlp(category): return [] -def apply_aat(category, merge=True): +def apply_aat(category, merge=True): # noqa: FBT002 """Expand the category using the AAT.""" if os.path.exists("aat.json"): with open("aat.json") as in_f: @@ -146,26 +146,29 @@ def apply_aat(category, merge=True): response = requests.get( "http://vocabsservices.getty.edu/AATService.asmx/AATGetTermMatch", params=[("term", f'"{category}"'), ("logop", "and"), ("notes", "")], + timeout=300, ) - if response.status_code == 200: - subjects = etree.fromstring(response.content).xpath("Subject/Subject_ID/text()") + if response.status_code == 200: # noqa: PLR2004 + subjects = etree.fromstring(response.content).xpath("Subject/Subject_ID/text()") # noqa: S320 hierarchies = [] for subject in subjects: response2 = requests.get( - "http://vocabsservices.getty.edu/AATService.asmx/AATGetSubject", params=[("subjectID", subject)] + "http://vocabsservices.getty.edu/AATService.asmx/AATGetSubject", + params=[("subjectID", subject)], + timeout=300, ) - if response.status_code == 200: - hierarchy_text = etree.fromstring(response2.content).xpath("Subject/Hierarchy/text()") + if response.status_code == 200: # noqa: PLR2004 + hierarchy_text = etree.fromstring(response2.content).xpath("Subject/Hierarchy/text()") # noqa: S320 if hierarchy_text: hierarchy = [] for entry in [h.strip() for h in hierarchy_text[0].split("|") if "<" not in h]: - entry = entry.lower() + entry = entry.lower() # noqa: PLW2901 if "(" in entry: - entry = entry[: entry.find("(")].strip() + entry = entry[: entry.find("(")].strip() # noqa: PLW2901 if entry.endswith(" facet"): - entry = entry[: entry.find(" facet")].strip() + entry = entry[: entry.find(" facet")].strip() # noqa: PLW2901 if entry.endswith(" genres"): - entry = entry[: entry.find(" genres")].strip() + entry = entry[: entry.find(" genres")].strip() # noqa: PLW2901 if entry not in hierarchy: hierarchy.append(entry) hierarchies.append(hierarchy) diff --git a/museum_map/cli/layout.py b/museum_map/cli/layout.py index 56cd848..32a224b 100644 --- a/museum_map/cli/layout.py +++ b/museum_map/cli/layout.py @@ -1,7 +1,7 @@ import asyncio import math from copy import deepcopy -from random import choice, sample +from random import choice import click from inflection import pluralize @@ -166,7 +166,7 @@ async def summarise_rooms(dbsession): rooms.scalars(), length=rooms_count.scalar_one(), label="Generating room summaries" ) as progress: for room in progress: - room.sample = choice(room.items) + room.sample = choice(room.items) # noqa: S311 dbsession.add(room) await dbsession.commit() @@ -190,7 +190,7 @@ async def summarise_floors(dbsession): size = await count_items(dbsession, group) while group.split in ["time", "similar", "attribute", "inner"]: parent_result = await dbsession.execute(select(Group).filter(Group.id == group.parent_id)) - group = parent_result.scalar_one() + group = parent_result.scalar_one() # noqa: PLW2901 if group in floor_groups: floor_groups[group] = floor_groups[group] + size else: @@ -202,7 +202,7 @@ async def summarise_floors(dbsession): for group, size in group_sizes: sub_total = sub_total + size dbsession.add(FloorTopic(label=pluralize_label(group.label), group=group, floor=floor, size=size)) - if sub_total / total > 0.66666: + if sub_total / total > 0.66666: # noqa: PLR2004 break items_result = await dbsession.execute( select(Item).filter(Item.room_id.in_([room.id for room in floor.rooms])) diff --git a/museum_map/cli/search.py b/museum_map/cli/search.py index 249569f..d2626f7 100644 --- a/museum_map/cli/search.py +++ b/museum_map/cli/search.py @@ -20,7 +20,7 @@ async def index_impl(config): index = await client.get_index("items") task = await index.delete() await wait_for_task(client, task.task_uid, timeout_in_ms=None) - except Exception: + except Exception: # noqa: S110 pass items_idx = await client.create_index("items", primary_key="mmap_id") stmt = select(Room).options(selectinload(Room.items)) diff --git a/museum_map/cli/server.py b/museum_map/cli/server.py index 1c0d737..199198d 100644 --- a/museum_map/cli/server.py +++ b/museum_map/cli/server.py @@ -1,11 +1,9 @@ from importlib import resources import click -from sqlalchemy.ext.asyncio import create_async_engine from tornado.ioloop import IOLoop from tornado.web import Application, StaticFileHandler -from museum_map.models import create_engine from museum_map.server.handlers import ( APICollectionHandler, APIConfigHandler, diff --git a/museum_map/models/__init__.py b/museum_map/models/__init__.py index bf4c23a..7ca6020 100644 --- a/museum_map/models/__init__.py +++ b/museum_map/models/__init__.py @@ -15,7 +15,7 @@ def create_engine(config) -> AsyncEngine: """Get a new singleton DB engine.""" - global engine + global engine # noqa: PLW0603 if engine is None: engine = create_async_engine(config["db"]["dsn"]) return engine @@ -26,7 +26,7 @@ def create_engine(config) -> AsyncEngine: def create_sessionmaker(config) -> Callable[[], AsyncSession]: """Get a new singleton DB session maker.""" - global async_sessionmaker + global async_sessionmaker # noqa: PLW0603 if async_sessionmaker is None: async_sessionmaker = sessionmaker(create_engine(config), expire_on_commit=False, class_=AsyncSession) return async_sessionmaker diff --git a/museum_map/models/floor.py b/museum_map/models/floor.py index 5775381..2034885 100644 --- a/museum_map/models/floor.py +++ b/museum_map/models/floor.py @@ -1,6 +1,5 @@ -from sqlalchemy import Column, ForeignKey, Index, Integer, Table, Unicode, UnicodeText +from sqlalchemy import Column, ForeignKey, Index, Integer, Table, Unicode from sqlalchemy.orm import relationship -from sqlalchemy_json import NestedMutableJson from museum_map.models.base import Base @@ -15,7 +14,7 @@ class Floor(Base): __tablename__ = "floors" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # noqa: A003 label = Column(Unicode(255)) level = Column(Integer) @@ -42,7 +41,7 @@ def as_jsonapi(self): class FloorTopic(Base): __tablename__ = "floor_topics" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # noqa: A003 group_id = Column(Integer, ForeignKey("groups.id")) floor_id = Column(Integer, ForeignKey("floors.id")) label = Column(Unicode(255)) diff --git a/museum_map/models/group.py b/museum_map/models/group.py index c61715d..1e1ac5e 100644 --- a/museum_map/models/group.py +++ b/museum_map/models/group.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Index, Integer, Unicode from sqlalchemy.orm import relationship -from sqlalchemy_json import NestedMutableJson from museum_map.models.base import Base @@ -8,7 +7,7 @@ class Group(Base): __tablename__ = "groups" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # noqa: A003 parent_id = Column(Integer, ForeignKey("groups.id")) value = Column(Unicode(255)) label = Column(Unicode(255)) diff --git a/museum_map/models/item.py b/museum_map/models/item.py index cf53b9f..6708ada 100644 --- a/museum_map/models/item.py +++ b/museum_map/models/item.py @@ -8,7 +8,7 @@ class Item(Base): __tablename__ = "items" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # noqa: A003 group_id = Column(Integer, ForeignKey("groups.id")) room_id = Column(Integer, ForeignKey("rooms.id")) attributes = Column(NestedMutableJson) diff --git a/museum_map/models/room.py b/museum_map/models/room.py index e40dc12..f185571 100644 --- a/museum_map/models/room.py +++ b/museum_map/models/room.py @@ -8,7 +8,7 @@ class Room(Base): __tablename__ = "rooms" - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key=True) # noqa: A003 floor_id = Column(Integer, ForeignKey("floors.id")) group_id = Column(Integer, ForeignKey("groups.id")) item_id = Column(Integer, ForeignKey("items.id")) diff --git a/museum_map/server/handlers.py b/museum_map/server/handlers.py index 4a080c9..c70bf2e 100644 --- a/museum_map/server/handlers.py +++ b/museum_map/server/handlers.py @@ -1,16 +1,12 @@ import logging import math import re -from configparser import ConfigParser -from datetime import datetime -from importlib import resources +from datetime import datetime, timezone from importlib.abc import Traversable from mimetypes import guess_type -from random import randint from meilisearch_python_async import Client from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import noload, selectinload from tornado import web @@ -72,7 +68,7 @@ async def get(self, types): query = query.filter(getattr(class_, column).in_([])) else: for value in values: - value = value.decode() + value = value.decode() # noqa: PLW2901 if value == "": query = query.filter(getattr(class_, column).in_([])) else: @@ -128,16 +124,16 @@ async def get(self): class APIPickHandler(RequestBase): - async def get(self, type): - if type in ["random", "todays"]: + async def get(self, pick_type): + if pick_type in ["random", "todays"]: async with create_sessionmaker(self.application.settings["config"])() as session: query, class_ = self.setup_query("items") if query is not None and class_ is not None: - if type == "random": + if pick_type == "random": query = query.order_by(func.random()).limit(12) - elif type == "todays": + elif pick_type == "todays": total = (await session.execute(select(func.count()).select_from(class_))).scalars().first() - row_nr = (math.floor(datetime.utcnow().timestamp() / 86400) % total) + 1 + row_nr = (math.floor(datetime.now(tz=timezone.utc).timestamp() / 86400) % total) + 1 query = query.order_by(class_.id).offset(row_nr).limit(1) result = await session.execute(query) items = [item.as_jsonapi() for item in result.scalars()] @@ -191,7 +187,7 @@ async def get(self: "FrontendHandler", path: str) -> None: :param path: The path to get. :type: path: str """ - self.xsrf_token + self.xsrf_token # noqa: B018 if not path.strip(): path = "/" try: @@ -231,8 +227,8 @@ async def _get_resource( if mimetype and mimetype[0]: self.set_header("Content-Type", mimetype[0]) self.write(data) - except IsADirectoryError: - raise FileNotFoundError() + except IsADirectoryError as err: + raise FileNotFoundError() from err def create_inject_item_html(config): @@ -255,7 +251,7 @@ async def inject_item_html(room_id: str, joke_id: str) -> str: """ - except Exception: + except Exception: # noqa: S110 pass return ""