Format code with ruff

alembic/env.py

@@ -1,11 +1,12 @@
+# ruff: noqa
 import os
 import sys
 from logging.config import fileConfig
 
-parent_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '../..'))
+parent_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "../.."))
 sys.path.append(parent_dir)
 
-from sqlalchemy import engine_from_config, create_engine
+from sqlalchemy import create_engine
 from alembic import context
 
 from jedeschule.pipelines.db_pipeline import Base
@@ -41,7 +42,7 @@ def run_migrations_offline():
     script output.
 
     """
-    url = os.getenv('DATABASE_URL')
+    url = os.getenv("DATABASE_URL")
     context.configure(
         url=url,
         target_metadata=target_metadata,
@@ -60,13 +61,11 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    url = os.getenv('DATABASE_URL')
+    url = os.getenv("DATABASE_URL")
     connectable = create_engine(url)
 
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()

alembic/versions/7debef0e3f50_create_initial_school_table.py

@@ -1,45 +1,47 @@
 """create initial school table
 
 Revision ID: 7debef0e3f50
-Revises: 
+Revises:
 Create Date: 2020-02-01 16:18:24.795672
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '7debef0e3f50'
+revision = "7debef0e3f50"
 down_revision = None
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('schools',
-    sa.Column('id', sa.String(), nullable=False),
-    sa.Column('name', sa.String(), nullable=True),
-    sa.Column('address', sa.String(), nullable=True),
-    sa.Column('address2', sa.String(), nullable=True),
-    sa.Column('zip', sa.String(), nullable=True),
-    sa.Column('city', sa.String(), nullable=True),
-    sa.Column('website', sa.String(), nullable=True),
-    sa.Column('email', sa.String(), nullable=True),
-    sa.Column('school_type', sa.String(), nullable=True),
-    sa.Column('legal_status', sa.String(), nullable=True),
-    sa.Column('provider', sa.String(), nullable=True),
-    sa.Column('fax', sa.String(), nullable=True),
-    sa.Column('phone', sa.String(), nullable=True),
-    sa.Column('director', sa.String(), nullable=True),
-    sa.Column('raw', sa.JSON(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
+    op.create_table(
+        "schools",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("address", sa.String(), nullable=True),
+        sa.Column("address2", sa.String(), nullable=True),
+        sa.Column("zip", sa.String(), nullable=True),
+        sa.Column("city", sa.String(), nullable=True),
+        sa.Column("website", sa.String(), nullable=True),
+        sa.Column("email", sa.String(), nullable=True),
+        sa.Column("school_type", sa.String(), nullable=True),
+        sa.Column("legal_status", sa.String(), nullable=True),
+        sa.Column("provider", sa.String(), nullable=True),
+        sa.Column("fax", sa.String(), nullable=True),
+        sa.Column("phone", sa.String(), nullable=True),
+        sa.Column("director", sa.String(), nullable=True),
+        sa.Column("raw", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
     )
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('schools')
+    op.drop_table("schools")
     # ### end Alembic commands ###

alembic/versions/8e4921f60766_add_last_update_timestamp.py

@@ -5,20 +5,23 @@
 Create Date: 2024-04-15 19:10:40.891044
 
 """
+
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '8e4921f60766'
-down_revision = 'b3913e0b45ac'
+revision = "8e4921f60766"
+down_revision = "b3913e0b45ac"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    op.add_column('schools', sa.Column('update_timestamp', sa.DateTime(), nullable=True))
+    op.add_column(
+        "schools", sa.Column("update_timestamp", sa.DateTime(), nullable=True)
+    )
 
 
 def downgrade():
-    op.drop_column('schools', 'update_timestamp')
+    op.drop_column("schools", "update_timestamp")

alembic/versions/b3913e0b45ac_add_location_to_school.py

@@ -5,26 +5,36 @@
 Create Date: 2021-02-14 09:09:07.672138
 
 """
+
 import geoalchemy2
 
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'b3913e0b45ac'
-down_revision = '7debef0e3f50'
+revision = "b3913e0b45ac"
+down_revision = "7debef0e3f50"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     conn = op.get_bind()
     conn.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
-    op.add_column('schools', sa.Column('location', geoalchemy2.types.Geometry(geometry_type='POINT', from_text='ST_GeomFromEWKT', name='geometry'), nullable=True))
+    op.add_column(
+        "schools",
+        sa.Column(
+            "location",
+            geoalchemy2.types.Geometry(
+                geometry_type="POINT", from_text="ST_GeomFromEWKT", name="geometry"
+            ),
+            nullable=True,
+        ),
+    )
 
 
 def downgrade():
-    op.drop_column('schools', 'location')
+    op.drop_column("schools", "location")
     conn = op.get_bind()
     conn.execute("DROP EXTENSION IF EXISTS postgis;")

jedeschule/pipelines/__init__.py

@@ -1,4 +1,3 @@
-from .jsonpipeline import JsonPipeline
-from .school_pipeline import SchoolPipeline
-from .db_pipeline import DatabasePipeline
-
+from .jsonpipeline import JsonPipeline as JsonPipeline
+from .school_pipeline import SchoolPipeline as SchoolPipeline
+from .db_pipeline import DatabasePipeline as DatabasePipeline

jedeschule/pipelines/db_pipeline.py

@@ -1,8 +1,9 @@
-from __future__ import annotations  # needed so that update_or_create can define School return type
+from __future__ import (
+    annotations,
+)  # needed so that update_or_create can define School return type
 
 import logging
 import os
-from datetime import datetime
 
 from geoalchemy2 import Geometry, WKTElement
 from sqlalchemy import String, Column, JSON, DateTime, func
@@ -11,7 +12,6 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from jedeschule.items import School as SchoolItem
 from jedeschule.pipelines.school_pipeline import SchoolPipelineItem
 
 Base = declarative_base()
@@ -26,7 +26,7 @@ def get_session():
 
 
 class School(Base):
-    __tablename__ = 'schools'
+    __tablename__ = "schools"
     id = Column(String, primary_key=True)
     name = Column(String)
     address = Column(String)
@@ -43,28 +43,30 @@ class School(Base):
     director = Column(String)
     raw = Column(JSON)
     update_timestamp = Column(DateTime, onupdate=func.now())
-    location = Column(Geometry('POINT'))
+    location = Column(Geometry("POINT"))
 
     @staticmethod
     def update_or_create(item: SchoolPipelineItem, session=None) -> School:
         if not session:
             session = get_session()
 
         school_data = {**item.info}
-        school = session.query(School).get(item.info['id'])
-        latitude = school_data.pop('latitude', None)
-        longitude = school_data.pop('longitude', None)
+        school = session.query(School).get(item.info["id"])
+        latitude = school_data.pop("latitude", None)
+        longitude = school_data.pop("longitude", None)
         if latitude is not None and longitude is not None:
             location = WKTElement(f"POINT({longitude} {latitude})", srid=4326)
-            school_data['location'] = location
+            school_data["location"] = location
         if school:
-            session.query(School).filter_by(id=item.info['id']).update({**school_data, 'raw': item.item})
+            session.query(School).filter_by(id=item.info["id"]).update(
+                {**school_data, "raw": item.item}
+            )
         else:
             school = School(**school_data, raw=item.item)
         return school
 
     def __str__(self):
-        return f'<School id={self.id}, name={self.name}>'
+        return f"<School id={self.id}, name={self.name}>"
 
 
 class DatabasePipeline:
@@ -77,7 +79,7 @@ def process_item(self, item: SchoolPipelineItem, spider):
             self.session.add(school)
             self.session.commit()
         except SQLAlchemyError as e:
-            logging.warning('Error when putting to DB')
+            logging.warning("Error when putting to DB")
             logging.warning(e)
             self.session.rollback()
         return school

jedeschule/pipelines/jsonpipeline.py

@@ -1,12 +1,15 @@
 from scrapy.exporters import JsonItemExporter
 import os
 
+
 class JsonPipeline(object):
     def open_spider(self, spider):
         if not os.path.exists("data"):
             os.makedirs("data")
-        self.file = open("data/" + spider.name + ".json", 'wb')
-        self.exporter = JsonItemExporter(self.file, encoding='utf-8', ensure_ascii=False)
+        self.file = open(f"data/{spider.name}.json", "wb")
+        self.exporter = JsonItemExporter(
+            self.file, encoding="utf-8", ensure_ascii=False
+        )
         self.exporter.start_exporting()
 
     def close_spider(self, spider):
@@ -15,4 +18,4 @@ def close_spider(self, spider):
 
     def process_item(self, item, spider):
         self.exporter.export_item(item)
-        return item
+        return item