🔼 Upgrade migration tooling

dataservice/__init__.py

@@ -18,7 +18,7 @@
 from dataservice.api.family_relationship.models import FamilyRelationship
 from dataservice.api.outcome.models import Outcome
 from dataservice.api.phenotype.models import Phenotype
-from dataservice.api.biospecimen.models import Biospecimen
+from dataservice.api.biospecimen.models import Biospecimen, Foo
 from dataservice.api.diagnosis.models import Diagnosis
 from dataservice.api.genomic_file.models import GenomicFile
 from dataservice.api.read_group.models import (

dataservice/api/biospecimen/models.py

@@ -8,6 +8,23 @@
 from sqlalchemy.dialects.postgresql import ARRAY
 
 
+class Foo(db.Model, Base):
+    """
+    Temporary
+    """
+
+    __tablename__ = 'foo'
+    __prefix__ = 'FS'
+
+    foo = db.Column(db.Text(), doc='Foo')
+    number = db.Column(db.Integer(), doc='Number')
+    date = db.Column(db.DateTime(), doc='The date the aliquot was shipped')
+    participant_id = db.Column(KfId(),
+                               db.ForeignKey('participant.kf_id'),
+                               nullable=False,
+                               doc='The kf_id of the biospecimen\'s donor')
+
+
 class Biospecimen(db.Model, Base):
     """
     Biospecimen entity.
@@ -49,6 +66,7 @@ class Biospecimen(db.Model, Base):
     __tablename__ = 'biospecimen'
     __prefix__ = 'BS'
 
+    foo = db.Column(db.Text(), doc='Foo')
     external_sample_id = db.Column(db.Text(),
                                    doc='Name given to sample by contributor')
     external_aliquot_id = db.Column(db.Text(),

migrations/alembic.ini

@@ -1,7 +1,6 @@
 # A generic, single database configuration.
 
 [alembic]
-script_location = migrations
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
 
@@ -12,7 +11,7 @@ script_location = migrations
 
 # Logging configuration
 [loggers]
-keys = root,sqlalchemy,alembic
+keys = root,sqlalchemy,alembic,flask_migrate
 
 [handlers]
 keys = console
@@ -35,6 +34,11 @@ level = INFO
 handlers =
 qualname = alembic
 
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)

migrations/env.py

@@ -1,8 +1,9 @@
-from __future__ import with_statement
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
 import logging
+from logging.config import fileConfig
+
+from flask import current_app
+
+from alembic import context
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -13,21 +14,43 @@
 fileConfig(config.config_file_name)
 logger = logging.getLogger('alembic.env')
 
+
+def get_engine():
+    try:
+        # this works with Flask-SQLAlchemy<3 and Alchemical
+        return current_app.extensions['migrate'].db.get_engine()
+    except TypeError:
+        # this works with Flask-SQLAlchemy>=3
+        return current_app.extensions['migrate'].db.engine
+
+
+def get_engine_url():
+    try:
+        return get_engine().url.render_as_string(hide_password=False).replace(
+            '%', '%%')
+    except AttributeError:
+        return str(get_engine().url).replace('%', '%%')
+
+
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
-config.set_main_option('sqlalchemy.url',
-                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+config.set_main_option('sqlalchemy.url', get_engine_url())
+target_db = current_app.extensions['migrate'].db
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
 
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -41,7 +64,9 @@ def run_migrations_offline():
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
+    context.configure(
+        url=url, target_metadata=get_metadata(), literal_binds=True
+    )
 
     with context.begin_transaction():
         context.run_migrations()
@@ -65,22 +90,19 @@ def process_revision_directives(context, revision, directives):
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
+    connectable = get_engine()
 
-    connection = engine.connect()
-    context.configure(connection=connection,
-                      target_metadata=target_metadata,
-                      compare_type=True,
-                      process_revision_directives=process_revision_directives,
-                      **current_app.extensions['migrate'].configure_args)
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=get_metadata(),
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
 
-    try:
         with context.begin_transaction():
             context.run_migrations()
-    finally:
-        connection.close()
+
 
 if context.is_offline_mode():
     run_migrations_offline()

migrations/versions/fe2b0ea83ad7_.py

@@ -0,0 +1,52 @@
+"""
+Temporary test migrations
+
+Revision ID: fe2b0ea83ad7
+Revises: 16e588cc6b85
+Create Date: 2023-05-18 19:55:06.865732
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+import dataservice
+
+# revision identifiers, used by Alembic.
+revision = 'fe2b0ea83ad7'
+down_revision = '16e588cc6b85'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('foo',
+                    sa.Column('uuid', postgresql.UUID(), nullable=True),
+                    sa.Column('created_at', sa.DateTime(), nullable=True),
+                    sa.Column('modified_at', sa.DateTime(), nullable=True),
+                    sa.Column('visible', sa.Boolean(),
+                              server_default='true', nullable=False),
+                    sa.Column('visibility_reason', sa.Text(), nullable=True),
+                    sa.Column('visibility_comment', sa.Text(), nullable=True),
+                    sa.Column('foo', sa.Text(), nullable=True),
+                    sa.Column('number', sa.Integer(), nullable=True),
+                    sa.Column('date', sa.DateTime(), nullable=True),
+                    sa.Column('participant_id', dataservice.api.common.model.KfId(
+                        length=11), nullable=False),
+                    sa.Column('kf_id', dataservice.api.common.model.KfId(
+                        length=11), nullable=False),
+                    sa.ForeignKeyConstraint(['participant_id'], [
+                                            'participant.kf_id'], ),
+                    sa.PrimaryKeyConstraint('kf_id'),
+                    sa.UniqueConstraint('uuid')
+                    )
+    op.add_column('biospecimen', sa.Column('foo', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('biospecimen', 'foo')
+    op.drop_table('foo')
+    # ### end Alembic commands ###

requirements.txt

@@ -1,24 +1,24 @@
-alembic==0.9.6
+alembic==1.10.4
 aniso8601==1.3.0
 apispec==0.29.0
-click==6.6
+click==7.1.2
 Flask==1.0.2
-Flask-Migrate==2.1.1
+Flask-Migrate==4.0.4
 Flask-Profile==0.2
 Flask-SQLAlchemy==2.3.2
 itsdangerous==0.24
 Mako==1.0.4
 MarkupSafe==0.23
 python-editor==1.0.3
 six==1.11.0
-SQLAlchemy==1.1.15
+SQLAlchemy==1.3.24
 Werkzeug==0.15.3
 base32-crockford==0.3.0
 gunicorn==19.7.1
 flask-marshmallow==0.8.0
 marshmallow==2.16.0
 marshmallow-sqlalchemy==0.13.2
-psycopg2==2.7.3.2
+psycopg2==2.9.6
 webargs==5.3.0
 boto3==1.7.8
 botocore==1.10.8

tests/biospecimen/test_biospecimen_models.py

@@ -373,7 +373,7 @@ def create_seqexp(self):
             sc = SequencingCenter(name="Baylor")
         se = SequencingExperiment(external_id="Test_seq_ex_o",
                                   experiment_strategy="WGS",
-                                  is_paired_end="True",
+                                  is_paired_end=True,
                                   platform="Test_platform",
                                   sequencing_center_id=sc.kf_id)
         sc.sequencing_experiments.extend([se])

tests/biospecimen/test_biospecimen_resources.py

@@ -442,7 +442,7 @@ def _create_save_to_db(self):
             db.session.commit()
         se = SequencingExperiment(external_id="Test_seq_ex_o",
                                   experiment_strategy="WGS",
-                                  is_paired_end="True",
+                                  is_paired_end=True,
                                   platform="Test_platform",
                                   sequencing_center_id=sc.kf_id)
         db.session.add(se)

tests/study/test_study_models.py

@@ -99,10 +99,12 @@ def test_delete_relations(self):
         # Delete study from participant
         p0 = participants[0]
         del p0.study
-        db.session.commit()
+        with self.assertRaises(IntegrityError):
+            db.session.commit()
 
         # Check database
-        self.assertNotIn(p0, Study.query.get(study.kf_id).participants)
+        db.session.rollback()
+        self.assertIn(p0, Study.query.get(study.kf_id).participants)
 
     def test_foreign_key_constraint(self):
         """

tests/study_file/test_study_file_models.py

@@ -110,7 +110,6 @@ def test_new_version(self):
         # Check that no new version was made
         assert self.indexd.Session().post.call_count == orig_post + 1
 
-
     def test_delete(self):
         """
         Test delete study_file
@@ -137,10 +136,12 @@ def test_delete_relations(self):
         # Delete study_file from study
         sf0 = study_files[0]
         del sf0.study
-        db.session.commit()
+        with self.assertRaises(IntegrityError):
+            db.session.commit()
 
         # Check database
-        self.assertNotIn(sf0, Study.query.get(study.kf_id).study_files)
+        db.session.rollback()
+        self.assertIn(sf0, Study.query.get(study.kf_id).study_files)
 
     def test_foreign_key_constraint(self):
         """