From 9a75173869eca4925bcd44a19c81ba66dce9eca7 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Tue, 14 Apr 2020 11:31:59 +0200 Subject: [PATCH] Deduplicate code for tests of archive migration code (#3924) Each archive migration was being tested on some archives included in this repo in `tools/fixtures/export/migrate` and the test was always the same. Simply take an archive, migrate it to the next version with the appropriate method and check that both metadata and data dictionaries of the migrated data match that of the reference archive. This same test was implemented for each migration method, but has now been centralized in a single test using pytest to parametrize the versions. The testing on external archives from `aiida-export-migration-testing` has also been streamlined a bit, by defining a new base class called `ArchiveMigrationTest` that provides some utility methods. --- .../importexport/migration/v03_to_v04.py | 5 +- .../tools/importexport/migration/__init__.py | 32 ++++++ .../importexport/migration/test_migrations.py | 62 ++++++++++++ .../importexport/migration/test_v01_to_v02.py | 59 ----------- .../importexport/migration/test_v02_to_v03.py | 73 ++------------ .../importexport/migration/test_v03_to_v04.py | 81 +-------------- .../importexport/migration/test_v04_to_v05.py | 98 ++----------------- .../importexport/migration/test_v05_to_v06.py | 76 ++------------ .../importexport/migration/test_v06_to_v07.py | 84 +++------------- .../importexport/migration/test_v07_to_v08.py | 72 ++------------ 10 files changed, 146 insertions(+), 496 deletions(-) create mode 100644 tests/tools/importexport/migration/test_migrations.py delete mode 100644 tests/tools/importexport/migration/test_v01_to_v02.py diff --git a/aiida/tools/importexport/migration/v03_to_v04.py b/aiida/tools/importexport/migration/v03_to_v04.py index cd7a8e32da..32745f73f7 100644 --- a/aiida/tools/importexport/migration/v03_to_v04.py +++ b/aiida/tools/importexport/migration/v03_to_v04.py @@ -432,7 +432,7 @@ def add_extras(data): data.update({'node_extras': node_extras, 'node_extras_conversion': node_extras_conversion}) -def migrate_v3_to_v4(metadata, data, folder, *args): # pylint: disable=unused-argument +def migrate_v3_to_v4(metadata, data, *args): """ Migration of export files from v0.3 to v0.4 @@ -446,6 +446,9 @@ def migrate_v3_to_v4(metadata, data, folder, *args): # pylint: disable=unused-a verify_metadata_version(metadata, old_version) update_metadata(metadata, new_version) + # The trajectory data migration requires the folder containing all the repository files of the archive + folder = args[0] + # Apply migrations in correct sequential order migration_base_data_plugin_type_string(data) migration_process_type(metadata, data) diff --git a/tests/tools/importexport/migration/__init__.py b/tests/tools/importexport/migration/__init__.py index 2776a55f97..3a12435017 100644 --- a/tests/tools/importexport/migration/__init__.py +++ b/tests/tools/importexport/migration/__init__.py @@ -7,3 +7,35 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### +"""Module with tests for export archive migrations.""" +from aiida.backends.testbase import AiidaTestCase +from aiida.tools.importexport.migration.utils import verify_metadata_version +from tests.utils.archives import get_json_files + + +class ArchiveMigrationTest(AiidaTestCase): + """Base class to write specific tests for a particular export archive migration.""" + + @classmethod + def setUpClass(cls, *args, **kwargs): + super().setUpClass(*args, **kwargs) + cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} + cls.core_archive = {'filepath': 'export/migrate'} + cls.maxDiff = None # pylint: disable=invalid-name + + def migrate(self, filename_archive, version_old, version_new, migration_method): + """Migrate one of the archives from `aiida-export-migration-tests`. + + :param filename_archive: the relative file name of the archive + :param version_old: version of the archive + :param version_new: version to migrate to + :param migration_method: the migration method that should convert between version_old and version_new + :return: the migrated metadata and data as a tuple + """ + metadata, data = get_json_files(filename_archive, **self.external_archive) + verify_metadata_version(metadata, version=version_old) + + migration_method(metadata, data) + verify_metadata_version(metadata, version=version_new) + + return metadata, data diff --git a/tests/tools/importexport/migration/test_migrations.py b/tests/tools/importexport/migration/test_migrations.py new file mode 100644 index 0000000000..75da75a855 --- /dev/null +++ b/tests/tools/importexport/migration/test_migrations.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# pylint: disable=redefined-outer-name +"""Test the export archive migrations on the archives included in `tests/fixtures/export/migrate`.""" +import copy +import pytest + +from aiida import get_version +from aiida.tools.importexport.common import Archive +from aiida.tools.importexport.migration.v01_to_v02 import migrate_v1_to_v2 +from aiida.tools.importexport.migration.v02_to_v03 import migrate_v2_to_v3 +from aiida.tools.importexport.migration.v03_to_v04 import migrate_v3_to_v4 +from aiida.tools.importexport.migration.v04_to_v05 import migrate_v4_to_v5 +from aiida.tools.importexport.migration.v05_to_v06 import migrate_v5_to_v6 +from aiida.tools.importexport.migration.v06_to_v07 import migrate_v6_to_v7 +from aiida.tools.importexport.migration.v07_to_v08 import migrate_v7_to_v8 +from aiida.tools.importexport.migration.utils import verify_metadata_version +from tests.utils.archives import get_json_files, get_archive_file + + +@pytest.fixture +def migration_data(request): + """For a given tuple of two subsequent versions and corresponding migration method, return metadata and data.""" + version_old, version_new, migration_method = request.param + + filepath_archive = 'export_v{}_simple.aiida'.format(version_new) + metadata_new, data_new = get_json_files(filepath_archive, filepath='export/migrate') + verify_metadata_version(metadata_new, version=version_new) + + filepath_archive = get_archive_file('export_v{}_simple.aiida'.format(version_old), filepath='export/migrate') + + with Archive(filepath_archive) as archive: + metadata_old = copy.deepcopy(archive.meta_data) + data_old = copy.deepcopy(archive.data) + + migration_method(metadata_old, data_old, archive.folder) + verify_metadata_version(metadata_old, version=version_new) + + yield version_old, version_new, metadata_old, metadata_new, data_old, data_new + + +@pytest.mark.parametrize( + 'migration_data', + (('0.1', '0.2', migrate_v1_to_v2), ('0.2', '0.3', migrate_v2_to_v3), ('0.3', '0.4', migrate_v3_to_v4), + ('0.4', '0.5', migrate_v4_to_v5), ('0.5', '0.6', migrate_v5_to_v6), ('0.6', '0.7', migrate_v6_to_v7), + ('0.7', '0.8', migrate_v7_to_v8)), + indirect=True +) +def test_migrations(migration_data): + """Test each migration method from the `aiida.tools.importexport.migration` module.""" + version_old, version_new, metadata_old, metadata_new, data_old, data_new = migration_data + + # Remove AiiDA version, since this may change regardless of the migration function + metadata_old.pop('aiida_version') + metadata_new.pop('aiida_version') + + # Assert conversion message in `metadata.json` is correct and then remove it for later assertions + metadata_new.pop('conversion_info') + message = 'Converted from version {} to {} with AiiDA v{}'.format(version_old, version_new, get_version()) + assert metadata_old.pop('conversion_info')[-1] == message, 'Conversion message after migration is wrong' + + assert metadata_old == metadata_new + assert data_old == data_new diff --git a/tests/tools/importexport/migration/test_v01_to_v02.py b/tests/tools/importexport/migration/test_v01_to_v02.py deleted file mode 100644 index 4dcc92e27a..0000000000 --- a/tests/tools/importexport/migration/test_v01_to_v02.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Test export file migration from export version 0.1 to 0.2""" - -from aiida import get_version -from aiida.backends.testbase import AiidaTestCase -from aiida.tools.importexport.migration.utils import verify_metadata_version -from aiida.tools.importexport.migration.v01_to_v02 import migrate_v1_to_v2 - -from tests.utils.archives import get_json_files - - -class TestMigrateV01toV02(AiidaTestCase): - """Test migration of export files from export version 0.1 to 0.2""" - - def test_migrate_v1_to_v2(self): - """Test function migrate_v1_to_v2""" - # Get metadata.json and data.json as dicts from v0.1 file archive - metadata_v1, data_v1 = get_json_files('export_v0.1_simple.aiida', filepath='export/migrate') - verify_metadata_version(metadata_v1, version='0.1') - - # Get metadata.json and data.json as dicts from v0.2 file archive - metadata_v2, data_v2 = get_json_files('export_v0.2_simple.aiida', filepath='export/migrate') - verify_metadata_version(metadata_v2, version='0.2') - - # Migrate to v0.2 - migrate_v1_to_v2(metadata_v1, data_v1) - verify_metadata_version(metadata_v1, version='0.2') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v1.pop('aiida_version') - metadata_v2.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - conversion_message = 'Converted from version 0.1 to 0.2 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v1.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v2.pop('conversion_info') - - # Assert changes were performed correctly - self.maxDiff = None # pylint: disable=invalid-name - self.assertDictEqual( - metadata_v1, - metadata_v2, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v1, data_v2, msg='After migration, data.json should equal intended data.json from archives' - ) diff --git a/tests/tools/importexport/migration/test_v02_to_v03.py b/tests/tools/importexport/migration/test_v02_to_v03.py index 0d5d00b6c3..8a0a0c0cc1 100644 --- a/tests/tools/importexport/migration/test_v02_to_v03.py +++ b/tests/tools/importexport/migration/test_v02_to_v03.py @@ -7,79 +7,21 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### -"""Test export file migration from export version 0.2 to 0.3""" # pylint: disable=too-many-branches - -from aiida.backends.testbase import AiidaTestCase -from aiida.tools.importexport.migration.utils import verify_metadata_version +"""Test export file migration from export version 0.2 to 0.3""" from aiida.tools.importexport.migration.v02_to_v03 import migrate_v2_to_v3 from tests.utils.archives import get_json_files +from . import ArchiveMigrationTest -class TestMigrateV02toV03(AiidaTestCase): - """Test migration of export files from export version 0.2 to 0.3""" - - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v2_to_v3(self): - """Test function migrate_v2_to_v3""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.2 file archive - metadata_v2, data_v2 = get_json_files('export_v0.2_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v2, version='0.2') - - # Get metadata.json and data.json as dicts from v0.3 file archive - metadata_v3, data_v3 = get_json_files('export_v0.3_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v3, version='0.3') - - # Migrate to v0.3 - migrate_v2_to_v3(metadata_v2, data_v2) - verify_metadata_version(metadata_v2, version='0.3') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v2.pop('aiida_version') - metadata_v3.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - conversion_message = 'Converted from version 0.2 to 0.3 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v2.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v3.pop('conversion_info') - - # Assert changes were performed correctly - self.maxDiff = None # pylint: disable=invalid-name - self.assertDictEqual( - metadata_v2, - metadata_v3, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v2, data_v3, msg='After migration, data.json should equal intended data.json from archives' - ) - - def test_migrate_v2_to_v3_complete(self): - """Test migration for file containing complete v0.2 era possibilities""" - - # Get metadata.json and data.json as dicts from v0.2 file archive - metadata, data = get_json_files('export_v0.2.aiida', **self.external_archive) - verify_metadata_version(metadata, version='0.2') +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" - # Migrate to v0.3 - migrate_v2_to_v3(metadata, data) - verify_metadata_version(metadata, version='0.3') + def test_migrate_external(self): + """Test the migration on the test archive provided by the external test package.""" + metadata, data = self.migrate('export_v0.2.aiida', '0.2', '0.3', migrate_v2_to_v3) - self.maxDiff = None # pylint: disable=invalid-name # Check link types legal_link_types = {'unspecified', 'createlink', 'returnlink', 'inputlink', 'calllink'} for link in data['links_uuid']: @@ -137,7 +79,6 @@ def test_compare_migration_with_aiida_made(self): metadata_v3.pop('aiida_version') self.assertDictEqual(metadata_v2, metadata_v3) - self.maxDiff = None # Compare 'data.json' self.assertEqual(len(data_v2), len(data_v3)) diff --git a/tests/tools/importexport/migration/test_v03_to_v04.py b/tests/tools/importexport/migration/test_v03_to_v04.py index 8ec51bfcaf..63a7f151b0 100644 --- a/tests/tools/importexport/migration/test_v03_to_v04.py +++ b/tests/tools/importexport/migration/test_v03_to_v04.py @@ -7,13 +7,11 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### -"""Test export file migration from export version 0.3 to 0.4""" # pylint: disable=too-many-locals,too-many-branches,too-many-statements - +"""Test export file migration from export version 0.3 to 0.4""" import tarfile import zipfile -from aiida.backends.testbase import AiidaTestCase from aiida.common.exceptions import NotExistent from aiida.common.folders import SandboxFolder from aiida.common.json import load as jsonload @@ -22,79 +20,13 @@ from aiida.tools.importexport.migration.v03_to_v04 import migrate_v3_to_v4 from tests.utils.archives import get_archive_file, get_json_files +from . import ArchiveMigrationTest -class TestMigrateV03toV04(AiidaTestCase): - """Test migration of export files from export version 0.3 to 0.4""" - - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v3_to_v4(self): - """Test function migrate_v3_to_v4""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.4 file archive - metadata_v4, data_v4 = get_json_files('export_v0.4_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v4, version='0.4') - - # Get metadata.json and data.json as dicts from v0.3 file archive - # Cannot use 'get_json_files' for 'export_v0.3_simple.aiida', - # because we need to pass the SandboxFolder to 'migrate_v3_to_v4' - dirpath_archive = get_archive_file('export_v0.3_simple.aiida', **self.core_archive) - - with SandboxFolder(sandbox_in_repo=False) as folder: - if zipfile.is_zipfile(dirpath_archive): - extract_zip(dirpath_archive, folder, silent=True) - elif tarfile.is_tarfile(dirpath_archive): - extract_tar(dirpath_archive, folder, silent=True) - else: - raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') - - try: - with open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: - data_v3 = jsonload(fhandle) - with open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: - metadata_v3 = jsonload(fhandle) - except IOError: - raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) - - verify_metadata_version(metadata_v3, version='0.3') - - # Migrate to v0.4 - migrate_v3_to_v4(metadata_v3, data_v3, folder) - verify_metadata_version(metadata_v3, version='0.4') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v3.pop('aiida_version') - metadata_v4.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - self.maxDiff = None # pylint: disable=invalid-name - conversion_message = 'Converted from version 0.3 to 0.4 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v3.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v4.pop('conversion_info') - - # Assert changes were performed correctly - self.assertDictEqual( - metadata_v3, - metadata_v4, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v3, data_v4, msg='After migration, data.json should equal intended data.json from archives' - ) +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" - def test_migrate_v3_to_v4_complete(self): + def test_migrate_external(self): """Test migration for file containing complete v0.3 era possibilities""" # Get metadata.json and data.json as dicts from v0.3 file archive @@ -138,7 +70,6 @@ def test_migrate_v3_to_v4_complete(self): ## Following checks are based on the archive-file ## Which means there are more legal entities, they are simply not relevant here. - self.maxDiff = None # pylint: disable=invalid-name # Check schema-changes new_node_attrs = {'node_type', 'process_type'} for change in new_node_attrs: @@ -331,13 +262,11 @@ def test_compare_migration_with_aiida_made(self): metadata_v4, data_v4 = get_json_files('export_v0.4.aiida', **self.external_archive) # Compare 'metadata.json' - self.maxDiff = None metadata_v3.pop('conversion_info') metadata_v3.pop('aiida_version') metadata_v4.pop('aiida_version') self.assertDictEqual(metadata_v3, metadata_v4) - self.maxDiff = None # Compare 'data.json' self.assertEqual(len(data_v3), len(data_v4)) diff --git a/tests/tools/importexport/migration/test_v04_to_v05.py b/tests/tools/importexport/migration/test_v04_to_v05.py index ab1d5f62e9..664d1f495a 100644 --- a/tests/tools/importexport/migration/test_v04_to_v05.py +++ b/tests/tools/importexport/migration/test_v04_to_v05.py @@ -8,104 +8,18 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Test export file migration from export version 0.4 to 0.5""" - -import tarfile -import zipfile - -from aiida.backends.testbase import AiidaTestCase -from aiida.common.exceptions import NotExistent -from aiida.common.folders import SandboxFolder -from aiida.common.json import load as jsonload -from aiida.tools.importexport.common.archive import extract_tar, extract_zip -from aiida.tools.importexport.migration.utils import verify_metadata_version from aiida.tools.importexport.migration.v04_to_v05 import migrate_v4_to_v5 -from tests.utils.archives import get_archive_file, get_json_files - - -class TestMigrateV04toV05(AiidaTestCase): - """Test migration of export files from export version 0.4 to 0.5""" - - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v4_to_v5(self): - """Test function migrate_v4_to_v5""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.5 file archive - metadata_v5, data_v5 = get_json_files('export_v0.5_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v5, version='0.5') - - # Get metadata.json and data.json as dicts from v0.4 file archive - # Cannot use 'get_json_files' for 'export_v0.4_simple.aiida', - # because we need to pass the SandboxFolder to 'migrate_v4_to_v5' - dirpath_archive = get_archive_file('export_v0.4_simple.aiida', **self.core_archive) - - with SandboxFolder(sandbox_in_repo=False) as folder: - if zipfile.is_zipfile(dirpath_archive): - extract_zip(dirpath_archive, folder, silent=True) - elif tarfile.is_tarfile(dirpath_archive): - extract_tar(dirpath_archive, folder, silent=True) - else: - raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') - - try: - with open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: - data_v4 = jsonload(fhandle) - with open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: - metadata_v4 = jsonload(fhandle) - except IOError: - raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) - - verify_metadata_version(metadata_v4, version='0.4') - - # Migrate to v0.5 - migrate_v4_to_v5(metadata_v4, data_v4) - verify_metadata_version(metadata_v4, version='0.5') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v4.pop('aiida_version') - metadata_v5.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - # Remove also 'conversion_info' from `metadata.json` of v0.5 file archive - self.maxDiff = None # pylint: disable=invalid-name - conversion_message = 'Converted from version 0.4 to 0.5 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v4.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v5.pop('conversion_info') - - # Assert changes were performed correctly - self.assertDictEqual( - metadata_v4, - metadata_v5, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v4, data_v5, msg='After migration, data.json should equal intended data.json from archives' - ) +from . import ArchiveMigrationTest - def test_migrate_v4_to_v5_complete(self): - """Test migration for file containing complete v0.4 era possibilities""" - # Get metadata.json and data.json as dicts from v0.4 file archive - metadata, data = get_json_files('export_v0.4.aiida', **self.external_archive) - verify_metadata_version(metadata, version='0.4') +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" - # Migrate to v0.5 - migrate_v4_to_v5(metadata, data) - verify_metadata_version(metadata, version='0.5') + def test_migrate_external(self): + """Test the migration on the test archive provided by the external test package.""" + metadata, data = self.migrate('export_v0.4.aiida', '0.4', '0.5', migrate_v4_to_v5) - self.maxDiff = None # pylint: disable=invalid-name # Check schema-changes removed_computer_attrs = {'transport_params'} removed_node_attrs = {'nodeversion', 'public'} diff --git a/tests/tools/importexport/migration/test_v05_to_v06.py b/tests/tools/importexport/migration/test_v05_to_v06.py index 23bea83d46..08f1490ded 100644 --- a/tests/tools/importexport/migration/test_v05_to_v06.py +++ b/tests/tools/importexport/migration/test_v05_to_v06.py @@ -8,65 +8,25 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Test export file migration from export version 0.5 to 0.6""" - from aiida.backends.general.migrations.calc_state import STATE_MAPPING -from aiida.backends.testbase import AiidaTestCase from aiida.tools.importexport.migration.utils import verify_metadata_version from aiida.tools.importexport.migration.v05_to_v06 import migrate_v5_to_v6 from tests.utils.archives import get_json_files +from . import ArchiveMigrationTest -class TestMigrateV05toV06(AiidaTestCase): - """Test migration of export files from export version 0.5 to 0.6""" - - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v5_to_v6(self): - """Test migration for file containing complete v0.5 era possibilities""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.5 file archive - metadata_v5, data_v5 = get_json_files('export_v0.5_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v5, version='0.5') +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" - # Get metadata.json and data.json as dicts from v0.6 file archive - metadata_v6, data_v6 = get_json_files('export_v0.6_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v6, version='0.6') + def test_migrate_external(self): + """Test the migration on the test archive provided by the external test package.""" + _, data = self.migrate('export_v0.5_manual.aiida', '0.5', '0.6', migrate_v5_to_v6) - # Migrate to v0.6 - migrate_v5_to_v6(metadata_v5, data_v5) - verify_metadata_version(metadata_v5, version='0.6') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v5.pop('aiida_version') - metadata_v6.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - self.maxDiff = None # pylint: disable=invalid-name - conversion_message = 'Converted from version 0.5 to 0.6 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v5.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v6.pop('conversion_info') - - # Assert changes were performed correctly - self.assertDictEqual( - metadata_v5, - metadata_v6, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v5, data_v6, msg='After migration, data.json should equal intended data.json from archives' - ) + # Explicitly check that conversion dictionaries were removed + illegal_data_dicts = {'node_attributes_conversion', 'node_extras_conversion'} + for dict_ in illegal_data_dicts: + self.assertNotIn(dict_, data, msg="dictionary '{}' should have been removed from data.json".format(dict_)) def test_migrate_v5_to_v6_calc_states(self): """Test the data migration of legacy `JobCalcState` attributes. @@ -141,19 +101,3 @@ def test_migrate_v5_to_v6_datetime(self): 'the archive `export_v0.5_simple.aiida` did not contain a node with the attribute ' '`scheduler_lastchecktime` which is required for this test.' ) - - def test_migrate_v5_to_v6_complete(self): - """Test migration for file containing complete v0.5 era possibilities""" - # Get metadata.json and data.json as dicts from v0.5 file archive - metadata, data = get_json_files('export_v0.5_manual.aiida', **self.external_archive) - verify_metadata_version(metadata, version='0.5') - - # Migrate to v0.6 - migrate_v5_to_v6(metadata, data) - verify_metadata_version(metadata, version='0.6') - - self.maxDiff = None # pylint: disable=invalid-name - # Explicitly check that conversion dictionaries were removed - illegal_data_dicts = {'node_attributes_conversion', 'node_extras_conversion'} - for dict_ in illegal_data_dicts: - self.assertNotIn(dict_, data, msg="dictionary '{}' should have been removed from data.json".format(dict_)) diff --git a/tests/tools/importexport/migration/test_v06_to_v07.py b/tests/tools/importexport/migration/test_v06_to_v07.py index e856b43826..34f2f10d87 100644 --- a/tests/tools/importexport/migration/test_v06_to_v07.py +++ b/tests/tools/importexport/migration/test_v06_to_v07.py @@ -8,78 +8,18 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Test export file migration from export version 0.6 to 0.7""" +from aiida.tools.importexport.migration.v06_to_v07 import migrate_v6_to_v7 + +from . import ArchiveMigrationTest + + +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" + + def test_migrate_external(self): + """Test the migration on the test archive provided by the external test package.""" + metadata, data = self.migrate('export_v0.6_manual.aiida', '0.6', '0.7', migrate_v6_to_v7) -from aiida.backends.testbase import AiidaTestCase -from aiida.tools.importexport.migration.utils import verify_metadata_version -from aiida.tools.importexport.migration.v06_to_v07 import ( - migrate_v6_to_v7, migration_data_migration_legacy_process_attributes -) - -from tests.utils.archives import get_json_files - - -class TestMigrateV06toV07(AiidaTestCase): - """Test migration of export files from export version 0.6 to 0.7""" - - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v6_to_v7(self): - """Test migration for file containing complete v0.6 era possibilities""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.6 file archive - metadata_v6, data_v6 = get_json_files('export_v0.6_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v6, version='0.6') - - # Get metadata.json and data.json as dicts from v0.7 file archive - metadata_v7, data_v7 = get_json_files('export_v0.7_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v7, version='0.7') - - # Migrate to v0.7 - migrate_v6_to_v7(metadata_v6, data_v6) - verify_metadata_version(metadata_v6, version='0.7') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v6.pop('aiida_version') - metadata_v7.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - self.maxDiff = None # pylint: disable=invalid-name - conversion_message = 'Converted from version 0.6 to 0.7 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v6.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v7.pop('conversion_info') - - # Assert changes were performed correctly - self.assertDictEqual( - metadata_v6, - metadata_v7, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v6, data_v7, msg='After migration, data.json should equal intended data.json from archives' - ) - - def test_migrate_v6_to_v7_complete(self): - """Test migration for file containing complete v0.6 era possibilities""" - # Get metadata.json and data.json as dicts from v0.6 file archive - metadata, data = get_json_files('export_v0.6_manual.aiida', **self.external_archive) - verify_metadata_version(metadata, version='0.6') - - # Migrate to v0.7 - migrate_v6_to_v7(metadata, data) - verify_metadata_version(metadata, version='0.7') - - self.maxDiff = None # pylint: disable=invalid-name # Check attributes of process.* nodes illegal_attrs = {'_sealed', '_finished', '_failed', '_aborted', '_do_abort'} new_attrs = {'sealed': True} @@ -119,6 +59,7 @@ def test_migrate_v6_to_v7_complete(self): def test_migration_0040_corrupt_archive(self): """Check CorruptArchive is raised for different cases during migration 0040""" from aiida.tools.importexport.common.exceptions import CorruptArchive + from aiida.tools.importexport.migration.v06_to_v07 import migration_data_migration_legacy_process_attributes # data has one "valid" entry, in the form of Node . # At least it has the needed key `node_type`. @@ -180,6 +121,7 @@ def test_migration_0040_corrupt_archive(self): def test_migration_0040_no_process_state(self): """Check old ProcessNodes without a `process_state` can be migrated""" + from aiida.tools.importexport.migration.v06_to_v07 import migration_data_migration_legacy_process_attributes # data has one "new" entry, in the form of Node . # data also has one "old" entry, in form of Node . # It doesn't have a `process_state` attribute (nor a `sealed` or `_sealed`) diff --git a/tests/tools/importexport/migration/test_v07_to_v08.py b/tests/tools/importexport/migration/test_v07_to_v08.py index 2068abb895..65ca6dcbda 100644 --- a/tests/tools/importexport/migration/test_v07_to_v08.py +++ b/tests/tools/importexport/migration/test_v07_to_v08.py @@ -8,76 +8,18 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Test export file migration from export version 0.7 to 0.8""" +from aiida.tools.importexport.migration.v07_to_v08 import migrate_v7_to_v8, migration_default_link_label -from aiida.backends.testbase import AiidaTestCase -from aiida.tools.importexport.migration.utils import verify_metadata_version -from aiida.tools.importexport.migration.v07_to_v08 import (migrate_v7_to_v8, migration_default_link_label) +from . import ArchiveMigrationTest -from tests.utils.archives import get_json_files +class TestMigrate(ArchiveMigrationTest): + """Tests specific for this archive migration.""" -class TestMigrateV07toV08(AiidaTestCase): - """Test migration of export files from export version 0.7 to 0.8""" + def test_migrate_external(self): + """Test the migration on the test archive provided by the external test package.""" + _, data = self.migrate('export_v0.7_manual.aiida', '0.7', '0.8', migrate_v7_to_v8) - @classmethod - def setUpClass(cls, *args, **kwargs): - super().setUpClass(*args, **kwargs) - - # Utility helpers - cls.external_archive = {'filepath': 'archives', 'external_module': 'aiida-export-migration-tests'} - cls.core_archive = {'filepath': 'export/migrate'} - - def test_migrate_v7_to_v8(self): - """Test migration for file containing complete v0.7 era possibilities""" - from aiida import get_version - - # Get metadata.json and data.json as dicts from v0.7 file archive - metadata_v7, data_v7 = get_json_files('export_v0.7_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v7, version='0.7') - - # Get metadata.json and data.json as dicts from v0.8 file archive - metadata_v8, data_v8 = get_json_files('export_v0.8_simple.aiida', **self.core_archive) - verify_metadata_version(metadata_v8, version='0.8') - - # Migrate to v0.8 - migrate_v7_to_v8(metadata_v7, data_v7) - verify_metadata_version(metadata_v7, version='0.8') - - # Remove AiiDA version, since this may change irregardless of the migration function - metadata_v7.pop('aiida_version') - metadata_v8.pop('aiida_version') - - # Assert conversion message in `metadata.json` is correct and then remove it for later assertions - self.maxDiff = None # pylint: disable=invalid-name - conversion_message = 'Converted from version 0.7 to 0.8 with AiiDA v{}'.format(get_version()) - self.assertEqual( - metadata_v7.pop('conversion_info')[-1], - conversion_message, - msg='The conversion message after migration is wrong' - ) - metadata_v8.pop('conversion_info') - - # Assert changes were performed correctly - self.assertDictEqual( - metadata_v7, - metadata_v8, - msg='After migration, metadata.json should equal intended metadata.json from archives' - ) - self.assertDictEqual( - data_v7, data_v8, msg='After migration, data.json should equal intended data.json from archives' - ) - - def test_migrate_v7_to_v8_complete(self): - """Test migration for file containing complete v0.7 era possibilities""" - # Get metadata.json and data.json as dicts from v0.7 file archive - metadata, data = get_json_files('export_v0.7_manual.aiida', **self.external_archive) - verify_metadata_version(metadata, version='0.7') - - # Migrate to v0.8 - migrate_v7_to_v8(metadata, data) - verify_metadata_version(metadata, version='0.8') - - self.maxDiff = None # pylint: disable=invalid-name # Check that no links have the label '_return', since it should now be 'result' illegal_label = '_return' for link in data.get('links_uuid'):