From 48ae33e9f5f20febbcc6daa2ade7c0d17325e433 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 5 Nov 2021 15:15:37 -0400 Subject: [PATCH 1/8] Update Migration V0021 Signed-off-by: Paul Bastide --- .../database/utils/db2/Db2CatalogAccess.java | 4 +- .../java/com/ibm/fhir/schema/app/Main.java | 152 ++++++----- .../schema/control/FhirSchemaGenerator.java | 10 +- .../schema/control/FhirSchemaVersion.java | 1 + ...14LogicalResourceIsDeletedLastUpdated.java | 5 +- .../MigrateV0021AbstractTypeRemoval.java | 247 ++++++++++++++++++ ...UnusedTableRemovalNeedsV0021Migration.java | 168 ++++++++++++ 7 files changed, 510 insertions(+), 77 deletions(-) create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2CatalogAccess.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2CatalogAccess.java index fb76f434b2a..063b0213219 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2CatalogAccess.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2CatalogAccess.java @@ -23,14 +23,14 @@ */ public class Db2CatalogAccess implements ICatalogAccess { private static final Logger logger = Logger.getLogger(Db2CatalogAccess.class.getName()); - + private Connection connection; private Db2CatalogDAO dao = new Db2CatalogDAO(new Db2Translator()); public Db2CatalogAccess(Connection c) { this.connection = c; } - + @Override public List getPartitionList(String schema, String table) { try { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index ad3ef2c3763..964df12c803 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -72,9 +72,9 @@ import com.ibm.fhir.database.utils.transaction.SimpleTransactionProvider; import com.ibm.fhir.database.utils.transaction.TransactionFactory; import com.ibm.fhir.database.utils.version.CreateControl; +import com.ibm.fhir.database.utils.version.CreateVersionHistory; import com.ibm.fhir.database.utils.version.CreateWholeSchemaVersion; import com.ibm.fhir.database.utils.version.SchemaConstants; -import com.ibm.fhir.database.utils.version.CreateVersionHistory; import com.ibm.fhir.database.utils.version.VersionHistoryService; import com.ibm.fhir.model.util.ModelSupport; import com.ibm.fhir.schema.api.ConcurrentUpdateException; @@ -96,11 +96,13 @@ import com.ibm.fhir.schema.control.InitializeLogicalResourceDenorms; import com.ibm.fhir.schema.control.JavaBatchSchemaGenerator; import com.ibm.fhir.schema.control.MigrateV0014LogicalResourceIsDeletedLastUpdated; +import com.ibm.fhir.schema.control.MigrateV0021AbstractTypeRemoval; import com.ibm.fhir.schema.control.OAuthSchemaGenerator; import com.ibm.fhir.schema.control.PopulateParameterNames; import com.ibm.fhir.schema.control.PopulateResourceTypes; import com.ibm.fhir.schema.control.SetTenantIdDb2; import com.ibm.fhir.schema.control.TenantInfo; +import com.ibm.fhir.schema.control.UnusedTableRemovalNeedsV0021Migration; import com.ibm.fhir.schema.model.ResourceType; import com.ibm.fhir.schema.model.Schema; import com.ibm.fhir.task.api.ITaskCollector; @@ -178,7 +180,7 @@ public class Main { private int vacuumCostLimit = 2000; private int vacuumThreshold = 1000; private Double vacuumScaleFactor = null; - + // How many seconds to wait to obtain the update lease private int waitForUpdateLeaseSeconds = 10; @@ -203,6 +205,9 @@ public class Main { private boolean revokeTenantKey; private boolean revokeAllTenantKeys; + // Forces the removal of tables if data exists + private boolean forceUnusedTableRemoval = false; + // Tenant Key Output or Input File private String tenantKeyFileName; private TenantKeyFileUtil tenantKeyFileUtil = new TenantKeyFileUtil(); @@ -217,7 +222,7 @@ public class Main { private int maxConnectionPoolSize = FhirSchemaConstants.DEFAULT_POOL_SIZE; private PoolConnectionProvider connectionPool; private ITransactionProvider transactionProvider; - + // Configuration to control how the LeaseManager operates private ILeaseManagerConfig leaseManagerConfig; @@ -269,10 +274,10 @@ protected void buildAdminSchemaModel(PhysicalDataModel pdm) { FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); gen.buildAdminSchema(pdm); } - + /** * Add the OAuth schema objects to the given {@link PhysicalDataModel} - * + * * @param pdm the model to build */ protected void buildOAuthSchemaModel(PhysicalDataModel pdm) { @@ -283,7 +288,7 @@ protected void buildOAuthSchemaModel(PhysicalDataModel pdm) { /** * Add the JavaBatch schema objects to the given {@link PhysicalDataModel} - * + * * @param pdm */ protected void buildJavaBatchSchemaModel(PhysicalDataModel pdm) { @@ -368,12 +373,12 @@ protected void createSchemas() { throw translator.translate(x); } } - + /** * Process the schemas configured to be updated */ protected void updateSchemas() { - + // Make sure that we have the CONTROL table created before we try any // schema update work IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); @@ -384,16 +389,16 @@ protected void updateSchemas() { if (updateFhirSchema) { updateFhirSchema(); } - + if (updateOauthSchema) { updateOauthSchema(); } - + if (updateJavaBatchSchema) { updateJavaBatchSchema(); } } - + /** * Add FHIR data schema objects to the given {@link PhysicalDataModel} * @param pdm the data model being built @@ -421,7 +426,7 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { throw new IllegalStateException("Unsupported db type: " + dbType); } } - + /** * Update the FHIR data schema */ @@ -433,13 +438,13 @@ protected void updateFhirSchema() { if (!leaseManager.waitForLease(waitForUpdateLeaseSeconds)) { throw new ConcurrentUpdateException("Concurrent update for FHIR data schema: '" + schema.getSchemaName() + "'"); } - + final String targetSchemaName = schema.getSchemaName(); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); } - + // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName); if (svm.isLatestSchema()) { @@ -449,39 +454,38 @@ protected void updateFhirSchema() { PhysicalDataModel pdm = new PhysicalDataModel(); buildFhirDataSchemaModel(pdm); boolean isNewDb = updateSchema(pdm); - + // If the db is multi-tenant, we populate the resource types and parameter names in allocate-tenant. // Otherwise, if its a new schema, populate the resource types and parameters names (codes) now if (!MULTITENANT_FEATURE_ENABLED.contains(dbType) && isNewDb) { populateResourceTypeAndParameterNameTableEntries(null); } - + if (MULTITENANT_FEATURE_ENABLED.contains(dbType)) { logger.info("Refreshing tenant partitions"); refreshTenants(); } - + // backfill the resource_change_log table if needed backfillResourceChangeLog(); - + // perform any updates we need related to the V0010 schema change (IS_DELETED flag) applyDataMigrationForV0010(); - + // V0014 IS_DELETED and LAST_UPDATED added to whole-system LOGICAL_RESOURCES applyDataMigrationForV0014(); - - // Log warning messages that unused tables will be removed in a future release. - // TODO: This will no longer be needed after the tables are removed (https://github.com/IBM/FHIR/issues/713). - logWarningMessagesForDeprecatedTables(); - + + // V0021 removes Abstract Type tables which are unused.s + applyTableRemovalForV0021(); + // Apply privileges if asked if (grantTo != null) { grantPrivilegesForFhirData(); } - + // Finally, update the whole schema version svm.updateSchemaVersion(); - + } } finally { leaseManager.cancelLease(); @@ -499,13 +503,13 @@ protected void updateOauthSchema() { if (!leaseManager.waitForLease(waitForUpdateLeaseSeconds)) { throw new ConcurrentUpdateException("Concurrent update for Liberty OAuth schema: '" + schema.getOauthSchemaName() + "'"); } - + final String targetSchemaName = schema.getOauthSchemaName(); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); } - + // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName); if (svm.isLatestSchema()) { @@ -514,12 +518,12 @@ protected void updateOauthSchema() { PhysicalDataModel pdm = new PhysicalDataModel(); buildOAuthSchemaModel(pdm); updateSchema(pdm); - + // Apply privileges if asked if (grantTo != null) { grantPrivilegesForOAuth(); } - + // Mark the schema as up-to-date svm.updateSchemaVersion(); } @@ -539,7 +543,7 @@ protected void updateJavaBatchSchema() { if (!leaseManager.waitForLease(waitForUpdateLeaseSeconds)) { throw new ConcurrentUpdateException("Concurrent update for Liberty JavaBatch schema: '" + schema.getJavaBatchSchemaName() + "'"); } - + final String targetSchemaName = schema.getJavaBatchSchemaName(); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { @@ -554,7 +558,7 @@ protected void updateJavaBatchSchema() { PhysicalDataModel pdm = new PhysicalDataModel(); buildJavaBatchSchemaModel(pdm); updateSchema(pdm); - + // Apply privileges if asked if (grantTo != null) { grantPrivilegesForBatch(); @@ -567,7 +571,7 @@ protected void updateJavaBatchSchema() { leaseManager.cancelLease(); } } - + /** * Update the schema associated with the given {@link PhysicalDataModel} * @return true if the database is new @@ -583,7 +587,7 @@ protected boolean updateSchema(PhysicalDataModel pdm) { IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); // Before we start anything, we need to make sure our schema history - // and control tables are in place. These tables are used to manage + // and control tables are in place. These tables are used to manage // all FHIR data, oauth and JavaBatch schemas we build try (ITransaction tx = transactionProvider.getTransaction()) { CreateVersionHistory.createTableIfNeeded(schema.getAdminSchemaName(), adapter); @@ -602,7 +606,7 @@ protected boolean updateSchema(PhysicalDataModel pdm) { applyModel(pdm, adapter, collector, vhs); // The physical database objects should now match what was defined in the PhysicalDataModel - + return isNewDb; } @@ -789,7 +793,7 @@ protected void updateProcedures() { IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); pdm.applyProcedures(adapter); pdm.applyFunctions(adapter); - + // Because we're replacing the procedures, we should also check if // we need to apply the associated privileges if (this.grantTo != null) { @@ -811,7 +815,7 @@ protected void updateProcedures() { /** * Build a common PhysicalDataModel containing all the requested schemas - * + * * @param pdm the model to construct * @param addFhirDataSchema include objects for the FHIR data schema * @param addOAuthSchema include objects for the Liberty OAuth schema @@ -821,11 +825,11 @@ protected void buildCommonModel(PhysicalDataModel pdm, boolean addFhirDataSchema if (addFhirDataSchema) { buildFhirDataSchemaModel(pdm); } - + if (addOAuthSchema) { buildOAuthSchemaModel(pdm); } - + if (addJavaBatchSchema) { buildJavaBatchSchemaModel(pdm); } @@ -842,7 +846,7 @@ protected void grantPrivilegesForFhirData() { PhysicalDataModel pdm = new PhysicalDataModel(); buildFhirDataSchemaModel(pdm); pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); - + // Grant SELECT on WHOLE_SCHEMA_VERSION to the FHIR server user // Note the constant comes from SchemaConstants on purpose CreateWholeSchemaVersion.grantPrivilegesTo(adapter, schema.getSchemaName(), SchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); @@ -864,14 +868,14 @@ protected void grantPrivilegesForOAuth() { PhysicalDataModel pdm = new PhysicalDataModel(); buildOAuthSchemaModel(pdm); pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); - + } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed tx.setRollbackOnly(); throw x; } } - + } /** @@ -909,15 +913,15 @@ protected void grantPrivileges() { grantFhirSchema = true; grantJavaBatchSchema = true; } - + if (grantFhirSchema) { grantPrivilegesForFhirData(); } - + if (grantOauthSchema) { grantPrivilegesForOAuth(); } - + if (grantJavaBatchSchema) { grantPrivilegesForBatch(); } @@ -1064,7 +1068,7 @@ protected void allocateTenant() { // IMPORTANT! Check the schema name aligns with the actual schema for this tenant checkSchemaForTenant(); - + // The key we'll use for this tenant. This key should be used in subsequent // activities related to this tenant, such as setting the tenant context. if (tenantKeyFileUtil.keyFileExists(tenantKeyFileName)) { @@ -1074,27 +1078,27 @@ protected void allocateTenant() { } else { tenantKey = getRandomKey(); } - + // The salt is used when we hash the tenantKey. We're just using SHA-256 for // the hash here, not multiple rounds of a password hashing algorithm. It's // sufficient in our case because we are using a 32-byte random value as the // key, giving 256 bits of entropy. final String tenantSalt = getRandomKey(); - + Db2Adapter adapter = new Db2Adapter(connectionPool); - + // Conditionally skip if the tenant name and key exist (this enables idempotency) boolean skip = checkIfTenantNameAndTenantKeyExists(adapter, tenantName, tenantKey, skipIfTenantExists); if (skip) { return; } - + if (tenantKeyFileName == null) { logger.info("Allocating new tenant: " + tenantName + " [key=" + tenantKey + "]"); } else { logger.info("Allocating new tenant: " + tenantName + " [tenantKeyFileName=" + tenantKeyFileName + "]"); } - + // Open a new transaction and associate it with our connection pool. Remember // that we don't support distributed transactions, so all connections within // this transaction must come from the same pool @@ -1103,7 +1107,7 @@ protected void allocateTenant() { try { tenantId = adapter.allocateTenant(schema.getAdminSchemaName(), schema.getSchemaName(), tenantName, tenantKey, tenantSalt, FhirSchemaConstants.TENANT_SEQUENCE); - + // The tenant-id is important because this is also used to identify the partition number logger.info("Tenant Id[" + tenantName + "] = [" + tenantId + "]"); } catch (DataAccessException x) { @@ -1112,23 +1116,23 @@ protected void allocateTenant() { throw x; } } - + // Build/update the tables as well as the stored procedures FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); PhysicalDataModel pdm = new PhysicalDataModel(); gen.buildSchema(pdm); - + // Get the data model to create the table partitions. This is threaded, so transactions are // handled within each thread by the adapter. This means we should probably pull some of // that logic out of the adapter and handle it at a higher level. Note...the extent size used // for the partitions needs to match the extent size of the original table tablespace (FHIR_TS) // so this must be constant. pdm.addTenantPartitions(adapter, schema.getSchemaName(), tenantId, FhirSchemaConstants.FHIR_TS_EXTENT_KB); - + // Fill any static data tables (which are also partitioned by tenant) // Prepopulate the Resource Type Tables and Parameters Name/Code Table populateResourceTypeAndParameterNameTableEntries(tenantId); - + // Now all the table partitions have been allocated, we can mark the tenant as ready try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { @@ -1139,13 +1143,13 @@ protected void allocateTenant() { throw x; } } - + if (grantTo != null) { // If the --grant-to has been given, we also need to apply that here, although // this really ought to be done when the schema is first built grantPrivileges(); } - + if (tenantKeyFileName == null) { logger.info("Allocated tenant: " + tenantName + " [key=" + tenantKey + "] with Id = " + tenantId); logger.info("The tenantKey JSON follows: \t\n{\"tenantKey\": \"" + tenantKey + "\"}"); @@ -1672,7 +1676,7 @@ protected void parseArgs(String[] args) { lmConfig.withHost(new HostnameHandler().getHostname()); lmConfig.withLeaseTimeSeconds(100); // default lmConfig.withStayAlive(true); // default - + for (int i = 0; i < args.length; i++) { int nextIdx = (i + 1); String arg = args[i]; @@ -1993,11 +1997,14 @@ protected void parseArgs(String[] args) { case "--skip-allocate-if-tenant-exists": skipIfTenantExists = true; break; + case "--force-unused-table-removal": + forceUnusedTableRemoval = true; + break; default: throw new IllegalArgumentException("Invalid argument: '" + arg + "'"); } } - + this.leaseManagerConfig = lmConfig.build(); } @@ -2534,14 +2541,25 @@ protected void logStatusMessage(int status) { } /** - * Log warning messages for deprecated tables. + * Removes RESOURCE and DOMAIN_RESOURCE tables from the fhir data schemas. */ - private void logWarningMessagesForDeprecatedTables() { - List deprecatedTables = - Arrays.asList("DOMAINRESOURCE_DATE_VALUES", "DOMAINRESOURCE_LATLNG_VALUES", "DOMAINRESOURCE_LOGICAL_RESOURCES", "DOMAINRESOURCE_NUMBER_VALUES", "DOMAINRESOURCE_QUANTITY_VALUES", "DOMAINRESOURCE_RESOURCE_TOKEN_REFS", "DOMAINRESOURCE_RESOURCES", "DOMAINRESOURCE_STR_VALUES", "RESOURCE_DATE_VALUES", "RESOURCE_LATLNG_VALUES", "RESOURCE_LOGICAL_RESOURCES", "RESOURCE_NUMBER_VALUES", "RESOURCE_QUANTITY_VALUES", "RESOURCE_RESOURCE_TOKEN_REFS", "RESOURCE_RESOURCES", "RESOURCE_STR_VALUES"); - for (String deprecatedTable : deprecatedTables) { - logger.warning("Table '" + deprecatedTable - + "' will be dropped in a future release. No data should be written to this table. If any data exists in the table, that data should be deleted."); + private void applyTableRemovalForV0021() { + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + String adminSchemaName = this.schema.getAdminSchemaName(); + String schemaName = this.schema.getSchemaName(); + UnusedTableRemovalNeedsV0021Migration needsMigrating = new UnusedTableRemovalNeedsV0021Migration(schemaName); + if (adapter.runStatement(needsMigrating)) { + logger.info("V0021 Migration: Removing Abstract Tables from schema '" + schemaName + "'"); + MigrateV0021AbstractTypeRemoval cmd = new MigrateV0021AbstractTypeRemoval(adapter, adminSchemaName, schemaName, forceUnusedTableRemoval); + adapter.runStatement(cmd); + } + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } } } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 5ec8cec6b31..4be36675741 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -124,8 +124,8 @@ public class FhirSchemaGenerator { // Build the multitenant variant of the schema private final boolean multitenant; - // TODO pass 'false' to getResourceTypes to avoid building tables for abstract resource types - private static final Set ALL_RESOURCE_TYPES = ModelSupport.getResourceTypes(true).stream() + // No abstract types + private static final Set ALL_RESOURCE_TYPES = ModelSupport.getResourceTypes(false).stream() .map(t -> ModelSupport.getTypeName(t).toUpperCase()) .collect(Collectors.toSet()); @@ -450,7 +450,7 @@ public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) { procedurePrivileges); pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); final ProcedureDef deleteResourceParameters = pd; - + pd = model.addProcedure(this.schemaName, ADD_ANY_RESOURCE, FhirSchemaVersion.V0001.vid(), @@ -607,7 +607,7 @@ public void addLogicalResources(PhysicalDataModel pdm) { if (priorVersion < FhirSchemaVersion.V0019.vid()) { statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); } - + if (priorVersion < FhirSchemaVersion.V0020.vid()) { statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); } @@ -807,7 +807,7 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { */ public void addResourceChangeLog(PhysicalDataModel pdm) { final String tableName = RESOURCE_CHANGE_LOG; - + // custom list of Withs because this table does not require fillfactor tuned in V0020 List customWiths = Arrays.asList( With.with("autovacuum_vacuum_scale_factor", "0.01"), // V0019 diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java index 50292d5f2f0..b215a8de5f3 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java @@ -39,6 +39,7 @@ public enum FhirSchemaVersion { ,V0018(18, "issue-1822 add optimized settings for postgres vacuum tables", true) ,V0019(19, "issue-1822 changes per the IBM Cloud Database Team", true) ,V0020(20, "issue-1834 Set PostgreSQL fillfactor", true) + ,V0021(21, "issue-713 remove Resource_LOGICAL_RESOURCES, DomainResource_LOGICAL_RESOURCES tables", false) ; // The version number recorded in the VERSION_HISTORY diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java index 4a4b05dc95f..7fb73e98a2e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0014LogicalResourceIsDeletedLastUpdated.java @@ -21,7 +21,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.DbType; -import com.ibm.fhir.database.utils.version.SchemaConstants; /** * Run a correlated update statement to update the new V0014 columns in LOGICAL_RESOURCES from the @@ -49,7 +48,8 @@ public class MigrateV0014LogicalResourceIsDeletedLastUpdated implements IDatabas /** * Public constructor * @param schemaName - * @param tableName + * @param resourceTypeName + * @param resourceTypeId */ public MigrateV0014LogicalResourceIsDeletedLastUpdated(String schemaName, String resourceTypeName, int resourceTypeId) { this.schemaName = schemaName; @@ -83,7 +83,6 @@ private void runCorrelatedUpdate(IDatabaseTranslator translator, Connection c) { + " WHERE tgt.logical_resource_id IN " + "(SELECT tgt2.logical_resource_id FROM " + tgtTable + " tgt2" + " WHERE tgt2.resource_type_id = " + this.resourceTypeId + " AND tgt2.is_deleted = 'X' " + translator.limit(MAX_CORRELATED_UPDATE_ROWS + "") + ")"; - ; try (PreparedStatement ps = c.prepareStatement(DML)) { int count = 1; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java new file mode 100644 index 00000000000..5000c8deda9 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -0,0 +1,247 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.DataAccessException; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.PartitionInfo; +import com.ibm.fhir.database.utils.db2.Db2CatalogAccess; +import com.ibm.fhir.database.utils.model.DbType; + +/** + * DOMAINRESOURCE_ and RESOURCE_ are abstract tables which are unused. + * V0021 does the following: + * 1 - Check the Tables are Empty + * 2 - Check if not empty or forced + * 3 - Remove the Tables/Indices and Views + */ +public class MigrateV0021AbstractTypeRemoval implements IDatabaseStatement { + + private static final Logger LOG = Logger.getLogger(MigrateV0021AbstractTypeRemoval.class.getName()); + + // The Adapter + private final IDatabaseAdapter adapter; + + // The FHIR Admin schema + private final String adminSchemaName; + + // The FHIR data schema + private final String schemaName; + + // Force + private final boolean force; + + // Total number of Tables that have data. + private int count = 0; + + /** + * Public constructor + * + * @param adapter + * @param adminSchemaName + * @param schemaName + * @param force - indicating if their is data in these tables, it'll still remove the table. + */ + public MigrateV0021AbstractTypeRemoval(IDatabaseAdapter adapter, String adminSchemaName, String schemaName, boolean force) { + this.adapter = adapter; + this.adminSchemaName = adminSchemaName; + this.schemaName = schemaName; + this.force = force; + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + switch (translator.getType()) { + case POSTGRESQL: + case DERBY: + checkDataTables(translator, c); + checkShouldThrowException(); + removeBaseArtifacts(translator, c); + break; + case DB2: + checkDataInAllPartitionedTenantTables(translator, c); + checkShouldThrowException(); + removePartitions(translator, c); + removeBaseArtifacts(translator, c); + break; + } + cleanupHistory(translator, c); + } + + /** + * Controls the throwing of Exceptions. + */ + private void checkShouldThrowException() { + if (count != 0 && !force) { + throw new IllegalArgumentException("The Data Tables for Resources or DomainResource contains data. use --force-unused-table-removal to force removal"); + } + } + + /** + * cleans up the the VersionHistoryService for a specific schema. + * + * This method is idempotent. run... run... run... and it'll cleanup the VHS tables - table and view, + * + * @param translator + * @param c + */ + private void cleanupHistory(IDatabaseTranslator translator, Connection c) { + // Clean up the Tables and Views and Index for the DomainResource and Resource Table Group. + final String sql = + "DELETE FROM FHIR_ADMIN.VERSION_HISTORY" + + " WHERE (OBJECT_NAME LIKE 'DOMAINRESOURCE_%'" + + " OR OBJECT_NAME LIKE 'RESOURCE_%')" + + " AND SCHEMA_NAME = ?"; + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, schemaName); + int vhsChanged = ps.executeUpdate(); + LOG.info("VersionHistoryServce: removed =[" + vhsChanged + "]"); + } catch (SQLException x) { + throw translator.translate(x); + } + } + + /** + * checks the data tables on all partitions + * @param translator + * @param c + */ + private void checkDataInAllPartitionedTenantTables(IDatabaseTranslator translator, Connection c) { + GetTenantList list = new GetTenantList(adminSchemaName); + List tenants = list.run(translator, c); + + for (TenantInfo tenant : tenants) { + final String stmtVariable = String.format("SET %s.SV_TENANT_ID = %d", adminSchemaName, tenant.getTenantId()); + try (Statement s = c.createStatement();) { + s.execute(stmtVariable); + } catch (SQLException x) { + throw translator.translate(x); + } + checkDataTables(translator, c); + } + } + + /** + * checks the data tables for any content in the deprecated table. + * + * @param translator + * @param c + */ + private void checkDataTables(IDatabaseTranslator translator, Connection c) { + for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { + final String table = schemaName + "." + deprecatedTable; + + // When checking for data... SYSCAT.TABLES->CARD was considered to be checked. + // However if the db hasn't collected statistics -1 is returned, and unreliable + // Instead we're going to check if it has at least one row... + final String sql = "SELECT * FROM " + table + " " + translator.limit("1"); + try (PreparedStatement ps = c.prepareStatement(sql)) { + if (ps.execute()) { + ResultSet rs = ps.getResultSet(); + if (rs.next()) { + LOG.warning("Data Table contains data '" + table + "'"); + count++; + } + } + } catch (SQLException x) { + throw translator.translate(x); + } + } + } + + /** + * removes the base artifacts from the database. + * @param translator + * @param c + */ + private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { + List tables = Arrays.asList("DOMAINRESOURCE_", "RESOURCE_"); + List valueTypes = Arrays.asList( + "DATE_VALUES", + "LATLNG_VALUES", + "NUMBER_VALUES", + "QUANTITY_VALUES", + "RESOURCE_TOKEN_REFS", + "STR_VALUES", + "PROFILES", + "TAGS", + "SECURITY"); + + // Run across both tables + for (String tablePrefix : tables) { + // Drop the Values Tables + for (String valueType : valueTypes) { + adapter.dropTable(schemaName, tablePrefix + valueType); + } + + // Drop the supporting tables + adapter.dropTable(schemaName, tablePrefix + "logical_resources"); + adapter.dropTable(schemaName, tablePrefix + "resources"); + + // Drop the View for the Table + adapter.dropView(schemaName, tablePrefix + "_token_values_V"); + } + + // Pattern for multitenant is to prefix tables with DRP_ + String prefix = ""; + if (translator.getType() == DbType.DB2) { + prefix = "DRP_"; + } + + // Drop the tables, when the tables don't exists the errors are swallowed + // and logs print warnings saying the tables don't exist. That's OK. + for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { + adapter.dropTable(schemaName, prefix + deprecatedTable); + } + } + + /** + * remove partitions + * @param translator + * @param c + */ + private void removePartitions(IDatabaseTranslator translator, Connection c) { + // Need to drop the partitions first and check all the partitions. + Db2CatalogAccess catalog = new Db2CatalogAccess(c); + for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { + List pis = catalog.getPartitionList(schemaName, deprecatedTable); + for (PartitionInfo pi : pis) { + try { + catalog.dropPartition(schemaName, deprecatedTable, pi); + } catch (DataAccessException ex) { + // SQLCODE=-20251, SQLSTATE=428G means it's the last partition https://www.ibm.com/docs/en/db2/11.1?topic=messages-sql20250-sql20499 + if (ex.getCause() instanceof SQLException) { + SQLException sqe = (SQLException) ex.getCause(); + if (sqe.getErrorCode() == -20251 && "428G2".equals(sqe.getSQLState())) { + LOG.info("Partition is last one and can't be dropped"); + continue; + } + if (sqe.getErrorCode() == -270 && "42997".equals(sqe.getSQLState())) { + LOG.info("Partition is already dropped"); + continue; + } + } + throw ex; + } + } + } + + // Remove the Base artifacts + removeBaseArtifacts(translator, c); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java new file mode 100644 index 00000000000..9ee8b9d6fae --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java @@ -0,0 +1,168 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; + +/** + * Checks to see if any of the tables exist in the target database. + */ +public class UnusedTableRemovalNeedsV0021Migration implements IDatabaseSupplier { + + // List of Tables to check for removal + public static final List DEPRECATED_TABLES = + Arrays.asList( + // DOMAINRESOURCE + "DOMAINRESOURCE_DATE_VALUES", + "DOMAINRESOURCE_LATLNG_VALUES", + "DOMAINRESOURCE_LOGICAL_RESOURCES", + "DOMAINRESOURCE_NUMBER_VALUES", + "DOMAINRESOURCE_PROFILES", + "DOMAINRESOURCE_QUANTITY_VALUES", + "DOMAINRESOURCE_RESOURCE_TOKEN_REFS", + "DOMAINRESOURCE_RESOURCES", + "DOMAINRESOURCE_SECURITY", + "DOMAINRESOURCE_STR_VALUES", + "DOMAINRESOURCE_TAGS", + // RESOURCE + "RESOURCE_DATE_VALUES", + "RESOURCE_LATLNG_VALUES", + "RESOURCE_LOGICAL_RESOURCES", + "RESOURCE_NUMBER_VALUES", + "RESOURCE_PROFILES", + "RESOURCE_QUANTITY_VALUES", + "RESOURCE_RESOURCE_TOKEN_REFS", + "RESOURCE_RESOURCES", + "RESOURCE_SECURITY", + "RESOURCE_STR_VALUES", + "RESOURCE_TAGS"); + + // Table Count + public static final int TABLE_COUNT = DEPRECATED_TABLES.size(); + + // The FHIR data schema + private final String schemaName; + + /** + * Public Constructor + * @param schemaName + */ + public UnusedTableRemovalNeedsV0021Migration(String schemaName) { + this.schemaName = schemaName; + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + switch (translator.getType()) { + case POSTGRESQL: + return checkPostgres(translator, c); + case DB2: + return checkDb2(translator, c); + case DERBY: + return checkDerby(translator, c); + default: + // if another database is supported, we won't have the tables at this point. + // Therefore it's always going to be false. + return false; + } + } + + /** + * Checks the db2 syscatalog for the tables. + * + * @param translator + * @param c + * @return + */ + public boolean checkDb2(IDatabaseTranslator translator, Connection c) { + final String sql = "SELECT tabname FROM SYSCAT.TABLES TABS" + + " WHERE TABS.tabschema = ? " + + " AND TABS.tabname in ( " + addParameterMarkers(TABLE_COUNT) + " )"; + return hasTables(translator, c, sql); + } + + /** + * Checks db2's sys catalog + * + * @param translator + * @param c + * @return + */ + public boolean checkDerby(IDatabaseTranslator translator, Connection c) { + // Grab the list of tables for the configured schema from the Derby sys catalog + final String sql = "" + + "SELECT tables.tablename FROM sys.systables AS tables " + + " JOIN sys.sysschemas AS schemas " + + " ON (tables.schemaid = schemas.schemaid) " + + " WHERE schemas.schemaname = ?" + + " AND schemas.tablename in (" + addParameterMarkers(TABLE_COUNT) + ")"; + return hasTables(translator, c, sql); + } + + /** + * Checks postgresql for the tables in the schema + * + * @param translator + * @param c + * @return + */ + public boolean checkPostgres(IDatabaseTranslator translator, Connection c) { + // Grab the list of tables for the configured schema from the PostgreSQL schema + // catalog + final String sql = "" + + "SELECT table_name FROM information_schema.tables " + + " WHERE table_schema = ?" + + " AND table_name in (" + addParameterMarkers(TABLE_COUNT) + ")"; + return hasTables(translator, c, sql); + } + + /** + * builds the parameter markers + * @param numOfParameters + * @return + */ + private String addParameterMarkers(int numOfParameters) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < numOfParameters; i++) { + builder.append(" ? "); + if (numOfParameters-1 != i) { + builder.append(","); + } + } + return builder.toString(); + } + + /** + * + * @param translator + * @param c + * @param sql + * @return + */ + private boolean hasTables(IDatabaseTranslator translator, Connection c, final String sql) { + try (PreparedStatement ps = c.prepareStatement(sql)) { + int i = 1; + ps.setString(i++, schemaName); + + for (String deprecatedTable : DEPRECATED_TABLES) { + ps.setString(i++, deprecatedTable); + } + ResultSet rs = ps.executeQuery(); + return rs.next(); + } catch (SQLException x) { + throw translator.translate(x); + } + } +} \ No newline at end of file From f94fb7fecc1baa701447481d7bc5878ec6749f6c Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 5 Nov 2021 16:07:25 -0400 Subject: [PATCH 2/8] Avoid creating unnecessary tables for Resource and DomainResource resource types #713 - Checks for DOMAINRESOURCE and RESOURCE tables view - Removes the artifacts - Bonus: added fix for null condition on a code in DefaultMemberMatchStrategy Signed-off-by: Paul Bastide --- .../java/com/ibm/fhir/schema/app/Main.java | 1 + .../MigrateV0021AbstractTypeRemoval.java | 68 +++++++++++++------ ...UnusedTableRemovalNeedsV0021Migration.java | 15 ++-- .../schema/derby/DerbyFhirDatabaseTest.java | 14 ++-- .../schema/derby/DerbySchemaVersionsTest.java | 12 ++-- .../strategy/DefaultMemberMatchStrategy.java | 3 +- 6 files changed, 72 insertions(+), 41 deletions(-) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index ce03d931753..132b49f6d3c 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -2561,6 +2561,7 @@ private void applyTableRemovalForV0021() { logger.info("V0021 Migration: Removing Abstract Tables from schema '" + schemaName + "'"); MigrateV0021AbstractTypeRemoval cmd = new MigrateV0021AbstractTypeRemoval(adapter, adminSchemaName, schemaName, forceUnusedTableRemoval); adapter.runStatement(cmd); + logger.info("V0021 Migration: Completed the Removal of the Abstract Tables from schema '" + schemaName + "'"); } } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java index 5000c8deda9..21ca78dae5c 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.List; import java.util.logging.Logger; +import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; @@ -34,6 +35,19 @@ public class MigrateV0021AbstractTypeRemoval implements IDatabaseStatement { private static final Logger LOG = Logger.getLogger(MigrateV0021AbstractTypeRemoval.class.getName()); + private static final List VALUE_TYPES = Arrays.asList( + "RESOURCE_TOKEN_REFS", + "DATE_VALUES", + "LATLNG_VALUES", + "NUMBER_VALUES", + "QUANTITY_VALUES", + "STR_VALUES", + "PROFILES", + "TAGS", + "SECURITY"); + + private static final List VALUE_TYPES_LOWER = VALUE_TYPES.stream().map(s -> s.toLowerCase()).collect(Collectors.toList()); + // The Adapter private final IDatabaseAdapter adapter; @@ -171,30 +185,16 @@ private void checkDataTables(IDatabaseTranslator translator, Connection c) { */ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { List tables = Arrays.asList("DOMAINRESOURCE_", "RESOURCE_"); - List valueTypes = Arrays.asList( - "DATE_VALUES", - "LATLNG_VALUES", - "NUMBER_VALUES", - "QUANTITY_VALUES", - "RESOURCE_TOKEN_REFS", - "STR_VALUES", - "PROFILES", - "TAGS", - "SECURITY"); + // Run across both tables for (String tablePrefix : tables) { - // Drop the Values Tables - for (String valueType : valueTypes) { - adapter.dropTable(schemaName, tablePrefix + valueType); - } - - // Drop the supporting tables - adapter.dropTable(schemaName, tablePrefix + "logical_resources"); - adapter.dropTable(schemaName, tablePrefix + "resources"); - // Drop the View for the Table - adapter.dropView(schemaName, tablePrefix + "_token_values_V"); + if (translator.getType() == DbType.POSTGRESQL) { + runDropTableResourceGroup(translator, c, schemaName.toLowerCase(), tablePrefix.toLowerCase(), VALUE_TYPES_LOWER); + } else { + runDropTableResourceGroup(translator, c, schemaName, tablePrefix, VALUE_TYPES); + } } // Pattern for multitenant is to prefix tables with DRP_ @@ -206,8 +206,34 @@ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { // Drop the tables, when the tables don't exists the errors are swallowed // and logs print warnings saying the tables don't exist. That's OK. for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { - adapter.dropTable(schemaName, prefix + deprecatedTable); + String table = prefix + deprecatedTable; + if (translator.getType() == DbType.POSTGRESQL) { + adapter.dropTable(schemaName.toLowerCase(), table.toLowerCase()); + } else { + adapter.dropTable(schemaName, table); + } + } + } + + /** + * run drop table resource group + * @param translator + * @param c + * @param schemaName + * @param tablePrefix + * @param valueTypes + */ + public void runDropTableResourceGroup(IDatabaseTranslator translator, Connection c, String schemaName, String tablePrefix, List valueTypes) { + adapter.dropView(schemaName, tablePrefix + "token_values_v"); + + // Drop the Values Tables + for (String valueType : valueTypes) { + adapter.dropTable(schemaName, tablePrefix + valueType); } + + // Drop the supporting tables + adapter.dropTable(schemaName, tablePrefix + "logical_resources"); + adapter.dropTable(schemaName, tablePrefix + "resources"); } /** diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java index 9ee8b9d6fae..6d1fd60faaf 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java @@ -15,6 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.model.DbType; /** * Checks to see if any of the tables exist in the target database. @@ -87,7 +88,7 @@ public Boolean run(IDatabaseTranslator translator, Connection c) { * @return */ public boolean checkDb2(IDatabaseTranslator translator, Connection c) { - final String sql = "SELECT tabname FROM SYSCAT.TABLES TABS" + final String sql = "SELECT 1 FROM SYSCAT.TABLES TABS" + " WHERE TABS.tabschema = ? " + " AND TABS.tabname in ( " + addParameterMarkers(TABLE_COUNT) + " )"; return hasTables(translator, c, sql); @@ -103,7 +104,7 @@ public boolean checkDb2(IDatabaseTranslator translator, Connection c) { public boolean checkDerby(IDatabaseTranslator translator, Connection c) { // Grab the list of tables for the configured schema from the Derby sys catalog final String sql = "" - + "SELECT tables.tablename FROM sys.systables AS tables " + + "SELECT 1 FROM sys.systables AS tables " + " JOIN sys.sysschemas AS schemas " + " ON (tables.schemaid = schemas.schemaid) " + " WHERE schemas.schemaname = ?" @@ -122,8 +123,8 @@ public boolean checkPostgres(IDatabaseTranslator translator, Connection c) { // Grab the list of tables for the configured schema from the PostgreSQL schema // catalog final String sql = "" - + "SELECT table_name FROM information_schema.tables " - + " WHERE table_schema = ?" + + "SELECT 1 FROM information_schema.tables " + + " WHERE table_schema = lower(?)" + " AND table_name in (" + addParameterMarkers(TABLE_COUNT) + ")"; return hasTables(translator, c, sql); } @@ -157,7 +158,11 @@ private boolean hasTables(IDatabaseTranslator translator, Connection c, final St ps.setString(i++, schemaName); for (String deprecatedTable : DEPRECATED_TABLES) { - ps.setString(i++, deprecatedTable); + if (translator.getType() == DbType.POSTGRESQL) { + ps.setString(i++, deprecatedTable.toLowerCase()); + } else { + ps.setString(i++, deprecatedTable); + } } ResultSet rs = ps.executeQuery(); return rs.next(); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java index 5d6449c05c0..6032398d173 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java @@ -9,7 +9,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import java.sql.Connection; @@ -31,7 +30,6 @@ import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; import com.ibm.fhir.database.utils.transaction.SimpleTransactionProvider; -import com.ibm.fhir.database.utils.version.CreateVersionHistory; import com.ibm.fhir.database.utils.version.CreateWholeSchemaVersion; import com.ibm.fhir.database.utils.version.VersionHistoryService; import com.ibm.fhir.schema.app.Main; @@ -62,7 +60,7 @@ public void testFhirSchema() throws Exception { try (DerbyFhirDatabase db = new DerbyFhirDatabase(DB_NAME)) { System.out.println("FHIR database exists."); checkDatabase(db, db.getSchemaName()); - + // Test the schema drop testDrop(db, db.getSchemaName()); } @@ -87,9 +85,9 @@ protected void testDrop(IConnectionProvider cp, String schemaName) throws SQLExc FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, schemaName, false); gen.buildSchema(pdm); pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); - + CreateWholeSchemaVersion.dropTable(schemaName, adapter); - + // Check that the schema is empty List schemaObjects = adapter.listSchemaObjects(schemaName); boolean schemaIsEmpty = schemaObjects.isEmpty(); @@ -100,7 +98,7 @@ protected void testDrop(IConnectionProvider cp, String schemaName) throws SQLExc } assertTrue(schemaIsEmpty); } - + // Now we're all done we can finally clean up the version-history for // our schema. Make sure we don't get a version for an object we know // we created @@ -137,10 +135,10 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S JdbcTarget tgt = new JdbcTarget(c); DerbyAdapter adapter = new DerbyAdapter(tgt); checkRefSequence(adapter); - + // Check that we have the correct number of tables. This will need to be updated // whenever tables, views or sequences are added or removed - assertEquals(adapter.listSchemaObjects(schemaName).size(), 1943); + assertEquals(adapter.listSchemaObjects(schemaName).size(), 1917); c.commit(); } catch (Throwable t) { c.rollback(); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java index abd06e68b5f..3a01f97d637 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java @@ -3,7 +3,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ - + package com.ibm.fhir.schema.derby; import static org.testng.Assert.assertEquals; @@ -27,7 +27,7 @@ public class DerbySchemaVersionsTest { private static final String TARGET_DIR = "target/derby/"; private static final String SCHEMA_NAME = "APP"; - + @Test public void test() throws Exception { String dbPath = TARGET_DIR + "fhirdb"; @@ -41,18 +41,18 @@ public void test() throws Exception { PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, 10); ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); SchemaVersionsManager svm = new SchemaVersionsManager(db.getTranslator(), connectionPool, transactionProvider, SCHEMA_NAME); - + svm.updateSchemaVersionId(FhirSchemaVersion.V0001); assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0001.vid()); svm.updateSchemaVersionId(FhirSchemaVersion.V0002); assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0002.vid()); svm.updateSchemaVersionId(FhirSchemaVersion.V0003); assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0003.vid()); - + // Make sure we can correctly determine the latest schema version value svm.updateSchemaVersion(); - assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0020.vid()); - + assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0021.vid()); + assertTrue(svm.isLatestSchema()); } } diff --git a/operation/fhir-operation-member-match/src/main/java/com/ibm/fhir/operation/davinci/hrex/provider/strategy/DefaultMemberMatchStrategy.java b/operation/fhir-operation-member-match/src/main/java/com/ibm/fhir/operation/davinci/hrex/provider/strategy/DefaultMemberMatchStrategy.java index 44a3fd08002..023409647c7 100644 --- a/operation/fhir-operation-member-match/src/main/java/com/ibm/fhir/operation/davinci/hrex/provider/strategy/DefaultMemberMatchStrategy.java +++ b/operation/fhir-operation-member-match/src/main/java/com/ibm/fhir/operation/davinci/hrex/provider/strategy/DefaultMemberMatchStrategy.java @@ -255,8 +255,9 @@ private void addIdValue(Identifier identifier) { // If there is more than one Coding it has to be MB. for (Coding coding : identifier.getType().getCoding()) { if (coding.getSystem() != null - && "http://terminology.hl7.org/CodeSystem/v2-0203" + && "http://terminology.hl7.org/CodeSystem/v2-0203" .equals(coding.getSystem().getValue()) + && coding.getCode() != null && "MB".equals(coding.getCode().getValue())) { // We only want to extract the code system v2-0203 with MB Uri sys = identifier.getSystem(); From ad7174e95c9a94170af3a39a5f0ef5d642529c82 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 5 Nov 2021 16:24:11 -0400 Subject: [PATCH 3/8] Update the Docs Signed-off-by: Paul Bastide --- fhir-persistence-schema/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fhir-persistence-schema/README.md b/fhir-persistence-schema/README.md index 0152dfc3b79..274f2e6889d 100644 --- a/fhir-persistence-schema/README.md +++ b/fhir-persistence-schema/README.md @@ -486,4 +486,10 @@ Note: the jar file is stored locally in `fhir-persistence-schema/target` or in t - grants.sql: `db2 -tvf grants.sql` - stored-procedures.sql: `db2 -td@ -vf stored-procedures.sql` + +# V0021 - Drops the DOMAINRESOURCE and RESOURCE tables + +If there is data in the DOMAINRESOURCE and RESOURCE table groups, which is unexpected, the administrator may run the tool with `--force-unused-table-removal` to force the removal of the unused tables. + + FHIR® is the registered trademark of HL7 and is used with the permission of HL7. From b93d84ec68a724856054685680e7f899c3af3638 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 5 Nov 2021 17:06:23 -0400 Subject: [PATCH 4/8] Apply suggestions from code review Signed-off-by: Paul Bastide Co-authored-by: Troy Biesterfeld <60481063+tbieste@users.noreply.github.com> --- .../src/main/java/com/ibm/fhir/schema/app/Main.java | 2 +- .../fhir/schema/control/MigrateV0021AbstractTypeRemoval.java | 4 ++-- .../schema/control/UnusedTableRemovalNeedsV0021Migration.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 132b49f6d3c..97139694933 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -479,7 +479,7 @@ protected void updateFhirSchema() { // V0014 IS_DELETED and LAST_UPDATED added to whole-system LOGICAL_RESOURCES applyDataMigrationForV0014(); - // V0021 removes Abstract Type tables which are unused.s + // V0021 removes Abstract Type tables which are unused. applyTableRemovalForV0021(); // Apply privileges if asked diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java index 21ca78dae5c..db4359d2078 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -69,7 +69,7 @@ public class MigrateV0021AbstractTypeRemoval implements IDatabaseStatement { * @param adapter * @param adminSchemaName * @param schemaName - * @param force - indicating if their is data in these tables, it'll still remove the table. + * @param force - indicating if there is data in these tables, it'll still remove the table. */ public MigrateV0021AbstractTypeRemoval(IDatabaseAdapter adapter, String adminSchemaName, String schemaName, boolean force) { this.adapter = adapter; @@ -102,7 +102,7 @@ public void run(IDatabaseTranslator translator, Connection c) { */ private void checkShouldThrowException() { if (count != 0 && !force) { - throw new IllegalArgumentException("The Data Tables for Resources or DomainResource contains data. use --force-unused-table-removal to force removal"); + throw new IllegalArgumentException("The Data Tables for Resources or DomainResource contains data. Use --force-unused-table-removal to force removal"); } } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java index 6d1fd60faaf..9038f0702d9 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java @@ -108,7 +108,7 @@ public boolean checkDerby(IDatabaseTranslator translator, Connection c) { + " JOIN sys.sysschemas AS schemas " + " ON (tables.schemaid = schemas.schemaid) " + " WHERE schemas.schemaname = ?" - + " AND schemas.tablename in (" + addParameterMarkers(TABLE_COUNT) + ")"; + + " AND tables.tablename in (" + addParameterMarkers(TABLE_COUNT) + ")"; return hasTables(translator, c, sql); } From e2d24ecbb91107dbbf1f07a5757e4a4d201f27bd Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Fri, 5 Nov 2021 19:39:59 -0400 Subject: [PATCH 5/8] Update for false on abstract Signed-off-by: Paul Bastide --- .../src/main/java/com/ibm/fhir/schema/app/Main.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 132b49f6d3c..53c2fdb0706 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -2103,10 +2103,9 @@ protected void applyDataMigrationForV0014() { private Set getResourceTypes() { Set result; if (this.resourceTypeSubset == null || this.resourceTypeSubset.isEmpty()) { - // pass 'false' to getResourceTypes to avoid building tables for abstract resource types // Should simplify FhirSchemaGenerator and always pass in this list. When switching // over to false, migration is required to drop the tables no longer required. - final boolean includeAbstractResourceTypes = true; + final boolean includeAbstractResourceTypes = false; result = ModelSupport.getResourceTypes(includeAbstractResourceTypes).stream().map(Class::getSimpleName).collect(Collectors.toSet()); } else { result = this.resourceTypeSubset; From 6d011a2079678a8bb97671338442a60de37ef561 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Mon, 8 Nov 2021 08:46:36 -0500 Subject: [PATCH 6/8] Update to support name check before drop in derby Update the Abstract Type to control errors Update the Properties to check for invalid names Signed-off-by: Paul Bastide --- build/derby-bootstrap.sh | 6 +++--- .../database/utils/derby/DerbyAdapter.java | 20 ++++++++++++++++--- .../java/com/ibm/fhir/schema/app/Main.java | 5 +++++ .../MigrateV0021AbstractTypeRemoval.java | 7 +++++-- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/build/derby-bootstrap.sh b/build/derby-bootstrap.sh index bd25986e48d..280b419091c 100755 --- a/build/derby-bootstrap.sh +++ b/build/derby-bootstrap.sh @@ -26,13 +26,13 @@ java -jar ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*- --update-schema java -jar ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/profile --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/reference --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/study1 --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index 2875168acbb..3062e3a5b18 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -176,6 +176,20 @@ public void dropTablespace(String tablespaceName) { logger.fine("Drop tablespace not supported in Derby"); } + @Override + public void dropTable(String schemaName, String tableName) { + final String nm = getQualifiedName(schemaName, tableName); + final String ddl = "DROP TABLE " + nm; + + if (doesTableExist(schemaName, tableName)) { + try { + runStatement(ddl); + } catch (UndefinedNameException x) { + logger.warning(ddl + "; TABLE not found"); + } + } + } + @Override public void detachPartition(String schemaName, String tableName, String partitionName, String newTableName) { warnOnce(MessageKey.PARTITIONING, "Detach partition not supported in Derby"); @@ -383,13 +397,13 @@ public List listSchemaObjects(String schemaName) { List result = new ArrayList<>(); DerbyListTablesForSchema listTables = new DerbyListTablesForSchema(schemaName); result.addAll(runStatement(listTables)); - + DerbyListViewsForSchema listViews = new DerbyListViewsForSchema(schemaName); result.addAll(runStatement(listViews)); - + DerbyListSequencesForSchema listSequences = new DerbyListSequencesForSchema(schemaName); result.addAll(runStatement(listSequences)); - + return result; } } \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 446bbe026a0..837c843117e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -2458,6 +2458,11 @@ protected void process() { String resourceTypesString = properties.getProperty("resourceTypes"); if (resourceTypesString != null && resourceTypesString.length() > 0) { resourceTypeSubset = new HashSet<>(Arrays.asList(resourceTypesString.split(","))); + + // Double check for Abstract Types + if (resourceTypeSubset.contains("DomainResource") || resourceTypeSubset.contains("Resource")) { + throw new IllegalArgumentException("--prop resourceTypes= should not include Abstract types"); + } } if (addKeyForTenant != null) { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java index db4359d2078..9a0f4d79811 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -173,7 +173,11 @@ private void checkDataTables(IDatabaseTranslator translator, Connection c) { } } } catch (SQLException x) { - throw translator.translate(x); + if (!translator.isUndefinedName(x)){ + throw translator.translate(x); + } else { + LOG.finest("Table already deleted: " + table); + } } } } @@ -186,7 +190,6 @@ private void checkDataTables(IDatabaseTranslator translator, Connection c) { private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { List tables = Arrays.asList("DOMAINRESOURCE_", "RESOURCE_"); - // Run across both tables for (String tablePrefix : tables) { // Drop the View for the Table From caf2bc2a4ea133e38eec8c416284620120866a27 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Mon, 8 Nov 2021 09:49:53 -0500 Subject: [PATCH 7/8] Update CI Build Scripts Signed-off-by: Paul Bastide --- build/audit/kafka/docker-compose.yml | 6 +++--- build/docker/docker-compose.yml | 6 +++--- build/migration/db2/4_current-migrate.sh | 8 ++++---- build/migration/postgres/4_current-migrate.sh | 8 ++++---- build/notifications/kafka/docker-compose.yml | 6 +++--- build/persistence/postgres/docker-compose.yml | 6 +++--- build/pre-integration-test.ps1 | 6 +++--- build/pre-integration-test.sh | 6 +++--- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/build/audit/kafka/docker-compose.yml b/build/audit/kafka/docker-compose.yml index 3c63ed4e80e..4b0ab58789f 100644 --- a/build/audit/kafka/docker-compose.yml +++ b/build/audit/kafka/docker-compose.yml @@ -68,15 +68,15 @@ services: command: bash -c " java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/profile --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/reference --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/study1 --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && /opt/ol/wlp/bin/server run" healthcheck: diff --git a/build/docker/docker-compose.yml b/build/docker/docker-compose.yml index b0ca82ce44e..58525e34945 100644 --- a/build/docker/docker-compose.yml +++ b/build/docker/docker-compose.yml @@ -31,15 +31,15 @@ services: command: bash -c " java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/profile --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/reference --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/study1 --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && /opt/ol/wlp/bin/server run" healthcheck: diff --git a/build/migration/db2/4_current-migrate.sh b/build/migration/db2/4_current-migrate.sh index 8e7b54e0707..5f345873296 100644 --- a/build/migration/db2/4_current-migrate.sh +++ b/build/migration/db2/4_current-migrate.sh @@ -56,19 +56,19 @@ java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-sche --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/profile --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/reference --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/study1 --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema # Reset to Original Directory popd > /dev/null # EOF -############################################################################### \ No newline at end of file +############################################################################### diff --git a/build/migration/postgres/4_current-migrate.sh b/build/migration/postgres/4_current-migrate.sh index e6442ca2e8b..0950901b306 100644 --- a/build/migration/postgres/4_current-migrate.sh +++ b/build/migration/postgres/4_current-migrate.sh @@ -70,19 +70,19 @@ java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-sche --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/profile --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/reference --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${WORKSPACE}/fhir/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/study1 --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema # Reset to Original Directory popd > /dev/null # EOF -############################################################################### \ No newline at end of file +############################################################################### diff --git a/build/notifications/kafka/docker-compose.yml b/build/notifications/kafka/docker-compose.yml index 8bb01797d63..a280df16dc8 100644 --- a/build/notifications/kafka/docker-compose.yml +++ b/build/notifications/kafka/docker-compose.yml @@ -68,15 +68,15 @@ services: command: bash -c " java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/profile --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/reference --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/study1 --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && /opt/ol/wlp/bin/server run" healthcheck: diff --git a/build/persistence/postgres/docker-compose.yml b/build/persistence/postgres/docker-compose.yml index bf3c21e2d19..48f7144835a 100644 --- a/build/persistence/postgres/docker-compose.yml +++ b/build/persistence/postgres/docker-compose.yml @@ -47,15 +47,15 @@ services: command: bash -c " java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/profile --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/reference --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && java -jar /opt/ibm-fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type derby --prop db.database=/output/derby/study1 --prop db.create=Y - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet --update-schema && /opt/ol/wlp/bin/server run" healthcheck: diff --git a/build/pre-integration-test.ps1 b/build/pre-integration-test.ps1 index 807c906e131..907d0bf5b49 100644 --- a/build/pre-integration-test.ps1 +++ b/build/pre-integration-test.ps1 @@ -58,15 +58,15 @@ java -jar $SCHEMATOOL ` --update-schema java -jar $SCHEMATOOL ` --db-type derby --prop db.database=${DB_LOC}\profile --prop db.create=Y ` - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource ` + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet ` --update-schema java -jar $SCHEMATOOL ` --db-type derby --prop db.database=${DB_LOC}\reference --prop db.create=Y ` - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource ` + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet ` --update-schema java -jar $SCHEMATOOL ` --db-type derby --prop db.database=${DB_LOC}\study1 --prop db.create=Y ` - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource ` + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet ` --update-schema # If the Config Exists, let's wipe it outfind diff --git a/build/pre-integration-test.sh b/build/pre-integration-test.sh index 32a4b8629ad..a5743aa969a 100755 --- a/build/pre-integration-test.sh +++ b/build/pre-integration-test.sh @@ -50,15 +50,15 @@ java -jar ${SIT}/fhir-server-dist/tools/fhir-persistence-schema-*-cli.jar \ --update-schema java -jar ${SIT}/fhir-server-dist/tools/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/profile --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Person,RelatedPerson,Organization,Location,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${SIT}/fhir-server-dist/tools/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/reference --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Medication,Observation,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema java -jar ${SIT}/fhir-server-dist/tools/fhir-persistence-schema-*-cli.jar \ --db-type derby --prop db.database=${DB_LOC}/study1 --prop db.create=Y \ - --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet,Resource \ + --prop resourceTypes=Patient,Group,Practitioner,PractitionerRole,Device,Organization,Location,Encounter,AllergyIntolerance,Observation,Condition,CarePlan,Provenance,Medication,MedicationAdministration,StructureDefinition,ElementDefinition,CodeSystem,ValueSet \ --update-schema echo "Copying configuration to install location..." From a2521f23f166e8598f17462c66f2ff5010c4c820 Mon Sep 17 00:00:00 2001 From: Paul Bastide Date: Mon, 8 Nov 2021 11:31:51 -0500 Subject: [PATCH 8/8] Update for lowercase names Signed-off-by: Paul Bastide --- .../MigrateV0021AbstractTypeRemoval.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java index 9a0f4d79811..295c1b170c2 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -158,25 +158,30 @@ private void checkDataInAllPartitionedTenantTables(IDatabaseTranslator translato */ private void checkDataTables(IDatabaseTranslator translator, Connection c) { for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { - final String table = schemaName + "." + deprecatedTable; + if (adapter.doesTableExist(schemaName, deprecatedTable)) { + String table = schemaName + "." + deprecatedTable; + if (translator.getType() == DbType.POSTGRESQL) { + table = schemaName.toLowerCase() + "." + deprecatedTable; + } - // When checking for data... SYSCAT.TABLES->CARD was considered to be checked. - // However if the db hasn't collected statistics -1 is returned, and unreliable - // Instead we're going to check if it has at least one row... - final String sql = "SELECT * FROM " + table + " " + translator.limit("1"); - try (PreparedStatement ps = c.prepareStatement(sql)) { - if (ps.execute()) { - ResultSet rs = ps.getResultSet(); - if (rs.next()) { - LOG.warning("Data Table contains data '" + table + "'"); - count++; + // When checking for data... SYSCAT.TABLES->CARD was considered to be checked. + // However if the db hasn't collected statistics -1 is returned, and unreliable + // Instead we're going to check if it has at least one row... + final String sql = "SELECT * FROM " + table + " " + translator.limit("1"); + try (PreparedStatement ps = c.prepareStatement(sql)) { + if (ps.execute()) { + ResultSet rs = ps.getResultSet(); + if (rs.next()) { + LOG.warning("Data Table contains data '" + table + "'"); + count++; + } + } + } catch (SQLException x) { + if (!translator.isUndefinedName(x)){ + throw translator.translate(x); + } else { + LOG.finest("Table already deleted: " + table); } - } - } catch (SQLException x) { - if (!translator.isUndefinedName(x)){ - throw translator.translate(x); - } else { - LOG.finest("Table already deleted: " + table); } } }