From c4a11be66686e2278ca580c8e9ff9c9f86005745 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 7 Mar 2022 11:29:07 +0000 Subject: [PATCH 01/40] issue #3437 experimental support for Citus database Signed-off-by: Robin Arnold --- .../java/com/ibm/fhir/bucket/app/Main.java | 3 +- .../bucket/persistence/AddBucketPath.java | 3 +- .../bucket/persistence/AddResourceBundle.java | 3 +- .../persistence/MergeResourceTypes.java | 3 +- .../bucket/persistence/RecordLogicalId.java | 3 +- .../database/utils/api/DistributionRules.java | 49 +++++++++ .../database/utils/api/IDatabaseAdapter.java | 4 +- .../utils/api/IDatabaseTranslator.java | 6 ++ .../database/utils/citus/CitusAdapter.java | 63 ++++++++++++ .../database/utils/citus/CitusTranslator.java | 21 ++++ .../citus/CreateDistributedTableDAO.java | 61 ++++++++++++ .../utils/citus/CreateReferenceTableDAO.java | 55 +++++++++++ .../fhir/database/utils/common/AddColumn.java | 19 ++-- .../common/DatabaseTranslatorFactory.java | 4 + .../database/utils/common/DropColumn.java | 3 +- .../database/utils/common/DropPrimaryKey.java | 3 +- .../fhir/database/utils/db2/Db2Adapter.java | 4 +- .../database/utils/db2/Db2Translator.java | 5 + .../database/utils/derby/DerbyAdapter.java | 4 +- .../database/utils/derby/DerbyTranslator.java | 5 + .../ibm/fhir/database/utils/model/DbType.java | 5 + .../utils/model/ForeignKeyConstraint.java | 16 +++ .../fhir/database/utils/model/IDataModel.java | 8 +- .../utils/model/PhysicalDataModel.java | 26 ++++- .../ibm/fhir/database/utils/model/Table.java | 99 +++++++++++++++++-- .../database/utils/pool/DatabaseSupport.java | 1 + .../utils/postgres/PostgresAdapter.java | 6 +- .../PostgresFillfactorSettingDAO.java | 6 +- .../utils/postgres/PostgresTranslator.java | 5 + .../postgres/PostgresVacuumSettingDAO.java | 3 +- .../database/utils/schema/LeaseManager.java | 1 + .../utils/schema/SchemaVersionsManager.java | 1 + .../jdbc/FHIRResourceDAOFactory.java | 30 ++++-- .../FHIRDbConnectionStrategyBase.java | 1 + .../jdbc/connection/FHIRDbFlavor.java | 6 ++ .../jdbc/connection/FHIRDbFlavorImpl.java | 5 + .../jdbc/dao/EraseResourceDAO.java | 2 +- .../jdbc/dao/impl/ParameterDAOImpl.java | 2 + .../jdbc/domain/SearchQueryRenderer.java | 1 + .../fhir/persistence/jdbc/test/spec/Main.java | 16 +-- .../jdbc/test/util/ParameterCounter.java | 7 ++ .../java/com/ibm/fhir/schema/app/Main.java | 58 +++++++---- .../com/ibm/fhir/schema/app/menu/Menu.java | 2 +- .../ibm/fhir/schema/app/util/CommonUtil.java | 8 +- .../control/FhirResourceTableGroup.java | 39 +++++--- .../schema/control/FhirSchemaGenerator.java | 43 +++++--- .../schema/control/FhirSchemaVersion.java | 1 + .../MigrateV0021AbstractTypeRemoval.java | 7 +- ...UnusedTableRemovalNeedsV0021Migration.java | 4 +- .../java/com/ibm/fhir/schema/patch/Main.java | 4 + .../schema/derby/DerbySchemaVersionsTest.java | 2 +- 51 files changed, 629 insertions(+), 107 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index dacdc0fc5cb..d4b78719128 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -606,6 +606,7 @@ public void configure() { setupDerbyRepository(); break; case POSTGRESQL: + case CITUS: setupPostgresRepository(); break; } @@ -810,7 +811,7 @@ private void buildSchema() { .map(FHIRResourceType.Value::value) .collect(Collectors.toSet()); - if (adapter.getTranslator().getType() == DbType.POSTGRESQL) { + if (adapter.getTranslator().isFamilyPostgreSQL()) { // Postgres doesn't support batched merges, so we go with a simpler UPSERT MergeResourceTypesPostgres mrt = new MergeResourceTypesPostgres(schemaName, resourceTypes); adapter.runStatement(mrt); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java index 93be50bedf7..76f2539e7a1 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -57,7 +56,7 @@ public Long run(IDatabaseTranslator translator, Connection c) { // try the old-fashioned way and handle duplicate key final String bucketPaths = DataDefinitionUtil.getQualifiedName(schemaName, "bucket_paths"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // For POSTGRES, if a statement fails it causes the whole transaction // to fail, so we need turn this into an UPSERT dml = "INSERT INTO " + bucketPaths + "(bucket_name, bucket_path) VALUES (?,?) ON CONFLICT(bucket_name, bucket_path) DO NOTHING"; diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java index 5215d12650d..478a50bc706 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java @@ -20,7 +20,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -76,7 +75,7 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { int version = 1; final String resourceBundles = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundles"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // For PostgresSQL, make sure we don't break the current transaction // if the statement fails...annoying dml = "INSERT INTO " + resourceBundles + "(" diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java index a209424cc2a..d1da59eb82a 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java @@ -18,7 +18,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -62,7 +61,7 @@ public void run(IDatabaseTranslator translator, Connection c) { try (PreparedStatement ps = c.prepareStatement(merge)) { // Assume the list is small enough to process in one batch for (String resourceType: resourceTypes) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { ps.setString(1, resourceType); } else { ps.setString(1, resourceType); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java index f9b71bbc7a6..10f80c6d8fc 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java @@ -16,7 +16,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -67,7 +66,7 @@ public void run(IDatabaseTranslator translator, Connection c) { final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "logical_resources"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // Use UPSERT syntax for Postgres to avoid breaking the transaction when // a statement fails dml = diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java new file mode 100644 index 00000000000..30855f64d3c --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java @@ -0,0 +1,49 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * Rules for distributing a table in a distributed RDBMS such as Citus + */ +public class DistributionRules { + + // If this table is distributed, which column is used for sharding + private final String distributionColumn; + // Is this table a reference table + private final boolean referenceTable; + + /** + * Public constructor + * @param distributionColumn + * @param referenceTable + */ + public DistributionRules(String distributionColumn, boolean referenceTable) { + this.distributionColumn = distributionColumn; + this.referenceTable = referenceTable; + if (this.referenceTable && this.distributionColumn != null) { + // variables are mutually exclusive + throw new IllegalArgumentException("Reference tables do not use a distributionColumn"); + } + } + + /** + * Getter for distributionColumn value + * @return + */ + public String getDistributionColumn() { + return this.distributionColumn; + } + + /** + * Getter for referenceTableValue + * @return + */ + public boolean isReferenceTable() { + return this.referenceTable; + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 0d2e74c5261..46f454dd8a1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -79,9 +79,11 @@ public interface IDatabaseAdapter { * @param tablespaceName * @param withs * @param checkConstraints + * @param distributionRules */ public void createTable(String schemaName, String name, String tenantColumnName, List columns, - PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints); + PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionRules distributionRules); /** * Add a new column to an existing table diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java index a03484d4df7..75e4a717767 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java @@ -196,6 +196,12 @@ public interface IDatabaseTranslator { */ DbType getType(); + /** + * True if the database type is part of the PostgreSQL family (POSTGRESQL, CITUS) + * @return + */ + boolean isFamilyPostgreSQL(); + /** * The name of the "DUAL" table...that special table giving us one row/column. * @return the name of the "DUAL" table for the database, or null if not supported diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java new file mode 100644 index 00000000000..82be749b95d --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -0,0 +1,63 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.IDatabaseTarget; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.With; +import com.ibm.fhir.database.utils.postgres.PostgresAdapter; + + +/** + * A database adapter implementation for Citus (distributed PostgreSQL) + */ +public class CitusAdapter extends PostgresAdapter { + + /** + * Public constructor + * @param target + */ + public CitusAdapter(IDatabaseTarget target) { + super(target); + } + + /** + * Public constructor + * @param cp + */ + public CitusAdapter(IConnectionProvider cp) { + super(cp); + } + + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionRules distributionRules) { + super.createTable(schemaName, name, tenantColumnName, columns, primaryKey, + identity, tablespaceName, withs, checkConstraints, distributionRules); + + // Apply the distribution rules, if any. Tables without distribution rules are created + // only on Citus controller nodes and never distributed to the worker nodes + if (distributionRules != null) { + if (distributionRules.isReferenceTable()) { + // A table that is fully replicated for each worker node + CreateReferenceTableDAO dao = new CreateReferenceTableDAO(schemaName, name); + runStatement(dao); + } else if (distributionRules.getDistributionColumn() != null && distributionRules.getDistributionColumn().length() > 0) { + // A table that is sharded using a hash on the distributionColumn value + CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, name, distributionRules.getDistributionColumn()); + runStatement(dao); + } + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java new file mode 100644 index 00000000000..5933ec50dd1 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java @@ -0,0 +1,21 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; + + +/** + * IDatabaseTranslator implementation supporting Citus + */ +public class CitusTranslator extends PostgresTranslator { + @Override + public DbType getType() { + return DbType.CITUS; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java new file mode 100644 index 00000000000..875162abd16 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java @@ -0,0 +1,61 @@ +/* + * (C) Copyright IBM Corp. 2019 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * DAO to add a new tenant key record + */ +public class CreateDistributedTableDAO implements IDatabaseStatement { + private final String schemaName; + private final String tableName; + private final String distributionKey; + + /** + * Public constructor + * + * @param schemaName + * @param tableName + * @param distributionKey + */ + public CreateDistributedTableDAO(String schemaName, String tableName, String distributionKey) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(tableName); + DataDefinitionUtil.assertValidName(distributionKey); + this.schemaName = schemaName.toLowerCase(); + this.tableName = tableName.toLowerCase(); + this.distributionKey = distributionKey.toLowerCase(); + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + // Run the Citus create_distributed_table UDF + final String table = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName); + StringBuilder sql = new StringBuilder(); + sql.append("SELECT create_distributed_table("); + sql.append("'").append(table).append("'"); + sql.append(", "); + sql.append("'").append(distributionKey).append("'"); + sql.append(")"); + + try (PreparedStatement ps = c.prepareStatement(sql.toString())) { + // It's a SELECT statement, but we don't care about the ResultSet + ps.executeQuery(); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java new file mode 100644 index 00000000000..718a3caee4c --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java @@ -0,0 +1,55 @@ +/* + * (C) Copyright IBM Corp. 2019 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * DAO to add a new tenant key record + */ +public class CreateReferenceTableDAO implements IDatabaseStatement { + private final String schemaName; + private final String tableName; + + /** + * Public constructor + * + * @param schemaName + * @param tableName + */ + public CreateReferenceTableDAO(String schemaName, String tableName) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(tableName); + this.schemaName = schemaName.toLowerCase(); + this.tableName = tableName.toLowerCase(); + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + // Run the Citus create_reference_table UDF + final String table = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName); + StringBuilder sql = new StringBuilder(); + sql.append("SELECT create_reference_table("); + sql.append("'").append(table).append("'"); + sql.append(")"); + + try (PreparedStatement ps = c.prepareStatement(sql.toString())) { + // It's a SELECT statement, but we don't care about the ResultSet + ps.executeQuery(); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java index 0d1b65deac5..e5303fb7606 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.model.ColumnBase; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresAdapter; /** @@ -46,14 +45,22 @@ public void run(IDatabaseTranslator translator, Connection c) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, tableName); // DatabaseTypeAdapter is needed to find the correct data type for the column. - IDatabaseTypeAdapter dbAdapter = null; - String driveClassName = translator.getDriverClassName(); - if (driveClassName.contains(DbType.DB2.value())) { + final IDatabaseTypeAdapter dbAdapter; + switch (translator.getType()) { + case DB2: dbAdapter = new Db2Adapter(); - } else if (driveClassName.contains(DbType.DERBY.value())) { + break; + case DERBY: dbAdapter = new DerbyAdapter(); - } else if (driveClassName.contains(DbType.POSTGRESQL.value())) { + break; + case POSTGRESQL: dbAdapter = new PostgresAdapter(); + break; + case CITUS: + dbAdapter = new PostgresAdapter(); + break; + default: + throw new IllegalArgumentException("Unsupported database type: " + translator.getType().name()); } String ddl = "ALTER TABLE " + qname + " ADD COLUMN " + columnDef(column, dbAdapter); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java index 78c71e83832..07d8b2e83c5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java @@ -7,6 +7,7 @@ package com.ibm.fhir.database.utils.common; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.db2.Db2Translator; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.database.utils.model.DbType; @@ -34,6 +35,9 @@ public static IDatabaseTranslator getTranslator(DbType type) { case POSTGRESQL: result = new PostgresTranslator(); break; + case CITUS: + result = new CitusTranslator(); + break; default: throw new IllegalStateException("DbType not supported: " + type); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java index 0871d0a81b1..c35c7e77fc1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java @@ -18,7 +18,6 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; /** * Drop columns from the schema.table @@ -68,7 +67,7 @@ public void run(IDatabaseTranslator translator, Connection c) { int dropCount = 0; for (String columnName : columnNames) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { if (postgresColumnExists(translator, c, columnName)) { ddl.append("\n\t" + "DROP COLUMN " + columnName); dropCount++; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java index 8da90c32c82..584afddfd53 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java @@ -16,7 +16,6 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; /** * Drop the primary key constraint on a table @@ -47,7 +46,7 @@ public void run(IDatabaseTranslator translator, Connection c) { // ought to be doing this via an adapter, which hides the differences between databases final String qname = DataDefinitionUtil.getQualifiedName(this.schemaName, this.tableName); final String ddl; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // There could be some schemas built between releases which don't have the ROW_ID PK // so for PostgreSQL we need to check if the constraint exists otherwise the whole // transaction fails. diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java index 8f5d927c783..f2a2c1408c5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java @@ -24,6 +24,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.DataAccessException; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -72,7 +73,8 @@ public Db2Adapter() { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionRules distributionRules) { // With DB2 we can implement support for multi-tenancy, which we do by injecting a MT_ID column // to the definition and partitioning on that column diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java index 4f2009b59c1..e5d3c0a2f33 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java @@ -300,4 +300,9 @@ public Optional maximumQueryParameters() { // Maximum number of host variable references in a dynamic SQL statement 32,767 return Optional.of(Integer.valueOf(32767)); } + + @Override + public boolean isFamilyPostgreSQL() { + return false; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index 3062e3a5b18..ff00b81e73e 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -15,6 +15,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -79,7 +80,8 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionRules distributionRules) { // Derby doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { warnOnce(MessageKey.MULTITENANCY, "Derby does not support multi-tenancy on: [" + name + "]"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java index 0f041f3731e..f94a6d8d488 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java @@ -223,4 +223,9 @@ public String pagination(int offset, int rowsPerPage) { return result.toString(); } + @Override + public boolean isFamilyPostgreSQL() { + return false; + } + } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java index 312f1e6a021..cc413af6307 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java @@ -20,6 +20,11 @@ public enum DbType { */ POSTGRESQL("postgresql"), + /** + * Citus (Distributed PostgreSQL) + */ + CITUS("citus"), + /** * IBM Db2 */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java index 374b4120250..8c383b3227c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java @@ -111,4 +111,20 @@ public String getQualifiedTargetName() { public void apply(String schemaName, String name, String tenantColumnName, IDatabaseAdapter target) { target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced); } + + /** + * Return true if the list of columns includes the column name, ignoring case + * @param distributionColumnName + * @return + */ + public boolean includesColumn(String columnName) { + // Linear search is OK because the list is very small and probably + // will be cheaper than maintaining both a list and set of values + for (String cn: this.columns) { + if (cn.equalsIgnoreCase(columnName)) { + return true; + } + } + return false; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java index 9e8537d2dcb..186504524a3 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -18,4 +18,10 @@ public interface IDataModel { * @return */ public Table findTable(String schemaName, String tableName); + + /** + * Is the target database distributed (e.g. with sharding)? + * @return + */ + public boolean isDistributed(); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index 4265012f5f8..0f76fdc3d5b 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -51,11 +51,14 @@ public class PhysicalDataModel implements IDataModel { // Common models that we rely on (e.g. for FK constraints) private final List federatedModels = new ArrayList<>(); + // Is this model configured to operate with a distributed (sharded) database? + private final boolean distributed; + /** * Default constructor. No federated models */ - public PhysicalDataModel() { - // No Op + public PhysicalDataModel(boolean distributed) { + this.distributed = distributed; } /** @@ -63,7 +66,19 @@ public PhysicalDataModel() { * @param federatedModels */ public PhysicalDataModel(PhysicalDataModel... federatedModels) { - this.federatedModels.addAll(Arrays.asList(federatedModels)); + boolean dist = false; + if (federatedModels != null) { + this.federatedModels.addAll(Arrays.asList(federatedModels)); + + // If any of the federated models are distributed, then assume we must be + for (PhysicalDataModel dm: federatedModels) { + if (dm.isDistributed()) { + dist = true; + break; + } + } + } + this.distributed = dist; } /** @@ -543,4 +558,9 @@ public void applyProcedureAndFunctionGrants(IDatabaseAdapter target, String grou public void dropTenantTablespace(IDatabaseAdapter adapter, int tenantId) { adapter.dropTenantTablespace(tenantId); } + + @Override + public boolean isDistributed() { + return this.distributed; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index 21db63313ef..b7186736823 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; @@ -47,6 +48,9 @@ public class Table extends BaseObject { // The column to use when making this table multi-tenant (if supported by the the target) private final String tenantColumnName; + // The rules to distribute the table in a distributed RDBMS implementation (Citus) + private final DistributionRules distributionRules; + // The With parameters on the table private final List withs; @@ -72,11 +76,14 @@ public class Table extends BaseObject { * @param migrations * @param withs * @param checkConstraints + * @param distributionRules */ - public Table(String schemaName, String name, int version, String tenantColumnName, Collection columns, PrimaryKeyDef pk, + public Table(String schemaName, String name, int version, String tenantColumnName, + Collection columns, PrimaryKeyDef pk, IdentityDef identity, Collection indexes, Collection fkConstraints, SessionVariableDef accessControlVar, Tablespace tablespace, List dependencies, Map tags, - Collection privileges, List migrations, List withs, List checkConstraints) { + Collection privileges, List migrations, List withs, List checkConstraints, + DistributionRules distributionRules) { super(schemaName, name, DatabaseObjectType.TABLE, version, migrations); this.tenantColumnName = tenantColumnName; this.columns.addAll(columns); @@ -88,6 +95,7 @@ public Table(String schemaName, String name, int version, String tenantColumnNam this.tablespace = tablespace; this.withs = withs; this.checkConstraints.addAll(checkConstraints); + this.distributionRules = distributionRules; // Adds all dependencies which aren't null. // The only circumstances where it is null is when it is self referencial (an FK on itself). @@ -125,7 +133,8 @@ public String getTenantColumnName() { public void apply(IDatabaseAdapter target) { final String tsName = this.tablespace == null ? null : this.tablespace.getName(); target.createTable(getSchemaName(), getObjectName(), this.tenantColumnName, this.columns, - this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints); + this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints, + this.distributionRules); // Now add any indexes associated with this table for (IndexDef idx: this.indexes) { @@ -241,6 +250,12 @@ public static class Builder extends VersionedSchemaObject { // Check constraints added to the table private List checkConstraints = new ArrayList<>(); + // The column to use for distribution when sharding + private String distributionColumnName; + + // Should this table be treated as a reference (replicated) table when sharding is enabled + private boolean distributionReference = false; + /** * Private constructor to force creation through factory method * @param schemaName @@ -270,6 +285,25 @@ public Builder setTablespace(Tablespace ts) { return this; } + /** + * Setter for the distributionColumnName + * @param cn + * @return + */ + public Builder setDistributionColumnName(String cn) { + this.distributionColumnName = cn; + return this; + } + + /** + * Set the distributionReference to true + * @return + */ + public Builder setDistributionReference() { + this.distributionReference = true; + return this; + } + public Builder addIntColumn(String columnName, boolean nullable) { ColumnDef cd = new ColumnDef(columnName); if (columns.contains(cd)) { @@ -689,15 +723,65 @@ public Table build(IDataModel dataModel) { // Check the FK references are valid List allDependencies = new ArrayList<>(); + // The list of FK constraints we are able to apply + List enabledFKConstraints = new ArrayList<>(); allDependencies.addAll(this.dependencies); - for (ForeignKeyConstraint c: this.fkConstraints.values()) { + // Set up the distribution rules for the table if the target model supports distribution + final DistributionRules distributionRules; + if (dataModel.isDistributed() && (this.distributionReference || this.distributionColumnName != null)) { + distributionRules = new DistributionRules(distributionColumnName, distributionReference); + } else { + distributionRules = null; + } + + // Filter the foreign key constraints to those allowed for the model. Distribution (e.g. Citus) adds + // certain restrictions on which foreign keys are supported, so we have no choice but to ignore them + for (ForeignKeyConstraint c: this.fkConstraints.values()) { Table target = dataModel.findTable(c.getTargetSchema(), c.getTargetTable()); if (target == null && !c.isSelf()) { String targetName = DataDefinitionUtil.getQualifiedName(c.getTargetSchema(), c.getTargetTable()); throw new IllegalArgumentException("Invalid foreign key constraint " + c.getConstraintName() + ": target table does not exist: " + targetName); } - allDependencies.add(target); + + if (!dataModel.isDistributed()) { + // ignore any distribution configuration because the target database is a plain RDBMS + allDependencies.add(target); + enabledFKConstraints.add(c); + } else { + // Make sure that FK references adhere to the restrictions imposed by distribution (replication or sharding) + if (distributionRules == null) { + // this table is not distributed, so we can handle the FK relationship as long + // as the target isn't sharded (replicated is OK) + if (target.distributionRules == null || target.distributionRules.isReferenceTable()) { + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } else if (distributionRules.isReferenceTable()) { + // This table is a reference (replicated) table. We can create FK relationships + // to other tables non-distributed and replicated tables + if (target.distributionRules == null || target.distributionRules.isReferenceTable()) { + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } else if (target.distributionRules != null) { + // This table is sharded. We can only create FK relationships to the target if + // the target is replicated, or has a matching sharding configuration + if (target.distributionRules.isReferenceTable()) { + // the target is replicated, so we can create a FK relationship to it + allDependencies.add(target); + enabledFKConstraints.add(c); + } else if (target.distributionRules.getDistributionColumn() != null + && target.distributionRules.getDistributionColumn().equalsIgnoreCase(this.distributionColumnName) + && c.includesColumn(this.distributionColumnName)) { + // Both tables are sharded. We can only support FK relationships if the source and target + // are "co-located" which means they must share the same distribution column which is also part of + // the foreign key + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } + } } if (this.tablespace != null) { @@ -707,8 +791,7 @@ public Table build(IDataModel dataModel) { // Our schema objects are immutable by design, so all initialization takes place // through the constructor return new Table(getSchemaName(), getObjectName(), this.version, this.tenantColumnName, buildColumns(), this.primaryKey, this.identity, this.indexes.values(), - this.fkConstraints.values(), this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints); - + enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionRules); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java index a968b234a25..859d52957d2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java @@ -72,6 +72,7 @@ public void init() { configureForDerby(); break; case POSTGRESQL: + case CITUS: configureForPostgresql(); break; default: diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index 14442c223a5..7d6ec441467 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -19,6 +19,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -28,7 +29,6 @@ import com.ibm.fhir.database.utils.common.AddForeignKeyConstraint; import com.ibm.fhir.database.utils.common.CommonDatabaseAdapter; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.common.DropForeignKeyConstraint; import com.ibm.fhir.database.utils.common.SchemaInfoObject; import com.ibm.fhir.database.utils.model.CheckConstraint; import com.ibm.fhir.database.utils.model.ColumnBase; @@ -39,7 +39,6 @@ import com.ibm.fhir.database.utils.model.Privilege; import com.ibm.fhir.database.utils.model.Table; import com.ibm.fhir.database.utils.model.With; -import com.ibm.fhir.database.utils.tenant.DropViewDAO; /** * A PostgreSql database target @@ -98,7 +97,8 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionRules distributionRules) { // PostgreSql doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java index 3a7ea2cd50e..1ce289c2148 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java @@ -10,14 +10,10 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; import java.util.logging.Logger; -import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.model.With; /** @@ -55,7 +51,7 @@ public PostgresFillfactorSettingDAO(String schema, String tableName, int fillfac @Override public void run(IDatabaseTranslator translator, Connection c) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // assume we need to set it unless it's been configured already boolean isFillfactor = true; LOG.fine(() -> "Checking the table fillfactor settings"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java index 1d6b4c85ef0..5bdc3f5500c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java @@ -274,4 +274,9 @@ public Optional maximumQueryParameters() { // it's Short.MAX_VALUE return Optional.of(Integer.valueOf(32767)); } + + @Override + public boolean isFamilyPostgreSQL() { + return true; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java index f0856868b83..af7b8079e38 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.model.With; /** @@ -60,7 +59,7 @@ public PostgresVacuumSettingDAO(String schema, String tableName, int vacuumCostL @Override public void run(IDatabaseTranslator translator, Connection c) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { boolean isScaleFactor = true; boolean isVacuumThreshold = true; boolean isVacuumCostLimit = true; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java index 27e5cceea6b..6b4ddbe3048 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java @@ -237,6 +237,7 @@ private GetLease getLeaseDAO() { final GetLease result; switch (this.translator.getType()) { case POSTGRESQL: + case CITUS: result = new GetLeasePostgresql(adminSchema, schemaName, config.getHost(), leaseId, leaseUntil); break; default: diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java index 8ee6dd03e84..c84e4103cd7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java @@ -70,6 +70,7 @@ public void updateSchemaVersionId(int versionId) { final UpdateSchemaVersion cmd; switch (this.translator.getType()) { case POSTGRESQL: + case CITUS: cmd = new UpdateSchemaVersionPostgresql(schemaName, versionId); break; default: diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java index 30ffa211339..85657fc9cc3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java @@ -11,6 +11,7 @@ import javax.transaction.TransactionSynchronizationRegistry; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.DatabaseTranslatorFactory; import com.ibm.fhir.database.utils.db2.Db2Translator; import com.ibm.fhir.database.utils.derby.DerbyTranslator; @@ -51,7 +52,7 @@ public class FHIRResourceDAOFactory { public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, ParameterTransactionDataImpl ptdi) throws IllegalArgumentException, FHIRPersistenceException { - ResourceDAO resourceDAO = null; + final ResourceDAO resourceDAO; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -62,8 +63,11 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; case POSTGRESQL: + case CITUS: resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return resourceDAO; } @@ -81,8 +85,8 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche public static ReindexResourceDAO getReindexResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, ParameterDAO parameterDao) { - IDatabaseTranslator translator = null; - ReindexResourceDAO result = null; + final IDatabaseTranslator translator; + final ReindexResourceDAO result; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -95,9 +99,12 @@ public static ReindexResourceDAO getReindexResourceDAO(Connection connection, St result = new ReindexResourceDAO(connection, translator, parameterDao, schemaName, flavor, cache, rrd); break; case POSTGRESQL: + case CITUS: translator = new PostgresTranslator(); result = new PostgresReindexResourceDAO(connection, translator, parameterDao, schemaName, flavor, cache, rrd); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return result; } @@ -122,7 +129,7 @@ public static IDatabaseTranslator getTranslatorForFlavor(FHIRDbFlavor flavor) { */ public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache) throws IllegalArgumentException, FHIRPersistenceException { - ResourceDAO resourceDAO = null; + final ResourceDAO resourceDAO; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -133,8 +140,11 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, cache, rrd); break; case POSTGRESQL: + case CITUS: resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return resourceDAO; } @@ -151,7 +161,7 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache) { - ResourceReferenceDAO rrd = null; + final ResourceReferenceDAO rrd; switch (flavor.getType()) { case DB2: rrd = new Db2ResourceReferenceDAO(new Db2Translator(), connection, schemaName, cache.getResourceReferenceCache(), adminSchemaName, cache.getParameterNameCache()); @@ -162,6 +172,11 @@ public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection case POSTGRESQL: rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); break; + case CITUS: + rrd = new PostgresResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return rrd; } @@ -174,7 +189,7 @@ public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection * @return */ public static FhirSequenceDAO getSequenceDAO(Connection connection, FHIRDbFlavor flavor) { - FhirSequenceDAO result = null; + final FhirSequenceDAO result; switch (flavor.getType()) { case DB2: // Derby syntax also works for Db2 @@ -184,8 +199,11 @@ public static FhirSequenceDAO getSequenceDAO(Connection connection, FHIRDbFlavor result = new com.ibm.fhir.persistence.jdbc.derby.FhirSequenceDAOImpl(connection); break; case POSTGRESQL: + case CITUS: result = new com.ibm.fhir.persistence.jdbc.postgres.FhirSequenceDAOImpl(connection); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return result; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java index bb4a72ca5e0..1fb92f16a32 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java @@ -195,6 +195,7 @@ public void applySearchOptimizerOptions(Connection c, boolean isCompartment) { switch (this.flavor.getType()) { case POSTGRESQL: + case CITUS: // PostgreSQL needs optimizer options set to address search performance issues // as described in issue 1911 final String pgName = FHIRConfiguration.PROPERTY_DATASOURCES + "/" + datastoreId + "/" + FHIRConfiguration.PROPERTY_JDBC_SEARCH_OPTIMIZER_OPTIONS; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java index 6a3471f5286..0398442cb6d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java @@ -26,4 +26,10 @@ public interface FHIRDbFlavor { * @return */ public DbType getType(); + + /** + * Is the dbType from the PostgreSQL family? + * @return + */ + public boolean isFamilyPostgreSQL(); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java index 269635154b6..5b4cc4495ba 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java @@ -34,4 +34,9 @@ public boolean isMultitenant() { public DbType getType() { return this.type; } + + @Override + public boolean isFamilyPostgreSQL() { + return this.type == DbType.POSTGRESQL || this.type == DbType.CITUS; + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java index 6f637bd2291..f748d95dbf9 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java @@ -369,7 +369,7 @@ public long erase(ResourceEraseRecord eraseRecord, EraseDTO eraseDto) throws Exc if (DbType.DB2.equals(getFlavor().getType()) && eraseDto.getVersion() == null) { runCallableStatement(CALL_DB2, erasedResourceGroupId); - } else if (DbType.POSTGRESQL.equals(getFlavor().getType()) && eraseDto.getVersion() == null) { + } else if (getFlavor().isFamilyPostgreSQL() && eraseDto.getVersion() == null) { runCallableStatement(CALL_POSTGRES, erasedResourceGroupId); } else { // Uses the Native Java to execute a Resource Erase diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java index 1be7a08cbe2..189c82b44ca 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java @@ -117,6 +117,7 @@ public int readOrAddParameterNameId(String parameterName) throws FHIRPersistence pnd = new DerbyParameterNamesDAO(connection, getSchemaName()); break; case POSTGRESQL: + case CITUS: pnd = new PostgresParameterNamesDAO(connection, getSchemaName()); break; default: @@ -154,6 +155,7 @@ public int readOrAddCodeSystemId(String codeSystemName) throws FHIRPersistenceDB csd = new DerbyCodeSystemDAO(connection, getSchemaName()); break; case POSTGRESQL: + case CITUS: csd = new PostgresCodeSystemDAO(connection, getSchemaName()); break; default: diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java index 63464e6ba38..fb1fa0362e9 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java @@ -2753,6 +2753,7 @@ private String getDataCol() { case DERBY: return "CAST(NULL AS BLOB) AS DATA"; case POSTGRESQL: + case CITUS: return "NULL::TEXT AS DATA"; default: throw new IllegalStateException("Database type not supported: " + translator.getType().name()); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java index 71c7fdc8664..b80d3ba8ca4 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java @@ -19,11 +19,6 @@ import java.util.logging.Level; import java.util.logging.Logger; -import jakarta.json.Json; -import jakarta.json.JsonObject; -import jakarta.json.JsonReader; -import jakarta.json.JsonReaderFactory; - import com.ibm.fhir.config.DefaultFHIRConfigProvider; import com.ibm.fhir.config.FHIRConfigProvider; import com.ibm.fhir.config.FHIRConfiguration; @@ -60,6 +55,11 @@ import com.ibm.fhir.schema.derby.DerbyFhirDatabase; import com.ibm.fhir.validation.test.ValidationProcessor; +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonReader; +import jakarta.json.JsonReaderFactory; + /** * Integration test using a multi-tenant schema in DB2 as the target for the * FHIR R4 Examples. @@ -124,7 +124,7 @@ public class Main { // mode of operation private static enum Operation { - DB2, DERBY, DERBYNETWORK, POSTGRESQL, PARSE + DB2, DERBY, DERBYNETWORK, POSTGRESQL, CITUS, PARSE } private Operation mode = Operation.DB2; @@ -226,6 +226,9 @@ protected void parseArgs(String[] args) { case "--postgresql": this.mode = Operation.POSTGRESQL; break; + case "--citus": + this.mode = Operation.CITUS; + break; case "--parse": this.mode = Operation.PARSE; break; @@ -303,6 +306,7 @@ protected void process() throws Exception { processDerbyNetwork(); break; case POSTGRESQL: + case CITUS: processPostgreSql(); break; case PARSE: diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java index 17064043ea5..087b7876850 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java @@ -25,6 +25,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -94,6 +95,12 @@ public void configure() { schemaName = "FHIRDATA"; } break; + case CITUS: + translator = new CitusTranslator(); + if (schemaName == null) { + schemaName = "FHIRDATA"; + } + break; case DB2: default: translator = new Db2Translator(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index dc151ea9dd7..a57244779d8 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -98,6 +98,7 @@ import com.ibm.fhir.database.utils.api.TenantStatus; import com.ibm.fhir.database.utils.api.UndefinedNameException; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; @@ -193,8 +194,8 @@ public class Main { // Indicates if the feature is enabled for the DbType public List MULTITENANT_FEATURE_ENABLED = Arrays.asList(DbType.DB2); - public List STORED_PROCEDURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL); - public List PRIVILEGES_FEATURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL); + public List STORED_PROCEDURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL, DbType.CITUS); + public List PRIVILEGES_FEATURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL, DbType.CITUS); // Properties accumulated as we parse args and read configuration files private final Properties properties = new Properties(); @@ -496,6 +497,7 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { logger.info("No database specific artifacts"); break; case POSTGRESQL: + case CITUS: gen.buildDatabaseSpecificArtifactsPostgres(pdm); break; default: @@ -518,15 +520,24 @@ protected void updateFhirSchema() { final String targetSchemaName = schema.getSchemaName(); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { + if (this.dbType == DbType.CITUS) { + // First version with Citus support is V0026 and we can't upgrade + // from before that + int currentSchemaVersion = svm.getVersionForSchema(); + if (currentSchemaVersion >= 0 && currentSchemaVersion < FhirSchemaVersion.V0026.vid()) { + throw new IllegalStateException("Cannot upgrade Citus databases with schema version < V0026"); + } + } + // Build/update the FHIR-related tables as well as the stored procedures - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); boolean isNewDb = updateSchema(pdm); @@ -595,7 +606,7 @@ protected void updateOauthSchema() { SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildOAuthSchemaModel(pdm); updateSchema(pdm); @@ -641,7 +652,7 @@ protected void updateJavaBatchSchema() { SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildJavaBatchSchemaModel(pdm); updateSchema(pdm); @@ -743,7 +754,7 @@ protected void populateResourceTypeAndParameterNameTableEntries(Integer tenantId * Typically used during development. */ protected void dropSchema() { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildCommonModel(pdm, dropFhirSchema, dropOauthSchema, dropJavaBatchSchema); // Dropping the schema in PostgreSQL can fail with an out of shared memory error @@ -872,7 +883,7 @@ protected void updateProcedures() { } // Build/update the tables as well as the stored procedures - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); // Since this is a stored procedure, we need the model. // We must pass in true to flag to the underlying layer that the // Procedures need to be generated. @@ -936,7 +947,7 @@ protected void grantPrivilegesForFhirData() { final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); @@ -958,7 +969,7 @@ protected void grantPrivilegesForOAuth() { final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildOAuthSchemaModel(pdm); pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); @@ -978,7 +989,7 @@ protected void grantPrivilegesForBatch() { final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildJavaBatchSchemaModel(pdm); pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_BATCH_GRANT_GROUP, grantTo); } catch (DataAccessException x) { @@ -1212,7 +1223,7 @@ protected void allocateTenant() { // Build/update the tables as well as the stored procedures FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); // Get the data model to create the table partitions. This is threaded, so transactions are @@ -1345,7 +1356,7 @@ protected void refreshTenants() { // It's crucial we use the correct schema for each particular tenant, which // is why we have to build the PhysicalDataModel separately for each tenant FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); Db2Adapter adapter = new Db2Adapter(connectionPool); @@ -1563,7 +1574,7 @@ protected void dropTenant() { // Build the model of the data (FHIRDATA) schema which is then used to drive the drop FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); // Detach the tenant partition from each of the data tables @@ -1582,7 +1593,7 @@ protected void dropDetachedPartitionTables() { TenantInfo tenantInfo = getTenantInfo(); FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); dropDetachedPartitionTables(pdm, tenantInfo); @@ -2088,6 +2099,8 @@ protected void parseArgs(String[] args) { case POSTGRESQL: translator = new PostgresTranslator(); break; + case CITUS: + translator = new CitusTranslator(); case DB2: default: break; @@ -2468,16 +2481,16 @@ private void doBackfill(IDatabaseAdapter adapter) { } /** - * updates the vacuum settings for postgres. + * updates the vacuum settings for postgres/citus. */ public void updateVacuumSettings() { - if (dbType != DbType.POSTGRESQL) { + if (dbType != DbType.POSTGRESQL && dbType != DbType.CITUS) { logger.severe("Updating the vacuum settings is only supported on postgres and the setting is for '" + dbType + "'"); return; } // Create the Physical Data Model - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildCommonModel(pdm, true, false, false); // Setup the Connection Pool @@ -2498,6 +2511,14 @@ public void updateVacuumSettings() { } } + /** + * Citus is a distributed implementation of PostgreSQL + * @return + */ + private boolean isDistributed() { + return this.dbType == DbType.CITUS; + } + /** * runs the vacuum update inside a single connection and single transaction. * @@ -2553,6 +2574,7 @@ private void generateDbSizeReport() { final ISizeCollector collector; switch (dbType) { case POSTGRESQL: + case CITUS: // assume for now this works for Citus collector = new PostgresSizeCollector(model); break; case DB2: diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java index 3f924bb7ffd..bc1dbab769d 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java @@ -82,7 +82,7 @@ public enum HelpMenu { MI_TENANT_KEY(TENANT_KEY, "tenantKey", "the tenant-key in the queries"), MI_TENANT_KEY_FILE(TENANT_KEY_FILE, "tenant-key-file-location", "sets the tenant key file location"), MI_LIST_TENANTS(LIST_TENANTS, "", "fetches list of tenants and current status"), - MI_DB_TYPE(DB_TYPE, "dbType" , "Either derby, postgresql, db2"), + MI_DB_TYPE(DB_TYPE, "dbType" , "Either derby, postgresql, db2, citus"), MI_DELETE_TENANT_META(DELETE_TENANT_META, "tenantName", "deletes tenant metadata given the tenantName"), MI_DROP_DETACHED(DROP_DETACHED, "tenantName", "(phase 2) drops the detached tenant partition tables given the tenantName"), MI_FREEZE_TENANT(FREEZE_TENANT, "", "Changes the tenant state to frozen, and subsequently (Db2 only)"), diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java index 035556a525e..37be2eb4d79 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusAdapter; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.common.LogFormatter; @@ -105,6 +106,7 @@ public static JdbcPropertyAdapter getPropertyAdapter(DbType dbType, Properties p case DERBY: return new DerbyPropertyAdapter(props); case POSTGRESQL: + case CITUS: return new PostgresPropertyAdapter(props); default: throw new IllegalStateException("Unsupported db type: " + dbType); @@ -119,6 +121,8 @@ public static IDatabaseAdapter getDbAdapter(DbType dbType, JdbcTarget target) { return new DerbyAdapter(target); case POSTGRESQL: return new PostgresAdapter(target); + case CITUS: + return new CitusAdapter(target); default: throw new IllegalStateException("Unsupported db type: " + dbType); } @@ -132,6 +136,8 @@ public static IDatabaseAdapter getDbAdapter(DbType dbType, IConnectionProvider c return new DerbyAdapter(connectionProvider); case POSTGRESQL: return new PostgresAdapter(connectionProvider); + case CITUS: + return new CitusAdapter(connectionProvider); default: throw new IllegalStateException("Unsupported db type: " + dbType); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index 1fd848c4e1b..d57601c342a 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -201,7 +201,7 @@ public void addLogicalResources(List group, String prefix) { // things sensible. Table.Builder builder = Table.builder(schemaName, tableName) .setTenantColumnName(MT_ID) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -214,6 +214,7 @@ public void addLogicalResources(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding // Add indexes to avoid dead lock issue of derby, and improve Db2 performance // Derby requires all columns used in where clause to be indexed, otherwise whole table lock will be // used instead of row lock, which can cause dead lock issue frequently during concurrent accesses. @@ -336,7 +337,7 @@ public void addResources(List group, String prefix) { final String tableName = prefix + _RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0024.vid()) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( RESOURCE_ID, false) @@ -352,6 +353,7 @@ public void addResources(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0024.vid()) { @@ -420,7 +422,7 @@ public void addStrValues(List group, String prefix) { // Parameters are tied to the logical resource Table tbl = Table.builder(schemaName, tableName) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) // .addBigIntColumn( ROW_ID, false) // Removed by issue-1683 - composites refactor .addIntColumn( PARAMETER_NAME_ID, false) @@ -437,6 +439,7 @@ public void addStrValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -509,7 +512,7 @@ public Table addResourceTokenRefs(List group, String prefix) { // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) @@ -524,6 +527,7 @@ public Table addResourceTokenRefs(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -588,7 +592,7 @@ public Table addProfiles(List group, String prefix) { // logical_resources (1) ---- (*) patient_profiles (*) ---- (0|1) common_canonical_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn( CANONICAL_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -601,6 +605,7 @@ public Table addProfiles(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -636,7 +641,7 @@ public Table addTags(List group, String prefix) { // logical_resources (1) ---- (*) patient_tags (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -647,6 +652,7 @@ public Table addTags(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -679,7 +685,7 @@ public Table addSecurity(List group, String prefix) { // logical_resources (1) ---- (*) patient_security (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -690,6 +696,7 @@ public Table addSecurity(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -784,7 +791,7 @@ public void addDateValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) @@ -800,6 +807,7 @@ public void addDateValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -850,7 +858,7 @@ public void addNumberValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) @@ -866,6 +874,7 @@ public void addNumberValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -924,7 +933,7 @@ public void addLatLngValues(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addDoubleColumn( LATITUDE_VALUE, true) @@ -940,6 +949,7 @@ public void addLatLngValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -996,7 +1006,7 @@ public void addQuantityValues(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( CODE, 255, false) @@ -1017,6 +1027,7 @@ public void addQuantityValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -1053,7 +1064,7 @@ public void addListLogicalResourceItems(List group, String pref final int lib = LOGICAL_ID_BYTES; Table tbl = Table.builder(schemaName, LIST_LOGICAL_RESOURCE_ITEMS) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres vacuum changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -1064,6 +1075,7 @@ public void addListLogicalResourceItems(List group, String pref .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -1096,7 +1108,7 @@ public void addPatientCurrentRefs(List group, String prefix) { // model with a foreign key to avoid order of insertion issues Table tbl = Table.builder(schemaName, PATIENT_CURRENT_REFS) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -1109,6 +1121,7 @@ public void addPatientCurrentRefs(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index fb0ad74194a..533e58079dd 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -540,7 +540,7 @@ public void addLogicalResources(PhysicalDataModel pdm) { final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD"; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) @@ -558,6 +558,7 @@ public void addLogicalResources(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(addWiths()) // add table tuning .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -641,7 +642,7 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) { final String tableName = COMMON_CANONICAL_VALUES; final String unqCanonicalUrl = "UNQ_" + tableName + "_URL"; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0014.vid()) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(CANONICAL_ID, false) .addVarcharColumn(URL, CANONICAL_URL_BYTES, false) @@ -650,6 +651,7 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(URL) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally NOP @@ -679,7 +681,7 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes (Original Table at V0014) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn( CANONICAL_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES @@ -693,6 +695,7 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -727,7 +730,7 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes, original table created at version V0014 + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES @@ -739,6 +742,7 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -770,7 +774,7 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes, original table created at version V0016 + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES @@ -782,6 +786,7 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -819,6 +824,9 @@ public void addResourceChangeLog(PhysicalDataModel pdm) { With.with("autovacuum_vacuum_cost_limit", "2000") // V0019 ); + // Note that for now, we elect to not distribute/shard this table because doing so + // would interfere with the queries supporting the history API which are based on + // index range scans across a contiguous range of records Table tbl = Table.builder(schemaName, tableName) .setTenantColumnName(MT_ID) .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes @@ -867,7 +875,7 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { // because it makes it very easy to find the most recent changes to resources associated with // a given patient (for example). Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( COMPARTMENT_NAME_ID, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -881,6 +889,7 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -913,7 +922,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) { final int msb = MAX_SEARCH_STRING_BYTES; Table tbl = Table.builder(schemaName, STR_VALUES) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( STR_VALUE, msb, true) @@ -929,6 +938,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -959,7 +969,7 @@ public Table addResourceDateValues(PhysicalDataModel model) { final String logicalResourcesTable = LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addTimestampColumn( DATE_START,6, true) @@ -974,6 +984,7 @@ public Table addResourceDateValues(PhysicalDataModel model) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion == 1) { @@ -1015,6 +1026,7 @@ resource_type VARCHAR(64) NOT NULL */ protected void addResourceTypes(PhysicalDataModel model) { resourceTypesTable = Table.builder(schemaName, RESOURCE_TYPES) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( RESOURCE_TYPE_ID, false) .addVarcharColumn( RESOURCE_TYPE, 64, false) @@ -1023,6 +1035,7 @@ protected void addResourceTypes(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1092,6 +1105,7 @@ protected void addParameterNames(PhysicalDataModel model) { String[] prfIncludeCols = {PARAMETER_NAME_ID}; parameterNamesTable = Table.builder(schemaName, PARAMETER_NAMES) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( PARAMETER_NAME, 255, false) @@ -1100,6 +1114,7 @@ protected void addParameterNames(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 treat this as a reference table .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1128,7 +1143,7 @@ code_system_name VARCHAR(255 OCTETS) NOT NULL */ protected void addCodeSystems(PhysicalDataModel model) { codeSystemsTable = Table.builder(schemaName, CODE_SYSTEMS) - .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( CODE_SYSTEM_ID, false) .addVarcharColumn(CODE_SYSTEM_NAME, 255, false) @@ -1137,6 +1152,7 @@ protected void addCodeSystems(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 treat this as a reference table .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1174,13 +1190,16 @@ protected void addCodeSystems(PhysicalDataModel model) { * the token_value represents its logical_id. This approach simplifies query writing when * following references. * + * If sharding is supported, this table is distributed by token_value which unfortunately + * means that it cannot be the target of any foreign key constraint (which needs to use + * the primary key COMMON_TOKEN_VALUE_ID). * @param pdm * @return the table definition */ public void addCommonTokenValues(PhysicalDataModel pdm) { final String tableName = COMMON_TOKEN_VALUES; commonTokenValuesTable = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0006.vid()) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn( COMMON_TOKEN_VALUE_ID, false) .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) @@ -1192,6 +1211,7 @@ public void addCommonTokenValues(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(TOKEN_VALUE) // V0026 shard using token_value .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1222,7 +1242,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { // logical_resources (0|1) ---- (*) resource_token_refs Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries @@ -1236,6 +1256,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(addWiths()) // table tuning .addMigration(priorVersion -> { // Replace the indexes initially defined in the V0006 version with better ones diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java index 130972d7738..151dc661225 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java @@ -45,6 +45,7 @@ public enum FhirSchemaVersion { ,V0023(23, "issue-2900 erased_resources to support $erase when offloading payloads", false) ,V0024(24, "issue-2900 for offloading add resource_payload_key to xx_resources", false) ,V0025(25, "issue-3158 stored proc updates to prevent deleting currently deleted resources", false) + ,V0026(26, "issue-nnnn extensions to support distribution/sharding", false) ; // The version number recorded in the VERSION_HISTORY diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java index 711bdf61b90..243eefc413f 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java @@ -84,6 +84,7 @@ public MigrateV0021AbstractTypeRemoval(IDatabaseAdapter adapter, String adminSch public void run(IDatabaseTranslator translator, Connection c) { switch (translator.getType()) { case POSTGRESQL: + case CITUS: case DERBY: checkDataTables(translator, c); checkShouldThrowException(); @@ -172,7 +173,7 @@ private void checkDataTables(IDatabaseTranslator translator, Connection c) { for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { if (adapter.doesTableExist(schemaName, deprecatedTable)) { String table = schemaName + "." + deprecatedTable; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { table = schemaName.toLowerCase() + "." + deprecatedTable; } @@ -210,7 +211,7 @@ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { // Run across both tables for (String tablePrefix : tables) { // Drop the View for the Table - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { runDropTableResourceGroup(translator, c, schemaName.toLowerCase(), tablePrefix.toLowerCase(), VALUE_TYPES_LOWER); } else { runDropTableResourceGroup(translator, c, schemaName, tablePrefix, VALUE_TYPES); @@ -227,7 +228,7 @@ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) { // and logs print warnings saying the tables don't exist. That's OK. for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) { String table = prefix + deprecatedTable; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { adapter.dropTable(schemaName.toLowerCase(), table.toLowerCase()); } else { adapter.dropTable(schemaName, table); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java index 4cb3a690874..ec39b39b4bc 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java @@ -15,7 +15,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; /** * Checks to see if any of the tables exist in the target database. @@ -73,6 +72,7 @@ public UnusedTableRemovalNeedsV0021Migration(String schemaName) { public Boolean run(IDatabaseTranslator translator, Connection c) { switch (translator.getType()) { case POSTGRESQL: + case CITUS: return checkPostgres(translator, c); case DB2: return checkDb2(translator, c); @@ -163,7 +163,7 @@ private boolean hasTables(IDatabaseTranslator translator, Connection c, final St ps.setString(i++, schemaName); for (String deprecatedTable : DEPRECATED_TABLES) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { ps.setString(i++, deprecatedTable.toLowerCase()); } else { ps.setString(i++, deprecatedTable); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java index 1c63acf8751..1de1c35113d 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java @@ -31,6 +31,7 @@ import com.ibm.fhir.database.utils.api.DatabaseNotReadyException; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.DropForeignKeyConstraint; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; @@ -222,6 +223,9 @@ protected void parseArgs(String[] args) { case POSTGRESQL: translator = new PostgresTranslator(); break; + case CITUS: + translator = new CitusTranslator(); + break; case DB2: default: break; diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java index c3b57d3d99d..ecc13bb7a6c 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java @@ -52,7 +52,7 @@ public void test() throws Exception { // Make sure we can correctly determine the latest schema version value svm.updateSchemaVersion(); - assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0025.vid()); + assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0026.vid()); assertFalse(svm.isSchemaOld()); } From 7244dcee90fb87bbff0296b3b35cfb78bbcd85b6 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 9 Mar 2022 13:24:42 +0000 Subject: [PATCH 02/40] issue #3437 schema build changes to support Citus distribution Signed-off-by: Robin Arnold --- .../database/utils/api/DistributionRules.java | 26 +++ .../database/utils/api/IDatabaseAdapter.java | 20 +- .../database/utils/citus/CitusAdapter.java | 140 ++++++++++-- .../utils/citus/ConfigureConnectionDAO.java | 48 ++++ .../citus/CreateDistributedTableDAO.java | 2 +- .../utils/citus/CreateReferenceTableDAO.java | 2 +- .../utils/common/CommonDatabaseAdapter.java | 11 +- .../database/utils/derby/DerbyAdapter.java | 6 +- .../fhir/database/utils/model/BaseObject.java | 7 +- .../database/utils/model/CreateIndex.java | 53 ++++- .../database/utils/model/IDatabaseObject.java | 8 +- .../fhir/database/utils/model/IndexDef.java | 11 +- .../utils/model/PhysicalDataModel.java | 13 +- .../ibm/fhir/database/utils/model/Table.java | 9 +- .../fhir/database/utils/model/Tablespace.java | 6 +- .../utils/pool/PoolConnectionProvider.java | 25 ++- .../utils/postgres/PostgresAdapter.java | 17 +- .../jdbc/dao/EraseResourceDAO.java | 10 +- .../java/com/ibm/fhir/schema/app/Main.java | 37 ++- .../schema/control/FhirSchemaConstants.java | 3 +- .../schema/control/FhirSchemaGenerator.java | 116 +++++++++- .../main/resources/citus/add_any_resource.sql | 212 ++++++++++++++++++ .../schema/derby/DerbyFhirDatabaseTest.java | 4 +- 23 files changed, 718 insertions(+), 68 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java create mode 100644 fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java index 30855f64d3c..b710bccee86 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java @@ -6,6 +6,9 @@ package com.ibm.fhir.database.utils.api; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; /** * Rules for distributing a table in a distributed RDBMS such as Citus @@ -46,4 +49,27 @@ public String getDistributionColumn() { public boolean isReferenceTable() { return this.referenceTable; } + + /** + * Is the table configured to be distributed (sharded) + * @return + */ + public boolean isDistributedTable() { + return this.distributionColumn != null; + } + + /** + * Asks if the distributionColumn is contained in the given collection of column names + + * @implNote case-insensitive + * @param columns + * @return + */ + public boolean includesDistributionColumn(Collection columns) { + if (this.distributionColumn != null) { + Set colSet = columns.stream().map(p -> p.toLowerCase()).collect(Collectors.toSet()); + return colSet.contains(this.distributionColumn.toLowerCase()); + } + return false; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 46f454dd8a1..3cc74d8ea96 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -85,6 +85,14 @@ public void createTable(String schemaName, String name, String tenantColumnName, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, DistributionRules distributionRules); + /** + * Apply any distribution rules configured for the named table + * @param schemaName + * @param tableName + * @param distributionRules + */ + public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules); + /** * Add a new column to an existing table * @param schemaName @@ -152,27 +160,29 @@ public void createTable(String schemaName, String name, String tenantColumnName, public void dropProcedure(String schemaName, String procedureName); /** - * + * Create a unique index * @param schemaName * @param tableName * @param indexName * @param tenantColumnName * @param indexColumns * @param includeColumns + * @param distributionRules */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, List includeColumns); + List indexColumns, List includeColumns, DistributionRules distributionRules); /** - * + * Create a unique index * @param schemaName * @param tableName * @param indexName * @param tenantColumnName * @param indexColumns + * @param distributionRules */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns); + List indexColumns, DistributionRules distributionRules); /** * diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java index 82be749b95d..a2e8119cb1f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -7,13 +7,18 @@ package com.ibm.fhir.database.utils.citus; import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTarget; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.model.CheckConstraint; import com.ibm.fhir.database.utils.model.ColumnBase; import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; import com.ibm.fhir.database.utils.model.PrimaryKeyDef; import com.ibm.fhir.database.utils.model.With; import com.ibm.fhir.database.utils.postgres.PostgresAdapter; @@ -23,6 +28,7 @@ * A database adapter implementation for Citus (distributed PostgreSQL) */ public class CitusAdapter extends PostgresAdapter { + private static final Logger logger = Logger.getLogger(CitusAdapter.class.getName()); /** * Public constructor @@ -40,24 +46,130 @@ public CitusAdapter(IConnectionProvider cp) { super(cp); } + @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, DistributionRules distributionRules) { - super.createTable(schemaName, name, tenantColumnName, columns, primaryKey, - identity, tablespaceName, withs, checkConstraints, distributionRules); - - // Apply the distribution rules, if any. Tables without distribution rules are created - // only on Citus controller nodes and never distributed to the worker nodes - if (distributionRules != null) { - if (distributionRules.isReferenceTable()) { - // A table that is fully replicated for each worker node - CreateReferenceTableDAO dao = new CreateReferenceTableDAO(schemaName, name); - runStatement(dao); - } else if (distributionRules.getDistributionColumn() != null && distributionRules.getDistributionColumn().length() > 0) { - // A table that is sharded using a hash on the distributionColumn value - CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, name, distributionRules.getDistributionColumn()); - runStatement(dao); + + // We don't use partitioning for multi-tenancy in our Citus implementation, so ignore the mt_id column + if (tenantColumnName != null) { + warnOnce(MessageKey.MULTITENANCY, "Citus does not support multi-tenancy: " + name); + } + + // Build a Citus-specific create table statement + String ddl = buildCitusCreateTableStatement(schemaName, name, columns, primaryKey, identity, withs, checkConstraints, distributionRules); + runStatement(ddl); + } + + /** + * Construct a CREATE TABLE statement using some Citus-specific business + * logic. + * @param schema + * @param name + * @param columns + * @param pkDef + * @param identity + * @param withs + * @param checkConstraints + * @param distributionRules + * @return + */ + private String buildCitusCreateTableStatement(String schema, String name, List columns, + PrimaryKeyDef pkDef, IdentityDef identity, List withs, + List checkConstraints, DistributionRules distributionRules) { + + if (identity != null && distributionRules != null && distributionRules.getDistributionColumn() != null) { + logger.warning("Citus: Ignoring IDENTITY columns on distributed table: '" + name + "." + identity.getColumnName()); + identity = null; + } + + StringBuilder result = new StringBuilder(); + result.append("CREATE TABLE "); + result.append(getQualifiedName(schema, name)); + result.append('('); + result.append(buildColumns(columns, identity)); + + if (pkDef != null) { + // Add the primary key definition after the columns. For Citus, if the table is + // distributed (sharded) then the distribution key MUST be one of the columns + // of the primary key. Make sure we lower-case things first so we guarantee a + // match where expected + String pkColString = String.join(",", pkDef.getColumns()); + Set pkSet = pkDef.getColumns().stream().map(c -> c.toLowerCase()).collect(Collectors.toSet()); + final String ldc = distributionRules == null || distributionRules.getDistributionColumn() == null ? null : distributionRules.getDistributionColumn().toLowerCase(); + if (ldc == null || pkSet.contains(ldc)) { + result.append(", CONSTRAINT "); + result.append(pkDef.getConstraintName()); + result.append(" PRIMARY KEY ("); + result.append(pkColString); + result.append(')'); + } else { + // Hopefully this is an intended data model design decision. Because it's so + // fundamental, we always want to warn about it. + logger.warning("Skipping primary key for table '" + name + + "' because it does not include required distribution column: '" + ldc + + "', only (" + pkColString + ")"); } } + + // Add any CHECK constraints + for (CheckConstraint cc: checkConstraints) { + result.append(", CONSTRAINT "); + result.append(cc.constraintName); + result.append(" CHECK ("); + result.append(cc.getConstraintExpression()); + result.append(")"); + } + result.append(')'); + + // Creates WITH (fillfactor=70, key2=val2); + if (withs != null && !withs.isEmpty()) { + StringBuilder builder = new StringBuilder(" WITH ("); + builder.append( + withs.stream() + .map(with -> with.buildWithComponent()) + .collect(Collectors.joining(","))); + builder.append(")"); + result.append(builder.toString()); + } + + return result.toString(); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionRules distributionRules) { + // For Citus, we are prevented from creating a unique index unless the index contains + // the distribution column + List columnNames = indexColumns.stream().map(ocd -> ocd.getColumnName()).collect(Collectors.toList()); + if (distributionRules != null && distributionRules.isDistributedTable() && !distributionRules.includesDistributionColumn(columnNames)) { + // Can only a normal index because it isn't partitioned by the distributionColumn + String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); + runStatement(ddl); + } else { + // Index is partitioned by the distributionColumn, so it can be unique + String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); + runStatement(ddl); + } + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules) { + // Apply the distribution rules. Tables without distribution rules are created + // only on Citus controller nodes and never distributed to the worker nodes. All + // the distribution changes are implemented in one transaction, which makes it much + // more efficient. + final String fullName = DataDefinitionUtil.getQualifiedName(schemaName, tableName); + if (distributionRules.isReferenceTable()) { + // A table that is fully replicated for each worker node + logger.info("Citus: distributing reference table '" + fullName + "'"); + CreateReferenceTableDAO dao = new CreateReferenceTableDAO(schemaName, tableName); + runStatement(dao); + } else if (distributionRules.getDistributionColumn() != null && distributionRules.getDistributionColumn().length() > 0) { + // A table that is sharded using a hash on the distributionColumn value + logger.info("Citus: Sharding table '" + fullName + "' using '" + distributionRules.getDistributionColumn() + "'"); + CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, tableName, distributionRules.getDistributionColumn()); + runStatement(dao); + } } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java new file mode 100644 index 00000000000..c578a9f7c06 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java @@ -0,0 +1,48 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; + +/** + * DAO to configure the Citus database connection when performing schema build + * activities. This must be performed before any of the following UDFs are called: + * - create_distributed_table + * - create_reference_table + * to avoid the following error: + * 
+ * org.postgresql.util.PSQLException: ERROR: cannot modify table "common_token_values" because there was a parallel operation on a distributed table in the transaction
+ * Detail: When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
+ * Hint: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+ *
+ */ +public class ConfigureConnectionDAO implements IDatabaseStatement { + + /** + * Public constructor + */ + public ConfigureConnectionDAO() { + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + final String SQL = "SET LOCAL citus.multi_shard_modify_mode TO 'sequential'"; + + try (Statement s = c.createStatement()) { + s.executeUpdate(SQL); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java index 875162abd16..11aa036ccc0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java index 718a3caee4c..654fe7fa38d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java index 7e5ebe58869..da25892a6ac 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseStatement; @@ -197,7 +198,7 @@ protected String buildCreateTableStatement(String schema, String name, List indexColumns, List includeColumns) { + List indexColumns, List includeColumns, DistributionRules distributionRules) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, includeColumns, true); runStatement(ddl); @@ -205,7 +206,7 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns) { + List indexColumns, DistributionRules distributionRules) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, true); runStatement(ddl); @@ -748,4 +749,8 @@ public void reorgTable(String schemaName, String tableName) { // NOP, unless overridden by a subclass (Db2Adapter, for example) } + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules) { + // NOP. Only used for distributed databases like Citus + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index ff00b81e73e..1201c155aaf 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -94,10 +94,10 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns) { + List includeColumns, DistributionRules distributionRules) { // Derby doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java index f0c52dd0050..abb6cf8bc12 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -323,4 +323,9 @@ public void addPrivilege(String groupName, Privilege p) { public void visit(Consumer c) { c.accept(this); } + + @Override + public void applyDistributionRules(IDatabaseAdapter target) { + // NOP. Only applies to Table + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java index 36a6bd7c1d4..de7def1348d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.common.CreateIndexStatement; @@ -26,20 +27,27 @@ public class CreateIndex extends BaseObject { // The name of the tenant column when used for multi-tenant databases private final String tenantColumnName; - + + // The table name the index will be created on private final String tableName; + // Distribution rules if the associated table is distributed + private final DistributionRules distributionRules; + /** * Protected constructor. Use the Builder to create instance. * @param schemaName * @param indexName * @param version + * @distributionRules */ - protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName) { + protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName, + DistributionRules distributionRules) { super(schemaName, versionTrackingName, DatabaseObjectType.INDEX, version); this.tableName = tableName; this.indexDef = indexDef; this.tenantColumnName = tenantColumnName; + this.distributionRules = distributionRules; } /** @@ -86,7 +94,7 @@ public String getTypeNameVersion() { @Override public void apply(IDatabaseAdapter target) { long start = System.nanoTime(); - indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target); + indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionRules); if (logger.isLoggable(Level.FINE)) { long end = System.nanoTime(); @@ -151,7 +159,12 @@ public static class Builder { // Special case to handle a previous defect where indexes were tracked using tableName in version_history private String versionTrackingName; - + // Set if the table is distributed + private String distributionColumn; + + // Set if the table is a distributed reference type table + private boolean distributionReference; + /** * @param schemaName the schemaName to set */ @@ -182,7 +195,27 @@ public Builder setVersionTrackingName(String name) { this.versionTrackingName = name; return this; } - + + /** + * Setter for distributionColumn + * @param name + * @return + */ + public Builder setDistributionColumn(String name) { + this.distributionColumn = name; + return this; + } + + /** + * Setter for distributionReference + * @param flag + * @return + */ + public Builder setDistributionReference(boolean flag) { + this.distributionReference = flag; + return this; + } + /** * @param version the version to set */ @@ -190,7 +223,6 @@ public Builder setVersion(int version) { this.version = version; return this; } - /** * @param unique the unique to set @@ -236,8 +268,13 @@ public CreateIndex build() { if (versionTrackingName == null) { versionTrackingName = this.indexName; } + DistributionRules distributionRules = null; + if (this.distributionReference || this.distributionColumn != null) { + distributionRules = new DistributionRules(distributionColumn, distributionReference); + } + return new CreateIndex(schemaName, versionTrackingName, tableName, version, - new IndexDef(indexName, indexCols, unique), tenantColumnName); + new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionRules); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java index 091ce64bff6..9be9e25ad8f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -49,6 +49,12 @@ public interface IDatabaseObject { */ public void applyTx(IDatabaseAdapter target, ITransactionProvider cp, IVersionHistoryService vhs); + /** + * Apply any distribution rules associated with the object (usually a table) + * @param target the target database we apply the operation to + */ + public void applyDistributionRules(IDatabaseAdapter target); + /** * Apply the change, but only if it has a newer version than we already have * recorded in the database diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java index 372e519ad50..b604f116aaa 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ import java.util.List; import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.common.CreateIndexStatement; @@ -68,13 +69,15 @@ public boolean isUnique() { * Apply this object to the given database target * @param tableName * @param target + * @param distributionRules */ - public void apply(String schemaName, String tableName, String tenantColumnName, IDatabaseAdapter target) { + public void apply(String schemaName, String tableName, String tenantColumnName, IDatabaseAdapter target, + DistributionRules distributionRules) { if (includeColumns != null && includeColumns.size() > 0) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionRules); } else if (unique) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); } else { target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index 0f76fdc3d5b..efeacb3c098 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -134,6 +134,17 @@ public void apply(IDatabaseAdapter target) { } } + /** + * Make a pass over all the objects again and apply any distribution rules they + * may have (e.g. for Citus) + * @param target + */ + public void applyDistributionRules(IDatabaseAdapter target) { + for (IDatabaseObject obj: allObjects) { + obj.applyDistributionRules(target); + } + } + /** * Apply all the objects linearly, but using the version history service to determine * what's new and what already exists diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index b7186736823..81f6a605a05 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -138,7 +138,7 @@ public void apply(IDatabaseAdapter target) { // Now add any indexes associated with this table for (IndexDef idx: this.indexes) { - idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionRules); } // Foreign key constraints @@ -940,4 +940,11 @@ public void visitReverse(DataModelVisitor v) { this.fkConstraints.forEach(fk -> v.visited(this, fk)); v.visited(this); } + + @Override + public void applyDistributionRules(IDatabaseAdapter target) { + if (this.distributionRules != null) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); + } + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java index ca1e6e44fde..65e4e4c62bf 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -95,4 +95,8 @@ public void visitReverse(DataModelVisitor v) { v.visited(this); } + @Override + public void applyDistributionRules(IDatabaseAdapter target) { + // NOP + } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java index ebe2d8a0c56..f6699829f48 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.DataAccessException; @@ -54,6 +55,7 @@ public class PoolConnectionProvider implements IConnectionProvider { // Should we reuse connections after an exception, or close them instead of returning them to the pool private boolean closeOnAnyError = false; + private Consumer newConnectionHandler; /** * Public constructor * @param cp @@ -73,6 +75,24 @@ public void setCloseOnAnyError() { this.closeOnAnyError = true; } + /** + * Setter for the newConnectionHandler + * @param handler + */ + public void setNewConnectionHandler(Consumer handler) { + this.newConnectionHandler = handler; + } + + /** + * Apply an configuration steps to a new connection + * @param c + */ + protected void configureConnection(Connection c) { + if (newConnectionHandler != null) { + newConnectionHandler.accept(c); + } + } + @Override public Connection getConnection() throws SQLException { // We use the same connection on a given thread each time it is requested @@ -148,6 +168,9 @@ public Connection getConnection() throws SQLException { } } + // Apply any configuration we want for a new connection + configureConnection(c); + long endTime = System.nanoTime(); double elapsed = (endTime-startTime) / 1e9; if (elapsed > 1.0) { diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index 7d6ec441467..b9494fa0de5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -47,7 +47,7 @@ public class PostgresAdapter extends CommonDatabaseAdapter { private static final Logger logger = Logger.getLogger(PostgresAdapter.class.getName()); // Different warning messages we track so that we only have to report them once - private enum MessageKey { + protected enum MessageKey { MULTITENANCY, CREATE_VAR, CREATE_PERM, @@ -65,6 +65,9 @@ private enum MessageKey { DROP_VARIABLE } + // Constant for better readability in method calls + protected static final boolean USE_SCHEMA_PREFIX = true; + // Just warn once for each unique message key. This cleans up build logs a lot private static final Set warned = ConcurrentHashMap.newKeySet(); @@ -112,9 +115,9 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns) { + List includeColumns, DistributionRules distributionRules) { // PostgreSql doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); } @Override @@ -300,10 +303,10 @@ public void runStatement(IDatabaseStatement stmt) { @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns) { + List indexColumns, DistributionRules distributionRules) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); // Postgresql doesn't support index name prefixed with the schema name. - String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, false); + String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); runStatement(ddl); } @@ -312,7 +315,7 @@ public void createIndex(String schemaName, String tableName, String indexName, S List indexColumns) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); // Postgresql doesn't support index name prefixed with the schema name. - String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, false); + String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); runStatement(ddl); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java index f748d95dbf9..5f5c74e6d38 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java @@ -389,7 +389,7 @@ public List getErasedResourceRecords(long erasedResourceGroup List result = new ArrayList<>(); final String SELECT_RECORDS = - "SELECT erased_resource_id, resource_type_id, logical_id, version_id " + + "SELECT resource_type_id, logical_id, version_id " + " FROM erased_resources " + " WHERE erased_resource_group_id = ?"; @@ -397,10 +397,10 @@ public List getErasedResourceRecords(long erasedResourceGroup stmt.setLong(1, erasedResourceGroupId); ResultSet rs = stmt.executeQuery(); while (rs.next()) { - long erasedResourceId = rs.getLong(1); - int resourceTypeId = rs.getInt(2); - String logicalId = rs.getString(3); - Integer versionId = rs.getInt(4); + long erasedResourceId = -1; // no longer used + int resourceTypeId = rs.getInt(1); + String logicalId = rs.getString(2); + Integer versionId = rs.getInt(3); if (rs.wasNull()) { versionId = null; } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index a57244779d8..d50634ee75e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -99,6 +99,7 @@ import com.ibm.fhir.database.utils.api.UndefinedNameException; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.citus.CitusTranslator; +import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; @@ -336,6 +337,10 @@ protected void configureConnectionPool() { JdbcConnectionProvider cp = new JdbcConnectionProvider(this.translator, adapter); this.connectionPool = new PoolConnectionProvider(cp, this.maxConnectionPoolSize); this.transactionProvider = new SimpleTransactionProvider(this.connectionPool); + +// if (this.dbType == DbType.CITUS) { +// connectionPool.setNewConnectionHandler(Main::configureCitusConnection); +// } } /** @@ -497,9 +502,11 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { logger.info("No database specific artifacts"); break; case POSTGRESQL: - case CITUS: gen.buildDatabaseSpecificArtifactsPostgres(pdm); break; + case CITUS: + gen.buildDatabaseSpecificArtifactsCitus(pdm); + break; default: throw new IllegalStateException("Unsupported db type: " + dbType); } @@ -709,11 +716,39 @@ protected boolean updateSchema(PhysicalDataModel pdm) { vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == 0; applyModel(pdm, adapter, collector, vhs); + applyDistributionRules(pdm); // The physical database objects should now match what was defined in the PhysicalDataModel return isNewDb; } + /** + * Apply any table distribution rules in one transaction + * @param pdm + */ + private void applyDistributionRules(PhysicalDataModel pdm) { + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + pdm.applyDistributionRules(adapter); + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + } + + /** + * Special case to initialize new connections created by the connection pool + * for Citus databases + * @param c + */ + private static void configureCitusConnection(Connection c) { + logger.info("Citus: Configuring new database connection"); + ConfigureConnectionDAO dao = new ConfigureConnectionDAO(); + dao.run(new CitusTranslator(), c); + } + /** * populates for the given tenantId the RESOURCE_TYPE table. * diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java index cb07d4f8c75..a7d5f7bcf79 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -66,6 +66,7 @@ public class FhirSchemaConstants { public static final int FRAGMENT_BYTES = 16; // R4 Logical Resources + public static final String LOGICAL_RESOURCE_SHARDS = "LOGICAL_RESOURCE_SHARDS"; public static final String LOGICAL_RESOURCES = "LOGICAL_RESOURCES"; public static final String REINDEX_TSTAMP = "REINDEX_TSTAMP"; public static final String REINDEX_TXID = "REINDEX_TXID"; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 533e58079dd..2d4e218c035 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -24,7 +24,6 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN; import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_GROUP_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK; @@ -40,6 +39,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SHARDS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES; @@ -387,6 +387,7 @@ public void buildSchema(PhysicalDataModel model) { addCodeSystems(model); addCommonTokenValues(model); addResourceTypes(model); + addLogicalResourceShards(model); addLogicalResources(model); // for system-level parameter search addReferencesSequence(model); addLogicalResourceCompartments(model); @@ -473,7 +474,7 @@ public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) { } public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { - // Add stored procedures/functions for postgresql. + // Add stored procedures/functions for postgresql and Citus // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. final String ROOT_DIR = "postgres/"; FunctionDef fd = model.addFunction(this.schemaName, @@ -525,6 +526,65 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); } + /** + * @implNote following the current pattern, which is why all this stuff is replicated + * @param model + */ + public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { + // Add stored procedures/functions for postgresql and Citus + // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. + final String ROOT_DIR = "postgres/"; + final String CITUS_ROOT_DIR = "citus/"; + FunctionDef fd = model.addFunction(this.schemaName, + ADD_CODE_SYSTEM, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), + procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_PARAMETER_NAME, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_RESOURCE_TYPE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // We currently only support functions with PostgreSQL, although this is really just a procedure + FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, + DELETE_RESOURCE_PARAMETERS, + FhirSchemaVersion.V0020.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges); + deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // Use the Citus-specific variant of add_any_resource (supports sharding) + fd = model.addFunction(this.schemaName, + ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ERASE_RESOURCE, + FhirSchemaVersion.V0013.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + } + /** * Add the system-wide logical_resources table. Note that LOGICAL_ID is * denormalized, stored in both LOGICAL_RESOURCES and _LOGICAL_RESOURCES. @@ -627,6 +687,48 @@ public void addLogicalResources(PhysicalDataModel pdm) { pdm.addObject(tbl); } + /** + * Adds a table to support sharding of the logical_id when running + * in a distributed RDBMS such as Citus. This table is sharded by + * LOGICAL_ID, which means we can use a primary key of + * {RESOURCE_TYPE_ID, LOGICAL_ID} which is required to ensure + * that we can lock the logical resource to avoid any concurrency + * issues. This is only used for distributed implementations. For + * the standard non-distributed solution, the locking is done + * using LOGICAL_RESOURCES. + * @param pdm + */ + public void addLogicalResourceShards(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCE_SHARDS; + final String mtId = this.multitenant ? MT_ID : null; + + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(mtId) + .addIntColumn(RESOURCE_TYPE_ID, false) + .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addPrimaryKey(tableName + "_PK", RESOURCE_TYPE_ID, LOGICAL_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_ID) // V0026 support for sharding + .addWiths(addWiths()) // add table tuning + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // NOP for now + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + /** * Create the COMMON_CANONICAL_VALUES table. Used from schema V0014 to normalize * meta.profile search parameters (similar to common_token_values). Only the url @@ -1324,16 +1426,14 @@ public void addErasedResources(PhysicalDataModel pdm) { // or resource_id values here, because those records may have // already been deleted by $erase. Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0023.vid()) + .setVersion(FhirSchemaVersion.V0026.vid()) .setTenantColumnName(mtId) - .addBigIntColumn(ERASED_RESOURCE_ID, false) - .setIdentityColumn(ERASED_RESOURCE_ID, Generated.ALWAYS) .addBigIntColumn(ERASED_RESOURCE_GROUP_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) .addIntColumn(VERSION_ID, true) - .addPrimaryKey(tableName + "_PK", ERASED_RESOURCE_ID) .addIndex(IDX + tableName + "_GID", ERASED_RESOURCE_GROUP_ID) + .setDistributionColumnName(ERASED_RESOURCE_GROUP_ID) .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) @@ -1342,6 +1442,8 @@ public void addErasedResources(PhysicalDataModel pdm) { .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Nothing yet + + // TODO migrate to simplified design (no PK, FK) return statements; }) .build(pdm); diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql new file mode 100644 index 00000000000..a31130f3ebb --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql @@ -0,0 +1,212 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2020, 2022 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to add a resource version and its associated parameters. These +-- parameters only ever point to the latest version of a resource, never to +-- previous versions, which are kept to support history queries. +-- implNote - Conventions: +-- p_... prefix used to represent input parameters +-- v_... prefix used to represent declared variables +-- t_... prefix used to represent temp variables +-- o_... prefix used to represent output parameters +-- Parameters: +-- p_logical_id: the logical id given to the resource by the FHIR server +-- p_payload: the BLOB (of JSON) which is the resource content +-- p_last_updated the last_updated time given by the FHIR server +-- p_is_deleted: the soft delete flag +-- p_version_id: the intended new version id of the resource (matching the JSON payload) +-- p_parameter_hash_b64 the Base64 encoded hash of parameter values +-- p_if_none_match the encoded If-None-Match value +-- o_logical_resource_id: output field returning the newly assigned logical_resource_id value +-- o_current_parameter_hash: Base64 current parameter hash if existing resource +-- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit +-- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) +-- Exceptions: +-- SQLSTATE 99001: on version conflict (concurrency) +-- SQLSTATE 99002: missing expected row (data integrity) +-- SQLSTATE 99004: delete a currently deleted resource (data integrity) +-- ---------------------------------------------------------------------------- + ( IN p_resource_type VARCHAR( 36), + IN p_logical_id VARCHAR(255), + IN p_payload BYTEA, + IN p_last_updated TIMESTAMP, + IN p_is_deleted CHAR( 1), + IN p_source_key VARCHAR( 64), + IN p_version INT, + IN p_parameter_hash_b64 VARCHAR( 44), + IN p_if_none_match INT, + IN p_resource_payload_key VARCHAR( 36), + OUT o_logical_resource_id BIGINT, + OUT o_current_parameter_hash VARCHAR( 44), + OUT o_interaction_status INT, + OUT o_if_none_match_version INT) + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + v_logical_resource_id BIGINT := NULL; + t_logical_resource_id BIGINT := NULL; + v_current_resource_id BIGINT := NULL; + v_resource_id BIGINT := NULL; + v_resource_type_id INT := NULL; + v_currently_deleted CHAR(1) := NULL; + v_new_resource INT := 0; + v_duplicate INT := 0; + v_current_version INT := 0; + v_change_type CHAR(1) := NULL; + + -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. + lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_shards WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + + BEGIN + -- default value unless we hit If-None-Match + o_interaction_status := 0; + + -- LOADED ON: {{DATE}} + v_schema_name := '{{SCHEMA_NAME}}'; + SELECT resource_type_id INTO v_resource_type_id + FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type; + + -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) + SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; + + -- Get a lock using the sharded logical_id in logical_resource_shards + OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO v_logical_resource_id; + CLOSE lock_cur; + + -- Create the resource if we don't have it already + IF v_logical_resource_id IS NULL + THEN + SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; + -- logical_resource_shards provides a sharded lookup mechanism for obtaining + -- the logical_resource_id when you know resource_type_id and logical_id + INSERT INTO {{SCHEMA_NAME}}.logical_resource_shards (resource_type_id, logical_id, logical_resource_id) + VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; + + -- The above insert could fail silently in a concurrent insert scenario, so we now just + -- need to try and obtain the lock (even though we may already have it if the above insert + -- succeeded. Whatever the case, we know there's a row there. + OPEN lock_cur (t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO t_logical_resource_id; + CLOSE lock_cur; + + -- check to see if it was us who actually created the record + IF v_logical_resource_id = t_logical_resource_id + THEN + -- create the corresponding entry in the global logical_resources table (which is distributed by + -- logical_resource_id). Because we created the logical_resource_shards record, we can + -- be certain the logical_resources record doesn't yet exist + INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64); + + -- similarly, create the corresponding record in the resource-type-specific logical_resources table + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + v_new_resource := 1; + ELSE + -- use the record created elsewhere + v_logical_resource_id := t_logical_resource_id; + END IF; + END IF; + + -- find the current parameter hash and deletion values from the logical_resources table + SELECT parameter_hash, is_deleted + INTO o_current_parameter_hash, v_currently_deleted; + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; + + -- Remember everying is locked at the logical resource level, so we are thread-safe here + IF v_new_resource = 0 THEN + -- as this is an existing resource, we need to know the current resource id. + -- This is only available at the resource-specific logical_resources level + EXECUTE + 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' + || ' WHERE logical_resource_id = $1 ' + INTO v_current_resource_id, v_current_version USING v_logical_resource_id; + + IF v_current_resource_id IS NULL OR v_current_version IS NULL + THEN + -- our concurrency protection means that this shouldn't happen + RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; + END IF; + + -- If-None-Match does not apply if the resource is currently deleted + IF v_currently_deleted = 'N' AND p_if_none_match = 0 + THEN + -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the + -- connection with a fatal error, so instead we use an out parameter to + -- indicate the match + o_interaction_status := 1; + o_if_none_match_version := v_current_version; + RETURN; + END IF; + + -- Concurrency check: + -- the version parameter we've been given (which is also embedded in the JSON payload) must be + -- one greater than the current version, otherwise we've hit a concurrent update race condition + IF p_version != v_current_version + 1 + THEN + RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; + END IF; + + -- Prevent creating a new deletion marker if the resource is currently deleted + IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' + THEN + RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; + END IF; + + IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash + THEN + -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) + -- TODO patch parameter sets instead of all delete/all insert. + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' + USING p_resource_type, v_logical_resource_id; + END IF; -- end if check parameter hash + END IF; -- end if existing resource + + EXECUTE + 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' + USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; + + + IF v_new_resource = 0 THEN + -- As this is an existing logical resource, we need to update the xx_logical_resource values to match + -- the values of the current resource. For new resources, these are added by the insert so we don't + -- need to update them here. + EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' + USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; + + -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level + EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' + USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; + END IF; + + -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event + -- related to resources changes (issue-1955) + IF p_is_deleted = 'Y' + THEN + v_change_type := 'D'; + ELSE + IF v_new_resource = 0 + THEN + v_change_type := 'U'; + ELSE + v_change_type := 'C'; + END IF; + END IF; + + INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) + VALUES (v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); + + -- Hand back the id of the logical resource we created earlier. In the new R4 schema + -- only the logical_resource_id is the target of any FK, so there's no need to return + -- the resource_id (which is now private to the _resources tables). + o_logical_resource_id := v_logical_resource_id; +END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java index 72520ea6d75..f4031df3d84 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -138,7 +138,7 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S // Check that we have the correct number of tables. This will need to be updated // whenever tables, views or sequences are added or removed - assertEquals(adapter.listSchemaObjects(schemaName).size(), 1918); + assertEquals(adapter.listSchemaObjects(schemaName).size(), 1919); c.commit(); } catch (Throwable t) { c.rollback(); From a7f23793392ac815de1bc7b19e4591dcca1c286c Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 14 Mar 2022 10:28:17 +0000 Subject: [PATCH 03/40] issue #3437 citus requires some custom ingestion logic Signed-off-by: Robin Arnold --- .../java/com/ibm/fhir/bucket/app/Main.java | 4 +- .../bucket/persistence/FhirBucketSchema.java | 62 ++++++- .../utils/api/SchemaApplyContext.java | 67 +++++++ .../citus/CreateDistributedTableDAO.java | 4 + .../utils/citus/CreateReferenceTableDAO.java | 4 + .../database/utils/derby/DerbyMaster.java | 6 +- .../utils/model/AlterSequenceStartWith.java | 9 +- .../utils/model/AlterTableAddColumn.java | 10 +- .../utils/model/AlterTableIdentityCache.java | 9 +- .../fhir/database/utils/model/BaseObject.java | 23 +-- .../database/utils/model/CreateIndex.java | 7 +- .../database/utils/model/DatabaseObject.java | 16 +- .../database/utils/model/FunctionDef.java | 9 +- .../database/utils/model/IDatabaseObject.java | 17 +- .../fhir/database/utils/model/NopObject.java | 9 +- .../database/utils/model/ObjectGroup.java | 24 ++- .../utils/model/PhysicalDataModel.java | 74 ++++++-- .../database/utils/model/ProcedureDef.java | 9 +- .../database/utils/model/RowArrayType.java | 9 +- .../fhir/database/utils/model/RowType.java | 9 +- .../fhir/database/utils/model/Sequence.java | 9 +- .../utils/model/SessionVariableDef.java | 7 +- .../ibm/fhir/database/utils/model/Table.java | 25 ++- .../fhir/database/utils/model/Tablespace.java | 16 +- .../ibm/fhir/database/utils/model/View.java | 9 +- .../database/utils/version/CreateControl.java | 6 +- .../utils/version/CreateVersionHistory.java | 6 +- .../version/CreateWholeSchemaVersion.java | 6 +- .../jdbc/FHIRResourceDAOFactory.java | 14 +- .../jdbc/citus/CitusResourceDAO.java | 90 ++++++++++ .../jdbc/citus/CitusResourceReferenceDAO.java | 165 ++++++++++++++++++ .../SetMultiShardModifyModeAction.java | 86 +++++++++ .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 6 +- .../jdbc/postgres/PostgresResourceDAO.java | 2 +- .../java/com/ibm/fhir/schema/app/Main.java | 50 +++++- .../ibm/fhir/schema/app/SchemaPrinter.java | 6 +- .../com/ibm/fhir/schema/app/menu/Menu.java | 1 + .../fhir/schema/control/AddForeignKey.java | 43 +++++ .../main/resources/citus/add_any_resource.sql | 19 +- .../schema/app/DataSchemaGeneratorTest.java | 10 +- .../app/JavaBatchSchemaGeneratorTest.java | 22 ++- .../schema/app/OAuthSchemaGeneratorTest.java | 10 +- .../schema/control/FhirSchemaServiceTest.java | 14 +- .../schema/control/ParallelBuildTest.java | 7 +- 44 files changed, 833 insertions(+), 177 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index d4b78719128..bedf631774e 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -56,6 +56,7 @@ import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; import com.ibm.fhir.database.utils.db2.Db2Adapter; @@ -786,7 +787,8 @@ private void buildSchema() { TaskService taskService = new TaskService(); ExecutorService pool = Executors.newFixedThreadPool(this.createSchemaThreads); ITaskCollector collector = taskService.makeTaskCollector(pool); - pdm.collect(collector, adapter, this.transactionProvider, vhs); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.collect(collector, adapter, context, this.transactionProvider, vhs); // FHIR in the hole! logger.info("Starting schema updates"); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java index e61026d77f6..b854dc906a2 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java @@ -1,16 +1,65 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.bucket.persistence; -import static com.ibm.fhir.bucket.persistence.SchemaConstants.*; - +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ALLOCATION_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_NAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATH; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATHS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATH_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.CREATED_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TEXT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TEXT_LEN; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ETAG; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FAILURE_COUNT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FILE_TYPE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FK; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HEARTBEAT_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HOSTNAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_CODE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_TEXT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_TEXT_LEN; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.IDX; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.JOB_ALLOCATION_SEQ; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LAST_MODIFIED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LINE_NUMBER; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCE_KEY; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOAD_COMPLETED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOAD_STARTED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_ID_BYTES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_RESOURCES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.NOT_NULL; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.NULLABLE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.OBJECT_NAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.OBJECT_SIZE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.PID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_ERRORS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_LOADS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_LOAD_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESPONSE_TIME_MS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ROWS_PROCESSED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.SCAN_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.STATUS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.UNQ; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.VERSION; import com.ibm.fhir.bucket.app.Main; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.Generated; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Sequence; @@ -257,8 +306,7 @@ protected void addResourceBundleErrors(PhysicalDataModel pdm) { * Apply the model to the database. Will generate the DDL and execute it * @param pdm */ - protected void applyModel(IDatabaseAdapter adapter, PhysicalDataModel pdm) { - pdm.apply(adapter); - } - + protected void applyModel(IDatabaseAdapter adapter, SchemaApplyContext context, PhysicalDataModel pdm) { + pdm.apply(adapter, context); + } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java new file mode 100644 index 00000000000..e7b0cf284e8 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java @@ -0,0 +1,67 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * Used to control how the schema gets applied + */ +public class SchemaApplyContext { + private final boolean includeForeignKeys; + + /** + * Protected constructor + * @param includeForeignKeys + */ + protected SchemaApplyContext(boolean includeForeignKeys) { + this.includeForeignKeys = includeForeignKeys; + } + public boolean isIncludeForeignKeys() { + return this.includeForeignKeys; + } + + /** + * Create a new {@link Builder} instance + * @return + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link SchemaApplyContext} + */ + public static class Builder { + private boolean includeForeignKeys; + + /** + * Setter for includeForeignKeys + * @param flag + * @return + */ + public Builder setIncludeForeignKeys(boolean flag) { + this.includeForeignKeys = flag; + return this; + } + /** + * Build an immutable instance of {@link SchemaApplyContext} using + * the current state of this + * @return + */ + public SchemaApplyContext build() { + return new SchemaApplyContext(this.includeForeignKeys); + } + } + + /** + * Get a default instance of the schema apply context + * @return + */ + public static SchemaApplyContext getDefault() { + return new SchemaApplyContext(true); + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java index 11aa036ccc0..d9e6589d582 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java @@ -9,6 +9,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; @@ -18,6 +19,8 @@ * DAO to add a new tenant key record */ public class CreateDistributedTableDAO implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(CreateDistributedTableDAO.class.getName()); + private final String schemaName; private final String tableName; private final String distributionKey; @@ -55,6 +58,7 @@ public void run(IDatabaseTranslator translator, Connection c) { } catch (SQLException x) { // Translate the exception into something a little more meaningful // for this database type and application + logger.severe("Call failed: " + sql.toString()); throw translator.translate(x); } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java index 654fe7fa38d..78023aba905 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java @@ -9,6 +9,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; @@ -18,6 +19,8 @@ * DAO to add a new tenant key record */ public class CreateReferenceTableDAO implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(CreateReferenceTableDAO.class.getName()); + private final String schemaName; private final String tableName; @@ -49,6 +52,7 @@ public void run(IDatabaseTranslator translator, Connection c) { } catch (SQLException x) { // Translate the exception into something a little more meaningful // for this database type and application + logger.severe("Call failed: " + sql.toString()); throw translator.translate(x); } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java index 5bc03f16667..c0921a2f53c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -23,6 +23,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.ConnectionProviderTarget; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcTarget; @@ -224,7 +225,8 @@ public void createSchema(IConnectionProvider pool, PhysicalDataModel pdm) { * @param pdm the data model we want to create */ public void createSchema(IConnectionProvider pool, IVersionHistoryService vhs, PhysicalDataModel pdm) { - runWithAdapter(pool, target -> pdm.applyWithHistory(target, vhs)); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + runWithAdapter(pool, target -> pdm.applyWithHistory(target, context, vhs)); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java index b31ee17a5c8..6ae473e02cd 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.Set; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Modify an existing sequence to start with a higher value @@ -39,13 +40,13 @@ public AlterSequenceStartWith(String schemaName, String sequenceName, int versio } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.alterSequenceRestartWith(getSchemaName(), getObjectName(), startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java index 639032a74f8..4acfdd0d68d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,11 +8,11 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Add new columns to an existing table. This alter will change the version history of the underlying table. @@ -62,7 +62,7 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { // To keep things simple, just add each column in its own statement for (ColumnBase c: columns) { target.alterTableAddColumn(getSchemaName(), getObjectName(), c); @@ -70,8 +70,8 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java index f30c40b65d2..7ff6328b185 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.Set; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Modify the CACHE property of an AS IDENTITY column (changes @@ -44,13 +45,13 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.alterTableColumnIdentityCache(getSchemaName(), getObjectName(), this.columnName, this.cache); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java index abb6cf8bc12..6706f8b2808 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java @@ -24,6 +24,7 @@ import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.LockException; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.thread.ThreadHandler; import com.ibm.fhir.task.api.ITaskCollector; @@ -186,24 +187,24 @@ public String toString() { } @Override - public ITaskGroup collect(final ITaskCollector tc, final IDatabaseAdapter target, final ITransactionProvider tp, final IVersionHistoryService vhs) { + public ITaskGroup collect(final ITaskCollector tc, final IDatabaseAdapter target, final SchemaApplyContext context, final ITransactionProvider tp, final IVersionHistoryService vhs) { // Make sure that anything we depend on gets processed first List children = null; if (!this.dependencies.isEmpty()) { children = new ArrayList<>(this.dependencies.size()); for (IDatabaseObject obj: dependencies) { - children.add(obj.collect(tc, target, tp, vhs)); + children.add(obj.collect(tc, target, context, tp, vhs)); } } // create a new task group representing this node, pointing to any dependencies // we collected above. We need to use the type and name for the task group, to // ensure we allow for the different namespaces (e.g. procedures vs tables). - return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, tp, vhs), children); + return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, context, tp, vhs), children); } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -212,7 +213,7 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi while (remainingAttempts-- > 0) { try (ITransaction tx = tp.getTransaction()) { try { - applyVersion(target, vhs); + applyVersion(target, context, vhs); remainingAttempts = 0; // exit the retry loop } catch (LockException x) { @@ -249,21 +250,15 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi } } - /** - * Apply the change, but only if it has a newer version than we already have - * recorded in the database - * @param target - * @param vhs the service used to manage the version history table - */ @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Only for Procedures do we skip the Version History Service check, and apply. if (vhs.applies(getSchemaName(), getObjectType().name(), getObjectName(), version) || getObjectType() == DatabaseObjectType.PROCEDURE) { logger.fine("Applying change [v" + version + "]: " + this.getTypeNameVersion()); // Apply this change to the target database - apply(vhs.getVersion(getSchemaName(), getObjectType().name(), getObjectName()), target); + apply(vhs.getVersion(getSchemaName(), getObjectType().name(), getObjectName()), target, context); // Check if the PROCEDURE is this exact version (Applies to FunctionDef and ProcedureDef) if (DatabaseObjectType.PROCEDURE.equals(getObjectType()) @@ -325,7 +320,7 @@ public void visit(Consumer c) { } @Override - public void applyDistributionRules(IDatabaseAdapter target) { + public void applyDistributionRules(IDatabaseAdapter target, int pass) { // NOP. Only applies to Table } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java index de7def1348d..c7cfca6b638 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java @@ -13,6 +13,7 @@ import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.CreateIndexStatement; /** @@ -92,7 +93,7 @@ public String getTypeNameVersion() { @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { long start = System.nanoTime(); indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionRules); @@ -104,8 +105,8 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java index 42012c047a5..b7331e941c0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -22,6 +22,7 @@ import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.LockException; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.thread.ThreadHandler; /** @@ -141,7 +142,7 @@ public String toString() { } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -150,7 +151,7 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi while (remainingAttempts-- > 0) { try (ITransaction tx = tp.getTransaction()) { try { - applyVersion(target, vhs); + applyVersion(target, context, vhs); remainingAttempts = 0; // exit the retry loop } catch (LockException x) { @@ -192,13 +193,13 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi * @param vhs the service used to manage the version history table */ @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // TODO find a better way to track database-level type stuff (not schema-specific) if (vhs.applies("__DATABASE__", getObjectType().name(), getObjectName(), version)) { logger.info("Applying change [v" + version + "]: "+ this.getTypeNameVersion()); // Apply this change to the target database - apply(vhs.getVersion("__DATABASE__", getObjectType().name(), getObjectName()), target); + apply(vhs.getVersion("__DATABASE__", getObjectType().name(), getObjectName()), target, context); // call back to the version history service to add the new version to the table // being used to track the change history @@ -215,4 +216,9 @@ public Map getTags() { public void visit(Consumer c) { c.accept(this); } + + @Override + public void applyDistributionRules(IDatabaseAdapter target, int pass) { + // NOP + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java index d57cc4514d4..e79b9dc4d09 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * The definition of a function, whose content is provided by a Supplier function @@ -34,18 +35,18 @@ public FunctionDef(String schemaName, String procedureName, int version, Supplie } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createOrReplaceFunction(getSchemaName(), getObjectName(), supplier); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } // Functions are applied with "Create or replace", so just do a regular apply - apply(target); + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java index 9be9e25ad8f..7cec93733ba 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java @@ -13,6 +13,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.api.ITaskGroup; @@ -31,15 +32,17 @@ public interface IDatabaseObject { * Apply the DDL for this object to the target database * @param priorVersion * @param target the database target + * @param context context to control the schema apply process */ - public void apply(IDatabaseAdapter target); + public void apply(IDatabaseAdapter target, SchemaApplyContext context); /** * Apply migration logic to bring the target database to the current level of this object * @param priorVersion * @param target the database target + * @param context to control the schema apply process */ - public void apply(Integer priorVersion, IDatabaseAdapter target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context); /** * Apply the DDL, but within its own transaction @@ -47,21 +50,23 @@ public interface IDatabaseObject { * @param cp of thread-specific transactions * @param vhs the service interface for adding this object to the version history table */ - public void applyTx(IDatabaseAdapter target, ITransactionProvider cp, IVersionHistoryService vhs); + public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider cp, IVersionHistoryService vhs); /** * Apply any distribution rules associated with the object (usually a table) * @param target the target database we apply the operation to + * @param pass multiple pass number */ - public void applyDistributionRules(IDatabaseAdapter target); + public void applyDistributionRules(IDatabaseAdapter target, int pass); /** * Apply the change, but only if it has a newer version than we already have * recorded in the database * @param target + * @param context * @param vhs the service used to manage the version history table */ - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs); + public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs); /** * DROP this object from the target database @@ -105,7 +110,7 @@ public interface IDatabaseObject { * @param tp * @param vhs */ - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs); + public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs); /** * Return the qualified name for this object (e.g. schema.name). diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java index 576afa4c0a3..a306a7e4af0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * A NOP (no operation) object which can be used to simplify dependencies @@ -29,12 +30,12 @@ public NopObject(String schemaName, String objectName) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @@ -44,7 +45,7 @@ public void drop(IDatabaseAdapter target) { } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // We're NOP so we do nothing on purpose } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java index f365393b011..25d1cdf92fc 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * A collection of {@link IDatabaseObject} which are applied in order within one transaction @@ -59,12 +60,12 @@ public ObjectGroup(String schemaName, String name, Collection g } @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Apply each member of our group to the target if it is a new version. // Version tracking is done at the individual level, not the group. for (IDatabaseObject obj: this.group) { - obj.applyVersion(target, vhs); + obj.applyVersion(target, context, vhs); } } @@ -86,18 +87,18 @@ public void grant(IDatabaseAdapter target, String groupName, String toUser) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { - obj.apply(target); + obj.apply(target, context); } } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { - obj.apply(priorVersion, target); + obj.apply(priorVersion, target, context); } } @@ -134,4 +135,11 @@ public void visitReverse(DataModelVisitor v) { group.get(i).visit(v); } } -} + + @Override + public void applyDistributionRules(IDatabaseAdapter target, int pass) { + for (IDatabaseObject obj: this.group) { + obj.applyDistributionRules(target, pass); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index efeacb3c098..250633a7930 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -20,8 +20,10 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.task.api.ITaskCollector; @@ -112,37 +114,46 @@ public void addObject(IDatabaseObject obj) { * time it takes to provision a schema. * @param tc collects and manages the object creation tasks and their dependencies * @param target the target database adapter + * @param context to control how the schema is built * @param tp * @param vhs */ - public void collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { for (IDatabaseObject obj: allObjects) { - obj.collect(tc, target, tp, vhs); + obj.collect(tc, target, context, tp, vhs); } } /** * Apply the entire model to the target in order * @param target + * @param context */ - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { logger.fine(String.format("Creating [%d/%d] %s", count++, total, obj.toString())); - obj.apply(target); + obj.apply(target, context); } } /** - * Make a pass over all the objects again and apply any distribution rules they + * Make a pass over all the objects and apply any distribution rules they * may have (e.g. for Citus) * @param target */ public void applyDistributionRules(IDatabaseAdapter target) { + + // make a first pass to apply reference rules + for (IDatabaseObject obj: allObjects) { + obj.applyDistributionRules(target, 0); + } + + // and another pass to apply sharding rules for (IDatabaseObject obj: allObjects) { - obj.applyDistributionRules(target); - } + obj.applyDistributionRules(target, 1); + } } /** @@ -151,12 +162,12 @@ public void applyDistributionRules(IDatabaseAdapter target) { * @param target * @param vhs */ - public void applyWithHistory(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyWithHistory(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { logger.fine(String.format("Creating [%d/%d] %s", count++, total, obj.toString())); - obj.applyVersion(target, vhs); + obj.applyVersion(target, context, vhs); } } @@ -164,13 +175,13 @@ public void applyWithHistory(IDatabaseAdapter target, IVersionHistoryService vhs * Apply all the procedures in the order in which they were added to the model * @param adapter */ - public void applyProcedures(IDatabaseAdapter adapter) { + public void applyProcedures(IDatabaseAdapter adapter, SchemaApplyContext context) { int total = procedures.size(); int count = 1; for (ProcedureDef obj: procedures) { logger.fine(String.format("Applying [%d/%d] %s", count++, total, obj.toString())); obj.drop(adapter); - obj.apply(adapter); + obj.apply(adapter, context); } } @@ -178,12 +189,12 @@ public void applyProcedures(IDatabaseAdapter adapter) { * Apply all the functions in the order in which they were added to the model * @param adapter */ - public void applyFunctions(IDatabaseAdapter adapter) { + public void applyFunctions(IDatabaseAdapter adapter, SchemaApplyContext context) { int total = functions.size(); int count = 1; for (FunctionDef obj: functions) { logger.fine(String.format("Applying [%d/%d] %s", count++, total, obj.toString())); - obj.apply(adapter); + obj.apply(adapter, context); } } @@ -216,6 +227,43 @@ public void drop(IDatabaseAdapter target, String tagGroup, String tag) { } } + + /** + * Split the drop in multiple (smaller) transactions, which can be helpful to + * reduce memory utilization in some scenarios + * @param target + * @param transactionProvider + * @param tagGroup + * @param tag + */ + public void dropSplitTransaction(IDatabaseAdapter target, ITransactionProvider transactionProvider, String tagGroup, String tag) { + + ArrayList copy = new ArrayList<>(); + copy.addAll(allObjects); + + int total = allObjects.size(); + int count = 1; + for (int i=total-1; i>=0; i--) { + IDatabaseObject obj = copy.get(i); + + // Each object (which often represents a group of tables) will be dropped + // in its own transaction...so clearly this needs to be an idempotent + // operation + try (ITransaction tx = transactionProvider.getTransaction()) { + try { + if (tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) { + logger.info(String.format("Dropping [%d/%d] %s", count++, total, obj.toString())); + obj.drop(target); + } else { + logger.info(String.format("Skipping [%d/%d] %s", count++, total, obj.toString())); + } + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + } + } /** * Drop all foreign key constraints on tables in this model. Typically done prior to dropping diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ProcedureDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ProcedureDef.java index 3cc9c1b22c4..00cd84fa3ca 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ProcedureDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ProcedureDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * The definition of a stored procedure, whose content is provided by a Supplier function @@ -34,7 +35,7 @@ public ProcedureDef(String schemaName, String procedureName, int version, Suppli } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { // Serialize the execution of the procedure, to try and avoid the // horrible deadlocks we keep getting synchronized (this) { @@ -44,14 +45,14 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } // we need to drop and then apply. drop(target); - apply(target); + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java index d90e57bd1ad..e4bbb8c8d3f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ package com.ibm.fhir.database.utils.model; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -35,16 +36,16 @@ public String toString() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createArrType(getSchemaName(), getObjectName(), rowTypeName, arraySize); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row array types is not supported"); } - apply(target); + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java index b2910828f7a..af6f4c9bff2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ import java.util.List; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Represents the ROW type used to pass parameters to the add_resource stored procedures @@ -25,16 +26,16 @@ public RowType(String schemaName, String typeName, int version, Collection 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row types is not supported"); } - apply(target); + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java index e0cd210a475..23a4dde9b28 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.Set; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Sequence related to the SQL sequence @@ -44,12 +45,12 @@ public Sequence(String schemaName, String sequenceName, int version, long startW } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createSequence(getSchemaName(), getObjectName(), this.startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading sequences is not supported"); } @@ -57,7 +58,7 @@ public void apply(Integer priorVersion, IDatabaseAdapter target) { // Only if VERSION1 then we want to apply, else fall through // Re-creating a sequence can have unintended consequences. if (this.version == 1 && (priorVersion == null || priorVersion == 0)) { - apply(target); + apply(target, context); } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java index e37ed61fb54..32a56865412 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.Set; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Adds a session variable to the database @@ -20,12 +21,12 @@ public SessionVariableDef(String schemaName, String variableName, int version) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index 81f6a605a05..d661bab3ccc 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -19,6 +19,7 @@ import com.ibm.fhir.database.utils.api.DistributionRules; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -130,7 +131,7 @@ public String getTenantColumnName() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { final String tsName = this.tablespace == null ? null : this.tablespace.getName(); target.createTable(getSchemaName(), getObjectName(), this.tenantColumnName, this.columns, this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints, @@ -141,9 +142,11 @@ public void apply(IDatabaseAdapter target) { idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionRules); } - // Foreign key constraints - for (ForeignKeyConstraint fkc: this.fkConstraints) { - fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + if (context.isIncludeForeignKeys()) { + // Foreign key constraints + for (ForeignKeyConstraint fkc: this.fkConstraints) { + fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + } } // Apply tenant access control if required @@ -160,9 +163,9 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion == null || priorVersion == 0) { - apply(target); + apply(target, context); } else if (this.getVersion() > priorVersion) { for (Migration step : migrations) { step.migrateFrom(priorVersion).stream().forEachOrdered(target::runStatement); @@ -942,9 +945,15 @@ public void visitReverse(DataModelVisitor v) { } @Override - public void applyDistributionRules(IDatabaseAdapter target) { + public void applyDistributionRules(IDatabaseAdapter target, int pass) { + // make sure all the reference tables are distributed first before + // we attempt to shard anything if (this.distributionRules != null) { - target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); + if (pass == 0 && this.distributionRules.isReferenceTable()) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); + } else if (pass == 1 && this.distributionRules.isDistributedTable()) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); + } } } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java index 65e4e4c62bf..32488a216a4 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java @@ -12,6 +12,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.api.ITaskGroup; @@ -34,7 +35,7 @@ public Tablespace(String tablespaceName, int version, int extentSizeKB) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { if (this.extentSizeKB > 0) { target.createTablespace(getName(), this.extentSizeKB); } @@ -45,11 +46,11 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0) { throw new UnsupportedOperationException("Modifying tablespaces is not supported"); } - apply(target); + apply(target, context); } @Override @@ -58,10 +59,10 @@ public void drop(IDatabaseAdapter target) { } @Override - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // no dependencies, so no need to recurse down List children = null; - return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, tp, vhs), children); + return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, context, tp, vhs), children); } @Override @@ -94,9 +95,4 @@ public void visit(DataModelVisitor v) { public void visitReverse(DataModelVisitor v) { v.visited(this); } - - @Override - public void applyDistributionRules(IDatabaseAdapter target) { - // NOP - } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java index 273bd9de2a0..b96c394e30a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -47,13 +48,13 @@ protected View(String schemaName, String objectName, int version, String selectC } @Override - public void apply(IDatabaseAdapter target) { + public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createOrReplaceView(getSchemaName(), getObjectName(), this.selectClause); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java index d5e22ec2ade..573c7a952a8 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -60,6 +61,7 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String adminSchem * @param target */ public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { + SchemaApplyContext context = SchemaApplyContext.getDefault(); PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, adminSchemaName, false); @@ -73,7 +75,7 @@ public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter // update tool could try to build the table. The solution is to make it // idempotent...if the table exists already, that's success try { - dataModel.apply(target); + dataModel.apply(target, context); } catch (Exception x) { if (t.exists(target)) { logger.info("Table '" + t.getQualifiedName() + "' already exists; skipping create"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java index 9c429eb19db..12124312ee7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ package com.ibm.fhir.database.utils.version; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -60,13 +61,14 @@ public static Table generateTable(PhysicalDataModel dataModel, String adminSchem */ public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); + SchemaApplyContext context = SchemaApplyContext.getDefault(); Table t = generateTable(dataModel, adminSchemaName, false); // apply this data model to the target if necessary - note - this bypasses the // version history table...because this is the table we're trying to create! if (!t.exists(target)) { - dataModel.apply(target); + dataModel.apply(target, context); } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java index acad4246cf3..c0e7d4c4130 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Privilege; import com.ibm.fhir.database.utils.model.Table; @@ -71,6 +72,7 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String schemaName */ public static void createTableIfNeeded(String schemaName, IDatabaseAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); + SchemaApplyContext context = SchemaApplyContext.getDefault(); Table t = buildTableDef(dataModel, schemaName, false); @@ -82,7 +84,7 @@ public static void createTableIfNeeded(String schemaName, IDatabaseAdapter targe // update tool could try to build the table. The solution is to make it // idempotent...if the table exists already, that's success try { - dataModel.apply(target); + dataModel.apply(target, context); } catch (Exception x) { if (t.exists(target)) { logger.info("Table '" + t.getQualifiedName() + "' already exists; skipping create"); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java index 85657fc9cc3..477f70c0234 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,8 @@ import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.citus.CitusResourceDAO; +import com.ibm.fhir.persistence.jdbc.citus.CitusResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.ReindexResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.FhirSequenceDAO; @@ -63,9 +65,11 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; case POSTGRESQL: - case CITUS: resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; + case CITUS: + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } @@ -140,9 +144,11 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, cache, rrd); break; case POSTGRESQL: - case CITUS: resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd); break; + case CITUS: + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, cache, rrd); + break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } @@ -173,7 +179,7 @@ public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); break; case CITUS: - rrd = new PostgresResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new CitusResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java new file mode 100644 index 00000000000..13ece6f3d02 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -0,0 +1,90 @@ +/* + * (C) Copyright IBM Corp. 2020, 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.citus; + +import java.sql.Connection; +import java.util.List; +import java.util.logging.Logger; + +import javax.transaction.TransactionSynchronizationRegistry; + +import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; +import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dto.Resource; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceDAO; + +/** + * Data access object for writing FHIR resources to Citus database using + * the stored procedure (or function, in this case) + */ +public class CitusResourceDAO extends PostgresResourceDAO { + private static final String CLASSNAME = CitusResourceDAO.class.getName(); + private static final Logger log = Logger.getLogger(CLASSNAME); + + // Read the current version of the resource (even if the resource has been deleted) + private static final String SQL_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.LOGICAL_ID = ? " + + " AND R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID"; // additional predicate using common Citus distribution column + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param flavor + * @param cache + * @param rrd + */ + public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd) { + super(connection, schemaName, flavor, cache, rrd); + } + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param flavor + * @param trxSynchRegistry + * @param cache + * @param rrd + * @param ptdi + */ + public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, + ParameterTransactionDataImpl ptdi) { + super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + } + + @Override + public Resource read(String logicalId, String resourceType) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "read"; + log.entering(CLASSNAME, METHODNAME); + + Resource resource = null; + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, logicalId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + return resource; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java new file mode 100644 index 00000000000..8743ad2532f --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java @@ -0,0 +1,165 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceReferenceDAO; + +/** + * Citus-specific extension of the {@link ResourceReferenceDAO} to work around + * some Citus distribution limitations + */ +public class CitusResourceReferenceDAO extends PostgresResourceReferenceDAO { + private static final Logger logger = Logger.getLogger(CitusResourceReferenceDAO.class.getName()); + + /** + * Public constructor + * + * @param t + * @param c + * @param schemaName + * @param cache + */ + public CitusResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { + super(t, c, schemaName, cache, parameterNameCache); + } + + @Override + public void doCodeSystemsUpsert(String paramList, Collection sortedSystemNames) { + // If we try using the PostgreSQL insert-as-select variant, Citus + // rejects the statement, so instead we simplify things by grabbing + // the id values from the sequence first, then simply submit as a + // batch. + List sequenceValues = new ArrayList<>(sortedSystemNames.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedSystemNames.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO code_systems (code_system_id, code_system_name) " + + " VALUES (?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (String csn: sortedSystemNames) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, csn); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } + + @Override + public void doCanonicalValuesUpsert(String paramList, Collection sortedURLS) { + // Because of how PostgreSQL MVCC implementation, the insert from negative outer + // join pattern doesn't work...you still hit conflicts. The PostgreSQL pattern + // for upsert is ON CONFLICT DO NOTHING, which is what we use here: + List sequenceValues = new ArrayList<>(sortedURLS.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedURLS.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO common_canonical_values (canonical_id, url) " + + " VALUES (?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (String csn: sortedURLS) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, csn); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void doCommonTokenValuesUpsert(String paramList, Collection sortedTokenValues) { + // In Citus, we can no longer use a generated id column, so we have to use + // values from the fhir-sequence and insert the values directly + List sequenceValues = new ArrayList<>(sortedTokenValues.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedTokenValues.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO common_token_values (common_token_value_id, token_value, code_system_id) " + + " VALUES (?, ?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (CommonTokenValue ctv: sortedTokenValues) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, ctv.getTokenValue()); + ps.setInt(3, ctv.getCodeSystemId()); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java new file mode 100644 index 00000000000..3a2b7b5e094 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java @@ -0,0 +1,86 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.connection; + +import java.sql.Connection; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.citus.CitusTranslator; +import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; +import com.ibm.fhir.database.utils.derby.DerbyAdapter; +import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.persistence.jdbc.derby.CreateCanonicalValuesTmp; +import com.ibm.fhir.persistence.jdbc.derby.CreateCodeSystemsTmp; +import com.ibm.fhir.persistence.jdbc.derby.CreateCommonTokenValuesTmp; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; + +/** + * For Citus connections, SET LOCAL citus.multi_shard_modify_mode TO 'sequential' + */ +public class SetMultiShardModifyModeAction extends ChainedAction { + private static final Logger log = Logger.getLogger(CreateTempTablesAction.class.getName()); + + /** + * Public constructor. No next action, so this will be the last action applied + */ + public SetMultiShardModifyModeAction() { + super(); + } + + /** + * Public constructor + * @param next the next action in the chain + */ + public SetMultiShardModifyModeAction(Action next) { + super(next); + } + + @Override + public void performOn(FHIRDbFlavor flavor, Connection connection) throws FHIRPersistenceDBConnectException { + + if (flavor.getType() == DbType.CITUS) { + // This is only used for Citus databases + log.fine("SET LOCAL citus.multi_shard_modify_mode TO 'sequential'"); + ConfigureConnectionDAO dao = new ConfigureConnectionDAO(); + dao.run(new CitusTranslator(), connection); + } + + // perform next action in the chain + super.performOn(flavor, connection); + } + + /** + * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCommonTokenValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCommonTokenValuesTmp(); + adapter.runStatement(cmd); + } + + /** + * Create the declared global temporary table CODE_SYSTEMS_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCodeSystemsTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCodeSystemsTmp(); + adapter.runStatement(cmd); + } + + /** + * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCanonicalValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCanonicalValuesTmp(); + adapter.runStatement(cmd); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index fcea8afb0a8..204a059463b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -119,6 +119,7 @@ import com.ibm.fhir.persistence.jdbc.connection.SchemaNameFromProps; import com.ibm.fhir.persistence.jdbc.connection.SchemaNameImpl; import com.ibm.fhir.persistence.jdbc.connection.SchemaNameSupplier; +import com.ibm.fhir.persistence.jdbc.connection.SetMultiShardModifyModeAction; import com.ibm.fhir.persistence.jdbc.connection.SetTenantAction; import com.ibm.fhir.persistence.jdbc.dao.EraseResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.ReindexResourceDAO; @@ -156,7 +157,6 @@ import com.ibm.fhir.persistence.payload.FHIRPayloadPersistence; import com.ibm.fhir.persistence.payload.PayloadPersistenceResponse; import com.ibm.fhir.persistence.payload.PayloadPersistenceResult; -import com.ibm.fhir.persistence.util.FHIRPersistenceUtil; import com.ibm.fhir.persistence.util.InputOutputByteStream; import com.ibm.fhir.persistence.util.LogicalIdentityProvider; import com.ibm.fhir.schema.control.FhirSchemaConstants; @@ -353,7 +353,9 @@ protected Action buildActionChain() { // reads/searches. result = new CreateTempTablesAction(result); - // For PostgreSQL + // For Citus SET LOCAL citus.multi_shard_modify_mode TO 'sequential' + result = new SetMultiShardModifyModeAction(result); + return result; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 8db642b5f25..768e43ac8e0 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -48,7 +48,7 @@ * the stored procedure (or function, in this case) */ public class PostgresResourceDAO extends ResourceDAOImpl { - private static final String CLASSNAME = PostgresResourceDAO.class.getSimpleName(); + private static final String CLASSNAME = PostgresResourceDAO.class.getName(); private static final Logger logger = Logger.getLogger(CLASSNAME); private static final String SQL_READ_RESOURCE_TYPE = "{CALL %s.add_resource_type(?, ?)}"; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index d50634ee75e..00cc877d6cb 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -22,6 +22,7 @@ import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_BATCH; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_FHIR; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_OAUTH; +import static com.ibm.fhir.schema.app.menu.Menu.DROP_SPLIT_TRANSACTION; import static com.ibm.fhir.schema.app.menu.Menu.DROP_TENANT; import static com.ibm.fhir.schema.app.menu.Menu.FORCE; import static com.ibm.fhir.schema.app.menu.Menu.FORCE_UNUSED_TABLE_REMOVAL; @@ -94,6 +95,7 @@ import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.api.TableSpaceRemovalException; import com.ibm.fhir.database.utils.api.TenantStatus; import com.ibm.fhir.database.utils.api.UndefinedNameException; @@ -136,6 +138,7 @@ import com.ibm.fhir.model.util.ModelSupport; import com.ibm.fhir.schema.app.menu.Menu; import com.ibm.fhir.schema.app.util.TenantKeyFileUtil; +import com.ibm.fhir.schema.control.AddForeignKey; import com.ibm.fhir.schema.control.BackfillResourceChangeLog; import com.ibm.fhir.schema.control.BackfillResourceChangeLogDb2; import com.ibm.fhir.schema.control.DisableForeignKey; @@ -283,6 +286,9 @@ public class Main { // Include detail output in the report (default is no) private boolean showDbSizeDetail = false; + // Split drops into multiple transactions? + private boolean dropSplitTransaction = false; + // Tenant Key Output or Input File private String tenantKeyFileName; private TenantKeyFileUtil tenantKeyFileUtil = new TenantKeyFileUtil(); @@ -386,7 +392,9 @@ protected void buildJavaBatchSchemaModel(PhysicalDataModel pdm) { */ protected void applyModel(PhysicalDataModel pdm, IDatabaseAdapter adapter, ITaskCollector collector, VersionHistoryService vhs) { logger.info("Collecting model update tasks"); - pdm.collect(collector, adapter, this.transactionProvider, vhs); + // If using a distributed RDBMS (Citus) then skip the initial FK creation + SchemaApplyContext context = SchemaApplyContext.builder().setIncludeForeignKeys(!isDistributed()).build(); + pdm.collect(collector, adapter, context, this.transactionProvider, vhs); // FHIR in the hole! logger.info("Starting model updates"); @@ -716,14 +724,17 @@ protected boolean updateSchema(PhysicalDataModel pdm) { vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == 0; applyModel(pdm, adapter, collector, vhs); - applyDistributionRules(pdm); + if (isDistributed()) { + applyDistributionRules(pdm); + } // The physical database objects should now match what was defined in the PhysicalDataModel return isNewDb; } /** - * Apply any table distribution rules in one transaction + * Apply any table distribution rules in one transaction and then add all the + * FK constraints that are needed * @param pdm */ private void applyDistributionRules(PhysicalDataModel pdm) { @@ -736,6 +747,21 @@ private void applyDistributionRules(PhysicalDataModel pdm) { throw x; } } + + // Now that all the tables have been distributed, it should be safe + // to apply the FK constraints + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + final String tenantColumnName = isMultitenant() ? "mt_id" : null; + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName); + pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + } /** @@ -830,7 +856,15 @@ protected void dropSchema() { if (dropFhirSchema) { // Just drop the objects associated with the FHIRDATA schema group final String schemaName = schema.getSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + if (this.dropSplitTransaction) { + // important that we use an adapter connected with the connection pool + // (which is connected to the transaction provider) + IDatabaseAdapter txAdapter = getDbAdapter(dbType, connectionPool); + pdm.dropSplitTransaction(txAdapter, this.transactionProvider, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } else { + // old fashioned drop where we do everything in one (big) transaction + pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } CreateWholeSchemaVersion.dropTable(schemaName, adapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); @@ -930,8 +964,9 @@ protected void updateProcedures() { try (Connection c = connectionPool.getConnection();) { try { IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); - pdm.applyProcedures(adapter); - pdm.applyFunctions(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.applyProcedures(adapter, context); + pdm.applyFunctions(adapter, context); // Because we're replacing the procedures, we should also check if // we need to apply the associated privileges @@ -2021,6 +2056,9 @@ protected void parseArgs(String[] args) { i++; } break; + case DROP_SPLIT_TRANSACTION: + this.dropSplitTransaction = true; + break; case POOL_SIZE: if (++i < args.length) { this.maxConnectionPoolSize = Integer.parseInt(args[i]); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java index 01b45b8579e..a0e28f8d470 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -46,6 +46,7 @@ import java.util.Properties; import java.util.concurrent.Executor; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -174,7 +175,8 @@ public void process() { JavaBatchSchemaGenerator javaBatchSchemaGenerator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); javaBatchSchemaGenerator.buildJavaBatchSchema(model); - model.apply(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.apply(adapter, context); } public void processApplyGrants() { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java index bc1dbab769d..228b7fc1f35 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java @@ -32,6 +32,7 @@ public class Menu { public static final String DROP_DETACHED = "--drop-detached"; public static final String FREEZE_TENANT = "--freeze-tenant"; public static final String DROP_TENANT = "--drop-tenant"; + public static final String DROP_SPLIT_TRANSACTION = "--drop-split-transaction"; public static final String REFRESH_TENANTS = "--refresh-tenants"; public static final String ALLOCATE_TENANT = "--allocate-tenant"; public static final String CONFIRM_DROP = "--confirm-drop"; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java new file mode 100644 index 00000000000..1b8d4822e5f --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java @@ -0,0 +1,43 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package com.ibm.fhir.schema.control; + +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.model.DataModelVisitorBase; +import com.ibm.fhir.database.utils.model.ForeignKeyConstraint; +import com.ibm.fhir.database.utils.model.Table; + +/** + * Visitor adapter used to add all the foreign key constraints + * associated with tables in the schema. + * + * Expects any transaction handling to be performed outside this class. + */ +public class AddForeignKey extends DataModelVisitorBase { + private static final Logger logger = Logger.getLogger(DropForeignKey.class.getName()); + + // The database adapter used to issue changes to the database + private final IDatabaseAdapter adapter; + private final String tenantColumnName; + + /** + * Public constructor + * @param adapter + */ + public AddForeignKey(IDatabaseAdapter adapter, String tenantColumnName) { + this.adapter = adapter; + this.tenantColumnName = tenantColumnName; + } + + @Override + public void visited(Table fromChildTable, ForeignKeyConstraint fk) { + // Enable (add) the FK constraint + logger.info(String.format("Adding foreign key: %s.%s[%s]", fromChildTable.getSchemaName(), fromChildTable.getObjectName(), fk.getConstraintName())); + fk.apply(fromChildTable.getSchemaName(), fromChildTable.getObjectName(), this.tenantColumnName, adapter); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql index a31130f3ebb..b963ad0f858 100644 --- a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql +++ b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql @@ -99,9 +99,10 @@ -- check to see if it was us who actually created the record IF v_logical_resource_id = t_logical_resource_id THEN - -- create the corresponding entry in the global logical_resources table (which is distributed by - -- logical_resource_id). Because we created the logical_resource_shards record, we can - -- be certain the logical_resources record doesn't yet exist + -- the record was created by this call, so now create the corresponding entry in the + -- global logical_resources table (which is distributed by logical_resource_id). + -- Because we created the logical_resource_shards record, we can be certain the + -- logical_resources record doesn't yet exist INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64); @@ -112,15 +113,15 @@ ELSE -- use the record created elsewhere v_logical_resource_id := t_logical_resource_id; + + -- find the current parameter hash and deletion values from the logical_resources table + SELECT parameter_hash, is_deleted + INTO o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; END IF; END IF; - -- find the current parameter hash and deletion values from the logical_resources table - SELECT parameter_hash, is_deleted - INTO o_current_parameter_hash, v_currently_deleted; - FROM {{SCHEMA_NAME}}.logical_resources - WHERE logical_resource_id = v_logical_resource_id; - -- Remember everying is locked at the logical resource level, so we are thread-safe here IF v_new_resource = 0 THEN -- as this is an existing resource, we need to know the current resource id. diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java index ffb936608f6..5e689c4d17a 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -37,9 +38,10 @@ public void testFHIRSchemaGeneratorCheckTags() { PhysicalDataModel pdm = new PhysicalDataModel(); FhirSchemaGenerator generator = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, true); generator.buildSchema(pdm); - pdm.apply(adapter); - pdm.applyFunctions(adapter); - pdm.applyProcedures(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.apply(adapter, context); + pdm.applyFunctions(adapter, context); + pdm.applyProcedures(adapter, context); pdm.visit(new ConfirmTagsVisitor()); } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java index 062ab9ba72e..ac78178c307 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020,2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -45,6 +45,7 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.AlterSequenceStartWith; @@ -89,9 +90,10 @@ public void testJavaBatchSchemaGeneratorDb2() { PhysicalDataModel pdm = new PhysicalDataModel(); JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); - pdm.apply(adapter); - pdm.applyFunctions(adapter); - pdm.applyProcedures(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.apply(adapter, context); + pdm.applyFunctions(adapter, context); + pdm.applyProcedures(adapter, context); if (DEBUG) { for (Entry command : commands.entrySet()) { @@ -120,8 +122,9 @@ public void testJavaBatchSchemaGeneratorPostgres() { PhysicalDataModel pdm = new PhysicalDataModel(); JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); - pdm.apply(adapter); - pdm.applyFunctions(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.apply(adapter, context); + pdm.applyFunctions(adapter, context); if (DEBUG) { for (Entry command : commands.entrySet()) { @@ -151,9 +154,10 @@ public void testJavaBatchSchemaGeneratorCheckTags() { PhysicalDataModel pdm = new PhysicalDataModel(); JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); - pdm.apply(adapter); - pdm.applyFunctions(adapter); - pdm.applyProcedures(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.apply(adapter, context); + pdm.applyFunctions(adapter, context); + pdm.applyProcedures(adapter, context); pdm.visit(new ConfirmTagsVisitor()); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java index 32e7cd491b9..e087b636098 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -37,9 +38,10 @@ public void testOAuthSchemaGeneratorCheckTags() { PhysicalDataModel pdm = new PhysicalDataModel(); OAuthSchemaGenerator generator = new OAuthSchemaGenerator(Main.OAUTH_SCHEMANAME); generator.buildOAuthSchema(pdm); - pdm.apply(adapter); - pdm.applyFunctions(adapter); - pdm.applyProcedures(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.apply(adapter, context); + pdm.applyFunctions(adapter, context); + pdm.applyProcedures(adapter, context); pdm.visit(new ConfirmTagsVisitor()); } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java index 301652a01a7..8d36f0ce5c0 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -22,8 +23,6 @@ import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; import com.ibm.fhir.database.utils.version.CreateVersionHistory; -import com.ibm.fhir.schema.control.FhirSchemaConstants; -import com.ibm.fhir.schema.control.FhirSchemaGenerator; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.core.service.TaskService; @@ -51,7 +50,8 @@ public void testDb2TableCreation() { // Pretend that our target is a DB2 database Db2Adapter adapter = new Db2Adapter(tgt); - model.apply(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.apply(adapter, context); } @Test @@ -71,7 +71,8 @@ public void testParallelTableCreation() { ITaskCollector collector = taskService.makeTaskCollector(pool); PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE)); Db2Adapter adapter = new Db2Adapter(tgt); - model.collect(collector, adapter, new TransactionProviderTest(), vhs); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.collect(collector, adapter, context, new TransactionProviderTest(), vhs); // FHIR in the hole! collector.startAndWait(); @@ -94,7 +95,8 @@ public void testDerbyTableCreation() { // Pretend that our target is a Derby database DerbyAdapter adapter = new DerbyAdapter(tgt); - model.apply(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.apply(adapter, context); } @Test diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java index 0fa95bb0c29..1734dcb1172 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,10 +13,10 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; -import com.ibm.fhir.schema.control.FhirSchemaGenerator; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.core.service.TaskService; @@ -45,7 +45,8 @@ public void testParallelTableCreation() { ITaskCollector collector = taskService.makeTaskCollector(pool); PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE)); Db2Adapter adapter = new Db2Adapter(tgt); - model.collect(collector, adapter, new TransactionProviderTest(), vhs); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.collect(collector, adapter, context, new TransactionProviderTest(), vhs); // FHIR in the hole! collector.startAndWait(); From 4398119a37170aa0f6c110f6cd257361b1f0d599 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Fri, 18 Mar 2022 09:58:29 +0000 Subject: [PATCH 04/40] issue #3437 distribute add_any_resource function in Citus Signed-off-by: Robin Arnold --- .../database/utils/api/IDatabaseAdapter.java | 8 + .../database/utils/citus/CitusAdapter.java | 46 + .../utils/common/CommonDatabaseAdapter.java | 5 + .../database/utils/model/FunctionDef.java | 13 +- .../utils/model/PhysicalDataModel.java | 19 +- .../FhirDistributedSchemaGenerator.java | 1544 +++++++++++++++++ .../schema/control/FhirSchemaGenerator.java | 32 +- .../main/resources/citus/add_any_resource.sql | 115 +- .../resources/citus/add_logical_resource.sql | 190 ++ 9 files changed, 1855 insertions(+), 117 deletions(-) create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java create mode 100644 fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 3cc74d8ea96..f741311e3df 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -499,6 +499,14 @@ public default boolean useSessionVariable() { */ public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier); + /** + * For Citus, functions can be distributed by one of their parameters (typically the first) + * @param schemaName + * @param functionName + * @param distributeByParamNumber + */ + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber); + /** * drops a given function * @param schemaName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java index a2e8119cb1f..37c2a8644c1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -6,6 +6,10 @@ package com.ibm.fhir.database.utils.citus; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; import java.util.Set; import java.util.logging.Logger; @@ -172,4 +176,46 @@ public void applyDistributionRules(String schemaName, String tableName, Distribu runStatement(dao); } } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + if (distributeByParamNumber < 1) { + throw new IllegalArgumentException("invalid distributeByParamNumber value: " + distributeByParamNumber); + } + // Need to get the signature text first in order to build the create_distribution_function + // statement + final String objectName = DataDefinitionUtil.getQualifiedName(schemaName, functionName); + final String SELECT = + "SELECT p.oid::regproc || '(' || pg_get_function_identity_arguments(p.oid) || ')' " + + " FROM pg_catalog.pg_proc p " + + " WHERE p.oid::regproc::text = LOWER(?)"; + + if (connectionProvider != null) { + try (Connection c = connectionProvider.getConnection()) { + String functionSig = null; + try (PreparedStatement ps = c.prepareStatement(SELECT)) { + ps.setString(1, objectName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + functionSig = rs.getString(1); + } + } + + if (functionSig != null) { + final String DISTRIBUTE = "SELECT create_distributed_function(?, ?)"; + try (PreparedStatement ps = c.prepareStatement(DISTRIBUTE)) { + ps.setString(1, functionSig); + ps.setString(2, "$" + distributeByParamNumber); + ps.executeQuery(DISTRIBUTE); + } + } else { + logger.warning("No matching function found for '" + objectName + "'"); + } + } catch (SQLException x) { + throw getTranslator().translate(x); + } + } else { + throw new IllegalStateException("distributeFunction requires a connectionProvider"); + } + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java index da25892a6ac..e1f19fbcf51 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java @@ -753,4 +753,9 @@ public void reorgTable(String schemaName, String tableName) { public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules) { // NOP. Only used for distributed databases like Citus } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + // NOP. Only used for distributed databases like Citus + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java index e79b9dc4d09..ae27ada8478 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java @@ -22,21 +22,30 @@ public class FunctionDef extends BaseObject { // supplier provides the procedure body when requested private Supplier supplier; + // When >0, indicates that this function should be distributed + private final int distributeByParamNum; + /** - * Public constructor + * Public constructor. Supports distribution of the function by the given parameter number + * * @param schemaName * @param procedureName * @param version * @param supplier + * @param distributeByParamNum */ - public FunctionDef(String schemaName, String procedureName, int version, Supplier supplier) { + public FunctionDef(String schemaName, String procedureName, int version, Supplier supplier, int distributeByParamNum) { super(schemaName, procedureName, DatabaseObjectType.PROCEDURE, version); this.supplier = supplier; + this.distributeByParamNum = distributeByParamNum; } @Override public void apply(IDatabaseAdapter target, SchemaApplyContext context) { target.createOrReplaceFunction(getSchemaName(), getObjectName(), supplier); + if (distributeByParamNum > 0) { + target.distributeFunction(getSchemaName(), getObjectName(), distributeByParamNum); + } } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index 250633a7930..e9e3fd3b766 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -468,7 +468,24 @@ public ProcedureDef addProcedure(String schemaName, String objectName, int versi */ public FunctionDef addFunction(String schemaName, String objectName, int version, Supplier templateProvider, Collection dependencies, Collection privileges) { - FunctionDef func = new FunctionDef(schemaName, objectName, version, templateProvider); + return addFunction(schemaName, objectName, version, templateProvider, dependencies, privileges, 0); + } + + /** + * adds the function to the model. + * + * @param schemaName + * @param objectName + * @param version + * @param templateProvider + * @param dependencies + * @param privileges + * @param distributeByParamNum + * @return + */ + public FunctionDef addFunction(String schemaName, String objectName, int version, Supplier templateProvider, + Collection dependencies, Collection privileges, int distributeByParamNum) { + FunctionDef func = new FunctionDef(schemaName, objectName, version, templateProvider, distributeByParamNum); privileges.forEach(p -> p.addToObject(func)); if (dependencies != null) { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java new file mode 100644 index 00000000000..58cebcd98bf --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java @@ -0,0 +1,1544 @@ +/* + * (C) Copyright IBM Corp. 2019, 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_URL_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TSTAMP; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TYPE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEMS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_NAME; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_CANONICAL_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPARTMENT_LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPARTMENT_NAME_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_END; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_START; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_GROUP_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.IDX; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.IS_DELETED; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LAST_UPDATED; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_COMPARTMENTS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SHARDS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.MT_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAMES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VERSION_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REINDEX_TSTAMP; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REINDEX_TXID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_CHANGE_LOG; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TOKEN_REFS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE_LCASE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANTS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_HASH; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_KEYS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_KEY_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_NAME; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_SALT; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_SEQUENCE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_STATUS; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.TOKEN_VALUE; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.URL; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_BYTES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_ID; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.common.AddColumn; +import com.ibm.fhir.database.utils.common.CreateIndexStatement; +import com.ibm.fhir.database.utils.common.DropColumn; +import com.ibm.fhir.database.utils.common.DropIndex; +import com.ibm.fhir.database.utils.common.DropTable; +import com.ibm.fhir.database.utils.model.AlterSequenceStartWith; +import com.ibm.fhir.database.utils.model.BaseObject; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.ColumnDefBuilder; +import com.ibm.fhir.database.utils.model.FunctionDef; +import com.ibm.fhir.database.utils.model.Generated; +import com.ibm.fhir.database.utils.model.GroupPrivilege; +import com.ibm.fhir.database.utils.model.IDatabaseObject; +import com.ibm.fhir.database.utils.model.NopObject; +import com.ibm.fhir.database.utils.model.ObjectGroup; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PhysicalDataModel; +import com.ibm.fhir.database.utils.model.Privilege; +import com.ibm.fhir.database.utils.model.ProcedureDef; +import com.ibm.fhir.database.utils.model.Sequence; +import com.ibm.fhir.database.utils.model.SessionVariableDef; +import com.ibm.fhir.database.utils.model.Table; +import com.ibm.fhir.database.utils.model.Tablespace; +import com.ibm.fhir.database.utils.model.With; +import com.ibm.fhir.database.utils.postgres.PostgresFillfactorSettingDAO; +import com.ibm.fhir.database.utils.postgres.PostgresVacuumSettingDAO; +import com.ibm.fhir.model.util.ModelSupport; + +/** + * Creates a distributed variant of the FHIR data schema. This schema distributes + * tables associated with certain resource types using a shard key (which in + * reality is a patient identifier which can be used to scope interactions) + * + * In general, the schema is largely similar to the + */ +public class FhirDistributedSchemaGenerator { + private static final Logger logger = Logger.getLogger(FhirDistributedSchemaGenerator.class.getName()); + + // The schema holding all the data-bearing tables + private final String schemaName; + + // The schema used for administration objects like the tenants table, variable etc + private final String adminSchemaName; + + // Build the multitenant variant of the schema + private final boolean multitenant; + + // No abstract types + private static final Set ALL_RESOURCE_TYPES = ModelSupport.getResourceTypes(false).stream() + .map(t -> ModelSupport.getTypeName(t).toUpperCase()) + .collect(Collectors.toSet()); + + private static final String ADD_CODE_SYSTEM = "ADD_CODE_SYSTEM"; + private static final String ADD_PARAMETER_NAME = "ADD_PARAMETER_NAME"; + private static final String ADD_RESOURCE_TYPE = "ADD_RESOURCE_TYPE"; + private static final String ADD_ANY_RESOURCE = "ADD_ANY_RESOURCE"; + + // Special procedure for Citus database support + private static final String ADD_LOGICAL_RESOURCE = "ADD_LOGICAL_RESOURCE"; + private static final String DELETE_RESOURCE_PARAMETERS = "DELETE_RESOURCE_PARAMETERS"; + private static final String ERASE_RESOURCE = "ERASE_RESOURCE"; + + // The tags we use to separate the schemas + public static final String SCHEMA_GROUP_TAG = "SCHEMA_GROUP"; + public static final String FHIRDATA_GROUP = "FHIRDATA"; + public static final String ADMIN_GROUP = "FHIR_ADMIN"; + + // ADMIN SCHEMA CONTENT + + // Sequence used by the admin tenant tables + private Sequence tenantSequence; + + // The session variable used for row access control. All tables depend on this + private SessionVariableDef sessionVariable; + + private Table tenantsTable; + private Table tenantKeysTable; + + private static final String SET_TENANT = "SET_TENANT"; + + // The set of dependencies common to all of our admin stored procedures + private Set adminProcedureDependencies = new HashSet<>(); + + // A NOP marker used to ensure procedures are only applied after all the create + // table statements are applied - to avoid DB2 catalog deadlocks + private IDatabaseObject allAdminTablesComplete; + + // Marker used to indicate that the admin schema is all done + private IDatabaseObject adminSchemaComplete; + + // The resource types to generate schema for + private final Set resourceTypes; + + // The common sequence used for allocated resource ids + private Sequence fhirSequence; + + // The sequence used for the reference tables (parameter_names, code_systems etc) + private Sequence fhirRefSequence; + + // The set of dependencies common to all of our resource procedures + private Set procedureDependencies = new HashSet<>(); + + private Table codeSystemsTable; + private Table parameterNamesTable; + private Table resourceTypesTable; + private Table commonTokenValuesTable; + + // A NOP marker used to ensure procedures are only applied after all the create + // table statements are applied - to avoid DB2 catalog deadlocks + private IDatabaseObject allTablesComplete; + + // Privileges needed by the stored procedures + private List procedurePrivileges = new ArrayList<>(); + + // Privileges needed for access to the FHIR resource data tables + private List resourceTablePrivileges = new ArrayList<>(); + + // Privileges needed for reading the sv_tenant_id variable + private List variablePrivileges = new ArrayList<>(); + + // Privileges needed for using the fhir sequence + private List sequencePrivileges = new ArrayList<>(); + + // The default tablespace used for everything not specific to a tenant + private Tablespace fhirTablespace; + + /** + * Generate the IBM FHIR Server Schema for all resourceTypes + * + * @param adminSchemaName + * @param schemaName + */ + public FhirDistributedSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant) { + this(adminSchemaName, schemaName, multitenant, ALL_RESOURCE_TYPES); + } + + /** + * Generate the IBM FHIR Server Schema with just the given resourceTypes + * + * @param adminSchemaName + * @param schemaName + */ + public FhirDistributedSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant, Set resourceTypes) { + this.adminSchemaName = adminSchemaName; + this.schemaName = schemaName; + this.multitenant = multitenant; + + // The FHIR user (e.g. "FHIRSERVER") will need these privileges to be granted to it. Note that + // we use the group identified by FHIR_USER_GRANT_GROUP here - these privileges can be applied + // to any DB2 user using an admin user, or another user with sufficient GRANT TO privileges. + + + // The FHIRSERVER user gets EXECUTE privilege specifically on the SET_TENANT procedure, which is + // owned by the admin user, not the FHIRSERVER user. + procedurePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.EXECUTE)); + + // FHIRSERVER needs INSERT, SELECT, UPDATE and DELETE on all the resource data tables + resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.INSERT)); + resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.SELECT)); + resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.UPDATE)); + resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.DELETE)); + + // FHIRSERVER gets only READ privilege to the SV_TENANT_ID variable. The only way FHIRSERVER can + // set (write to) SV_TENANT_ID is by calling the SET_TENANT stored procedure, which requires + // both TENANT_NAME and TENANT_KEY to be provided. + variablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.READ)); + + // FHIRSERVER gets to use the FHIR sequence + sequencePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.USAGE)); + + this.resourceTypes = resourceTypes; + } + + /** + * Build the admin part of the schema. One admin schema can support multiple FHIRDATA + * schemas. It is also possible to have multiple admin schemas (on a dev system, + * for example, although in production there would probably be just one admin schema + * in a given database + * @param model + */ + public void buildAdminSchema(PhysicalDataModel model) { + // All tables are added to this new tablespace (which has a small extent size. + // Each tenant partition gets its own tablespace + fhirTablespace = new Tablespace(FhirSchemaConstants.FHIR_TS, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_TS_EXTENT_KB); + fhirTablespace.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + model.addObject(fhirTablespace); + + addTenantSequence(model); + addTenantTable(model); + addTenantKeysTable(model); + addVariable(model); + + // Add a NopObject which acts as a single dependency marker for the procedure objects to depend on + this.allAdminTablesComplete = new NopObject(adminSchemaName, "allAdminTablesComplete"); + this.allAdminTablesComplete.addDependencies(adminProcedureDependencies); + this.allAdminTablesComplete.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + model.addObject(allAdminTablesComplete); + + // The set_tenant procedure can be created after all the admin tables are done + final String ROOT_DIR = "db2/"; + ProcedureDef setTenant = model.addProcedure(this.adminSchemaName, SET_TENANT, 2, + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, adminSchemaName, + ROOT_DIR + SET_TENANT.toLowerCase() + ".sql", null), + Arrays.asList(allAdminTablesComplete), + procedurePrivileges); + setTenant.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + + // A final marker which is used to block any FHIR data schema activity until the admin schema is completed + this.adminSchemaComplete = new NopObject(adminSchemaName, "adminSchemaComplete"); + this.adminSchemaComplete.addDependencies(Arrays.asList(setTenant)); + this.adminSchemaComplete.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + model.addObject(adminSchemaComplete); + } + + /** + * Add the session variable we need. This variable is used to support multi-tenancy + * via the row-based access control permission predicate. + * @param model + */ + public void addVariable(PhysicalDataModel model) { + this.sessionVariable = new SessionVariableDef(adminSchemaName, "SV_TENANT_ID", FhirSchemaVersion.V0001.vid()); + this.sessionVariable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + variablePrivileges.forEach(p -> p.addToObject(this.sessionVariable)); + + // Make sure any admin procedures are built after the session variable + adminProcedureDependencies.add(this.sessionVariable); + model.addObject(this.sessionVariable); + } + + /** + * Create a table to manage the list of tenants. The tenant id is used + * as a partition value for all the other tables + * @param model + */ + protected void addTenantTable(PhysicalDataModel model) { + + this.tenantsTable = Table.builder(adminSchemaName, TENANTS) + .addIntColumn( MT_ID, false) + .addVarcharColumn( TENANT_NAME, 36, false) // probably a UUID + .addVarcharColumn( TENANT_STATUS, 16, false) + .addUniqueIndex(IDX + "TENANT_TN", TENANT_NAME) + .addPrimaryKey("TENANT_PK", MT_ID) + .setTablespace(fhirTablespace) + .build(model); + + this.tenantsTable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + this.adminProcedureDependencies.add(tenantsTable); + model.addTable(tenantsTable); + model.addObject(tenantsTable); + } + + /** + * Each tenant can have multiple access keys which are used to authenticate and authorize + * clients for access to the data for a given tenant. We support multiple keys per tenant + * as a way to allow key rotation in the configuration without impacting service continuity + * @param model + */ + protected void addTenantKeysTable(PhysicalDataModel model) { + + this.tenantKeysTable = Table.builder(adminSchemaName, TENANT_KEYS) + .addIntColumn( TENANT_KEY_ID, false) // PK + .addIntColumn( MT_ID, false) // FK to TENANTS + .addVarcharColumn( TENANT_SALT, 44, false) // 32 bytes == 44 Base64 symbols + .addVarbinaryColumn( TENANT_HASH, 32, false) // SHA-256 => 32 bytes + .addUniqueIndex(IDX + "TENANT_KEY_SALT", TENANT_SALT) // we want every salt to be unique + .addUniqueIndex(IDX + "TENANT_KEY_TIDH", MT_ID, TENANT_HASH) // for set_tenant query + .addPrimaryKey("TENANT_KEY_PK", TENANT_KEY_ID) + .addForeignKeyConstraint(FK + TENANT_KEYS + "_TNID", adminSchemaName, TENANTS, MT_ID) // dependency + .setTablespace(fhirTablespace) + .build(model); + + this.tenantKeysTable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + this.adminProcedureDependencies.add(tenantKeysTable); + model.addTable(tenantKeysTable); + model.addObject(tenantKeysTable); + } + + /** + 
+    CREATE SEQUENCE fhir_sequence
+             AS BIGINT
+     START WITH 1
+          CACHE 1000
+       NO CYCLE;
+
+ * + * @param pdm + */ + protected void addTenantSequence(PhysicalDataModel pdm) { + this.tenantSequence = new Sequence(adminSchemaName, TENANT_SEQUENCE, FhirSchemaVersion.V0001.vid(), 1, 1000); + this.tenantSequence.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); + adminProcedureDependencies.add(tenantSequence); + sequencePrivileges.forEach(p -> p.addToObject(tenantSequence)); + + pdm.addObject(tenantSequence); + } + + /** + * Create the schema using the given target + * @param model + */ + public void buildSchema(PhysicalDataModel model) { + // Build the complete physical model so that we know it's consistent + buildAdminSchema(model); + addFhirSequence(model); + addFhirRefSequence(model); + addParameterNames(model); + addCodeSystems(model); + addCommonTokenValues(model); + addResourceTypes(model); + addLogicalResourceShards(model); + addLogicalResources(model); // for system-level parameter search + addReferencesSequence(model); + addLogicalResourceCompartments(model); + addResourceChangeLog(model); // track changes for easier export + addCommonCanonicalValues(model); // V0014 + addLogicalResourceProfiles(model); // V0014 + addLogicalResourceTags(model); // V0014 + addLogicalResourceSecurity(model); // V0016 + addErasedResources(model); // V0023 + + Table globalStrValues = addResourceStrValues(model); // for system-level _profile parameters + Table globalDateValues = addResourceDateValues(model); // for system-level date parameters + + // new normalized table for supporting token data (replaces TOKEN_VALUES) + Table globalResourceTokenRefs = addResourceTokenRefs(model); + + // The three "global" tables aren't true dependencies, but this was the easiest way to force sequential processing + // and avoid a pesky deadlock issue we were hitting while adding foreign key constraints on the global tables + addResourceTables(model, globalStrValues, globalDateValues, globalResourceTokenRefs); + + // All the table objects and types should be ready now, so create our NOP + // which is used as a single dependency for all procedures. This means + // procedures won't start until all the create table/type etc statements + // are done...hopefully reducing the number of deadlocks we see. + this.allTablesComplete = new NopObject(schemaName, "allTablesComplete"); + this.allTablesComplete.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.allTablesComplete.addDependencies(procedureDependencies); + model.addObject(allTablesComplete); + } + + public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) { + // These procedures just depend on the table they are manipulating and the fhir sequence. But + // to avoid deadlocks, we only apply them after all the tables are done, so we make all + // procedures depend on the allTablesComplete marker. + final String ROOT_DIR = "db2/"; + ProcedureDef pd = model.addProcedure(this.schemaName, + ADD_CODE_SYSTEM, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + pd = model.addProcedure(this.schemaName, + ADD_PARAMETER_NAME, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + pd = model.addProcedure(this.schemaName, + ADD_RESOURCE_TYPE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + pd = model.addProcedure(this.schemaName, + DELETE_RESOURCE_PARAMETERS, + FhirSchemaVersion.V0020.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + final ProcedureDef deleteResourceParameters = pd; + + pd = model.addProcedure(this.schemaName, + ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + pd = model.addProcedure(this.schemaName, + ERASE_RESOURCE, + FhirSchemaVersion.V0013.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), + procedurePrivileges); + pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + } + + public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { + // Add stored procedures/functions for PostgreSQL + // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. + final String ROOT_DIR = "postgres/"; + FunctionDef fd = model.addFunction(this.schemaName, + ADD_CODE_SYSTEM, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), + procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_PARAMETER_NAME, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_RESOURCE_TYPE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // We currently only support functions with PostgreSQL, although this is really just a procedure + FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, + DELETE_RESOURCE_PARAMETERS, + FhirSchemaVersion.V0020.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges); + deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ERASE_RESOURCE, + FhirSchemaVersion.V0013.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + } + + /** + * @implNote following the current pattern, which is why all this stuff is replicated + * @param model + */ + public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { + // Add stored procedures/functions for postgresql and Citus + // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. + final String ROOT_DIR = "postgres/"; + final String CITUS_ROOT_DIR = "citus/"; + FunctionDef fd = model.addFunction(this.schemaName, + ADD_CODE_SYSTEM, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), + procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_PARAMETER_NAME, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_RESOURCE_TYPE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // Add the delete resource parameters function and distribute using logical_resource_id (param $2) + FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, + DELETE_RESOURCE_PARAMETERS, + FhirSchemaVersion.V0020.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges, 2); + deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // Use the Citus-specific function which is distributed using logical_resource_id (param $1) + fd = model.addFunction(this.schemaName, ADD_LOGICAL_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), + procedurePrivileges, 1); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + final FunctionDef addLogicalResource = fd; + + // Use the Citus-specific variant of add_any_resource and distribute using logical_resource_id (param $1) + fd = model.addFunction(this.schemaName, ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete, addLogicalResource), + procedurePrivileges, 1); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ERASE_RESOURCE, + FhirSchemaVersion.V0013.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + } + + /** + * Add the system-wide logical_resources table. Note that LOGICAL_ID is + * denormalized, stored in both LOGICAL_RESOURCES and _LOGICAL_RESOURCES. + * This avoids an additional join, and simplifies the migration to this + * new schema model. + * @param pdm + */ + public void addLogicalResources(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCES; + final String mtId = this.multitenant ? MT_ID : null; + + final String IDX_LOGICAL_RESOURCES_RITS = "IDX_" + LOGICAL_RESOURCES + "_RITS"; + final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD"; + + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addIntColumn(RESOURCE_TYPE_ID, false) + .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) + .addTimestampColumn(REINDEX_TSTAMP, false, "CURRENT_TIMESTAMP") // new column for V0006 + .addBigIntColumn(REINDEX_TXID, false, "0") // new column for V0006 + .addTimestampColumn(LAST_UPDATED, true) // new column for V0014 + .addCharColumn(IS_DELETED, 1, false, "'X'") + .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true) // new column for V0015 + .addPrimaryKey(tableName + "_PK", LOGICAL_RESOURCE_ID) + .addUniqueIndex("UNQ_" + LOGICAL_RESOURCES, RESOURCE_TYPE_ID, LOGICAL_ID) + .addIndex(IDX_LOGICAL_RESOURCES_RITS, new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null)) + .addIndex(IDX_LOGICAL_RESOURCES_LUPD, new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null)) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addWiths(addWiths()) // add table tuning + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion == FhirSchemaVersion.V0001.vid()) { + // Add statements to migrate from version V0001 to V0006 of this object + List cols = ColumnDefBuilder.builder() + .addTimestampColumn(REINDEX_TSTAMP, false, "CURRENT_TIMESTAMP") + .addBigIntColumn(REINDEX_TXID, false, "0") + .buildColumns(); + + statements.add(new AddColumn(schemaName, tableName, cols.get(0))); + statements.add(new AddColumn(schemaName, tableName, cols.get(1))); + + // Add the new index on REINDEX_TSTAMP. This index is special because it's the + // first index in our schema to use DESC. + List indexCols = Arrays.asList(new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null)); + statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_RITS, tableName, mtId, indexCols)); + } + + if (priorVersion < FhirSchemaVersion.V0009.vid()) { + // Get rid of the old global token values parameter table which no longer + // used + statements.add(new DropTable(schemaName, "TOKEN_VALUES")); + } + + if (priorVersion < FhirSchemaVersion.V0014.vid()) { + // Add LAST_UPDATED and IS_DELETED to whole-system logical_resources + List cols = ColumnDefBuilder.builder() + .addTimestampColumn(LAST_UPDATED, true) + .addCharColumn(IS_DELETED, 1, false, "'X'") + .buildColumns(); + + statements.add(new AddColumn(schemaName, tableName, cols.get(0))); + statements.add(new AddColumn(schemaName, tableName, cols.get(1))); + + // New index on the LAST_UPDATED. We don't need to include resource-type. If + // you know the resource type, you'll be querying the resource-specific + // xx_logical_resources table instead + List indexCols = Arrays.asList(new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null)); + statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_LUPD, tableName, mtId, indexCols)); + } + + if (priorVersion < FhirSchemaVersion.V0015.vid()) { + // Add PARAM_HASH logical_resources + List cols = ColumnDefBuilder.builder() + .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true) + .buildColumns(); + statements.add(new AddColumn(schemaName, tableName, cols.get(0))); + } + + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + + /** + * Adds a table to support sharding of the logical_id when running + * in a distributed RDBMS such as Citus. This table is sharded by + * LOGICAL_ID, which means we can use a primary key of + * {RESOURCE_TYPE_ID, LOGICAL_ID} which is required to ensure + * that we can lock the logical resource to avoid any concurrency + * issues. This is only used for distributed implementations. For + * the standard non-distributed solution, the locking is done + * using LOGICAL_RESOURCES. + * @param pdm + */ + public void addLogicalResourceShards(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCE_SHARDS; + final String mtId = this.multitenant ? MT_ID : null; + + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(mtId) + .addIntColumn(RESOURCE_TYPE_ID, false) + .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addPrimaryKey(tableName + "_PK", RESOURCE_TYPE_ID, LOGICAL_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_ID) // V0026 support for sharding + .addWiths(addWiths()) // add table tuning + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // NOP for now + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + + /** + * Create the COMMON_CANONICAL_VALUES table. Used from schema V0014 to normalize + * meta.profile search parameters (similar to common_token_values). Only the url + * is included by design. The (optional) version and fragment values are stored + * in the parameter mapping table (logical_resource_profiles) in order to support + * inequalities on version while still using a literal CANONICAL_ID = x predicate. + * These canonical ids are cached in the server, so search queries won't need to + * join to this table. The URL is typically a long string, so by normalizing and + * storing/indexing it once, we reduce space consumption. + * @param pdm + */ + public void addCommonCanonicalValues(PhysicalDataModel pdm) { + final String tableName = COMMON_CANONICAL_VALUES; + final String unqCanonicalUrl = "UNQ_" + tableName + "_URL"; + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn(CANONICAL_ID, false) + .addVarcharColumn(URL, CANONICAL_URL_BYTES, false) + .addPrimaryKey(tableName + "_PK", CANONICAL_ID) + .addUniqueIndex(unqCanonicalUrl, URL) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(URL) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // Intentionally NOP + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + + /** + * A single-parameter table supporting _profile search parameter values + * Add the LOGICAL_RESOURCE_PROFILES table to the given {@link PhysicalDataModel}. + * This table maps logical resources to meta.profile values stored as canonical URIs + * in COMMON_CANONICAL_VALUES. Canonical values can include optional version and fragment + * values as described here: https://www.hl7.org/fhir/datatypes.html#canonical + * @param pdm + * @return + */ + public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { + + final String tableName = LOGICAL_RESOURCE_PROFILES; + + // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn( CANONICAL_ID, false) // FK referencing COMMON_CANONICAL_VALUES + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES + .addVarcharColumn( VERSION, VERSION_BYTES, true) + .addVarcharColumn( FRAGMENT, FRAGMENT_BYTES, true) + .addIndex(IDX + tableName + "_CCVLR", CANONICAL_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, CANONICAL_ID) + .addForeignKeyConstraint(FK + tableName + "_CCV", schemaName, COMMON_CANONICAL_VALUES, CANONICAL_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + /** + * A single-parameter table supporting _tag search parameter values. + * Tags are tokens, but because they may not be very selective we use a + * separate table in order to avoid messing up cardinality estimates + * in the query optimizer. + * @param pdm + * @return + */ + public Table addLogicalResourceTags(PhysicalDataModel pdm) { + + final String tableName = LOGICAL_RESOURCE_TAGS; + + // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES + .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + /** + * Add the dedicated common_token_values mapping table for security search parameters + * @param pdm + * @return + */ + public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCE_SECURITY; + + // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES + .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + /** + * Add the resource_change_log table. This table supports tracking of every change made + * to a resource at the global level, making it much easier to stream a list of changes + * from a known point. + * @param pdm + */ + public void addResourceChangeLog(PhysicalDataModel pdm) { + final String tableName = RESOURCE_CHANGE_LOG; + + // custom list of Withs because this table does not require fillfactor tuned in V0020 + List customWiths = Arrays.asList( + With.with("autovacuum_vacuum_scale_factor", "0.01"), // V0019 + With.with("autovacuum_vacuum_threshold", "1000"), // V0019 + With.with("autovacuum_vacuum_cost_limit", "2000") // V0019 + ); + + // Note that for now, we elect to not distribute/shard this table because doing so + // would interfere with the queries supporting the history API which are based on + // index range scans across a contiguous range of records + Table tbl = Table.builder(schemaName, tableName) + .setTenantColumnName(MT_ID) + .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes + .addBigIntColumn(RESOURCE_ID, false) + .addIntColumn(RESOURCE_TYPE_ID, false) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addTimestampColumn(CHANGE_TSTAMP, false) + .addIntColumn(VERSION_ID, false) + .addCharColumn(CHANGE_TYPE, 1, false) + .addPrimaryKey(tableName + "_PK", RESOURCE_ID) + .addUniqueIndex("UNQ_" + RESOURCE_CHANGE_LOG + "_CTRTRI", CHANGE_TSTAMP, RESOURCE_TYPE_ID, RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(customWiths) // Does not require fillfactor tuning + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + + /** + * Adds the system level logical_resource_compartments table which identifies to + * which compartments a give resource belongs. A resource may belong to many + * compartments. + * @param pdm + * @return Table the table that was added to the PhysicalDataModel + */ + public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCE_COMPARTMENTS; + + // note COMPARTMENT_LOGICAL_RESOURCE_ID represents the compartment (e.g. the Patient) + // that this resource exists within. This compartment resource may be a ghost resource...i.e. one + // which has a record in LOGICAL_RESOURCES but currently does not have any resource + // versions because we haven't yet loaded the resource itself. The timestamp is included + // because it makes it very easy to find the most recent changes to resources associated with + // a given patient (for example). + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( COMPARTMENT_NAME_ID, false) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addTimestampColumn(LAST_UPDATED, false) + .addBigIntColumn(COMPARTMENT_LOGICAL_RESOURCE_ID, false) + .addUniqueIndex(IDX + tableName + "_LRNMLR", LOGICAL_RESOURCE_ID, COMPARTMENT_NAME_ID, COMPARTMENT_LOGICAL_RESOURCE_ID) + .addUniqueIndex(IDX + tableName + "_NMCOMPLULR", COMPARTMENT_NAME_ID, COMPARTMENT_LOGICAL_RESOURCE_ID, LAST_UPDATED, LOGICAL_RESOURCE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .addForeignKeyConstraint(FK + tableName + "_COMP", schemaName, LOGICAL_RESOURCES, COMPARTMENT_LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + + + /** + * Add system-wide RESOURCE_STR_VALUES table to support _profile + * properties (which are of type REFERENCE). + * @param pdm + * @return Table the table that was added to the PhysicalDataModel + */ + public Table addResourceStrValues(PhysicalDataModel pdm) { + final int msb = MAX_SEARCH_STRING_BYTES; + + Table tbl = Table.builder(schemaName, STR_VALUES) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( PARAMETER_NAME_ID, false) + .addVarcharColumn( STR_VALUE, msb, true) + .addVarcharColumn( STR_VALUE_LCASE, msb, true) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addIndex(IDX + STR_VALUES + "_PSR", PARAMETER_NAME_ID, STR_VALUE, LOGICAL_RESOURCE_ID) + .addIndex(IDX + STR_VALUES + "_PLR", PARAMETER_NAME_ID, STR_VALUE_LCASE, LOGICAL_RESOURCE_ID) + .addIndex(IDX + STR_VALUES + "_RPS", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, STR_VALUE) + .addIndex(IDX + STR_VALUES + "_RPL", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, STR_VALUE_LCASE) + .addForeignKeyConstraint(FK + STR_VALUES + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + STR_VALUES + "_RID", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, STR_VALUES, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, STR_VALUES, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + /** + * Add the table for data search parameters at the (system-wide) resource level + * @param model + * @return Table the table that was added to the PhysicalDataModel + */ + public Table addResourceDateValues(PhysicalDataModel model) { + final String tableName = DATE_VALUES; + final String logicalResourcesTable = LOGICAL_RESOURCES; + + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( PARAMETER_NAME_ID, false) + .addTimestampColumn( DATE_START,6, true) + .addTimestampColumn( DATE_END,6, true) + .addBigIntColumn(LOGICAL_RESOURCE_ID, false) + .addIndex(IDX + tableName + "_PSER", PARAMETER_NAME_ID, DATE_START, DATE_END, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_PESR", PARAMETER_NAME_ID, DATE_END, DATE_START, LOGICAL_RESOURCE_ID) + .addIndex(IDX + tableName + "_RPSE", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, DATE_START, DATE_END) + .addForeignKeyConstraint(FK + tableName + "_PN", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + tableName + "_R", schemaName, logicalResourcesTable, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // New Column for V0017 + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion == 1) { + statements.add(new DropIndex(schemaName, IDX + tableName + "_PVR")); + statements.add(new DropIndex(schemaName, IDX + tableName + "_RPV")); + statements.add(new DropColumn(schemaName, tableName, DATE_VALUE_DROPPED_COLUMN)); + } + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(model); + + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + model.addTable(tbl); + model.addObject(tbl); + + return tbl; + } + + /** + * 
+        CREATE TABLE resource_types (
+            resource_type_id INT NOT NULL
+            CONSTRAINT pk_resource_type PRIMARY KEY,
+            resource_type   VARCHAR(64) NOT NULL
+        );
+
+        -- make sure resource_type values are unique
+        CREATE UNIQUE INDEX unq_resource_types_rt ON resource_types(resource_type);
+
+ * + * @param model + */ + protected void addResourceTypes(PhysicalDataModel model) { + resourceTypesTable = Table.builder(schemaName, RESOURCE_TYPES) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( RESOURCE_TYPE_ID, false) + .addVarcharColumn( RESOURCE_TYPE, 64, false) + .addUniqueIndex(IDX + "unq_resource_types_rt", RESOURCE_TYPE) + .addPrimaryKey(RESOURCE_TYPES + "_PK", RESOURCE_TYPE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 supporting for sharding + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // Intentionally a NOP + return statements; + }) + .build(model); + + // TODO Table should be immutable, so add support to the Builder for this + this.resourceTypesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(resourceTypesTable); + model.addTable(resourceTypesTable); + model.addObject(resourceTypesTable); + } + + /** + * Add the collection of tables for each of the listed + * FHIR resource types + * @param model + */ + protected void addResourceTables(PhysicalDataModel model, IDatabaseObject... dependency) { + if (this.sessionVariable == null) { + throw new IllegalStateException("Session variable must be defined before adding resource tables"); + } + + // The sessionVariable is used to enable access control on every table, so we + // provide it as a dependency + FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, this.multitenant, sessionVariable, + this.procedureDependencies, this.fhirTablespace, this.resourceTablePrivileges, addWiths()); + for (String resourceType: this.resourceTypes) { + + resourceType = resourceType.toUpperCase().trim(); + if (!ALL_RESOURCE_TYPES.contains(resourceType.toUpperCase())) { + logger.warning("Passed resource type '" + resourceType + "' does not match any known FHIR resource types; creating anyway"); + } + + ObjectGroup group = frg.addResourceType(resourceType); + group.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // Add additional dependencies the group doesn't yet know about + group.addDependencies(Arrays.asList(this.codeSystemsTable, this.parameterNamesTable, this.resourceTypesTable, this.commonTokenValuesTable)); + + // Add all other dependencies that were explicitly passed + group.addDependencies(Arrays.asList(dependency)); + + // Make this group a dependency for all the stored procedures. + this.procedureDependencies.add(group); + model.addObject(group); + } + } + + /** + * + * + CREATE TABLE parameter_names ( + parameter_name_id INT NOT NULL + CONSTRAINT pk_parameter_name PRIMARY KEY, + parameter_name VARCHAR(255 OCTETS) NOT NULL + ); + + CREATE UNIQUE INDEX unq_parameter_name_rtnm ON parameter_names(parameter_name) INCLUDE (parameter_name_id); + + * @param model + */ + protected void addParameterNames(PhysicalDataModel model) { + // The index which also used by the database to support the primary key constraint + String[] prfIndexCols = {PARAMETER_NAME}; + String[] prfIncludeCols = {PARAMETER_NAME_ID}; + + parameterNamesTable = Table.builder(schemaName, PARAMETER_NAMES) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( PARAMETER_NAME_ID, false) + .addVarcharColumn( PARAMETER_NAME, 255, false) + .addUniqueIndex(IDX + "PARAMETER_NAME_RTNM", Arrays.asList(prfIndexCols), Arrays.asList(prfIncludeCols)) + .addPrimaryKey(PARAMETER_NAMES + "_PK", PARAMETER_NAME_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 treat this as a reference table + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // Intentionally a NOP + return statements; + }) + .build(model); + + this.parameterNamesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(parameterNamesTable); + + model.addTable(parameterNamesTable); + model.addObject(parameterNamesTable); + } + + /** + * Add the code_systems table to the database schema + CREATE TABLE code_systems ( + code_system_id INT NOT NULL + CONSTRAINT pk_code_system PRIMARY KEY, + code_system_name VARCHAR(255 OCTETS) NOT NULL + ); + + CREATE UNIQUE INDEX unq_code_system_cinm ON code_systems(code_system_name); + + * @param model + */ + protected void addCodeSystems(PhysicalDataModel model) { + codeSystemsTable = Table.builder(schemaName, CODE_SYSTEMS) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( CODE_SYSTEM_ID, false) + .addVarcharColumn(CODE_SYSTEM_NAME, 255, false) + .addUniqueIndex(IDX + "CODE_SYSTEM_CINM", CODE_SYSTEM_NAME) + .addPrimaryKey(CODE_SYSTEMS + "_PK", CODE_SYSTEM_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionReference() // V0026 treat this as a reference table + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, CODE_SYSTEMS, 2000, null, 1000)); + } + return statements; + }) + .build(model); + + this.codeSystemsTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(codeSystemsTable); + model.addTable(codeSystemsTable); + model.addObject(codeSystemsTable); + } + + /** + * Table used to store normalized values for tokens, shared by all the + * _TOKEN_VALUES tables. Although this requires an additional + * join, it cuts down on space by avoiding repeating long strings (e.g. urls). + * This also helps to reduce the total sizes of the indexes, helping to improve + * cache hit rates for a given buffer cache size. + * Token values may or may not have an associated code system, in which case, + * it assigned a default system. This is why CODE_SYSTEM_ID is not nullable and + * has a FK constraint. + * + * We never need to find all token values for a given code-system, so there's no need + * for a second index (CODE_SYSTEM_ID, TOKEN_VALUE). Do not add it. + * + * Because different parameter names may reference the same token value (e.g. + * 'Observation.subject' and 'Claim.patient' are both patient references), the + * common token value is not distinguished by a parameter_name_id. + * + * Where common token values are used to represent local relationships between two resources, + * the code_system encodes the resource type of the referenced resource and + * the token_value represents its logical_id. This approach simplifies query writing when + * following references. + * + * If sharding is supported, this table is distributed by token_value which unfortunately + * means that it cannot be the target of any foreign key constraint (which needs to use + * the primary key COMMON_TOKEN_VALUE_ID). + * @param pdm + * @return the table definition + */ + public void addCommonTokenValues(PhysicalDataModel pdm) { + final String tableName = COMMON_TOKEN_VALUES; + commonTokenValuesTable = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addBigIntColumn( COMMON_TOKEN_VALUE_ID, false) + .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) + .addIntColumn( CODE_SYSTEM_ID, false) + .addVarcharColumn( TOKEN_VALUE, MAX_TOKEN_VALUE_BYTES, false) + .addUniqueIndex(IDX + tableName + "_TVCP", TOKEN_VALUE, CODE_SYSTEM_ID) + .addPrimaryKey(tableName + "_PK", COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_CSID", schemaName, CODE_SYSTEMS, CODE_SYSTEM_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(TOKEN_VALUE) // V0026 shard using token_value + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // Intentionally a NOP + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + commonTokenValuesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + pdm.addTable(commonTokenValuesTable); + pdm.addObject(commonTokenValuesTable); + } + + /** + * Add the system-wide RESOURCE_TOKEN_REFS table which is used for + * _tag and _security search properties in R4 (new table + * for issue #1366 V0006 schema change). Replaces the + * previous TOKEN_VALUES table. All token values are now + * normalized in the COMMON_TOKEN_VALUES table. Because this + * is for system-level params, there's no need to support + * composite params + * @param pdm + * @return Table the table that was added to the PhysicalDataModel + */ + public Table addResourceTokenRefs(PhysicalDataModel pdm) { + + final String tableName = RESOURCE_TOKEN_REFS; + + // logical_resources (0|1) ---- (*) resource_token_refs + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .addIntColumn( PARAMETER_NAME_ID, false) + .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version + .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0009 change + .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0009 change + .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .addForeignKeyConstraint(FK + tableName + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .addWiths(addWiths()) // table tuning + .addMigration(priorVersion -> { + // Replace the indexes initially defined in the V0006 version with better ones + List statements = new ArrayList<>(); + if (priorVersion == FhirSchemaVersion.V0006.vid()) { + // Migrate the index definitions as part of the V0008 version of the schema + // This table was originally introduced as part of the V0006 schema, which + // is what we use as the match for the priorVersion + statements.add(new DropIndex(schemaName, IDX + tableName + "_TVLR")); + statements.add(new DropIndex(schemaName, IDX + tableName + "_LRTV")); + + final String mtId = multitenant ? MT_ID : null; + // Replace the original TVLR index on (common_token_value_id, parameter_name_id, logical_resource_id) + List tplr = Arrays.asList( + new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null), + new OrderedColumnDef(PARAMETER_NAME_ID, OrderedColumnDef.Direction.ASC, null), + new OrderedColumnDef(LOGICAL_RESOURCE_ID, OrderedColumnDef.Direction.ASC, null) + ); + statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_TPLR", tableName, mtId, tplr)); + + // Replace the original LRTV index with a new index on (logical_resource_id, parameter_name_id, common_token_value_id) + List lrpt = Arrays.asList( + new OrderedColumnDef(LOGICAL_RESOURCE_ID, OrderedColumnDef.Direction.ASC, null), + new OrderedColumnDef(PARAMETER_NAME_ID, OrderedColumnDef.Direction.ASC, null), + new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null) + ); + statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_LRPT", tableName, mtId, lrpt)); + } + if (priorVersion < FhirSchemaVersion.V0019.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + } + if (priorVersion < FhirSchemaVersion.V0020.vid()) { + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + + return tbl; + } + + /** + * The erased_resources table is used to track which logical resources and corresponding + * resource versions have been erased using the $erase operation. This table should + * typically be empty and only used temporarily by the erase DAO/procedures to indicate + * which rows have been erased. The entries in this table are then used to delete + * any offloaded payload entries. + * @param pdm + */ + public void addErasedResources(PhysicalDataModel pdm) { + final String tableName = ERASED_RESOURCES; + final String mtId = this.multitenant ? MT_ID : null; + + // Each erase operation is allocated an ERASED_RESOURCE_GROUP_ID + // value which can be used to retrieve the resource and/or + // resource-versions erased in a particular call. The rows + // can then be deleted once the erasure of any offloaded + // payload is confirmed. Note that we don't use logical_resource_id + // or resource_id values here, because those records may have + // already been deleted by $erase. + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0026.vid()) + .setTenantColumnName(mtId) + .addBigIntColumn(ERASED_RESOURCE_GROUP_ID, false) + .addIntColumn(RESOURCE_TYPE_ID, false) + .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) + .addIntColumn(VERSION_ID, true) + .addIndex(IDX + tableName + "_GID", ERASED_RESOURCE_GROUP_ID) + .setDistributionColumnName(ERASED_RESOURCE_GROUP_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) + .enableAccessControl(this.sessionVariable) + .addWiths(addWiths()) // add table tuning + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + // Nothing yet + + // TODO migrate to simplified design (no PK, FK) + return statements; + }) + .build(pdm); + + // TODO should not need to add as a table and an object. Get the table to add itself? + tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + this.procedureDependencies.add(tbl); + pdm.addTable(tbl); + pdm.addObject(tbl); + } + + /** + * 
+    CREATE SEQUENCE fhir_sequence
+             AS BIGINT
+     START WITH 1
+          CACHE 20000
+       NO CYCLE;
+     *
+ * + * @param pdm + */ + protected void addFhirSequence(PhysicalDataModel pdm) { + this.fhirSequence = new Sequence(schemaName, FHIR_SEQUENCE, FhirSchemaVersion.V0001.vid(), 1, 1000); + this.fhirSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + procedureDependencies.add(fhirSequence); + sequencePrivileges.forEach(p -> p.addToObject(fhirSequence)); + + pdm.addObject(fhirSequence); + } + + protected void addFhirRefSequence(PhysicalDataModel pdm) { + this.fhirRefSequence = new Sequence(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE); + this.fhirRefSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + procedureDependencies.add(fhirRefSequence); + sequencePrivileges.forEach(p -> p.addToObject(fhirRefSequence)); + pdm.addObject(fhirRefSequence); + + // Schema V0003 does an alter to bump up the start value of the reference sequence + // to avoid a conflict with parameter names not in the pre-populated set + // fix for issue-1263. This will only be applied if the current version of the + // the FHIR_REF_SEQUENCE is <= 2. + BaseObject alter = new AlterSequenceStartWith(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0003.vid(), + FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE, 1); + alter.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + procedureDependencies.add(alter); + alter.addDependency(fhirRefSequence); // only alter after the sequence is initially created + + // Because the sequence might be dropped and recreated, we need to inject privileges + // so that they are applied when this ALTER SEQUENCE is processed. + sequencePrivileges.forEach(p -> p.addToObject(alter)); + pdm.addObject(alter); + } + + /** + * Add the sequence used by the new local/external references data model + * @param pdm + */ + protected void addReferencesSequence(PhysicalDataModel pdm) { + Sequence seq = new Sequence(schemaName, FhirSchemaConstants.REFERENCES_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.REFERENCES_SEQUENCE_START, FhirSchemaConstants.REFERENCES_SEQUENCE_CACHE, FhirSchemaConstants.REFERENCES_SEQUENCE_INCREMENT); + seq.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + procedureDependencies.add(seq); + sequencePrivileges.forEach(p -> p.addToObject(seq)); + pdm.addObject(seq); + } + + /** + * The defaults with addWiths. Added to every table in a PostgreSQL schema + * @return + */ + protected List addWiths() { + // NOTE! If you change this table remember that you also need to bump the + // schema version of every table that uses this list of Withs. This includes + // adding a corresponding migration step. + return Arrays.asList( + With.with("autovacuum_vacuum_scale_factor", "0.01"), // V0019 + With.with("autovacuum_vacuum_threshold", "1000"), // V0019 + With.with("autovacuum_vacuum_cost_limit", "2000"), // V0019 + With.with(FhirSchemaConstants.PG_FILLFACTOR_PROP, Integer.toString(FhirSchemaConstants.PG_FILLFACTOR_VALUE)) // V0020 + ); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 2d4e218c035..c4cb8d51da0 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -136,6 +136,9 @@ public class FhirSchemaGenerator { private static final String ADD_PARAMETER_NAME = "ADD_PARAMETER_NAME"; private static final String ADD_RESOURCE_TYPE = "ADD_RESOURCE_TYPE"; private static final String ADD_ANY_RESOURCE = "ADD_ANY_RESOURCE"; + + // Special procedure for Citus database support + private static final String ADD_LOGICAL_RESOURCE = "ADD_LOGICAL_RESOURCE"; private static final String DELETE_RESOURCE_PARAMETERS = "DELETE_RESOURCE_PARAMETERS"; private static final String ERASE_RESOURCE = "ERASE_RESOURCE"; @@ -474,7 +477,7 @@ public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) { } public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { - // Add stored procedures/functions for postgresql and Citus + // Add stored procedures/functions for PostgreSQL // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. final String ROOT_DIR = "postgres/"; FunctionDef fd = model.addFunction(this.schemaName, @@ -559,22 +562,31 @@ public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - // We currently only support functions with PostgreSQL, although this is really just a procedure + // Add the delete resource parameters function and distribute using logical_resource_id (param $2) FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, DELETE_RESOURCE_PARAMETERS, FhirSchemaVersion.V0020.vid(), () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges); + procedurePrivileges, 2); deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - // Use the Citus-specific variant of add_any_resource (supports sharding) - fd = model.addFunction(this.schemaName, - ADD_ANY_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + // Use the Citus-specific function which is distributed using logical_resource_id (param $1) + fd = model.addFunction(this.schemaName, ADD_LOGICAL_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), + procedurePrivileges, 1); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + final FunctionDef addLogicalResource = fd; + + // Use the Citus-specific variant of add_any_resource and distribute using logical_resource_id (param $1) + fd = model.addFunction(this.schemaName, ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete, addLogicalResource), + procedurePrivileges, 1); fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); fd = model.addFunction(this.schemaName, diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql index b963ad0f858..6e3d70aac49 100644 --- a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql +++ b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql @@ -99,115 +99,22 @@ -- check to see if it was us who actually created the record IF v_logical_resource_id = t_logical_resource_id THEN - -- the record was created by this call, so now create the corresponding entry in the - -- global logical_resources table (which is distributed by logical_resource_id). - -- Because we created the logical_resource_shards record, we can be certain the - -- logical_resources record doesn't yet exist - INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) - VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64); - - -- similarly, create the corresponding record in the resource-type-specific logical_resources table - EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' - || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; v_new_resource := 1; ELSE - -- use the record created elsewhere + -- resource was created by another thread, so use that id instead v_logical_resource_id := t_logical_resource_id; - - -- find the current parameter hash and deletion values from the logical_resources table - SELECT parameter_hash, is_deleted - INTO o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources - WHERE logical_resource_id = v_logical_resource_id; END IF; END IF; - -- Remember everying is locked at the logical resource level, so we are thread-safe here - IF v_new_resource = 0 THEN - -- as this is an existing resource, we need to know the current resource id. - -- This is only available at the resource-specific logical_resources level - EXECUTE - 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' - || ' WHERE logical_resource_id = $1 ' - INTO v_current_resource_id, v_current_version USING v_logical_resource_id; - - IF v_current_resource_id IS NULL OR v_current_version IS NULL - THEN - -- our concurrency protection means that this shouldn't happen - RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; - END IF; - - -- If-None-Match does not apply if the resource is currently deleted - IF v_currently_deleted = 'N' AND p_if_none_match = 0 - THEN - -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the - -- connection with a fatal error, so instead we use an out parameter to - -- indicate the match - o_interaction_status := 1; - o_if_none_match_version := v_current_version; - RETURN; - END IF; - - -- Concurrency check: - -- the version parameter we've been given (which is also embedded in the JSON payload) must be - -- one greater than the current version, otherwise we've hit a concurrent update race condition - IF p_version != v_current_version + 1 - THEN - RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; - END IF; + -- add_logical_resource has 13 IN parameters followed by 3 OUT parameters + EXECUTE 'SELECT * FROM {{SCHEMA_NAME}}.add_logical_resource($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)' + INTO o_current_parameter_hash, + o_interaction_status, + o_if_none_match_version + USING v_logical_resource_id, v_new_resource, + p_resource_type, v_resource_type_id, p_logical_id, + p_payload, p_last_updated, p_is_deleted, + p_source_key, p_version, p_parameter_hash_b64, + p_if_none_match, p_resource_payload_key; - -- Prevent creating a new deletion marker if the resource is currently deleted - IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' - THEN - RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; - END IF; - - IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash - THEN - -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) - -- TODO patch parameter sets instead of all delete/all insert. - EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' - USING p_resource_type, v_logical_resource_id; - END IF; -- end if check parameter hash - END IF; -- end if existing resource - - EXECUTE - 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' - || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' - USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; - - - IF v_new_resource = 0 THEN - -- As this is an existing logical resource, we need to update the xx_logical_resource values to match - -- the values of the current resource. For new resources, these are added by the insert so we don't - -- need to update them here. - EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' - USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; - - -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level - EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' - USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; - END IF; - - -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event - -- related to resources changes (issue-1955) - IF p_is_deleted = 'Y' - THEN - v_change_type := 'D'; - ELSE - IF v_new_resource = 0 - THEN - v_change_type := 'U'; - ELSE - v_change_type := 'C'; - END IF; - END IF; - - INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) - VALUES (v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); - - -- Hand back the id of the logical resource we created earlier. In the new R4 schema - -- only the logical_resource_id is the target of any FK, so there's no need to return - -- the resource_id (which is now private to the _resources tables). - o_logical_resource_id := v_logical_resource_id; END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql new file mode 100644 index 00000000000..f28a2a70800 --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql @@ -0,0 +1,190 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2022 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to add a resource version and its associated parameters. These +-- parameters only ever point to the latest version of a resource, never to +-- previous versions, which are kept to support history queries. +-- implNote - Conventions: +-- p_... prefix used to represent input parameters +-- v_... prefix used to represent declared variables +-- t_... prefix used to represent temp variables +-- o_... prefix used to represent output parameters +-- Parameters: +-- p_logical_resource_id: the known primary key for the logical resource +-- p_new_resource: 1 if this is a newly created resource +-- p_resource_type: the resource type name +-- p_resource_type_id: the resource type id +-- p_logical_id: the logical id given to the resource by the FHIR server +-- p_payload: the BLOB (of JSON) which is the resource content if any +-- p_last_updated the last_updated time given by the FHIR server +-- p_is_deleted: the soft delete flag +-- p_version_id: the intended new version id of the resource (matching the JSON payload) +-- p_parameter_hash_b64 the Base64 encoded hash of parameter values +-- p_if_none_match the encoded If-None-Match value +-- o_current_parameter_hash: Base64 current parameter hash if existing resource +-- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit +-- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) +-- Exceptions: +-- SQLSTATE 99001: on version conflict (concurrency) +-- SQLSTATE 99002: missing expected row (data integrity) +-- SQLSTATE 99004: delete a currently deleted resource (data integrity) +-- Citus Distribed Function: +-- For Citus, we split the ingestion logic into two stored procedures (functions) +-- so that we can distribute these functions using the same distribution (sharding) +-- key as the tables they interact with +-- add_any_resource - distributed by logical_id, which matches the sharding +-- of logical_resource_shards +-- add_logical_resource - distributed by logical_resource_id, which matches the +-- sharding of all the other tables used by statements in +-- the procedure. +-- ---------------------------------------------------------------------------- + ( IN p_logical_resource_id BIGINT, + IN p_new_resource INT, + IN p_resource_type VARCHAR( 36), + IN p_resource_type_id INT, + IN p_logical_id VARCHAR(255), + IN p_payload BYTEA, + IN p_last_updated TIMESTAMP, + IN p_is_deleted CHAR( 1), + IN p_source_key VARCHAR( 64), + IN p_version INT, + IN p_parameter_hash_b64 VARCHAR( 44), + IN p_if_none_match INT, + IN p_resource_payload_key VARCHAR( 36), + OUT o_current_parameter_hash VARCHAR( 44), + OUT o_interaction_status INT, + OUT o_if_none_match_version INT) + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + v_logical_resource_id BIGINT := NULL; + t_logical_resource_id BIGINT := NULL; + v_current_resource_id BIGINT := NULL; + v_resource_id BIGINT := NULL; + v_currently_deleted CHAR(1) := NULL; + v_duplicate INT := 0; + v_current_version INT := 0; + v_change_type CHAR(1) := NULL; + + -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. + lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_shards WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + + BEGIN + -- default value unless we hit If-None-Match + o_interaction_status := 0; + + -- LOADED ON: {{DATE}} + v_schema_name := '{{SCHEMA_NAME}}'; + + -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) + SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; + + -- Create the resource if we don't have it already + IF p_new_resource = 1 + THEN + -- create the corresponding entry in the + -- global logical_resources table (which is distributed by logical_resource_id). + -- Because we created the logical_resource_shards record, we can be certain the + -- logical_resources record doesn't yet exist + INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (v_logical_resource_id, p_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64); + + -- similarly, create the corresponding record in the resource-type-specific logical_resources table + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + ELSE + -- find the current parameter hash and deletion values from the logical_resources table + SELECT parameter_hash, is_deleted + INTO o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; + + -- as this is an existing resource, we need to know the current resource id. + -- This is only available at the resource-specific logical_resources level + EXECUTE + 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' + || ' WHERE logical_resource_id = $1 ' + INTO v_current_resource_id, v_current_version USING v_logical_resource_id; + + IF v_current_resource_id IS NULL OR v_current_version IS NULL + THEN + -- our concurrency protection means that this shouldn't happen + RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; + END IF; + + -- If-None-Match does not apply if the resource is currently deleted + IF v_currently_deleted = 'N' AND p_if_none_match = 0 + THEN + -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the + -- connection with a fatal error, so instead we use an out parameter to + -- indicate the match + o_interaction_status := 1; + o_if_none_match_version := v_current_version; + RETURN; + END IF; + + -- Concurrency check: + -- the version parameter we've been given (which is also embedded in the JSON payload) must be + -- one greater than the current version, otherwise we've hit a concurrent update race condition + IF p_version != v_current_version + 1 + THEN + RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; + END IF; + + -- Prevent creating a new deletion marker if the resource is currently deleted + IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' + THEN + RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; + END IF; + + IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash + THEN + -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) + -- TODO patch parameter sets instead of all delete/all insert. + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' + USING p_resource_type, v_logical_resource_id; + END IF; -- end if check parameter hash + END IF; -- end if new resource + + -- create the new resource version record + EXECUTE + 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' + USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; + + + IF p_new_resource = 0 THEN + -- As this is an existing logical resource, we need to update the xx_logical_resource values to match + -- the values of the current resource. For new resources, these are added by the insert so we don't + -- need to update them here. + EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' + USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; + + -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level + EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' + USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; + END IF; + + -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event + -- related to resources changes (issue-1955) + IF p_is_deleted = 'Y' + THEN + v_change_type := 'D'; + ELSE + IF p_new_resource = 0 + THEN + v_change_type := 'U'; + ELSE + v_change_type := 'C'; + END IF; + END IF; + + INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) + VALUES (v_resource_id, p_last_updated, p_resource_type_id, v_logical_resource_id, p_version, v_change_type); +END $$; \ No newline at end of file From 68819abc2f1c859263598931f2cddb61f8987eb0 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 5 May 2022 14:40:38 +0100 Subject: [PATCH 05/40] issue #3437 distributed schema using shard_key Signed-off-by: Robin Arnold --- .../java/com/ibm/fhir/bucket/app/Main.java | 14 +- .../bucket/persistence/FhirBucketSchema.java | 6 +- .../test/FhirBucketSchemaTest.java | 5 +- .../patient/PatientExportPartitionMapper.java | 2 +- .../system/SystemExportPartitionMapper.java | 2 +- .../ibm/fhir/config/FHIRConfiguration.java | 8 + .../ibm/fhir/config/FHIRRequestContext.java | 22 + .../utils/api/DistributionContext.java | 42 + .../database/utils/api/DistributionRules.java | 75 - .../database/utils/api/DistributionType.java | 16 + .../database/utils/api/IDatabaseAdapter.java | 16 +- .../database/utils/api/ISchemaAdapter.java | 611 +++++++ .../fhir/database/utils/api/SchemaType.java | 20 + .../database/utils/citus/CitusAdapter.java | 61 +- .../utils/common/CommonDatabaseAdapter.java | 8 +- .../utils/common/PlainSchemaAdapter.java | 366 ++++ .../fhir/database/utils/db2/Db2Adapter.java | 4 +- .../database/utils/derby/DerbyAdapter.java | 8 +- .../database/utils/derby/DerbyMaster.java | 21 +- .../utils/model/AlterSequenceStartWith.java | 10 +- .../utils/model/AlterTableAddColumn.java | 10 +- .../utils/model/AlterTableIdentityCache.java | 10 +- .../fhir/database/utils/model/BaseObject.java | 14 +- .../utils/model/ColumnDefBuilder.java | 3 + .../fhir/database/utils/model/ColumnType.java | 3 +- .../database/utils/model/CreateIndex.java | 49 +- .../database/utils/model/DatabaseObject.java | 8 +- .../utils/model/ForeignKeyConstraint.java | 33 +- .../database/utils/model/FunctionDef.java | 10 +- .../database/utils/model/IDatabaseObject.java | 18 +- .../fhir/database/utils/model/IndexDef.java | 16 +- .../fhir/database/utils/model/NopObject.java | 10 +- .../database/utils/model/ObjectGroup.java | 14 +- .../utils/model/PhysicalDataModel.java | 25 +- .../database/utils/model/ProcedureDef.java | 10 +- .../database/utils/model/RowArrayType.java | 8 +- .../fhir/database/utils/model/RowType.java | 8 +- .../fhir/database/utils/model/Sequence.java | 10 +- .../utils/model/SessionVariableDef.java | 10 +- .../utils/model/SmallIntBooleanColumn.java | 35 + .../database/utils/model/SmallIntColumn.java | 13 +- .../ibm/fhir/database/utils/model/Table.java | 148 +- .../fhir/database/utils/model/Tablespace.java | 12 +- .../ibm/fhir/database/utils/model/View.java | 8 +- .../utils/postgres/PostgresAdapter.java | 10 +- .../database/utils/version/CreateControl.java | 4 +- .../utils/version/CreateVersionHistory.java | 4 +- .../version/CreateWholeSchemaVersion.java | 8 +- .../jdbc/FHIRResourceDAOFactory.java | 20 +- .../jdbc/citus/CitusResourceDAO.java | 8 +- .../FHIRDbConnectionStrategyBase.java | 7 +- .../jdbc/connection/FHIRDbFlavor.java | 7 + .../jdbc/connection/FHIRDbFlavorImpl.java | 24 +- ...RDbTenantDatasourceConnectionStrategy.java | 22 +- .../FHIRDbTestConnectionStrategy.java | 6 +- .../dao/impl/ParameterTransportVisitor.java | 109 ++ .../jdbc/dao/impl/ResourceDAOImpl.java | 2 +- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 126 +- .../SearchParametersTransportAdapter.java | 156 ++ .../jdbc/postgres/PostgresResourceDAO.java | 145 +- .../jdbc/search/test/JDBCSearchNearTest.java | 4 +- .../jdbc/test/erase/EraseTestMain.java | 3 +- .../java/com/ibm/fhir/schema/app/Main.java | 134 +- .../ibm/fhir/schema/app/SchemaPrinter.java | 39 +- .../com/ibm/fhir/schema/app/menu/Menu.java | 2 + .../ibm/fhir/schema/app/util/CommonUtil.java | 46 + .../build/DistributedSchemaAdapter.java | 125 ++ .../fhir/schema/build/FhirSchemaAdapter.java | 25 + .../fhir/schema/control/AddForeignKey.java | 11 +- .../FhirDistributedSchemaGenerator.java | 1544 ----------------- .../control/FhirResourceTableGroup.java | 41 +- .../schema/control/FhirSchemaConstants.java | 3 + .../schema/control/FhirSchemaGenerator.java | 85 +- .../control/JavaBatchSchemaGenerator.java | 4 +- .../fhir/schema/derby/DerbyFhirDatabase.java | 8 +- .../postgres/add_any_resource_distributed.sql | 212 +++ ...delete_resource_parameters_distributed.sql | 63 + .../postgres/erase_resource_distributed.sql | 89 + .../schema/app/DataSchemaGeneratorTest.java | 14 +- .../app/JavaBatchSchemaGeneratorTest.java | 27 +- .../schema/app/OAuthSchemaGeneratorTest.java | 11 +- .../schema/control/FhirSchemaServiceTest.java | 28 +- .../schema/control/ParallelBuildTest.java | 8 +- .../schema/derby/DerbyFhirDatabaseTest.java | 10 +- .../fhir/schema/derby/DerbyMigrationTest.java | 3 +- .../ibm/fhir/persistence/FHIRPersistence.java | 9 +- .../context/FHIRPersistenceContext.java | 7 + .../FHIRPersistenceContextFactory.java | 9 +- .../impl/FHIRPersistenceContextImpl.java | 26 + .../fhir/persistence/index/DateParameter.java | 46 + .../persistence/index/FHIRIndexProvider.java | 23 + .../index/FHIRRemoteIndexService.java | 47 + .../index/IndexProviderResponse.java | 45 + .../persistence/index/LocationParameter.java | 45 + .../persistence/index/NumberParameter.java | 60 + .../index/ParameterValueVisitorAdapter.java | 78 + .../persistence/index/QuantityParameter.java | 90 + .../persistence/index/RemoteIndexData.java | 57 + .../persistence/index/RemoteIndexMessage.java | 44 + .../index/SearchParameterValue.java | 49 + .../index/SearchParametersTransport.java | 350 ++++ .../persistence/index/StringParameter.java | 29 + .../persistence/index/TokenParameter.java | 62 + .../test/FHIRPersistenceContextTest.java | 3 +- .../persistence/test/MockPersistenceImpl.java | 4 +- .../test/common/AbstractChangesTest.java | 12 +- .../test/common/AbstractEraseTest.java | 16 +- .../test/common/AbstractPersistenceTest.java | 2 +- fhir-remote-index/pom.xml | 113 ++ .../remote/index/api/IMessageHandler.java | 18 + .../com/ibm/fhir/remote/index/app/Main.java | 283 +++ .../index/database/BaseMessageHandler.java | 152 ++ .../DistributedPostgresMessageHandler.java | 150 ++ .../DistributedPostgresParameterBatch.java | 46 + .../index/kafka/RemoteIndexConsumer.java | 180 ++ fhir-server/pom.xml | 4 + .../filter/rest/FHIRRestServletFilter.java | 18 + .../kafka/FHIRRemoteIndexKafkaService.java | 160 ++ .../index/kafka/KafkaPropertyAdapter.java | 64 + .../listener/FHIRServletContextListener.java | 43 + .../ServerRegistryResourceProvider.java | 4 +- .../ibm/fhir/server/util/FHIRRestHelper.java | 38 +- .../fhir/server/test/MockPersistenceImpl.java | 4 +- .../test/ServerResolveFunctionTest.java | 3 +- .../fhir/smart/test/MockPersistenceImpl.java | 4 +- 125 files changed, 5212 insertions(+), 2184 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java delete mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java delete mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java create mode 100644 fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql create mode 100644 fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql create mode 100644 fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java create mode 100644 fhir-remote-index/pom.xml create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java create mode 100644 fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java create mode 100644 fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index 1c1c036351e..c1661a2e550 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -56,11 +56,13 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.db2.Db2PropertyAdapter; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -148,6 +150,9 @@ public class Main { // The adapter configured for the type of database we're using private IDatabaseAdapter adapter; + // The (plain) schema adapter which wraps the database adapter + private ISchemaAdapter schemaAdapter; + // The number of threads to use for the schema creation step private int createSchemaThreads = 1; @@ -671,6 +676,7 @@ public void setupDerbyRepository() { this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.connectionPool.setCloseOnAnyError(); this.adapter = new DerbyAdapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } @@ -734,7 +740,7 @@ protected VersionHistoryService createVersionHistoryService() { // Create the version history table if it doesn't yet exist try (ITransaction tx = transactionProvider.getTransaction()) { try { - CreateVersionHistory.createTableIfNeeded(schemaName, this.adapter); + CreateVersionHistory.createTableIfNeeded(schemaName, this.schemaAdapter); } catch (Exception x) { logger.log(Level.SEVERE, "failed to create version history table", x); tx.setRollbackOnly(); @@ -763,8 +769,8 @@ public void bootstrapDb() { try (ITransaction tx = transactionProvider.getTransaction()) { try { adapter.createSchema(schemaName); - CreateControl.createTableIfNeeded(schemaName, adapter); - CreateWholeSchemaVersion.createTableIfNeeded(schemaName, adapter); + CreateControl.createTableIfNeeded(schemaName, schemaAdapter); + CreateWholeSchemaVersion.createTableIfNeeded(schemaName, schemaAdapter); success = true; } catch (Exception x) { logger.log(Level.SEVERE, "failed to create schema management tables", x); @@ -825,7 +831,7 @@ private void buildSchema() { ExecutorService pool = Executors.newFixedThreadPool(this.createSchemaThreads); ITaskCollector collector = taskService.makeTaskCollector(pool); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.collect(collector, adapter, context, this.transactionProvider, vhs); + pdm.collect(collector, schemaAdapter, context, this.transactionProvider, vhs); // FHIR in the hole! logger.info("Starting schema updates"); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java index b854dc906a2..625a7de9347 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java @@ -58,7 +58,7 @@ import static com.ibm.fhir.bucket.persistence.SchemaConstants.VERSION; import com.ibm.fhir.bucket.app.Main; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.Generated; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -304,9 +304,11 @@ protected void addResourceBundleErrors(PhysicalDataModel pdm) { /** * Apply the model to the database. Will generate the DDL and execute it + * @param adapter + * @param context * @param pdm */ - protected void applyModel(IDatabaseAdapter adapter, SchemaApplyContext context, PhysicalDataModel pdm) { + protected void applyModel(ISchemaAdapter adapter, SchemaApplyContext context, PhysicalDataModel pdm) { pdm.apply(adapter, context); } } diff --git a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java index a81754e3171..0b2b98d23b2 100644 --- a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java +++ b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java @@ -53,9 +53,11 @@ import com.ibm.fhir.bucket.persistence.ResourceRec; import com.ibm.fhir.bucket.persistence.ResourceTypeRec; import com.ibm.fhir.bucket.persistence.ResourceTypesReader; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider; import com.ibm.fhir.database.utils.derby.DerbyMaster; @@ -369,7 +371,8 @@ protected VersionHistoryService createVersionHistoryService() throws SQLExceptio try { JdbcTarget target = new JdbcTarget(c); DerbyAdapter derbyAdapter = new DerbyAdapter(target); - CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, derbyAdapter); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(derbyAdapter); + CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, schemaAdapter); c.commit(); } catch (SQLException x) { logger.log(Level.SEVERE, "failed to create version history table", x); diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java index 7490e2bfb0c..2552a86c65b 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java @@ -83,7 +83,7 @@ public PartitionPlan mapPartitions() throws Exception { List target = new ArrayList<>(); try { for (String resourceType : resourceTypes) { - List resourceResults = fhirPersistence.changes(1, null, null, null, + List resourceResults = fhirPersistence.changes(null, 1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); // Early Exit Logic diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java index 817f49feae4..0e1277f56c2 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java @@ -78,7 +78,7 @@ public PartitionPlan mapPartitions() throws Exception { List target = new ArrayList<>(); try { for (String resourceType : resourceTypes) { - List resourceResults = fhirPersistence.changes(1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); + List resourceResults = fhirPersistence.changes(null, 1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); // Early Exit Logic if (!resourceResults.isEmpty()) { diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java index 2b3059577ec..14ae58a5d07 100644 --- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java +++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java @@ -25,6 +25,7 @@ public class FHIRConfiguration { // Core server properties public static final String PROPERTY_ORIGINAL_REQUEST_URI_HEADER_NAME = "fhirServer/core/originalRequestUriHeaderName"; public static final String PROPERTY_TENANT_ID_HEADER_NAME = "fhirServer/core/tenantIdHeaderName"; + public static final String PROPERTY_SHARD_KEY_HEADER_NAME = "fhirServer/core/shardKeyHeaderName"; public static final String PROPERTY_DATASTORE_ID_HEADER_NAME = "fhirServer/core/datastoreIdHeaderName"; public static final String PROPERTY_DEFAULT_TENANT_ID = "fhirServer/core/defaultTenantId"; public static final String PROPERTY_DEFAULT_PRETTY_PRINT = "fhirServer/core/defaultPrettyPrint"; @@ -109,6 +110,12 @@ public class FHIRConfiguration { public static final String PROPERTY_NATS_KEYSTORE = "fhirServer/notifications/nats/keystoreLocation"; public static final String PROPERTY_NATS_KEYSTORE_PW = "fhirServer/notifications/nats/keystorePassword"; + // Configuration properties for the Kafka-based async index service + public static final String PROPERTY_REMOTE_INDEX_SERVICE_TYPE = "fhirServer/remoteIndexService/type"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME = "fhirServer/remoteIndexService/kafka/topicName"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS = "fhirServer/remoteIndexService/kafka/connectionProperties"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_MODE = "fhirServer/remoteIndexService/kafka/mode"; + // Operations config properties public static final String PROPERTY_OPERATIONS_EVERYTHING = "fhirServer/operations/everything"; public static final String PROPERTY_OPERATIONS_EVERYTHING_INCLUDE_TYPES = "includeTypes"; @@ -132,6 +139,7 @@ public class FHIRConfiguration { public static final String DEFAULT_TENANT_ID_HEADER_NAME = "X-FHIR-TENANT-ID"; public static final String DEFAULT_DATASTORE_ID_HEADER_NAME = "X-FHIR-DSID"; public static final String DEFAULT_PRETTY_RESPONSE_HEADER_NAME = "X-FHIR-FORMATTED"; + public static final String DEFAULT_SHARD_KEY_HEADER_NAME = "X-FHIR-SHARD-KEY"; public static final String FHIR_SERVER_DEFAULT_CONFIG = "config/default/fhir-server-config.json"; diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java index f7f6d8888a7..2675254c682 100644 --- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java +++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java @@ -36,6 +36,10 @@ public class FHIRRequestContext { // The datastore to be used for this request. Usually "default" private String dataStoreId; + + // An optional shard key passed with the request for use with distributed schemas + private String requestShardKey; + private String requestUniqueId; private String originalRequestUri; private Map> httpHeaders; @@ -144,6 +148,24 @@ public void setDataStoreId(String dataStoreId) throws FHIRException { } } + /** + * Set the shard key string value provided by the request + * @param k + */ + public void setRequestShardKey(String k) { + this.requestShardKey = k; + } + + /** + * Get the shard key string value provided by the request. This value is + * not filtered in any way because the value eventually gets hashed into + * a short (2 byte integer number) before being used. + * @return + */ + public String getRequestShardKey() { + return this.requestShardKey; + } + /** * set an Operation Context property * @param name diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java new file mode 100644 index 00000000000..4475d9c37e7 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java @@ -0,0 +1,42 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * Carrier for the distribution context passed to some adapter methods + */ +public class DistributionContext { + // The type of distribution to be applied for a particular table + private final DistributionType distributionType; + // The column name to be used for distribution when the distributionType is DISTRIBUTED + private final String distributionColumnName; + + /** + * Public constructor + * @param distributionType + * @param distributionColumnName + */ + public DistributionContext(DistributionType distributionType, String distributionColumnName) { + this.distributionType = distributionType; + this.distributionColumnName = distributionColumnName; + } + + /** + * @return the distributionType + */ + public DistributionType getDistributionType() { + return distributionType; + } + + /** + * @return the distributionColumnName + */ + public String getDistributionColumnName() { + return distributionColumnName; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java deleted file mode 100644 index b710bccee86..00000000000 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionRules.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * (C) Copyright IBM Corp. 2022 - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.ibm.fhir.database.utils.api; - -import java.util.Collection; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Rules for distributing a table in a distributed RDBMS such as Citus - */ -public class DistributionRules { - - // If this table is distributed, which column is used for sharding - private final String distributionColumn; - // Is this table a reference table - private final boolean referenceTable; - - /** - * Public constructor - * @param distributionColumn - * @param referenceTable - */ - public DistributionRules(String distributionColumn, boolean referenceTable) { - this.distributionColumn = distributionColumn; - this.referenceTable = referenceTable; - if (this.referenceTable && this.distributionColumn != null) { - // variables are mutually exclusive - throw new IllegalArgumentException("Reference tables do not use a distributionColumn"); - } - } - - /** - * Getter for distributionColumn value - * @return - */ - public String getDistributionColumn() { - return this.distributionColumn; - } - - /** - * Getter for referenceTableValue - * @return - */ - public boolean isReferenceTable() { - return this.referenceTable; - } - - /** - * Is the table configured to be distributed (sharded) - * @return - */ - public boolean isDistributedTable() { - return this.distributionColumn != null; - } - - /** - * Asks if the distributionColumn is contained in the given collection of column names - - * @implNote case-insensitive - * @param columns - * @return - */ - public boolean includesDistributionColumn(Collection columns) { - if (this.distributionColumn != null) { - Set colSet = columns.stream().map(p -> p.toLowerCase()).collect(Collectors.toSet()); - return colSet.contains(this.distributionColumn.toLowerCase()); - } - return false; - } -} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java new file mode 100644 index 00000000000..0442b40e42a --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java @@ -0,0 +1,16 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + +/** + * The type of distribution to use for a table + */ +public enum DistributionType { + NONE, // table will not be distributed at all + REFERENCE, // table will be replicated + DISTRIBUTED // table will be sharded by a known column +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 898dfcaeb3b..292f6e56289 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -79,19 +79,19 @@ public interface IDatabaseAdapter { * @param tablespaceName * @param withs * @param checkConstraints - * @param distributionRules + * @param distributionContext */ public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, - DistributionRules distributionRules); + DistributionContext distributionContext); /** * Apply any distribution rules configured for the named table * @param schemaName * @param tableName - * @param distributionRules + * @param distributionContext */ - public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules); + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext); /** * Add a new column to an existing table @@ -167,10 +167,10 @@ public void createTable(String schemaName, String name, String tenantColumnName, * @param tenantColumnName * @param indexColumns * @param includeColumns - * @param distributionRules + * @param distributionContext */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, List includeColumns, DistributionRules distributionRules); + List indexColumns, List includeColumns, DistributionContext distributionContext); /** * Create a unique index @@ -179,10 +179,10 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN * @param indexName * @param tenantColumnName * @param indexColumns - * @param distributionRules + * @param distributionContext */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, DistributionRules distributionRules); + List indexColumns, DistributionContext distributionContext); /** * diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java new file mode 100644 index 00000000000..1e314e8050b --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java @@ -0,0 +1,611 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.ibm.fhir.database.utils.common.SchemaInfoObject; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.Privilege; +import com.ibm.fhir.database.utils.model.Table; +import com.ibm.fhir.database.utils.model.With; + +/** + * Adapter to create a particular flavor of the FHIR schema + */ +public interface ISchemaAdapter { + + /** + * Create a new tablespace with the given name + * + * @param tablespaceName + */ + public void createTablespace(String tablespaceName); + + /** + * Create a new tablespace using the given extent size + * + * @param tablespaceName + * @param extentSizeKB + */ + public void createTablespace(String tablespaceName, int extentSizeKB); + + /** + * Drop an existing tablespace, including all of its contents + * + * @param tablespaceName + */ + public void dropTablespace(String tablespaceName); + + /** + * Detach the partition + * + * @param schemaName + * @param tableName + * @param partitionName + * @param newTableName + */ + public void detachPartition(String schemaName, String tableName, String partitionName, String newTableName); + + /** + * Build the create table DDL + * + * @param schemaName + * @param name + * @param tenantColumnName optional column name to enable multi-tenancy + * @param columns + * @param primaryKey + * @param identity + * @param tablespaceName + * @param withs + * @param checkConstraints + * @param distributionRules + */ + public void createTable(String schemaName, String name, String tenantColumnName, List columns, + PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionType distributionRules); + + /** + * Apply any distribution rules configured for the named table + * @param schemaName + * @param tableName + * @param distributionRules + */ + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionRules); + + /** + * Add a new column to an existing table + * @param schemaName + * @param tableName + * @param column + */ + public void alterTableAddColumn(String schemaName, String tableName, ColumnBase column); + + /** + * Reorg the table if the underlying database supports it. Required after + * columns are added/removed from a table. + * @param schemaName + * @param tableName + */ + public void reorgTable(String schemaName, String tableName); + + /** + * Create ROW type used for passing values to stored procedures e.g.: + * + * 
+     * CREATE OR REPLACE TYPE .t_str_values AS ROW (parameter_name_id INTEGER,
+     * str_value VARCHAR(511 OCTETS), str_value_lcase VARCHAR(511 OCTETS))
+     *
+ * + * @param schemaName + * @param typeName + * @param columns + */ + public void createRowType(String schemaName, String typeName, List columns); + + /** + * Create ARRAY type used for passing values to stored procedures e.g.: CREATE + * OR REPLACE TYPE .t_str_values_arr AS .t_str_values ARRAY[256] + * + * @param schemaName + * @param typeName + * @param valueType + * @param arraySize + */ + public void createArrType(String schemaName, String typeName, String valueType, int arraySize); + + /** + * Drop the type object from the schema + * + * @param schemaName + * @param typeName + */ + public void dropType(String schemaName, String typeName); + + /** + * Create the stored procedure using the DDL text provided by the supplier + * + * @param schemaName + * @param procedureName + * @param supplier + */ + public void createOrReplaceProcedure(String schemaName, String procedureName, Supplier supplier); + + /** + * Drop the given procedure + * + * @param schemaName + * @param procedureName + */ + public void dropProcedure(String schemaName, String procedureName); + + /** + * Create a unique index + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param includeColumns + * @param distributionRules + */ + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, List includeColumns, DistributionType distributionRules); + + /** + * Create a unique index + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param distributionRules + */ + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionType distributionRules); + + /** + * Create an index on the named schema.table object + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param distributionType + */ + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionType distributionType); + + /** + * + * 
+     * CREATE VARIABLE ptng.session_tenant INT DEFAULT NULL;
+     *
+ * + * @param schemaName + * @param variableName + */ + public void createIntVariable(String schemaName, String variableName); + + /** + * + * 
+     * CREATE OR REPLACE PERMISSION ROW_ACCESS ON ptng.patients FOR ROWS WHERE patients.mt_id =
+     * ptng.session_tenant ENFORCED FOR ALL ACCESS ENABLE;
+     *
+ * + * @param schemaName + * @param permissionName + * @param tableName + * @param predicate + */ + public void createOrReplacePermission(String schemaName, String permissionName, String tableName, String predicate); + + /** + * + * + * 
 ALTER TABLE  ACTIVATE ROW ACCESS CONTROL
+     *
+ * + * @param schemaName + * @param tableName + */ + public void activateRowAccessControl(String schemaName, String tableName); + + /** + * Deactivate row access control on a table ALTER TABLE DEACTIVATE ROW + * ACCESS CONTROL + * + * @param schemaName + * @param tableName + */ + public void deactivateRowAccessControl(String schemaName, String tableName); + + /** + * Build the DML statement for setting a session variable + * + * @param schemaName + * @param variableName + * @param value + */ + public void setIntVariable(String schemaName, String variableName, int value); + + /** + * Drop table from the schema + * + * @param schemaName + * @param name + */ + public void dropTable(String schemaName, String name); + + /** + * Drop permission object from the schema + * + * @param schemaName + * @param permissionName + */ + public void dropPermission(String schemaName, String permissionName); + + /** + * @param schemaName + * @param variableName + */ + public void dropVariable(String schemaName, String variableName); + + /** + * + * @param constraintName + * @param schemaName + * @param name + * @param targetSchema + * @param targetTable + * @param targetColumnName + * @param tenantColumnName + * @param columns + * @param enforced + * @param distributionType distribution type of the source table + * @param targetIsReference + */ + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, + String targetTable, String targetColumnName, String tenantColumnName, List columns, + boolean enforced, DistributionType distributionType, boolean targetIsReference); + + /** + * Allocate a new tenant + * + * @param adminSchemaName + * @param schemaName + * @param tenantName + * @param tenantKey + * @param tenantSalt + * @param idSequenceName + * @return + */ + public int allocateTenant(String adminSchemaName, String schemaName, String tenantName, String tenantKey, + String tenantSalt, String idSequenceName); + + /** + * Delete all the metadata associated with the given tenant identifier, as long as the + * tenant status is DROPPED. + * @param tenantId + */ + public void deleteTenantMeta(String adminSchemaName, int tenantId); + + /** + * Get the tenant id for the given schema and tenant name + * + * @param adminSchemaName + * @param tenantName + * @return + */ + public int findTenantId(String adminSchemaName, String tenantName); + + /** + * Create the partitions on each of these tables + * + * @param tables + * @param schemaName + * @param newTenantId + * @param extentSizeKB + */ + public void createTenantPartitions(Collection tables, String schemaName, int newTenantId, int extentSizeKB); + + /** + * Add a new tenant partition to each of the tables in the collection. Idempotent, so can + * be run to add partitions for existing tenants to new tables + * @param tables + * @param schemaName + * @param newTenantId + */ + public void addNewTenantPartitions(Collection
tables, String schemaName, int newTenantId); + + /** + * Detach the partition associated with the tenantId from each of the given tables + * + * @param tables + * @param schemaName + * @param tenantId + * @param tenantStagingTable + */ + public void removeTenantPartitions(Collection
tables, String schemaName, int tenantId); + + /** + * Drop the tables which were created by the detach partition operation (as + * part of tenant deprovisioning). + * @param tables + * @param schemaName + * @param tenantId + */ + public void dropDetachedPartitions(Collection
tables, String schemaName, int tenantId); + + /** + * Update the tenant status + * + * @param adminSchemaName + * @param tenantId + * @param status + */ + public void updateTenantStatus(String adminSchemaName, int tenantId, TenantStatus status); + + /** + * + * @param schemaName + * @param sequenceName + * @param startWith the START WITH value for the sequence + * @param cache the sequence CACHE value + */ + public void createSequence(String schemaName, String sequenceName, long startWith, int cache, int incrementBy); + + /** + * + * @param schemaName + * @param sequenceName + */ + public void dropSequence(String schemaName, String sequenceName); + + /** + * Sets/resets the sequence to start with the given value. + * @param schemaName + * @param sequenceName + * @param restartWith + * @param cache + */ + public void alterSequenceRestartWith(String schemaName, String sequenceName, long restartWith, int cache, int incrementBy); + + /** + * Grant the list of privileges on the named object to the user. This is a + * general purpose method which can be used to specify privileges for any object + * type which doesn't need the object type to be specified in the grant DDL. + * + * @param schemaName + * @param tableName + * @param privileges + * @param toUser + */ + public void grantObjectPrivileges(String schemaName, String tableName, Collection privileges, String toUser); + + /** + * Grant the collection of privileges on the named procedure to the user + * + * @param schemaName + * @param procedureName + * @param privileges + * @param toUser + */ + public void grantProcedurePrivileges(String schemaName, String procedureName, Collection privileges, + String toUser); + + /** + * Grant the collection of privileges on the named variable to the user + * + * @param schemaName + * @param variableName + * @param privileges + * @param toUser + */ + public void grantVariablePrivileges(String schemaName, String variableName, Collection privileges, + String toUser); + + /** + * Grant the collection of privileges on the named variable to the user + * + * @param schemaName + * @param objectName + * @param group + * @param toUser + */ + public void grantSequencePrivileges(String schemaName, String objectName, Collection group, + String toUser); + + /** + * Grants USAGE on the given schemaName to the given user + * @param schemaName + */ + public void grantSchemaUsage(String schemaName, String grantToUser); + + /** + * Grant access to all sequences in the named schema + * @param schemaName + * @param grantToUser + */ + public void grantAllSequenceUsage(String schemaName, String grantToUser); + + /** + * Check if the table currently exists + * + * @param schemaName + * @param objectName + * @return + */ + public boolean doesTableExist(String schemaName, String objectName); + + /** + * Create a database schema + * + * @param schemaName + */ + public void createSchema(String schemaName); + + /** + * create a unique constraint on a table. + * + * @param constraintName + * @param columns + * @param schemaName + * @param name + */ + public void createUniqueConstraint(String constraintName, List columns, String schemaName, String name); + + /** + * checks connectivity to the database and that it is compatible + * @param adminSchema + * @return + */ + public boolean checkCompatibility(String adminSchema); + + /** + * + * @return a false, if not used, or true if used with the persistence layer. + */ + public default boolean useSessionVariable() { + return false; + } + + /** + * creates or replaces the SQL function + * @param schemaName + * @param objectName + * @param supplier + */ + public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier); + + /** + * For Citus, functions can be distributed by one of their parameters (typically the first) + * @param schemaName + * @param functionName + * @param distributeByParamNumber + */ + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber); + + /** + * drops a given function + * @param schemaName + * @param functionName + */ + public void dropFunction(String schemaName, String functionName); + + /** + * grants permissions on a given function + * @param schemaName + * @param functionName + * @param privileges + * @param toUser + */ + public void grantFunctionPrivileges(String schemaName, String functionName, Collection privileges, String toUser); + + /** + * Drop the tablespace associated with the given tenantId + * @param tenantId + */ + public void dropTenantTablespace(int tenantId); + + /** + * Disable the FK with the given constraint name + * @param tableName + * @param constraintName + */ + public void disableForeignKey(String schemaName, String tableName, String constraintName); + + /** + * Drop the FK on the table with the given constraint name + * @param schemaName + * @param tableName + * @param constraintName + */ + public void dropForeignKey(String schemaName, String tableName, String constraintName); + + /** + * Enable the FK with the given constraint name + * @param schemaName + * @param tableName + * @param constraintName + */ + public void enableForeignKey(String schemaName, String tableName, String constraintName); + + /** + * + * @param schemaName + * @param tableName + */ + public void setIntegrityOff(String schemaName, String tableName); + + /** + * + * @param schemaName + * @param tableName + */ + public void setIntegrityUnchecked(String schemaName, String tableName); + + /** + * Change the CACHE value of the named identity generated always column + * @param schemaName + * @param objectName + * @param columnName + * @param cache + */ + public void alterTableColumnIdentityCache(String schemaName, String objectName, String columnName, int cache); + + /** + * Drop the named index + * @param schemaName + * @param indexName + */ + public void dropIndex(String schemaName, String indexName); + + /** + * Create the view as defined by the selectClause + * @param schemaName + * @param objectName + * @param selectClause + */ + public void createView(String schemaName, String objectName, String selectClause); + + /** + * Drop the view from the database + * @param schemaName + * @param objectName + */ + public void dropView(String schemaName, String objectName); + + /** + * Create or replace the view + * @param schemaName + * @param objectName + * @param selectClause + */ + public void createOrReplaceView(String schemaName, String objectName, String selectClause); + + /** + * List the objects present in the given schema + * @param schemaName + * @return + */ + List listSchemaObjects(String schemaName); + + /** + * Run the given statement against the database represented by this adapter + * + * @param statement + */ + public void runStatement(IDatabaseStatement statement); +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java new file mode 100644 index 00000000000..bd85c6d7e6c --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java @@ -0,0 +1,20 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * The flavor of database schema + * PLAIN - the schema we typically deploy to Derby or PostgreSQL + * MULTITENANT - on Db2 supporting multiple tenants using partitioning and RBAC + * DISTRIBUTED - for use with distributed technologies like Citus DB + */ +public enum SchemaType { + PLAIN, + MULTITENANT, + DISTRIBUTED +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java index 37c2a8644c1..e23fbaa4277 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -10,12 +10,14 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.List; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTarget; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; @@ -53,16 +55,21 @@ public CitusAdapter(IConnectionProvider cp) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, - DistributionRules distributionRules) { + DistributionContext distributionContext) { // We don't use partitioning for multi-tenancy in our Citus implementation, so ignore the mt_id column if (tenantColumnName != null) { warnOnce(MessageKey.MULTITENANCY, "Citus does not support multi-tenancy: " + name); } - // Build a Citus-specific create table statement - String ddl = buildCitusCreateTableStatement(schemaName, name, columns, primaryKey, identity, withs, checkConstraints, distributionRules); - runStatement(ddl); + if (distributionContext != null) { + // Build a Citus-specific create table statement + String ddl = buildCitusCreateTableStatement(schemaName, name, columns, primaryKey, identity, withs, checkConstraints, distributionContext); + runStatement(ddl); + } else { + // building a plain schema, so we can use the standard PostgreSQL method + super.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, distributionContext); + } } /** @@ -75,14 +82,16 @@ public void createTable(String schemaName, String name, String tenantColumnName, * @param identity * @param withs * @param checkConstraints - * @param distributionRules + * @param distributionType * @return */ private String buildCitusCreateTableStatement(String schema, String name, List columns, PrimaryKeyDef pkDef, IdentityDef identity, List withs, - List checkConstraints, DistributionRules distributionRules) { + List checkConstraints, DistributionContext distributionContext) { - if (identity != null && distributionRules != null && distributionRules.getDistributionColumn() != null) { + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); + if (identity != null && distributionType == DistributionType.DISTRIBUTED) { logger.warning("Citus: Ignoring IDENTITY columns on distributed table: '" + name + "." + identity.getColumnName()); identity = null; } @@ -100,7 +109,7 @@ private String buildCitusCreateTableStatement(String schema, String name, List pkSet = pkDef.getColumns().stream().map(c -> c.toLowerCase()).collect(Collectors.toSet()); - final String ldc = distributionRules == null || distributionRules.getDistributionColumn() == null ? null : distributionRules.getDistributionColumn().toLowerCase(); + final String ldc = distributionType == DistributionType.DISTRIBUTED ? distributionColumnName.toLowerCase() : null; if (ldc == null || pkSet.contains(ldc)) { result.append(", CONSTRAINT "); result.append(pkDef.getConstraintName()); @@ -142,11 +151,13 @@ private String buildCitusCreateTableStatement(String schema, String name, List indexColumns, DistributionRules distributionRules) { + List indexColumns, DistributionContext distributionContext) { // For Citus, we are prevented from creating a unique index unless the index contains // the distribution column + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); List columnNames = indexColumns.stream().map(ocd -> ocd.getColumnName()).collect(Collectors.toList()); - if (distributionRules != null && distributionRules.isDistributedTable() && !distributionRules.includesDistributionColumn(columnNames)) { + if (distributionType == DistributionType.DISTRIBUTED && !includesDistributionColumn(distributionColumnName, columnNames)) { // Can only a normal index because it isn't partitioned by the distributionColumn String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); runStatement(ddl); @@ -158,21 +169,23 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN } @Override - public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules) { + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext) { // Apply the distribution rules. Tables without distribution rules are created // only on Citus controller nodes and never distributed to the worker nodes. All // the distribution changes are implemented in one transaction, which makes it much // more efficient. + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); final String fullName = DataDefinitionUtil.getQualifiedName(schemaName, tableName); - if (distributionRules.isReferenceTable()) { + if (distributionType == DistributionType.REFERENCE) { // A table that is fully replicated for each worker node logger.info("Citus: distributing reference table '" + fullName + "'"); CreateReferenceTableDAO dao = new CreateReferenceTableDAO(schemaName, tableName); runStatement(dao); - } else if (distributionRules.getDistributionColumn() != null && distributionRules.getDistributionColumn().length() > 0) { + } else if (distributionType == DistributionType.DISTRIBUTED) { // A table that is sharded using a hash on the distributionColumn value - logger.info("Citus: Sharding table '" + fullName + "' using '" + distributionRules.getDistributionColumn() + "'"); - CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, tableName, distributionRules.getDistributionColumn()); + logger.info("Citus: Sharding table '" + fullName + "' using '" + distributionColumnName + "'"); + CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, tableName, distributionColumnName); runStatement(dao); } } @@ -218,4 +231,20 @@ public void distributeFunction(String schemaName, String functionName, int distr throw new IllegalStateException("distributeFunction requires a connectionProvider"); } } + + /** + * Asks if the distributionColumnName is contained in the given collection of column names + * + * @implNote case-insensitive + * @param distributionColumnName + * @param columns + * @return + */ + public boolean includesDistributionColumn(String distributionColumnName, Collection columns) { + if (distributionColumnName != null) { + Set colSet = columns.stream().map(p -> p.toLowerCase()).collect(Collectors.toSet()); + return colSet.contains(distributionColumnName.toLowerCase()); + } + return false; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java index daf22ca9348..40cde9f3997 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java @@ -19,7 +19,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseStatement; @@ -198,7 +198,7 @@ protected String buildCreateTableStatement(String schema, String name, List indexColumns, List includeColumns, DistributionRules distributionRules) { + List indexColumns, List includeColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, includeColumns, true); runStatement(ddl); @@ -206,7 +206,7 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, DistributionRules distributionRules) { + List indexColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, true); runStatement(ddl); @@ -750,7 +750,7 @@ public void reorgTable(String schemaName, String tableName) { } @Override - public void applyDistributionRules(String schemaName, String tableName, DistributionRules distributionRules) { + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext) { // NOP. Only used for distributed databases like Citus } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java new file mode 100644 index 00000000000..e2690e72516 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java @@ -0,0 +1,366 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.TenantStatus; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.Privilege; +import com.ibm.fhir.database.utils.model.Table; +import com.ibm.fhir.database.utils.model.With; + + +/** + * Adapter to build the plain version of the FHIR schema. Uses + * the IDatabaseAdapter to hide the specifics of a particular + * database flavor (like Db2, PostgreSQL, Derby etc). + */ +public class PlainSchemaAdapter implements ISchemaAdapter { + + // The adapter we use to execute database-specific DDL + protected final IDatabaseAdapter databaseAdapter; + + /** + * Public constructor + * + * @param databaseAdapter + */ + public PlainSchemaAdapter(IDatabaseAdapter databaseAdapter) { + this.databaseAdapter = databaseAdapter; + } + + @Override + public void createTablespace(String tablespaceName) { + databaseAdapter.createTablespace(tablespaceName); + } + + @Override + public void createTablespace(String tablespaceName, int extentSizeKB) { + databaseAdapter.createTablespace(tablespaceName, extentSizeKB); + } + + @Override + public void dropTablespace(String tablespaceName) { + databaseAdapter.dropTablespace(tablespaceName); + } + + @Override + public void detachPartition(String schemaName, String tableName, String partitionName, String newTableName) { + databaseAdapter.detachPartition(schemaName, tableName, partitionName, newTableName); + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionRules) { + databaseAdapter.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, null); + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionRules) { + databaseAdapter.applyDistributionRules(schemaName, tableName, null); + } + + @Override + public void alterTableAddColumn(String schemaName, String tableName, ColumnBase column) { + databaseAdapter.alterTableAddColumn(schemaName, tableName, column); + } + + @Override + public void reorgTable(String schemaName, String tableName) { + databaseAdapter.reorgTable(schemaName, tableName); + } + + @Override + public void createRowType(String schemaName, String typeName, List columns) { + databaseAdapter.createRowType(schemaName, typeName, columns); + } + + @Override + public void createArrType(String schemaName, String typeName, String valueType, int arraySize) { + databaseAdapter.createArrType(schemaName, typeName, valueType, arraySize); + } + + @Override + public void dropType(String schemaName, String typeName) { + databaseAdapter.dropType(schemaName, typeName); + } + + @Override + public void createOrReplaceProcedure(String schemaName, String procedureName, Supplier supplier) { + databaseAdapter.createOrReplaceProcedure(schemaName, procedureName, supplier); + } + + @Override + public void dropProcedure(String schemaName, String procedureName) { + databaseAdapter.dropProcedure(schemaName, procedureName); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionRules) { + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, null); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionRules) { + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, null); + } + + @Override + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { + databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + } + + @Override + public void createIntVariable(String schemaName, String variableName) { + databaseAdapter.createIntVariable(schemaName, variableName); + } + + @Override + public void createOrReplacePermission(String schemaName, String permissionName, String tableName, String predicate) { + databaseAdapter.createOrReplacePermission(schemaName, permissionName, tableName, predicate); + } + + @Override + public void activateRowAccessControl(String schemaName, String tableName) { + databaseAdapter.activateRowAccessControl(schemaName, tableName); + } + + @Override + public void deactivateRowAccessControl(String schemaName, String tableName) { + databaseAdapter.deactivateRowAccessControl(schemaName, tableName); + } + + @Override + public void setIntVariable(String schemaName, String variableName, int value) { + databaseAdapter.setIntVariable(schemaName, variableName, value); + } + + @Override + public void dropTable(String schemaName, String name) { + databaseAdapter.dropTable(schemaName, name); + } + + @Override + public void dropPermission(String schemaName, String permissionName) { + databaseAdapter.dropPermission(schemaName, permissionName); + } + + @Override + public void dropVariable(String schemaName, String variableName) { + databaseAdapter.dropVariable(schemaName, variableName); + } + + @Override + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, + String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { + databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced); + } + + @Override + public int allocateTenant(String adminSchemaName, String schemaName, String tenantName, String tenantKey, String tenantSalt, String idSequenceName) { + return databaseAdapter.allocateTenant(adminSchemaName, schemaName, tenantName, tenantKey, tenantSalt, idSequenceName); + } + + @Override + public void deleteTenantMeta(String adminSchemaName, int tenantId) { + databaseAdapter.deleteTenantMeta(adminSchemaName, tenantId); + } + + @Override + public int findTenantId(String adminSchemaName, String tenantName) { + return databaseAdapter.findTenantId(adminSchemaName, tenantName); + } + + @Override + public void createTenantPartitions(Collection
tables, String schemaName, int newTenantId, int extentSizeKB) { + databaseAdapter.createTenantPartitions(tables, schemaName, newTenantId, extentSizeKB); + } + + @Override + public void addNewTenantPartitions(Collection
tables, String schemaName, int newTenantId) { + databaseAdapter.addNewTenantPartitions(tables, schemaName, newTenantId); + } + + @Override + public void removeTenantPartitions(Collection
tables, String schemaName, int tenantId) { + databaseAdapter.removeTenantPartitions(tables, schemaName, tenantId); + } + + @Override + public void dropDetachedPartitions(Collection
tables, String schemaName, int tenantId) { + databaseAdapter.dropDetachedPartitions(tables, schemaName, tenantId); + } + + @Override + public void updateTenantStatus(String adminSchemaName, int tenantId, TenantStatus status) { + databaseAdapter.updateTenantStatus(adminSchemaName, tenantId, status); + } + + @Override + public void createSequence(String schemaName, String sequenceName, long startWith, int cache, int incrementBy) { + databaseAdapter.createSequence(schemaName, sequenceName, startWith, cache, incrementBy); + } + + @Override + public void dropSequence(String schemaName, String sequenceName) { + databaseAdapter.dropSequence(schemaName, sequenceName); + } + + @Override + public void alterSequenceRestartWith(String schemaName, String sequenceName, long restartWith, int cache, int incrementBy) { + databaseAdapter.alterSequenceRestartWith(schemaName, sequenceName, restartWith, cache, incrementBy); + } + + @Override + public void grantObjectPrivileges(String schemaName, String tableName, Collection privileges, String toUser) { + databaseAdapter.grantObjectPrivileges(schemaName, tableName, privileges, toUser); + } + + @Override + public void grantProcedurePrivileges(String schemaName, String procedureName, Collection privileges, String toUser) { + databaseAdapter.grantProcedurePrivileges(schemaName, procedureName, privileges, toUser); + } + + @Override + public void grantVariablePrivileges(String schemaName, String variableName, Collection privileges, String toUser) { + databaseAdapter.grantVariablePrivileges(schemaName, variableName, privileges, toUser); + } + + @Override + public void grantSequencePrivileges(String schemaName, String objectName, Collection group, String toUser) { + databaseAdapter.grantSequencePrivileges(schemaName, objectName, group, toUser); + } + + @Override + public void grantSchemaUsage(String schemaName, String grantToUser) { + databaseAdapter.grantSchemaUsage(schemaName, grantToUser); + + } + + @Override + public void grantAllSequenceUsage(String schemaName, String grantToUser) { + databaseAdapter.grantAllSequenceUsage(schemaName, grantToUser); + } + + @Override + public boolean doesTableExist(String schemaName, String objectName) { + return databaseAdapter.doesTableExist(schemaName, objectName); + } + + @Override + public void createSchema(String schemaName) { + databaseAdapter.createSchema(schemaName); + } + + @Override + public void createUniqueConstraint(String constraintName, List columns, String schemaName, String name) { + databaseAdapter.createUniqueConstraint(constraintName, columns, schemaName, name); + } + + @Override + public boolean checkCompatibility(String adminSchema) { + return databaseAdapter.checkCompatibility(adminSchema); + } + + @Override + public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier) { + databaseAdapter.createOrReplaceFunction(schemaName, objectName, supplier); + } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + databaseAdapter.distributeFunction(schemaName, functionName, distributeByParamNumber); + } + + @Override + public void dropFunction(String schemaName, String functionName) { + databaseAdapter.dropFunction(schemaName, functionName); + } + + @Override + public void grantFunctionPrivileges(String schemaName, String functionName, Collection privileges, String toUser) { + databaseAdapter.grantFunctionPrivileges(schemaName, functionName, privileges, toUser); + } + + @Override + public void dropTenantTablespace(int tenantId) { + databaseAdapter.dropTenantTablespace(tenantId); + } + + @Override + public void disableForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.disableForeignKey(schemaName, tableName, constraintName); + } + + @Override + public void dropForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.dropForeignKey(schemaName, tableName, constraintName); + } + + @Override + public void enableForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.enableForeignKey(schemaName, tableName, constraintName); + } + + @Override + public void setIntegrityOff(String schemaName, String tableName) { + databaseAdapter.setIntegrityOff(schemaName, tableName); + } + + @Override + public void setIntegrityUnchecked(String schemaName, String tableName) { + databaseAdapter.setIntegrityUnchecked(schemaName, tableName); + } + + @Override + public void alterTableColumnIdentityCache(String schemaName, String objectName, String columnName, int cache) { + databaseAdapter.alterTableColumnIdentityCache(schemaName, objectName, columnName, cache); + } + + @Override + public void dropIndex(String schemaName, String indexName) { + databaseAdapter.dropIndex(schemaName, indexName); + } + + @Override + public void createView(String schemaName, String objectName, String selectClause) { + databaseAdapter.createView(schemaName, objectName, selectClause); + } + + @Override + public void dropView(String schemaName, String objectName) { + databaseAdapter.dropView(schemaName, objectName); + } + + @Override + public void createOrReplaceView(String schemaName, String objectName, String selectClause) { + databaseAdapter.createOrReplaceView(schemaName, objectName, selectClause); + } + + @Override + public List listSchemaObjects(String schemaName) { + return databaseAdapter.listSchemaObjects(schemaName); + } + + @Override + public void runStatement(IDatabaseStatement statement) { + databaseAdapter.runStatement(statement); + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java index f2a2c1408c5..95549cbd386 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.DataAccessException; -import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -74,7 +74,7 @@ public Db2Adapter() { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, - DistributionRules distributionRules) { + DistributionContext distributionContext) { // With DB2 we can implement support for multi-tenancy, which we do by injecting a MT_ID column // to the definition and partitioning on that column diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index 1201c155aaf..33790279c78 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -15,7 +15,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -81,7 +81,7 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, - DistributionRules distributionRules) { + DistributionContext distributionContext) { // Derby doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { warnOnce(MessageKey.MULTITENANCY, "Derby does not support multi-tenancy on: [" + name + "]"); @@ -94,10 +94,10 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns, DistributionRules distributionRules) { + List includeColumns, DistributionContext distributionContext) { // Derby doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionContext); } @Override diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java index c0921a2f53c..f9e4ba0c308 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java @@ -22,11 +22,13 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.ConnectionProviderTarget; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -234,7 +236,7 @@ public void createSchema(IConnectionProvider pool, IVersionHistoryService vhs, P * @param pool provides database connections * @param fn the command to execute */ - public void runWithAdapter(IConnectionProvider pool, Consumer fn) { + public void runWithAdapter(IConnectionProvider pool, Consumer fn) { // We need to obtain connections from the same pool as the version history service // so we can avoid deadlocks for certain DDL like DROP INDEX @@ -244,7 +246,7 @@ public void runWithAdapter(IConnectionProvider pool, Consumer // call the Function we've been given using the adapter we just wrapped // around the connection. - fn.accept(adapter); + fn.accept(wrap(adapter)); } catch (DataAccessException x) { logger.log(Level.SEVERE, "Error while running", x); throw x; @@ -257,7 +259,7 @@ public void runWithAdapter(IConnectionProvider pool, Consumer * * @param fn */ - public void runWithAdapter(java.util.function.Consumer fn) { + public void runWithAdapter(java.util.function.Consumer fn) { IConnectionProvider cp = new DerbyConnectionProvider(this, null); ConnectionProviderTarget target = new ConnectionProviderTarget(cp); @@ -272,7 +274,18 @@ public void runWithAdapter(java.util.function.Consumer fn) { // call the Function we've been given using the adapter we just wrapped // around the connection. Each statement executes in its own connection/transaction. - fn.accept(adapter); + // We also need to wrap the DerbyAdapter in a plain schema adapter + fn.accept(wrap(adapter)); + } + + /** + * Utility method to wrap the database adapter in a plain schema adapter + * which acts as a pass-through to the underlying databaseAdapter + * @param databaseAdapter + * @return + */ + public static ISchemaAdapter wrap(IDatabaseAdapter databaseAdapter) { + return new PlainSchemaAdapter(databaseAdapter); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java index 6ae473e02cd..58346d69130 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java @@ -8,7 +8,7 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -40,22 +40,22 @@ public AlterSequenceStartWith(String schemaName, String sequenceName, int versio } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.alterSequenceRestartWith(getSchemaName(), getObjectName(), startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP. Sequence will be dropped by the object initially creating it } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantSequencePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java index 4acfdd0d68d..92c6ef64df5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java @@ -11,7 +11,7 @@ import java.util.List; import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -62,7 +62,7 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // To keep things simple, just add each column in its own statement for (ColumnBase c: columns) { target.alterTableAddColumn(getSchemaName(), getObjectName(), c); @@ -70,17 +70,17 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { // NOP } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java index 7ff6328b185..81f3ac67d4c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java @@ -8,7 +8,7 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -45,22 +45,22 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.alterTableColumnIdentityCache(getSchemaName(), getObjectName(), this.columnName, this.cache); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { // NOP } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java index 6706f8b2808..709518f2d27 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java @@ -19,7 +19,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; @@ -187,7 +187,7 @@ public String toString() { } @Override - public ITaskGroup collect(final ITaskCollector tc, final IDatabaseAdapter target, final SchemaApplyContext context, final ITransactionProvider tp, final IVersionHistoryService vhs) { + public ITaskGroup collect(final ITaskCollector tc, final ISchemaAdapter target, final SchemaApplyContext context, final ITransactionProvider tp, final IVersionHistoryService vhs) { // Make sure that anything we depend on gets processed first List children = null; if (!this.dependencies.isEmpty()) { @@ -204,7 +204,7 @@ public ITaskGroup collect(final ITaskCollector tc, final IDatabaseAdapter target } @Override - public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -251,7 +251,7 @@ public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransa } @Override - public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Only for Procedures do we skip the Version History Service check, and apply. if (vhs.applies(getSchemaName(), getObjectType().name(), getObjectName(), version) || getObjectType() == DatabaseObjectType.PROCEDURE) { @@ -278,7 +278,7 @@ public Map getTags() { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // The group is optional. Some objects may not have a group corresponding with // the requested groupName, in which case no privileges will be granted @@ -295,7 +295,7 @@ public void grant(IDatabaseAdapter target, String groupName, String toUser) { * @param group * @param toUser */ - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantObjectPrivileges(this.schemaName, this.objectName, group, toUser); } @@ -320,7 +320,7 @@ public void visit(Consumer c) { } @Override - public void applyDistributionRules(IDatabaseAdapter target, int pass) { + public void applyDistributionRules(ISchemaAdapter target, int pass) { // NOP. Only applies to Table } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java index d6ac9edb090..e4cc22806b5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java @@ -209,6 +209,9 @@ public List buildColumns() { case SMALLINT: column = new SmallIntColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); break; + case SMALLINT_BOOLEAN: + column = new SmallIntBooleanColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); + break; case DOUBLE: column = new DoubleColumn(cd.getName(), cd.isNullable()); break; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java index 4a74fb17ecc..2b91bf0b37f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java @@ -20,5 +20,6 @@ public enum ColumnType { TIMESTAMP, BLOB, CLOB, - SMALLINT + SMALLINT, + SMALLINT_BOOLEAN // for JavaBatch } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java index c7cfca6b638..98649f37d44 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java @@ -11,8 +11,8 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.DistributionRules; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.CreateIndexStatement; @@ -33,22 +33,22 @@ public class CreateIndex extends BaseObject { private final String tableName; // Distribution rules if the associated table is distributed - private final DistributionRules distributionRules; + private final DistributionType distributionType; /** * Protected constructor. Use the Builder to create instance. * @param schemaName * @param indexName * @param version - * @distributionRules + * @param distributionType */ protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName, - DistributionRules distributionRules) { + DistributionType distributionType) { super(schemaName, versionTrackingName, DatabaseObjectType.INDEX, version); this.tableName = tableName; this.indexDef = indexDef; this.tenantColumnName = tenantColumnName; - this.distributionRules = distributionRules; + this.distributionType = distributionType; } /** @@ -93,9 +93,9 @@ public String getTypeNameVersion() { @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { long start = System.nanoTime(); - indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionRules); + indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionType); if (logger.isLoggable(Level.FINE)) { long end = System.nanoTime(); @@ -105,12 +105,12 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { long start = System.nanoTime(); indexDef.drop(getSchemaName(), target); @@ -161,10 +161,7 @@ public static class Builder { private String versionTrackingName; // Set if the table is distributed - private String distributionColumn; - - // Set if the table is a distributed reference type table - private boolean distributionReference; + private DistributionType distributionType = DistributionType.NONE; /** * @param schemaName the schemaName to set @@ -198,22 +195,12 @@ public Builder setVersionTrackingName(String name) { } /** - * Setter for distributionColumn - * @param name + * Setter for distributionType + * @param dt * @return */ - public Builder setDistributionColumn(String name) { - this.distributionColumn = name; - return this; - } - - /** - * Setter for distributionReference - * @param flag - * @return - */ - public Builder setDistributionReference(boolean flag) { - this.distributionReference = flag; + public Builder setDistributionType(DistributionType dt) { + this.distributionType = dt; return this; } @@ -269,13 +256,9 @@ public CreateIndex build() { if (versionTrackingName == null) { versionTrackingName = this.indexName; } - DistributionRules distributionRules = null; - if (this.distributionReference || this.distributionColumn != null) { - distributionRules = new DistributionRules(distributionColumn, distributionReference); - } return new CreateIndex(schemaName, versionTrackingName, tableName, version, - new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionRules); + new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionType); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java index b7331e941c0..3c2707443ac 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java @@ -17,7 +17,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; @@ -142,7 +142,7 @@ public String toString() { } @Override - public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -193,7 +193,7 @@ public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransa * @param vhs the service used to manage the version history table */ @Override - public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // TODO find a better way to track database-level type stuff (not schema-specific) if (vhs.applies("__DATABASE__", getObjectType().name(), getObjectName(), version)) { logger.info("Applying change [v" + version + "]: "+ this.getTypeNameVersion()); @@ -218,7 +218,7 @@ public void visit(Consumer c) { } @Override - public void applyDistributionRules(IDatabaseAdapter target, int pass) { + public void applyDistributionRules(ISchemaAdapter target, int pass) { // NOP } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java index 8c383b3227c..3975c3a4f88 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -10,7 +10,8 @@ import java.util.Arrays; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -25,6 +26,9 @@ public class ForeignKeyConstraint extends Constraint { private final String targetColumnName; private final List columns = new ArrayList<>(); + // Flag to indicate that the target is a REFERENCE table when using distribution (like Citus) + private boolean targetIsReference; + /** * @param constraintName * @param enforced @@ -68,6 +72,22 @@ public String getTargetColumnName() { public boolean isSelf() { return self; } + + /** + * Is the target table distributed as a REFERENCE table (Citus) + * @return + */ + public boolean isTargetReference() { + return this.targetIsReference; + } + + /** + * Set the flag to indicate if the target table is a reference type + * @param flag + */ + public void setTargetReference(boolean flag) { + this.targetIsReference = flag; + } /** * Getter for the target table name * @return @@ -105,11 +125,16 @@ public String getQualifiedTargetName() { } /** + * Apply the FK constraint to the given target + * @param schemaName * @param name + * @param tenantColumnName * @param target + * @param sourceDistributionType */ - public void apply(String schemaName, String name, String tenantColumnName, IDatabaseAdapter target) { - target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced); + public void apply(String schemaName, String name, String tenantColumnName, ISchemaAdapter target, DistributionType sourceDistributionType) { + target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced, sourceDistributionType, + targetIsReference); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java index ae27ada8478..35b22a8c6a0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java @@ -10,7 +10,7 @@ import java.util.function.Supplier; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -41,7 +41,7 @@ public FunctionDef(String schemaName, String procedureName, int version, Supplie } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createOrReplaceFunction(getSchemaName(), getObjectName(), supplier); if (distributeByParamNum > 0) { target.distributeFunction(getSchemaName(), getObjectName(), distributeByParamNum); @@ -49,7 +49,7 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } @@ -59,12 +59,12 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropFunction(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantFunctionPrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java index 7cec93733ba..74d0a8a7b9c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.function.Consumer; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.SchemaApplyContext; @@ -34,7 +34,7 @@ public interface IDatabaseObject { * @param target the database target * @param context context to control the schema apply process */ - public void apply(IDatabaseAdapter target, SchemaApplyContext context); + public void apply(ISchemaAdapter target, SchemaApplyContext context); /** * Apply migration logic to bring the target database to the current level of this object @@ -42,7 +42,7 @@ public interface IDatabaseObject { * @param target the database target * @param context to control the schema apply process */ - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context); /** * Apply the DDL, but within its own transaction @@ -50,14 +50,14 @@ public interface IDatabaseObject { * @param cp of thread-specific transactions * @param vhs the service interface for adding this object to the version history table */ - public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider cp, IVersionHistoryService vhs); + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider cp, IVersionHistoryService vhs); /** * Apply any distribution rules associated with the object (usually a table) * @param target the target database we apply the operation to * @param pass multiple pass number */ - public void applyDistributionRules(IDatabaseAdapter target, int pass); + public void applyDistributionRules(ISchemaAdapter target, int pass); /** * Apply the change, but only if it has a newer version than we already have @@ -66,13 +66,13 @@ public interface IDatabaseObject { * @param context * @param vhs the service used to manage the version history table */ - public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs); + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs); /** * DROP this object from the target database * @param target */ - public void drop(IDatabaseAdapter target); + public void drop(ISchemaAdapter target); /** * Grant the given privileges to the user @@ -80,7 +80,7 @@ public interface IDatabaseObject { * @param groupName * @param toUser */ - public void grant(IDatabaseAdapter target, String groupName, String toUser); + public void grant(ISchemaAdapter target, String groupName, String toUser); /** * Visit this object, calling the consumer for itself, or its children if any @@ -110,7 +110,7 @@ public interface IDatabaseObject { * @param tp * @param vhs */ - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs); + public ITaskGroup collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs); /** * Return the qualified name for this object (e.g. schema.name). diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java index b604f116aaa..dd6f478fc8c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java @@ -11,8 +11,8 @@ import java.util.List; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.DistributionRules; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.common.CreateIndexStatement; /** @@ -71,16 +71,16 @@ public boolean isUnique() { * @param target * @param distributionRules */ - public void apply(String schemaName, String tableName, String tenantColumnName, IDatabaseAdapter target, - DistributionRules distributionRules) { + public void apply(String schemaName, String tableName, String tenantColumnName, ISchemaAdapter target, + DistributionType distributionType) { if (includeColumns != null && includeColumns.size() > 0) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionRules); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionType); } else if (unique) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType); } else { - target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType); } } @@ -89,7 +89,7 @@ else if (unique) { * @param schemaName * @param target */ - public void drop(String schemaName, IDatabaseAdapter target) { + public void drop(String schemaName, ISchemaAdapter target) { target.dropIndex(schemaName, indexName); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java index a306a7e4af0..feb0186c140 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java @@ -6,7 +6,7 @@ package com.ibm.fhir.database.utils.model; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.SchemaApplyContext; @@ -30,22 +30,22 @@ public NopObject(String schemaName, String objectName) { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // We're NOP so we do nothing on purpose } @Override - public void applyTx(IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // We're NOP so we do nothing on purpose } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java index 25d1cdf92fc..407f8679406 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java @@ -13,7 +13,7 @@ import java.util.Set; import java.util.function.Consumer; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.SchemaApplyContext; @@ -60,7 +60,7 @@ public ObjectGroup(String schemaName, String name, Collection g } @Override - public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Apply each member of our group to the target if it is a new version. // Version tracking is done at the individual level, not the group. @@ -70,7 +70,7 @@ public void applyVersion(IDatabaseAdapter target, SchemaApplyContext context, IV } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // Apply each member of the group, but going in reverse for (int i=group.size()-1; i>=0; i--) { group.get(i).drop(target); @@ -78,7 +78,7 @@ public void drop(IDatabaseAdapter target) { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // Override the BaseObject behavior because we need to propagate the grant request // to the indivual objects we have aggregated for (IDatabaseObject obj: this.group) { @@ -87,7 +87,7 @@ public void grant(IDatabaseAdapter target, String groupName, String toUser) { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { obj.apply(target, context); @@ -95,7 +95,7 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { obj.apply(priorVersion, target, context); @@ -137,7 +137,7 @@ public void visitReverse(DataModelVisitor v) { } @Override - public void applyDistributionRules(IDatabaseAdapter target, int pass) { + public void applyDistributionRules(ISchemaAdapter target, int pass) { for (IDatabaseObject obj: this.group) { obj.applyDistributionRules(target, pass); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index e9e3fd3b766..cec5dc379ab 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; @@ -118,7 +119,7 @@ public void addObject(IDatabaseObject obj) { * @param tp * @param vhs */ - public void collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { + public void collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { for (IDatabaseObject obj: allObjects) { obj.collect(tc, target, context, tp, vhs); } @@ -129,7 +130,7 @@ public void collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyConte * @param target * @param context */ - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { @@ -143,7 +144,7 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { * may have (e.g. for Citus) * @param target */ - public void applyDistributionRules(IDatabaseAdapter target) { + public void applyDistributionRules(ISchemaAdapter target) { // make a first pass to apply reference rules for (IDatabaseObject obj: allObjects) { @@ -162,7 +163,7 @@ public void applyDistributionRules(IDatabaseAdapter target) { * @param target * @param vhs */ - public void applyWithHistory(IDatabaseAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { + public void applyWithHistory(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { @@ -175,7 +176,7 @@ public void applyWithHistory(IDatabaseAdapter target, SchemaApplyContext context * Apply all the procedures in the order in which they were added to the model * @param adapter */ - public void applyProcedures(IDatabaseAdapter adapter, SchemaApplyContext context) { + public void applyProcedures(ISchemaAdapter adapter, SchemaApplyContext context) { int total = procedures.size(); int count = 1; for (ProcedureDef obj: procedures) { @@ -189,7 +190,7 @@ public void applyProcedures(IDatabaseAdapter adapter, SchemaApplyContext context * Apply all the functions in the order in which they were added to the model * @param adapter */ - public void applyFunctions(IDatabaseAdapter adapter, SchemaApplyContext context) { + public void applyFunctions(ISchemaAdapter adapter, SchemaApplyContext context) { int total = functions.size(); int count = 1; for (FunctionDef obj: functions) { @@ -206,7 +207,7 @@ public void applyFunctions(IDatabaseAdapter adapter, SchemaApplyContext context) * @param tagGroup * @param tag */ - public void drop(IDatabaseAdapter target, String tagGroup, String tag) { + public void drop(ISchemaAdapter target, String tagGroup, String tag) { // The simplest way to reverse the list is add everything into an array list // which we then simply traverse end to start ArrayList copy = new ArrayList<>(); @@ -236,7 +237,7 @@ public void drop(IDatabaseAdapter target, String tagGroup, String tag) { * @param tagGroup * @param tag */ - public void dropSplitTransaction(IDatabaseAdapter target, ITransactionProvider transactionProvider, String tagGroup, String tag) { + public void dropSplitTransaction(ISchemaAdapter target, ITransactionProvider transactionProvider, String tagGroup, String tag) { ArrayList copy = new ArrayList<>(); copy.addAll(allObjects); @@ -272,7 +273,7 @@ public void dropSplitTransaction(IDatabaseAdapter target, ITransactionProvider t * @param tagGroup * @param tag */ - public void dropForeignKeyConstraints(IDatabaseAdapter target, String tagGroup, String tag) { + public void dropForeignKeyConstraints(ISchemaAdapter target, String tagGroup, String tag) { // The simplest way to reverse the list is add everything into an array list // which we then simply traverse end to start ArrayList copy = new ArrayList<>(); @@ -335,7 +336,7 @@ public void visitReverse(DataModelVisitor v) { * Drop the lot * @param target */ - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { drop(target, null, null); } @@ -597,7 +598,7 @@ public void processObjectsWithTag(String tagName, String tagValue, Consumer 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } @@ -56,12 +56,12 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropProcedure(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantProcedurePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java index e4bbb8c8d3f..9be488c4e7b 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java @@ -6,7 +6,7 @@ package com.ibm.fhir.database.utils.model; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; @@ -36,12 +36,12 @@ public String toString() { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createArrType(getSchemaName(), getObjectName(), rowTypeName, arraySize); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row array types is not supported"); } @@ -49,7 +49,7 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropType(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java index af6f4c9bff2..6f0ae923192 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java @@ -10,7 +10,7 @@ import java.util.Collection; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -26,12 +26,12 @@ public RowType(String schemaName, String typeName, int version, Collection 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row types is not supported"); } @@ -39,7 +39,7 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropType(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java index 23a4dde9b28..42da00b22a2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java @@ -8,7 +8,7 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -45,12 +45,12 @@ public Sequence(String schemaName, String sequenceName, int version, long startW } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createSequence(getSchemaName(), getObjectName(), this.startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading sequences is not supported"); } @@ -63,12 +63,12 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropSequence(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantSequencePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java index 32a56865412..f3bd594d21a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java @@ -8,7 +8,7 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -21,22 +21,22 @@ public SessionVariableDef(String schemaName, String variableName, int version) { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropVariable(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { if (target.useSessionVariable()) { target.grantVariablePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java new file mode 100644 index 00000000000..2ff6c98c7f1 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java @@ -0,0 +1,35 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.model; + +import com.ibm.fhir.database.utils.api.IDatabaseTypeAdapter; +import com.ibm.fhir.database.utils.postgres.PostgresAdapter; + +/** + * Column acting as either a boolean or smallint depending on the underlying + * database type + */ +public class SmallIntBooleanColumn extends ColumnBase { + /** + * @param name + * @param nullable + * @param defaultValue + */ + public SmallIntBooleanColumn(String name, boolean nullable, String defaultValue) { + super(name, nullable, defaultValue); + } + + @Override + public String getTypeInfo(IDatabaseTypeAdapter adapter) { + if (adapter instanceof PostgresAdapter) { + this.resetDefaultValue(); + return "BOOLEAN"; + } else { + return "SMALLINT"; + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java index a44f1b14152..753718b1764 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,10 +7,9 @@ package com.ibm.fhir.database.utils.model; import com.ibm.fhir.database.utils.api.IDatabaseTypeAdapter; -import com.ibm.fhir.database.utils.postgres.PostgresAdapter; /** - * Small Int Column + * Small Int Column (2 bytes signed integer) */ public class SmallIntColumn extends ColumnBase { /** @@ -24,11 +23,7 @@ public SmallIntColumn(String name, boolean nullable, String defaultValue) { @Override public String getTypeInfo(IDatabaseTypeAdapter adapter) { - if (adapter instanceof PostgresAdapter) { - this.resetDefaultValue(); - return "BOOLEAN"; - } else { - return "SMALLINT"; - } + // TODO ask the adapter for the type name to use for a smallint type column. + return "SMALLINT"; } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index d661bab3ccc..8a2393f0cbb 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -17,8 +17,8 @@ import java.util.Set; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.DistributionRules; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; @@ -50,7 +50,7 @@ public class Table extends BaseObject { private final String tenantColumnName; // The rules to distribute the table in a distributed RDBMS implementation (Citus) - private final DistributionRules distributionRules; + private final DistributionType distributionType; // The With parameters on the table private final List withs; @@ -77,14 +77,14 @@ public class Table extends BaseObject { * @param migrations * @param withs * @param checkConstraints - * @param distributionRules + * @param distributionType */ public Table(String schemaName, String name, int version, String tenantColumnName, Collection columns, PrimaryKeyDef pk, IdentityDef identity, Collection indexes, Collection fkConstraints, SessionVariableDef accessControlVar, Tablespace tablespace, List dependencies, Map tags, Collection privileges, List migrations, List withs, List checkConstraints, - DistributionRules distributionRules) { + DistributionType distributionType) { super(schemaName, name, DatabaseObjectType.TABLE, version, migrations); this.tenantColumnName = tenantColumnName; this.columns.addAll(columns); @@ -96,7 +96,7 @@ public Table(String schemaName, String name, int version, String tenantColumnNam this.tablespace = tablespace; this.withs = withs; this.checkConstraints.addAll(checkConstraints); - this.distributionRules = distributionRules; + this.distributionType = distributionType; // Adds all dependencies which aren't null. // The only circumstances where it is null is when it is self referencial (an FK on itself). @@ -130,22 +130,30 @@ public String getTenantColumnName() { return this.tenantColumnName; } + /** + * Getter for the table's distributionType + * @return + */ + public DistributionType getDistributionType() { + return this.distributionType; + } + @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { final String tsName = this.tablespace == null ? null : this.tablespace.getName(); target.createTable(getSchemaName(), getObjectName(), this.tenantColumnName, this.columns, this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints, - this.distributionRules); + this.distributionType); // Now add any indexes associated with this table for (IndexDef idx: this.indexes) { - idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionRules); + idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType); } if (context.isIncludeForeignKeys()) { // Foreign key constraints for (ForeignKeyConstraint fkc: this.fkConstraints) { - fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType); } } @@ -163,7 +171,7 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion == null || priorVersion == 0) { apply(target, context); } else if (this.getVersion() > priorVersion) { @@ -185,7 +193,7 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { if (this.accessControlVar != null) { target.deactivateRowAccessControl(getSchemaName(), getObjectName()); @@ -253,11 +261,8 @@ public static class Builder extends VersionedSchemaObject { // Check constraints added to the table private List checkConstraints = new ArrayList<>(); - // The column to use for distribution when sharding - private String distributionColumnName; - - // Should this table be treated as a reference (replicated) table when sharding is enabled - private boolean distributionReference = false; + // The type of distribution to use for this table when using a distributed database + private DistributionType distributionType = DistributionType.NONE; /** * Private constructor to force creation through factory method @@ -293,17 +298,8 @@ public Builder setTablespace(Tablespace ts) { * @param cn * @return */ - public Builder setDistributionColumnName(String cn) { - this.distributionColumnName = cn; - return this; - } - - /** - * Set the distributionReference to true - * @return - */ - public Builder setDistributionReference() { - this.distributionReference = true; + public Builder setDistributionType(DistributionType dt) { + this.distributionType = dt; return this; } @@ -336,6 +332,30 @@ public Builder addSmallIntColumn(String columnName, Integer defaultValue, boolea return this; } + /** + * Variant used by JavaBatch which is BOOLEAN in PostgreSQL but SMALLINT elsewhere + * @param columnName + * @param defaultValue + * @param nullable + * @return + */ + public Builder addSmallIntBooleanColumn(String columnName, Integer defaultValue, boolean nullable) { + ColumnDef cd = new ColumnDef(columnName); + if (columns.contains(cd)) { + throw new IllegalArgumentException("Duplicate column: " + columnName); + } + + cd.setNullable(nullable); + + if (defaultValue != null) { + cd.setDefaultVal(Integer.toString(defaultValue)); + } + + cd.setColumnType(ColumnType.SMALLINT_BOOLEAN); + columns.add(cd); + return this; + } + public Builder addBigIntColumn(String columnName, boolean nullable) { addBigIntColumn(columnName, nullable, null); return this; @@ -730,14 +750,6 @@ public Table build(IDataModel dataModel) { List enabledFKConstraints = new ArrayList<>(); allDependencies.addAll(this.dependencies); - // Set up the distribution rules for the table if the target model supports distribution - final DistributionRules distributionRules; - if (dataModel.isDistributed() && (this.distributionReference || this.distributionColumnName != null)) { - distributionRules = new DistributionRules(distributionColumnName, distributionReference); - } else { - distributionRules = null; - } - // Filter the foreign key constraints to those allowed for the model. Distribution (e.g. Citus) adds // certain restrictions on which foreign keys are supported, so we have no choice but to ignore them for (ForeignKeyConstraint c: this.fkConstraints.values()) { @@ -747,42 +759,43 @@ public Table build(IDataModel dataModel) { throw new IllegalArgumentException("Invalid foreign key constraint " + c.getConstraintName() + ": target table does not exist: " + targetName); } + // Determine the distribution type of the FK target. If target is null, it must mean that this is a FK to self + DistributionType targetDistributionType = target != null ? target.getDistributionType() : this.distributionType; + if (targetDistributionType == DistributionType.REFERENCE) { + // Mark the constraint as pointing to a REFERENCE table (which won't include a shard + // column). FK relationships are therefore local to each distributed node (Citus) + c.setTargetReference(true); + } + if (!dataModel.isDistributed()) { // ignore any distribution configuration because the target database is a plain RDBMS - allDependencies.add(target); + if (target != null) { + // only add dependency if target is something else. If target is null, it means + // a FK reference to self and so no dependency is needed + allDependencies.add(target); + } enabledFKConstraints.add(c); } else { // Make sure that FK references adhere to the restrictions imposed by distribution (replication or sharding) - if (distributionRules == null) { + if (distributionType == DistributionType.NONE) { // this table is not distributed, so we can handle the FK relationship as long // as the target isn't sharded (replicated is OK) - if (target.distributionRules == null || target.distributionRules.isReferenceTable()) { + if (targetDistributionType == DistributionType.REFERENCE) { allDependencies.add(target); enabledFKConstraints.add(c); } - } else if (distributionRules.isReferenceTable()) { + } else if (distributionType == DistributionType.REFERENCE) { // This table is a reference (replicated) table. We can create FK relationships - // to other tables non-distributed and replicated tables - if (target.distributionRules == null || target.distributionRules.isReferenceTable()) { - allDependencies.add(target); - enabledFKConstraints.add(c); - } - } else if (target.distributionRules != null) { - // This table is sharded. We can only create FK relationships to the target if - // the target is replicated, or has a matching sharding configuration - if (target.distributionRules.isReferenceTable()) { - // the target is replicated, so we can create a FK relationship to it - allDependencies.add(target); - enabledFKConstraints.add(c); - } else if (target.distributionRules.getDistributionColumn() != null - && target.distributionRules.getDistributionColumn().equalsIgnoreCase(this.distributionColumnName) - && c.includesColumn(this.distributionColumnName)) { - // Both tables are sharded. We can only support FK relationships if the source and target - // are "co-located" which means they must share the same distribution column which is also part of - // the foreign key + // to other replicated tables + if (targetDistributionType == DistributionType.REFERENCE) { allDependencies.add(target); enabledFKConstraints.add(c); } + } else if (targetDistributionType != DistributionType.NONE) { + // This table is distributed, and the target is either distributed or a reference + // table. In either case we can support the FK because the tables will be co-located + allDependencies.add(target); + enabledFKConstraints.add(c); } } } @@ -794,7 +807,7 @@ public Table build(IDataModel dataModel) { // Our schema objects are immutable by design, so all initialization takes place // through the constructor return new Table(getSchemaName(), getObjectName(), this.version, this.tenantColumnName, buildColumns(), this.primaryKey, this.identity, this.indexes.values(), - enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionRules); + enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionType); } /** @@ -819,6 +832,9 @@ protected List buildColumns() { case SMALLINT: column = new SmallIntColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); break; + case SMALLINT_BOOLEAN: + column = new SmallIntBooleanColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); + break; case DOUBLE: column = new DoubleColumn(cd.getName(), cd.isNullable()); break; @@ -927,7 +943,7 @@ public Builder addWiths(List withs) { * @param target * @return */ - public boolean exists(IDatabaseAdapter target) { + public boolean exists(ISchemaAdapter target) { return target.doesTableExist(getSchemaName(), getObjectName()); } @@ -945,15 +961,13 @@ public void visitReverse(DataModelVisitor v) { } @Override - public void applyDistributionRules(IDatabaseAdapter target, int pass) { + public void applyDistributionRules(ISchemaAdapter target, int pass) { // make sure all the reference tables are distributed first before // we attempt to shard anything - if (this.distributionRules != null) { - if (pass == 0 && this.distributionRules.isReferenceTable()) { - target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); - } else if (pass == 1 && this.distributionRules.isDistributedTable()) { - target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionRules); - } + if (pass == 0 && this.distributionType == DistributionType.REFERENCE) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType); + } else if (pass == 1 && this.distributionType == DistributionType.DISTRIBUTED) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType); } } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java index 32488a216a4..5da8395af35 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java @@ -9,7 +9,7 @@ import java.util.Collection; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.SchemaApplyContext; @@ -35,7 +35,7 @@ public Tablespace(String tablespaceName, int version, int extentSizeKB) { } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { if (this.extentSizeKB > 0) { target.createTablespace(getName(), this.extentSizeKB); } @@ -46,7 +46,7 @@ public void apply(IDatabaseAdapter target, SchemaApplyContext context) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0) { throw new UnsupportedOperationException("Modifying tablespaces is not supported"); } @@ -54,12 +54,12 @@ public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyCont } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropTablespace(getName()); } @Override - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { + public ITaskGroup collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // no dependencies, so no need to recurse down List children = null; return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, context, tp, vhs), children); @@ -76,7 +76,7 @@ public void fetchDependenciesTo(Collection out) { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // NOP } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java index b96c394e30a..2b7f07a50db 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java @@ -15,7 +15,7 @@ import java.util.Set; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; @@ -48,17 +48,17 @@ protected View(String schemaName, String objectName, int version, String selectC } @Override - public void apply(IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createOrReplaceView(getSchemaName(), getObjectName(), this.selectClause); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target, SchemaApplyContext context) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropView(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index b8879df7168..1c2df6d87d6 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -19,7 +19,7 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.DistributionRules; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -101,7 +101,7 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, - DistributionRules distributionRules) { + DistributionContext distributionContext) { // PostgreSql doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { @@ -115,9 +115,9 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns, DistributionRules distributionRules) { + List includeColumns, DistributionContext distributionContext) { // PostgreSql doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionRules); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionContext); } @Override @@ -303,7 +303,7 @@ public void runStatement(IDatabaseStatement stmt) { @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, DistributionRules distributionRules) { + List indexColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); // Postgresql doesn't support index name prefixed with the schema name. String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java index 573c7a952a8..bc88066a46a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java @@ -8,7 +8,7 @@ import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -60,7 +60,7 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String adminSchem * @param adminSchemaName * @param target */ - public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String adminSchemaName, ISchemaAdapter target) { SchemaApplyContext context = SchemaApplyContext.getDefault(); PhysicalDataModel dataModel = new PhysicalDataModel(); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java index 12124312ee7..6a66e508328 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java @@ -6,7 +6,7 @@ package com.ibm.fhir.database.utils.version; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -59,7 +59,7 @@ public static Table generateTable(PhysicalDataModel dataModel, String adminSchem * @param adminSchemaName * @param target */ - public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String adminSchemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); SchemaApplyContext context = SchemaApplyContext.getDefault(); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java index c0e7d4c4130..8b45b692512 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java @@ -8,7 +8,7 @@ import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Privilege; @@ -70,7 +70,7 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String schemaName * @param schemaName * @param target */ - public static void createTableIfNeeded(String schemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String schemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); SchemaApplyContext context = SchemaApplyContext.getDefault(); @@ -100,7 +100,7 @@ public static void createTableIfNeeded(String schemaName, IDatabaseAdapter targe * @param schemaName * @param target */ - public static void dropTable(String schemaName, IDatabaseAdapter target) { + public static void dropTable(String schemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, schemaName, false); @@ -120,7 +120,7 @@ public static void dropTable(String schemaName, IDatabaseAdapter target) { * @param groupName * @param toUser */ - public static void grantPrivilegesTo(IDatabaseAdapter target, String schemaName, String groupName, String toUser) { + public static void grantPrivilegesTo(ISchemaAdapter target, String schemaName, String groupName, String toUser) { PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, schemaName, false); t.grant(target, groupName, toUser); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java index 477f70c0234..9f15e820797 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java @@ -44,15 +44,19 @@ public class FHIRResourceDAOFactory { /** * Construct a new ResourceDAO implementation matching the database type * @param connection valid connection to the database + * @param adminSchemaName * @param schemaName the name of the schema containing the FHIR resource tables * @param flavor the type and capability of the database and schema * @param trxSynchRegistry + * @param cache + * @param ptdi + * @param shardKey * @return a concrete implementation of {@link ResourceDAO} * @throws IllegalArgumentException * @throws FHIRPersistenceException */ public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, - FHIRPersistenceJDBCCache cache, ParameterTransactionDataImpl ptdi) + FHIRPersistenceJDBCCache cache, ParameterTransactionDataImpl ptdi, Short shardKey) throws IllegalArgumentException, FHIRPersistenceException { final ResourceDAO resourceDAO; @@ -65,10 +69,10 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; case POSTGRESQL: - resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); break; case CITUS: - resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); @@ -124,15 +128,19 @@ public static IDatabaseTranslator getTranslatorForFlavor(FHIRDbFlavor flavor) { /** * Construct a new ResourceDAO implementation matching the database type + * * @param connection valid connection to the database + * @param adminSchemaName * @param schemaName the name of the schema containing the FHIR resource tables * @param flavor the type and capability of the database and schema + * @param cache + * @param shardKey * @return a concrete implementation of {@link ResourceDAO} * @throws IllegalArgumentException * @throws FHIRPersistenceException */ public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, - FHIRPersistenceJDBCCache cache) throws IllegalArgumentException, FHIRPersistenceException { + FHIRPersistenceJDBCCache cache, Short shardKey) throws IllegalArgumentException, FHIRPersistenceException { final ResourceDAO resourceDAO; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); @@ -144,10 +152,10 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, cache, rrd); break; case POSTGRESQL: - resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd); + resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd, shardKey); break; case CITUS: - resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, cache, rrd); + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, cache, rrd, shardKey); break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java index 13ece6f3d02..56aac5545f5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -47,8 +47,8 @@ public class CitusResourceDAO extends PostgresResourceDAO { * @param cache * @param rrd */ - public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd) { - super(connection, schemaName, flavor, cache, rrd); + public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, Short shardKey) { + super(connection, schemaName, flavor, cache, rrd, shardKey); } /** @@ -63,8 +63,8 @@ public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor f * @param ptdi */ public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, - ParameterTransactionDataImpl ptdi) { - super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + ParameterTransactionDataImpl ptdi, Short shardKey) { + super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java index 1fb92f16a32..f038aae225a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java @@ -18,6 +18,7 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.config.PropertyGroup; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; @@ -97,17 +98,17 @@ private FHIRDbFlavor createFlavor() throws FHIRPersistenceDataAccessException { if (dsPG != null) { try { - boolean multitenant = false; String typeValue = dsPG.getStringProperty("type"); + SchemaType schemaType = SchemaType.PLAIN; DbType type = DbType.from(typeValue); if (type == DbType.DB2) { // We make this absolute for now. May change in the future if we // support a single-tenant schema in DB2. - multitenant = true; + schemaType = SchemaType.MULTITENANT; } - result = new FHIRDbFlavorImpl(type, multitenant); + result = new FHIRDbFlavorImpl(type, schemaType); } catch (Exception x) { log.log(Level.SEVERE, "No type property found for datastore '" + datastoreId + "'", x); throw new FHIRPersistenceDataAccessException("Datastore configuration issue. Details in server logs"); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java index 0398442cb6d..662a310cd7a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java @@ -6,6 +6,7 @@ package com.ibm.fhir.persistence.jdbc.connection; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; /** @@ -21,6 +22,12 @@ public interface FHIRDbFlavor { */ public boolean isMultitenant(); + /** + * What type of schema is this + * @return + */ + public SchemaType getSchemaType(); + /** * What type of database is this? * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java index 5b4cc4495ba..777da38703d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java @@ -6,6 +6,7 @@ package com.ibm.fhir.persistence.jdbc.connection; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; /** @@ -14,20 +15,24 @@ */ public class FHIRDbFlavorImpl implements FHIRDbFlavor { - // does the database schema support multi-tenancy - private final boolean multitenant; - // basic type of the database (DB2, Derby etc) private final DbType type; - public FHIRDbFlavorImpl(DbType type, boolean multitenant) { + private final SchemaType schemaType; + + /** + * Public constructor + * @param type + * @param schemaType + */ + public FHIRDbFlavorImpl(DbType type, SchemaType schemaType) { this.type = type; - this.multitenant = multitenant; + this.schemaType = schemaType; } @Override public boolean isMultitenant() { - return this.multitenant; + return this.schemaType == SchemaType.MULTITENANT; } @Override @@ -39,4 +44,9 @@ public DbType getType() { public boolean isFamilyPostgreSQL() { return this.type == DbType.POSTGRESQL || this.type == DbType.CITUS; } -} + + @Override + public SchemaType getSchemaType() { + return this.schemaType; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java index dc31f4899fd..ba1ee36d426 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java @@ -20,6 +20,7 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.config.PropertyGroup; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.exception.FHIRException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; @@ -212,17 +213,26 @@ private FHIRDbFlavor createFlavor() throws FHIRPersistenceDataAccessException { if (dsPG != null) { try { - boolean multitenant = false; - String typeValue = dsPG.getStringProperty("type"); + SchemaType schemaType = SchemaType.PLAIN; + String schemaTypeValue = dsPG.getStringProperty("schemaType", null); + if (schemaTypeValue != null) { + schemaType = SchemaType.valueOf(schemaTypeValue.toUpperCase()); + } + String typeValue = dsPG.getStringProperty("type"); DbType type = DbType.from(typeValue); if (type == DbType.DB2) { - // We make this absolute for now. May change in the future if we - // support a single-tenant schema in DB2. - multitenant = true; + // For Db2 we currently only support MULTITENANT so we force the schemaType + schemaType = SchemaType.MULTITENANT; + } else { + // Make sure for any other database of type we're not being asked to use the + // multitenant variant + if (schemaType == SchemaType.MULTITENANT) { + throw new FHIRPersistenceDataAccessException("schemaType MULTITENANT is only supported for Db2"); + } } - result = new FHIRDbFlavorImpl(type, multitenant); + result = new FHIRDbFlavorImpl(type, schemaType); } catch (Exception x) { log.log(Level.SEVERE, "No type property found for datastore '" + datastoreId + "'", x); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java index 0553bad6011..6aa5aa7bbf6 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java @@ -11,6 +11,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; @@ -51,8 +52,9 @@ public FHIRDbTestConnectionStrategy(IConnectionProvider cp, Action action) { this.connectionProvider = cp; this.action = action; - // we don't support multi-tenancy in our unit-test database - flavor = new FHIRDbFlavorImpl(cp.getTranslator().getType(), false); + // we don't support multi-tenancy or distribution in our unit-test database, + // so we use PLAIN for the schema type + flavor = new FHIRDbFlavorImpl(cp.getTranslator().getType(), SchemaType.PLAIN); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java new file mode 100644 index 00000000000..d35691f1bc0 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java @@ -0,0 +1,109 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.impl; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; +import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; +import com.ibm.fhir.persistence.jdbc.dto.DateParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValueVisitor; +import com.ibm.fhir.persistence.jdbc.dto.LocationParmVal; +import com.ibm.fhir.persistence.jdbc.dto.NumberParmVal; +import com.ibm.fhir.persistence.jdbc.dto.QuantityParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; +import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; +import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; + + +/** + * A visitor to map parameters to a format suitable for transport to another + * system (e.g. for remote indexing) + */ +public class ParameterTransportVisitor implements ExtractedParameterValueVisitor { + private final ParameterValueVisitorAdapter adapter; + + // tracks the number of composites so we know what next composite_id to use + private int compositeIdCounter = 0; + + // Tracks the name of the composite parameter currently being processed + private String currentCompositeParameterName = null; + + /** + * Public constructor + * @param adapter + */ + public ParameterTransportVisitor(ParameterValueVisitorAdapter adapter) { + this.adapter = adapter; + } + + @Override + public void visit(StringParmVal stringParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.stringValue(stringParameter.getName(), stringParameter.getValueString(), compositeId); + } + + @Override + public void visit(NumberParmVal numberParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.numberValue(numberParameter.getName(), + numberParameter.getValueNumber(), + numberParameter.getValueNumberLow(), + numberParameter.getValueNumberHigh(), + compositeId); + } + + @Override + public void visit(DateParmVal dateParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.dateValue(dateParameter.getName(), dateParameter.getValueDateStart(), dateParameter.getValueDateEnd(), compositeId); + } + + @Override + public void visit(TokenParmVal tokenParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.tokenValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), compositeId); + } + + @Override + public void visit(QuantityParmVal quantityParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.quantityValue(quantityParameter.getName(), quantityParameter.getValueSystem(), quantityParameter.getValueCode(), quantityParameter.getValueNumber(), + quantityParameter.getValueNumberLow(), quantityParameter.getValueNumberHigh(), compositeId); + + } + + @Override + public void visit(LocationParmVal locationParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.locationValue(locationParameter.getName(), locationParameter.getValueLatitude(), locationParameter.getValueLongitude(), compositeId); + } + + @Override + public void visit(ReferenceParmVal ref) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.referenceValue(ref.getName(), ref.getRefValue(), compositeId); + } + + @Override + public void visit(CompositeParmVal compositeParameter) throws FHIRPersistenceException { + if (this.currentCompositeParameterName != null) { + throw new FHIRPersistenceException("found nested composite parameter which isn't supported. " + + "current:[" + currentCompositeParameterName + "]" + + " nested:[" + compositeParameter.getName() + "]"); + } + + // Each parameter contained within this composite will be assigned the same + // compositeIdCounter value + this.compositeIdCounter++; + this.currentCompositeParameterName = compositeParameter.getName(); + for (ExtractedParameterValue epv: compositeParameter.getComponent()) { + epv.accept(this); + } + this.currentCompositeParameterName = null; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index c8f227f682c..d1420edec3b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -762,7 +762,7 @@ public List searchForIds(Select dataQuery) throws FHIRPersistenceDataAcces * @param logicalResourceId * @throws SQLException */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { + private void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { // bind parameters diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index bd96947d35d..07b84e43cd7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -55,6 +55,7 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.api.UndefinedNameException; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.model.DbType; @@ -105,6 +106,9 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceDeletedException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceNotFoundException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; +import com.ibm.fhir.persistence.index.IndexProviderResponse; +import com.ibm.fhir.persistence.index.RemoteIndexData; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.FHIRResourceDAOFactory; import com.ibm.fhir.persistence.jdbc.JDBCConstants; @@ -132,6 +136,7 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.FetchResourcePayloadsDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.JDBCIdentityCacheImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterDAOImpl; +import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterTransportVisitor; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; @@ -236,6 +241,9 @@ public class FHIRPersistenceJDBCImpl implements FHIRPersistence, SchemaNameSuppl // A list of EraseResourceRec referencing offload resource records to erase if the current transaction commits private final List eraseResourceRecs = new ArrayList<>(); + // A list of the remote index messages we need to check we get ACKs for + private final List remoteIndexMessageList = new ArrayList<>(); + /** * Constructor for use when running as web application in WLP. * @throws Exception @@ -374,7 +382,7 @@ public SingleResourceResult create(FHIRPersistenceContex log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction @@ -401,7 +409,7 @@ public SingleResourceResult create(FHIRPersistenceContex // The DAO objects are now created on-the-fly (not expensive to construct) and // given the connection to use while processing this request - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); // Persist the Resource DTO. @@ -413,6 +421,7 @@ public SingleResourceResult create(FHIRPersistenceContex + ", version=" + resourceDTO.getVersionId()); } + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getShardKey(), searchParameters); SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -446,6 +455,35 @@ public SingleResourceResult create(FHIRPersistenceContex } } + /** + * Convert the extracted parameters into a package we can send to a remote service + * for processing then send to that service (if so configured) + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param shardKey + * @param searchParameters + */ + private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, Short shardKey, + ExtractedSearchParameters searchParameters) throws FHIRPersistenceException { + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + if (remoteIndexService != null) { + // convert the parameters into a form that will be easy to ship to a remote service + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, shardKey); + ParameterTransportVisitor visitor = new ParameterTransportVisitor(adapter); + for (ExtractedParameterValue pv: searchParameters.getParameters()) { + pv.accept(visitor); + } + + // Note that the remote index service is supposed to be multi-tenant, using + // the tenantId from the request context on this thread, so we don't need + // to pass that here + final String partitionKey = resourceType + "/" + logicalId; + IndexProviderResponse ipr = remoteIndexService.submit(new RemoteIndexData(partitionKey, adapter.build())); + remoteIndexMessageList.add(ipr); // we'll check for an ACK just before we commit the transaction + } + } + /** * Prefill the cache if required * @throws FHIRPersistenceException @@ -453,7 +491,7 @@ public SingleResourceResult create(FHIRPersistenceContex private void doCachePrefill() throws FHIRPersistenceException { if (cache.needToPrefill()) { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(/*context=*/null, connection); } catch(FHIRPersistenceException e) { throw e; } catch(Throwable e) { @@ -511,24 +549,35 @@ private com.ibm.fhir.persistence.jdbc.dto.Resource createResourceDTO(Class SingleResourceResult update(FHIRPersistenceContex log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction this.payloadPersistenceResponses.add(context.getOffloadResponse()); } - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); // Since 1869, the resource is already correctly configured so no need to modify it @@ -606,6 +655,9 @@ public SingleResourceResult update(FHIRPersistenceContex } } + // If configured, send the extracted parameters to the remote indexing service + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getShardKey(), searchParameters); + SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -659,10 +711,10 @@ public MultiResourceResult search(FHIRPersistenceContext context, Class void delete(FHIRPersistenceContext context, Class log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction this.payloadPersistenceResponses.add(context.getOffloadResponse()); } - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); // Create a new Resource DTO instance to represent the deletion marker. final int newVersionId = versionId + 1; @@ -1125,8 +1177,8 @@ public SingleResourceResult read(FHIRPersistenceContext } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); resourceDTO = resourceDao.read(logicalId, resourceType.getSimpleName()); boolean resourceIsDeleted = resourceDTO != null && resourceDTO.isDeleted(); @@ -1197,8 +1249,8 @@ public MultiResourceResult history(FHIRPersistenceContext context, Class SingleResourceResult vread(FHIRPersistenceContext } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); version = Integer.parseInt(versionId); resourceDTO = resourceDao.versionRead(logicalId, resourceType.getSimpleName(), version); @@ -2522,14 +2574,14 @@ public String getSchemaForRequestContext(Connection connection) throws FHIRPersi /** * Prefill the caches */ - public void doCachePrefill(Connection connection) throws FHIRPersistenceException { + public void doCachePrefill(FHIRPersistenceContext context, Connection connection) throws FHIRPersistenceException { // Perform the cache prefill just once (for a given tenant). This isn't synchronous, so // there's a chance for other threads to slip in before the prefill completes. Those threads // just end up repeating the prefill - a little extra work one time to avoid unnecessary locking // Note - this is done as the first thing in a transaction so there's no concern about reading // uncommitted values. if (cache.needToPrefill()) { - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); FHIRPersistenceJDBCCacheUtil.prefill(resourceDao, parameterDao, cache); cache.clearNeedToPrefill(); @@ -2570,8 +2622,8 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); ReindexResourceDAO reindexDAO = FHIRResourceDAOFactory.getReindexResourceDAO(connection, FhirSchemaConstants.FHIR_ADMIN, schemaNameSupplier.getSchemaForRequestContext(connection), connectionStrategy.getFlavor(), this.trxSynchRegistry, this.cache, parameterDao); // Obtain a resource we will reindex in this request/transaction. The record is locked as part @@ -2852,6 +2904,22 @@ public void onCommit(Collection records, Collection records, Collection resourceType, java.time.Instant fromLastModified, java.time.Instant toLastModified, Function processor) throws FHIRPersistenceException { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(null, connection); // translator is required to handle some simple SQL syntax differences. This is easier // than creating separate DAO implementations for each database type IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); @@ -2883,11 +2951,11 @@ public ResourcePayload fetchResourcePayloads(Class resourceT } @Override - public List changes(int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, + public List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, Long changeIdMarker, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); // translator is required to handle some simple SQL syntax differences. This is easier // than creating separate DAO implementations for each database type final List resourceTypeIds; @@ -2920,13 +2988,13 @@ public List changes(int resourceCount, java.time.Instan } @Override - public ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceException { + public ResourceEraseRecord erase(FHIRPersistenceContext context, EraseDTO eraseDto) throws FHIRPersistenceException { final String METHODNAME = "erase"; log.entering(CLASSNAME, METHODNAME); ResourceEraseRecord eraseRecord = new ResourceEraseRecord(); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); IResourceReferenceDAO rrd = makeResourceReferenceDAO(connection); EraseResourceDAO eraseDao = new EraseResourceDAO(connection, FhirSchemaConstants.FHIR_ADMIN, translator, @@ -3020,12 +3088,12 @@ private boolean allSearchParmsAreGlobal(List queryParms) { } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { final String METHODNAME = "retrieveIndex"; log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); RetrieveIndexDAO dao = new RetrieveIndexDAO(translator, schemaNameSupplier.getSchemaForRequestContext(connection), resourceTypeName, count, notModifiedAfter, afterIndexId, this.cache); return dao.run(connection); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java new file mode 100644 index 00000000000..4413d78f05c --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java @@ -0,0 +1,156 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.impl; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.util.logging.Logger; + +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.SearchParametersTransport; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + + +/** + * Visitor implementation to build an instance of {@link SearchParametersTransport} to + * provide support for shipping a set of search parameter values off to a remote + * index service. This allows the parameters to be stored in the database in a + * separate transaction, and allows the inserts to be batched together, providing + * improved throughput. + */ +public class SearchParametersTransportAdapter implements ParameterValueVisitorAdapter { + private static final Logger logger = Logger.getLogger(SearchParametersTransportAdapter.class.getName()); + + // The builder we use to collect all the visited parameter values + private final SearchParametersTransport.Builder builder; + + /** + * Public constructor + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param shardKey + */ + public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, Short shardKey) { + builder = SearchParametersTransport.builder() + .withResourceType(resourceType) + .withLogicalId(logicalId) + .withLogicalResourceId(logicalResourceId) + .withShardKey(shardKey); + } + + /** + * Build the SearchParametersTransport instance from the current state of builder + * @return + */ + public SearchParametersTransport build() { + return builder.build(); + } + + @Override + public void stringValue(String name, String valueString, Integer compositeId) { + StringParameter value = new StringParameter(); + value.setName(name); + value.setValue(valueString); + value.setCompositeId(compositeId); + builder.addStringValue(value); + } + + @Override + public void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId) { + NumberParameter value = new NumberParameter(); + value.setName(name); + value.setValue(valueNumber); + value.setLowValue(valueNumberLow); + value.setHighValue(valueNumberHigh); + value.setCompositeId(compositeId); + builder.addNumberValue(value); + } + + @Override + public void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId) { + DateParameter value = new DateParameter(); + value.setName(name); + value.setValueDateStart(valueDateStart); + value.setValueDateEnd(valueDateEnd); + value.setCompositeId(compositeId); + builder.addDateValue(value); + } + + @Override + public void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId) { + TokenParameter value = new TokenParameter(); + value.setName(name); + value.setValueSystem(valueSystem); + value.setValueCode(valueCode); + value.setCompositeId(compositeId); + builder.addTokenValue(value); + } + + @Override + public void quantityValue(String name, String valueSystem, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, + Integer compositeId) { + QuantityParameter value = new QuantityParameter(); + value.setName(name); + value.setValueSystem(valueSystem); + value.setValueCode(valueCode); + value.setValueNumber(valueNumber); + value.setValueNumberLow(valueNumberLow); + value.setValueNumberHigh(valueNumberHigh); + value.setCompositeId(compositeId); + builder.addQuantityValue(value); + } + + @Override + public void locationValue(String name, Double valueLatitude, Double valueLongitude, Integer compositeId) { + LocationParameter value = new LocationParameter(); + value.setName(name); + value.setValueLatitude(valueLatitude); + value.setValueLongitude(valueLongitude); + value.setCompositeId(compositeId); + builder.addLocationValue(value); + } + + @Override + public void referenceValue(String name, ReferenceValue refValue, Integer compositeId) { + if (refValue == null) { + return; + } + + // The ReferenceValue has already been processed to convert the reference to + // the required standard form, ready for insertion as a token value. + + String refResourceType = refValue.getTargetResourceType(); + String refLogicalId = refValue.getValue(); + Integer refVersion = refValue.getVersion(); + + // Ignore references containing only a "display" element (apparently supported by the spec, + // but contains nothing useful to store because there's no searchable value). + // See ParameterVisitorBatchDAO + if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { + // protect against code regression. Invalid/improper references should be + // filtered out already. + logger.warning("Invalid reference parameter type: name='" + name + "' type=" + refValue.getType().name()); + throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); + } + + TokenParameter value = new TokenParameter(); + value.setName(name); + value.setValueSystem(refResourceType); + value.setValueCode(refLogicalId); + value.setRefVersionId(refVersion); + value.setCompositeId(compositeId); + builder.addTokenValue(value); + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 768e43ac8e0..c5f160fe19e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -22,6 +22,7 @@ import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.InteractionStatus; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; @@ -55,17 +56,46 @@ public class PostgresResourceDAO extends ResourceDAOImpl { // 13 args (9 in, 4 out) private static final String SQL_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; + // 14 args (10 in, 4 out) + private static final String SQL_DISTRIBUTED_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; + + private static final String SQL_DISTRIBUTED_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.SHARD_KEY = ? " + + " AND LR.LOGICAL_ID = ? " + + " AND R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " + + " AND R.SHARD_KEY = LR.SHARD_KEY "; + + // Read a specific version of the resource + private static final String SQL_DISTRIBUTED_VERSION_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.SHARD_KEY = ? " + + " AND LR.LOGICAL_ID = ? " + + " AND R.VERSION_ID = ? " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " + + " AND R.SHARD_KEY = LR.SHARD_KEY "; // DAO used to obtain sequence values from FHIR_REF_SEQUENCE private FhirRefSequenceDAO fhirRefSequenceDAO; - public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd) { + // The shard key used with distributed databases + private final Short shardKey; + + public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, Short shardKey) { super(connection, schemaName, flavor, cache, rrd); + this.shardKey = shardKey; } public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, - ParameterTransactionDataImpl ptdi) { + ParameterTransactionDataImpl ptdi, Short shardKey) { super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + this.shardKey = shardKey; } @Override @@ -87,41 +117,53 @@ public Resource insert(Resource resource, List paramete // hit the procedure Objects.requireNonNull(getResourceTypeId(resource.getResourceType())); - stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); + if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (this.shardKey == null) { + throw new FHIRPersistenceException("Shard key value required when schema type is DISTRIBUTED"); + } + stmtString = String.format(SQL_DISTRIBUTED_INSERT_WITH_PARAMETERS, getSchemaName()); + } else { + stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); + } stmt = connection.prepareCall(stmtString); - stmt.setString(1, resource.getResourceType()); - stmt.setString(2, resource.getLogicalId()); + int arg = 1; + if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + stmt.setShort(arg++, shardKey); + } + stmt.setString(arg++, resource.getResourceType()); + stmt.setString(arg++, resource.getLogicalId()); if (resource.getDataStream() != null) { - stmt.setBinaryStream(3, resource.getDataStream().inputStream()); + stmt.setBinaryStream(arg++, resource.getDataStream().inputStream()); } else { // payload was offloaded to another data store - stmt.setNull(3, Types.BINARY); + stmt.setNull(arg++, Types.BINARY); } lastUpdated = resource.getLastUpdated(); - stmt.setTimestamp(4, lastUpdated, CalendarHelper.getCalendarForUTC()); - stmt.setString(5, resource.isDeleted() ? "Y": "N"); - stmt.setString(6, UUID.randomUUID().toString()); - stmt.setInt(7, resource.getVersionId()); - stmt.setString(8, parameterHashB64); - setInt(stmt, 9, ifNoneMatch); - setString(stmt, 10, resource.getResourcePayloadKey()); - stmt.registerOutParameter(11, Types.BIGINT); - stmt.registerOutParameter(12, Types.VARCHAR); // The old parameter_hash - stmt.registerOutParameter(13, Types.INTEGER); // o_interaction_status - stmt.registerOutParameter(14, Types.INTEGER); // o_if_none_match_version + stmt.setTimestamp(arg++, lastUpdated, CalendarHelper.getCalendarForUTC()); + stmt.setString(arg++, resource.isDeleted() ? "Y": "N"); + stmt.setString(arg++, UUID.randomUUID().toString()); + stmt.setInt(arg++, resource.getVersionId()); + stmt.setString(arg++, parameterHashB64); + setInt(stmt, arg++, ifNoneMatch); + setString(stmt, arg++, resource.getResourcePayloadKey()); + + // TODO use a helper function which can return the arg index to help clean up the syntax + stmt.registerOutParameter(arg, Types.BIGINT); final int resourceIdIndex = arg++; + stmt.registerOutParameter(arg, Types.VARCHAR); final int oldParameterHashIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int interactionStatusIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int ifNoneMatchVersionIndex = arg++; dbCallStartTime = System.nanoTime(); stmt.execute(); dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(11)); - - if (stmt.getInt(13) == 1) { // interaction status + resource.setId(stmt.getLong(resourceIdIndex)); + if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status // no change, so skip parameter updates resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); - resource.setIfNoneMatchVersion(stmt.getInt(14)); // current version + resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version } else { resource.setInteractionStatus(InteractionStatus.MODIFIED); @@ -129,8 +171,11 @@ public Resource insert(Resource resource, List paramete // To keep things simple for the postgresql use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - final String currentParameterHash = stmt.getString(12); - if (parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + // For now we bypass parameter work for DISTRIBUTED schemas because the plan + // is to make loading async for better ingestion performance + final String currentParameterHash = stmt.getString(oldParameterHashIndex); + if (getFlavor().getSchemaType() != SchemaType.DISTRIBUTED + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() || !parameterHashB64.equals(currentParameterHash))) { // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); @@ -168,6 +213,56 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { return resource; } + @Override + public Resource read(String logicalId, String resourceType) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "read"; + logger.entering(CLASSNAME, METHODNAME); + + Resource resource = null; + if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_DISTRIBUTED_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, shardKey, logicalId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + logger.exiting(CLASSNAME, METHODNAME); + } + } else { + resource = super.read(logicalId, resourceType); + } + return resource; + } + + @Override + public Resource versionRead(String logicalId, String resourceType, int versionId) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "versionRead"; + logger.entering(CLASSNAME, METHODNAME); + + Resource resource = null; + if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + String stmtString = null; + + try { + stmtString = String.format(SQL_DISTRIBUTED_VERSION_READ, resourceType, resourceType); + List resources = this.runQuery(stmtString, shardKey, logicalId, versionId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + logger.exiting(CLASSNAME, METHODNAME); + } + } else { + resource = super.versionRead(logicalId, resourceType, versionId); + } + return resource; + + } + /** * Delete all parameters for the given resourceId from the parameters table * @@ -176,7 +271,7 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { * @param logicalResourceId * @throws SQLException */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { + private void deleteFromParameterTableX(Connection conn, String tableName, long logicalResourceId) throws SQLException { final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { // bind parameters diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java index 00ae1b1fbba..496b60a83dc 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java @@ -132,7 +132,7 @@ public void teardown() throws Exception { FHIRSearchContext ctx = searchHelper.parseQueryParameters(Location.class, Collections.emptyMap(), true, true); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(null, ctx); + FHIRPersistenceContextFactory.createPersistenceContext(null, ctx, null); com.ibm.fhir.model.type.Instant lastUpdated = FHIRPersistenceUtil.getUpdateTime(); persistence.delete(persistenceContext, savedResource.getClass(), savedResource.getId(), FHIRPersistenceSupport.getMetaVersionId(savedResource), lastUpdated); if (persistence.isTransactional()) { @@ -164,7 +164,7 @@ public MultiResourceResult runQueryTestMultiples(String searchParamCode, String. public MultiResourceResult runQueryTest(Map> queryParms) throws Exception { FHIRSearchContext ctx = searchHelper.parseQueryParameters(Location.class, queryParms, true, true); - FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, ctx); + FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, ctx, null); MultiResourceResult result = persistence.search(persistenceContext, Location.class); return result; } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java index 141979182ab..e1a637dec0f 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java @@ -18,6 +18,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; @@ -87,7 +88,7 @@ protected void erase() throws Exception { try (Connection c = createConnection()) { System.out.println("Got a Connection"); try { - FHIRDbFlavor flavor = new FHIRDbFlavorImpl(dbType, true); + FHIRDbFlavor flavor = new FHIRDbFlavorImpl(dbType, SchemaType.PLAIN); EraseResourceDAO dao = new EraseResourceDAO(c, FhirSchemaConstants.FHIR_ADMIN, translator, schemaName, flavor, new MockLocalCache(), null); ResourceEraseRecord record = new ResourceEraseRecord(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index e1ee53d72f4..cec3c1ffce1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -16,6 +16,7 @@ import static com.ibm.fhir.schema.app.menu.Menu.CREATE_SCHEMA_OAUTH; import static com.ibm.fhir.schema.app.menu.Menu.DB_TYPE; import static com.ibm.fhir.schema.app.menu.Menu.DELETE_TENANT_META; +import static com.ibm.fhir.schema.app.menu.Menu.DISTRIBUTED; import static com.ibm.fhir.schema.app.menu.Menu.DROP_ADMIN; import static com.ibm.fhir.schema.app.menu.Menu.DROP_DETACHED; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA; @@ -60,6 +61,7 @@ import static com.ibm.fhir.schema.app.util.CommonUtil.getDbAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.getPropertyAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.getRandomKey; +import static com.ibm.fhir.schema.app.util.CommonUtil.getSchemaAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.loadDriver; import static com.ibm.fhir.schema.app.util.CommonUtil.logClasspath; @@ -94,9 +96,11 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.api.TableSpaceRemovalException; import com.ibm.fhir.database.utils.api.TenantStatus; import com.ibm.fhir.database.utils.api.UndefinedNameException; @@ -311,6 +315,9 @@ public class Main { // Configuration to control how the LeaseManager operates private ILeaseManagerConfig leaseManagerConfig; + // Do we want to build the distributed flavor of the FHIR data schema? + private boolean distributed = false; + // ----------------------------------------------------------------------------------------------------------------- // The following method is related to the common methods and functions /** @@ -355,7 +362,8 @@ protected void configureConnectionPool() { */ protected void buildAdminSchemaModel(PhysicalDataModel pdm) { // Add the tenant and tenant_keys tables and any other admin schema stuff - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); + SchemaType schemaType = isMultitenant() ? SchemaType.MULTITENANT : SchemaType.PLAIN; + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), schemaType); gen.buildAdminSchema(pdm); } @@ -389,7 +397,7 @@ protected void buildJavaBatchSchemaModel(PhysicalDataModel pdm) { * @param collector * @param vhs */ - protected void applyModel(PhysicalDataModel pdm, IDatabaseAdapter adapter, ITaskCollector collector, VersionHistoryService vhs) { + protected void applyModel(PhysicalDataModel pdm, ISchemaAdapter adapter, ITaskCollector collector, VersionHistoryService vhs) { logger.info("Collecting model update tasks"); // If using a distributed RDBMS (Citus) then skip the initial FK creation SchemaApplyContext context = SchemaApplyContext.builder().setIncludeForeignKeys(!isDistributed()).build(); @@ -476,9 +484,9 @@ protected void updateSchemas() { // Make sure that we have the CONTROL table created before we try any // schema update work - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateControl.createTableIfNeeded(schema.getAdminSchemaName(), adapter); + CreateControl.createTableIfNeeded(schema.getAdminSchemaName(), schemaAdapter); } catch (UniqueConstraintViolationException x) { // Race condition - two or more instances trying to create the CONTROL table throw new ConcurrentUpdateException("Concurrent update - create control table"); @@ -504,9 +512,9 @@ protected void updateSchemas() { protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { FhirSchemaGenerator gen; if (resourceTypeSubset == null || resourceTypeSubset.isEmpty()) { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType()); } else { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant(), resourceTypeSubset); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType(), resourceTypeSubset); } gen.buildSchema(pdm); @@ -541,9 +549,9 @@ protected void updateFhirSchema() { } final String targetSchemaName = schema.getSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing @@ -562,7 +570,7 @@ protected void updateFhirSchema() { // Build/update the FHIR-related tables as well as the stored procedures PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); - boolean isNewDb = updateSchema(pdm); + boolean isNewDb = updateSchema(pdm, getSchemaType()); if (this.exitStatus == EXIT_OK) { // If the db is multi-tenant, we populate the resource types and parameter names in allocate-tenant. @@ -622,18 +630,18 @@ protected void updateOauthSchema() { } final String targetSchemaName = schema.getOauthSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildOAuthSchemaModel(pdm); - updateSchema(pdm); + updateSchema(pdm, SchemaType.PLAIN); if (this.exitStatus == EXIT_OK) { // Apply privileges if asked @@ -668,18 +676,18 @@ protected void updateJavaBatchSchema() { } final String targetSchemaName = schema.getJavaBatchSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildJavaBatchSchemaModel(pdm); - updateSchema(pdm); + updateSchema(pdm, SchemaType.PLAIN); if (this.exitStatus == EXIT_OK) { // Apply privileges if asked @@ -705,7 +713,7 @@ protected void updateJavaBatchSchema() { * Update the schema associated with the given {@link PhysicalDataModel} * @return true if the database is new */ - protected boolean updateSchema(PhysicalDataModel pdm) { + protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) { // The objects are applied in parallel, which relies on each object // expressing its dependencies correctly. Changes are only applied @@ -716,12 +724,13 @@ protected boolean updateSchema(PhysicalDataModel pdm) { ExecutorService pool = Executors.newFixedThreadPool(this.threadPoolSize); ITaskCollector collector = taskService.makeTaskCollector(pool); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(schemaType, dbType, connectionPool); // Before we start anything, we need to make sure our schema history // and control tables are in place. These tables are used to manage // all FHIR data, oauth and JavaBatch schemas we build try (ITransaction tx = transactionProvider.getTransaction()) { - CreateVersionHistory.createTableIfNeeded(schema.getAdminSchemaName(), adapter); + CreateVersionHistory.createTableIfNeeded(schema.getAdminSchemaName(), schemaAdapter); } // Current version history for the data schema @@ -735,7 +744,7 @@ protected boolean updateSchema(PhysicalDataModel pdm) { boolean isNewDb = vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == null || vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == 0; - applyModel(pdm, adapter, collector, vhs); + applyModel(pdm, schemaAdapter, collector, vhs); if (isDistributed()) { applyDistributionRules(pdm); } @@ -752,8 +761,8 @@ protected boolean updateSchema(PhysicalDataModel pdm) { private void applyDistributionRules(PhysicalDataModel pdm) { try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); - pdm.applyDistributionRules(adapter); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); + pdm.applyDistributionRules(schemaAdapter); } catch (RuntimeException x) { tx.setRollbackOnly(); throw x; @@ -765,7 +774,7 @@ private void applyDistributionRules(PhysicalDataModel pdm) { try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { final String tenantColumnName = isMultitenant() ? "mt_id" : null; - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter adapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName); pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); } catch (RuntimeException x) { @@ -860,6 +869,8 @@ protected void dropSchema() { try { JdbcTarget target = new JdbcTarget(c); IDatabaseAdapter adapter = getDbAdapter(dbType, target); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), adapter); + ISchemaAdapter plainSchemaAdapter = getSchemaAdapter(SchemaType.PLAIN, adapter); VersionHistoryService vhs = new VersionHistoryService(schema.getAdminSchemaName(), schema.getSchemaName(), schema.getOauthSchemaName(), schema.getJavaBatchSchemaName()); vhs.setTransactionProvider(transactionProvider); @@ -871,13 +882,15 @@ protected void dropSchema() { if (this.dropSplitTransaction) { // important that we use an adapter connected with the connection pool // (which is connected to the transaction provider) - IDatabaseAdapter txAdapter = getDbAdapter(dbType, connectionPool); - pdm.dropSplitTransaction(txAdapter, this.transactionProvider, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + ISchemaAdapter poolSchemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); + pdm.dropSplitTransaction(poolSchemaAdapter, this.transactionProvider, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); } else { // old fashioned drop where we do everything in one (big) transaction - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); } - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + + // Drop the whole-schema-version table + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -887,8 +900,8 @@ protected void dropSchema() { if (dropOauthSchema) { // Just drop the objects associated with the OAUTH schema group final String schemaName = schema.getOauthSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, OAuthSchemaGenerator.OAUTH_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, OAuthSchemaGenerator.OAUTH_GROUP); + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -898,8 +911,8 @@ protected void dropSchema() { if (dropJavaBatchSchema) { // Just drop the objects associated with the BATCH schema group final String schemaName = schema.getJavaBatchSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, JavaBatchSchemaGenerator.BATCH_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + pdm.drop(plainSchemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, JavaBatchSchemaGenerator.BATCH_GROUP); + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -910,7 +923,7 @@ protected void dropSchema() { // Just drop the objects associated with the ADMIN schema group CreateVersionHistory.generateTable(pdm, ADMIN_SCHEMANAME, true); CreateControl.buildTableDef(pdm, ADMIN_SCHEMANAME, true); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.ADMIN_GROUP); + pdm.drop(plainSchemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.ADMIN_GROUP); if (!checkSchemaIsEmpty(adapter, schema.getAdminSchemaName())) { throw new DataAccessException("Schema '" + schema.getAdminSchemaName() + "' not empty after drop"); } @@ -975,15 +988,15 @@ protected void updateProcedures() { try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try (Connection c = connectionPool.getConnection();) { try { - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.applyProcedures(adapter, context); - pdm.applyFunctions(adapter, context); + pdm.applyProcedures(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); // Because we're replacing the procedures, we should also check if // we need to apply the associated privileges if (this.grantTo != null) { - pdm.applyProcedureAndFunctionGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + pdm.applyProcedureAndFunctionGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); } } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -1026,16 +1039,16 @@ protected void buildCommonModel(PhysicalDataModel pdm, boolean addFhirDataSchema */ protected void grantPrivilegesForFhirData() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); // Grant SELECT on WHOLE_SCHEMA_VERSION to the FHIR server user // Note the constant comes from SchemaConstants on purpose - CreateWholeSchemaVersion.grantPrivilegesTo(adapter, schema.getSchemaName(), SchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + CreateWholeSchemaVersion.grantPrivilegesTo(schemaAdapter, schema.getSchemaName(), SchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed tx.setRollbackOnly(); @@ -1048,12 +1061,12 @@ protected void grantPrivilegesForFhirData() { * Apply grants to the OAuth schema objects */ protected void grantPrivilegesForOAuth() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildOAuthSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -1068,15 +1081,15 @@ protected void grantPrivilegesForOAuth() { * Apply grants to the JavaBatch schema objects */ protected void grantPrivilegesForBatch() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildJavaBatchSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_BATCH_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_BATCH_GRANT_GROUP, grantTo); // special case for the JavaBatch schema in PostgreSQL - adapter.grantAllSequenceUsage(schema.getJavaBatchSchemaName(), grantTo); + schemaAdapter.grantAllSequenceUsage(schema.getJavaBatchSchemaName(), grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -1127,6 +1140,20 @@ protected boolean isMultitenant() { return MULTITENANT_FEATURE_ENABLED.contains(this.dbType); } + /** + * What type of schema do we want to build? + * @return + */ + protected SchemaType getSchemaType() { + if (isMultitenant()) { + return SchemaType.MULTITENANT; + } else if (isDistributed()) { + return SchemaType.DISTRIBUTED; + } else { + return SchemaType.PLAIN; + } + } + // ----------------------------------------------------------------------------------------------------------------- // The following methods are related to Multi-Tenant only. /** @@ -1308,7 +1335,7 @@ protected void allocateTenant() { } // Build/update the tables as well as the stored procedures - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1441,7 +1468,7 @@ protected void refreshTenants() { if (ti.getTenantSchema() != null && (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema()))) { // It's crucial we use the correct schema for each particular tenant, which // is why we have to build the PhysicalDataModel separately for each tenant - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), isMultitenant()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), getSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1659,7 +1686,7 @@ protected void dropTenant() { TenantInfo tenantInfo = freezeTenant(); // Build the model of the data (FHIRDATA) schema which is then used to drive the drop - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1678,7 +1705,7 @@ protected void dropTenant() { protected void dropDetachedPartitionTables() { TenantInfo tenantInfo = getTenantInfo(); - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -2131,6 +2158,9 @@ protected void parseArgs(String[] args) { case CONFIRM_DROP: this.confirmDrop = true; break; + case DISTRIBUTED: + this.distributed = true; + break; case ALLOCATE_TENANT: if (++i < args.length) { this.tenantName = args[i]; @@ -2616,11 +2646,11 @@ public void updateVacuumSettings() { } /** - * Citus is a distributed implementation of PostgreSQL + * Should we build the distributed variant of the FHIR data schema * @return */ private boolean isDistributed() { - return this.dbType == DbType.CITUS; + return this.distributed; } /** diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java index a0e28f8d470..d3323afe176 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java @@ -46,9 +46,12 @@ import java.util.Properties; import java.util.concurrent.Executor; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; +import com.ibm.fhir.database.utils.derby.DerbyMaster; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.version.CreateVersionHistory; import com.ibm.fhir.database.utils.version.VersionHistoryService; @@ -90,7 +93,7 @@ public class SchemaPrinter { private static final String STORED_PROCEDURE_DELIMITER = "@"; private final boolean toFile; - private final boolean multitenant; + private final SchemaType schemaType; private File schemaFile = new File("schema.sql"); private File spFile = new File("stored-procedures.sql"); private File grantFile = new File("grants.sql"); @@ -105,11 +108,15 @@ public class SchemaPrinter { /** * constructor that switches behavior toFile our output stream. + * + * @param toFile + * @param schemaType + * @throws FileNotFoundException */ - public SchemaPrinter(boolean toFile, boolean multitenant) throws FileNotFoundException { + public SchemaPrinter(boolean toFile, SchemaType schemaType) throws FileNotFoundException { this.toFile = toFile; - this.multitenant = multitenant; + this.schemaType = schemaType; if (this.toFile) { out = new PrintStream(new FileOutputStream(schemaFile)); @@ -154,9 +161,10 @@ public void process() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = DerbyMaster.wrap(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -166,7 +174,7 @@ public void process() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, multitenant); + FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, this.schemaType); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -176,7 +184,7 @@ public void process() { JavaBatchSchemaGenerator javaBatchSchemaGenerator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); javaBatchSchemaGenerator.buildJavaBatchSchema(model); SchemaApplyContext context = SchemaApplyContext.getDefault(); - model.apply(adapter, context); + model.apply(schemaAdapter, context); } public void processApplyGrants() { @@ -184,9 +192,10 @@ public void processApplyGrants() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = DerbyMaster.wrap(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -196,7 +205,7 @@ public void processApplyGrants() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, multitenant); + FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, schemaType); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -208,7 +217,7 @@ public void processApplyGrants() { // clear it out. commands.clear(); - model.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, "FHIRUSER"); + model.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, "FHIRUSER"); } /** @@ -262,6 +271,7 @@ public void print() { public static void main(String[] args) { boolean outputToFile = false; boolean multitenant = false; + boolean distributed = false; String outputFile = ""; // If there are files @@ -274,13 +284,22 @@ public static void main(String[] args) { case "--multitenant": multitenant = true; break; + case "--distributed": + distributed = true; + break; default: throw new IllegalArgumentException("Invalid argument: " + arg); } } + if (multitenant && distributed) { + throw new IllegalArgumentException("--multitenant and --distributed are mutually exclusive"); + } + + SchemaType schemaType = multitenant ? SchemaType.MULTITENANT : distributed ? SchemaType.DISTRIBUTED : SchemaType.PLAIN; + try { - SchemaPrinter printer = new SchemaPrinter(outputToFile, multitenant); + SchemaPrinter printer = new SchemaPrinter(outputToFile, schemaType); printer.process(); printer.print(); printer.processApplyGrants(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java index 0490765f477..2390b165898 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java @@ -62,6 +62,7 @@ public class Menu { public static final String HELP = "--help"; public static final String SHOW_DB_SIZE = "--show-db-size"; public static final String SHOW_DB_SIZE_DETAIL = "--show-db-size-detail"; + public static final String DISTRIBUTED = "--distributed"; public Menu() { // NOP @@ -116,6 +117,7 @@ public enum HelpMenu { MI_CREATE_SCHEMA_FHIR(CREATE_SCHEMA_FHIR, "schemaName", "Create the FHIR Data Schema"), MI_CREATE_SCHEMA_BATCH(CREATE_SCHEMA_BATCH, "schemaName", "Create the Batch Schema"), MI_CREATE_SCHEMA_OAUTH(CREATE_SCHEMA_OAUTH, "schemaName", "Create the OAuth Schema"), + MI_DISTRIBUTED(DISTRIBUTED, "", "Build the distributed variant of the FHIR data schema"), MI_SHOW_DB_SIZE(SHOW_DB_SIZE, "", "Generate report with a breakdown of database size"), MI_SHOW_DB_SIZE_DETAIL(SHOW_DB_SIZE_DETAIL, "", "Include detailed table and index info in size report"); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java index 37be2eb4d79..9042fbf1cde 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java @@ -19,6 +19,8 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.citus.CitusAdapter; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.common.JdbcTarget; @@ -30,6 +32,9 @@ import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresAdapter; import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; +import com.ibm.fhir.schema.build.DistributedSchemaAdapter; +import com.ibm.fhir.schema.build.FhirSchemaAdapter; +import com.ibm.fhir.schema.control.FhirSchemaConstants; /** * @@ -127,6 +132,47 @@ public static IDatabaseAdapter getDbAdapter(DbType dbType, JdbcTarget target) { throw new IllegalStateException("Unsupported db type: " + dbType); } } + /** + * Get the schema adapter which will build the schema variant described by + * the given schemaType + * @param schemaType + * @param dbType + * @param connectionProvider + * @return + */ + public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, DbType dbType, IConnectionProvider connectionProvider) { + IDatabaseAdapter dbAdapter = getDbAdapter(dbType, connectionProvider); + switch (schemaType) { + case PLAIN: + return new FhirSchemaAdapter(dbAdapter); + case MULTITENANT: + return new FhirSchemaAdapter(dbAdapter); + case DISTRIBUTED: + return new DistributedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + default: + throw new IllegalArgumentException("Unsupported schema type: " + schemaType); + } + } + + /** + * Wrap the given databaseAdapter in an ISchemaAdapter implementation selected + * by the given schemaType + * @param schemaType + * @param dbAdapter + * @return + */ + public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, IDatabaseAdapter dbAdapter) { + switch (schemaType) { + case PLAIN: + return new FhirSchemaAdapter(dbAdapter); + case MULTITENANT: + return new FhirSchemaAdapter(dbAdapter); + case DISTRIBUTED: + return new DistributedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + default: + throw new IllegalArgumentException("Unsupported schema type: " + schemaType); + } + } public static IDatabaseAdapter getDbAdapter(DbType dbType, IConnectionProvider connectionProvider) { switch (dbType) { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java new file mode 100644 index 00000000000..6a637d46e33 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java @@ -0,0 +1,125 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.SmallIntColumn; +import com.ibm.fhir.database.utils.model.With; + +/** + * Adapter implementation used to build the distributed variant of + * the IBM FHIR Server RDBMS schema. + * + * This schema adds a distribution key column to every table identified as + * distributed. This column is also added to every index and FK relationship + * as needed. We use a smallint (2 bytes) which represents a signed integer + * holding values in the range [-32768, 32767]. This provides sufficient spread, + * assuming we won't be using a database with thousands of nodes. + */ +public class DistributedSchemaAdapter extends FhirSchemaAdapter { + + // The distribution column to add to each table marked as distributed + final String distributionColumnName; + + /** + * @param databaseAdapter + */ + public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter, String distributionColumnName) { + super(databaseAdapter); + this.distributionColumnName = distributionColumnName; + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType) { + // If the table is distributed, we need to inject the distribution column into the columns list. This same + // column will need to be injected into each of the index definitions + List actualColumns = new ArrayList<>(); + if (distributionType == DistributionType.DISTRIBUTED) { + ColumnBase distributionColumn = new SmallIntColumn(distributionColumnName, false, null); + actualColumns.add(distributionColumn); + if (primaryKey != null) { + // we need to alter the primary so it includes the distribution column + // as the last member + List newCols = new ArrayList<>(primaryKey.getColumns()); + newCols.add(distributionColumnName); + primaryKey = new PrimaryKeyDef(primaryKey.getConstraintName(), newCols); + } + } + + actualColumns.addAll(columns); + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createTable(schemaName, name, tenantColumnName, actualColumns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, includeColumns, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, dc); + } + + @Override + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { + // for non-unique indexes, we don't need to include the distribution column +// List actualColumns = new ArrayList<>(indexColumns); +// if (distributionType == DistributionType.DISTRIBUTED) { +// // inject the distribution column into the index definition +// actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); +// } + + // Create the index using the modified set of index columns + databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + } + + @Override + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, + String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { + // If both this and the target table are distributed, we need to add the distributionColumnName + // to the FK relationship definition. If the target is a reference, it won't have the shard_key + // column because the table is fully replicated across all nodes and therefore any FK relationship + // can be based on the original PK definition without the extra sharding column. + List newCols = new ArrayList<>(columns); + if (distributionType == DistributionType.DISTRIBUTED && !targetIsReference) { + newCols.add(distributionColumnName); + } + databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, newCols, enforced); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java new file mode 100644 index 00000000000..386b435327b --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java @@ -0,0 +1,25 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; + +/** + * Represents an adapter used to build the standard FHIR schema + */ +public class FhirSchemaAdapter extends PlainSchemaAdapter { + + /** + * Public constructor + * + * @param databaseAdapter + */ + public FhirSchemaAdapter(IDatabaseAdapter databaseAdapter) { + super(databaseAdapter); + } +} diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java index 1b8d4822e5f..cea31177fc8 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java @@ -7,7 +7,7 @@ import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.model.DataModelVisitorBase; import com.ibm.fhir.database.utils.model.ForeignKeyConstraint; import com.ibm.fhir.database.utils.model.Table; @@ -22,14 +22,13 @@ public class AddForeignKey extends DataModelVisitorBase { private static final Logger logger = Logger.getLogger(DropForeignKey.class.getName()); // The database adapter used to issue changes to the database - private final IDatabaseAdapter adapter; + private final ISchemaAdapter adapter; private final String tenantColumnName; - /** * Public constructor * @param adapter */ - public AddForeignKey(IDatabaseAdapter adapter, String tenantColumnName) { + public AddForeignKey(ISchemaAdapter adapter, String tenantColumnName) { this.adapter = adapter; this.tenantColumnName = tenantColumnName; } @@ -37,7 +36,9 @@ public AddForeignKey(IDatabaseAdapter adapter, String tenantColumnName) { @Override public void visited(Table fromChildTable, ForeignKeyConstraint fk) { // Enable (add) the FK constraint + // TODO handle distributed tables...need the src and target distribution types + // so we know whether or not to inject the distribution column into the FK logger.info(String.format("Adding foreign key: %s.%s[%s]", fromChildTable.getSchemaName(), fromChildTable.getObjectName(), fk.getConstraintName())); - fk.apply(fromChildTable.getSchemaName(), fromChildTable.getObjectName(), this.tenantColumnName, adapter); + fk.apply(fromChildTable.getSchemaName(), fromChildTable.getObjectName(), this.tenantColumnName, adapter, fromChildTable.getDistributionType()); } } \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java deleted file mode 100644 index 58cebcd98bf..00000000000 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirDistributedSchemaGenerator.java +++ /dev/null @@ -1,1544 +0,0 @@ -/* - * (C) Copyright IBM Corp. 2019, 2022 - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.ibm.fhir.schema.control; - -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CANONICAL_URL_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TSTAMP; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CHANGE_TYPE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEMS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.CODE_SYSTEM_NAME; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_CANONICAL_VALUES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMMON_TOKEN_VALUE_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPARTMENT_LOGICAL_RESOURCE_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.COMPARTMENT_NAME_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_END; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_START; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_GROUP_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.FRAGMENT_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.IDX; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.IS_DELETED; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LAST_UPDATED; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_COMPARTMENTS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SHARDS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.MT_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_HASH_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAMES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.PARAMETER_NAME_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VERSION_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.REINDEX_TSTAMP; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.REINDEX_TXID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_CHANGE_LOG; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TOKEN_REFS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_TYPE_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.STR_VALUE_LCASE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANTS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_HASH; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_KEYS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_KEY_ID; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_NAME; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_SALT; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_SEQUENCE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TENANT_STATUS; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.TOKEN_VALUE; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.URL; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_BYTES; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.VERSION_ID; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import com.ibm.fhir.database.utils.api.IDatabaseStatement; -import com.ibm.fhir.database.utils.common.AddColumn; -import com.ibm.fhir.database.utils.common.CreateIndexStatement; -import com.ibm.fhir.database.utils.common.DropColumn; -import com.ibm.fhir.database.utils.common.DropIndex; -import com.ibm.fhir.database.utils.common.DropTable; -import com.ibm.fhir.database.utils.model.AlterSequenceStartWith; -import com.ibm.fhir.database.utils.model.BaseObject; -import com.ibm.fhir.database.utils.model.ColumnBase; -import com.ibm.fhir.database.utils.model.ColumnDefBuilder; -import com.ibm.fhir.database.utils.model.FunctionDef; -import com.ibm.fhir.database.utils.model.Generated; -import com.ibm.fhir.database.utils.model.GroupPrivilege; -import com.ibm.fhir.database.utils.model.IDatabaseObject; -import com.ibm.fhir.database.utils.model.NopObject; -import com.ibm.fhir.database.utils.model.ObjectGroup; -import com.ibm.fhir.database.utils.model.OrderedColumnDef; -import com.ibm.fhir.database.utils.model.PhysicalDataModel; -import com.ibm.fhir.database.utils.model.Privilege; -import com.ibm.fhir.database.utils.model.ProcedureDef; -import com.ibm.fhir.database.utils.model.Sequence; -import com.ibm.fhir.database.utils.model.SessionVariableDef; -import com.ibm.fhir.database.utils.model.Table; -import com.ibm.fhir.database.utils.model.Tablespace; -import com.ibm.fhir.database.utils.model.With; -import com.ibm.fhir.database.utils.postgres.PostgresFillfactorSettingDAO; -import com.ibm.fhir.database.utils.postgres.PostgresVacuumSettingDAO; -import com.ibm.fhir.model.util.ModelSupport; - -/** - * Creates a distributed variant of the FHIR data schema. This schema distributes - * tables associated with certain resource types using a shard key (which in - * reality is a patient identifier which can be used to scope interactions) - * - * In general, the schema is largely similar to the - */ -public class FhirDistributedSchemaGenerator { - private static final Logger logger = Logger.getLogger(FhirDistributedSchemaGenerator.class.getName()); - - // The schema holding all the data-bearing tables - private final String schemaName; - - // The schema used for administration objects like the tenants table, variable etc - private final String adminSchemaName; - - // Build the multitenant variant of the schema - private final boolean multitenant; - - // No abstract types - private static final Set ALL_RESOURCE_TYPES = ModelSupport.getResourceTypes(false).stream() - .map(t -> ModelSupport.getTypeName(t).toUpperCase()) - .collect(Collectors.toSet()); - - private static final String ADD_CODE_SYSTEM = "ADD_CODE_SYSTEM"; - private static final String ADD_PARAMETER_NAME = "ADD_PARAMETER_NAME"; - private static final String ADD_RESOURCE_TYPE = "ADD_RESOURCE_TYPE"; - private static final String ADD_ANY_RESOURCE = "ADD_ANY_RESOURCE"; - - // Special procedure for Citus database support - private static final String ADD_LOGICAL_RESOURCE = "ADD_LOGICAL_RESOURCE"; - private static final String DELETE_RESOURCE_PARAMETERS = "DELETE_RESOURCE_PARAMETERS"; - private static final String ERASE_RESOURCE = "ERASE_RESOURCE"; - - // The tags we use to separate the schemas - public static final String SCHEMA_GROUP_TAG = "SCHEMA_GROUP"; - public static final String FHIRDATA_GROUP = "FHIRDATA"; - public static final String ADMIN_GROUP = "FHIR_ADMIN"; - - // ADMIN SCHEMA CONTENT - - // Sequence used by the admin tenant tables - private Sequence tenantSequence; - - // The session variable used for row access control. All tables depend on this - private SessionVariableDef sessionVariable; - - private Table tenantsTable; - private Table tenantKeysTable; - - private static final String SET_TENANT = "SET_TENANT"; - - // The set of dependencies common to all of our admin stored procedures - private Set adminProcedureDependencies = new HashSet<>(); - - // A NOP marker used to ensure procedures are only applied after all the create - // table statements are applied - to avoid DB2 catalog deadlocks - private IDatabaseObject allAdminTablesComplete; - - // Marker used to indicate that the admin schema is all done - private IDatabaseObject adminSchemaComplete; - - // The resource types to generate schema for - private final Set resourceTypes; - - // The common sequence used for allocated resource ids - private Sequence fhirSequence; - - // The sequence used for the reference tables (parameter_names, code_systems etc) - private Sequence fhirRefSequence; - - // The set of dependencies common to all of our resource procedures - private Set procedureDependencies = new HashSet<>(); - - private Table codeSystemsTable; - private Table parameterNamesTable; - private Table resourceTypesTable; - private Table commonTokenValuesTable; - - // A NOP marker used to ensure procedures are only applied after all the create - // table statements are applied - to avoid DB2 catalog deadlocks - private IDatabaseObject allTablesComplete; - - // Privileges needed by the stored procedures - private List procedurePrivileges = new ArrayList<>(); - - // Privileges needed for access to the FHIR resource data tables - private List resourceTablePrivileges = new ArrayList<>(); - - // Privileges needed for reading the sv_tenant_id variable - private List variablePrivileges = new ArrayList<>(); - - // Privileges needed for using the fhir sequence - private List sequencePrivileges = new ArrayList<>(); - - // The default tablespace used for everything not specific to a tenant - private Tablespace fhirTablespace; - - /** - * Generate the IBM FHIR Server Schema for all resourceTypes - * - * @param adminSchemaName - * @param schemaName - */ - public FhirDistributedSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant) { - this(adminSchemaName, schemaName, multitenant, ALL_RESOURCE_TYPES); - } - - /** - * Generate the IBM FHIR Server Schema with just the given resourceTypes - * - * @param adminSchemaName - * @param schemaName - */ - public FhirDistributedSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant, Set resourceTypes) { - this.adminSchemaName = adminSchemaName; - this.schemaName = schemaName; - this.multitenant = multitenant; - - // The FHIR user (e.g. "FHIRSERVER") will need these privileges to be granted to it. Note that - // we use the group identified by FHIR_USER_GRANT_GROUP here - these privileges can be applied - // to any DB2 user using an admin user, or another user with sufficient GRANT TO privileges. - - - // The FHIRSERVER user gets EXECUTE privilege specifically on the SET_TENANT procedure, which is - // owned by the admin user, not the FHIRSERVER user. - procedurePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.EXECUTE)); - - // FHIRSERVER needs INSERT, SELECT, UPDATE and DELETE on all the resource data tables - resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.INSERT)); - resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.SELECT)); - resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.UPDATE)); - resourceTablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.DELETE)); - - // FHIRSERVER gets only READ privilege to the SV_TENANT_ID variable. The only way FHIRSERVER can - // set (write to) SV_TENANT_ID is by calling the SET_TENANT stored procedure, which requires - // both TENANT_NAME and TENANT_KEY to be provided. - variablePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.READ)); - - // FHIRSERVER gets to use the FHIR sequence - sequencePrivileges.add(new GroupPrivilege(FhirSchemaConstants.FHIR_USER_GRANT_GROUP, Privilege.USAGE)); - - this.resourceTypes = resourceTypes; - } - - /** - * Build the admin part of the schema. One admin schema can support multiple FHIRDATA - * schemas. It is also possible to have multiple admin schemas (on a dev system, - * for example, although in production there would probably be just one admin schema - * in a given database - * @param model - */ - public void buildAdminSchema(PhysicalDataModel model) { - // All tables are added to this new tablespace (which has a small extent size. - // Each tenant partition gets its own tablespace - fhirTablespace = new Tablespace(FhirSchemaConstants.FHIR_TS, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_TS_EXTENT_KB); - fhirTablespace.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - model.addObject(fhirTablespace); - - addTenantSequence(model); - addTenantTable(model); - addTenantKeysTable(model); - addVariable(model); - - // Add a NopObject which acts as a single dependency marker for the procedure objects to depend on - this.allAdminTablesComplete = new NopObject(adminSchemaName, "allAdminTablesComplete"); - this.allAdminTablesComplete.addDependencies(adminProcedureDependencies); - this.allAdminTablesComplete.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - model.addObject(allAdminTablesComplete); - - // The set_tenant procedure can be created after all the admin tables are done - final String ROOT_DIR = "db2/"; - ProcedureDef setTenant = model.addProcedure(this.adminSchemaName, SET_TENANT, 2, - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, adminSchemaName, - ROOT_DIR + SET_TENANT.toLowerCase() + ".sql", null), - Arrays.asList(allAdminTablesComplete), - procedurePrivileges); - setTenant.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - - // A final marker which is used to block any FHIR data schema activity until the admin schema is completed - this.adminSchemaComplete = new NopObject(adminSchemaName, "adminSchemaComplete"); - this.adminSchemaComplete.addDependencies(Arrays.asList(setTenant)); - this.adminSchemaComplete.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - model.addObject(adminSchemaComplete); - } - - /** - * Add the session variable we need. This variable is used to support multi-tenancy - * via the row-based access control permission predicate. - * @param model - */ - public void addVariable(PhysicalDataModel model) { - this.sessionVariable = new SessionVariableDef(adminSchemaName, "SV_TENANT_ID", FhirSchemaVersion.V0001.vid()); - this.sessionVariable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - variablePrivileges.forEach(p -> p.addToObject(this.sessionVariable)); - - // Make sure any admin procedures are built after the session variable - adminProcedureDependencies.add(this.sessionVariable); - model.addObject(this.sessionVariable); - } - - /** - * Create a table to manage the list of tenants. The tenant id is used - * as a partition value for all the other tables - * @param model - */ - protected void addTenantTable(PhysicalDataModel model) { - - this.tenantsTable = Table.builder(adminSchemaName, TENANTS) - .addIntColumn( MT_ID, false) - .addVarcharColumn( TENANT_NAME, 36, false) // probably a UUID - .addVarcharColumn( TENANT_STATUS, 16, false) - .addUniqueIndex(IDX + "TENANT_TN", TENANT_NAME) - .addPrimaryKey("TENANT_PK", MT_ID) - .setTablespace(fhirTablespace) - .build(model); - - this.tenantsTable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - this.adminProcedureDependencies.add(tenantsTable); - model.addTable(tenantsTable); - model.addObject(tenantsTable); - } - - /** - * Each tenant can have multiple access keys which are used to authenticate and authorize - * clients for access to the data for a given tenant. We support multiple keys per tenant - * as a way to allow key rotation in the configuration without impacting service continuity - * @param model - */ - protected void addTenantKeysTable(PhysicalDataModel model) { - - this.tenantKeysTable = Table.builder(adminSchemaName, TENANT_KEYS) - .addIntColumn( TENANT_KEY_ID, false) // PK - .addIntColumn( MT_ID, false) // FK to TENANTS - .addVarcharColumn( TENANT_SALT, 44, false) // 32 bytes == 44 Base64 symbols - .addVarbinaryColumn( TENANT_HASH, 32, false) // SHA-256 => 32 bytes - .addUniqueIndex(IDX + "TENANT_KEY_SALT", TENANT_SALT) // we want every salt to be unique - .addUniqueIndex(IDX + "TENANT_KEY_TIDH", MT_ID, TENANT_HASH) // for set_tenant query - .addPrimaryKey("TENANT_KEY_PK", TENANT_KEY_ID) - .addForeignKeyConstraint(FK + TENANT_KEYS + "_TNID", adminSchemaName, TENANTS, MT_ID) // dependency - .setTablespace(fhirTablespace) - .build(model); - - this.tenantKeysTable.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - this.adminProcedureDependencies.add(tenantKeysTable); - model.addTable(tenantKeysTable); - model.addObject(tenantKeysTable); - } - - /** - 
-    CREATE SEQUENCE fhir_sequence
-             AS BIGINT
-     START WITH 1
-          CACHE 1000
-       NO CYCLE;
-
- * - * @param pdm - */ - protected void addTenantSequence(PhysicalDataModel pdm) { - this.tenantSequence = new Sequence(adminSchemaName, TENANT_SEQUENCE, FhirSchemaVersion.V0001.vid(), 1, 1000); - this.tenantSequence.addTag(SCHEMA_GROUP_TAG, ADMIN_GROUP); - adminProcedureDependencies.add(tenantSequence); - sequencePrivileges.forEach(p -> p.addToObject(tenantSequence)); - - pdm.addObject(tenantSequence); - } - - /** - * Create the schema using the given target - * @param model - */ - public void buildSchema(PhysicalDataModel model) { - // Build the complete physical model so that we know it's consistent - buildAdminSchema(model); - addFhirSequence(model); - addFhirRefSequence(model); - addParameterNames(model); - addCodeSystems(model); - addCommonTokenValues(model); - addResourceTypes(model); - addLogicalResourceShards(model); - addLogicalResources(model); // for system-level parameter search - addReferencesSequence(model); - addLogicalResourceCompartments(model); - addResourceChangeLog(model); // track changes for easier export - addCommonCanonicalValues(model); // V0014 - addLogicalResourceProfiles(model); // V0014 - addLogicalResourceTags(model); // V0014 - addLogicalResourceSecurity(model); // V0016 - addErasedResources(model); // V0023 - - Table globalStrValues = addResourceStrValues(model); // for system-level _profile parameters - Table globalDateValues = addResourceDateValues(model); // for system-level date parameters - - // new normalized table for supporting token data (replaces TOKEN_VALUES) - Table globalResourceTokenRefs = addResourceTokenRefs(model); - - // The three "global" tables aren't true dependencies, but this was the easiest way to force sequential processing - // and avoid a pesky deadlock issue we were hitting while adding foreign key constraints on the global tables - addResourceTables(model, globalStrValues, globalDateValues, globalResourceTokenRefs); - - // All the table objects and types should be ready now, so create our NOP - // which is used as a single dependency for all procedures. This means - // procedures won't start until all the create table/type etc statements - // are done...hopefully reducing the number of deadlocks we see. - this.allTablesComplete = new NopObject(schemaName, "allTablesComplete"); - this.allTablesComplete.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.allTablesComplete.addDependencies(procedureDependencies); - model.addObject(allTablesComplete); - } - - public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) { - // These procedures just depend on the table they are manipulating and the fhir sequence. But - // to avoid deadlocks, we only apply them after all the tables are done, so we make all - // procedures depend on the allTablesComplete marker. - final String ROOT_DIR = "db2/"; - ProcedureDef pd = model.addProcedure(this.schemaName, - ADD_CODE_SYSTEM, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - pd = model.addProcedure(this.schemaName, - ADD_PARAMETER_NAME, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - pd = model.addProcedure(this.schemaName, - ADD_RESOURCE_TYPE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - pd = model.addProcedure(this.schemaName, - DELETE_RESOURCE_PARAMETERS, - FhirSchemaVersion.V0020.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - final ProcedureDef deleteResourceParameters = pd; - - pd = model.addProcedure(this.schemaName, - ADD_ANY_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - pd = model.addProcedure(this.schemaName, - ERASE_RESOURCE, - FhirSchemaVersion.V0013.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), - procedurePrivileges); - pd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - } - - public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { - // Add stored procedures/functions for PostgreSQL - // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. - final String ROOT_DIR = "postgres/"; - FunctionDef fd = model.addFunction(this.schemaName, - ADD_CODE_SYSTEM, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), - procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_PARAMETER_NAME, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_RESOURCE_TYPE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // We currently only support functions with PostgreSQL, although this is really just a procedure - FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, - DELETE_RESOURCE_PARAMETERS, - FhirSchemaVersion.V0020.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges); - deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_ANY_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ERASE_RESOURCE, - FhirSchemaVersion.V0013.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - } - - /** - * @implNote following the current pattern, which is why all this stuff is replicated - * @param model - */ - public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { - // Add stored procedures/functions for postgresql and Citus - // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. - final String ROOT_DIR = "postgres/"; - final String CITUS_ROOT_DIR = "citus/"; - FunctionDef fd = model.addFunction(this.schemaName, - ADD_CODE_SYSTEM, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), - procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_PARAMETER_NAME, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_RESOURCE_TYPE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // Add the delete resource parameters function and distribute using logical_resource_id (param $2) - FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, - DELETE_RESOURCE_PARAMETERS, - FhirSchemaVersion.V0020.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges, 2); - deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // Use the Citus-specific function which is distributed using logical_resource_id (param $1) - fd = model.addFunction(this.schemaName, ADD_LOGICAL_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), - procedurePrivileges, 1); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - final FunctionDef addLogicalResource = fd; - - // Use the Citus-specific variant of add_any_resource and distribute using logical_resource_id (param $1) - fd = model.addFunction(this.schemaName, ADD_ANY_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete, addLogicalResource), - procedurePrivileges, 1); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ERASE_RESOURCE, - FhirSchemaVersion.V0013.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - } - - /** - * Add the system-wide logical_resources table. Note that LOGICAL_ID is - * denormalized, stored in both LOGICAL_RESOURCES and _LOGICAL_RESOURCES. - * This avoids an additional join, and simplifies the migration to this - * new schema model. - * @param pdm - */ - public void addLogicalResources(PhysicalDataModel pdm) { - final String tableName = LOGICAL_RESOURCES; - final String mtId = this.multitenant ? MT_ID : null; - - final String IDX_LOGICAL_RESOURCES_RITS = "IDX_" + LOGICAL_RESOURCES + "_RITS"; - final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD"; - - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addIntColumn(RESOURCE_TYPE_ID, false) - .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) - .addTimestampColumn(REINDEX_TSTAMP, false, "CURRENT_TIMESTAMP") // new column for V0006 - .addBigIntColumn(REINDEX_TXID, false, "0") // new column for V0006 - .addTimestampColumn(LAST_UPDATED, true) // new column for V0014 - .addCharColumn(IS_DELETED, 1, false, "'X'") - .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true) // new column for V0015 - .addPrimaryKey(tableName + "_PK", LOGICAL_RESOURCE_ID) - .addUniqueIndex("UNQ_" + LOGICAL_RESOURCES, RESOURCE_TYPE_ID, LOGICAL_ID) - .addIndex(IDX_LOGICAL_RESOURCES_RITS, new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null)) - .addIndex(IDX_LOGICAL_RESOURCES_LUPD, new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null)) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) - .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addWiths(addWiths()) // add table tuning - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion == FhirSchemaVersion.V0001.vid()) { - // Add statements to migrate from version V0001 to V0006 of this object - List cols = ColumnDefBuilder.builder() - .addTimestampColumn(REINDEX_TSTAMP, false, "CURRENT_TIMESTAMP") - .addBigIntColumn(REINDEX_TXID, false, "0") - .buildColumns(); - - statements.add(new AddColumn(schemaName, tableName, cols.get(0))); - statements.add(new AddColumn(schemaName, tableName, cols.get(1))); - - // Add the new index on REINDEX_TSTAMP. This index is special because it's the - // first index in our schema to use DESC. - List indexCols = Arrays.asList(new OrderedColumnDef(REINDEX_TSTAMP, OrderedColumnDef.Direction.DESC, null)); - statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_RITS, tableName, mtId, indexCols)); - } - - if (priorVersion < FhirSchemaVersion.V0009.vid()) { - // Get rid of the old global token values parameter table which no longer - // used - statements.add(new DropTable(schemaName, "TOKEN_VALUES")); - } - - if (priorVersion < FhirSchemaVersion.V0014.vid()) { - // Add LAST_UPDATED and IS_DELETED to whole-system logical_resources - List cols = ColumnDefBuilder.builder() - .addTimestampColumn(LAST_UPDATED, true) - .addCharColumn(IS_DELETED, 1, false, "'X'") - .buildColumns(); - - statements.add(new AddColumn(schemaName, tableName, cols.get(0))); - statements.add(new AddColumn(schemaName, tableName, cols.get(1))); - - // New index on the LAST_UPDATED. We don't need to include resource-type. If - // you know the resource type, you'll be querying the resource-specific - // xx_logical_resources table instead - List indexCols = Arrays.asList(new OrderedColumnDef(LAST_UPDATED, OrderedColumnDef.Direction.ASC, null)); - statements.add(new CreateIndexStatement(schemaName, IDX_LOGICAL_RESOURCES_LUPD, tableName, mtId, indexCols)); - } - - if (priorVersion < FhirSchemaVersion.V0015.vid()) { - // Add PARAM_HASH logical_resources - List cols = ColumnDefBuilder.builder() - .addVarcharColumn(PARAMETER_HASH, PARAMETER_HASH_BYTES, true) - .buildColumns(); - statements.add(new AddColumn(schemaName, tableName, cols.get(0))); - } - - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - } - - /** - * Adds a table to support sharding of the logical_id when running - * in a distributed RDBMS such as Citus. This table is sharded by - * LOGICAL_ID, which means we can use a primary key of - * {RESOURCE_TYPE_ID, LOGICAL_ID} which is required to ensure - * that we can lock the logical resource to avoid any concurrency - * issues. This is only used for distributed implementations. For - * the standard non-distributed solution, the locking is done - * using LOGICAL_RESOURCES. - * @param pdm - */ - public void addLogicalResourceShards(PhysicalDataModel pdm) { - final String tableName = LOGICAL_RESOURCE_SHARDS; - final String mtId = this.multitenant ? MT_ID : null; - - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(mtId) - .addIntColumn(RESOURCE_TYPE_ID, false) - .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addPrimaryKey(tableName + "_PK", RESOURCE_TYPE_ID, LOGICAL_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) - .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_ID) // V0026 support for sharding - .addWiths(addWiths()) // add table tuning - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // NOP for now - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - } - - /** - * Create the COMMON_CANONICAL_VALUES table. Used from schema V0014 to normalize - * meta.profile search parameters (similar to common_token_values). Only the url - * is included by design. The (optional) version and fragment values are stored - * in the parameter mapping table (logical_resource_profiles) in order to support - * inequalities on version while still using a literal CANONICAL_ID = x predicate. - * These canonical ids are cached in the server, so search queries won't need to - * join to this table. The URL is typically a long string, so by normalizing and - * storing/indexing it once, we reduce space consumption. - * @param pdm - */ - public void addCommonCanonicalValues(PhysicalDataModel pdm) { - final String tableName = COMMON_CANONICAL_VALUES; - final String unqCanonicalUrl = "UNQ_" + tableName + "_URL"; - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn(CANONICAL_ID, false) - .addVarcharColumn(URL, CANONICAL_URL_BYTES, false) - .addPrimaryKey(tableName + "_PK", CANONICAL_ID) - .addUniqueIndex(unqCanonicalUrl, URL) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(URL) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // Intentionally NOP - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - } - - /** - * A single-parameter table supporting _profile search parameter values - * Add the LOGICAL_RESOURCE_PROFILES table to the given {@link PhysicalDataModel}. - * This table maps logical resources to meta.profile values stored as canonical URIs - * in COMMON_CANONICAL_VALUES. Canonical values can include optional version and fragment - * values as described here: https://www.hl7.org/fhir/datatypes.html#canonical - * @param pdm - * @return - */ - public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { - - final String tableName = LOGICAL_RESOURCE_PROFILES; - - // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn( CANONICAL_ID, false) // FK referencing COMMON_CANONICAL_VALUES - .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES - .addVarcharColumn( VERSION, VERSION_BYTES, true) - .addVarcharColumn( FRAGMENT, FRAGMENT_BYTES, true) - .addIndex(IDX + tableName + "_CCVLR", CANONICAL_ID, LOGICAL_RESOURCE_ID) - .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, CANONICAL_ID) - .addForeignKeyConstraint(FK + tableName + "_CCV", schemaName, COMMON_CANONICAL_VALUES, CANONICAL_ID) - .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - /** - * A single-parameter table supporting _tag search parameter values. - * Tags are tokens, but because they may not be very selective we use a - * separate table in order to avoid messing up cardinality estimates - * in the query optimizer. - * @param pdm - * @return - */ - public Table addLogicalResourceTags(PhysicalDataModel pdm) { - - final String tableName = LOGICAL_RESOURCE_TAGS; - - // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES - .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES - .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) - .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - /** - * Add the dedicated common_token_values mapping table for security search parameters - * @param pdm - * @return - */ - public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { - final String tableName = LOGICAL_RESOURCE_SECURITY; - - // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES - .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES - .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) - .addIndex(IDX + tableName + "_LRCCV", LOGICAL_RESOURCE_ID, COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - /** - * Add the resource_change_log table. This table supports tracking of every change made - * to a resource at the global level, making it much easier to stream a list of changes - * from a known point. - * @param pdm - */ - public void addResourceChangeLog(PhysicalDataModel pdm) { - final String tableName = RESOURCE_CHANGE_LOG; - - // custom list of Withs because this table does not require fillfactor tuned in V0020 - List customWiths = Arrays.asList( - With.with("autovacuum_vacuum_scale_factor", "0.01"), // V0019 - With.with("autovacuum_vacuum_threshold", "1000"), // V0019 - With.with("autovacuum_vacuum_cost_limit", "2000") // V0019 - ); - - // Note that for now, we elect to not distribute/shard this table because doing so - // would interfere with the queries supporting the history API which are based on - // index range scans across a contiguous range of records - Table tbl = Table.builder(schemaName, tableName) - .setTenantColumnName(MT_ID) - .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes - .addBigIntColumn(RESOURCE_ID, false) - .addIntColumn(RESOURCE_TYPE_ID, false) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addTimestampColumn(CHANGE_TSTAMP, false) - .addIntColumn(VERSION_ID, false) - .addCharColumn(CHANGE_TYPE, 1, false) - .addPrimaryKey(tableName + "_PK", RESOURCE_ID) - .addUniqueIndex("UNQ_" + RESOURCE_CHANGE_LOG + "_CTRTRI", CHANGE_TSTAMP, RESOURCE_TYPE_ID, RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(customWiths) // Does not require fillfactor tuning - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - } - - /** - * Adds the system level logical_resource_compartments table which identifies to - * which compartments a give resource belongs. A resource may belong to many - * compartments. - * @param pdm - * @return Table the table that was added to the PhysicalDataModel - */ - public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { - final String tableName = LOGICAL_RESOURCE_COMPARTMENTS; - - // note COMPARTMENT_LOGICAL_RESOURCE_ID represents the compartment (e.g. the Patient) - // that this resource exists within. This compartment resource may be a ghost resource...i.e. one - // which has a record in LOGICAL_RESOURCES but currently does not have any resource - // versions because we haven't yet loaded the resource itself. The timestamp is included - // because it makes it very easy to find the most recent changes to resources associated with - // a given patient (for example). - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( COMPARTMENT_NAME_ID, false) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addTimestampColumn(LAST_UPDATED, false) - .addBigIntColumn(COMPARTMENT_LOGICAL_RESOURCE_ID, false) - .addUniqueIndex(IDX + tableName + "_LRNMLR", LOGICAL_RESOURCE_ID, COMPARTMENT_NAME_ID, COMPARTMENT_LOGICAL_RESOURCE_ID) - .addUniqueIndex(IDX + tableName + "_NMCOMPLULR", COMPARTMENT_NAME_ID, COMPARTMENT_LOGICAL_RESOURCE_ID, LAST_UPDATED, LOGICAL_RESOURCE_ID) - .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .addForeignKeyConstraint(FK + tableName + "_COMP", schemaName, LOGICAL_RESOURCES, COMPARTMENT_LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - - - /** - * Add system-wide RESOURCE_STR_VALUES table to support _profile - * properties (which are of type REFERENCE). - * @param pdm - * @return Table the table that was added to the PhysicalDataModel - */ - public Table addResourceStrValues(PhysicalDataModel pdm) { - final int msb = MAX_SEARCH_STRING_BYTES; - - Table tbl = Table.builder(schemaName, STR_VALUES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( PARAMETER_NAME_ID, false) - .addVarcharColumn( STR_VALUE, msb, true) - .addVarcharColumn( STR_VALUE_LCASE, msb, true) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addIndex(IDX + STR_VALUES + "_PSR", PARAMETER_NAME_ID, STR_VALUE, LOGICAL_RESOURCE_ID) - .addIndex(IDX + STR_VALUES + "_PLR", PARAMETER_NAME_ID, STR_VALUE_LCASE, LOGICAL_RESOURCE_ID) - .addIndex(IDX + STR_VALUES + "_RPS", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, STR_VALUE) - .addIndex(IDX + STR_VALUES + "_RPL", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, STR_VALUE_LCASE) - .addForeignKeyConstraint(FK + STR_VALUES + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) - .addForeignKeyConstraint(FK + STR_VALUES + "_RID", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, STR_VALUES, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, STR_VALUES, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - /** - * Add the table for data search parameters at the (system-wide) resource level - * @param model - * @return Table the table that was added to the PhysicalDataModel - */ - public Table addResourceDateValues(PhysicalDataModel model) { - final String tableName = DATE_VALUES; - final String logicalResourcesTable = LOGICAL_RESOURCES; - - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( PARAMETER_NAME_ID, false) - .addTimestampColumn( DATE_START,6, true) - .addTimestampColumn( DATE_END,6, true) - .addBigIntColumn(LOGICAL_RESOURCE_ID, false) - .addIndex(IDX + tableName + "_PSER", PARAMETER_NAME_ID, DATE_START, DATE_END, LOGICAL_RESOURCE_ID) - .addIndex(IDX + tableName + "_PESR", PARAMETER_NAME_ID, DATE_END, DATE_START, LOGICAL_RESOURCE_ID) - .addIndex(IDX + tableName + "_RPSE", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, DATE_START, DATE_END) - .addForeignKeyConstraint(FK + tableName + "_PN", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) - .addForeignKeyConstraint(FK + tableName + "_R", schemaName, logicalResourcesTable, LOGICAL_RESOURCE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion == 1) { - statements.add(new DropIndex(schemaName, IDX + tableName + "_PVR")); - statements.add(new DropIndex(schemaName, IDX + tableName + "_RPV")); - statements.add(new DropColumn(schemaName, tableName, DATE_VALUE_DROPPED_COLUMN)); - } - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(model); - - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - model.addTable(tbl); - model.addObject(tbl); - - return tbl; - } - - /** - * 
-        CREATE TABLE resource_types (
-            resource_type_id INT NOT NULL
-            CONSTRAINT pk_resource_type PRIMARY KEY,
-            resource_type   VARCHAR(64) NOT NULL
-        );
-
-        -- make sure resource_type values are unique
-        CREATE UNIQUE INDEX unq_resource_types_rt ON resource_types(resource_type);
-
- * - * @param model - */ - protected void addResourceTypes(PhysicalDataModel model) { - resourceTypesTable = Table.builder(schemaName, RESOURCE_TYPES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( RESOURCE_TYPE_ID, false) - .addVarcharColumn( RESOURCE_TYPE, 64, false) - .addUniqueIndex(IDX + "unq_resource_types_rt", RESOURCE_TYPE) - .addPrimaryKey(RESOURCE_TYPES + "_PK", RESOURCE_TYPE_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 supporting for sharding - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // Intentionally a NOP - return statements; - }) - .build(model); - - // TODO Table should be immutable, so add support to the Builder for this - this.resourceTypesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(resourceTypesTable); - model.addTable(resourceTypesTable); - model.addObject(resourceTypesTable); - } - - /** - * Add the collection of tables for each of the listed - * FHIR resource types - * @param model - */ - protected void addResourceTables(PhysicalDataModel model, IDatabaseObject... dependency) { - if (this.sessionVariable == null) { - throw new IllegalStateException("Session variable must be defined before adding resource tables"); - } - - // The sessionVariable is used to enable access control on every table, so we - // provide it as a dependency - FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, this.multitenant, sessionVariable, - this.procedureDependencies, this.fhirTablespace, this.resourceTablePrivileges, addWiths()); - for (String resourceType: this.resourceTypes) { - - resourceType = resourceType.toUpperCase().trim(); - if (!ALL_RESOURCE_TYPES.contains(resourceType.toUpperCase())) { - logger.warning("Passed resource type '" + resourceType + "' does not match any known FHIR resource types; creating anyway"); - } - - ObjectGroup group = frg.addResourceType(resourceType); - group.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // Add additional dependencies the group doesn't yet know about - group.addDependencies(Arrays.asList(this.codeSystemsTable, this.parameterNamesTable, this.resourceTypesTable, this.commonTokenValuesTable)); - - // Add all other dependencies that were explicitly passed - group.addDependencies(Arrays.asList(dependency)); - - // Make this group a dependency for all the stored procedures. - this.procedureDependencies.add(group); - model.addObject(group); - } - } - - /** - * - * - CREATE TABLE parameter_names ( - parameter_name_id INT NOT NULL - CONSTRAINT pk_parameter_name PRIMARY KEY, - parameter_name VARCHAR(255 OCTETS) NOT NULL - ); - - CREATE UNIQUE INDEX unq_parameter_name_rtnm ON parameter_names(parameter_name) INCLUDE (parameter_name_id); - - * @param model - */ - protected void addParameterNames(PhysicalDataModel model) { - // The index which also used by the database to support the primary key constraint - String[] prfIndexCols = {PARAMETER_NAME}; - String[] prfIncludeCols = {PARAMETER_NAME_ID}; - - parameterNamesTable = Table.builder(schemaName, PARAMETER_NAMES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( PARAMETER_NAME_ID, false) - .addVarcharColumn( PARAMETER_NAME, 255, false) - .addUniqueIndex(IDX + "PARAMETER_NAME_RTNM", Arrays.asList(prfIndexCols), Arrays.asList(prfIncludeCols)) - .addPrimaryKey(PARAMETER_NAMES + "_PK", PARAMETER_NAME_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 treat this as a reference table - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // Intentionally a NOP - return statements; - }) - .build(model); - - this.parameterNamesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(parameterNamesTable); - - model.addTable(parameterNamesTable); - model.addObject(parameterNamesTable); - } - - /** - * Add the code_systems table to the database schema - CREATE TABLE code_systems ( - code_system_id INT NOT NULL - CONSTRAINT pk_code_system PRIMARY KEY, - code_system_name VARCHAR(255 OCTETS) NOT NULL - ); - - CREATE UNIQUE INDEX unq_code_system_cinm ON code_systems(code_system_name); - - * @param model - */ - protected void addCodeSystems(PhysicalDataModel model) { - codeSystemsTable = Table.builder(schemaName, CODE_SYSTEMS) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( CODE_SYSTEM_ID, false) - .addVarcharColumn(CODE_SYSTEM_NAME, 255, false) - .addUniqueIndex(IDX + "CODE_SYSTEM_CINM", CODE_SYSTEM_NAME) - .addPrimaryKey(CODE_SYSTEMS + "_PK", CODE_SYSTEM_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 treat this as a reference table - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, CODE_SYSTEMS, 2000, null, 1000)); - } - return statements; - }) - .build(model); - - this.codeSystemsTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(codeSystemsTable); - model.addTable(codeSystemsTable); - model.addObject(codeSystemsTable); - } - - /** - * Table used to store normalized values for tokens, shared by all the - * _TOKEN_VALUES tables. Although this requires an additional - * join, it cuts down on space by avoiding repeating long strings (e.g. urls). - * This also helps to reduce the total sizes of the indexes, helping to improve - * cache hit rates for a given buffer cache size. - * Token values may or may not have an associated code system, in which case, - * it assigned a default system. This is why CODE_SYSTEM_ID is not nullable and - * has a FK constraint. - * - * We never need to find all token values for a given code-system, so there's no need - * for a second index (CODE_SYSTEM_ID, TOKEN_VALUE). Do not add it. - * - * Because different parameter names may reference the same token value (e.g. - * 'Observation.subject' and 'Claim.patient' are both patient references), the - * common token value is not distinguished by a parameter_name_id. - * - * Where common token values are used to represent local relationships between two resources, - * the code_system encodes the resource type of the referenced resource and - * the token_value represents its logical_id. This approach simplifies query writing when - * following references. - * - * If sharding is supported, this table is distributed by token_value which unfortunately - * means that it cannot be the target of any foreign key constraint (which needs to use - * the primary key COMMON_TOKEN_VALUE_ID). - * @param pdm - * @return the table definition - */ - public void addCommonTokenValues(PhysicalDataModel pdm) { - final String tableName = COMMON_TOKEN_VALUES; - commonTokenValuesTable = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addBigIntColumn( COMMON_TOKEN_VALUE_ID, false) - .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) - .addIntColumn( CODE_SYSTEM_ID, false) - .addVarcharColumn( TOKEN_VALUE, MAX_TOKEN_VALUE_BYTES, false) - .addUniqueIndex(IDX + tableName + "_TVCP", TOKEN_VALUE, CODE_SYSTEM_ID) - .addPrimaryKey(tableName + "_PK", COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_CSID", schemaName, CODE_SYSTEMS, CODE_SYSTEM_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(TOKEN_VALUE) // V0026 shard using token_value - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // Intentionally a NOP - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - commonTokenValuesTable.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - pdm.addTable(commonTokenValuesTable); - pdm.addObject(commonTokenValuesTable); - } - - /** - * Add the system-wide RESOURCE_TOKEN_REFS table which is used for - * _tag and _security search properties in R4 (new table - * for issue #1366 V0006 schema change). Replaces the - * previous TOKEN_VALUES table. All token values are now - * normalized in the COMMON_TOKEN_VALUES table. Because this - * is for system-level params, there's no need to support - * composite params - * @param pdm - * @return Table the table that was added to the PhysicalDataModel - */ - public Table addResourceTokenRefs(PhysicalDataModel pdm) { - - final String tableName = RESOURCE_TOKEN_REFS; - - // logical_resources (0|1) ---- (*) resource_token_refs - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .setTenantColumnName(MT_ID) - .addIntColumn( PARAMETER_NAME_ID, false) - .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries - .addBigIntColumn( LOGICAL_RESOURCE_ID, false) - .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version - .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0009 change - .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0009 change - .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) - .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) - .addForeignKeyConstraint(FK + tableName + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding - .addWiths(addWiths()) // table tuning - .addMigration(priorVersion -> { - // Replace the indexes initially defined in the V0006 version with better ones - List statements = new ArrayList<>(); - if (priorVersion == FhirSchemaVersion.V0006.vid()) { - // Migrate the index definitions as part of the V0008 version of the schema - // This table was originally introduced as part of the V0006 schema, which - // is what we use as the match for the priorVersion - statements.add(new DropIndex(schemaName, IDX + tableName + "_TVLR")); - statements.add(new DropIndex(schemaName, IDX + tableName + "_LRTV")); - - final String mtId = multitenant ? MT_ID : null; - // Replace the original TVLR index on (common_token_value_id, parameter_name_id, logical_resource_id) - List tplr = Arrays.asList( - new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null), - new OrderedColumnDef(PARAMETER_NAME_ID, OrderedColumnDef.Direction.ASC, null), - new OrderedColumnDef(LOGICAL_RESOURCE_ID, OrderedColumnDef.Direction.ASC, null) - ); - statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_TPLR", tableName, mtId, tplr)); - - // Replace the original LRTV index with a new index on (logical_resource_id, parameter_name_id, common_token_value_id) - List lrpt = Arrays.asList( - new OrderedColumnDef(LOGICAL_RESOURCE_ID, OrderedColumnDef.Direction.ASC, null), - new OrderedColumnDef(PARAMETER_NAME_ID, OrderedColumnDef.Direction.ASC, null), - new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null) - ); - statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_LRPT", tableName, mtId, lrpt)); - } - if (priorVersion < FhirSchemaVersion.V0019.vid()) { - statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); - } - if (priorVersion < FhirSchemaVersion.V0020.vid()) { - statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); - } - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - - return tbl; - } - - /** - * The erased_resources table is used to track which logical resources and corresponding - * resource versions have been erased using the $erase operation. This table should - * typically be empty and only used temporarily by the erase DAO/procedures to indicate - * which rows have been erased. The entries in this table are then used to delete - * any offloaded payload entries. - * @param pdm - */ - public void addErasedResources(PhysicalDataModel pdm) { - final String tableName = ERASED_RESOURCES; - final String mtId = this.multitenant ? MT_ID : null; - - // Each erase operation is allocated an ERASED_RESOURCE_GROUP_ID - // value which can be used to retrieve the resource and/or - // resource-versions erased in a particular call. The rows - // can then be deleted once the erasure of any offloaded - // payload is confirmed. Note that we don't use logical_resource_id - // or resource_id values here, because those records may have - // already been deleted by $erase. - Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) - .setTenantColumnName(mtId) - .addBigIntColumn(ERASED_RESOURCE_GROUP_ID, false) - .addIntColumn(RESOURCE_TYPE_ID, false) - .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) - .addIntColumn(VERSION_ID, true) - .addIndex(IDX + tableName + "_GID", ERASED_RESOURCE_GROUP_ID) - .setDistributionColumnName(ERASED_RESOURCE_GROUP_ID) - .setTablespace(fhirTablespace) - .addPrivileges(resourceTablePrivileges) - .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) - .enableAccessControl(this.sessionVariable) - .addWiths(addWiths()) // add table tuning - .addMigration(priorVersion -> { - List statements = new ArrayList<>(); - // Nothing yet - - // TODO migrate to simplified design (no PK, FK) - return statements; - }) - .build(pdm); - - // TODO should not need to add as a table and an object. Get the table to add itself? - tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - this.procedureDependencies.add(tbl); - pdm.addTable(tbl); - pdm.addObject(tbl); - } - - /** - * 
-    CREATE SEQUENCE fhir_sequence
-             AS BIGINT
-     START WITH 1
-          CACHE 20000
-       NO CYCLE;
-     *
- * - * @param pdm - */ - protected void addFhirSequence(PhysicalDataModel pdm) { - this.fhirSequence = new Sequence(schemaName, FHIR_SEQUENCE, FhirSchemaVersion.V0001.vid(), 1, 1000); - this.fhirSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - procedureDependencies.add(fhirSequence); - sequencePrivileges.forEach(p -> p.addToObject(fhirSequence)); - - pdm.addObject(fhirSequence); - } - - protected void addFhirRefSequence(PhysicalDataModel pdm) { - this.fhirRefSequence = new Sequence(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE); - this.fhirRefSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - procedureDependencies.add(fhirRefSequence); - sequencePrivileges.forEach(p -> p.addToObject(fhirRefSequence)); - pdm.addObject(fhirRefSequence); - - // Schema V0003 does an alter to bump up the start value of the reference sequence - // to avoid a conflict with parameter names not in the pre-populated set - // fix for issue-1263. This will only be applied if the current version of the - // the FHIR_REF_SEQUENCE is <= 2. - BaseObject alter = new AlterSequenceStartWith(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0003.vid(), - FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE, 1); - alter.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - procedureDependencies.add(alter); - alter.addDependency(fhirRefSequence); // only alter after the sequence is initially created - - // Because the sequence might be dropped and recreated, we need to inject privileges - // so that they are applied when this ALTER SEQUENCE is processed. - sequencePrivileges.forEach(p -> p.addToObject(alter)); - pdm.addObject(alter); - } - - /** - * Add the sequence used by the new local/external references data model - * @param pdm - */ - protected void addReferencesSequence(PhysicalDataModel pdm) { - Sequence seq = new Sequence(schemaName, FhirSchemaConstants.REFERENCES_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.REFERENCES_SEQUENCE_START, FhirSchemaConstants.REFERENCES_SEQUENCE_CACHE, FhirSchemaConstants.REFERENCES_SEQUENCE_INCREMENT); - seq.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - procedureDependencies.add(seq); - sequencePrivileges.forEach(p -> p.addToObject(seq)); - pdm.addObject(seq); - } - - /** - * The defaults with addWiths. Added to every table in a PostgreSQL schema - * @return - */ - protected List addWiths() { - // NOTE! If you change this table remember that you also need to bump the - // schema version of every table that uses this list of Withs. This includes - // adding a corresponding migration step. - return Arrays.asList( - With.with("autovacuum_vacuum_scale_factor", "0.01"), // V0019 - With.with("autovacuum_vacuum_threshold", "1000"), // V0019 - With.with("autovacuum_vacuum_cost_limit", "2000"), // V0019 - With.with(FhirSchemaConstants.PG_FILLFACTOR_PROP, Integer.toString(FhirSchemaConstants.PG_FILLFACTOR_VALUE)) // V0020 - ); - } -} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index d57601c342a..ecae01396d2 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -73,6 +73,7 @@ import java.util.List; import java.util.Set; +import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.common.AddColumn; import com.ibm.fhir.database.utils.common.CreateIndexStatement; @@ -200,8 +201,9 @@ public void addLogicalResources(List group, String prefix) { // We also have a FK constraint pointing back to that table to try and keep // things sensible. Table.Builder builder = Table.builder(schemaName, tableName) - .setTenantColumnName(MT_ID) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -214,7 +216,6 @@ public void addLogicalResources(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding // Add indexes to avoid dead lock issue of derby, and improve Db2 performance // Derby requires all columns used in where clause to be indexed, otherwise whole table lock will be // used instead of row lock, which can cause dead lock issue frequently during concurrent accesses. @@ -339,6 +340,7 @@ public void addResources(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( RESOURCE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -353,7 +355,6 @@ public void addResources(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0024.vid()) { @@ -424,6 +425,7 @@ public void addStrValues(List group, String prefix) { .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding // .addBigIntColumn( ROW_ID, false) // Removed by issue-1683 - composites refactor .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( STR_VALUE, msb, true) @@ -439,7 +441,6 @@ public void addStrValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -514,6 +515,7 @@ public Table addResourceTokenRefs(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -527,7 +529,6 @@ public Table addResourceTokenRefs(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -594,6 +595,7 @@ public Table addProfiles(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn( CANONICAL_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addVarcharColumn( VERSION, VERSION_BYTES, true) @@ -605,7 +607,6 @@ public Table addProfiles(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -643,6 +644,7 @@ public Table addTags(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -652,7 +654,6 @@ public Table addTags(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -687,6 +688,7 @@ public Table addSecurity(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -696,7 +698,6 @@ public Table addSecurity(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -792,8 +793,9 @@ public void addDateValues(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addTimestampColumn( DATE_START, true) .addTimestampColumn( DATE_END, true) @@ -807,7 +809,6 @@ public void addDateValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -859,8 +860,9 @@ public void addNumberValues(List group, String prefix) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addDoubleColumn( NUMBER_VALUE, true) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -874,7 +876,6 @@ public void addNumberValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -932,9 +933,10 @@ public void addLatLngValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addDoubleColumn( LATITUDE_VALUE, true) .addDoubleColumn( LONGITUDE_VALUE, true) @@ -949,7 +951,6 @@ public void addLatLngValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -1005,9 +1006,10 @@ public void addQuantityValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( CODE, 255, false) .addDoubleColumn( QUANTITY_VALUE, true) @@ -1027,7 +1029,6 @@ public void addQuantityValues(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -1065,8 +1066,9 @@ public void addListLogicalResourceItems(List group, String pref Table tbl = Table.builder(schemaName, LIST_LOGICAL_RESOURCE_ITEMS) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIntColumn( RESOURCE_TYPE_ID, false) .addVarcharColumn( ITEM_LOGICAL_ID, lib, true) @@ -1075,7 +1077,6 @@ public void addListLogicalResourceItems(List group, String pref .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -1109,8 +1110,9 @@ public void addPatientCurrentRefs(List group, String prefix) { Table tbl = Table.builder(schemaName, PATIENT_CURRENT_REFS) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding - .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addVarcharColumn( CURRENT_PROBLEMS_LIST, lib, true) .addVarcharColumn( CURRENT_MEDICATIONS_LIST, lib, true) @@ -1121,7 +1123,6 @@ public void addPatientCurrentRefs(List group, String prefix) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java index d68cd3fc0c0..61e5c33ba17 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java @@ -195,4 +195,7 @@ public class FhirSchemaConstants { public static final String ERASED_RESOURCES = "ERASED_RESOURCES"; public static final String ERASED_RESOURCE_ID = "ERASED_RESOURCE_ID"; public static final String ERASED_RESOURCE_GROUP_ID = "ERASED_RESOURCE_GROUP_ID"; + + // Data Distribution/Sharding Constants + public static final String SHARD_KEY = "SHARD_KEY"; } \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index c4cb8d51da0..d149072491e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -83,7 +83,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.AddColumn; import com.ibm.fhir.database.utils.common.CreateIndexStatement; import com.ibm.fhir.database.utils.common.DropColumn; @@ -124,8 +126,8 @@ public class FhirSchemaGenerator { // The schema used for administration objects like the tenants table, variable etc private final String adminSchemaName; - // Build the multitenant variant of the schema - private final boolean multitenant; + // Which variant of the schema do we want to build + private final SchemaType schemaType; // No abstract types private static final Set ALL_RESOURCE_TYPES = ModelSupport.getResourceTypes(false).stream() @@ -212,8 +214,8 @@ public class FhirSchemaGenerator { * @param adminSchemaName * @param schemaName */ - public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant) { - this(adminSchemaName, schemaName, multitenant, ALL_RESOURCE_TYPES); + public FhirSchemaGenerator(String adminSchemaName, String schemaName, SchemaType schemaType) { + this(adminSchemaName, schemaName, schemaType, ALL_RESOURCE_TYPES); } /** @@ -222,10 +224,10 @@ public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean mu * @param adminSchemaName * @param schemaName */ - public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant, Set resourceTypes) { + public FhirSchemaGenerator(String adminSchemaName, String schemaName, SchemaType schemaType, Set resourceTypes) { this.adminSchemaName = adminSchemaName; this.schemaName = schemaName; - this.multitenant = multitenant; + this.schemaType = schemaType; // The FHIR user (e.g. "FHIRSERVER") will need these privileges to be granted to it. Note that // we use the group identified by FHIR_USER_GRANT_GROUP here - these privileges can be applied @@ -505,10 +507,22 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); // We currently only support functions with PostgreSQL, although this is really just a procedure + final String deleteResourceParametersScript; + final String addAnyResourceScript; + final String eraseResourceScript; + if (model.isDistributed()) { + deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + "_distributed.sql"; + addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + "_distributed.sql"; + eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + "_distributed.sql"; + } else { + deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql"; + addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + ".sql"; + eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql"; + } FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, DELETE_RESOURCE_PARAMETERS, FhirSchemaVersion.V0020.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, deleteResourceParametersScript, null), Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); @@ -516,15 +530,14 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { fd = model.addFunction(this.schemaName, ADD_ANY_RESOURCE, FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() - + ".sql", null), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addAnyResourceScript, null), Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); fd = model.addFunction(this.schemaName, ERASE_RESOURCE, FhirSchemaVersion.V0013.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, eraseResourceScript, null), Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); } @@ -597,6 +610,14 @@ public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); } + /** + * Are we building the Db2-specific multitenant schema variant + * @return + */ + private boolean isMultitenant() { + return this.schemaType == SchemaType.MULTITENANT; + } + /** * Add the system-wide logical_resources table. Note that LOGICAL_ID is * denormalized, stored in both LOGICAL_RESOURCES and _LOGICAL_RESOURCES. @@ -606,7 +627,7 @@ public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { */ public void addLogicalResources(PhysicalDataModel pdm) { final String tableName = LOGICAL_RESOURCES; - final String mtId = this.multitenant ? MT_ID : null; + final String mtId = isMultitenant() ? MT_ID : null; final String IDX_LOGICAL_RESOURCES_RITS = "IDX_" + LOGICAL_RESOURCES + "_RITS"; final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD"; @@ -614,6 +635,7 @@ public void addLogicalResources(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -630,7 +652,6 @@ public void addLogicalResources(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(addWiths()) // add table tuning .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -712,11 +733,12 @@ public void addLogicalResources(PhysicalDataModel pdm) { */ public void addLogicalResourceShards(PhysicalDataModel pdm) { final String tableName = LOGICAL_RESOURCE_SHARDS; - final String mtId = this.multitenant ? MT_ID : null; + final String mtId = isMultitenant() ? MT_ID : null; Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(mtId) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addIntColumn(RESOURCE_TYPE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -725,7 +747,6 @@ public void addLogicalResourceShards(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_ID) // V0026 support for sharding .addWiths(addWiths()) // add table tuning .addMigration(priorVersion -> { List statements = new ArrayList<>(); @@ -758,6 +779,7 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(CANONICAL_ID, false) .addVarcharColumn(URL, CANONICAL_URL_BYTES, false) .addPrimaryKey(tableName + "_PK", CANONICAL_ID) @@ -765,7 +787,6 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(URL) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally NOP @@ -797,6 +818,7 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn( CANONICAL_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addVarcharColumn( VERSION, VERSION_BYTES, true) @@ -809,7 +831,6 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -846,6 +867,7 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -856,7 +878,6 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -890,6 +911,7 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -900,7 +922,6 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -938,12 +959,11 @@ public void addResourceChangeLog(PhysicalDataModel pdm) { With.with("autovacuum_vacuum_cost_limit", "2000") // V0019 ); - // Note that for now, we elect to not distribute/shard this table because doing so - // would interfere with the queries supporting the history API which are based on - // index range scans across a contiguous range of records + // Each shard gets its own history Table tbl = Table.builder(schemaName, tableName) .setTenantColumnName(MT_ID) .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes + .setDistributionType(DistributionType.DISTRIBUTED) .addBigIntColumn(RESOURCE_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -991,6 +1011,7 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addIntColumn( COMPARTMENT_NAME_ID, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addTimestampColumn(LAST_UPDATED, false) @@ -1003,7 +1024,6 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1052,7 +1072,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1098,7 +1118,7 @@ public Table addResourceDateValues(PhysicalDataModel model) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion == 1) { @@ -1149,7 +1169,7 @@ protected void addResourceTypes(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 supporting for sharding + .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1176,7 +1196,7 @@ protected void addResourceTables(PhysicalDataModel model, IDatabaseObject... dep // The sessionVariable is used to enable access control on every table, so we // provide it as a dependency - FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, this.multitenant, sessionVariable, + FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, isMultitenant(), sessionVariable, this.procedureDependencies, this.fhirTablespace, this.resourceTablePrivileges, addWiths()); for (String resourceType: this.resourceTypes) { @@ -1228,7 +1248,7 @@ protected void addParameterNames(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 treat this as a reference table + .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1266,7 +1286,7 @@ protected void addCodeSystems(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionReference() // V0026 treat this as a reference table + .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1325,7 +1345,7 @@ public void addCommonTokenValues(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(TOKEN_VALUE) // V0026 shard using token_value + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 shard using token_value .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1358,6 +1378,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -1370,7 +1391,6 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionColumnName(LOGICAL_RESOURCE_ID) // V0026 support for sharding .addWiths(addWiths()) // table tuning .addMigration(priorVersion -> { // Replace the indexes initially defined in the V0006 version with better ones @@ -1382,7 +1402,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { statements.add(new DropIndex(schemaName, IDX + tableName + "_TVLR")); statements.add(new DropIndex(schemaName, IDX + tableName + "_LRTV")); - final String mtId = multitenant ? MT_ID : null; + final String mtId = isMultitenant() ? MT_ID : null; // Replace the original TVLR index on (common_token_value_id, parameter_name_id, logical_resource_id) List tplr = Arrays.asList( new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null), @@ -1428,7 +1448,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { */ public void addErasedResources(PhysicalDataModel pdm) { final String tableName = ERASED_RESOURCES; - final String mtId = this.multitenant ? MT_ID : null; + final String mtId = isMultitenant() ? MT_ID : null; // Each erase operation is allocated an ERASED_RESOURCE_GROUP_ID // value which can be used to retrieve the resource and/or @@ -1445,7 +1465,6 @@ public void addErasedResources(PhysicalDataModel pdm) { .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) .addIntColumn(VERSION_ID, true) .addIndex(IDX + tableName + "_GID", ERASED_RESOURCE_GROUP_ID) - .setDistributionColumnName(ERASED_RESOURCE_GROUP_ID) .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java index 8427b8a45aa..4a50338d39a 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java @@ -245,7 +245,7 @@ public void addStepThreadExecutionTable(PhysicalDataModel model) { .addBigIntColumn(M_WRITESKIP, NOT_NULL) // M_WRITESKIP BIGINT NOT NULL .addBigIntColumn(FK_JOBEXECID, NOT_NULL) // FK_JOBEXECID BIGINT NOT NULL .addBigIntColumn(FK_TOPLVL_STEPEXECID, NULL) // FK_TOPLVL_STEPEXECID BIGINT - .addSmallIntColumn(ISPARTITIONEDSTEP, 0, NULL) // ISPARTITIONEDSTEP SMALLINT DEFAULT 0 + .addSmallIntBooleanColumn(ISPARTITIONEDSTEP, 0, NULL) // ISPARTITIONEDSTEP SMALLINT DEFAULT 0 .addPrimaryKey(PK + STEPTHREADEXECUTION_TABLE, STEPEXECID) // PRIMARY KEY (STEPEXECID) .addIndex("STE_FKJOBEXECID_IX", FK_JOBEXECID) // STE_FKJOBEXECID_IX (FK_JOBEXECID) .addIndex("STE_FKTLSTEPEID_IX", FK_TOPLVL_STEPEXECID) // STE_FKTLSTEPEID_IX (FK_TOPLVL_STEPEXECID) @@ -391,7 +391,7 @@ public void addStepThreadInstanceTable(PhysicalDataModel model) { .addBlobColumn(CHECKPOINTDATA, 2147483647, 10240, NULL) // CHECKPOINTDATA BLOB(2147483647) .addBigIntColumn(FK_JOBINSTANCEID, NOT_NULL) // FK_JOBINSTANCEID BIGINT NOT NULL .addBigIntColumn(FK_LATEST_STEPEXECID, NOT_NULL) // FK_LATEST_STEPEXECID BIGINT NOT NULL - .addSmallIntColumn(PARTITIONED, 0, NULL) //PARTITIONED SMALLINT DEFAULT 0 NOT NULL + .addSmallIntBooleanColumn(PARTITIONED, 0, NULL) //PARTITIONED SMALLINT DEFAULT 0 NOT NULL .addIntColumn(PARTITIONPLANSIZE, NULL) // PARTITIONPLANSIZE INTEGER .addIntColumn(STARTCOUNT,NULL) // STARTCOUNT INTEGER .addIndex("STI_FKINSTANCEID_IX", FK_JOBINSTANCEID) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java index 78583215f48..14122855eed 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java @@ -7,7 +7,6 @@ package com.ibm.fhir.schema.derby; import java.sql.Connection; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -19,6 +18,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider; @@ -87,11 +87,11 @@ public DerbyFhirDatabase(String dbPath, Set resourceTypeNames) throws S // Database objects for the admin schema (shared across multiple tenants in the same DB) PhysicalDataModel pdm = new PhysicalDataModel(); if (resourceTypeNames == null) { - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); gen.buildSchema(pdm); } else { // just build out a subset of tables - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false, resourceTypeNames); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN, resourceTypeNames); gen.buildSchema(pdm); } @@ -177,7 +177,7 @@ public VersionHistoryService createVersionHistoryService() throws SQLException { try { JdbcTarget target = new JdbcTarget(c); DerbyAdapter derbyAdapter = new DerbyAdapter(target); - CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, derbyAdapter); + CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, DerbyMaster.wrap(derbyAdapter)); c.commit(); } catch (SQLException x) { logger.log(Level.SEVERE, "failed to create version history table", x); diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql new file mode 100644 index 00000000000..94ecff46ab6 --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql @@ -0,0 +1,212 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2022 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Provides support for the distributed database schema variant (e.g. Citus) +-- which uses a shard_key value to distribute the data across multiple +-- database nodes. +-- +-- For Citus, this function can also be tagged as distributed because +-- all the SQL/DML uses either a reference table, or a table distributed +-- by the p_shard_key parameter value. +-- +-- Procedure to add a resource version and its associated parameters. These +-- parameters only ever point to the latest version of a resource, never to +-- previous versions, which are kept to support history queries. +-- implNote - Conventions: +-- p_... prefix used to represent input parameters +-- v_... prefix used to represent declared variables +-- t_... prefix used to represent temp variables +-- o_... prefix used to represent output parameters +-- Parameters: +-- p_shard_key: the key used to distribute resources by sharding +-- p_logical_id: the logical id given to the resource by the FHIR server +-- p_payload: the BLOB (of JSON) which is the resource content +-- p_last_updated the last_updated time given by the FHIR server +-- p_is_deleted: the soft delete flag +-- p_version_id: the intended new version id of the resource (matching the JSON payload) +-- p_parameter_hash_b64 the Base64 encoded hash of parameter values +-- p_if_none_match the encoded If-None-Match value +-- o_logical_resource_id: output field returning the newly assigned logical_resource_id value +-- o_current_parameter_hash: Base64 current parameter hash if existing resource +-- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit +-- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) +-- Exceptions: +-- SQLSTATE 99001: on version conflict (concurrency) +-- SQLSTATE 99002: missing expected row (data integrity) +-- SQLSTATE 99004: delete a currently deleted resource (data integrity) +-- ---------------------------------------------------------------------------- + ( IN p_shard_key SMALLINT, + IN p_resource_type VARCHAR( 36), + IN p_logical_id VARCHAR(255), + IN p_payload BYTEA, + IN p_last_updated TIMESTAMP, + IN p_is_deleted CHAR( 1), + IN p_source_key VARCHAR( 64), + IN p_version INT, + IN p_parameter_hash_b64 VARCHAR( 44), + IN p_if_none_match INT, + IN p_resource_payload_key VARCHAR( 36), + OUT o_logical_resource_id BIGINT, + OUT o_current_parameter_hash VARCHAR( 44), + OUT o_interaction_status INT, + OUT o_if_none_match_version INT) + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + v_logical_resource_id BIGINT := NULL; + t_logical_resource_id BIGINT := NULL; + v_current_resource_id BIGINT := NULL; + v_resource_id BIGINT := NULL; + v_resource_type_id INT := NULL; + v_currently_deleted CHAR(1) := NULL; + v_new_resource INT := 0; + v_duplicate INT := 0; + v_current_version INT := 0; + v_change_type CHAR(1) := NULL; + + -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. + lock_cur CURSOR (t_shard_key SMALLINT, t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id, parameter_hash, is_deleted FROM {{SCHEMA_NAME}}.logical_resources WHERE shard_key = t_shard_key AND resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + +BEGIN + -- default value unless we hit If-None-Match + o_interaction_status := 0; + + -- LOADED ON: {{DATE}} + v_schema_name := '{{SCHEMA_NAME}}'; + SELECT resource_type_id INTO v_resource_type_id + FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type; + + -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) + SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; + + -- Get a lock at the system-wide logical resource level + OPEN lock_cur(t_shard_key := p_shard_key, t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted; + CLOSE lock_cur; + + -- Create the resource if we don't have it already + IF v_logical_resource_id IS NULL + THEN + SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; + -- remember that we have a concurrent system...so there is a possibility + -- that another thread snuck in before us and created the logical resource. This + -- is easy to handle, just turn around and read it + INSERT INTO {{SCHEMA_NAME}}.logical_resources (shard_key, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (p_shard_key, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; + + -- if row existed, we still need to obtain a lock on it. Because logical resource records are + -- never deleted, we don't need to worry about it disappearing again before we grab the row lock + OPEN lock_cur (t_shard_key := p_shard_key, t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted; + CLOSE lock_cur; + + -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL + o_current_parameter_hash := NULL; + + IF v_logical_resource_id = t_logical_resource_id + THEN + -- we created the logical resource and therefore we already own the lock. So now we can + -- safely create the corresponding record in the resource-type-specific logical_resources table + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (shard_key, logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' USING p_shard_key, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + v_new_resource := 1; + ELSE + v_logical_resource_id := t_logical_resource_id; + END IF; + END IF; + + -- Remember everying is locked at the logical resource level, so we are thread-safe here + IF v_new_resource = 0 THEN + -- as this is an existing resource, we need to know the current resource id. + -- This is only available at the resource-specific logical_resources level + EXECUTE + 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' + || ' WHERE shard_key = $1 AND logical_resource_id = $2 ' + INTO v_current_resource_id, v_current_version USING p_shard_key, v_logical_resource_id; + + IF v_current_resource_id IS NULL OR v_current_version IS NULL + THEN + -- our concurrency protection means that this shouldn't happen + RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; + END IF; + + -- If-None-Match does not apply if the resource is currently deleted + IF v_currently_deleted = 'N' AND p_if_none_match = 0 + THEN + -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the + -- connection with a fatal error, so instead we use an out parameter to + -- indicate the match + o_interaction_status := 1; + o_if_none_match_version := v_current_version; + RETURN; + END IF; + + -- Concurrency check: + -- the version parameter we've been given (which is also embedded in the JSON payload) must be + -- one greater than the current version, otherwise we've hit a concurrent update race condition + IF p_version != v_current_version + 1 + THEN + RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; + END IF; + + -- Prevent creating a new deletion marker if the resource is currently deleted + IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' + THEN + RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; + END IF; + + IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash + THEN + -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) + -- TODO patch parameter sets instead of all delete/all insert. + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2, $3)' + USING p_shard_key, p_resource_type, v_logical_resource_id; + END IF; -- end if check parameter hash + END IF; -- end if existing resource + + EXECUTE + 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (shard_key, resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7, $8)' + USING p_shard_key, v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; + + + IF v_new_resource = 0 THEN + -- As this is an existing logical resource, we need to update the xx_logical_resource values to match + -- the values of the current resource. For new resources, these are added by the insert so we don't + -- need to update them here. + EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE shard_key = $5 AND logical_resource_id = $6' + USING v_resource_id, p_is_deleted, p_last_updated, p_version, p_shard_key, v_logical_resource_id; + + -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level + EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE shard_key = $4 AND logical_resource_id = $5' + USING p_is_deleted, p_last_updated, p_parameter_hash_b64, p_shard_key, v_logical_resource_id; + END IF; + + -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event + -- related to resources changes (issue-1955) + IF p_is_deleted = 'Y' + THEN + v_change_type := 'D'; + ELSE + IF v_new_resource = 0 AND v_currently_deleted = 'N' + THEN + v_change_type := 'U'; + ELSE + v_change_type := 'C'; + END IF; + END IF; + + INSERT INTO {{SCHEMA_NAME}}.resource_change_log(shard_key, resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) + VALUES (p_shard_key, v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); + + -- Hand back the id of the logical resource we created earlier. In the new R4 schema + -- only the logical_resource_id is the target of any FK, so there's no need to return + -- the resource_id (which is now private to the _resources tables). + o_logical_resource_id := v_logical_resource_id; +END $$; diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql new file mode 100644 index 00000000000..abf1f6b50ac --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql @@ -0,0 +1,63 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2021 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to delete all search parameters values for a given resource. +-- This variant is for use with the distributed schema variant typically +-- deployed to a distributed database service like Citus. +-- +-- p_shard_key: the key used for distribution (sharding) +-- p_resource_type: the resource type name +-- p_logical_resource_id: the database id of the resource for which the parameters are to be deleted +-- ---------------------------------------------------------------------------- + ( IN p_shard_key SMALLINT, + IN p_resource_type VARCHAR( 36), + IN p_logical_resource_id BIGINT, + OUT o_logical_resource_id BIGINT) + RETURNS BIGINT + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + +BEGIN + v_schema_name := '{{SCHEMA_NAME}}'; + + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + + -- because we're a function, pass back a result + o_logical_resource_id := p_logical_resource_id; +END $$; diff --git a/fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql new file mode 100644 index 00000000000..03967ad27fd --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql @@ -0,0 +1,89 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2021 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to remove a resource, history and parameters values +-- +-- p_shard_key: the sharding key used for distribution +-- p_resource_type: the resource type +-- p_logical_id: the resource logical id +-- o_deleted: the total number of resource versions that are deleted +-- ---------------------------------------------------------------------------- + ( IN p_shard_key SMALLINT, + IN p_resource_type VARCHAR( 36), + IN p_logical_id VARCHAR( 255), + IN p_erased_resource_group_id BIGINT, + OUT o_deleted BIGINT) + RETURNS BIGINT + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + v_logical_resource_id BIGINT := NULL; + v_resource_type_id BIGINT := -1; + v_total BIGINT := 0; + +BEGIN + v_schema_name := '{{SCHEMA_NAME}}'; + + -- Prep 1: Get the v_resource_type_id + SELECT resource_type_id INTO v_resource_type_id + FROM {{SCHEMA_NAME}}.resource_types + WHERE resource_type = p_resource_type; + + -- Prep 2: Get the logical from the system-wide logical resource level + SELECT logical_resource_id INTO v_logical_resource_id + FROM {{SCHEMA_NAME}}.logical_resources + WHERE shard_key = p_shard_key + AND resource_type_id = v_resource_type_id + AND logical_id = p_logical_id + FOR UPDATE; + + IF NOT FOUND + THEN + v_total := -1; + ELSE + -- Step 1: Delete from resource_change_log + -- Delete is done before the RESOURCES table entries disappear + -- This uses the primary_keys of each table to conditional-delete + EXECUTE + 'DELETE FROM {{SCHEMA_NAME}}.RESOURCE_CHANGE_LOG ' + || ' WHERE SHARD_KEY = $1 AND RESOURCE_ID IN ( ' + || ' SELECT RESOURCE_ID ' + || ' FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES ' + || ' WHERE SHARD_KEY = $2 ' + || ' AND LOGICAL_RESOURCE_ID = $3) ' + USING p_shard_key, p_shard_key, v_logical_resource_id; + + -- Step 1.1: Record the versions we need to delete if we are doing payload offload + EXECUTE 'INSERT INTO {{SCHEMA_NAME}}.erased_resources(shard_key, erased_resource_group_id, resource_type_id, logical_id, version_id) ' + || ' SELECT SHARD_KEY, $1, $2, $3, version_id ' + || ' FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES ' + || ' WHERE SHARD_KEY = $4, LOGICAL_RESOURCE_ID = $5 ' + USING p_erased_resource_group_id, v_resource_type_id, p_logical_id, p_shard_key, v_logical_resource_id; + + -- Step 2: Delete All Versions from Resources Table + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2' + USING p_shard_key, v_logical_resource_id; + GET DIAGNOSTICS v_total = ROW_COUNT; + + -- The delete_resource_parameters call is a function, so we have to use a select here, not call + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2, $3)' + USING p_shard_key, p_resource_type, v_logical_resource_id; + + -- Step 4: Delete from Logical Resources table + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_LOGICAL_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2' + USING p_shard_key, v_logical_resource_id; + + -- Step 5: Delete from Global Logical Resources + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.LOGICAL_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2 AND RESOURCE_TYPE_ID = $3' + USING p_shard_key, v_logical_resource_id, v_resource_type_id; + END IF; + + -- Return the total number of deleted versions + o_deleted := v_total; +END $$; diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java index 5e689c4d17a..4ba936270e3 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java @@ -8,8 +8,11 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.version.CreateVersionHistory; @@ -25,9 +28,10 @@ public void testFHIRSchemaGeneratorCheckTags() { PrintConnection connection = test.new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -36,12 +40,12 @@ public void testFHIRSchemaGeneratorCheckTags() { vhs.setTarget(adapter); PhysicalDataModel pdm = new PhysicalDataModel(); - FhirSchemaGenerator generator = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, true); + FhirSchemaGenerator generator = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, SchemaType.PLAIN); generator.buildSchema(pdm); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.apply(adapter, context); - pdm.applyFunctions(adapter, context); - pdm.applyProcedures(adapter, context); + pdm.apply(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); + pdm.applyProcedures(schemaAdapter, context); pdm.visit(new ConfirmTagsVisitor()); } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java index ac78178c307..d72035c5b59 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java @@ -45,8 +45,10 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.AlterSequenceStartWith; import com.ibm.fhir.database.utils.model.AlterTableIdentityCache; @@ -77,9 +79,10 @@ public void testJavaBatchSchemaGeneratorDb2() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -91,9 +94,9 @@ public void testJavaBatchSchemaGeneratorDb2() { JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.apply(adapter, context); - pdm.applyFunctions(adapter, context); - pdm.applyProcedures(adapter, context); + pdm.apply(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); + pdm.applyProcedures(schemaAdapter, context); if (DEBUG) { for (Entry command : commands.entrySet()) { @@ -109,9 +112,10 @@ public void testJavaBatchSchemaGeneratorPostgres() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); PostgresAdapter adapter = new PostgresAdapter(target); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -123,8 +127,8 @@ public void testJavaBatchSchemaGeneratorPostgres() { JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.apply(adapter, context); - pdm.applyFunctions(adapter, context); + pdm.apply(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); if (DEBUG) { for (Entry command : commands.entrySet()) { @@ -141,9 +145,10 @@ public void testJavaBatchSchemaGeneratorCheckTags() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -155,9 +160,9 @@ public void testJavaBatchSchemaGeneratorCheckTags() { JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); generator.buildJavaBatchSchema(pdm); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.apply(adapter, context); - pdm.applyFunctions(adapter, context); - pdm.applyProcedures(adapter, context); + pdm.apply(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); + pdm.applyProcedures(schemaAdapter, context); pdm.visit(new ConfirmTagsVisitor()); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java index e087b636098..8c21ec516ab 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java @@ -8,8 +8,10 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.version.CreateVersionHistory; @@ -25,9 +27,10 @@ public void testOAuthSchemaGeneratorCheckTags() { PrintConnection connection = test.new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -39,9 +42,9 @@ public void testOAuthSchemaGeneratorCheckTags() { OAuthSchemaGenerator generator = new OAuthSchemaGenerator(Main.OAUTH_SCHEMANAME); generator.buildOAuthSchema(pdm); SchemaApplyContext context = SchemaApplyContext.getDefault(); - pdm.apply(adapter, context); - pdm.applyFunctions(adapter, context); - pdm.applyProcedures(adapter, context); + pdm.apply(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); + pdm.applyProcedures(schemaAdapter, context); pdm.visit(new ConfirmTagsVisitor()); } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java index 8d36f0ce5c0..fe041e08e12 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java @@ -15,7 +15,10 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -41,7 +44,7 @@ public void testDb2TableCreation() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -50,8 +53,9 @@ public void testDb2TableCreation() { // Pretend that our target is a DB2 database Db2Adapter adapter = new Db2Adapter(tgt); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); SchemaApplyContext context = SchemaApplyContext.getDefault(); - model.apply(adapter, context); + model.apply(schemaAdapter, context); } @Test @@ -60,7 +64,7 @@ public void testParallelTableCreation() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -71,8 +75,9 @@ public void testParallelTableCreation() { ITaskCollector collector = taskService.makeTaskCollector(pool); PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE)); Db2Adapter adapter = new Db2Adapter(tgt); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); SchemaApplyContext context = SchemaApplyContext.getDefault(); - model.collect(collector, adapter, context, new TransactionProviderTest(), vhs); + model.collect(collector, schemaAdapter, context, new TransactionProviderTest(), vhs); // FHIR in the hole! collector.startAndWait(); @@ -86,7 +91,7 @@ public void testDerbyTableCreation() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -95,8 +100,9 @@ public void testDerbyTableCreation() { // Pretend that our target is a Derby database DerbyAdapter adapter = new DerbyAdapter(tgt); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); SchemaApplyContext context = SchemaApplyContext.getDefault(); - model.apply(adapter, context); + model.apply(schemaAdapter, context); } @Test @@ -104,7 +110,7 @@ public void testTenantPartitioning() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -127,7 +133,7 @@ public void testDrop() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -136,7 +142,8 @@ public void testDrop() { // Pretend that our target is a DB2 database Db2Adapter adapter = new Db2Adapter(tgt); - model.drop(adapter); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); + model.drop(schemaAdapter); } @Test @@ -146,7 +153,8 @@ public void testVersionHistorySchema() { // Pretend that our target is a DB2 database Db2Adapter adapter = new Db2Adapter(tgt); - CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, adapter); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); + CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, schemaAdapter); } } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java index 1734dcb1172..3427ace4e28 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java @@ -13,7 +13,10 @@ import org.testng.annotations.Test; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -34,7 +37,7 @@ public void testParallelTableCreation() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -45,8 +48,9 @@ public void testParallelTableCreation() { ITaskCollector collector = taskService.makeTaskCollector(pool); PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE)); Db2Adapter adapter = new Db2Adapter(tgt); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); SchemaApplyContext context = SchemaApplyContext.getDefault(); - model.collect(collector, adapter, context, new TransactionProviderTest(), vhs); + model.collect(collector, schemaAdapter, context, new TransactionProviderTest(), vhs); // FHIR in the hole! collector.startAndWait(); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java index f4031df3d84..fe51963b68e 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java @@ -19,10 +19,13 @@ import org.testng.annotations.Test; import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.GetSequenceNextValueDAO; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.common.SchemaInfoObject; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.derby.DerbyMaster; @@ -76,17 +79,18 @@ protected void testDrop(IConnectionProvider cp, String schemaName) throws SQLExc PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, 10); ITransactionProvider transactionProvider = new SimpleTransactionProvider(cp); DerbyAdapter adapter = new DerbyAdapter(connectionPool); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter); VersionHistoryService vhs = new VersionHistoryService(ADMIN_SCHEMA_NAME, schemaName); vhs.setTransactionProvider(transactionProvider); vhs.setTarget(adapter); try (ITransaction tx = transactionProvider.getTransaction()) { PhysicalDataModel pdm = new PhysicalDataModel(); - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, schemaName, false); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, schemaName, SchemaType.PLAIN); gen.buildSchema(pdm); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + CreateWholeSchemaVersion.dropTable(schemaName, schemaAdapter); // Check that the schema is empty List schemaObjects = adapter.listSchemaObjects(schemaName); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java index 37c18593649..1aefbcd5075 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java @@ -30,6 +30,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider; import com.ibm.fhir.database.utils.derby.DerbyMaster; @@ -174,7 +175,7 @@ public void testMigrateFhirSchema() throws Exception { private void createOrUpgradeSchema(DerbyMaster db, IConnectionProvider pool, VersionHistoryService vhs, Set resourceTypes) throws SQLException { - FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false, resourceTypes); + FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN, resourceTypes); PhysicalDataModel pdm = new PhysicalDataModel(); gen.buildSchema(pdm); diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java index fe52ad1a198..beb96d26f31 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java @@ -244,6 +244,7 @@ default boolean isChangesSupported() { /** * Fetch up to resourceCount records from the RESOURCE_CHANGE_LOG table. * + * @param context the FHIRPersistenceContext associated with the current request * @param resourceCount the max number of resource change records to fetch * @param sinceLastModified filter records with record.lastUpdate >= sinceLastModified. Optional. * @param beforeLastModified filter records with record.lastUpdate <= beforeLastModified. Optional. @@ -254,7 +255,7 @@ default boolean isChangesSupported() { * @return a list containing up to resourceCount elements describing resources which have changed * @throws FHIRPersistenceException */ - List changes(int resourceCount, java.time.Instant sinceLastModified, + List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, Long changeIdMarker, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException; @@ -262,17 +263,19 @@ List changes(int resourceCount, java.time.Instant since /** * Erases part or a whole of a resource in the data layer. * + * @param context * @param eraseDto the details of the user input * @return a record indicating the success or partial success of the erase * @throws FHIRPersistenceException */ - default ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceException { + default ResourceEraseRecord erase(FHIRPersistenceContext context, EraseDTO eraseDto) throws FHIRPersistenceException { throw new FHIRPersistenceException("Erase is not supported"); } /** * Retrieves a list of index IDs available for reindexing. * + * @param context the FHIRPersistenceContext associated with this request * @param count the maximum nuber of index IDs to retrieve * @param notModifiedAfter only retrieve index IDs for resources not last updated after the specified timestamp * @param afterIndexId retrieve index IDs starting after this specified index ID, or null to start with first index ID @@ -280,7 +283,7 @@ default ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceExcep * @return list of index IDs available for reindexing * @throws FHIRPersistenceException */ - List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) + List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException; /** diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java index d618e3cf24a..871bc15574b 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java @@ -54,4 +54,11 @@ public interface FHIRPersistenceContext { * @return */ PayloadPersistenceResponse getOffloadResponse(); + + /** + * Get the key used for sharding used by the distributed schema variant. If + * the tenant is not configured for distribution, the value will be null + * @return any Short value in [Short.MIN_VALUE, Short.MAX_VALUE] or null + */ + Short getShardKey(); } \ No newline at end of file diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java index 836acc15289..b9825d886b3 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java @@ -71,10 +71,12 @@ public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEve * Returns a FHIRPersistenceContext that contains a FHIRPersistenceEvent and a FHIRSearchContext. * @param event the FHIRPersistenceEvent instance to be contained in the FHIRPersistenceContext instance * @param searchContext the FHIRSearchContext instance to be contained in the FHIRPersistenceContext instance + * @param shardKey */ - public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext) { + public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext, Short shardKey) { return FHIRPersistenceContextImpl.builder(event) .withSearchContext(searchContext) + .withShardKey(shardKey) .build(); } @@ -103,11 +105,14 @@ public static FHIRHistoryContext createHistoryContext() { * @param event the FHIRPersistenceEvent instance to be contained in the FHIRPersistenceContext instance * @param includeDeleted flag to tell the persistence layer to include deleted resources in the operation results * @param searchContext the FHIRSearchContext instance to be contained in the FHIRPersistenceContext instance + * @param shardKey the sharding value used for the distributed schema */ - public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, boolean includeDeleted, FHIRSearchContext searchContext) { + public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, boolean includeDeleted, FHIRSearchContext searchContext, + Short shardKey) { return FHIRPersistenceContextImpl.builder(event) .withIncludeDeleted(includeDeleted) .withSearchContext(searchContext) + .withShardKey(shardKey) .build(); } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java index db7b8f761c4..64d5d65b217 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java @@ -23,6 +23,7 @@ public class FHIRPersistenceContextImpl implements FHIRPersistenceContext { private FHIRSearchContext searchContext; private boolean includeDeleted = false; private Integer ifNoneMatch; + private Short shardKey; // The response from the payload persistence (offloading) call, if any private PayloadPersistenceResponse offloadResponse; @@ -46,6 +47,7 @@ public static class Builder { private boolean includeDeleted; private Integer ifNoneMatch; private PayloadPersistenceResponse offloadResponse; + private Short shardKey; /** * Protected constructor @@ -72,6 +74,7 @@ public FHIRPersistenceContext build() { impl.setIfNoneMatch(ifNoneMatch); impl.setIncludeDeleted(includeDeleted); impl.setOffloadResponse(offloadResponse); + impl.setShardKey(shardKey); return impl; } @@ -116,6 +119,16 @@ public Builder withIncludeDeleted(boolean includeDeleted) { return this; } + /** + * Build with the shardKey value + * @param shardKey + * @return + */ + public Builder withShardKey(Short shardKey) { + this.shardKey = shardKey; + return this; + } + /** * Build with the given offloadResponse * @param offloadResponse @@ -185,6 +198,11 @@ public boolean includeDeleted() { return includeDeleted; } + @Override + public Short getShardKey() { + return this.shardKey; + } + /** * Setter for the includeDeleted flag * @param includeDeleted @@ -193,6 +211,14 @@ public void setIncludeDeleted(boolean includeDeleted) { this.includeDeleted = includeDeleted; } + /** + * Set the shardKey value + * @param value + */ + public void setShardKey(Short value) { + this.shardKey = value; + } + /** * Setter for the If-None-Match header value * @param ifNoneMatch diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java new file mode 100644 index 00000000000..4f04d962b78 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java @@ -0,0 +1,46 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.sql.Timestamp; + +/** + * A date search parameter value + */ +public class DateParameter extends SearchParameterValue { + private Timestamp valueDateStart; + private Timestamp valueDateEnd; + + /** + * @return the valueDateStart + */ + public Timestamp getValueDateStart() { + return valueDateStart; + } + + /** + * @param valueDateStart the valueDateStart to set + */ + public void setValueDateStart(Timestamp valueDateStart) { + this.valueDateStart = valueDateStart; + } + + /** + * @return the valueDateEnd + */ + public Timestamp getValueDateEnd() { + return valueDateEnd; + } + + /** + * @param valueDateEnd the valueDateEnd to set + */ + public void setValueDateEnd(Timestamp valueDateEnd) { + this.valueDateEnd = valueDateEnd; + } + +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java new file mode 100644 index 00000000000..e4347a470e8 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java @@ -0,0 +1,23 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.util.concurrent.CompletableFuture; + +/** + * Interface to support dispatching of resource parameter blocks to another + * service for offline processing. + */ +public interface FHIRIndexProvider { + + /** + * Submit the index data request to the async indexing service we represent + * @param data + * @return A CompletableFuture which completes when the request is acknowledged to have been received by the async service + */ + CompletableFuture submit(RemoteIndexData data); +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java new file mode 100644 index 00000000000..b37a5c89d3d --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java @@ -0,0 +1,47 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import com.ibm.fhir.config.FHIRRequestContext; + +/** + * Service interface to support shipping resource search parameter values to an + * external service where they can be loaded into the database asynchronously. + * Implementations are expected to be tenant aware. They must use the tenantId + * value from the current {@link FHIRRequestContext} + */ +public abstract class FHIRRemoteIndexService { + + // For now we just publish this as a static service + // TODO we should be injecting these services to something like the request context + private static FHIRRemoteIndexService serviceInstance; + +// private ConcurrentHashMap + /** + * Initialize the serviceInstance value + * @param instance + */ + public static void setServiceInstance(FHIRRemoteIndexService instance) { + serviceInstance = instance; + } + + /** + * Get the serviceInstance value + * @return + */ + public static FHIRRemoteIndexService getServiceInstance() { + return serviceInstance; + } + + /** + * Submit the index data request to the async indexing service we represent + * @implNote implementations must use tenantId from {@link FHIRRequestContext} + * @param data + * @return A wrapper for a CompletableFuture which completes when the request is acknowledged to have been received by the async service + */ + public abstract IndexProviderResponse submit(RemoteIndexData data); +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java new file mode 100644 index 00000000000..5242c5fc47f --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java @@ -0,0 +1,45 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +/** + * Response from submitting IndexData to a FHIRIndexProvider implementation + */ +public class IndexProviderResponse { + private final RemoteIndexData data; + private final CompletableFuture ack; + /** + * Public constructor for a successful response + * @param tenantId + * @param data + */ + public IndexProviderResponse(RemoteIndexData data, CompletableFuture ack) { + this.data = data; + this.ack = ack; + } + + /** + * Get the request data for which this is the response + * @return + */ + public RemoteIndexData getData() { + return this.data; + } + + /** + * Get acknowledgement that the message was received by the service + * we sent it to + * @throws InterruptedException + * @throws ExecutionException + */ + public void getAck() throws InterruptedException, ExecutionException { + ack.get(); + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java new file mode 100644 index 00000000000..d4058472e81 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java @@ -0,0 +1,45 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A LatLng location search parameter + */ +public class LocationParameter extends SearchParameterValue { + private Double valueLatitude; + private Double valueLongitude; + + /** + * @return the valueLatitude + */ + public Double getValueLatitude() { + return valueLatitude; + } + + /** + * @param valueLatitude the valueLatitude to set + */ + public void setValueLatitude(Double valueLatitude) { + this.valueLatitude = valueLatitude; + } + + /** + * @return the valueLongitude + */ + public Double getValueLongitude() { + return valueLongitude; + } + + /** + * @param valueLongitude the valueLongitude to set + */ + public void setValueLongitude(Double valueLongitude) { + this.valueLongitude = valueLongitude; + } + +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java new file mode 100644 index 00000000000..153cb5e9ba4 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java @@ -0,0 +1,60 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.math.BigDecimal; + +/** + * A number search parameter + */ +public class NumberParameter extends SearchParameterValue { + private BigDecimal value; + private BigDecimal lowValue; + private BigDecimal highValue; + + /** + * @return the value + */ + public BigDecimal getValue() { + return value; + } + + /** + * @param value the value to set + */ + public void setValue(BigDecimal value) { + this.value = value; + } + + /** + * @return the lowValue + */ + public BigDecimal getLowValue() { + return lowValue; + } + + /** + * @param lowValue the lowValue to set + */ + public void setLowValue(BigDecimal lowValue) { + this.lowValue = lowValue; + } + + /** + * @return the highValue + */ + public BigDecimal getHighValue() { + return highValue; + } + + /** + * @param highValue the highValue to set + */ + public void setHighValue(BigDecimal highValue) { + this.highValue = highValue; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java new file mode 100644 index 00000000000..b105dcde235 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java @@ -0,0 +1,78 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.math.BigDecimal; +import java.sql.Timestamp; + +import com.ibm.fhir.search.util.ReferenceValue; + +/** + * Used by a parameter value visitor to translate the parameter values + * to a new form + */ +public interface ParameterValueVisitorAdapter { + + /** + * @param name + * @param valueString + * @param compositeId + */ + void stringValue(String name, String valueString, Integer compositeId); + + /** + * @param name + * @param valueNumber + * @param valueNumberLow + * @param valueNumberHigh + * @param compositeId + */ + void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId); + + /** + * @param name + * @param valueDateStart + * @param valueDateEnd + * @param compositeId + */ + void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId); + + /** + * @param name + * @param valueSystem + * @param valueCode + * @param compositeId + */ + void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId); + + /** + * @param name + * @param valueSystem + * @param valueCode + * @param valueNumber + * @param valueNumberLow + * @param valueNumberHigh + * @param compositeId + */ + void quantityValue(String name, String valueSystem, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, + Integer compositeId); + + /** + * @param name + * @param valueLatitude + * @param valueLongitude + * @param compositeId + */ + void locationValue(String name, Double valueLatitude, Double valueLongitude, Integer compositeId); + + /** + * @param name + * @param refValue + * @param compositeId + */ + void referenceValue(String name, ReferenceValue refValue, Integer compositeId); +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java new file mode 100644 index 00000000000..f4f931e0983 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java @@ -0,0 +1,90 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.math.BigDecimal; + +/** + * A quantity search parameter value + */ +public class QuantityParameter extends SearchParameterValue { + private BigDecimal valueNumber; + private BigDecimal valueNumberLow; + private BigDecimal valueNumberHigh; + private String valueSystem; + private String valueCode; + + /** + * @return the valueNumber + */ + public BigDecimal getValueNumber() { + return valueNumber; + } + + /** + * @param valueNumber the valueNumber to set + */ + public void setValueNumber(BigDecimal valueNumber) { + this.valueNumber = valueNumber; + } + + /** + * @return the valueNumberLow + */ + public BigDecimal getValueNumberLow() { + return valueNumberLow; + } + + /** + * @param valueNumberLow the valueNumberLow to set + */ + public void setValueNumberLow(BigDecimal valueNumberLow) { + this.valueNumberLow = valueNumberLow; + } + + /** + * @return the valueNumberHigh + */ + public BigDecimal getValueNumberHigh() { + return valueNumberHigh; + } + + /** + * @param valueNumberHigh the valueNumberHigh to set + */ + public void setValueNumberHigh(BigDecimal valueNumberHigh) { + this.valueNumberHigh = valueNumberHigh; + } + + /** + * @return the valueSystem + */ + public String getValueSystem() { + return valueSystem; + } + + /** + * @param valueSystem the valueSystem to set + */ + public void setValueSystem(String valueSystem) { + this.valueSystem = valueSystem; + } + + /** + * @return the valueCode + */ + public String getValueCode() { + return valueCode; + } + + /** + * @param valueCode the valueCode to set + */ + public void setValueCode(String valueCode) { + this.valueCode = valueCode; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java new file mode 100644 index 00000000000..f9e79c496df --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java @@ -0,0 +1,57 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +/** + * Supplier of index data + */ +public class RemoteIndexData { + // The partition key used to route the data + private final String partitionKey; + + // The data object holding all the extracted search parameters + private final SearchParametersTransport searchParameters; + + @Override + public String toString() { + final StringBuilder result = new StringBuilder(); + result.append("partitionKey:[").append(partitionKey).append("]"); + result.append(" resource:["); + result.append(searchParameters.getResourceType()).append("/").append(searchParameters.getLogicalId()); + result.append("]"); + return result.toString(); + } + + /** + * Public constructor + * @param partitionKey + * @param searchParameters + */ + public RemoteIndexData(String partitionKey, SearchParametersTransport searchParameters) { + this.searchParameters = searchParameters; + this.partitionKey = partitionKey; + } + + /** + * Get the search parameter block representing the data we want to send + * to the remote indexing service + * @return + */ + public SearchParametersTransport getSearchParameters() { + return this.searchParameters; + } + + /** + * Get the key used to select which partition we want to send to. Partitions + * are important because we want to see the IndexData processed in order within + * a particular partition + * @return + */ + public String getPartitionKey() { + return this.partitionKey; + } +} \ No newline at end of file diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java new file mode 100644 index 00000000000..3c35a98599f --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +/** + * The Kafka message we send to the remote index service + */ +public class RemoteIndexMessage { + private String tenantId; + private SearchParametersTransport data; + + /** + * @return the tenantId + */ + public String getTenantId() { + return tenantId; + } + + /** + * @param tenantId the tenantId to set + */ + public void setTenantId(String tenantId) { + this.tenantId = tenantId; + } + + /** + * @return the data + */ + public SearchParametersTransport getData() { + return data; + } + + /** + * @param data the data to set + */ + public void setData(SearchParametersTransport data) { + this.data = data; + } + +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java new file mode 100644 index 00000000000..0e7feea6e55 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java @@ -0,0 +1,49 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * The base class for our search parameter values. These index model classes + * are designed to reflect the raw values we want the remote indexing + * service to store + */ +public class SearchParameterValue { + // The name of the parameter + private String name; + + // The composite id used to tie together values belonging to the same composite parameter. Null for ordinary params. + private Integer compositeId; + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the compositeId + */ + public Integer getCompositeId() { + return compositeId; + } + + /** + * @param compositeId the compositeId to set + */ + public void setCompositeId(Integer compositeId) { + this.compositeId = compositeId; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java new file mode 100644 index 00000000000..2fa53df3088 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java @@ -0,0 +1,350 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a collection of search parameters extracted from a FHIR resource + * held in a form that is easy to serialize/deserialize to a wire format + * (e.g. JSON) for sending to a remote/async indexing service. + * @implNote because we want to serialize/deserialize this object + * as JSON, we need to keep it simple + */ +public class SearchParametersTransport { + + // The FHIR resource type name + private String resourceType; + + // The logical id of the resource + private String logicalId; + + // The database identifier assigned to this resource + private long logicalResourceId; + + // The key value used for sharding the data when using a distributed database + private Short shardKey; + + private List stringValues; + private List numberValues; + private List quantityValues; + private List tokenValues; + private List dateValues; + private List locationValues; + + /** + * Factory method to create a {@link Builder} instance + * @return + */ + public static Builder builder() { + return new Builder(); + } + + /** + * A builder to make it easier to construct a {@link SearchParametersTransport} + */ + public static class Builder { + private List stringValues = new ArrayList<>(); + private List numberValues = new ArrayList<>(); + private List quantityValues = new ArrayList<>(); + private List tokenValues = new ArrayList<>(); + private List dateValues = new ArrayList<>(); + private List locationValues = new ArrayList<>(); + + private String resourceType; + private String logicalId; + private long logicalResourceId = -1; + private Short shardKey; + + /** + * Set the resourceType + * @param resourceType + * @return + */ + public Builder withResourceType(String resourceType) { + this.resourceType = resourceType; + return this; + } + + /** + * Set the logicalId + * @param logicalId + * @return + */ + public Builder withLogicalId(String logicalId) { + this.logicalId = logicalId; + return this; + } + + /** + * Set the logicalResourceId + * @param logicalResourceId + * @return + */ + public Builder withLogicalResourceId(long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + return this; + } + + /** + * Set the shardKey + * @param shardKey + * @return + */ + public Builder withShardKey(Short shardKey) { + this.shardKey = shardKey; + return this; + } + + /** + * Add a string parameter value + * @param value + * @return + */ + public Builder addStringValue(StringParameter value) { + stringValues.add(value); + return this; + } + + /** + * Add a number parameter value + * @param value + * @return + */ + public Builder addNumberValue(NumberParameter value) { + numberValues.add(value); + return this; + } + + /** + * Add a quantity parameter value + * @param value + * @return + */ + public Builder addQuantityValue(QuantityParameter value) { + quantityValues.add(value); + return this; + } + + /** + * Add a token parameter value + * @param value + * @return + */ + public Builder addTokenValue(TokenParameter value) { + tokenValues.add(value); + return this; + } + + /** + * Add a date parameter value + * @param value + * @return + */ + public Builder addDateValue(DateParameter value) { + dateValues.add(value); + return this; + } + + /** + * Add a location parameter value + * @param value + * @return + */ + public Builder addLocationValue(LocationParameter value) { + locationValues.add(value); + return this; + } + + /** + * Builder a new {@link SearchParametersTransport} instance based on the current state + * of this {@link Builder}. + * @return + */ + public SearchParametersTransport build() { + if (this.logicalResourceId < 0) { + throw new IllegalStateException("Must set logicalResourceId"); + } + if (this.resourceType == null) { + throw new IllegalStateException("Must set resourceType"); + } + if (this.logicalId == null) { + throw new IllegalStateException("Must set logicalId"); + } + + SearchParametersTransport result = new SearchParametersTransport(); + result.resourceType = this.resourceType; + result.logicalId = this.logicalId; + result.logicalResourceId = this.logicalResourceId; + result.shardKey = this.shardKey; + + if (this.stringValues.size() > 0) { + result.stringValues = new ArrayList<>(this.stringValues); + } + if (this.numberValues.size() > 0) { + result.numberValues = new ArrayList<>(this.numberValues); + } + if (this.quantityValues.size() > 0) { + result.quantityValues = new ArrayList<>(this.quantityValues); + } + if (this.tokenValues.size() > 0) { + result.tokenValues = new ArrayList<>(this.tokenValues); + } + if (this.dateValues.size() > 0) { + result.dateValues = new ArrayList<>(this.dateValues); + } + if (this.locationValues.size() > 0) { + result.locationValues = new ArrayList<>(this.locationValues); + } + return result; + } + } + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + + /** + * @param resourceType the resourceType to set + */ + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } + + + /** + * @param logicalId the logicalId to set + */ + public void setLogicalId(String logicalId) { + this.logicalId = logicalId; + } + + + /** + * @return the logicalResourceId + */ + public long getLogicalResourceId() { + return logicalResourceId; + } + + + /** + * @param logicalResourceId the logicalResourceId to set + */ + public void setLogicalResourceId(long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + } + + + /** + * @return the stringValues + */ + public List getStringValues() { + return stringValues; + } + + + /** + * @param stringValues the stringValues to set + */ + public void setStringValues(List stringValues) { + this.stringValues = stringValues; + } + + + /** + * @return the numberValues + */ + public List getNumberValues() { + return numberValues; + } + + + /** + * @param numberValues the numberValues to set + */ + public void setNumberValues(List numberValues) { + this.numberValues = numberValues; + } + + + /** + * @return the quantityValues + */ + public List getQuantityValues() { + return quantityValues; + } + + + /** + * @param quantityValues the quantityValues to set + */ + public void setQuantityValues(List quantityValues) { + this.quantityValues = quantityValues; + } + + + /** + * @return the tokenValues + */ + public List getTokenValues() { + return tokenValues; + } + + + /** + * @param tokenValues the tokenValues to set + */ + public void setTokenValues(List tokenValues) { + this.tokenValues = tokenValues; + } + + + /** + * @return the dateValues + */ + public List getDateValues() { + return dateValues; + } + + + /** + * @param dateValues the dateValues to set + */ + public void setDateValues(List dateValues) { + this.dateValues = dateValues; + } + + + /** + * @return the locationValues + */ + public List getLocationValues() { + return locationValues; + } + + + /** + * @param locationValues the locationValues to set + */ + public void setLocationValues(List locationValues) { + this.locationValues = locationValues; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java new file mode 100644 index 00000000000..4a56ba46e7a --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java @@ -0,0 +1,29 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A string search parameter used for transporting values for remote indexing + */ +public class StringParameter extends SearchParameterValue { + private String value; + + /** + * @return the value + */ + public String getValue() { + return value; + } + + /** + * @param value the value to set + */ + public void setValue(String value) { + this.value = value; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java new file mode 100644 index 00000000000..20232be10de --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java @@ -0,0 +1,62 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A token search parameter value + */ +public class TokenParameter extends SearchParameterValue { + private String valueSystem; + private String valueCode; + + // for storing versioned references + private Integer refVersionId; + + /** + * @return the valueSystem + */ + public String getValueSystem() { + return valueSystem; + } + + /** + * @param valueSystem the valueSystem to set + */ + public void setValueSystem(String valueSystem) { + this.valueSystem = valueSystem; + } + + /** + * @return the valueCode + */ + public String getValueCode() { + return valueCode; + } + + /** + * @param valueCode the valueCode to set + */ + public void setValueCode(String valueCode) { + this.valueCode = valueCode; + } + + /** + * @return the refVersionId + */ + public Integer getRefVersionId() { + return refVersionId; + } + + /** + * @param refVersionId the refVersionId to set + */ + public void setRefVersionId(Integer refVersionId) { + this.refVersionId = refVersionId; + } + +} diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java index f6c9bc0c01b..d1d2df0c9f8 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java @@ -78,7 +78,7 @@ public void test4() { FHIRSearchContext sc = FHIRSearchContextFactory.createSearchContext(); assertNotNull(sc); - FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc); + FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc, (short)13); assertNotNull(ctxt); assertNotNull(ctxt.getPersistenceEvent()); assertEquals(pe, ctxt.getPersistenceEvent()); @@ -86,5 +86,6 @@ public void test4() { assertEquals(sc, ctxt.getSearchContext()); assertFalse(ctxt.includeDeleted()); assertNull(ctxt.getHistoryContext()); + assertEquals(13, Short.toUnsignedInt(ctxt.getShardKey())); } } diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java index 79b39892440..895ff7e8ff0 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java @@ -93,13 +93,13 @@ public ResourcePayload fetchResourcePayloads(Class resourceT } @Override - public List changes(int resourceCount, Instant fromLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames, + public List changes(FHIRPersistenceContext context, int resourceCount, Instant fromLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException { return Collections.emptyList(); } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { return Collections.emptyList(); } diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java index 6a6b13156f8..a26f1491579 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java @@ -104,7 +104,7 @@ public void testSomeData() throws Exception { final List resourceTypeNames = Collections.emptyList(); final boolean excludeTransactionTimeoutWindow = false; final HistorySortOrder historySortOrder = HistorySortOrder.NONE; - List result = persistence.changes(100, sinceLastModified, beforeLastModified, changeIdMarker, resourceTypeNames, excludeTransactionTimeoutWindow, historySortOrder); + List result = persistence.changes(null, 100, sinceLastModified, beforeLastModified, changeIdMarker, resourceTypeNames, excludeTransactionTimeoutWindow, historySortOrder); assertNotNull(result); assertTrue(result.size() >= 7); assertTrue(result.size() <= 100); @@ -119,7 +119,7 @@ public void testChanges() throws Exception { final Long afterResourceId = null; final String resourceTypeName = null; - List result = persistence.changes(7, sinceLastModified, null, null, null, false, HistorySortOrder.NONE); + List result = persistence.changes(null, 7, sinceLastModified, null, null, null, false, HistorySortOrder.NONE); assertNotNull(result); // 4 CREATE @@ -160,7 +160,7 @@ public void testLimit() throws Exception { Long afterResourceId = null; final String resourceTypeName = null; - List result = persistence.changes(4, sinceLastModified, null, null, null, false, HistorySortOrder.NONE); + List result = persistence.changes(null, 4, sinceLastModified, null, null, null, false, HistorySortOrder.NONE); assertNotNull(result); // Limit was set to 4, so we should only get partial data @@ -169,14 +169,14 @@ public void testLimit() throws Exception { // Make another call now to get the remaining 3 changes sinceLastModified = result.get(3).getChangeTstamp(); afterResourceId = result.get(3).getChangeId(); - result = persistence.changes(3, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE); + result = persistence.changes(null, 3, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE); assertNotNull(result); assertEquals(result.size(), 3); // And a final call to make sure we get nothing sinceLastModified = result.get(2).getChangeTstamp(); afterResourceId = result.get(2).getChangeId(); - result = persistence.changes(100, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE); + result = persistence.changes(null, 100, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE); assertNotNull(result); assertEquals(result.size(), 0); } @@ -189,7 +189,7 @@ public void testResourceTypeFilter() throws Exception { final String resourceTypeName = resource1.getClass().getSimpleName(); List resourceTypeNames = Arrays.asList(resourceTypeName); - List result = persistence.changes(10, sinceLastModified, null, afterResourceId, resourceTypeNames, false, HistorySortOrder.NONE); + List result = persistence.changes(null, 10, sinceLastModified, null, afterResourceId, resourceTypeNames, false, HistorySortOrder.NONE); assertNotNull(result); assertEquals(result.size(), 7); } diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java index 448a97419cd..877df87e687 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java @@ -126,7 +126,7 @@ public void testEraseResourceWithHistory() throws Exception { EraseDTO dto = new EraseDTO(); dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), 3); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE); @@ -141,7 +141,7 @@ public void testEraseSingleResource() throws Exception { EraseDTO dto = new EraseDTO(); dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), 1); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE); @@ -158,7 +158,7 @@ public void testEraseLastIsDeleted() throws Exception { EraseDTO dto = new EraseDTO(); dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), 2); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE); @@ -171,7 +171,7 @@ public void testEraseNotExists() throws Exception { EraseDTO dto = new EraseDTO(); dto.setLogicalId("---NOTEXISTS"); dto.setResourceType("Basic"); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_FOUND); } @@ -189,7 +189,7 @@ public void testEraseSpecificVersionGreater() throws Exception { dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); dto.setVersion(4); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), -1); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_SUPPORTED_GREATER); @@ -209,7 +209,7 @@ public void testEraseSpecificVersion() throws Exception { dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); dto.setVersion(2); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), 1); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.VERSION); @@ -232,7 +232,7 @@ public void testEraseLatestSpecificVersion() throws Exception { dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); dto.setVersion(3); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), -1); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_SUPPORTED_LATEST); @@ -248,7 +248,7 @@ public void testEraseSingleResourceWithVersion1() throws Exception { dto.setLogicalId(resource1.getId()); dto.setResourceType("Basic"); dto.setVersion(1); - ResourceEraseRecord eraseRecord = persistence.erase(dto); + ResourceEraseRecord eraseRecord = persistence.erase(null, dto); assertNotNull(eraseRecord); assertEquals((int) eraseRecord.getTotal(), 1); assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE); diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java index 89b5c56009e..0b424be8db8 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java @@ -97,7 +97,7 @@ protected FHIRPersistenceContext getPersistenceContextIfNoneMatch() throws Excep return FHIRPersistenceContextFactory.createPersistenceContext(null, ifNoneMatch); } protected FHIRPersistenceContext getPersistenceContextForSearch(FHIRSearchContext ctxt) { - return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt); + return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt, null); } protected FHIRPersistenceContext getPersistenceContextForHistory(FHIRHistoryContext ctxt) { return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt); diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml new file mode 100644 index 00000000000..36db98f99db --- /dev/null +++ b/fhir-remote-index/pom.xml @@ -0,0 +1,113 @@ + + 4.0.0 + + fhir-remote-index + + com.ibm.fhir + fhir-parent + 4.11.0-SNAPSHOT + ../fhir-parent + + + + UTF-8 + + + + + ${project.groupId} + fhir-persistence + ${project.version} + + + ${project.groupId} + fhir-model + ${project.version} + + + ${project.groupId} + fhir-validation + ${project.version} + + + ${project.groupId} + fhir-config + ${project.version} + provided + + + ${project.groupId} + fhir-database-utils + ${project.version} + + + com.google.code.gson + gson + + + org.apache.derby + derby + + + com.ibm.db2 + jcc + + + org.postgresql + postgresql + + + org.apache.kafka + kafka-clients + + + org.testng + testng + test + + + org.apache.derby + derbytools + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + cli + + + com.ibm.fhir.remote.index.app.Main + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java new file mode 100644 index 00000000000..7b0b8a520b5 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java @@ -0,0 +1,18 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.api; + +import java.util.List; + +/** + * Our interface for handling messages received by the consumer. Used + * to decouple the Kafka consumer from the database persistence logic + */ +public interface IMessageHandler { + + void process(List messages); +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java new file mode 100644 index 00000000000..75c960ddf62 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -0,0 +1,283 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.app; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.PartitionInfo; + +import com.ibm.fhir.database.utils.thread.ThreadHandler; +import com.ibm.fhir.remote.index.api.IMessageHandler; +import com.ibm.fhir.remote.index.kafka.RemoteIndexConsumer; + +/** + * Main class for the FHIR remote index service Kafka consumer + */ +public class Main { + private static final Logger logger = Logger.getLogger(Main.class.getName()); + // Properties holding the Kafka connection information + private final Properties kafkaProperties = new Properties(); + // Properties holding the JDBC connection information + private final Properties databaseProperties = new Properties(); + + private String topicName; + + // The standard consumer group which all the remote index consumers should use + private String consumerGroup = "remote-index-service-cg"; + + private int consumerCount = 1; + + private Duration pollDuration; + private long maxBatchCollectTimeMs = 5000; + + // the list of consumers + private final List consumers = new ArrayList<>(); + + // track the number of consumers that are still running + private AtomicInteger stillRunningCounter; + + private volatile boolean running; + + // Exit if we drop below this number of running consumers + private int minRunningConsumerThreshold = 1; + + /** + * Parse the given command line arguments + * @param args + */ + public void parseArgs(String[] args) { + int a = 0; + while (a < args.length) { + final String arg = args[a++]; + switch (arg) { + case "--kafka-properties": + if (a < args.length && !args[a].startsWith("--")) { + loadKafkaProperties(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --kafka-properties"); + } + break; + case "--database-properties": + if (a < args.length && !args[a].startsWith("--")) { + loadDatabaseProperties(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --database-properties"); + } + break; + case "--topic-name": + if (a < args.length && !args[a].startsWith("--")) { + topicName = args[a++]; + } else { + throw new IllegalArgumentException("Missing value for --topic-name"); + } + break; + case "--consumer-group": + if (a < args.length && !args[a].startsWith("--")) { + consumerGroup = args[a++]; + } else { + throw new IllegalArgumentException("Missing value for --topic-name"); + } + break; + case "--consumer-count": + if (a < args.length && !args[a].startsWith("--")) { + consumerCount = Integer.parseInt(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --topic-name"); + } + break; + default: + throw new IllegalArgumentException("Bad arg: '" + arg + "'"); + } + } + } + + /** + * Read kafka properties from the given file + * @param filename + */ + protected void loadKafkaProperties(String filename) { + try (InputStream is = new FileInputStream(filename)) { + kafkaProperties.load(is); + } catch (IOException x) { + throw new IllegalArgumentException(x); + } + } + + /** + * Read database properties from the given file + * @param filename + */ + protected void loadDatabaseProperties(String filename) { + try (InputStream is = new FileInputStream(filename)) { + databaseProperties.load(is); + } catch (IOException x) { + throw new IllegalArgumentException(x); + } + } + + /** + * Keep consuming from Kafka forever...or until we see too many + * consumers fail + */ + public void run() { + dumpProperties("kafka", kafkaProperties); + dumpProperties("database", databaseProperties); + + // One thread per consumer + ExecutorService pool = Executors.newCachedThreadPool(); + for (int i=0; i kc = buildConsumer(); + if (i == 0) { + // use the first consumer to check we have partitions for the configured topic + doPartitionCheck(kc); + } + IMessageHandler handler = buildHandler(); + RemoteIndexConsumer consumer = new RemoteIndexConsumer(kc, handler, () -> failedConsumerCallback(), topicName, maxBatchCollectTimeMs, pollDuration); + pool.submit(consumer); + // Keep track of the consumer, so that we can signal a shutdown if we need to + consumers.add(consumer); + } + + // Hold in a slow poll loop, ideally forever. We only exit if too + // many consumers fail + while (running) { + ThreadHandler.safeSleep(1000); + } + + // Make sure anything still running is stopped + logger.warning("Too many consumers have failed, so stopping everything"); + for (RemoteIndexConsumer consumer: consumers) { + consumer.shutdown(); + } + + // Try to make the exit as clean as possible, although this may not happen + // because we're likely in some sort of failure scenario here (e.g. network + // partition, brokers failed, database down etc). + int waitForTerminationSeconds = 30; + logger.info("Waiting " + waitForTerminationSeconds + " seconds for consumers to stop"); + try { + pool.awaitTermination(waitForTerminationSeconds, TimeUnit.SECONDS); + } catch (InterruptedException x) { + logger.warning("Interrupted waiting for consumer pool to terminate"); + } + logger.info("All consumers stopped"); + } + + /** + * Create a new consumer + * @return + */ + public KafkaConsumer buildConsumer() { + + Properties kp = new Properties(); + kp.putAll(kafkaProperties); + + // Inject the properties we want to force here + kp.put("enable.auto.commit", "false"); + kp.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + kp.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + kp.put("group.id", this.consumerGroup); + + KafkaConsumer consumer = new KafkaConsumer<>(kp); + return consumer; + } + + /** + * Get the partitions for the named topic to check if the topic actually exists + */ + private void doPartitionCheck(KafkaConsumer consumer) { + // Checking for topic existence before subscribing + List partitions = consumer.partitionsFor(topicName); + if (partitions == null || partitions.isEmpty()) { + logger.severe("Topic not found: '" + topicName + "'"); + throw new IllegalStateException("topic not found"); + } else { + // dump the list of partitions configured for this topic + for (PartitionInfo pi: partitions) { + logger.info("Topic '" + topicName + "' has partition " + pi.toString()); + } + } + } + + + /** + * Log the properties which can help with debugging deployment issues. + * Hides secrets + * @param which the type of properties + * @param p the properties to dump + */ + protected void dumpProperties(String which, Properties p) { + + if (which != null && p != null) { + StringBuilder buffer = new StringBuilder(); + buffer.append("{"); + Iterator keys = p.keySet().iterator(); + boolean first = true; + while (keys.hasNext()) { + String key = (String) keys.next(); + String value = p.getProperty(key); + if (key.toLowerCase().contains("password")) { + value = "[*******]"; + } + if (first) { + first = false; + } else { + buffer.append(", "); + } + buffer.append("\"").append(key).append("\""); + buffer.append(": "); + buffer.append("\"").append(value).append("\""); + } + buffer.append("}"); + logger.info(which + ": " + buffer.toString()); + } + } + + public IMessageHandler buildHandler() { + // + return null; + } + + private void failedConsumerCallback() { + if (this.stillRunningCounter.decrementAndGet() < minRunningConsumerThreshold) { + // Signal termination of the entire program + logger.severe("Too many consumers have failed. Terminating"); + this.running = false; + } + } + + /** + * @param args + */ + public static void main(String[] args) { + Main m = new Main(); + try { + m.parseArgs(args); + m.run(); + } catch (Throwable t) { + logger.log(Level.SEVERE, "terminating", t); + } finally { + // Any exit means something failed, so we call this an error so a container + // environment can react accordingly + System.exit(1); + } + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java new file mode 100644 index 00000000000..0f46b1bf603 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -0,0 +1,152 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.util.List; + +import com.google.gson.Gson; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParametersTransport; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.IMessageHandler; + + +/** + * + */ +public abstract class BaseMessageHandler implements IMessageHandler { + + @Override + public void process(List messages) { + for (String payload: messages) { + RemoteIndexMessage message = unmarshall(payload); + process(message); + } + pushBatch(); + commit(); + } + + /** + * Push any data we've accumulated from processing messages. + */ + protected abstract void pushBatch(); + + /** + * Unmarshall the json payload string into a RemoteIndexMessage + * @param payload + * @return + */ + private RemoteIndexMessage unmarshall(String jsonPayload) { + Gson gson = new Gson(); + return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + } + /** + * Process the data + * @param message + */ + private void process(RemoteIndexMessage message) { + SearchParametersTransport params = message.getData(); + if (params.getStringValues() != null) { + for (StringParameter p: params.getStringValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getDateValues() != null) { + for (DateParameter p: params.getDateValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getNumberValues() != null) { + for (NumberParameter p: params.getNumberValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getQuantityValues() != null) { + for (QuantityParameter p: params.getQuantityValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getTokenValues() != null) { + for (TokenParameter p: params.getTokenValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getLocationValues() != null) { + for (LocationParameter p: params.getLocationValues()) { + process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + } + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p); + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p); + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p); + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p); + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p); + + /** + * @param tenantId + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p); + + private void commit() { + + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java new file mode 100644 index 00000000000..8112df767d2 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -0,0 +1,150 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; + +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TokenParameter; + +/** + * Loads search parameter values into the target FHIR schema on + * a PostgreSQL database. + */ +public class DistributedPostgresMessageHandler extends BaseMessageHandler { + + // the connection to use for the inserts + private final Connection connection; + + // rarely used so no need for {@code java.sql.PreparedStatement} or batching on this one + // even on Location resources its only there once by default + private final String insertLocation; + + // Searchable string attributes stored at the system level + private final PreparedStatement systemStrings; + private int systemStringCount; + + // Searchable date attributes stored at the system level + private final PreparedStatement systemDates; + private int systemDateCount; + + /** + * Public constructor + * + * @param connection + */ + public DistributedPostgresMessageHandler(Connection connection) { + this.connection = connection; + insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + strings = connection.prepareStatement(insertString); + + insertNumber = multitenant ? + "INSERT INTO " + tablePrefix + "_number_values (mt_id, parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?,?)" + : + "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?)"; + numbers = c.prepareStatement(insertNumber); + + insertDate = multitenant ? + "INSERT INTO " + tablePrefix + "_date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?)" + : + "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + dates = c.prepareStatement(insertDate); + + insertQuantity = multitenant ? + "INSERT INTO " + tablePrefix + "_quantity_values (mt_id, parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?,?,?,?)" + : + "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?,?,?)"; + quantities = c.prepareStatement(insertQuantity); + + insertLocation = multitenant ? "INSERT INTO " + tablePrefix + "_latlng_values (mt_id, parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?)" + : "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + + if (storeWholeSystemParams) { + // System level string attributes + String insertSystemString = multitenant ? + "INSERT INTO str_values (mt_id, parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" + : + "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; + systemStrings = c.prepareStatement(insertSystemString); + + // System level date attributes + String insertSystemDate = multitenant ? + "INSERT INTO date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" + : + "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)"; + systemDates = c.prepareStatement(insertSystemDate); + } else { + systemStrings = null; + systemDates = null; + } + } + /** + * Compute the shard key value use to distribute resources among nodes + * of the database + * @param resourceType + * @param logicalId + * @return + */ + private short encodeShardKey(String resourceType, String logicalId) { + final String requestShardKey = resourceType + "/" + logicalId; + return Short.valueOf((short)requestShardKey.hashCode()); + } + + @Override + protected void pushBatch() { + // Push any data we've accumulated so far. This may occur + // if we cross a volume threshold, and will always occur as + // the last step before the current transaction is committed, + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } + + @Override + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) { + final short shardKey = encodeShardKey(resourceType, logicalId); + // TODO Auto-generated method stub + + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java new file mode 100644 index 00000000000..549875a1c9d --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java @@ -0,0 +1,46 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; + +/** + * Parameter batch statements configured for a given resource type + */ +public class DistributedPostgresParameterBatch { + private final Connection connection; + private final String resourceType; + private final PreparedStatement strings; + private int stringCount; + + private final PreparedStatement numbers; + private int numberCount; + + private final PreparedStatement dates; + private int dateCount; + + private final PreparedStatement quantities; + private int quantityCount; + + /** + * Public constructor + * @param connection + * @param resourceType + */ + public DistributedPostgresParameterBatch(Connection connection, String resourceType) { + this.connection = connection; + this.resourceType = resourceType; + } + + /** + * Push the current batch + */ + public void pushBatch() { + + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java new file mode 100644 index 00000000000..6cfe482bc4d --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java @@ -0,0 +1,180 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package com.ibm.fhir.remote.index.kafka; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; + +import com.ibm.fhir.remote.index.api.IMessageHandler; + +/** + * Kafka consumer reading remote index messages, batches the data and + * loads it into the configured database. + */ +public class RemoteIndexConsumer implements Runnable { + private static final Logger logger = Logger.getLogger(RemoteIndexConsumer.class.getName()); + + // Nanoseconds in a second + private static final long NANOS = 1000000000L; + + // the remote index service Kafka topic name + private final String topicName; + + // The max time to spend collecting a batch before submitting + private final long maxBatchCollectTimeMs; + + // The consumer object representing the connection to Kafka + private final KafkaConsumer kafkaConsumer; + + // The handler we use to process messages we receive from Kafka + private final IMessageHandler messageHandler; + + private final Duration pollWaitTime; + + // Flag used to exit kafka loop on termination + private volatile boolean running = true; + + // The map we use to track offsets as we process messages + private Map trackingOffsetMap = new HashMap<>(); + private Map rollbackOffsetMap = new HashMap<>(); + + // Use a list to stage the results of a poll() + private final LinkedHashSet> recordBuffer = new LinkedHashSet<>(); + private final List> currentBatch = new ArrayList<>(); + private final Set assignedPartitions = new HashSet<>(); + + // Track the number of sequential commit failures + private int commitFailures = 0; + + // If we get 5 failures in a row, we disconnect + private static final int COMMIT_FAILURE_DISCONNECT_THRESHOLD = 5; + + // A callback used to signal that this consumer has failed + private final Runnable consumerFailedCallback; + + /** + * A listener to track the partitions assigned to this cluster member + * Callbacks happen as part of the consumer#poll() call, so no concurrency concerns + */ + private class PartitionChangeListener implements ConsumerRebalanceListener { + + @Override + public void onPartitionsRevoked(Collection partitions) { + for (TopicPartition tp: partitions) { + logger.info("Revoking partition: " + tp.topic() + ":" + tp.partition()); + } + assignedPartitions.removeAll(partitions); + } + + @Override + public void onPartitionsAssigned(Collection partitions) { + for (TopicPartition tp: partitions) { + logger.info("Assigning partition: " + tp.topic() + ":" + tp.partition()); + } + assignedPartitions.addAll(partitions); + } + } + + /** + * Public constructor + * + * @param kafkaConsumer + * @param messageHandler + * @param consumerFailedCallback + * @param topicName + * @param maxBatchCollectTimeMs + * @param pollWaitTime + */ + public RemoteIndexConsumer(KafkaConsumer kafkaConsumer, IMessageHandler messageHandler, + Runnable consumerFailedCallback, String topicName, long maxBatchCollectTimeMs, Duration pollWaitTime) { + this.kafkaConsumer = kafkaConsumer; + this.messageHandler = messageHandler; + this.consumerFailedCallback = consumerFailedCallback; + this.topicName = topicName; + this.maxBatchCollectTimeMs = maxBatchCollectTimeMs; + this.pollWaitTime = pollWaitTime; + } + + /** + * poll the consumer and forward any messages we receive to the message handler + */ + private void consume() { + ConsumerRecords records = kafkaConsumer.poll(pollWaitTime); + List messages = new ArrayList<>(); + + for (ConsumerRecord record : records) { + messages.add(record.value()); + } + messageHandler.process(messages); + kafkaConsumer.commitAsync(); + } + + + public void shutdown() { + this.running = false; + try { + kafkaConsumer.wakeup(); + } catch (Throwable x) { + logger.warning("Error waking up kafka consumer: " + x.getMessage()); + } + } + + /** + * Get an array of the TopicPartition tuples currently assigned to this node + * @return the assigned TopicPartitions, or null if the consumer.assignment is empty + */ + private TopicPartition[] getConsumerAssignment() { + Set assignment = kafkaConsumer.assignment(); + + if (assignment.size() > 0) { + return assignment.toArray(new TopicPartition[assignment.size()]); + } + else { + return null; + } + } + + @Override + public void run() { + logger.info("Subscribing consumer to topic '" + this.topicName + "'"); + kafkaConsumer.subscribe(Collections.singletonList(topicName), new PartitionChangeListener()); + + logger.info("Starting consumer loop"); + while (running) { + try { + consume(); + } catch (Throwable t) { + // If we end up here, it means we think it's an unrecoverable error which + // we want to signal back to the main controller. If enough consumer + // threads fail, this will lead to the whole program to terminate + // which in typical deployments will mean that the container will + // hopefully be restarted + logger.log(Level.SEVERE, "unexpected error in consumer loop", t); + this.running = false; + this.consumerFailedCallback.run(); + } + } + logger.info("Consumer thread terminated"); + } + +} diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml index 1caa356399f..5e3906e6b23 100644 --- a/fhir-server/pom.xml +++ b/fhir-server/pom.xml @@ -117,6 +117,10 @@ org.apache.httpcomponents httpclient + + com.google.code.gson + gson + ${project.groupId} fhir-examples diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java b/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java index c05f8a4bbd5..6fd4f41db26 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java @@ -57,6 +57,7 @@ public class FHIRRestServletFilter extends HttpFilter { private static String tenantIdHeaderName = null; private static String datastoreIdHeaderName = null; private static String originalRequestUriHeaderName = null; + private static String shardKeyHeaderName = null; private static final String preferHeaderName = "Prefer"; private static final String preferHandlingHeaderSectionName = "handling"; private static final String preferReturnHeaderSectionName = "return"; @@ -72,6 +73,7 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F long initialTime = System.nanoTime(); + String shardKey = null; String tenantId = defaultTenantId; String dsId = FHIRConfiguration.DEFAULT_DATASTORE_ID; String requestUrl = getRequestURL(request); @@ -93,6 +95,11 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F dsId = t; } + t = request.getHeader(shardKeyHeaderName); + if (t != null) { + shardKey = t; + } + t = getOriginalRequestURI(request); if (t != null) { originalRequestUri = t; @@ -115,6 +122,9 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F requestDescription.append(originalRequestUri); } requestDescription.append("]"); + if (shardKey != null) { + requestDescription.append(" shardKey:[").append(shardKey).append("]"); + } String encodedRequestDescription = Encode.forHtml(requestDescription.toString()); log.info("Received request: " + encodedRequestDescription); @@ -129,6 +139,10 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F // Don't try using FHIRConfigHelper before setting the context! FHIRRequestContext.set(context); + if (shardKey != null) { + context.setRequestShardKey(shardKey); + } + context.setOriginalRequestUri(originalRequestUri); // Set the request headers. @@ -501,6 +515,10 @@ public void init(FilterConfig filterConfig) throws ServletException { defaultTenantId = config.getStringProperty(FHIRConfiguration.PROPERTY_DEFAULT_TENANT_ID, FHIRConfiguration.DEFAULT_TENANT_ID); log.info("Configured default tenant-id value is: " + defaultTenantId); + + shardKeyHeaderName = config.getStringProperty(FHIRConfiguration.PROPERTY_SHARD_KEY_HEADER_NAME, + FHIRConfiguration.DEFAULT_SHARD_KEY_HEADER_NAME); + log.info("Configured shard-key header name is: '" + shardKeyHeaderName + "'"); } catch (Exception e) { throw new ServletException("Servlet filter initialization error.", e); } diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java new file mode 100644 index 00000000000..d6da57d2e6f --- /dev/null +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java @@ -0,0 +1,160 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.server.index.kafka; + +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +import com.google.gson.Gson; +import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; +import com.ibm.fhir.persistence.index.IndexProviderResponse; +import com.ibm.fhir.persistence.index.RemoteIndexData; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.server.index.kafka.KafkaPropertyAdapter.Mode; + + +/** + * Forwards parameter blocks to a partitioned Kafka topic. This allows us to + * skip the expensive parameter insert operations during ingestion and offload + * them to a separate process where we can process the operations more efficiently + * by using larger batches and different concurrency control mechanisms + */ +public class FHIRRemoteIndexKafkaService extends FHIRRemoteIndexService { + private static final Logger logger = Logger.getLogger(FHIRRemoteIndexKafkaService.class.getName()); + + private String topicName = null; + private Producer producer; + private KafkaPropertyAdapter.Mode mode; + + /** + * Default constructor + */ + public FHIRRemoteIndexKafkaService() { + } + + /** + * Initialize the provider + * @param properties + */ + public void init(KafkaPropertyAdapter properties) { + this.mode = properties.getMode(); + this.topicName = properties.getTopicName(); + Properties kafkaProps = new Properties(); + properties.putPropertiesTo(kafkaProps); + + if (logger.isLoggable(Level.FINER)) { + logger.finer("Kafka async index publisher is configured with the following properties:\n" + kafkaProps.toString()); + logger.finer("Topic name: " + this.topicName); + } + + String bootstrapServers = kafkaProps.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); + if (bootstrapServers == null) { + throw new IllegalStateException("The " + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " property was missing from the index service Kafka connection properties."); + } + + if (this.mode != Mode.LOGONLY) { + producer = new KafkaProducer<>(kafkaProps); + } + } + + /** + * Performs any necessary "shutdown" logic to disconnect from the topic. + */ + public void shutdown() { + logger.entering(this.getClass().getName(), "shutdown"); + + try { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Shutting down Kafka index service for topic: '" + topicName + "'."); + } + if (producer != null) { + producer.close(); + } + } finally { + logger.exiting(this.getClass().getName(), "shutdown"); + } + } + + /** + * Render the data value to a JSON string which is the wire format we + * use for remote indexing messages + * @param message + * @return + */ + public String marshallToString(RemoteIndexMessage message) { + final Gson gson = new Gson(); + return gson.toJson(message); + } + + @Override + public IndexProviderResponse submit(final RemoteIndexData data) { + // We rely on the default Kafka partitioner, which in our case will + // select a partition based on a hash of the key, which should be + // something like "Patient/a-patient-logical-id" + final String tenantId = FHIRRequestContext.get().getTenantId(); + RemoteIndexMessage msg = new RemoteIndexMessage(); + msg.setTenantId(tenantId); + msg.setData(data.getSearchParameters()); + final String message = marshallToString(msg); + + if (this.mode == Mode.ACTIVE) { + ProducerRecord record = new ProducerRecord(topicName, data.getPartitionKey(), message); + Future rmd = producer.send(record); + + // convert the future we get from the producer into a CompletableFuture + // and then map this to the response type we use (information hiding... + // the caller shouldn't be exposed to the fact that we're using Kafka) + CompletableFuture cf = backToThe(rmd) + .thenApply(v -> { + return null; + }); + return new IndexProviderResponse(data, cf); + } else { + // just log the message to help debug + logger.info("Remote index request: " + message); + return new IndexProviderResponse(data, CompletableFuture.completedFuture(null)); + } + } + + /** + * Convert a Future into a CompletableFuture + * @param + * @param f + * @return + */ + public static CompletableFuture backToThe(Future f) { + // This composition means we don't call f.get() until the get + // is called on the CompletableFuture result + return CompletableFuture.completedFuture(null).thenCompose(noValue -> { + try { + logger.finest("Waiting for index service Kafka send to complete"); + return CompletableFuture.completedFuture(f.get()); + } catch (InterruptedException e) { + // the only time we should be interrupted is during shutdown, so no + // need to log here because it should be expected that we'll fail + return CompletableFuture.failedFuture(e); + } catch (ExecutionException e) { + // Log the issue right away so that it can be time-correlated with other issues + logger.log(Level.SEVERE, "Failed async submission to Kafka", e); + return CompletableFuture.failedFuture(e.getCause()); + } finally { + logger.finest("completed"); + } + }); + } +} \ No newline at end of file diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java new file mode 100644 index 00000000000..97b3900a97e --- /dev/null +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java @@ -0,0 +1,64 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.server.index.kafka; + +import java.util.Properties; + +import org.apache.kafka.clients.producer.ProducerConfig; + +/** + * Wrapper around a {@link Properties} making them easier to consume + */ +public class KafkaPropertyAdapter { + private final Properties properties; + private final String topicName; + private final Mode mode; + + public static enum Mode { + ACTIVE, + LOGONLY + } + + /** + * Public constructor + * @param topicName + * @param properties + */ + public KafkaPropertyAdapter(String topicName, Properties properties, Mode mode) { + this.topicName = topicName; + this.properties = properties; + this.mode = mode; + } + + /** + * Fill the given kafkaProperties object with the configuration properties + * held by this adapter + * @param kafkaProps + */ + public void putPropertiesTo(Properties kafkaProps) { + kafkaProps.put(ProducerConfig.CLIENT_ID_CONFIG, "fhir-server"); + kafkaProps.putAll(this.properties); + // Make sure we always use these serializers, even if the property + // has been defined in this.properties + kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + } + + /** + * @return the topicName + */ + public String getTopicName() { + return topicName; + } + + /** + * @return the mode + */ + public Mode getMode() { + return mode; + } +} diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java index f85ba74354e..9e456f3f8c5 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java @@ -12,6 +12,9 @@ import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_EXTENDED_CODEABLE_CONCEPT_VALIDATION; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_CONNECTIONPROPS; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_ENABLED; +import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS; +import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_MODE; +import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_TOPICNAME; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_CHANNEL; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_CLIENT; @@ -23,6 +26,7 @@ import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TLS_ENABLED; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE_PW; +import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_REMOTE_INDEX_SERVICE_TYPE; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_REGISTRY_RESOURCE_PROVIDER_ENABLED; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_RESOLVE_FUNCTION_ENABLED; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_WEBSOCKET_ENABLED; @@ -54,9 +58,13 @@ import com.ibm.fhir.model.util.FHIRUtil; import com.ibm.fhir.model.util.ModelSupport; import com.ibm.fhir.path.function.registry.FHIRPathFunctionRegistry; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.helper.FHIRPersistenceHelper; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.registry.FHIRRegistry; import com.ibm.fhir.search.util.SearchHelper; +import com.ibm.fhir.server.index.kafka.FHIRRemoteIndexKafkaService; +import com.ibm.fhir.server.index.kafka.KafkaPropertyAdapter; import com.ibm.fhir.server.notification.kafka.FHIRNotificationKafkaPublisher; import com.ibm.fhir.server.notification.websocket.FHIRNotificationServiceEndpointConfig; import com.ibm.fhir.server.notifications.nats.FHIRNotificationNATSPublisher; @@ -81,12 +89,14 @@ public class FHIRServletContextListener implements ServletContextListener { private static final String ATTRNAME_WEBSOCKET_SERVERCONTAINER = "javax.websocket.server.ServerContainer"; private static final String DEFAULT_KAFKA_TOPICNAME = "fhirNotifications"; + private static final String DEFAULT_KAFKA_INDEX_SERVICE_TOPICNAME = "fhirIndex"; private static final String DEFAULT_NATS_CHANNEL = "fhirNotifications"; private static final String DEFAULT_NATS_CLUSTER = "nats-streaming"; private static final String DEFAULT_NATS_CLIENT = "fhir-server"; public static final String FHIR_SERVER_INIT_COMPLETE = "com.ibm.fhir.webappInitComplete"; private static FHIRNotificationKafkaPublisher kafkaPublisher = null; private static FHIRNotificationNATSPublisher natsPublisher = null; + private static FHIRRemoteIndexService remoteIndexService = null; private List graphTermServiceProviders = new ArrayList<>(); private List remoteTermServiceProviders = new ArrayList<>(); @@ -215,6 +225,39 @@ public void contextInitialized(ServletContextEvent event) { log.info("Bypassing NATS notification init."); } + // If the Kafka async indexing service is enabled, set it up so that it's ready to go + // before we starting processing any requests. + String remoteIndexServiceType = fhirConfig.getStringProperty(PROPERTY_REMOTE_INDEX_SERVICE_TYPE, null); + if (remoteIndexServiceType != null) { + if ("kafka".equals(remoteIndexServiceType)) { + String topicName = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME, DEFAULT_KAFKA_INDEX_SERVICE_TOPICNAME); + String mode = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_MODE, "active"); + + // Gather up the Kafka connection properties for the async index service + Properties kafkaProps = new Properties(); + PropertyGroup pg = fhirConfig.getPropertyGroup(PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS); + if (pg != null) { + List connectionProps = pg.getProperties(); + if (connectionProps != null) { + for (PropertyEntry entry : connectionProps) { + kafkaProps.setProperty(entry.getName(), entry.getValue().toString()); + } + } + } + + log.info("Initializing Kafka async indexing service."); + FHIRRemoteIndexKafkaService s = new FHIRRemoteIndexKafkaService(); + s.init(new KafkaPropertyAdapter(topicName, kafkaProps, KafkaPropertyAdapter.Mode.valueOf(mode))); + // Now the service is ready, we can publish it + remoteIndexService = s; + FHIRRemoteIndexService.setServiceInstance(remoteIndexService); + } else { + throw new FHIRPersistenceException("Invalid value for remote index service property '" + PROPERTY_REMOTE_INDEX_SERVICE_TYPE + "'"); + } + } else { + log.info("Bypassing Kafka async indexing service configuration."); + } + Boolean checkReferenceTypes = fhirConfig.getBooleanProperty(PROPERTY_CHECK_REFERENCE_TYPES, Boolean.TRUE); FHIRModelConfig.setCheckReferenceTypes(checkReferenceTypes); diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java b/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java index e3fde506560..c4bb8fe6390 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java @@ -120,7 +120,7 @@ private List computeRegistryResources(Class getRegistryResources(Class doRead(String type, String id, getInterceptorMgr().fireBeforeReadEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, includeDeleted, searchContext); + FHIRPersistenceContextFactory.createPersistenceContext(event, includeDeleted, searchContext, encodeRequestShardKey(requestContext)); result = persistence.read(persistenceContext, resourceType, id); if (!result.isSuccess() && throwExcOnNull) { throw new FHIRPersistenceResourceNotFoundException("Resource '" + type + "/" + id + "' not found."); @@ -1179,6 +1180,22 @@ private SingleResourceResult doRead(String type, String id, log.exiting(this.getClass().getName(), "doRead"); } } + /** + * Encode the shard key value if it has been set in the request context + * @param context + * @return + */ + private Short encodeRequestShardKey(FHIRRequestContext context) { + Short result = null; + final String requestShardKey = context.getRequestShardKey(); + if (requestShardKey != null) { + result = Short.valueOf((short)requestShardKey.hashCode()); + if (log.isLoggable(Level.FINEST)) { + log.finest("shardKey:[" + requestShardKey + "] hash:[" + result + "]"); + } + } + return result; + } @Override public Resource doVRead(String type, String id, String versionId, MultivaluedMap queryParameters) @@ -1215,7 +1232,7 @@ public Resource doVRead(String type, String id, String versionId, MultivaluedMap getInterceptorMgr().fireBeforeVreadEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext); + FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, encodeRequestShardKey(requestContext)); SingleResourceResult srr = persistence.vread(persistenceContext, resourceType, id, versionId); if (!srr.isSuccess() || srr.getResource() == null && !srr.isDeleted()) { throw new FHIRPersistenceResourceNotFoundException("Resource '" @@ -1360,7 +1377,7 @@ public Bundle doSearch(String type, String compartment, String compartmentId, getInterceptorMgr().fireBeforeSearchEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext); + FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, encodeRequestShardKey(requestContext)); MultiResourceResult searchResult = persistence.search(persistenceContext, resourceType); @@ -3012,6 +3029,8 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r FHIRPersistenceEvent event = new FHIRPersistenceEvent(null, buildPersistenceEventProperties(resourceType == null ? "Resource" : resourceType, null, null, null, historyContext)); getInterceptorMgr().fireBeforeHistoryEvent(event); + // Build a context + FHIRPersistenceContext context = FHIRPersistenceContextImpl.builder(event).withShardKey(encodeRequestShardKey(requestContext)).build(); // Start a new txn in the persistence layer if one is not already active. Integer count = historyContext.getCount(); @@ -3028,7 +3047,7 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r if (resourceType != null) { // Use the resource type on the path, ignoring any _type parameter - records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), Collections.singletonList(resourceType), historyContext.isExcludeTransactionTimeoutWindow(), + records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), Collections.singletonList(resourceType), historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder()); } else if (historyContext.getResourceTypes().size() > 0) { // New API allows us to filter using multiple resource type names, but first we @@ -3036,12 +3055,12 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r for (String rt: historyContext.getResourceTypes()) { validateInteraction(Interaction.HISTORY, rt); } - records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), historyContext.getResourceTypes(), + records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), historyContext.getResourceTypes(), historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder()); } else { // no resource type filter final List NULL_RESOURCE_TYPE_NAMES = null; - records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), NULL_RESOURCE_TYPE_NAMES, historyContext.isExcludeTransactionTimeoutWindow(), + records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), NULL_RESOURCE_TYPE_NAMES, historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder()); } @@ -3273,6 +3292,7 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r @Override public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseDTO eraseDto) throws FHIROperationException { // @implNote doReindex has a nice pattern to handle some retries in case of deadlock exceptions + FHIRPersistenceContext context = null; final int TX_ATTEMPTS = 5; int attempt = 1; ResourceEraseRecord eraseRecord = new ResourceEraseRecord(); @@ -3281,7 +3301,7 @@ public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseD try { txn = new FHIRTransactionHelper(getTransaction()); txn.begin(); - eraseRecord = persistence.erase(eraseDto); + eraseRecord = persistence.erase(context, eraseDto); attempt = TX_ATTEMPTS; // end the retry loop } catch (FHIRPersistenceDataAccessException x) { if (x.isTransactionRetryable() && attempt < TX_ATTEMPTS) { @@ -3305,11 +3325,13 @@ public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseD public List doRetrieveIndex(FHIROperationContext operationContext, String resourceTypeName, int count, Instant notModifiedAfter, Long afterIndexId) throws Exception { List indexIds = null; + FHIRPersistenceContext context = null; + FHIRTransactionHelper txn = null; try { txn = new FHIRTransactionHelper(getTransaction()); txn.begin(); - indexIds = persistence.retrieveIndex(count, notModifiedAfter, afterIndexId, resourceTypeName); + indexIds = persistence.retrieveIndex(context, count, notModifiedAfter, afterIndexId, resourceTypeName); } finally { if (txn != null) { txn.end(); diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java index 584f5023062..44d677d318d 100644 --- a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java +++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java @@ -166,7 +166,7 @@ public ResourcePayload fetchResourcePayloads(Class resourceT } @Override - public List changes(int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, + public List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, Long changeIdMarker, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException { // NOP @@ -174,7 +174,7 @@ public List changes(int resourceCount, java.time.Instan } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { // NOP return null; } diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java index a6abf24e76d..8b5818a677d 100644 --- a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java +++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java @@ -433,6 +433,7 @@ public ResourcePayload fetchResourcePayloads( @Override public List changes( + FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, @@ -474,7 +475,7 @@ private SingleResourceResult createOrUpdate(T resource) } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { throw new UnsupportedOperationException(); } diff --git a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java index c5b2d5643d6..f7c2e8e35c5 100644 --- a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java +++ b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java @@ -182,13 +182,13 @@ public ResourcePayload fetchResourcePayloads(Class resourceT } @Override - public List changes(int resourceCount, Instant sinceLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames, + public List changes(FHIRPersistenceContext context, int resourceCount, Instant sinceLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException { return null; } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { return null; } From 0752bbb73d868f8c711f1a75d8a792c571aa14b7 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Tue, 10 May 2022 16:44:18 +0100 Subject: [PATCH 06/40] issue #3437 remote index kafka consumer Signed-off-by: Robin Arnold --- .../index/SearchParameterValue.java | 25 + fhir-remote-index/pom.xml | 4 + .../index/api/BatchParameterProcessor.java | 86 +++ .../remote/index/api/BatchParameterValue.java | 40 ++ .../remote/index/api/IMessageHandler.java | 9 +- .../fhir/remote/index/api/IdentityCache.java | 44 ++ .../com/ibm/fhir/remote/index/app/Main.java | 94 ++- .../index/batch/BatchDateParameter.java | 39 + .../index/batch/BatchLocationParameter.java | 39 + .../index/batch/BatchNumberParameter.java | 39 + .../index/batch/BatchQuantityParameter.java | 43 ++ .../index/batch/BatchStringParameter.java | 39 + .../index/batch/BatchTokenParameter.java | 43 ++ .../remote/index/cache/IdentityCacheImpl.java | 64 ++ .../index/database/BaseMessageHandler.java | 47 +- .../remote/index/database/CacheLoader.java | 42 ++ .../index/database/CodeSystemValue.java | 48 ++ .../index/database/CommonTokenValue.java | 66 ++ .../index/database/CommonTokenValueKey.java | 47 ++ .../DistributedPostgresMessageHandler.java | 671 +++++++++++++++--- .../DistributedPostgresParameterBatch.java | 247 ++++++- ...stributedPostgresSystemParameterBatch.java | 139 ++++ .../database/JDBCBatchParameterProcessor.java | 177 +++++ .../index/database/ParameterNameValue.java | 47 ++ .../index/database/ResourceTokenValue.java | 76 ++ .../index/kafka/RemoteIndexConsumer.java | 263 +++---- 26 files changed, 2225 insertions(+), 253 deletions(-) create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java index 0e7feea6e55..ccf7cbc6dde 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java @@ -19,6 +19,9 @@ public class SearchParameterValue { // The composite id used to tie together values belonging to the same composite parameter. Null for ordinary params. private Integer compositeId; + // True if this parameter should also be stored at the whole-system level + private Boolean wholeSystem; + /** * @return the name */ @@ -46,4 +49,26 @@ public Integer getCompositeId() { public void setCompositeId(Integer compositeId) { this.compositeId = compositeId; } + + /** + * @return the wholeSystem + */ + public Boolean getWholeSystem() { + return wholeSystem; + } + + /** + * Returns true iff the wholeSystem property is not null and true + * @return + */ + public boolean isSystemParam() { + return this.wholeSystem != null && this.wholeSystem.booleanValue(); + } + + /** + * @param wholeSystem the wholeSystem to set + */ + public void setWholeSystem(Boolean wholeSystem) { + this.wholeSystem = wholeSystem; + } } diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml index 36db98f99db..8485903f03c 100644 --- a/fhir-remote-index/pom.xml +++ b/fhir-remote-index/pom.xml @@ -62,6 +62,10 @@ org.apache.kafka kafka-clients + + com.github.ben-manes.caffeine + caffeine + org.testng testng diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java new file mode 100644 index 00000000000..54d6300c8b7 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java @@ -0,0 +1,86 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.api; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.database.CodeSystemValue; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * Processes batched parameters + */ +public interface BatchParameterProcessor { + /** + * Compute the shard key value use to distribute resources among nodes + * of the database + * @param resourceType + * @param logicalId + * @return + */ + short encodeShardKey(String resourceType, String logicalId); + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException; + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) throws FHIRPersistenceException; + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue codeSystemValue) throws FHIRPersistenceException; + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) throws FHIRPersistenceException; + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) throws FHIRPersistenceException; + + /** + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java new file mode 100644 index 00000000000..39ef9a5bed8 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java @@ -0,0 +1,40 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.api; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A parameter value batched for later processing + */ +public abstract class BatchParameterValue { + protected final ParameterNameValue parameterNameValue; + protected final String resourceType; + protected final String logicalId; + protected final long logicalResourceId; + + /** + * Protected constructor + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + */ + protected BatchParameterValue(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue) { + this.resourceType = resourceType; + this.logicalId = logicalId; + this.logicalResourceId = logicalResourceId; + this.parameterNameValue = parameterNameValue; + } + + /** + * Apply this parameter value to the target processor + * @param processor + */ + public abstract void apply(BatchParameterProcessor processor) throws FHIRPersistenceException; +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java index 7b0b8a520b5..b503b5bcb75 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java @@ -8,11 +8,18 @@ import java.util.List; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; + /** * Our interface for handling messages received by the consumer. Used * to decouple the Kafka consumer from the database persistence logic */ public interface IMessageHandler { - void process(List messages); + void process(List messages) throws FHIRPersistenceException; + + /** + * Close any resources held by the handler + */ + void close(); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java new file mode 100644 index 00000000000..ace72468da9 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.api; + + +/** + * Interface to hides the implementation of various caches we use during + * ingestion persistence + */ +public interface IdentityCache { + /** + * Get the parameter_name_id value for the given parameterName + * @param parameterName + * @return the parameter_name_id or null if the value is not found in the cache + */ + Integer getParameterNameId(String parameterName); + + /** + * Get the code_system_id value for the given codeSystem value + * @param codeSystem + * @return the code_system_id or null if the value is not found in the cache + */ + Integer getCodeSystemId(String codeSystem); + + /** + * Get the common_token_value_id for the given codeSystem and tokenValue + * @param shardKey + * @param codeSystem + * @param tokenValue + * @return the common_token_value_id or null if the value is not found in the cache + */ + Long getCommonTokenValueId(short shardKey, String codeSystem, String tokenValue); + + /** + * Add the given parameterName to parameterNameId mapping to the cache + * @param parameterName + * @param parameterNameId + */ + void addParameterName(String parameterName, int parameterNameId); +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 75c960ddf62..f8c5a9d66f0 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -9,10 +9,13 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.sql.Connection; +import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -24,8 +27,17 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.PartitionInfo; +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.database.utils.thread.ThreadHandler; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.remote.index.api.IMessageHandler; +import com.ibm.fhir.remote.index.cache.IdentityCacheImpl; +import com.ibm.fhir.remote.index.database.CacheLoader; +import com.ibm.fhir.remote.index.database.DistributedPostgresMessageHandler; import com.ibm.fhir.remote.index.kafka.RemoteIndexConsumer; /** @@ -45,7 +57,7 @@ public class Main { private int consumerCount = 1; - private Duration pollDuration; + private Duration pollDuration = Duration.ofSeconds(10); private long maxBatchCollectTimeMs = 5000; // the list of consumers @@ -54,11 +66,16 @@ public class Main { // track the number of consumers that are still running private AtomicInteger stillRunningCounter; - private volatile boolean running; + private volatile boolean running = true; // Exit if we drop below this number of running consumers private int minRunningConsumerThreshold = 1; + private IdentityCacheImpl identityCache; + // Database Configuration + private IDatabaseTranslator translator; + private IConnectionProvider connectionProvider; + /** * Parse the given command line arguments * @param args @@ -93,14 +110,14 @@ public void parseArgs(String[] args) { if (a < args.length && !args[a].startsWith("--")) { consumerGroup = args[a++]; } else { - throw new IllegalArgumentException("Missing value for --topic-name"); + throw new IllegalArgumentException("Missing value for --consumer-group"); } break; case "--consumer-count": if (a < args.length && !args[a].startsWith("--")) { consumerCount = Integer.parseInt(args[a++]); } else { - throw new IllegalArgumentException("Missing value for --topic-name"); + throw new IllegalArgumentException("Missing value for --consumer-count"); } break; default: @@ -133,13 +150,28 @@ protected void loadDatabaseProperties(String filename) { } } + /** + * Get the configured schema name for where we need to use it explicitly + * @return + * @throws FHIRPersistenceException + */ + private String getSchemaName() throws FHIRPersistenceException { + String result = databaseProperties.getProperty("currentSchema"); + if (result == null) { + throw new FHIRPersistenceException("currentSchema value missing in database properties"); + } + return result; + } + /** * Keep consuming from Kafka forever...or until we see too many * consumers fail */ - public void run() { + public void run() throws FHIRPersistenceException { dumpProperties("kafka", kafkaProperties); dumpProperties("database", databaseProperties); + configureForPostgres(); + initIdentityCache(); // One thread per consumer ExecutorService pool = Executors.newCachedThreadPool(); @@ -181,11 +213,27 @@ public void run() { logger.info("All consumers stopped"); } + /** + * Set up the identity cache and preload it with all the parameter_names + * currently in the database + * @throws FHIRPersistenceException + */ + private void initIdentityCache() throws FHIRPersistenceException { + logger.info("Initializing identity cache"); + identityCache = new IdentityCacheImpl(1000, Duration.ofSeconds(3600), 10000, Duration.ofSeconds(3600)); + CacheLoader loader = new CacheLoader(identityCache); + try (Connection connection = connectionProvider.getConnection()) { + loader.apply(connection); + connection.commit(); + } catch (SQLException x) { + throw new FHIRPersistenceException("cache init failed", x); + } + } /** * Create a new consumer * @return */ - public KafkaConsumer buildConsumer() { + private KafkaConsumer buildConsumer() { Properties kp = new Properties(); kp.putAll(kafkaProperties); @@ -200,6 +248,35 @@ public KafkaConsumer buildConsumer() { return consumer; } + private void configureForPostgres() { + this.translator = new PostgresTranslator(); + try { + Class.forName(translator.getDriverClassName()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException(e); + } + + PostgresPropertyAdapter propertyAdapter = new PostgresPropertyAdapter(databaseProperties); + connectionProvider = new JdbcConnectionProvider(translator, propertyAdapter); + } + + /** + * Instantiate a new message handler for use by a consumer thread. Each handler gets + * its own database connection. + * @return + * @throws FHIRPersistenceException + */ + private IMessageHandler buildHandler() throws FHIRPersistenceException { + Objects.requireNonNull(identityCache, "must set up identityCache first"); + try { + // Each handler gets a dedicated database connection so we don't have + // to deal with contention when grabbing connections from a pool + return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache); + } catch (SQLException x) { + throw new FHIRPersistenceException("get connection failed", x); + } + } + /** * Get the partitions for the named topic to check if the topic actually exists */ @@ -251,11 +328,6 @@ protected void dumpProperties(String which, Properties p) { } } - public IMessageHandler buildHandler() { - // - return null; - } - private void failedConsumerCallback() { if (this.stillRunningCounter.decrementAndGet() < minRunningConsumerThreshold) { // Signal termination of the entire program diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java new file mode 100644 index 00000000000..9d08826eaba --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java @@ -0,0 +1,39 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A date parameter we are collecting to batch + */ +public class BatchDateParameter extends BatchParameterValue { + private final DateParameter parameter; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + public BatchDateParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java new file mode 100644 index 00000000000..524de945f18 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java @@ -0,0 +1,39 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A location parameter we are collecting to batch + */ +public class BatchLocationParameter extends BatchParameterValue { + private final LocationParameter parameter; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + public BatchLocationParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java new file mode 100644 index 00000000000..d1aa15f3323 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java @@ -0,0 +1,39 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A number parameter we are collecting to batch + */ +public class BatchNumberParameter extends BatchParameterValue { + private final NumberParameter parameter; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + public BatchNumberParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java new file mode 100644 index 00000000000..cbbf486e6b0 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java @@ -0,0 +1,43 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.CodeSystemValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A quantity parameter we are collecting to batch + */ +public class BatchQuantityParameter extends BatchParameterValue { + private final QuantityParameter parameter; + private final CodeSystemValue codeSystemValue; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param csv + */ + public BatchQuantityParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue csv) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.codeSystemValue = csv; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, codeSystemValue); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java new file mode 100644 index 00000000000..2b37a8dbb9a --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java @@ -0,0 +1,39 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A string parameter we are collecting to batch + */ +public class BatchStringParameter extends BatchParameterValue { + private final StringParameter parameter; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + public BatchStringParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java new file mode 100644 index 00000000000..50f0c14ae71 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java @@ -0,0 +1,43 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A token parameter we are collecting to batch + */ +public class BatchTokenParameter extends BatchParameterValue { + private final TokenParameter parameter; + private final CommonTokenValue commonTokenValue; + + /** + * Canonical constructor + * + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param commonTokenValue + */ + public BatchTokenParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) { + super(resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.commonTokenValue = commonTokenValue; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java new file mode 100644 index 00000000000..624e2e31ca0 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java @@ -0,0 +1,64 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.cache; + +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.database.CommonTokenValueKey; + +/** + * Implementation of a cache we use to reduce the number of databases accesses + * required to find the id for a given object key + */ +public class IdentityCacheImpl implements IdentityCache { + private final ConcurrentHashMap parameterNames = new ConcurrentHashMap<>(); + private final Cache codeSystemCache; + private final Cache commonTokenValueCache; + private static final Integer NULL_INT = null; + private static final Long NULL_LONG = null; + + /** + * Public constructor + */ + public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDuration, + long maxCommonTokenCacheSize, Duration commonTokenCacheDuration) { + codeSystemCache = Caffeine.newBuilder() + .maximumSize(maxCodeSystemCacheSize) + .expireAfterWrite(codeSystemCacheDuration) + .build(); + commonTokenValueCache = Caffeine.newBuilder() + .maximumSize(maxCommonTokenCacheSize) + .expireAfterWrite(commonTokenCacheDuration) + .build(); + } + + @Override + public Integer getParameterNameId(String parameterName) { + // This should only miss if the parameter name value doesn't actually + // exist. Because the set is relatively small, we store everything. + return parameterNames.get(parameterName); + } + + @Override + public Integer getCodeSystemId(String codeSystem) { + return codeSystemCache.get(codeSystem, k -> NULL_INT); + } + + @Override + public Long getCommonTokenValueId(short shardKey, String codeSystem, String tokenValue) { + return commonTokenValueCache.get(new CommonTokenValueKey(shardKey, codeSystem, tokenValue), k -> NULL_LONG); + } + + @Override + public void addParameterName(String parameterName, int parameterNameId) { + parameterNames.put(parameterName, parameterNameId); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index 0f46b1bf603..89a813b0eaa 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -9,6 +9,7 @@ import java.util.List; import com.google.gson.Gson; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; @@ -21,24 +22,36 @@ /** - * + * Base for the Kafka message handler to load message data into + * a database via JDBC. */ public abstract class BaseMessageHandler implements IMessageHandler { @Override - public void process(List messages) { - for (String payload: messages) { - RemoteIndexMessage message = unmarshall(payload); - process(message); + public void process(List messages) throws FHIRPersistenceException { + try { + for (String payload: messages) { + RemoteIndexMessage message = unmarshall(payload); + process(message); + } + pushBatch(); + } catch (Throwable t) { + setRollbackOnly(); + throw t; + } finally { + endTransaction(); } - pushBatch(); - commit(); } + /** + * Mark the transaction for rollback + */ + protected abstract void setRollbackOnly(); + /** * Push any data we've accumulated from processing messages. */ - protected abstract void pushBatch(); + protected abstract void pushBatch() throws FHIRPersistenceException; /** * Unmarshall the json payload string into a RemoteIndexMessage @@ -53,7 +66,7 @@ private RemoteIndexMessage unmarshall(String jsonPayload) { * Process the data * @param message */ - private void process(RemoteIndexMessage message) { + private void process(RemoteIndexMessage message) throws FHIRPersistenceException { SearchParametersTransport params = message.getData(); if (params.getStringValues() != null) { for (StringParameter p: params.getStringValues()) { @@ -99,7 +112,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException; /** * @param tenantId @@ -108,7 +121,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException; /** * @param tenantId @@ -117,7 +130,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException; /** * @param tenantId @@ -126,7 +139,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException; /** * @param tenantId @@ -135,7 +148,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException; /** * @param tenantId @@ -144,9 +157,7 @@ private void process(RemoteIndexMessage message) { * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p); + protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException; - private void commit() { - - } + protected abstract void endTransaction() throws FHIRPersistenceException; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java new file mode 100644 index 00000000000..aa4e67fa46a --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java @@ -0,0 +1,42 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.remote.index.api.IdentityCache; + +/** + * Preload the cache + */ +public class CacheLoader { + private final IdentityCache cache; + + /** + * Public constructor + * @param cache + */ + public CacheLoader(IdentityCache cache) { + this.cache = cache; + } + + public void apply(Connection connection) throws FHIRPersistenceException { + final String SQL = "SELECT parameter_name, parameter_name_id FROM parameter_names"; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + cache.addParameterName(rs.getString(1), rs.getInt(2)); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("fetch parameter names failed", x); + } + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java new file mode 100644 index 00000000000..472fa977d10 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java @@ -0,0 +1,48 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + + +/** + * Holder for the code_system_id obtained from the database. This is + * modelled as a mutable record so that we can reference this record + * many times, and resolve it once (either from cache lookup, database + * lookup, or a database create). + */ +public class CodeSystemValue { + private final String codeSystem; + private Integer codeSystemId; + + /** + * Public constructor + * @param codeSystem + */ + public CodeSystemValue(String codeSystem) { + this.codeSystem = codeSystem; + } + + /** + * @return the codeSystemId or null if it is current unknown + */ + public Integer getCodeSystemId() { + return codeSystemId; + } + + /** + * @param codeSystemId the codeSystemId to set + */ + public void setCodeSystemId(int codeSystemId) { + this.codeSystemId = codeSystemId; + } + + /** + * @return the codeSystem + */ + public String getCodeSystem() { + return codeSystem; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java new file mode 100644 index 00000000000..4a271217aae --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java @@ -0,0 +1,66 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +/** + * Represents a common_token_value record which may or may not yet exist + * in the database. If it exists in the database, we may not yet have + * retrieved its common_token_value_id. + */ +public class CommonTokenValue { + private final short shardKey; + private final CodeSystemValue codeSystemValue; + private final String tokenValue; + private Long commonTokenValueId; + + /** + * Public constructor + * @param shardKey + * @param codeSystemValue + * @param tokenValue + */ + public CommonTokenValue(short shardKey, CodeSystemValue codeSystemValue, String tokenValue) { + this.shardKey = shardKey; + this.codeSystemValue = codeSystemValue; + this.tokenValue = tokenValue; + } + + /** + * @return the commonTokenValueId + */ + public Long getCommonTokenValueId() { + return commonTokenValueId; + } + + /** + * @param commonTokenValueId the commonTokenValueId to set + */ + public void setCommonTokenValueId(Long commonTokenValueId) { + this.commonTokenValueId = commonTokenValueId; + } + + /** + * @return the codeSystemValue + */ + public CodeSystemValue getCodeSystemValue() { + return codeSystemValue; + } + + /** + * @return the tokenValue + */ + public String getTokenValue() { + return tokenValue; + } + + /** + * @return the shardKey + */ + public short getShardKey() { + return shardKey; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java new file mode 100644 index 00000000000..de3b6fa6907 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java @@ -0,0 +1,47 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.util.Objects; + +/** + * A key used to identify a common_token_value record in our distributed schema + * variant + */ +public class CommonTokenValueKey { + private final short shardKey; + private final String codeSystem; + private final String tokenValue; + + /** + * Public constructor + * @param shardKey + * @param codeSystem + * @param tokenValue + */ + public CommonTokenValueKey(short shardKey, String codeSystem, String tokenValue) { + this.shardKey = shardKey; + this.codeSystem = Objects.requireNonNull(codeSystem); + this.tokenValue = Objects.requireNonNull(tokenValue); + } + + @Override + public int hashCode() { + return Objects.hash(codeSystem, tokenValue, shardKey); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof CommonTokenValueKey) { + CommonTokenValueKey that = (CommonTokenValueKey)obj; + return this.shardKey == that.shardKey + && this.codeSystem.equals(that.codeSystem) + && this.tokenValue.equals(that.tokenValue); + } + return false; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index 8112df767d2..64c4cab9591 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -6,145 +6,648 @@ package com.ibm.fhir.remote.index.database; +import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.SearchParameterValue; import com.ibm.fhir.persistence.index.StringParameter; import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.batch.BatchDateParameter; +import com.ibm.fhir.remote.index.batch.BatchLocationParameter; +import com.ibm.fhir.remote.index.batch.BatchNumberParameter; +import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; +import com.ibm.fhir.remote.index.batch.BatchStringParameter; +import com.ibm.fhir.remote.index.batch.BatchTokenParameter; /** * Loads search parameter values into the target FHIR schema on * a PostgreSQL database. */ public class DistributedPostgresMessageHandler extends BaseMessageHandler { + private static final Logger logger = Logger.getLogger(DistributedPostgresMessageHandler.class.getName()); // the connection to use for the inserts private final Connection connection; - // rarely used so no need for {@code java.sql.PreparedStatement} or batching on this one - // even on Location resources its only there once by default - private final String insertLocation; + // We're a PostgreSQL DAO, so we now which translator to use + private final IDatabaseTranslator translator = new PostgresTranslator(); - // Searchable string attributes stored at the system level - private final PreparedStatement systemStrings; - private int systemStringCount; + // The FHIR data schema + private final String schemaName; - // Searchable date attributes stored at the system level - private final PreparedStatement systemDates; - private int systemDateCount; + // the cache we use for various lookups + private final IdentityCache identityCache; + + // All parameter names we've seen (cleared if there's a rollback) + private final Map parameterNameMap = new HashMap<>(); + + // A map of code system name to the value holding its codeSystemId from the database + private final Map codeSystemValueMap = new HashMap<>(); + + // A map to support lookup of CommonTokenValue records by key + private final Map commonTokenValueMap = new HashMap<>(); + + // All parameter names in the current transaction for which we don't yet know the parameter_name_id + private final List unresolvedParameterNames = new ArrayList<>(); + + // A list of all the CodeSystemValues for which we don't yet know the code_system_id + private final List unresolvedSystemValues = new ArrayList<>(); + + // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id + private final List unresolvedTokenValues = new ArrayList<>(); + + // The processed values we've collected + private final List batchedParameterValues = new ArrayList<>(); + + // The processor used to process the batched parameter values after all the reference values are created + private final JDBCBatchParameterProcessor batchProcessor; + + private final int maxCodeSystemsPerStatement = 512; + private final int maxCommonTokenValuesPerStatement = 256; + private boolean rollbackOnly; /** * Public constructor * * @param connection */ - public DistributedPostgresMessageHandler(Connection connection) { + public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache) { this.connection = connection; - insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; - strings = connection.prepareStatement(insertString); - - insertNumber = multitenant ? - "INSERT INTO " + tablePrefix + "_number_values (mt_id, parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?,?)" - : - "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?)"; - numbers = c.prepareStatement(insertNumber); - - insertDate = multitenant ? - "INSERT INTO " + tablePrefix + "_date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?)" - : - "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; - dates = c.prepareStatement(insertDate); - - insertQuantity = multitenant ? - "INSERT INTO " + tablePrefix + "_quantity_values (mt_id, parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?,?,?,?)" - : - "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?,?,?)"; - quantities = c.prepareStatement(insertQuantity); - - insertLocation = multitenant ? "INSERT INTO " + tablePrefix + "_latlng_values (mt_id, parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?,?)" - : "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; - - if (storeWholeSystemParams) { - // System level string attributes - String insertSystemString = multitenant ? - "INSERT INTO str_values (mt_id, parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" - : - "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; - systemStrings = c.prepareStatement(insertSystemString); - - // System level date attributes - String insertSystemDate = multitenant ? - "INSERT INTO date_values (mt_id, parameter_name_id, date_start, date_end, logical_resource_id) VALUES (" + adminSchemaName + ".sv_tenant_id,?,?,?,?)" - : - "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)"; - systemDates = c.prepareStatement(insertSystemDate); - } else { - systemStrings = null; - systemDates = null; + this.schemaName = schemaName; + this.identityCache = cache; + this.batchProcessor = new JDBCBatchParameterProcessor(connection); + } + + @Override + protected void setRollbackOnly() { + this.rollbackOnly = true; + } + + @Override + public void close() { + try { + batchProcessor.close(); + } catch (Throwable t) { + logger.log(Level.SEVERE, "close batchProcessor failed" , t); } } + + @Override + protected void endTransaction() throws FHIRPersistenceException { + try { + if (!this.rollbackOnly) { + logger.fine("Committing transaction"); + connection.commit(); + // any values from parameter_names, code_systems and common_token_values + // are now committed to the database, so we can publish their record ids + // to the shared cache which makes them accessible from other threads + publishCachedValues(); + } else { + // something went wrong...try to roll back the transaction before we close + // everything + try { + connection.rollback(); + } catch (SQLException x) { + // It could very well be that we've lost touch with the database in which case + // the rollback will also fail. Not much we can do, although we don't bother + // with a stack trace here because it's just more noise for the log file. + logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); + } + } + } catch (SQLException x) { + throw new FHIRPersistenceException("commit failed", x); + } finally { + unresolvedParameterNames.clear(); + unresolvedSystemValues.clear(); + unresolvedTokenValues.clear(); + } + } + /** - * Compute the shard key value use to distribute resources among nodes - * of the database - * @param resourceType - * @param logicalId - * @return + * After the transaction has been committed, we can publish certain values to the + * shared identity caches */ - private short encodeShardKey(String resourceType, String logicalId) { - final String requestShardKey = resourceType + "/" + logicalId; - return Short.valueOf((short)requestShardKey.hashCode()); + public void publishCachedValues() { + // all the unresolvedParameterNames should be resolved at this point + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); + } + this.unresolvedParameterNames.clear(); } @Override - protected void pushBatch() { + protected void pushBatch() throws FHIRPersistenceException { // Push any data we've accumulated so far. This may occur // if we cross a volume threshold, and will always occur as // the last step before the current transaction is committed, + // Process the token values so that we can establish + // any entries we need for common_token_values + resolveParameterNames(); + resolveCodeSystems(); + resolveCommonTokenValues(); + + // Now that all the lookup values should've been resolved, we can go ahead + // and push the parameters to the JDBC batch insert statements via the + // batchProcessor + for (BatchParameterValue v: this.batchedParameterValues) { + v.apply(batchProcessor); + } + batchProcessor.pushBatch(); } + /** + * Get the parameter name value for the given parameter value + * @param p + * @return + */ + private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { + ParameterNameValue result = parameterNameMap.get(p.getName()); + if (result == null) { + result = new ParameterNameValue(p.getName()); + parameterNameMap.put(p.getName(), result); + + // let's see if the id is available in the shared identity cache + Integer parameterNameId = identityCache.getParameterNameId(p.getName()); + if (parameterNameId != null) { + result.setParameterNameId(parameterNameId); + } else { + // ids will be created later (so that we can process them in order) + unresolvedParameterNames.add(result); + } + } + return result; + } + @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub - + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchStringParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub - + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchLocationParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub - + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { + short shardKey = batchProcessor.encodeShardKey(resourceType, logicalId); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTokenParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub - + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); + this.batchedParameterValues.add(new BatchQuantityParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub - + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchNumberParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) { - final short shardKey = encodeShardKey(resourceType, logicalId); - // TODO Auto-generated method stub + protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchDateParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + /** + * Get the CodeSystemValue we've assigned for the given codeSystem value. This + * may not yet have the actual code_system_id from the database yet - any values + * we don't have will be assigned in a later phase (so we can do things neatly + * in bulk). + * @param codeSystem + * @return + */ + private CodeSystemValue lookupCodeSystemValue(String codeSystem) { + CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); + if (result == null) { + result = new CodeSystemValue(codeSystem); + this.codeSystemValueMap.put(codeSystem, result); + + // Take this opportunity to see if we have a cached value for this codeSystem + Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); + if (codeSystemId != null) { + result.setCodeSystemId(codeSystemId); + } else { + // Stash for later resolution + this.unresolvedSystemValues.add(result); + } + } + return result; + } + + /** + * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. + * The returned value may not yet have the actual common_token_value_id yet - we fetch + * these values later and create new database records as necessary. + * @param codeSystem + * @param tokenValue + * @return + */ + private CommonTokenValue lookupCommonTokenValue(short shardKey, String codeSystem, String tokenValue) { + CommonTokenValueKey key = new CommonTokenValueKey(shardKey, codeSystem, tokenValue); + CommonTokenValue result = this.commonTokenValueMap.get(key); + if (result == null) { + CodeSystemValue csv = lookupCodeSystemValue(codeSystem); + result = new CommonTokenValue(shardKey, csv, tokenValue); + this.commonTokenValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long commonTokenValueId = identityCache.getCommonTokenValueId(shardKey, codeSystem, tokenValue); + if (commonTokenValueId != null) { + result.setCommonTokenValueId(commonTokenValueId); + } else { + this.unresolvedTokenValues.add(result); + } + } + return result; + } + + /** + * Make sure we have values for all the code_systems we have collected + * in the current + * batch + * @throws FHIRPersistenceException + */ + private void resolveCodeSystems() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCodeSystemIds(unresolvedSystemValues); + + if (!missing.isEmpty()) { + addMissingCodeSystems(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCodeSystemIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all code system values"); + } + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system value + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT code_system_id, code_system FROM code_systems WHERE code_system IN ("); + for (int i=0; i 0) { + query.append(","); + } + query.append("?"); + } + query.append(")"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (CodeSystemValue csv: values) { + ps.setString(param++, csv.getCodeSystem()); + } + return ps; + } + + /** + * These code systems weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + private void addMissingCodeSystems(List missing) throws FHIRPersistenceException { + List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); + // Sort the code system values first to help avoid deadlocks + Collections.sort(values); // natural ordering for String is fine here + + final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO code_systems (code_system_id, code_system) VALUES ("); + insert.append(nextVal); // next sequence value + insert.append(",?) ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (String codeSystem: values) { + ps.setString(1, codeSystem); + ps.addBatch(); + if (++count == this.maxCodeSystemsPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); + if (csv != null) { + csv.setCodeSystemId(rs.getInt(1)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("code systems query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CodeSystemValue csv: sub) { + if (csv.getCodeSystemId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed", x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Make sure we have values for all the common_token_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonTokenValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCommonTokenValueIds(unresolvedTokenValues); + + if (!missing.isEmpty()) { + // Sort first to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getTokenValue().compareTo(b.getTokenValue()); + if (result == 0) { + result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + } + return result; + }); + addMissingCommonTokenValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCommonTokenValueIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all common token values"); + } + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT shard_key, code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatement buildCommonTokenValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + // need the code_system name - so we join back to the code_systems table as well + query.append("SELECT c.shard_key, cs.code_system, c.token_value, c.common_token_value_id "); + query.append(" FROM common_token_values c"); + query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); + query.append(" JOIN VALUES ("); + boolean first = true; + for (CommonTokenValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id + query.append(",?)"); // bind variable for the token-value + } + query.append(") AS v(code_system_id, token_value) "); + query.append(" ON c.code_system_id = v.code_system_id AND c.token_value = v.token_value"); + + // Create the prepared statement and bind the values + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (CommonTokenValue ctv: values) { + ps.setString(param++, ctv.getTokenValue()); + } + return ps; } -} + + private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildCommonTokenValueSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonTokenValueKey key = new CommonTokenValueKey(rs.getShort(1), rs.getString(2), rs.getString(3)); + CommonTokenValue ctv = this.commonTokenValueMap.get(key); + if (ctv != null) { + ctv.setCommonTokenValueId(rs.getLong(4)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common token values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonTokenValue ctv: sub) { + if (ctv.getCommonTokenValueId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common token values fetch failed", x); + throw new FHIRPersistenceException("common token values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + private void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) VALUES ("); + insert.append("?,?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (CommonTokenValue ctv: missing) { + ps.setShort(1, ctv.getShardKey()); + ps.setInt(2, ctv.getCodeSystemValue().getCodeSystemId()); + ps.setString(3, ctv.getTokenValue()); + ps.addBatch(); + if (++count == this.maxCommonTokenValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common token values"); + } + } + + /** + * Make sure all the parameter names we've seen in the batch exist + * in the database and have ids. + * @throws FHIRPersistenceException + */ + private void resolveParameterNames() throws FHIRPersistenceException { + // We expect parameter names to have a very high cache hit rate and + // so we simplify processing by simply iterating one-by-one for the + // values we still need to resolve. The most important point here is + // to do this in a sorted order to avoid deadlock issues because this + // could be happening across multiple consumer threads at the same time. + Collections.sort(this.unresolvedParameterNames, (a,b) -> { + return a.getParameterName().compareTo(b.getParameterName()); + }); + + try { + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); + if (parameterNameId == null) { + parameterNameId = createParameterName(pnv.getParameterName()); + } + pnv.setParameterNameId(parameterNameId); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("error resolving parameter names", x); + } + } + + private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { + String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ps.setString(1, parameterName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return rs.getInt(1); + } + } + + // no entry in parameter_names + return null; + } + + /** + * Create the parameter name using the stored procedure which handles any concurrency + * issue we may have + * @param parameterName + * @return + */ + private Integer createParameterName(String parameterName) throws SQLException { + final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; + Integer parameterNameId; + try (CallableStatement stmt = connection.prepareCall(CALL)) { + stmt.setString(1, parameterName); + stmt.registerOutParameter(2, Types.INTEGER); + stmt.execute(); + parameterNameId = stmt.getInt(2); + } + + return parameterNameId; + } + +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java index 549875a1c9d..2be9839a91a 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java @@ -6,8 +6,15 @@ package com.ibm.fhir.remote.index.database; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; + +import com.ibm.fhir.database.utils.common.CalendarHelper; /** * Parameter batch statements configured for a given resource type @@ -15,32 +22,254 @@ public class DistributedPostgresParameterBatch { private final Connection connection; private final String resourceType; - private final PreparedStatement strings; + + private PreparedStatement strings; private int stringCount; - private final PreparedStatement numbers; + private PreparedStatement numbers; private int numberCount; - private final PreparedStatement dates; + private PreparedStatement dates; private int dateCount; - private final PreparedStatement quantities; + private PreparedStatement quantities; private int quantityCount; + private PreparedStatement locations; + private int locationCount; + + private PreparedStatement resourceTokenRefs; + private int resourceTokenRefCount; + /** * Public constructor - * @param connection + * @param c * @param resourceType */ - public DistributedPostgresParameterBatch(Connection connection, String resourceType) { - this.connection = connection; + public DistributedPostgresParameterBatch(Connection c, String resourceType) { + this.connection = c; this.resourceType = resourceType; } /** * Push the current batch */ - public void pushBatch() { - + public void pushBatch() throws SQLException { + if (stringCount > 0) { + strings.executeBatch(); + stringCount = 0; + } + if (numberCount > 0) { + numbers.executeBatch(); + numberCount = 0; + } + if (dateCount > 0) { + dates.executeBatch(); + dateCount = 0; + } + if (quantityCount > 0) { + quantities.executeBatch(); + quantityCount = 0; + } + if (locationCount > 0) { + locations.executeBatch(); + locationCount = 0; + } + if (resourceTokenRefCount > 0) { + resourceTokenRefs.executeBatch(); + resourceTokenRefCount = 0; + } + } + + /** + * Resets the state of the DAO by closing all statements and + * setting any batch counts to 0 + */ + public void reset() { + if (strings != null) { + try { + strings.close(); + } catch (SQLException x) { + // NOP + } finally { + strings = null; + stringCount = 0; + } + } + + if (numbers != null) { + try { + numbers.close(); + } catch (SQLException x) { + // NOP + } finally { + numbers = null; + numberCount = 0; + } + } + + if (dates != null) { + try { + dates.close(); + } catch (SQLException x) { + // NOP + } finally { + dates = null; + dateCount = 0; + } + } + + if (quantities != null) { + try { + quantities.close(); + } catch (SQLException x) { + // NOP + } finally { + quantities = null; + quantityCount = 0; + } + } + + if (locations != null) { + try { + locations.close(); + } catch (SQLException x) { + // NOP + } finally { + locations = null; + locationCount = 0; + } + } + + if (resourceTokenRefs != null) { + try { + resourceTokenRefs.close(); + } catch (SQLException x) { + // NOP + } finally { + resourceTokenRefs = null; + resourceTokenRefCount = 0; + } + } + } + + /** + * Set the compositeId on the given PreparedStatement, handling a value if necessary + * @param ps + * @param index + * @param compositeId + * @throws SQLException + */ + private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException { + if (compositeId != null) { + ps.setInt(index, compositeId); + } else { + ps.setNull(index, Types.INTEGER); + } + } + + public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId, short shardKey) throws SQLException { + if (strings == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)"; + strings = connection.prepareStatement(insertString); + } + + strings.setInt(1, parameterNameId); + strings.setString(2, strValue); + strings.setString(3, strValueLower); + strings.setLong(4, logicalResourceId); + setComposite(strings, 5, compositeId); + strings.setShort(6, shardKey); + strings.addBatch(); + stringCount++; + } + + public void addNumber(long logicalResourceId, int parameterNameId, BigDecimal value, BigDecimal valueLow, BigDecimal valueHigh, Integer compositeId, short shardKey) throws SQLException { + if (numbers == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertNumber = "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?,?)"; + numbers = connection.prepareStatement(insertNumber); + } + numbers.setInt(1, parameterNameId); + numbers.setBigDecimal(2, value); + numbers.setBigDecimal(3, valueLow); + numbers.setBigDecimal(4, valueHigh); + numbers.setLong(5, logicalResourceId); + setComposite(numbers, 6, compositeId); + numbers.setShort(7, shardKey); + numbers.addBatch(); + numberCount++; + } + + public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId, short shardKey) throws SQLException { + if (dates == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertDate = "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)"; + dates = connection.prepareStatement(insertDate); + } + + final Calendar UTC = CalendarHelper.getCalendarForUTC(); + dates.setInt(1, parameterNameId); + dates.setTimestamp(2, dateStart, UTC); + dates.setTimestamp(3, dateEnd, UTC); + dates.setLong(4, logicalResourceId); + setComposite(dates, 5, compositeId); + dates.setShort(6, shardKey); + dates.addBatch(); + dateCount++; + } + + public void addQuantity(long logicalResourceId, int parameterNameId, Integer codeSystemId, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId, short shardKey) throws SQLException { + if (quantities == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertQuantity = "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?,?,?,?)"; + quantities = connection.prepareStatement(insertQuantity); + } + + quantities.setInt(1, parameterNameId); + quantities.setInt(2, codeSystemId); + quantities.setString(3, valueCode); + quantities.setBigDecimal(4, valueNumber); + quantities.setBigDecimal(5, valueNumberLow); + quantities.setBigDecimal(6, valueNumberHigh); + quantities.setLong(7, logicalResourceId); + setComposite(quantities, 8, compositeId); + quantities.setShort(9, shardKey); + quantities.addBatch(); + quantityCount++; + } + + public void addLocation(long logicalResourceId, int parameterNameId, Double lat, Double lng, Integer compositeId, short shardKey) throws SQLException { + if (locations == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertLocation = "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)"; + locations = connection.prepareStatement(insertLocation); + } + + locations.setInt(1, parameterNameId); + locations.setDouble(2, lat); + locations.setDouble(3, lng); + locations.setLong(4, logicalResourceId); + setComposite(locations, 5, compositeId); + locations.setShort(6, shardKey); + locations.addBatch(); + locationCount++; + } + + public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId, Integer refVersionId, Integer compositeId, short shardKey) throws SQLException { + if (resourceTokenRefs == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, ref_version_id, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)"; + resourceTokenRefs = connection.prepareStatement(tokenString); + } + resourceTokenRefs.setInt(1, parameterNameId); + resourceTokenRefs.setLong(2, commonTokenValueId); + setComposite(resourceTokenRefs, 3, refVersionId); + resourceTokenRefs.setLong(4, logicalResourceId); + setComposite(resourceTokenRefs, 5, compositeId); + resourceTokenRefs.setShort(6, shardKey); + resourceTokenRefs.addBatch(); + resourceTokenRefCount++; } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java new file mode 100644 index 00000000000..5a9b554a2d4 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java @@ -0,0 +1,139 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; + +import com.ibm.fhir.database.utils.common.CalendarHelper; + +/** + * Batch insert statements for system-level parameters + * @implNote targets the distributed variant of the schema + * where each table includes a shard_key column + */ +public class DistributedPostgresSystemParameterBatch { + private final Connection connection; + + private PreparedStatement systemStrings; + private int systemStringCount; + + private PreparedStatement systemDates; + private int systemDateCount; + + /** + * Public constructor + * @param c + */ + public DistributedPostgresSystemParameterBatch(Connection c) { + this.connection = c; + } + + /** + * Push the current batch + */ + public void pushBatch() throws SQLException { + if (systemStringCount > 0) { + systemStrings.executeBatch(); + systemStringCount = 0; + } + if (systemDateCount > 0) { + systemDates.executeBatch(); + systemDateCount = 0; + } + } + + /** + * Clear the current batch + */ + public void clearBatch() throws SQLException { + if (systemStringCount > 0) { + systemStrings.clearBatch(); + systemStringCount = 0; + } + if (systemDateCount > 0) { + systemDates.clearBatch(); + systemDateCount = 0; + } + } + /** + * Closes all the statements currently open + */ + public void close() { + + if (systemStrings != null) { + try { + systemStrings.close(); + } catch (SQLException x) { + // NOP + } finally { + systemStrings = null; + } + } + + if (systemDates != null) { + try { + systemDates.close(); + } catch (SQLException x) { + // NOP + } finally { + systemDates = null; + } + } + } + + /** + * Set the compositeId on the given PreparedStatement, handling a value if necessary + * @param ps + * @param index + * @param compositeId + * @throws SQLException + */ + private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException { + if (compositeId != null) { + ps.setInt(index, compositeId); + } else { + ps.setNull(index, Types.INTEGER); + } + } + + public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId, short shardKey) throws SQLException { + // System level string attributes + if (systemStrings == null) { + final String insertSystemString = "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, shard_key) VALUES (?,?,?,?,?)"; + systemStrings = connection.prepareStatement(insertSystemString); + } + systemStrings.setInt(1, parameterNameId); + systemStrings.setString(2, strValue); + systemStrings.setString(3, strValueLower); + systemStrings.setLong(4, logicalResourceId); + setComposite(systemStrings, 5, compositeId); + systemStrings.setShort(6, shardKey); + systemStrings.addBatch(); + systemStringCount++; + } + + public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId, short shardKey) throws SQLException { + if (systemDates == null) { + final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id, shard_key) VALUES (?,?,?,?,?)"; + systemDates = connection.prepareStatement(insertSystemDate); + } + final Calendar UTC = CalendarHelper.getCalendarForUTC(); + systemDates.setInt(1, parameterNameId); + systemDates.setTimestamp(2, dateStart, UTC); + systemDates.setTimestamp(3, dateEnd, UTC); + systemDates.setLong(4, logicalResourceId); + setComposite(systemDates, 5, compositeId); + systemDates.setShort(6, shardKey); + systemDates.addBatch(); + systemDateCount++; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java new file mode 100644 index 00000000000..973510610d9 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java @@ -0,0 +1,177 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; + + +/** + * Processes batched parameters by pushing the values to various + * JDBC statements + */ +public class JDBCBatchParameterProcessor implements BatchParameterProcessor { + // A cache of the resource-type specific DAOs we've created + private final Map daoMap = new HashMap<>(); + + // Encapculates the statements for inserting whole-system level search params + private final DistributedPostgresSystemParameterBatch systemDao; + + // Resource types we've touched in the current batch + private final Set resourceTypesInBatch = new HashSet<>(); + + // The database connection this consumer thread is using + private final Connection connection; + + /** + * Public constructor + * @param connection + */ + public JDBCBatchParameterProcessor(Connection connection) { + this.connection = connection; + this.systemDao = new DistributedPostgresSystemParameterBatch(connection); + } + + /** + * Close any resources we're holding to support a cleaner exit + */ + public void close() { + for (Map.Entry entry: daoMap.entrySet()) { + entry.getValue().reset(); + } + systemDao.close(); + } + + /** + * Push any statements that have been batched but not yet executed + * @throws FHIRPersistenceException + */ + public void pushBatch() throws FHIRPersistenceException { + try { + for (String resourceType: resourceTypesInBatch) { + DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + try { + dao.pushBatch(); + } catch (SQLException x) { + throw new FHIRPersistenceException("pushBatch failed for '" + resourceType + "'"); + } + } + + try { + systemDao.pushBatch(); + } catch (SQLException x) { + throw new FHIRPersistenceException("batch insert for whole-system parameters", x); + } + } finally { + // Reset the set of active resource-types ready for the next batch + resourceTypesInBatch.clear(); + } + } + + private DistributedPostgresParameterBatch getParameterBatchDao(String resourceType) { + DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + if (dao == null) { + dao = new DistributedPostgresParameterBatch(connection, resourceType); + daoMap.put(resourceType, dao); + } + return dao; + } + + @Override + public short encodeShardKey(String resourceType, String logicalId) { + final String requestShardKey = resourceType + "/" + logicalId; + return Short.valueOf((short)requestShardKey.hashCode()); + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey); + + if (parameter.isSystemParam()) { + systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); + } + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId(), shardKey); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); + } + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId(), shardKey); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting quantity params for '" + resourceType + "'"); + } + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId(), shardKey); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting location params for '" + resourceType + "'"); + } + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId(), shardKey); + if (p.isSystemParam()) { + systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId(), shardKey); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'"); + } + } + + @Override + public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + final short shardKey = encodeShardKey(resourceType, logicalId); + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId(), shardKey); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'"); + } + } + +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java new file mode 100644 index 00000000000..8abe2d46457 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java @@ -0,0 +1,47 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + + +/** + * Represents a record in parameter_names for which we don't know + * the parameter_name_id value, or which we need to create + */ +public class ParameterNameValue { + private final String parameterName; + private Integer parameterNameId; + + /** + * Public constructor + * @param parameterName + */ + public ParameterNameValue(String parameterName) { + this.parameterName = parameterName; + } + + /** + * @return the parameterName + */ + public String getParameterName() { + return parameterName; + } + + /** + * @return the parameterNameId + */ + public Integer getParameterNameId() { + return parameterNameId; + } + + /** + * @param parameterNameId the parameterNameId to set + */ + public void setParameterNameId(Integer parameterNameId) { + this.parameterNameId = parameterNameId; + } + +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java new file mode 100644 index 00000000000..3e287e51d39 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java @@ -0,0 +1,76 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import com.ibm.fhir.persistence.index.TokenParameter; + +/** + * Record representing a new row in _resource_token_refs + */ +public class ResourceTokenValue { + private final String resourceType; + private final String logicalId; + private final long logicalResourceId; + private final TokenParameter tokenParameter; + private final CommonTokenValue commonTokenValue; + + /** + * Public constructor + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param tokenParameter + * @param commonTokenValue + */ + public ResourceTokenValue(String resourceType, String logicalId, long logicalResourceId, TokenParameter tokenParameter, CommonTokenValue commonTokenValue) { + this.resourceType = resourceType; + this.logicalId = logicalId; + this.logicalResourceId = logicalResourceId; + this.tokenParameter = tokenParameter; + this.commonTokenValue = commonTokenValue; + } + + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } + + + /** + * @return the logicalResourceId + */ + public long getLogicalResourceId() { + return logicalResourceId; + } + + + /** + * @return the tokenParameter + */ + public TokenParameter getTokenParameter() { + return tokenParameter; + } + + + /** + * @return the commonTokenValue + */ + public CommonTokenValue getCommonTokenValue() { + return commonTokenValue; + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java index 6cfe482bc4d..47f0ef97561 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java @@ -9,12 +9,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -23,134 +19,111 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.common.TopicPartition; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.remote.index.api.IMessageHandler; /** * Kafka consumer reading remote index messages, batches the data and * loads it into the configured database. */ -public class RemoteIndexConsumer implements Runnable { - private static final Logger logger = Logger.getLogger(RemoteIndexConsumer.class.getName()); - - // Nanoseconds in a second - private static final long NANOS = 1000000000L; - - // the remote index service Kafka topic name - private final String topicName; - - // The max time to spend collecting a batch before submitting - private final long maxBatchCollectTimeMs; - - // The consumer object representing the connection to Kafka - private final KafkaConsumer kafkaConsumer; - - // The handler we use to process messages we receive from Kafka - private final IMessageHandler messageHandler; - - private final Duration pollWaitTime; - - // Flag used to exit kafka loop on termination - private volatile boolean running = true; - - // The map we use to track offsets as we process messages - private Map trackingOffsetMap = new HashMap<>(); - private Map rollbackOffsetMap = new HashMap<>(); - - // Use a list to stage the results of a poll() - private final LinkedHashSet> recordBuffer = new LinkedHashSet<>(); - private final List> currentBatch = new ArrayList<>(); - private final Set assignedPartitions = new HashSet<>(); - - // Track the number of sequential commit failures - private int commitFailures = 0; - - // If we get 5 failures in a row, we disconnect - private static final int COMMIT_FAILURE_DISCONNECT_THRESHOLD = 5; - - // A callback used to signal that this consumer has failed - private final Runnable consumerFailedCallback; - - /** - * A listener to track the partitions assigned to this cluster member - * Callbacks happen as part of the consumer#poll() call, so no concurrency concerns - */ - private class PartitionChangeListener implements ConsumerRebalanceListener { - - @Override - public void onPartitionsRevoked(Collection partitions) { - for (TopicPartition tp: partitions) { - logger.info("Revoking partition: " + tp.topic() + ":" + tp.partition()); - } - assignedPartitions.removeAll(partitions); - } - - @Override - public void onPartitionsAssigned(Collection partitions) { - for (TopicPartition tp: partitions) { - logger.info("Assigning partition: " + tp.topic() + ":" + tp.partition()); - } - assignedPartitions.addAll(partitions); - } - } - - /** - * Public constructor - * - * @param kafkaConsumer - * @param messageHandler - * @param consumerFailedCallback - * @param topicName - * @param maxBatchCollectTimeMs - * @param pollWaitTime - */ - public RemoteIndexConsumer(KafkaConsumer kafkaConsumer, IMessageHandler messageHandler, - Runnable consumerFailedCallback, String topicName, long maxBatchCollectTimeMs, Duration pollWaitTime) { - this.kafkaConsumer = kafkaConsumer; - this.messageHandler = messageHandler; - this.consumerFailedCallback = consumerFailedCallback; - this.topicName = topicName; - this.maxBatchCollectTimeMs = maxBatchCollectTimeMs; - this.pollWaitTime = pollWaitTime; - } - - /** - * poll the consumer and forward any messages we receive to the message handler - */ - private void consume() { - ConsumerRecords records = kafkaConsumer.poll(pollWaitTime); - List messages = new ArrayList<>(); - - for (ConsumerRecord record : records) { - messages.add(record.value()); - } - messageHandler.process(messages); - kafkaConsumer.commitAsync(); - } - - - public void shutdown() { - this.running = false; - try { - kafkaConsumer.wakeup(); - } catch (Throwable x) { - logger.warning("Error waking up kafka consumer: " + x.getMessage()); - } - } - +public class RemoteIndexConsumer implements Runnable, OffsetCommitCallback { + + private static final Logger logger = Logger.getLogger(RemoteIndexConsumer.class.getName()); + + // Nanoseconds in a second + private static final long NANOS = 1000000000L; + + // the remote index service Kafka topic name + private final String topicName; + + // The max time to spend collecting a batch before submitting + private final long maxBatchCollectTimeMs; + + // The consumer object representing the connection to Kafka + private final KafkaConsumer kafkaConsumer; + + // The handler we use to process messages we receive from Kafka + private final IMessageHandler messageHandler; + + private final Duration pollWaitTime; + + // Flag used to exit kafka loop on termination + private volatile boolean running = true; + + // The number of commit failures since the last successful one + private int sequentialCommitFailures = 0; + + // If we get 5 failures in a row, we disconnect + private static final int COMMIT_FAILURE_DISCONNECT_THRESHOLD = 5; + + // A callback used to signal that this consumer has failed + private final Runnable consumerFailedCallback; + /** - * Get an array of the TopicPartition tuples currently assigned to this node - * @return the assigned TopicPartitions, or null if the consumer.assignment is empty + * A listener to track the partitions assigned to this cluster member + * Callbacks happen as part of the consumer#poll() call, so no concurrency concerns */ - private TopicPartition[] getConsumerAssignment() { - Set assignment = kafkaConsumer.assignment(); - - if (assignment.size() > 0) { - return assignment.toArray(new TopicPartition[assignment.size()]); + private class PartitionChangeListener implements ConsumerRebalanceListener { + + @Override + public void onPartitionsRevoked(Collection partitions) { + for (TopicPartition tp : partitions) { + logger.info("Revoking partition: " + tp.topic() + ":" + tp.partition()); + } } - else { - return null; + + @Override + public void onPartitionsAssigned(Collection partitions) { + for (TopicPartition tp : partitions) { + logger.info("Assigning partition: " + tp.topic() + ":" + tp.partition()); + } + } + } + + /** + * Public constructor + * + * @param kafkaConsumer + * @param messageHandler + * @param consumerFailedCallback + * @param topicName + * @param maxBatchCollectTimeMs + * @param pollWaitTime + */ + public RemoteIndexConsumer(KafkaConsumer kafkaConsumer, IMessageHandler messageHandler, + Runnable consumerFailedCallback, String topicName, long maxBatchCollectTimeMs, Duration pollWaitTime) { + this.kafkaConsumer = kafkaConsumer; + this.messageHandler = messageHandler; + this.consumerFailedCallback = consumerFailedCallback; + this.topicName = topicName; + this.maxBatchCollectTimeMs = maxBatchCollectTimeMs; + this.pollWaitTime = pollWaitTime; + } + + /** + * poll the consumer and forward any messages we receive to the message handler + */ + private void consume() throws FHIRPersistenceException { + ConsumerRecords records = kafkaConsumer.poll(pollWaitTime); + List messages = new ArrayList<>(); + + for (ConsumerRecord record : records) { + messages.add(record.value()); + } + messageHandler.process(messages); + // TODO, obviously + // kafkaConsumer.commitAsync(this); + } + + public void shutdown() { + this.running = false; + try { + kafkaConsumer.wakeup(); + } catch (Throwable x) { + logger.warning("Error waking up kafka consumer: " + x.getMessage()); } } @@ -162,19 +135,49 @@ public void run() { logger.info("Starting consumer loop"); while (running) { try { - consume(); + if (this.sequentialCommitFailures > COMMIT_FAILURE_DISCONNECT_THRESHOLD) { + logger.severe("Too many commit failures. Stopping consumer"); + this.running = false; + } else { + consume(); + } } catch (Throwable t) { - // If we end up here, it means we think it's an unrecoverable error which - // we want to signal back to the main controller. If enough consumer - // threads fail, this will lead to the whole program to terminate - // which in typical deployments will mean that the container will - // hopefully be restarted + // If we end up here, it means we think it's an unrecoverable error so + // clear the running flag allowing us to exit logger.log(Level.SEVERE, "unexpected error in consumer loop", t); this.running = false; - this.consumerFailedCallback.run(); + } finally { + if (!running) { + // explicitly closing the consumer here should allow for faster error recovery + // (assuming, of course, that the brokers are still reachable from this node) + kafkaConsumer.close(); + + // close the handler, cleaning up any resources it is holding onto + messageHandler.close(); + + // signal back to the main controller. If enough consumer + // threads fail, this will lead to the whole program to terminate + // which in typical deployments will mean that the container will + // hopefully be restarted + consumerFailedCallback.run(); + } } } - logger.info("Consumer thread terminated"); + logger.info("Consumer closed and thread terminated"); } -} + @Override + public void onComplete(Map offsets, Exception exception) { + // called on this consumer's thread, so no need for any synchronization + if (exception == null) { + // successful commit, so reset the failure counter + this.sequentialCommitFailures = 0; + } else { + this.sequentialCommitFailures++; + logger.warning("Commit failure sequential=[" + this.sequentialCommitFailures + "] reason=[" + exception.getMessage() + "]"); + if (logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, "sequentialCommitFailures=" + sequentialCommitFailures, exception); + } + } + } +} \ No newline at end of file From 6343ab9dfbe6877f4522e20573884fa399469edc Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 12 May 2022 17:26:32 +0100 Subject: [PATCH 07/40] issue #3437 remote index support for profile, tags and security Signed-off-by: Robin Arnold --- .../utils/common/PreparedStatementHelper.java | 126 +++++++ .../dao/impl/ParameterTransportVisitor.java | 60 +++- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 26 +- .../SearchParametersTransportAdapter.java | 75 +++-- .../context/FHIRPersistenceContext.java | 4 +- .../FHIRPersistenceContextFactory.java | 12 +- .../impl/FHIRPersistenceContextImpl.java | 18 +- .../persistence/index/CanonicalSupport.java | 87 +++++ .../fhir/persistence/index/DateParameter.java | 15 +- .../persistence/index/LocationParameter.java | 13 + .../persistence/index/NumberParameter.java | 15 + .../index/ParameterValueVisitorAdapter.java | 40 ++- .../persistence/index/ProfileParameter.java | 79 +++++ .../persistence/index/QuantityParameter.java | 19 ++ .../index/RemoteIndexConstants.java | 17 + .../persistence/index/RemoteIndexMessage.java | 15 + .../index/SearchParameterValue.java | 12 + .../index/SearchParametersTransport.java | 117 ++++++- .../persistence/index/SecurityParameter.java | 57 ++++ .../persistence/index/StringParameter.java | 11 + .../fhir/persistence/index/TagParameter.java | 57 ++++ .../persistence/index/TokenParameter.java | 15 + .../test/FHIRPersistenceContextTest.java | 4 +- fhir-remote-index/pom.xml | 5 + .../index/api/BatchParameterProcessor.java | 65 +++- .../remote/index/api/BatchParameterValue.java | 5 +- .../fhir/remote/index/api/IdentityCache.java | 7 + .../com/ibm/fhir/remote/index/app/Main.java | 24 +- .../index/batch/BatchDateParameter.java | 7 +- .../index/batch/BatchLocationParameter.java | 7 +- .../index/batch/BatchNumberParameter.java | 7 +- .../index/batch/BatchProfileParameter.java | 45 +++ .../index/batch/BatchQuantityParameter.java | 7 +- .../index/batch/BatchSecurityParameter.java | 44 +++ .../index/batch/BatchStringParameter.java | 7 +- .../remote/index/batch/BatchTagParameter.java | 44 +++ .../index/batch/BatchTokenParameter.java | 7 +- .../remote/index/cache/IdentityCacheImpl.java | 16 +- .../index/database/BaseMessageHandler.java | 123 ++++++- .../index/database/CommonCanonicalValue.java | 67 ++++ .../database/CommonCanonicalValueKey.java | 44 +++ .../DistributedPostgresMessageHandler.java | 318 ++++++++++++++++-- .../DistributedPostgresParameterBatch.java | 108 ++++++ ...stributedPostgresSystemParameterBatch.java | 106 +++++- .../database/JDBCBatchParameterProcessor.java | 147 +++++++- .../database/PreparedStatementWrapper.java | 58 ++++ .../index/kafka/RemoteIndexConsumer.java | 57 ++-- .../kafka/FHIRRemoteIndexKafkaService.java | 2 + .../ibm/fhir/server/util/FHIRRestHelper.java | 26 +- 49 files changed, 2022 insertions(+), 225 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java new file mode 100644 index 00000000000..86297e2ebcf --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java @@ -0,0 +1,126 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; + +/** + * Collection of utility functions to simply setting values on a PreparedStatement + */ +public class PreparedStatementHelper { + // The PreparedStatement we delegate everything to + private final Calendar UTC = CalendarHelper.getCalendarForUTC(); + private final PreparedStatement ps; + private int index = 1; + + /** + * Public constructor + * @param ps + */ + public PreparedStatementHelper(PreparedStatement ps) { + this.ps = ps; + } + + /** + * Set the (possibly null) int value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setInt(Integer value) throws SQLException { + if (value != null) { + ps.setInt(index, value); + } else { + ps.setNull(index, Types.INTEGER); + } + index++; + return this; + } + + /** + * Set the (possibly null) long value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setLong(Long value) throws SQLException { + if (value != null) { + ps.setLong(index, value); + } else { + ps.setNull(index, Types.BIGINT); + } + index++; + return this; + } + + /** + * Set the (possibly null) long value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setShort(Short value) throws SQLException { + if (value != null) { + ps.setShort(index, value); + } else { + ps.setNull(index, Types.SMALLINT); + } + index++; + return this; + } + + /** + * Set the (possibly null) String value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setString(String value) throws SQLException { + if (value != null) { + ps.setString(index, value); + } else { + ps.setNull(index, Types.VARCHAR); + } + index++; + return this; + } + + /** + * Set the (possibly null) int value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setTimestamp(Timestamp value) throws SQLException { + if (value != null) { + ps.setTimestamp(index, value, UTC); + } else { + ps.setNull(index, Types.TIMESTAMP); + } + index++; + return this; + } + + /** + * Add a new batch entry based on the current state of the {@link PreparedStatement}. + * Note that we don't return this on purpose...because addBatch should be last in + * any sequence of setXX(...) calls. + * @throws SQLException + */ + public void addBatch() throws SQLException { + ps.addBatch(); + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java index d35691f1bc0..f85586a1b1e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java @@ -6,8 +6,13 @@ package com.ibm.fhir.persistence.jdbc.dao.impl; +import java.util.logging.Logger; + import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.CanonicalSupport; import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; import com.ibm.fhir.persistence.jdbc.dto.DateParmVal; import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; @@ -18,6 +23,9 @@ import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; +import com.ibm.fhir.search.SearchConstants; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; /** @@ -25,6 +33,10 @@ * system (e.g. for remote indexing) */ public class ParameterTransportVisitor implements ExtractedParameterValueVisitor { + private static final Logger logger = Logger.getLogger(ParameterTransportVisitor.class.getName()); + private static final Boolean IS_WHOLE_SYSTEM = Boolean.TRUE; + + // The adapter to which we delegate each of our visit calls private final ParameterValueVisitorAdapter adapter; // tracks the number of composites so we know what next composite_id to use @@ -43,8 +55,15 @@ public ParameterTransportVisitor(ParameterValueVisitorAdapter adapter) { @Override public void visit(StringParmVal stringParameter) throws FHIRPersistenceException { - Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; - adapter.stringValue(stringParameter.getName(), stringParameter.getValueString(), compositeId); + + if (SearchConstants.PROFILE.equals(stringParameter.getName())) { + // special case to store profile parameters in their own table + ProfileParameter pp = CanonicalSupport.createProfileParameter(stringParameter.getName(), stringParameter.getValueString()); + adapter.profileValue(pp.getName(), pp.getUrl(), pp.getVersion(), pp.getFragment(), IS_WHOLE_SYSTEM); + } else { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.stringValue(stringParameter.getName(), stringParameter.getValueString(), compositeId, stringParameter.isWholeSystem()); + } } @Override @@ -60,13 +79,25 @@ public void visit(NumberParmVal numberParameter) throws FHIRPersistenceException @Override public void visit(DateParmVal dateParameter) throws FHIRPersistenceException { Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; - adapter.dateValue(dateParameter.getName(), dateParameter.getValueDateStart(), dateParameter.getValueDateEnd(), compositeId); + adapter.dateValue(dateParameter.getName(), dateParameter.getValueDateStart(), dateParameter.getValueDateEnd(), compositeId, dateParameter.isWholeSystem()); } @Override public void visit(TokenParmVal tokenParameter) throws FHIRPersistenceException { Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; - adapter.tokenValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), compositeId); + // tag and profile search params are often low-selectivity (many resources sharing the same value) so + // we put them into their own tables to allow better cardinality estimation by the query + // optimizer + switch (tokenParameter.getName()) { + case SearchConstants.TAG: + adapter.tagValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), IS_WHOLE_SYSTEM); + break; + case SearchConstants.SECURITY: + adapter.securityValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), IS_WHOLE_SYSTEM); + break; + default: + adapter.tokenValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), compositeId); + } } @Override @@ -84,9 +115,26 @@ public void visit(LocationParmVal locationParameter) throws FHIRPersistenceExcep } @Override - public void visit(ReferenceParmVal ref) throws FHIRPersistenceException { + public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; - adapter.referenceValue(ref.getName(), ref.getRefValue(), compositeId); + + // The ReferenceValue has already been processed to convert the reference to + // the required standard form, ready for insertion as a token value. + ReferenceValue refValue = rpv.getRefValue(); + String resourceType = rpv.getResourceType(); + String refResourceType = refValue.getTargetResourceType(); + String refLogicalId = refValue.getValue(); + Integer refVersion = refValue.getVersion(); + if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { + // protect against code regression. Invalid/improper references should be + // filtered out already. + logger.warning("Invalid reference parameter type: '" + resourceType + "." + rpv.getName() + "' type=" + refValue.getType().name()); + throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); + } + if (refResourceType == null) { + refResourceType = JDBCConstants.DEFAULT_TOKEN_SYSTEM; + } + adapter.referenceValue(rpv.getName(), refResourceType, refLogicalId, refVersion, compositeId); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 07b84e43cd7..ace21a63db7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -421,7 +421,7 @@ public SingleResourceResult create(FHIRPersistenceContex + ", version=" + resourceDTO.getVersionId()); } - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getShardKey(), searchParameters); + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getRequestShard(), searchParameters); SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -464,12 +464,12 @@ public SingleResourceResult create(FHIRPersistenceContex * @param shardKey * @param searchParameters */ - private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, Short shardKey, + private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, String requestShard, ExtractedSearchParameters searchParameters) throws FHIRPersistenceException { FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); if (remoteIndexService != null) { // convert the parameters into a form that will be easy to ship to a remote service - SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, shardKey); + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, requestShard); ParameterTransportVisitor visitor = new ParameterTransportVisitor(adapter); for (ExtractedParameterValue pv: searchParameters.getParameters()) { pv.accept(visitor); @@ -547,6 +547,22 @@ private com.ibm.fhir.persistence.jdbc.dto.Resource createResourceDTO(Class SingleResourceResult update(FHIRPersistenceContex } // If configured, send the extracted parameters to the remote indexing service - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getShardKey(), searchParameters); + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getRequestShard(), searchParameters); SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java index 4413d78f05c..c2fa972ff8a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java @@ -8,29 +8,28 @@ import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.logging.Logger; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; +import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; import com.ibm.fhir.persistence.index.SearchParametersTransport; +import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; -import com.ibm.fhir.search.util.ReferenceValue; -import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; /** - * Visitor implementation to build an instance of {@link SearchParametersTransport} to + * Visitor adapter implementation to build an instance of {@link SearchParametersTransport} to * provide support for shipping a set of search parameter values off to a remote * index service. This allows the parameters to be stored in the database in a * separate transaction, and allows the inserts to be batched together, providing * improved throughput. */ public class SearchParametersTransportAdapter implements ParameterValueVisitorAdapter { - private static final Logger logger = Logger.getLogger(SearchParametersTransportAdapter.class.getName()); // The builder we use to collect all the visited parameter values private final SearchParametersTransport.Builder builder; @@ -40,14 +39,14 @@ public class SearchParametersTransportAdapter implements ParameterValueVisitorAd * @param resourceType * @param logicalId * @param logicalResourceId - * @param shardKey + * @param requestShard */ - public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, Short shardKey) { + public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, String requestShard) { builder = SearchParametersTransport.builder() .withResourceType(resourceType) .withLogicalId(logicalId) .withLogicalResourceId(logicalResourceId) - .withShardKey(shardKey); + .withRequestShard(requestShard); } /** @@ -59,11 +58,12 @@ public SearchParametersTransport build() { } @Override - public void stringValue(String name, String valueString, Integer compositeId) { + public void stringValue(String name, String valueString, Integer compositeId, boolean wholeSystem) { StringParameter value = new StringParameter(); value.setName(name); value.setValue(valueString); value.setCompositeId(compositeId); + value.setWholeSystem(wholeSystem); builder.addStringValue(value); } @@ -79,12 +79,13 @@ public void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNum } @Override - public void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId) { + public void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId, boolean wholeSystem) { DateParameter value = new DateParameter(); value.setName(name); value.setValueDateStart(valueDateStart); value.setValueDateEnd(valueDateEnd); value.setCompositeId(compositeId); + value.setWholeSystem(wholeSystem); builder.addDateValue(value); } @@ -98,6 +99,37 @@ public void tokenValue(String name, String valueSystem, String valueCode, Intege builder.addTokenValue(value); } + @Override + public void tagValue(String name, String valueSystem, String valueCode, boolean wholeSystem) { + TagParameter value = new TagParameter(); + value.setName(name); + value.setValueSystem(valueSystem); + value.setValueCode(valueCode); + value.setWholeSystem(wholeSystem); + builder.addTagValue(value); + } + + @Override + public void profileValue(String name, String url, String version, String fragment, boolean wholeSystem) { + ProfileParameter value = new ProfileParameter(); + value.setName(name); + value.setUrl(url); + value.setVersion(version); + value.setFragment(fragment); + value.setWholeSystem(wholeSystem); + builder.addProfileValue(value); + } + + @Override + public void securityValue(String name, String valueSystem, String valueCode, boolean wholeSystem) { + SecurityParameter value = new SecurityParameter(); + value.setName(name); + value.setValueSystem(valueSystem); + value.setValueCode(valueCode); + value.setWholeSystem(wholeSystem); + builder.addSecurityValue(value); + } + @Override public void quantityValue(String name, String valueSystem, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId) { @@ -123,28 +155,7 @@ public void locationValue(String name, Double valueLatitude, Double valueLongitu } @Override - public void referenceValue(String name, ReferenceValue refValue, Integer compositeId) { - if (refValue == null) { - return; - } - - // The ReferenceValue has already been processed to convert the reference to - // the required standard form, ready for insertion as a token value. - - String refResourceType = refValue.getTargetResourceType(); - String refLogicalId = refValue.getValue(); - Integer refVersion = refValue.getVersion(); - - // Ignore references containing only a "display" element (apparently supported by the spec, - // but contains nothing useful to store because there's no searchable value). - // See ParameterVisitorBatchDAO - if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { - // protect against code regression. Invalid/improper references should be - // filtered out already. - logger.warning("Invalid reference parameter type: name='" + name + "' type=" + refValue.getType().name()); - throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); - } - + public void referenceValue(String name, String refResourceType, String refLogicalId, Integer refVersion, Integer compositeId) { TokenParameter value = new TokenParameter(); value.setName(name); value.setValueSystem(refResourceType); diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java index 871bc15574b..9b8b9f819ba 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java @@ -58,7 +58,7 @@ public interface FHIRPersistenceContext { /** * Get the key used for sharding used by the distributed schema variant. If * the tenant is not configured for distribution, the value will be null - * @return any Short value in [Short.MIN_VALUE, Short.MAX_VALUE] or null + * @return the shard key value specified in the request */ - Short getShardKey(); + String getRequestShard(); } \ No newline at end of file diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java index b9825d886b3..f8e2038e809 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java @@ -71,12 +71,12 @@ public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEve * Returns a FHIRPersistenceContext that contains a FHIRPersistenceEvent and a FHIRSearchContext. * @param event the FHIRPersistenceEvent instance to be contained in the FHIRPersistenceContext instance * @param searchContext the FHIRSearchContext instance to be contained in the FHIRPersistenceContext instance - * @param shardKey + * @param requestShard the requested shard; can be null */ - public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext, Short shardKey) { + public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext, String requestShard) { return FHIRPersistenceContextImpl.builder(event) .withSearchContext(searchContext) - .withShardKey(shardKey) + .withRequestShard(requestShard) .build(); } @@ -105,14 +105,14 @@ public static FHIRHistoryContext createHistoryContext() { * @param event the FHIRPersistenceEvent instance to be contained in the FHIRPersistenceContext instance * @param includeDeleted flag to tell the persistence layer to include deleted resources in the operation results * @param searchContext the FHIRSearchContext instance to be contained in the FHIRPersistenceContext instance - * @param shardKey the sharding value used for the distributed schema + * @param requestShard the sharding value used for the distributed schema */ public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, boolean includeDeleted, FHIRSearchContext searchContext, - Short shardKey) { + String requestShard) { return FHIRPersistenceContextImpl.builder(event) .withIncludeDeleted(includeDeleted) .withSearchContext(searchContext) - .withShardKey(shardKey) + .withRequestShard(requestShard) .build(); } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java index 64d5d65b217..2bdc4e3e37c 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java @@ -23,7 +23,7 @@ public class FHIRPersistenceContextImpl implements FHIRPersistenceContext { private FHIRSearchContext searchContext; private boolean includeDeleted = false; private Integer ifNoneMatch; - private Short shardKey; + private String requestShard; // The response from the payload persistence (offloading) call, if any private PayloadPersistenceResponse offloadResponse; @@ -47,7 +47,7 @@ public static class Builder { private boolean includeDeleted; private Integer ifNoneMatch; private PayloadPersistenceResponse offloadResponse; - private Short shardKey; + private String requestShard; /** * Protected constructor @@ -74,7 +74,7 @@ public FHIRPersistenceContext build() { impl.setIfNoneMatch(ifNoneMatch); impl.setIncludeDeleted(includeDeleted); impl.setOffloadResponse(offloadResponse); - impl.setShardKey(shardKey); + impl.setRequestShard(requestShard); return impl; } @@ -124,8 +124,8 @@ public Builder withIncludeDeleted(boolean includeDeleted) { * @param shardKey * @return */ - public Builder withShardKey(Short shardKey) { - this.shardKey = shardKey; + public Builder withRequestShard(String requestShard) { + this.requestShard = requestShard; return this; } @@ -199,8 +199,8 @@ public boolean includeDeleted() { } @Override - public Short getShardKey() { - return this.shardKey; + public String getRequestShard() { + return this.requestShard; } /** @@ -215,8 +215,8 @@ public void setIncludeDeleted(boolean includeDeleted) { * Set the shardKey value * @param value */ - public void setShardKey(Short value) { - this.shardKey = value; + public void setRequestShard(String value) { + this.requestShard = value; } /** diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java new file mode 100644 index 00000000000..c8f9848b489 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java @@ -0,0 +1,87 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; + +/** + * Utility methods supporting the processing of profile search + * parameters which are stored using common_canonical_values + */ +public class CanonicalSupport { + + /** + * Split the given string value to extract the profile url, version + * and fragment parts if they exist + * @param stringValue + * @return + */ + public static ProfileParameter createProfileParameter(String name, String stringValue) { + ProfileParameter result; + try { + result = parseCanonicalValue(stringValue); + } catch (FHIRPersistenceException e) { + // Not a valid version/fragment format - just use the input string as the whole uri + result = new ProfileParameter(); + result.setUrl(stringValue); + } + result.setName(name); + return result; + } + + /** + * Parse the canonical value. + * @param canonicalValue + * @return + */ + private static ProfileParameter parseCanonicalValue(String canonicalValue) throws FHIRPersistenceException { + String uri = canonicalValue; + String version = null; + String fragment = null; + + // Parse the canonical value to extract the URI|VERSION#FRAGMENT pieces + if (canonicalValue != null) { + int vindex = canonicalValue.indexOf('|'); + int findex = canonicalValue.indexOf('#'); + if (vindex == 0 || findex == 0 || vindex > findex && findex > -1) { + throw new FHIRPersistenceException("Invalid canonical URI"); + } + + // Extract version if given + if (vindex > 0) { + if (findex > -1) { + version = canonicalValue.substring(vindex+1, findex); // everything after the | but before the # + } else { + version = canonicalValue.substring(vindex+1); // everything after the | + } + if (version.isEmpty()) { + version = null; + } + uri = canonicalValue.substring(0, vindex); // everything before the | + } + + // Extract fragment if given + if (findex > 0) { + fragment = canonicalValue.substring(findex+1); + if (fragment.isEmpty()) { + fragment = null; + } + + if (vindex < 0) { + // fragment but no version + uri = canonicalValue.substring(0, findex); // everything before the # + } + } + } + + ProfileParameter result = new ProfileParameter(); + result.setUrl(uri); + result.setVersion(version); + result.setFragment(fragment); + return result; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java index 4f04d962b78..ddb50e5b147 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java @@ -14,7 +14,20 @@ public class DateParameter extends SearchParameterValue { private Timestamp valueDateStart; private Timestamp valueDateEnd; - + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Date["); + addDescription(result); + result.append(","); + result.append(valueDateStart); + result.append(","); + result.append(valueDateEnd); + result.append("]"); + return result.toString(); + } + /** * @return the valueDateStart */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java index d4058472e81..7e54303d046 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java @@ -14,6 +14,19 @@ public class LocationParameter extends SearchParameterValue { private Double valueLatitude; private Double valueLongitude; + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Location["); + addDescription(result); + result.append(","); + result.append(valueLatitude); + result.append(","); + result.append(valueLongitude); + result.append("]"); + return result.toString(); + } + /** * @return the valueLatitude */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java index 153cb5e9ba4..e5fa2bdd317 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java @@ -16,6 +16,21 @@ public class NumberParameter extends SearchParameterValue { private BigDecimal lowValue; private BigDecimal highValue; + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Number["); + addDescription(result); + result.append(","); + result.append(value); + result.append(","); + result.append(lowValue); + result.append(","); + result.append(highValue); + result.append("]"); + return result.toString(); + } + /** * @return the value */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java index b105dcde235..173a7334ab8 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java @@ -9,8 +9,6 @@ import java.math.BigDecimal; import java.sql.Timestamp; -import com.ibm.fhir.search.util.ReferenceValue; - /** * Used by a parameter value visitor to translate the parameter values * to a new form @@ -21,8 +19,9 @@ public interface ParameterValueVisitorAdapter { * @param name * @param valueString * @param compositeId + * @param wholeSystem */ - void stringValue(String name, String valueString, Integer compositeId); + void stringValue(String name, String valueString, Integer compositeId, boolean wholeSystem); /** * @param name @@ -38,8 +37,9 @@ public interface ParameterValueVisitorAdapter { * @param valueDateStart * @param valueDateEnd * @param compositeId + * @param wholeSystem */ - void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId); + void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId, boolean wholeSystem); /** * @param name @@ -49,6 +49,32 @@ public interface ParameterValueVisitorAdapter { */ void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId); + /** + * @param name + * @param valueSystem + * @param valueCode + * @param compositeId + * @param wholeSystem + */ + void tagValue(String name, String valueSystem, String valueCode, boolean wholeSystem); + + /** + * @param name + * @param url + * @param version + * @param fragment + * @param wholeSystem + */ + void profileValue(String name, String url, String version, String fragment, boolean wholeSystem); + + /** + * @param name + * @param valueSystem + * @param valueCode + * @param wholeSystem + */ + void securityValue(String name, String valueSystem, String valueCode, boolean wholeSystem); + /** * @param name * @param valueSystem @@ -71,8 +97,10 @@ void quantityValue(String name, String valueSystem, String valueCode, BigDecimal /** * @param name - * @param refValue + * @param refResourceType + * @param refLogicalId + * @param refVersion * @param compositeId */ - void referenceValue(String name, ReferenceValue refValue, Integer compositeId); + void referenceValue(String name, String refResourceType, String refLogicalId, Integer refVersion, Integer compositeId); } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java new file mode 100644 index 00000000000..5e19afb6516 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java @@ -0,0 +1,79 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A profile search parameter value + */ +public class ProfileParameter extends SearchParameterValue { + + private String url; + + // profile version value + private String version; + + // profile fragment value + private String fragment; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Profile["); + addDescription(result); + result.append(","); + result.append(url); + result.append(","); + result.append(version); + result.append(","); + result.append(fragment); + result.append("]"); + return result.toString(); + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param url the url to set + */ + public void setUrl(String url) { + this.url = url; + } + + /** + * @return the version + */ + public String getVersion() { + return version; + } + + /** + * @param version the version to set + */ + public void setVersion(String version) { + this.version = version; + } + + /** + * @return the fragment + */ + public String getFragment() { + return fragment; + } + + /** + * @param fragment the fragment to set + */ + public void setFragment(String fragment) { + this.fragment = fragment; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java index f4f931e0983..b2386f0ed0e 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java @@ -18,6 +18,25 @@ public class QuantityParameter extends SearchParameterValue { private String valueSystem; private String valueCode; + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Quantity["); + addDescription(result); + result.append(","); + result.append(valueNumber); + result.append(","); + result.append(valueNumberLow); + result.append(","); + result.append(valueNumberHigh); + result.append(","); + result.append(valueSystem); + result.append(","); + result.append(valueCode); + result.append("]"); + return result.toString(); + } + /** * @return the valueNumber */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java new file mode 100644 index 00000000000..34fd200749c --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java @@ -0,0 +1,17 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * Constants associated with the remote index service + */ +public class RemoteIndexConstants { + + // the current version of remote index messages we push to Kafka + public static final int MESSAGE_VERSION = 1; +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java index 3c35a98599f..7b09b92bc51 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java @@ -11,6 +11,7 @@ */ public class RemoteIndexMessage { private String tenantId; + private int messageVersion; private SearchParametersTransport data; /** @@ -40,5 +41,19 @@ public SearchParametersTransport getData() { public void setData(SearchParametersTransport data) { this.data = data; } + + /** + * @return the messageVersion + */ + public int getMessageVersion() { + return messageVersion; + } + + /** + * @param messageVersion the messageVersion to set + */ + public void setMessageVersion(int messageVersion) { + this.messageVersion = messageVersion; + } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java index ccf7cbc6dde..c13bd06e634 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java @@ -22,6 +22,18 @@ public class SearchParameterValue { // True if this parameter should also be stored at the whole-system level private Boolean wholeSystem; + /** + * Add the base description of this parameter to the given {@link StringBuilder} + * @param sb + */ + protected void addDescription(StringBuilder sb) { + sb.append(name); + sb.append(","); + sb.append(compositeId); + sb.append(","); + sb.append(wholeSystem); + } + /** * @return the name */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java index 2fa53df3088..6fc40b9494b 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java @@ -28,7 +28,7 @@ public class SearchParametersTransport { private long logicalResourceId; // The key value used for sharding the data when using a distributed database - private Short shardKey; + private String requestShard; private List stringValues; private List numberValues; @@ -36,6 +36,9 @@ public class SearchParametersTransport { private List tokenValues; private List dateValues; private List locationValues; + private List tagValues = new ArrayList<>(); + private List profileValues = new ArrayList<>(); + private List securityValues = new ArrayList<>(); /** * Factory method to create a {@link Builder} instance @@ -55,11 +58,14 @@ public static class Builder { private List tokenValues = new ArrayList<>(); private List dateValues = new ArrayList<>(); private List locationValues = new ArrayList<>(); + private List tagValues = new ArrayList<>(); + private List profileValues = new ArrayList<>(); + private List securityValues = new ArrayList<>(); private String resourceType; private String logicalId; private long logicalResourceId = -1; - private Short shardKey; + private String requestShard; /** * Set the resourceType @@ -96,8 +102,8 @@ public Builder withLogicalResourceId(long logicalResourceId) { * @param shardKey * @return */ - public Builder withShardKey(Short shardKey) { - this.shardKey = shardKey; + public Builder withRequestShard(String shardValue) { + this.requestShard = shardValue; return this; } @@ -141,6 +147,36 @@ public Builder addTokenValue(TokenParameter value) { return this; } + /** + * Add a tag parameter value + * @param value + * @return + */ + public Builder addTagValue(TagParameter value) { + tagValues.add(value); + return this; + } + + /** + * Add a profile parameter value + * @param value + * @return + */ + public Builder addProfileValue(ProfileParameter value) { + profileValues.add(value); + return this; + } + + /** + * Add a security parameter value + * @param value + * @return + */ + public Builder addSecurityValue(SecurityParameter value) { + securityValues.add(value); + return this; + } + /** * Add a date parameter value * @param value @@ -181,7 +217,7 @@ public SearchParametersTransport build() { result.resourceType = this.resourceType; result.logicalId = this.logicalId; result.logicalResourceId = this.logicalResourceId; - result.shardKey = this.shardKey; + result.setRequestShard(this.requestShard); if (this.stringValues.size() > 0) { result.stringValues = new ArrayList<>(this.stringValues); @@ -201,6 +237,15 @@ public SearchParametersTransport build() { if (this.locationValues.size() > 0) { result.locationValues = new ArrayList<>(this.locationValues); } + if (this.tagValues.size() > 0) { + result.setTagValues(new ArrayList<>(this.tagValues)); + } + if (this.profileValues.size() > 0) { + result.setProfileValues(new ArrayList<>(this.profileValues)); + } + if (this.securityValues.size() > 0) { + result.setSecurityValues(new ArrayList<>(this.securityValues)); + } return result; } } @@ -347,4 +392,66 @@ public List getLocationValues() { public void setLocationValues(List locationValues) { this.locationValues = locationValues; } + + + /** + * @return the requestShard + */ + public String getRequestShard() { + return requestShard; + } + + + /** + * @param shardValue the request shard value to set + */ + public void setRequestShard(String shardValue) { + this.requestShard = shardValue; + } + + + /** + * @return the tagValues + */ + public List getTagValues() { + return tagValues; + } + + + /** + * @param tagValues the tagValues to set + */ + public void setTagValues(List tagValues) { + this.tagValues = tagValues; + } + + + /** + * @return the profileValues + */ + public List getProfileValues() { + return profileValues; + } + + + /** + * @param profileValues the profileValues to set + */ + public void setProfileValues(List profileValues) { + this.profileValues = profileValues; + } + + /** + * @return the securityValues + */ + public List getSecurityValues() { + return securityValues; + } + + /** + * @param profileValues the profileValues to set + */ + public void setSecurityValues(List securityValues) { + this.securityValues = securityValues; + } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java new file mode 100644 index 00000000000..1d024762750 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java @@ -0,0 +1,57 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A security search parameter value + */ +public class SecurityParameter extends SearchParameterValue { + private String valueSystem; + private String valueCode; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Security["); + addDescription(result); + result.append(","); + result.append(valueSystem); + result.append(","); + result.append(valueCode); + result.append("]"); + return result.toString(); + } + + /** + * @return the valueSystem + */ + public String getValueSystem() { + return valueSystem; + } + + /** + * @param valueSystem the valueSystem to set + */ + public void setValueSystem(String valueSystem) { + this.valueSystem = valueSystem; + } + + /** + * @return the valueCode + */ + public String getValueCode() { + return valueCode; + } + + /** + * @param valueCode the valueCode to set + */ + public void setValueCode(String valueCode) { + this.valueCode = valueCode; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java index 4a56ba46e7a..bb8077c041c 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java @@ -13,6 +13,17 @@ public class StringParameter extends SearchParameterValue { private String value; + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("String["); + addDescription(result); + result.append(","); + result.append(value); + result.append("]"); + return result.toString(); + } + /** * @return the value */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java new file mode 100644 index 00000000000..539ea2c3b9d --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java @@ -0,0 +1,57 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A token search parameter value + */ +public class TagParameter extends SearchParameterValue { + private String valueSystem; + private String valueCode; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Tag["); + addDescription(result); + result.append(","); + result.append(valueSystem); + result.append(","); + result.append(valueCode); + result.append("]"); + return result.toString(); + } + + /** + * @return the valueSystem + */ + public String getValueSystem() { + return valueSystem; + } + + /** + * @param valueSystem the valueSystem to set + */ + public void setValueSystem(String valueSystem) { + this.valueSystem = valueSystem; + } + + /** + * @return the valueCode + */ + public String getValueCode() { + return valueCode; + } + + /** + * @param valueCode the valueCode to set + */ + public void setValueCode(String valueCode) { + this.valueCode = valueCode; + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java index 20232be10de..0d12749f5af 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java @@ -16,6 +16,21 @@ public class TokenParameter extends SearchParameterValue { // for storing versioned references private Integer refVersionId; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Token["); + addDescription(result); + result.append(","); + result.append(valueSystem); + result.append(","); + result.append(valueCode); + result.append(","); + result.append(refVersionId); + result.append("]"); + return result.toString(); + } /** * @return the valueSystem diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java index d1d2df0c9f8..c34851565c4 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java @@ -78,7 +78,7 @@ public void test4() { FHIRSearchContext sc = FHIRSearchContextFactory.createSearchContext(); assertNotNull(sc); - FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc, (short)13); + FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc, "pat42"); assertNotNull(ctxt); assertNotNull(ctxt.getPersistenceEvent()); assertEquals(pe, ctxt.getPersistenceEvent()); @@ -86,6 +86,6 @@ public void test4() { assertEquals(sc, ctxt.getSearchContext()); assertFalse(ctxt.includeDeleted()); assertNull(ctxt.getHistoryContext()); - assertEquals(13, Short.toUnsignedInt(ctxt.getShardKey())); + assertEquals("pat42", ctxt.getRequestShard()); } } diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml index 8485903f03c..3d70c573e3c 100644 --- a/fhir-remote-index/pom.xml +++ b/fhir-remote-index/pom.xml @@ -42,6 +42,11 @@ fhir-database-utils ${project.version} + + ${project.groupId} + fhir-search + ${project.version} + com.google.code.gson gson diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java index 54d6300c8b7..a3613203d5b 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java @@ -10,10 +10,14 @@ import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; import com.ibm.fhir.remote.index.database.CodeSystemValue; +import com.ibm.fhir.remote.index.database.CommonCanonicalValue; import com.ibm.fhir.remote.index.database.CommonTokenValue; import com.ibm.fhir.remote.index.database.ParameterNameValue; @@ -21,66 +25,111 @@ * Processes batched parameters */ public interface BatchParameterProcessor { + /** * Compute the shard key value use to distribute resources among nodes * of the database + * @param requestShard + * @return + */ + Short encodeShardKey(String requestShard); + + /** + * @param requestShard * @param resourceType * @param logicalId - * @return + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException; + + /** + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + */ + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) throws FHIRPersistenceException; + + /** + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter */ - short encodeShardKey(String resourceType, String logicalId); + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue codeSystemValue) throws FHIRPersistenceException; /** + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) throws FHIRPersistenceException; /** + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) throws FHIRPersistenceException; /** + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue codeSystemValue) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; /** + * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter + * @param commonTokenValue + * @throws FHIRPersistenceException */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; /** + * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter + * @param commonCanonicalValue + * @throws FHIRPersistenceException */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter parameter, CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException; /** + * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter + * @param commonCanonicalValue + * @throws FHIRPersistenceException */ - void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java index 39ef9a5bed8..5b8af6f9dcb 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java @@ -13,6 +13,7 @@ * A parameter value batched for later processing */ public abstract class BatchParameterValue { + protected final String requestShard; protected final ParameterNameValue parameterNameValue; protected final String resourceType; protected final String logicalId; @@ -20,12 +21,14 @@ public abstract class BatchParameterValue { /** * Protected constructor + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue */ - protected BatchParameterValue(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue) { + protected BatchParameterValue(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue) { + this.requestShard = requestShard; this.resourceType = resourceType; this.logicalId = logicalId; this.logicalResourceId = logicalResourceId; diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java index ace72468da9..cb3e5679f78 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java @@ -41,4 +41,11 @@ public interface IdentityCache { * @param parameterNameId */ void addParameterName(String parameterName, int parameterNameId); + + /** + * @param shardKey + * @param url + * @return + */ + Long getCommonCanonicalValueId(short shardKey, String url); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index f8c5a9d66f0..013b881d13e 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -69,7 +69,8 @@ public class Main { private volatile boolean running = true; // Exit if we drop below this number of running consumers - private int minRunningConsumerThreshold = 1; + private float minRunningConsumerRatio = 0.5f; + private int minRunningConsumerThreshold; private IdentityCacheImpl identityCache; // Database Configuration @@ -173,6 +174,12 @@ public void run() throws FHIRPersistenceException { configureForPostgres(); initIdentityCache(); + // Keep track of how many consumers are still running. If too many fail, + // we stop everything and exit which allows our operating environment + // to handle things perhaps by restarting us somewhere else + stillRunningCounter = new AtomicInteger(this.consumerCount); + this.minRunningConsumerThreshold = Math.max(1, Math.round(this.consumerCount * minRunningConsumerRatio)); + // One thread per consumer ExecutorService pool = Executors.newCachedThreadPool(); for (int i=0; i buildConsumer() { kp.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kp.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kp.put("group.id", this.consumerGroup); + kp.put("auto.offset.reset", "earliest"); KafkaConsumer consumer = new KafkaConsumer<>(kp); return consumer; @@ -328,11 +340,17 @@ protected void dumpProperties(String which, Properties p) { } } + /** + * Called from a consumer thread when it is about to exit + */ private void failedConsumerCallback() { - if (this.stillRunningCounter.decrementAndGet() < minRunningConsumerThreshold) { + final int remainingConsumersStillRunning = stillRunningCounter.decrementAndGet(); + if (remainingConsumersStillRunning < minRunningConsumerThreshold) { // Signal termination of the entire program logger.severe("Too many consumers have failed. Terminating"); this.running = false; + } else { + logger.info("Remaining consumer count: " + remainingConsumersStillRunning); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java index 9d08826eaba..ed65a2f0a8b 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java @@ -21,19 +21,20 @@ public class BatchDateParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - public BatchDateParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchDateParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java index 524de945f18..3afcac8848c 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java @@ -21,19 +21,20 @@ public class BatchLocationParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - public BatchLocationParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchLocationParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java index d1aa15f3323..3235dd08c8b 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java @@ -21,19 +21,20 @@ public class BatchNumberParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - public BatchNumberParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchNumberParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java new file mode 100644 index 00000000000..14c0e508693 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java @@ -0,0 +1,45 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.CommonCanonicalValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A profile parameter we are collecting to batch + */ +public class BatchProfileParameter extends BatchParameterValue { + private final ProfileParameter parameter; + private final CommonCanonicalValue commonCanonicalValue; + + /** + * Canonical constructor + * + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param commonCanonicalValue + */ + public BatchProfileParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, + ParameterNameValue parameterNameValue, ProfileParameter parameter, CommonCanonicalValue commonCanonicalValue) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.commonCanonicalValue = commonCanonicalValue; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonCanonicalValue); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java index cbbf486e6b0..80b568f1b95 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java @@ -23,6 +23,7 @@ public class BatchQuantityParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId @@ -30,14 +31,14 @@ public class BatchQuantityParameter extends BatchParameterValue { * @param parameter * @param csv */ - public BatchQuantityParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue csv) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchQuantityParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue csv) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; this.codeSystemValue = csv; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, codeSystemValue); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, codeSystemValue); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java new file mode 100644 index 00000000000..eb5d86dfec3 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.SecurityParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A security parameter we are collecting to batch + */ +public class BatchSecurityParameter extends BatchParameterValue { + private final SecurityParameter parameter; + private final CommonTokenValue commonTokenValue; + + /** + * Canonical constructor + * + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param commonTokenValue + */ + public BatchSecurityParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter parameter, CommonTokenValue commonTokenValue) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.commonTokenValue = commonTokenValue; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java index 2b37a8dbb9a..bfd4a15feb0 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java @@ -21,19 +21,20 @@ public class BatchStringParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param parameterNameValue * @param parameter */ - public BatchStringParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchStringParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java new file mode 100644 index 00000000000..d2a1aa1d982 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.TagParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A tag parameter we are collecting to batch + */ +public class BatchTagParameter extends BatchParameterValue { + private final TagParameter parameter; + private final CommonTokenValue commonTokenValue; + + /** + * Canonical constructor + * + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param commonTokenValue + */ + public BatchTagParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter parameter, CommonTokenValue commonTokenValue) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.commonTokenValue = commonTokenValue; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java index 50f0c14ae71..7dc6bfaa27a 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java @@ -23,6 +23,7 @@ public class BatchTokenParameter extends BatchParameterValue { /** * Canonical constructor * + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId @@ -30,14 +31,14 @@ public class BatchTokenParameter extends BatchParameterValue { * @param parameter * @param commonTokenValue */ - public BatchTokenParameter(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) { - super(resourceType, logicalId, logicalResourceId, parameterNameValue); + public BatchTokenParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); this.parameter = parameter; this.commonTokenValue = commonTokenValue; } @Override public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { - processor.process(resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue); + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java index 624e2e31ca0..0b47e4d8cee 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java @@ -12,6 +12,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey; import com.ibm.fhir.remote.index.database.CommonTokenValueKey; /** @@ -22,6 +23,7 @@ public class IdentityCacheImpl implements IdentityCache { private final ConcurrentHashMap parameterNames = new ConcurrentHashMap<>(); private final Cache codeSystemCache; private final Cache commonTokenValueCache; + private final Cache commonCanonicalValueCache; private static final Integer NULL_INT = null; private static final Long NULL_LONG = null; @@ -29,7 +31,8 @@ public class IdentityCacheImpl implements IdentityCache { * Public constructor */ public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDuration, - long maxCommonTokenCacheSize, Duration commonTokenCacheDuration) { + long maxCommonTokenCacheSize, Duration commonTokenCacheDuration, + long maxCommonCanonicalCacheSize, Duration commonCanonicalCacheDuration) { codeSystemCache = Caffeine.newBuilder() .maximumSize(maxCodeSystemCacheSize) .expireAfterWrite(codeSystemCacheDuration) @@ -38,6 +41,10 @@ public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDur .maximumSize(maxCommonTokenCacheSize) .expireAfterWrite(commonTokenCacheDuration) .build(); + commonCanonicalValueCache = Caffeine.newBuilder() + .maximumSize(maxCommonCanonicalCacheSize) + .expireAfterWrite(commonCanonicalCacheDuration) + .build(); } @Override @@ -61,4 +68,9 @@ public Long getCommonTokenValueId(short shardKey, String codeSystem, String toke public void addParameterName(String parameterName, int parameterNameId) { parameterNames.put(parameterName, parameterNameId); } -} + + @Override + public Long getCommonCanonicalValueId(short shardKey, String url) { + return commonCanonicalValueCache.get(new CommonCanonicalValueKey(shardKey, url), k -> NULL_LONG); + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index 89a813b0eaa..11ee02c0fc6 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -7,16 +7,21 @@ package com.ibm.fhir.remote.index.database; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import com.google.gson.Gson; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; import com.ibm.fhir.persistence.index.RemoteIndexMessage; import com.ibm.fhir.persistence.index.SearchParametersTransport; +import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; import com.ibm.fhir.remote.index.api.IMessageHandler; @@ -26,13 +31,25 @@ * a database via JDBC. */ public abstract class BaseMessageHandler implements IMessageHandler { + private final Logger logger = Logger.getLogger(BaseMessageHandler.class.getName()); + private static final int MIN_SUPPORTED_MESSAGE_VERSION = 1; @Override public void process(List messages) throws FHIRPersistenceException { try { + startBatch(); for (String payload: messages) { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Processing message payload: " + payload); + } RemoteIndexMessage message = unmarshall(payload); - process(message); + if (message != null) { + if (message.getMessageVersion() >= MIN_SUPPORTED_MESSAGE_VERSION) { + process(message); + } else { + logger.warning("Message version [" + message.getMessageVersion() + "] not supported, ignoring payload=[" + payload + "]"); + } + } } pushBatch(); } catch (Throwable t) { @@ -43,6 +60,11 @@ public void process(List messages) throws FHIRPersistenceException { } } + /** + * Called before we start processing a new batch of messages + */ + protected abstract void startBatch(); + /** * Mark the transaction for rollback */ @@ -60,7 +82,15 @@ public void process(List messages) throws FHIRPersistenceException { */ private RemoteIndexMessage unmarshall(String jsonPayload) { Gson gson = new Gson(); - return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + try { + return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + } catch (Throwable t) { + // We need to sink this error to avoid poison messages from + // blocking the queues. + // TODO. Perhaps push this to a dedicated error topic + logger.severe("Not a RemoteIndexMessage. Ignoring: '" + jsonPayload + "'"); + } + return null; } /** * Process the data @@ -70,94 +100,159 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException SearchParametersTransport params = message.getData(); if (params.getStringValues() != null) { for (StringParameter p: params.getStringValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } if (params.getDateValues() != null) { for (DateParameter p: params.getDateValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } if (params.getNumberValues() != null) { for (NumberParameter p: params.getNumberValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } if (params.getQuantityValues() != null) { for (QuantityParameter p: params.getQuantityValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } if (params.getTokenValues() != null) { for (TokenParameter p: params.getTokenValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } if (params.getLocationValues() != null) { for (LocationParameter p: params.getLocationValues()) { - process(message.getTenantId(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getTagValues() != null) { + for (TagParameter p: params.getTagValues()) { + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getProfileValues() != null) { + for (ProfileParameter p: params.getProfileValues()) { + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } + + if (params.getSecurityValues() != null) { + for (SecurityParameter p: params.getSecurityValues()) { + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } } /** * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException; /** * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException; /** + * * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p + * @throws FHIRPersistenceException */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException; /** + * * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p + * @throws FHIRPersistenceException */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException; /** + * * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p + * @throws FHIRPersistenceException */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException; /** * @param tenantId + * @param requestShard * @param resourceType * @param logicalId * @param logicalResourceId * @param p */ - protected abstract void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException; + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException; + /** + * @param tenantId + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException; + + /** + * @param tenantId + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException; + + /** + * @param tenantId + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + */ + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException; + + /** + * Tell the persistence layer to commit the current transaction, or perform a rollback + * if setRollbackOnly() has been called + * @throws FHIRPersistenceException + */ protected abstract void endTransaction() throws FHIRPersistenceException; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java new file mode 100644 index 00000000000..449d43d36ed --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java @@ -0,0 +1,67 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +/** + * Represents a common_canonical_value record which may or may not yet exist + * in the database. If it exists in the database, we may not yet have + * retrieved its canonical_id. + */ +public class CommonCanonicalValue { + private final short shardKey; + private final String url; + private Long canonicalId; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append(shardKey); + result.append(","); + result.append(url); + result.append(","); + result.append(canonicalId); + return result.toString(); + } + /** + * Public constructor + * @param shardKey + * @param codeSystemValue + * @param tokenValue + */ + public CommonCanonicalValue(short shardKey, String url) { + this.shardKey = shardKey; + this.url = url; + } + + /** + * @return the canonicalId + */ + public Long getCanonicalId() { + return canonicalId; + } + + /** + * @param canonicalId the canonicalId to set + */ + public void setCanonicalId(Long canonicalId) { + this.canonicalId = canonicalId; + } + + /** + * @return the shardKey + */ + public short getShardKey() { + return shardKey; + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java new file mode 100644 index 00000000000..a6d3b8d19e6 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.util.Objects; + +/** + * A key used to identify a common_canonical_value record in our distributed schema + * variant + */ +public class CommonCanonicalValueKey { + private final short shardKey; + private final String url; + + /** + * Public constructor + * @param shardKey + * @param codeSystem + * @param tokenValue + */ + public CommonCanonicalValueKey(short shardKey, String url) { + this.shardKey = shardKey; + this.url = Objects.requireNonNull(url); + } + + @Override + public int hashCode() { + return Objects.hash(url, shardKey); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof CommonCanonicalValueKey) { + CommonCanonicalValueKey that = (CommonCanonicalValueKey)obj; + return this.shardKey == that.shardKey + && this.url.equals(that.url); + } + return false; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index 64c4cab9591..5e4185aaf84 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -27,17 +27,23 @@ import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; import com.ibm.fhir.persistence.index.SearchParameterValue; +import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; import com.ibm.fhir.remote.index.api.BatchParameterValue; import com.ibm.fhir.remote.index.api.IdentityCache; import com.ibm.fhir.remote.index.batch.BatchDateParameter; import com.ibm.fhir.remote.index.batch.BatchLocationParameter; import com.ibm.fhir.remote.index.batch.BatchNumberParameter; +import com.ibm.fhir.remote.index.batch.BatchProfileParameter; import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; +import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; import com.ibm.fhir.remote.index.batch.BatchStringParameter; +import com.ibm.fhir.remote.index.batch.BatchTagParameter; import com.ibm.fhir.remote.index.batch.BatchTokenParameter; /** @@ -68,6 +74,9 @@ public class DistributedPostgresMessageHandler extends BaseMessageHandler { // A map to support lookup of CommonTokenValue records by key private final Map commonTokenValueMap = new HashMap<>(); + // A map to support lookup of CommonCanonicalValue records by key + private final Map commonCanonicalValueMap = new HashMap<>(); + // All parameter names in the current transaction for which we don't yet know the parameter_name_id private final List unresolvedParameterNames = new ArrayList<>(); @@ -76,6 +85,9 @@ public class DistributedPostgresMessageHandler extends BaseMessageHandler { // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id private final List unresolvedTokenValues = new ArrayList<>(); + + // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id + private final List unresolvedCanonicalValues = new ArrayList<>(); // The processed values we've collected private final List batchedParameterValues = new ArrayList<>(); @@ -85,6 +97,7 @@ public class DistributedPostgresMessageHandler extends BaseMessageHandler { private final int maxCodeSystemsPerStatement = 512; private final int maxCommonTokenValuesPerStatement = 256; + private final int maxCommonCanonicalValuesPerStatement = 256; private boolean rollbackOnly; /** @@ -99,6 +112,17 @@ public DistributedPostgresMessageHandler(Connection connection, String schemaNam this.batchProcessor = new JDBCBatchParameterProcessor(connection); } + @Override + protected void startBatch() { + // always start with a clean slate + batchedParameterValues.clear(); + unresolvedParameterNames.clear(); + unresolvedSystemValues.clear(); + unresolvedTokenValues.clear(); + unresolvedCanonicalValues.clear(); + batchProcessor.startBatch(); + } + @Override protected void setRollbackOnly() { this.rollbackOnly = true; @@ -115,10 +139,13 @@ public void close() { @Override protected void endTransaction() throws FHIRPersistenceException { + boolean committed = false; try { if (!this.rollbackOnly) { logger.fine("Committing transaction"); connection.commit(); + committed = true; + // any values from parameter_names, code_systems and common_token_values // are now committed to the database, so we can publish their record ids // to the shared cache which makes them accessible from other threads @@ -131,16 +158,23 @@ protected void endTransaction() throws FHIRPersistenceException { } catch (SQLException x) { // It could very well be that we've lost touch with the database in which case // the rollback will also fail. Not much we can do, although we don't bother - // with a stack trace here because it's just more noise for the log file. + // with a stack trace here because it's just more noise for the log file, and + // the exception that triggered the rollback is already going to be propagated + // and logged. logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); } } } catch (SQLException x) { throw new FHIRPersistenceException("commit failed", x); } finally { - unresolvedParameterNames.clear(); - unresolvedSystemValues.clear(); - unresolvedTokenValues.clear(); + if (!committed) { + // The maps may contain ids that were not committed to the database so + // we should clean them out in case we decide to reuse this consumer + this.parameterNameMap.clear(); + this.codeSystemValueMap.clear(); + this.commonTokenValueMap.clear(); + this.commonCanonicalValueMap.clear(); + } } } @@ -153,7 +187,6 @@ public void publishCachedValues() { for (ParameterNameValue pnv: this.unresolvedParameterNames) { identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); } - this.unresolvedParameterNames.clear(); } @Override @@ -166,6 +199,7 @@ protected void pushBatch() throws FHIRPersistenceException { resolveParameterNames(); resolveCodeSystems(); resolveCommonTokenValues(); + resolveCommonCanonicalValues(); // Now that all the lookup values should've been resolved, we can go ahead // and push the parameters to the JDBC batch insert statements via the @@ -182,6 +216,9 @@ protected void pushBatch() throws FHIRPersistenceException { * @return */ private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("get ParameterNameValue for [" + p.toString() + "]"); + } ParameterNameValue result = parameterNameMap.get(p.getName()); if (result == null) { result = new ParameterNameValue(p.getName()); @@ -200,42 +237,66 @@ private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHI } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchStringParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchLocationParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { - short shardKey = batchProcessor.encodeShardKey(resourceType, logicalId); + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchTokenParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonCanonicalValue ctv = lookupCommonCanonicalValue(shardKey, p.getUrl()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { ParameterNameValue parameterNameValue = getParameterNameId(p); CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); - this.batchedParameterValues.add(new BatchQuantityParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); + this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchNumberParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } @Override - protected void process(String tenantId, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchDateParameter(resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); } /** @@ -291,6 +352,24 @@ private CommonTokenValue lookupCommonTokenValue(short shardKey, String codeSyste return result; } + private CommonCanonicalValue lookupCommonCanonicalValue(short shardKey, String url) { + CommonCanonicalValueKey key = new CommonCanonicalValueKey(shardKey, url); + CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); + if (result == null) { + result = new CommonCanonicalValue(shardKey, url); + this.commonCanonicalValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long canonicalId = identityCache.getCommonCanonicalValueId(shardKey, url); + if (canonicalId != null) { + result.setCanonicalId(canonicalId); + } else { + this.unresolvedCanonicalValues.add(result); + } + } + return result; + } + /** * Make sure we have values for all the code_systems we have collected * in the current @@ -316,7 +395,7 @@ private void resolveCodeSystems() throws FHIRPersistenceException { } /** - * Build and prepare a statement to fetch the code_system_id and code_system value + * Build and prepare a statement to fetch the code_system_id and code_system_name * from the code_systems table for all the given (unresolved) code system values * @param values * @return @@ -324,7 +403,7 @@ private void resolveCodeSystems() throws FHIRPersistenceException { */ private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { StringBuilder query = new StringBuilder(); - query.append("SELECT code_system_id, code_system FROM code_systems WHERE code_system IN ("); + query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); for (int i=0; i 0) { query.append(","); @@ -355,7 +434,7 @@ private void addMissingCodeSystems(List missing) throws FHIRPer final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO code_systems (code_system_id, code_system) VALUES ("); + insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); insert.append(nextVal); // next sequence value insert.append(",?) ON CONFLICT DO NOTHING"); @@ -470,14 +549,15 @@ private void resolveCommonTokenValues() throws FHIRPersistenceException { * @return SELECT shard_key, code_system, token_value, common_token_value_id * @throws SQLException */ - private PreparedStatement buildCommonTokenValueSelectStatement(List values) throws SQLException { + private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { StringBuilder query = new StringBuilder(); // need the code_system name - so we join back to the code_systems table as well - query.append("SELECT c.shard_key, cs.code_system, c.token_value, c.common_token_value_id "); + query.append("SELECT c.shard_key, cs.code_system_name, c.token_value, c.common_token_value_id "); query.append(" FROM common_token_values c"); query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); - query.append(" JOIN VALUES ("); - + query.append(" JOIN (VALUES "); + + // Create a (codeSystem, shardKey, tokenValue) tuple for each of the CommonTokenValue records boolean first = true; for (CommonTokenValue ctv: values) { if (first) { @@ -487,19 +567,22 @@ private PreparedStatement buildCommonTokenValueSelectStatement(List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { @@ -512,7 +595,9 @@ private List fetchCommonTokenValueIds(List u int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive offset += subSize; // set up for the next iteration - try (PreparedStatement ps = buildCommonTokenValueSelectStatement(sub)) { + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { + sql = ps.getStatementText(); ResultSet rs = ps.executeQuery(); // We can't rely on the order of result rows matching the order of the in-list, // so we have to go back to our map to look up each CodeSystemValue @@ -542,7 +627,7 @@ private List fetchCommonTokenValueIds(List u } } } catch (SQLException x) { - logger.log(Level.SEVERE, "common token values fetch failed", x); + logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); throw new FHIRPersistenceException("common token values fetch failed"); } } @@ -561,8 +646,9 @@ private void addMissingCommonTokenValues(List missing) throws final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) VALUES ("); - insert.append("?,?,?,"); + insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) "); + insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number + insert.append(" VALUES (?,?,?,"); insert.append(nextVal); // next sequence value insert.append(") ON CONFLICT DO NOTHING"); @@ -589,6 +675,172 @@ private void addMissingCommonTokenValues(List missing) throws } } + /** + * Make sure we have values for all the common_canonical_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonCanonicalValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCanonicalIds(unresolvedCanonicalValues); + + if (!missing.isEmpty()) { + // Sort on (url, shard_key) to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getUrl().compareTo(b.getUrl()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + return result; + }); + addMissingCommonCanonicalValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCanonicalIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all canonical values"); + } + } + + private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonCanonicalValueKey key = new CommonCanonicalValueKey(rs.getShort(1), rs.getString(2)); + CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); + if (ctv != null) { + ctv.setCanonicalId(rs.getLong(3)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonCanonicalValue ctv: sub) { + if (ctv.getCanonicalId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common canonical values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT shard_key, code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT c.shard_key, c.url, c.canonical_id "); + query.append(" FROM common_canonical_values c "); + query.append(" JOIN (VALUES "); + + // Create a (shardKey, url) tuple for each of the CommonCanonicalValue records + boolean first = true; + for (CommonCanonicalValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getShardKey()); // literal for shard_key + query.append(",?)"); // bind variable for the uri + } + query.append(") AS v(shard_key, url) "); + query.append(" ON (c.url = v.url AND c.shard_key = v.shard_key)"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + logger.finer(() -> "fetch common canonical values [" + statementText + "]"); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonCanonicalValue ctv: values) { + ps.setString(param++, ctv.getUrl()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + private void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (shard_key, url, canonical_id) "); + insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number + insert.append(" VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + final String DML = insert.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("addMissingCanonicalIds: " + DML); + } + try (PreparedStatement ps = connection.prepareStatement(DML)) { + int count = 0; + for (CommonCanonicalValue ctv: missing) { + logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); + ps.setShort(1, ctv.getShardKey()); + ps.setString(2, ctv.getUrl()); + ps.addBatch(); + if (++count == this.maxCommonCanonicalValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common canonical values"); + } + } + /** * Make sure all the parameter names we've seen in the batch exist * in the database and have ids. diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java index 2be9839a91a..ece3b5558fe 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java @@ -15,6 +15,7 @@ import java.util.Calendar; import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; /** * Parameter batch statements configured for a given resource type @@ -41,6 +42,15 @@ public class DistributedPostgresParameterBatch { private PreparedStatement resourceTokenRefs; private int resourceTokenRefCount; + private PreparedStatement tags; + private int tagCount; + + private PreparedStatement profiles; + private int profileCount; + + private PreparedStatement security; + private int securityCount; + /** * Public constructor * @param c @@ -79,6 +89,18 @@ public void pushBatch() throws SQLException { resourceTokenRefs.executeBatch(); resourceTokenRefCount = 0; } + if (tagCount > 0) { + tags.executeBatch(); + tagCount = 0; + } + if (profileCount > 0) { + profiles.executeBatch(); + profileCount = 0; + } + if (securityCount > 0) { + security.executeBatch(); + securityCount = 0; + } } /** @@ -151,6 +173,36 @@ public void reset() { resourceTokenRefCount = 0; } } + if (tags != null) { + try { + tags.close(); + } catch (SQLException x) { + // NOP + } finally { + tags = null; + tagCount = 0; + } + } + if (profiles != null) { + try { + profiles.close(); + } catch (SQLException x) { + // NOP + } finally { + profiles = null; + profileCount = 0; + } + } + if (security != null) { + try { + security.close(); + } catch (SQLException x) { + // NOP + } finally { + security = null; + securityCount = 0; + } + } } /** @@ -167,6 +219,20 @@ private void setComposite(PreparedStatement ps, int index, Integer compositeId) ps.setNull(index, Types.INTEGER); } } + /** + * Utility method to set a string value and handle null + * @param ps + * @param index + * @param value + * @throws SQLException + */ + private void setString(PreparedStatement ps, int index, String value) throws SQLException { + if (value == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, value); + } + } public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId, short shardKey) throws SQLException { if (strings == null) { @@ -272,4 +338,46 @@ public void addResourceTokenRef(long logicalResourceId, int parameterNameId, lon resourceTokenRefs.addBatch(); resourceTokenRefCount++; } + + public void addTag(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException { + if (tags == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_tags (common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)"; + tags = connection.prepareStatement(tokenString); + } + tags.setLong(1, commonTokenValueId); + tags.setLong(2, logicalResourceId); + tags.setShort(3, shardKey); + tags.addBatch(); + tagCount++; + } + + public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment, short shardKey) throws SQLException { + if (profiles == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_profiles (canonical_id, logical_resource_id, shard_key, version, fragment) VALUES (?,?,?,?,?)"; + profiles = connection.prepareStatement(tokenString); + } + profiles.setLong(1, canonicalId); + profiles.setLong(2, logicalResourceId); + profiles.setShort(3, shardKey); + setString(profiles, 4, version); + setString(profiles, 5, fragment); + profiles.addBatch(); + profileCount++; + } + + public void addSecurity(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException { + if (tags == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String INS = "INSERT INTO " + tablePrefix + "_security (common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)"; + security = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(security); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .setShort(shardKey) + .addBatch(); + securityCount++; + } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java index 5a9b554a2d4..8effe2e2afb 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java @@ -14,6 +14,7 @@ import java.util.Calendar; import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; /** * Batch insert statements for system-level parameters @@ -29,6 +30,15 @@ public class DistributedPostgresSystemParameterBatch { private PreparedStatement systemDates; private int systemDateCount; + private PreparedStatement systemProfiles; + private int systemProfileCount; + + private PreparedStatement systemTags; + private int systemTagCount; + + private PreparedStatement systemSecurity; + private int systemSecurityCount; + /** * Public constructor * @param c @@ -49,21 +59,20 @@ public void pushBatch() throws SQLException { systemDates.executeBatch(); systemDateCount = 0; } - } - - /** - * Clear the current batch - */ - public void clearBatch() throws SQLException { - if (systemStringCount > 0) { - systemStrings.clearBatch(); - systemStringCount = 0; + if (systemTagCount > 0) { + systemTags.executeBatch(); + systemTagCount = 0; } - if (systemDateCount > 0) { - systemDates.clearBatch(); - systemDateCount = 0; + if (systemProfileCount > 0) { + systemProfiles.executeBatch(); + systemProfileCount = 0; + } + if (systemSecurityCount > 0) { + systemSecurity.executeBatch(); + systemSecurityCount = 0; } } + /** * Closes all the statements currently open */ @@ -76,6 +85,7 @@ public void close() { // NOP } finally { systemStrings = null; + systemStringCount = 0; } } @@ -86,6 +96,37 @@ public void close() { // NOP } finally { systemDates = null; + systemDateCount = 0; + } + } + if (systemTags != null) { + try { + systemTags.close(); + } catch (SQLException x) { + // NOP + } finally { + systemTags = null; + systemTagCount = 0; + } + } + if (systemProfiles != null) { + try { + systemProfiles.close(); + } catch (SQLException x) { + // NOP + } finally { + systemProfiles = null; + systemProfileCount = 0; + } + } + if (systemSecurity != null) { + try { + systemSecurity.close(); + } catch (SQLException x) { + // NOP + } finally { + systemSecurity = null; + systemProfileCount = 0; } } } @@ -136,4 +177,45 @@ public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateS systemDates.addBatch(); systemDateCount++; } + + public void addTag(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException { + if (systemTags == null) { + final String INS = "INSERT INTO logical_resource_tags(common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)"; + systemTags = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemTags); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .setShort(shardKey) + .addBatch(); + systemTagCount++; + } + + public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment, short shardKey) throws SQLException { + if (systemProfiles == null) { + final String INS = "INSERT INTO logical_resource_profiles(canonical_id, logical_resource_id, shard_key, version, fragment) VALUES (?,?,?,?,?)"; + systemProfiles = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemProfiles); + psh.setLong(canonicalId) + .setLong(logicalResourceId) + .setShort(shardKey) + .setString(version) + .setString(fragment) + .addBatch(); + systemProfileCount++; + } + + public void addSecurity(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException { + if (systemTags == null) { + final String INS = "INSERT INTO logical_resource_security(common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)"; + systemSecurity = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemSecurity); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .setShort(shardKey) + .addBatch(); + systemSecurityCount++; + } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java index 973510610d9..55ccbc8f3ed 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java @@ -12,13 +12,18 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; import com.ibm.fhir.remote.index.api.BatchParameterProcessor; @@ -28,6 +33,8 @@ * JDBC statements */ public class JDBCBatchParameterProcessor implements BatchParameterProcessor { + private static final Logger logger = Logger.getLogger(JDBCBatchParameterProcessor.class.getName()); + // A cache of the resource-type specific DAOs we've created private final Map daoMap = new HashMap<>(); @@ -59,6 +66,13 @@ public void close() { systemDao.close(); } + /** + * Start processing a new batch + */ + public void startBatch() { + resourceTypesInBatch.clear(); + } + /** * Push any statements that have been batched but not yet executed * @throws FHIRPersistenceException @@ -66,6 +80,9 @@ public void close() { public void pushBatch() throws FHIRPersistenceException { try { for (String resourceType: resourceTypesInBatch) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Pushing batch for [" + resourceType + "]"); + } DistributedPostgresParameterBatch dao = daoMap.get(resourceType); try { dao.pushBatch(); @@ -75,6 +92,7 @@ public void pushBatch() throws FHIRPersistenceException { } try { + logger.fine("Pushing batch for whole-system parameters"); systemDao.pushBatch(); } catch (SQLException x) { throw new FHIRPersistenceException("batch insert for whole-system parameters", x); @@ -86,6 +104,7 @@ public void pushBatch() throws FHIRPersistenceException { } private DistributedPostgresParameterBatch getParameterBatchDao(String resourceType) { + resourceTypesInBatch.add(resourceType); DistributedPostgresParameterBatch dao = daoMap.get(resourceType); if (dao == null) { dao = new DistributedPostgresParameterBatch(connection, resourceType); @@ -95,16 +114,24 @@ private DistributedPostgresParameterBatch getParameterBatchDao(String resourceTy } @Override - public short encodeShardKey(String resourceType, String logicalId) { - final String requestShardKey = resourceType + "/" + logicalId; - return Short.valueOf((short)requestShardKey.hashCode()); + public Short encodeShardKey(String requestShard) { + if (requestShard != null) { + return Short.valueOf((short)requestShard.hashCode()); + } else { + return null; + } } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process string parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + parameter.toString() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey); if (parameter.isSystemParam()) { @@ -116,10 +143,15 @@ public void process(String resourceType, String logicalId, long logicalResourceI } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process number parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId(), shardKey); } catch (SQLException x) { throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); @@ -127,10 +159,15 @@ public void process(String resourceType, String logicalId, long logicalResourceI } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process quantity parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId(), shardKey); } catch (SQLException x) { throw new FHIRPersistenceException("Failed inserting quantity params for '" + resourceType + "'"); @@ -138,10 +175,15 @@ public void process(String resourceType, String logicalId, long logicalResourceI } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process location parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId(), shardKey); } catch (SQLException x) { throw new FHIRPersistenceException("Failed inserting location params for '" + resourceType + "'"); @@ -149,10 +191,15 @@ public void process(String resourceType, String logicalId, long logicalResourceI } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process date parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId(), shardKey); if (p.isSystemParam()) { systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId(), shardKey); @@ -163,15 +210,85 @@ public void process(String resourceType, String logicalId, long logicalResourceI } @Override - public void process(String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p, + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p, CommonTokenValue commonTokenValue) throws FHIRPersistenceException { - final short shardKey = encodeShardKey(resourceType, logicalId); + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process token parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + try { DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId(), shardKey); } catch (SQLException x) { throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'"); } } + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process tag parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); + dao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); + + if (p.isSystemParam()) { + systemDao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting tag params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter p, + CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process profile parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonCanonicalValue.getCanonicalId() + "]"); + } + + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); + dao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment(), shardKey); + if (p.isSystemParam()) { + systemDao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment(), shardKey); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting profile params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process security parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + + try { + DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); + dao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); + + if (p.isSystemParam()) { + systemDao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'"); + } + } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java new file mode 100644 index 00000000000..4705a8f4a26 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java @@ -0,0 +1,58 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Wraps a {@link PreparedStatement} together with the statement text for easier + * logging when there are errors + */ +public class PreparedStatementWrapper implements AutoCloseable { + private final String statementText; + private final PreparedStatement preparedStatement; + + /** + * Canonical constructor + * @param statementText + * @param ps + */ + public PreparedStatementWrapper(String statementText, PreparedStatement ps) { + this.statementText = statementText; + this.preparedStatement = ps; + } + + /** + * @return the statementText + */ + public String getStatementText() { + return statementText; + } + + /** + * @return the preparedStatement + */ + public PreparedStatement getPreparedStatement() { + return preparedStatement; + } + + @Override + public void close() throws SQLException { + this.preparedStatement.close(); + } + + /** + * Convenience method to delegate the call to the wrapped statement + * @return + * @throws SQLException + */ + public ResultSet executeQuery() throws SQLException { + return preparedStatement.executeQuery(); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java index 47f0ef97561..4884c5a54ed 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java @@ -103,30 +103,6 @@ public RemoteIndexConsumer(KafkaConsumer kafkaConsumer, IMessage this.pollWaitTime = pollWaitTime; } - /** - * poll the consumer and forward any messages we receive to the message handler - */ - private void consume() throws FHIRPersistenceException { - ConsumerRecords records = kafkaConsumer.poll(pollWaitTime); - List messages = new ArrayList<>(); - - for (ConsumerRecord record : records) { - messages.add(record.value()); - } - messageHandler.process(messages); - // TODO, obviously - // kafkaConsumer.commitAsync(this); - } - - public void shutdown() { - this.running = false; - try { - kafkaConsumer.wakeup(); - } catch (Throwable x) { - logger.warning("Error waking up kafka consumer: " + x.getMessage()); - } - } - @Override public void run() { logger.info("Subscribing consumer to topic '" + this.topicName + "'"); @@ -148,6 +124,8 @@ public void run() { this.running = false; } finally { if (!running) { + logger.warning("Stopping consumer loop"); + // explicitly closing the consumer here should allow for faster error recovery // (assuming, of course, that the brokers are still reachable from this node) kafkaConsumer.close(); @@ -166,6 +144,37 @@ public void run() { logger.info("Consumer closed and thread terminated"); } + /** + * poll the consumer and forward any messages we receive to the message handler + */ + private void consume() throws FHIRPersistenceException { + logger.finer("Polling Kafka"); + ConsumerRecords records = kafkaConsumer.poll(pollWaitTime); + logger.finer(() -> "Kafka poll records count: " + records.count()); + + // Extract the message payloads from each ConsumerRecord + List messages = new ArrayList<>(); + for (ConsumerRecord record : records) { + messages.add(record.value()); + } + if (messages.size() > 0) { + messageHandler.process(messages); + } + kafkaConsumer.commitAsync(this); + } + + /** + * Shut down this consumer, interrupting the Kafka poll wait + */ + public void shutdown() { + this.running = false; + try { + kafkaConsumer.wakeup(); + } catch (Throwable x) { + logger.warning("Error waking up kafka consumer: " + x.getMessage()); + } + } + @Override public void onComplete(Map offsets, Exception exception) { // called on this consumer's thread, so no need for any synchronization diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java index d6da57d2e6f..385a3d671ec 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java @@ -23,6 +23,7 @@ import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.index.IndexProviderResponse; +import com.ibm.fhir.persistence.index.RemoteIndexConstants; import com.ibm.fhir.persistence.index.RemoteIndexData; import com.ibm.fhir.persistence.index.RemoteIndexMessage; import com.ibm.fhir.server.index.kafka.KafkaPropertyAdapter.Mode; @@ -108,6 +109,7 @@ public IndexProviderResponse submit(final RemoteIndexData data) { // something like "Patient/a-patient-logical-id" final String tenantId = FHIRRequestContext.get().getTenantId(); RemoteIndexMessage msg = new RemoteIndexMessage(); + msg.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); msg.setTenantId(tenantId); msg.setData(data.getSearchParameters()); final String message = marshallToString(msg); diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java index 64f2186f76d..4a389423df1 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java @@ -369,7 +369,7 @@ public FHIRRestOperationResponse doCreatePersist(FHIRPersistenceEvent event, Lis final FHIRPersistenceContext persistenceContext = FHIRPersistenceContextImpl.builder(event) .withOffloadResponse(offloadResponse) - .withShardKey(encodeRequestShardKey(requestContext)) + .withRequestShard(requestContext.getRequestShardKey()) .build(); // For 1869 bundle processing, the resource is updated first and is no longer mutated by the @@ -1151,7 +1151,7 @@ private SingleResourceResult doRead(String type, String id, getInterceptorMgr().fireBeforeReadEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, includeDeleted, searchContext, encodeRequestShardKey(requestContext)); + FHIRPersistenceContextFactory.createPersistenceContext(event, includeDeleted, searchContext, requestContext.getRequestShardKey()); result = persistence.read(persistenceContext, resourceType, id); if (!result.isSuccess() && throwExcOnNull) { throw new FHIRPersistenceResourceNotFoundException("Resource '" + type + "/" + id + "' not found."); @@ -1180,22 +1180,6 @@ private SingleResourceResult doRead(String type, String id, log.exiting(this.getClass().getName(), "doRead"); } } - /** - * Encode the shard key value if it has been set in the request context - * @param context - * @return - */ - private Short encodeRequestShardKey(FHIRRequestContext context) { - Short result = null; - final String requestShardKey = context.getRequestShardKey(); - if (requestShardKey != null) { - result = Short.valueOf((short)requestShardKey.hashCode()); - if (log.isLoggable(Level.FINEST)) { - log.finest("shardKey:[" + requestShardKey + "] hash:[" + result + "]"); - } - } - return result; - } @Override public Resource doVRead(String type, String id, String versionId, MultivaluedMap queryParameters) @@ -1232,7 +1216,7 @@ public Resource doVRead(String type, String id, String versionId, MultivaluedMap getInterceptorMgr().fireBeforeVreadEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, encodeRequestShardKey(requestContext)); + FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, requestContext.getRequestShardKey()); SingleResourceResult srr = persistence.vread(persistenceContext, resourceType, id, versionId); if (!srr.isSuccess() || srr.getResource() == null && !srr.isDeleted()) { throw new FHIRPersistenceResourceNotFoundException("Resource '" @@ -1377,7 +1361,7 @@ public Bundle doSearch(String type, String compartment, String compartmentId, getInterceptorMgr().fireBeforeSearchEvent(event); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, encodeRequestShardKey(requestContext)); + FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, requestContext.getRequestShardKey()); MultiResourceResult searchResult = persistence.search(persistenceContext, resourceType); @@ -3030,7 +3014,7 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r new FHIRPersistenceEvent(null, buildPersistenceEventProperties(resourceType == null ? "Resource" : resourceType, null, null, null, historyContext)); getInterceptorMgr().fireBeforeHistoryEvent(event); // Build a context - FHIRPersistenceContext context = FHIRPersistenceContextImpl.builder(event).withShardKey(encodeRequestShardKey(requestContext)).build(); + FHIRPersistenceContext context = FHIRPersistenceContextImpl.builder(event).withRequestShard(requestContext.getRequestShardKey()).build(); // Start a new txn in the persistence layer if one is not already active. Integer count = historyContext.getCount(); From a8acdb895f1cb7166fd1583a61903cf098112001 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 16 May 2022 00:19:04 +0100 Subject: [PATCH 08/40] issue #3437 remote index wait until server transaction commits Signed-off-by: Robin Arnold --- .../utils/common/ResultSetReader.java | 103 ++++++++++++ .../jdbc/citus/CitusResourceDAO.java | 2 +- .../connection/FHIRDbConnectionStrategy.java | 2 +- .../FHIRDbConnectionStrategyBase.java | 2 +- .../jdbc/connection/FHIRDbHelper.java | 2 +- ...RDbTenantDatasourceConnectionStrategy.java | 2 +- .../FHIRDbTestConnectionStrategy.java | 2 +- .../FHIRTestTransactionAdapter.java | 2 +- .../FHIRUserTransactionAdapter.java | 2 +- .../jdbc/dao/api/CodeSystemDAO.java | 2 +- .../jdbc/dao/api/ParameterDAO.java | 2 +- .../jdbc/dao/api/ParameterNameDAO.java | 2 +- .../persistence/jdbc/dao/api/ResourceDAO.java | 2 +- .../jdbc/dao/impl/CodeSystemDAOImpl.java | 2 +- .../jdbc/dao/impl/FHIRDbDAOImpl.java | 2 +- .../dao/impl/FetchResourcePayloadsDAO.java | 2 +- .../jdbc/dao/impl/JDBCIdentityCacheImpl.java | 2 +- .../jdbc/dao/impl/ParameterDAOImpl.java | 2 +- .../jdbc/dao/impl/ParameterNameDAOImpl.java | 2 +- .../dao/impl/ParameterVisitorBatchDAO.java | 2 +- .../jdbc/dao/impl/ResourceDAOImpl.java | 2 +- .../jdbc/dao/impl/ResourceReferenceDAO.java | 2 +- .../jdbc/db2/Db2ResourceReferenceDAO.java | 2 +- .../jdbc/derby/DerbyCodeSystemDAO.java | 2 +- .../jdbc/derby/DerbyParameterNamesDAO.java | 2 +- .../jdbc/derby/DerbyResourceDAO.java | 2 +- .../jdbc/derby/DerbyResourceReferenceDAO.java | 2 +- .../FHIRPersistenceFKVException.java | 1 + .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 23 ++- .../SearchParametersTransportAdapter.java | 12 +- .../jdbc/postgres/PostgresCodeSystemDAO.java | 2 +- .../postgres/PostgresParameterNamesDAO.java | 2 +- .../jdbc/postgres/PostgresResourceDAO.java | 2 +- .../postgres/PostgresResourceNoProcDAO.java | 2 +- .../PostgresResourceReferenceDAO.java | 2 +- .../FHIRPersistenceDataAccessException.java | 5 +- .../index/FHIRRemoteIndexService.java | 1 - .../persistence/index/RemoteIndexMessage.java | 12 +- .../index/SearchParametersTransport.java | 106 ++++++++++++ .../com/ibm/fhir/remote/index/app/Main.java | 13 +- .../index/database/BaseMessageHandler.java | 158 +++++++++++++++--- .../DistributedPostgresMessageHandler.java | 133 ++++++++++++++- .../DistributedPostgresParameterBatch.java | 2 +- .../database/JDBCBatchParameterProcessor.java | 13 +- .../index/database/LogicalResourceValue.java | 151 +++++++++++++++++ .../ibm/fhir/server/util/FHIRRestHelper.java | 2 +- 46 files changed, 727 insertions(+), 70 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java rename {fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc => fhir-persistence/src/main/java/com/ibm/fhir/persistence}/exception/FHIRPersistenceDataAccessException.java (91%) create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java new file mode 100644 index 00000000000..ec847754aa3 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java @@ -0,0 +1,103 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Calendar; + +/** + * Simplifies reading values from a {@link ResultSet} + */ +public class ResultSetReader { + private final Calendar UTC = CalendarHelper.getCalendarForUTC(); + + // The ResultSet we're reading from + private final ResultSet rs; + + private int index = 0; + + /** + * Canonical constructor + * @param rs + */ + public ResultSetReader(ResultSet rs) { + this.rs = rs; + } + + /** + * Invoke {@link ResultSet#next()} + * @return true if the ResultSet has a row + * @throws SQLException + */ + public boolean next() throws SQLException { + index = 1; + return rs.next(); + } + + /** + * Get a string column value and increment the column index + * @return + * @throws SQLException + */ + public String getString() throws SQLException { + return rs.getString(index++); + } + + /** + * Get a Short column value and increment the column index + * @return + * @throws SQLException + */ + public Short getShort() throws SQLException { + Short result = rs.getShort(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get an Integer column value and increment the column index + * @return + * @throws SQLException + */ + public Integer getInt() throws SQLException { + Integer result = rs.getInt(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Long column value and increment the column index + * @return + * @throws SQLException + */ + public Long getLong() throws SQLException { + Long result = rs.getLong(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Timestamp column value and increment the column index + * @return + * @throws SQLException + */ + public Timestamp getTimestamp() throws SQLException { + Timestamp result = rs.getTimestamp(index++, UTC); + if (rs.wasNull()) { + result = null; + } + return result; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java index 56aac5545f5..94f5e4330d2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -12,12 +12,12 @@ import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceDAO; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java index 71ae4223cf5..3855bff078e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java @@ -9,9 +9,9 @@ import java.sql.Connection; import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.FHIRDbDAOImpl; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Abstraction used to obtain JDBC connections. The database being connected * is determined by the datasource currently referenced by the {@link FHIRRequestContext} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java index f038aae225a..9dabc9bf543 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java @@ -20,8 +20,8 @@ import com.ibm.fhir.config.PropertyGroup; import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.postgresql.SetPostgresOptimizerOptions; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java index 6a97cd6a08c..093089dcee2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java @@ -13,10 +13,10 @@ import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.model.util.FHIRUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBCleanupException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Helper functions used for managing FHIR database interactions diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java index ba1ee36d426..9f73a16a182 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java @@ -23,10 +23,10 @@ import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.exception.FHIRException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.impl.FHIRDbDAOImpl; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java index 6aa5aa7bbf6..5583613a409 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java @@ -12,8 +12,8 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Hides the logic behind obtaining a JDBC {@link Connection} from the DAO code. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java index 65789cca91c..19d7df6eb81 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java @@ -14,8 +14,8 @@ import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.transaction.SimpleTransactionProvider; import com.ibm.fhir.persistence.FHIRPersistenceTransaction; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Hides the logic behind obtaining a JDBC {@link Connection} from the DAO code. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java index ed242e55fd6..318ac3b4100 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java @@ -17,9 +17,9 @@ import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.persistence.FHIRPersistenceTransaction; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.CacheTransactionSync; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java index 6bc1b3b3dd5..05412f3b01c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java @@ -8,8 +8,8 @@ import java.util.Map; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines APIs specific to parameter_names table. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java index 16437517034..c2a24c1f7b5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java @@ -8,9 +8,9 @@ import java.util.Map; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines methods for creating, updating, diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java index 68247ab66c4..36607b0394b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java @@ -8,7 +8,7 @@ import java.util.Map; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines APIs specific to parameter_names table. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java index 34d50abd725..f9b3b8fd72d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java @@ -12,12 +12,12 @@ import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.persistence.context.FHIRPersistenceContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface provides methods creating, updating, and retrieving rows in the FHIR Resource tables. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java index bc0916f5a8a..0eb1b8983dc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java @@ -16,8 +16,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.CodeSystemDAO; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This DAO uses a connection provided to its constructor. It's therefore diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java index faa10d7ad16..0b741a5fdda 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java @@ -25,13 +25,13 @@ import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.model.util.FHIRUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDbDAO; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBCleanupException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This class is a root Data Access Object for managing JDBC access to the FHIR database. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java index ff67d2317f7..d3526c2dc53 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java @@ -24,8 +24,8 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.ResourcePayload; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * DAO to fetch resource ids using a time range and optional current resource id as a filter. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java index 61ce7545ba5..e81f8974b2a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java @@ -16,6 +16,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; @@ -24,7 +25,6 @@ import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java index 189c82b44ca..ca7908b81e5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java @@ -12,6 +12,7 @@ import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.CodeSystemDAO; @@ -20,7 +21,6 @@ import com.ibm.fhir.persistence.jdbc.derby.DerbyCodeSystemDAO; import com.ibm.fhir.persistence.jdbc.derby.DerbyParameterNamesDAO; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.postgres.PostgresCodeSystemDAO; import com.ibm.fhir.persistence.jdbc.postgres.PostgresParameterNamesDAO; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java index 952d81ba666..443f9ba5d58 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java @@ -16,8 +16,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Database interaction for parameter_names. Caching etc is handled diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index 0aa713b84e7..5647e32ab51 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -25,6 +25,7 @@ import com.ibm.fhir.config.FHIRConfigHelper; import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; @@ -39,7 +40,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.schema.control.FhirSchemaConstants; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index d1420edec3b..65dde69f7de 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -35,6 +35,7 @@ import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.persistence.InteractionStatus; import com.ibm.fhir.persistence.context.FHIRPersistenceContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; @@ -47,7 +48,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.util.InputOutputByteStream; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 3abca7a9001..475df62f6f7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -27,6 +27,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; @@ -34,7 +35,6 @@ import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.schema.control.FhirSchemaConstants; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index 15f737af91e..ecb16929ec6 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -18,6 +18,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; @@ -28,7 +29,6 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java index 831f084e3b6..33ec752c1f6 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java @@ -11,9 +11,9 @@ import java.sql.ResultSet; import java.sql.SQLException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.FhirRefSequenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.CodeSystemDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Derby variant DAO used to manage code_systems records. Uses diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java index 996965a81f3..450772adb2f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java @@ -11,9 +11,9 @@ import java.sql.ResultSet; import java.sql.SQLException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.FhirRefSequenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * For R4 we have replaced the old Derby (Java) stored procedure with diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index 77478df554b..980ca60195a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -28,6 +28,7 @@ import com.ibm.fhir.database.utils.derby.DerbyMaster; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceDeletedException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; @@ -43,7 +44,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java index 4eada2d250a..9a256dfc2a8 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; @@ -33,7 +34,6 @@ import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceReferenceDAO; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java index f2cfe4275a5..6dc29ec453f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java @@ -9,6 +9,7 @@ import java.util.Collection; import com.ibm.fhir.model.resource.OperationOutcome; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; /** * This exception class is thrown when Foreign Key violations are encountered while attempting to access data in the FHIR DB. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index ace21a63db7..12317282d4b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -102,6 +102,7 @@ import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.context.FHIRPersistenceContextFactory; import com.ibm.fhir.persistence.erase.EraseDTO; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceDeletedException; @@ -152,7 +153,6 @@ import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.JDBCParameterBuildingVisitor; @@ -421,7 +421,8 @@ public SingleResourceResult create(FHIRPersistenceContex + ", version=" + resourceDTO.getVersionId()); } - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getRequestShard(), searchParameters); + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -461,15 +462,19 @@ public SingleResourceResult create(FHIRPersistenceContex * @param resourceType * @param logicalId * @param logicalResourceId - * @param shardKey + * @param versionId + * @param lastUpdated + * @param requestShard * @param searchParameters */ - private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, String requestShard, + private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, + int versionId, java.time.Instant lastUpdated, String requestShard, ExtractedSearchParameters searchParameters) throws FHIRPersistenceException { FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); if (remoteIndexService != null) { // convert the parameters into a form that will be easy to ship to a remote service - SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, requestShard); + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, + versionId, lastUpdated, requestShard, searchParameters.getParameterHashB64()); ParameterTransportVisitor visitor = new ParameterTransportVisitor(adapter); for (ExtractedParameterValue pv: searchParameters.getParameters()) { pv.accept(visitor); @@ -478,8 +483,8 @@ private void sendParametersToRemoteIndexService(String resourceType, String logi // Note that the remote index service is supposed to be multi-tenant, using // the tenantId from the request context on this thread, so we don't need // to pass that here - final String partitionKey = resourceType + "/" + logicalId; - IndexProviderResponse ipr = remoteIndexService.submit(new RemoteIndexData(partitionKey, adapter.build())); + final String kafkaPartitionKey = resourceType + "/" + logicalId; + IndexProviderResponse ipr = remoteIndexService.submit(new RemoteIndexData(kafkaPartitionKey, adapter.build())); remoteIndexMessageList.add(ipr); // we'll check for an ACK just before we commit the transaction } } @@ -672,7 +677,8 @@ public SingleResourceResult update(FHIRPersistenceContex } // If configured, send the extracted parameters to the remote indexing service - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), context.getRequestShard(), searchParameters); + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) @@ -2862,6 +2868,7 @@ private void transactionCompleted(Boolean committed) { // important to clear this list after each transaction because batch bundles // use the same FHIRPersistenceJDBCImpl instance for each entry + remoteIndexMessageList.clear(); payloadPersistenceResponses.clear(); eraseResourceRecs.clear(); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java index c2fa972ff8a..c982a072c41 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java @@ -8,6 +8,7 @@ import java.math.BigDecimal; import java.sql.Timestamp; +import java.time.Instant; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; @@ -39,14 +40,21 @@ public class SearchParametersTransportAdapter implements ParameterValueVisitorAd * @param resourceType * @param logicalId * @param logicalResourceId + * @param versionId + * @param lastUpdated * @param requestShard + * @param parameterHash */ - public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, String requestShard) { + public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, + int versionId, Instant lastUpdated, String requestShard, String parameterHash) { builder = SearchParametersTransport.builder() .withResourceType(resourceType) .withLogicalId(logicalId) .withLogicalResourceId(logicalResourceId) - .withRequestShard(requestShard); + .withVersionId(versionId) + .withLastUpdated(lastUpdated) + .withRequestShard(requestShard) + .withParameterHash(parameterHash); } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java index 3dfc8f7f7c0..01b4019f598 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java @@ -12,8 +12,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.CodeSystemDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * PostgreSql variant DAO used to manage code_systems records. Uses diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java index ffa9161402d..35d1d9a4916 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java @@ -12,8 +12,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; public class PostgresParameterNamesDAO extends ParameterNameDAOImpl { private static final String CLASSNAME = PostgresParameterNamesDAO.class.getName(); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index c5f160fe19e..3ec8b0c441b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -25,6 +25,7 @@ import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; @@ -40,7 +41,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java index 5e24825a8ef..34fde9eeb23 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java @@ -27,6 +27,7 @@ import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; @@ -42,7 +43,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java index 6a3ccf2746b..c5c578fb614 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java @@ -14,13 +14,13 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Postgres-specific extension of the {@link ResourceReferenceDAO} to work around diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java similarity index 91% rename from fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java rename to fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java index c97e5fa5fcc..c59849c0cb7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java @@ -1,15 +1,14 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2022 * * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.persistence.jdbc.exception; +package com.ibm.fhir.persistence.exception; import java.util.Collection; import com.ibm.fhir.model.resource.OperationOutcome; -import com.ibm.fhir.persistence.exception.FHIRPersistenceException; /** * This exception class represents failures encountered while attempting to access (read, write) data in the FHIR DB. diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java index b37a5c89d3d..35188481b27 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java @@ -20,7 +20,6 @@ public abstract class FHIRRemoteIndexService { // TODO we should be injecting these services to something like the request context private static FHIRRemoteIndexService serviceInstance; -// private ConcurrentHashMap /** * Initialize the serviceInstance value * @param instance diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java index 7b09b92bc51..8cfbdf248fb 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java @@ -13,7 +13,17 @@ public class RemoteIndexMessage { private String tenantId; private int messageVersion; private SearchParametersTransport data; - + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("tenant["); + result.append(tenantId); + result.append("] "); + result.append(data.toString()); + return result.toString(); + } + /** * @return the tenantId */ diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java index 6fc40b9494b..d4fb995daa3 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java @@ -6,6 +6,7 @@ package com.ibm.fhir.persistence.index; +import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -26,6 +27,15 @@ public class SearchParametersTransport { // The database identifier assigned to this resource private long logicalResourceId; + + // The current version of the resource + private int versionId; + + // The parameter hash computed for this set of parameters + private String parameterHash; + + // The last_updated time in a fixed format for transport + private String lastUpdated; // The key value used for sharding the data when using a distributed database private String requestShard; @@ -48,6 +58,27 @@ public static Builder builder() { return new Builder(); } + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("resourceType["); + result.append(resourceType); + result.append("] "); + result.append("logicalId["); + result.append(logicalId); + result.append("] "); + result.append("versionId["); + result.append(versionId); + result.append("] "); +// result.append("parameterHash["); +// result.append(parameterHash); +// result.append("] "); +// result.append("lastUpdated["); +// result.append(lastUpdated); +// result.append("] "); + return result.toString(); + } + /** * A builder to make it easier to construct a {@link SearchParametersTransport} */ @@ -66,6 +97,9 @@ public static class Builder { private String logicalId; private long logicalResourceId = -1; private String requestShard; + private int versionId; + private String parameterHash; + private String lastUpdated; /** * Set the resourceType @@ -77,6 +111,21 @@ public Builder withResourceType(String resourceType) { return this; } + /** + * Set the parameterHash + * @param hash + * @return + */ + public Builder withParameterHash(String hash) { + this.parameterHash = hash; + return this; + } + + public Builder withLastUpdated(Instant lastUpdated) { + this.lastUpdated = lastUpdated.toString(); + return this; + } + /** * Set the logicalId * @param logicalId @@ -87,6 +136,16 @@ public Builder withLogicalId(String logicalId) { return this; } + /** + * Set the versionId + * @param versionId + * @return + */ + public Builder withVersionId(int versionId) { + this.versionId = versionId; + return this; + } + /** * Set the logicalResourceId * @param logicalResourceId @@ -217,7 +276,10 @@ public SearchParametersTransport build() { result.resourceType = this.resourceType; result.logicalId = this.logicalId; result.logicalResourceId = this.logicalResourceId; + result.setVersionId(this.versionId); result.setRequestShard(this.requestShard); + result.setParameterHash(this.parameterHash); + result.setLastUpdated(this.lastUpdated); if (this.stringValues.size() > 0) { result.stringValues = new ArrayList<>(this.stringValues); @@ -454,4 +516,48 @@ public List getSecurityValues() { public void setSecurityValues(List securityValues) { this.securityValues = securityValues; } + + + /** + * @return the versionId + */ + public int getVersionId() { + return versionId; + } + + + /** + * @param versionId the versionId to set + */ + public void setVersionId(int versionId) { + this.versionId = versionId; + } + + /** + * @return the parameterHash + */ + public String getParameterHash() { + return parameterHash; + } + + /** + * @param parameterHash the parameterHash to set + */ + public void setParameterHash(String parameterHash) { + this.parameterHash = parameterHash; + } + + /** + * @return the lastUpdated + */ + public String getLastUpdated() { + return lastUpdated; + } + + /** + * @param lastUpdated the lastUpdated to set + */ + public void setLastUpdated(String lastUpdated) { + this.lastUpdated = lastUpdated; + } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 013b881d13e..891467bc8c8 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -60,6 +60,10 @@ public class Main { private Duration pollDuration = Duration.ofSeconds(10); private long maxBatchCollectTimeMs = 5000; + // The max time we wait for the database to catch up with what was sent to Kafka + // Must be a little longer than the the Liberty transaction timeout + private long maxReadyTimeMs = 180000; + // the list of consumers private final List consumers = new ArrayList<>(); @@ -121,6 +125,13 @@ public void parseArgs(String[] args) { throw new IllegalArgumentException("Missing value for --consumer-count"); } break; + case "--max-ready-time-ms": + if (a < args.length && !args[a].startsWith("--")) { + maxReadyTimeMs = Long.parseLong(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --max-ready-time-ms"); + } + break; default: throw new IllegalArgumentException("Bad arg: '" + arg + "'"); } @@ -283,7 +294,7 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { try { // Each handler gets a dedicated database connection so we don't have // to deal with contention when grabbing connections from a pool - return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache); + return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); } catch (SQLException x) { throw new FHIRPersistenceException("get connection failed", x); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index 11ee02c0fc6..7e0362e0bf4 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -6,11 +6,15 @@ package com.ibm.fhir.remote.index.database; +import java.security.SecureRandom; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import com.google.gson.Gson; +import com.ibm.fhir.database.utils.thread.ThreadHandler; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; @@ -34,30 +38,75 @@ public abstract class BaseMessageHandler implements IMessageHandler { private final Logger logger = Logger.getLogger(BaseMessageHandler.class.getName()); private static final int MIN_SUPPORTED_MESSAGE_VERSION = 1; + // If we fail 10 times due to deadlocks, then something is seriously wrong + private static final int MAX_TX_ATTEMPTS = 10; + private SecureRandom random = new SecureRandom(); + + private final long maxReadyWaitMs; + /** + * Protected constructor + * @param maxReadyWaitMs the max time in ms to wait for the upstream transaction to make the data ready + */ + protected BaseMessageHandler(long maxReadyWaitMs) { + this.maxReadyWaitMs = maxReadyWaitMs; + } + @Override public void process(List messages) throws FHIRPersistenceException { - try { - startBatch(); - for (String payload: messages) { - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Processing message payload: " + payload); - } - RemoteIndexMessage message = unmarshall(payload); - if (message != null) { - if (message.getMessageVersion() >= MIN_SUPPORTED_MESSAGE_VERSION) { - process(message); - } else { - logger.warning("Message version [" + message.getMessageVersion() + "] not supported, ignoring payload=[" + payload + "]"); - } + List unmarshalled = new ArrayList<>(messages.size()); + for (String payload: messages) { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Processing message payload: " + payload); + } + RemoteIndexMessage message = unmarshall(payload); + if (message != null) { + if (message.getMessageVersion() >= MIN_SUPPORTED_MESSAGE_VERSION) { + unmarshalled.add(message); + } else { + logger.warning("Message version [" + message.getMessageVersion() + "] not supported, ignoring payload=[" + payload + "]"); } } - pushBatch(); - } catch (Throwable t) { - setRollbackOnly(); - throw t; - } finally { - endTransaction(); } + processWithRetry(unmarshalled); + } + + /** + * Process the batch of messages with support for retries in the case + * of a retryable error such as a database deadlock + * + * @param messages + * @throws FHIRPersistenceException + */ + private void processWithRetry(List messages) throws FHIRPersistenceException { + int attempt = 1; + do { + try { + if (attempt > 1) { + // introduce a random delay before we re-attempt to process the batch. This + // may help to avoid subsequent deadlocks if there are multiple transactions + // involved + final long delay = random.nextInt(10) * 1000l; + logger.fine(() -> "Deadlock retry backoff ms: " + delay); + ThreadHandler.safeSleep(delay); + } + startBatch(); + processMessages(messages); + pushBatch(); + + attempt = MAX_TX_ATTEMPTS; // exit our do...while + } catch (FHIRPersistenceDataAccessException x) { + setRollbackOnly(); + // see if this is a retryable error + if (x.isTransactionRetryable() && attempt++ < MAX_TX_ATTEMPTS) { + logger.warning("tx failed, but retry permitted: " + x.getMessage()); + resetBatch(); // clear up any cruft from the previous attempt + } else { + throw x; + } + } finally { + endTransaction(); + } + } while (attempt < MAX_TX_ATTEMPTS); } /** @@ -70,6 +119,12 @@ public void process(List messages) throws FHIRPersistenceException { */ protected abstract void setRollbackOnly(); + /** + * Reset the state of the handler following a failure so that the batch can + * be retried + */ + protected abstract void resetBatch(); + /** * Push any data we've accumulated from processing messages. */ @@ -92,6 +147,71 @@ private RemoteIndexMessage unmarshall(String jsonPayload) { } return null; } + + /** + * Process the list of messages + * @param messages + * @throws FHIRPersistenceException + */ + private void processMessages(List messages) throws FHIRPersistenceException { + // We need to do a quick scan of all the messages to make sure that + // the logical resource records for each already exist. If prepare + // returns false, it means one of two things: + // 1. we received the message before the server transaction committed + // 2. the server transaction failed/rolled back, so we'll never be ready + long timeoutTime = System.nanoTime() + this.maxReadyWaitMs * 1000000; + + // Messages which match the current version info in the database + List okToProcess = new ArrayList<>(); + + // resources which don't yet exist of their version is older than the message + List notReady = new ArrayList<>(); + + // make at least one attempt + do { + if (okToProcess.size() > 0) { + okToProcess.clear(); // reset ready for next prepare call + } + if (notReady.size() > 0) { + notReady.clear(); // reset ready for next prepare call + } + + // Ask the handle to check which messages match the database + // and are therefore ready to be processed + prepare(messages, okToProcess, notReady); + + // If the ready check fails just sleep for a bit because we need + // to wait until the upstream transaction commits. This means we + // may need to keep waiting for a long time which unfortunately + // stalls processing this partition + if (notReady.size() > 0) { + long snoozeMs = Math.min(1000l, (timeoutTime - System.nanoTime()) / 1000000); + // short sleep to wait for the upstream transaction to complete + ThreadHandler.safeSleep(snoozeMs); + } + } while (notReady.size() > 0 && System.nanoTime() < timeoutTime); + + // okToProcess contains those messages for which we see the upstream transaction + // has committed. + for (RemoteIndexMessage message: okToProcess) { + process(message); + } + + // Make a note of which messages we were unable to process because the upstream + // transaction did not commit before our maxReadyWaitMs timeout + for (RemoteIndexMessage message: notReady) { + logger.warning("Timed out waiting for upstream transaction to commit data for: " + message.toString()); + } + } + + /** + * Check to see if the database is ready to process the messages + * @param IN: messages to check + * @param OUT: okToMessages the messages matching the current database + * @param OUT: notReady the messages for which the upstream transaction has yet to commit + */ + protected abstract void prepare(List messages, List okToProcess, List notReady) throws FHIRPersistenceException; + /** * Process the data * @param message diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index 5e4185aaf84..7dcbc907100 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -15,13 +15,16 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.ResultSetReader; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.DateParameter; @@ -29,6 +32,7 @@ import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; import com.ibm.fhir.persistence.index.SearchParameterValue; import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; @@ -104,8 +108,12 @@ public class DistributedPostgresMessageHandler extends BaseMessageHandler { * Public constructor * * @param connection + * @param schemaName + * @param cache + * @param maxReadyTimeMs */ - public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache) { + public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(maxReadyTimeMs); this.connection = connection; this.schemaName = schemaName; this.identityCache = cache; @@ -902,4 +910,127 @@ private Integer createParameterName(String parameterName) throws SQLException { return parameterNameId; } + @Override + protected void resetBatch() { + // Called when a transaction has been rolled back because of a deadlock + // or other retryable error and we want to try and process the batch again + batchProcessor.reset(); + } + + /** + * Build the check ready query + * @param messagesByResourceType + * @return + */ + private String buildCheckReadyQuery(Map> messagesByResourceType) { + StringBuilder select = new StringBuilder(); + // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // patient_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (1,2,3,4) + // UNION ALL + // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // observation_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (5,6,7) + boolean first = true; + for (Map.Entry> entry: messagesByResourceType.entrySet()) { + final String resourceType = entry.getKey(); + final List messages = entry.getValue(); + final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); + if (first) { + first = false; + } else { + select.append(" UNION ALL "); + } + select.append(" SELECT lr.shard_key, lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); + select.append(" FROM logical_resources AS lr, "); + select.append(resourceType).append("_logical_resources AS xlr "); + select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); + select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); + } + + return select.toString(); + } + + @Override + protected void prepare(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { + // Get a list of all the resources for which we can see the current logical resource data. + // If the resource doesn't yet exist or its version meta doesn't the message + // then we add to the notReady list. If the resource version meta already + // exceeds the message, then we'll skip processing altogether because it + // means that there should be another message in the queue with more + // up-to-date parameters + Map messageMap = new HashMap<>(); + Map> messagesByResourceType = new HashMap<>(); + for (RemoteIndexMessage msg: messages) { + Long logicalResourceId = msg.getData().getLogicalResourceId(); + messageMap.put(logicalResourceId, msg); + + // split out the messages per resource type because we need to read from xx_logical_resources + List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); + values.add(msg); + } + + Set found = new HashSet<>(); + final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); + logger.fine(() -> "check ready query: " + checkReadyQuery); + try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { + ResultSet rs = ps.executeQuery(); + // wrap the ResultSet in a reader for easier consumption + ResultSetReader rsReader = new ResultSetReader(rs); + while (rsReader.next()) { + LogicalResourceValue lrv = LogicalResourceValue.builder() + .withShardKey(rsReader.getShort()) + .withLogicalResourceId(rsReader.getLong()) + .withResourceType(rsReader.getString()) + .withLogicalId(rsReader.getString()) + .withVersionId(rsReader.getInt()) + .withLastUpdated(rsReader.getTimestamp()) + .withParameterHash(rsReader.getString()) + .build(); + RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); + if (m == null) { + throw new IllegalStateException("query returned a logical resource which we didn't request"); + } + + // Check the values from the database to see if they match + // the information in the message. + if (m.getData().getVersionId() == lrv.getVersionId()) { + // only process this message if the parameter hash and lastUpdated + // times match - which is a good check that we're storing parameters + // from the correct transaction. If these don't match, we can simply + // say we found the data but don't need to process the message. + final String lastUpdated = lrv.getLastUpdated().toString(); + if (lrv.getParameterHash().equals(m.getData().getParameterHash()) + && lastUpdated.equals(m.getData().getLastUpdated())) { + okToProcess.add(m); + } + found.add(lrv.getLogicalResourceId()); // won't be marked as missing + } else if (m.getData().getVersionId() > lrv.getVersionId()) { + // we can skip processing this record because the database has already + // been updated with a newer version. Identify the record as having been + // found so we don't keep waiting for it + found.add(lrv.getLogicalResourceId()); + } + // if the version in the database is prior to version in the message we + // received it means that the server transaction hasn't been committed... + // so we have to wait just as though it were missing altogether + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); + throw new FHIRPersistenceException("prepare query failed"); + } + + if (found.size() < messages.size()) { + // identify the missing records and add to the notReady list + for (RemoteIndexMessage m: messages) { + if (!found.contains(m.getData().getLogicalResourceId())) { + notReady.add(m); + } + } + } + } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java index ece3b5558fe..feebe7fb947 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java @@ -107,7 +107,7 @@ public void pushBatch() throws SQLException { * Resets the state of the DAO by closing all statements and * setting any batch counts to 0 */ - public void reset() { + public void close() { if (strings != null) { try { strings.close(); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java index 55ccbc8f3ed..de559a51db4 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java @@ -61,7 +61,7 @@ public JDBCBatchParameterProcessor(Connection connection) { */ public void close() { for (Map.Entry entry: daoMap.entrySet()) { - entry.getValue().reset(); + entry.getValue().close(); } systemDao.close(); } @@ -73,6 +73,17 @@ public void startBatch() { resourceTypesInBatch.clear(); } + /** + * Make sure that each statement that may contain data is cleared before we + * retry a batch + */ + public void reset() { + for (String resourceType: resourceTypesInBatch) { + DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + dao.close(); + } + systemDao.close(); + } /** * Push any statements that have been batched but not yet executed * @throws FHIRPersistenceException diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java new file mode 100644 index 00000000000..7491e2f8d79 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java @@ -0,0 +1,151 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Timestamp; + +/** + * A DTO representing a record from logical_resources + */ +public class LogicalResourceValue { + private final short shardKey; + private final long logicalResourceId; + private final String resourceType; + private final String logicalId; + private final int versionId; + private final Timestamp lastUpdated; + private final String parameterHash; + public static class Builder { + private short shardKey; + private long logicalResourceId; + private String resourceType; + private String logicalId; + private int versionId; + private Timestamp lastUpdated; + private String parameterHash; + + public Builder withShardKey(short shardKey) { + this.shardKey = shardKey; + return this; + } + public Builder withLogicalResourceId(long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + return this; + } + public Builder withResourceType(String resourceType) { + this.resourceType = resourceType; + return this; + } + public Builder withLogicalId(String logicalId) { + this.logicalId = logicalId; + return this; + } + public Builder withVersionId(int versionId) { + this.versionId = versionId; + return this; + } + public Builder withLastUpdated(Timestamp lastUpdated) { + this.lastUpdated = lastUpdated; + return this; + } + public Builder withParameterHash(String parameterHash) { + this.parameterHash = parameterHash; + return this; + } + + /** + * Create a new {@link LogicalResourceValue} using the current state of this {@link Builder} + * @return + */ + public LogicalResourceValue build() { + return new LogicalResourceValue(shardKey, logicalResourceId, resourceType, logicalId, versionId, lastUpdated, parameterHash); + } + } + + /** + * Factor function to create a fresh instance of a {@link Builder} + * @return + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Public constructor + * @param shardKey + * @param logicalResourceId + * @param resourceType + * @param logicalId + * @param lastUpdated + * @param parameterHash + */ + public LogicalResourceValue(short shardKey, long logicalResourceId, String resourceType, String logicalId, int versionId, Timestamp lastUpdated, String parameterHash) { + this.shardKey = shardKey; + this.logicalResourceId = logicalResourceId; + this.resourceType = resourceType; + this.logicalId = logicalId; + this.versionId = versionId; + this.lastUpdated = lastUpdated; + this.parameterHash = parameterHash; + } + + + /** + * @return the shardKey + */ + public short getShardKey() { + return shardKey; + } + + + /** + * @return the logicalResourceId + */ + public long getLogicalResourceId() { + return logicalResourceId; + } + + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } + + + /** + * @return the lastUpdated + */ + public Timestamp getLastUpdated() { + return lastUpdated; + } + + + /** + * @return the parameterHash + */ + public String getParameterHash() { + return parameterHash; + } + + /** + * @return the versionId + */ + public int getVersionId() { + return versionId; + } + +} diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java index 4a389423df1..c7be5d311cd 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java @@ -107,11 +107,11 @@ import com.ibm.fhir.persistence.context.FHIRSystemHistoryContext; import com.ibm.fhir.persistence.context.impl.FHIRPersistenceContextImpl; import com.ibm.fhir.persistence.erase.EraseDTO; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceIfNoneMatchException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceNotFoundException; import com.ibm.fhir.persistence.helper.FHIRTransactionHelper; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.payload.PayloadPersistenceResponse; import com.ibm.fhir.persistence.util.FHIRPersistenceUtil; import com.ibm.fhir.profile.ProfileSupport; From 7125f67a9527113265dcd566acc20e4ab2f39b27 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Sat, 21 May 2022 00:09:00 +0100 Subject: [PATCH 09/40] issue #3437 add logical_resource_ident and ref param support Signed-off-by: Robin Arnold --- .../database/utils/api/IDatabaseAdapter.java | 9 + .../database/utils/api/ISchemaAdapter.java | 9 + .../fhir/database/utils/api/SchemaType.java | 4 +- .../utils/common/PlainSchemaAdapter.java | 5 + .../fhir/database/utils/db2/Db2Adapter.java | 8 + .../db2/Db2DoesForeignKeyConstraintExist.java | 61 + .../database/utils/derby/DerbyAdapter.java | 8 + .../DerbyDoesForeignKeyConstraintExist.java | 66 + .../utils/model/ForeignKeyConstraint.java | 7 +- .../ibm/fhir/database/utils/model/Table.java | 25 +- .../utils/postgres/PostgresAdapter.java | 9 + .../postgres/PostgresDoesConstraintExist.java | 66 + ...PostgresDoesForeignKeyConstraintExist.java | 14 +- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 1 + .../jdbc/postgres/PostgresResourceDAO.java | 47 +- .../java/com/ibm/fhir/schema/app/Main.java | 201 ++- .../com/ibm/fhir/schema/app/menu/Menu.java | 4 +- .../ibm/fhir/schema/app/util/CommonUtil.java | 9 +- .../build/DistributedSchemaAdapter.java | 115 +- .../schema/build/ShardedSchemaAdapter.java | 125 ++ .../control/FhirResourceTableGroup.java | 96 +- .../schema/control/FhirSchemaConstants.java | 9 +- .../schema/control/FhirSchemaGenerator.java | 149 +- .../schema/control/FhirSchemaVersion.java | 3 +- ...GetLogicalResourceNeedsV0027Migration.java | 53 + .../MigrateV0027LogicalResourceIdent.java | 48 + .../postgres/add_any_resource_distributed.sql | 131 +- ...delete_resource_parameters_distributed.sql | 69 +- .../delete_resource_parameters_sharded.sql | 63 + ...ributed.sql => erase_resource_sharded.sql} | 0 .../app/JavaBatchSchemaGeneratorTest.java | 34 +- .../schema/derby/DerbyFhirDatabaseTest.java | 2 +- .../schema/derby/DerbySchemaVersionsTest.java | 2 +- .../persistence/index/ReferenceParameter.java | 76 + .../index/SearchParametersTransport.java | 41 +- .../SearchParametersTransportAdapter.java | 22 +- .../index/api/BatchParameterProcessor.java | 14 + .../com/ibm/fhir/remote/index/app/Main.java | 22 +- .../index/batch/BatchReferenceParameter.java | 44 + .../index/database/BaseMessageHandler.java | 27 +- .../DistributedPostgresMessageHandler.java | 964 +------------ .../database/LogicalResourceIdentKey.java | 42 + .../database/LogicalResourceIdentValue.java | 118 ++ .../PlainBatchParameterProcessor.java | 312 +++++ .../database/PlainPostgresMessageHandler.java | 1218 +++++++++++++++++ .../database/PlainPostgresParameterBatch.java | 416 ++++++ .../PlainPostgresSystemParameterBatch.java | 216 +++ .../ShardedBatchParameterProcessor.java} | 70 +- .../ShardedPostgresMessageHandler.java | 1088 +++++++++++++++ .../ShardedPostgresParameterBatch.java} | 46 +- .../ShardedPostgresSystemParameterBatch.java} | 6 +- .../index/MessageSerializationTest.java | 65 + .../ibm/fhir/server/util/FHIRRestHelper.java | 1 + 53 files changed, 4862 insertions(+), 1398 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java create mode 100644 fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql rename fhir-persistence-schema/src/main/resources/postgres/{erase_resource_distributed.sql => erase_resource_sharded.sql} (100%) create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java rename {fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl => fhir-persistence/src/main/java/com/ibm/fhir/persistence/index}/SearchParametersTransportAdapter.java (87%) create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java rename fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/{database/JDBCBatchParameterProcessor.java => sharded/ShardedBatchParameterProcessor.java} (79%) create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java rename fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/{database/DistributedPostgresParameterBatch.java => sharded/ShardedPostgresParameterBatch.java} (89%) rename fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/{database/DistributedPostgresSystemParameterBatch.java => sharded/ShardedPostgresSystemParameterBatch.java} (97%) create mode 100644 fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 292f6e56289..27d87d62597 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -565,6 +565,15 @@ public default boolean useSessionVariable() { */ public void enableForeignKey(String schemaName, String tableName, String constraintName); + /** + * Does the named foreign key constraint exist + * @param schemaName + * @param tableName + * @param constraintName + * @return + */ + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName); + /** * * @param schemaName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java index 1e314e8050b..ef537a5f015 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java @@ -542,6 +542,15 @@ public default boolean useSessionVariable() { */ public void enableForeignKey(String schemaName, String tableName, String constraintName); + /** + * Check to see if the named foreign key constraint already exists + * @param schemaName + * @param tableName + * @param constraintName + * @return + */ + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName); + /** * * @param schemaName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java index bd85c6d7e6c..39fce034ae2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java @@ -12,9 +12,11 @@ * PLAIN - the schema we typically deploy to Derby or PostgreSQL * MULTITENANT - on Db2 supporting multiple tenants using partitioning and RBAC * DISTRIBUTED - for use with distributed technologies like Citus DB + * SHARDED - explicitly sharded using an injected shard_key column */ public enum SchemaType { PLAIN, MULTITENANT, - DISTRIBUTED + DISTRIBUTED, + SHARDED } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java index e2690e72516..8bfdb632f12 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java @@ -319,6 +319,11 @@ public void enableForeignKey(String schemaName, String tableName, String constra databaseAdapter.enableForeignKey(schemaName, tableName, constraintName); } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + return databaseAdapter.doesForeignKeyConstraintExist(schemaName, tableName, constraintName); + } + @Override public void setIntegrityOff(String schemaName, String tableName) { databaseAdapter.setIntegrityOff(schemaName, tableName); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java index 95549cbd386..7491b9774f7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java @@ -606,6 +606,14 @@ public void enableForeignKey(String schemaName, String tableName, String constra runStatement(ddl); } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + Db2DoesForeignKeyConstraintExist test = new Db2DoesForeignKeyConstraintExist(schemaName, tableName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(test); + return val != null && val.booleanValue(); + } + /* (non-Javadoc) * @see com.ibm.fhir.database.utils.api.IDatabaseAdapter#setIntegrityOff(java.lang.String, java.lang.String) */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java new file mode 100644 index 00000000000..e0ee134188c --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java @@ -0,0 +1,61 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.db2; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Check the Db2 catalog to see if the configured constraint exists + */ +public class Db2DoesForeignKeyConstraintExist implements IDatabaseSupplier { + + // The constraint identity + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + */ + public Db2DoesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toUpperCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toUpperCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toUpperCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + // Grab the list of tables for the configured schema from the DB2 catalog + final String sql = "" + + "SELECT 1 FROM SYSCAT.REFERENCES " + + " WHERE tabschema = ? " + + " AND tabname = ? " + + " AND constname = ? "; + + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index 33790279c78..15f1058642a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -310,6 +310,14 @@ public void createForeignKeyConstraint(String constraintName, String schemaName, } } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + DerbyDoesForeignKeyConstraintExist test = new DerbyDoesForeignKeyConstraintExist(schemaName, tableName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(test); + return val != null && val.booleanValue(); + } + @Override protected List prefixTenantColumn(String tenantColumnName, List columns) { // No tenant support, so simply return the columns list unchanged, without prefixing diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java new file mode 100644 index 00000000000..82f84fc2407 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java @@ -0,0 +1,66 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.derby; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Inspect the Derby catalog to see if the configured constraint exists + */ +public class DerbyDoesForeignKeyConstraintExist implements IDatabaseSupplier { + + // The constraint identity + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + */ + public DerbyDoesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toUpperCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toUpperCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toUpperCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + // Check the catalog to see if the named constraint exists + final String sql = "" + + "SELECT 1 " + + " FROM sys.sysschemas s," + + " sys.sysconstraints c," + + " sys.systables t " + + " WHERE t.schemaid = s.schemaid " + + " AND c.tableid = t.tableid " + + " AND s.schemaname = ? " + + " AND t.tablename = ? " + + " AND c.constraintname = ? "; + + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java index 3975c3a4f88..68ca87906ae 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java @@ -133,8 +133,11 @@ public String getQualifiedTargetName() { * @param sourceDistributionType */ public void apply(String schemaName, String name, String tenantColumnName, ISchemaAdapter target, DistributionType sourceDistributionType) { - target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced, sourceDistributionType, - targetIsReference); + // make this idempotent to support upgrade scenarios + if (!target.doesForeignKeyConstraintExist(schemaName, name, getConstraintName())) { + target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced, sourceDistributionType, + targetIsReference); + } } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index 8a2393f0cbb..48b609eb965 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -52,6 +52,9 @@ public class Table extends BaseObject { // The rules to distribute the table in a distributed RDBMS implementation (Citus) private final DistributionType distributionType; + // If set, overrides the column used to distribute the data in a sharded database + private final String distributionColumnName; + // The With parameters on the table private final List withs; @@ -78,13 +81,14 @@ public class Table extends BaseObject { * @param withs * @param checkConstraints * @param distributionType + * @param distributionColumnName */ public Table(String schemaName, String name, int version, String tenantColumnName, Collection columns, PrimaryKeyDef pk, IdentityDef identity, Collection indexes, Collection fkConstraints, SessionVariableDef accessControlVar, Tablespace tablespace, List dependencies, Map tags, Collection privileges, List migrations, List withs, List checkConstraints, - DistributionType distributionType) { + DistributionType distributionType, String distributionColumnName) { super(schemaName, name, DatabaseObjectType.TABLE, version, migrations); this.tenantColumnName = tenantColumnName; this.columns.addAll(columns); @@ -97,6 +101,7 @@ public Table(String schemaName, String name, int version, String tenantColumnNam this.withs = withs; this.checkConstraints.addAll(checkConstraints); this.distributionType = distributionType; + this.distributionColumnName = distributionColumnName; // Adds all dependencies which aren't null. // The only circumstances where it is null is when it is self referencial (an FK on itself). @@ -264,6 +269,9 @@ public static class Builder extends VersionedSchemaObject { // The type of distribution to use for this table when using a distributed database private DistributionType distributionType = DistributionType.NONE; + // Allows the standard distribution column to be overridden + private String distributionColumnName; + /** * Private constructor to force creation through factory method * @param schemaName @@ -294,7 +302,7 @@ public Builder setTablespace(Tablespace ts) { } /** - * Setter for the distributionColumnName + * Setter for the distributionType * @param cn * @return */ @@ -303,6 +311,16 @@ public Builder setDistributionType(DistributionType dt) { return this; } + /** + * Setter for the distributionColumnName value + * @param columnName + * @return + */ + public Builder setDistributionColumnName(String columnName) { + this.distributionColumnName = columnName; + return this; + } + public Builder addIntColumn(String columnName, boolean nullable) { ColumnDef cd = new ColumnDef(columnName); if (columns.contains(cd)) { @@ -807,7 +825,8 @@ public Table build(IDataModel dataModel) { // Our schema objects are immutable by design, so all initialization takes place // through the constructor return new Table(getSchemaName(), getObjectName(), this.version, this.tenantColumnName, buildColumns(), this.primaryKey, this.identity, this.indexes.values(), - enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionType); + enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionType, + distributionColumnName); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index 1c2df6d87d6..b1ba13d5431 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -385,6 +385,15 @@ public void enableForeignKey(String schemaName, String tableName, String constra throw new UnsupportedOperationException("Disable FK currently not supported for this adapter."); } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + // check the catalog to see if the named constraint exists + PostgresDoesForeignKeyConstraintExist fkExists = new PostgresDoesForeignKeyConstraintExist(schemaName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(fkExists); + return val != null && val.booleanValue(); + } + @Override public void setIntegrityOff(String schemaName, String tableName) { // not expecting this to be called for this adapter diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java new file mode 100644 index 00000000000..dc2c2203fcd --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java @@ -0,0 +1,66 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.postgres; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * PostgreSQL catalog query to determine if the named constraint exists for the given + * schema and table + */ +public class PostgresDoesConstraintExist implements IDatabaseSupplier { + + // Identity of the constraint + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + * @param tableName + * @param constraintName + */ + public PostgresDoesConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toLowerCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toLowerCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toLowerCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + final String SQL = "" + + "SELECT 1 FROM " + + " pg_catalog.pg_constraint con " + + " JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid " + + " JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace " + + " WHERE nsp.nspname = ? " + + " AND rel.relname = ? " + + " AND con.conname = ? "; + + try (PreparedStatement ps = c.prepareStatement(SQL)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java index 7e578671e30..f29f4c340db 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java @@ -31,16 +31,16 @@ public class PostgresDoesForeignKeyConstraintExist implements IDatabaseSupplier< * @param schemaName */ public PostgresDoesForeignKeyConstraintExist(String schemaName, String constraintName) { - this.schemaName = DataDefinitionUtil.assertValidName(schemaName); - this.constraintName = DataDefinitionUtil.assertValidName(constraintName); + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toLowerCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toLowerCase(); } @Override public Boolean run(IDatabaseTranslator translator, Connection c) { - Boolean result = Boolean.FALSE; + Boolean result; final String sql = "" - + "SELECT 1 " - + " FROM pg_constraint " + + "SELECT 1 FROM " + + " pg_constraint " + " WHERE contype = 'f' " + " AND connamespace = ?::regnamespace " + " AND conname = ? "; @@ -49,9 +49,7 @@ public Boolean run(IDatabaseTranslator translator, Connection c) { ps.setString(1, schemaName); ps.setString(2, constraintName); ResultSet rs = ps.executeQuery(); - if(rs.next()) { - result = Boolean.TRUE; - } + result = Boolean.valueOf(rs.next()); } catch (SQLException x) { throw translator.translate(x); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 12317282d4b..d8f00cae064 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -110,6 +110,7 @@ import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.index.IndexProviderResponse; import com.ibm.fhir.persistence.index.RemoteIndexData; +import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.FHIRResourceDAOFactory; import com.ibm.fhir.persistence.jdbc.JDBCConstants; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 3ec8b0c441b..114ae79728e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -57,9 +57,9 @@ public class PostgresResourceDAO extends ResourceDAOImpl { // 13 args (9 in, 4 out) private static final String SQL_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; // 14 args (10 in, 4 out) - private static final String SQL_DISTRIBUTED_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; + private static final String SQL_SHARDED_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; - private static final String SQL_DISTRIBUTED_READ = "" + private static final String SQL_SHARDED_READ = "" + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + " FROM %s_RESOURCES R, " @@ -70,7 +70,7 @@ public class PostgresResourceDAO extends ResourceDAOImpl { + " AND R.SHARD_KEY = LR.SHARD_KEY "; // Read a specific version of the resource - private static final String SQL_DISTRIBUTED_VERSION_READ = "" + private static final String SQL_SHARDED_VERSION_READ = "" + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + " FROM %s_RESOURCES R, " @@ -84,7 +84,7 @@ public class PostgresResourceDAO extends ResourceDAOImpl { // DAO used to obtain sequence values from FHIR_REF_SEQUENCE private FhirRefSequenceDAO fhirRefSequenceDAO; - // The shard key used with distributed databases + // The (optional) shard key used with sharded databases private final Short shardKey; public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, Short shardKey) { @@ -117,17 +117,17 @@ public Resource insert(Resource resource, List paramete // hit the procedure Objects.requireNonNull(getResourceTypeId(resource.getResourceType())); - if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { if (this.shardKey == null) { - throw new FHIRPersistenceException("Shard key value required when schema type is DISTRIBUTED"); + throw new FHIRPersistenceException("Shard key value required when schema type is SHARDED"); } - stmtString = String.format(SQL_DISTRIBUTED_INSERT_WITH_PARAMETERS, getSchemaName()); + stmtString = String.format(SQL_SHARDED_INSERT_WITH_PARAMETERS, getSchemaName()); } else { stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); } stmt = connection.prepareCall(stmtString); int arg = 1; - if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { stmt.setShort(arg++, shardKey); } stmt.setString(arg++, resource.getResourceType()); @@ -171,10 +171,10 @@ public Resource insert(Resource resource, List paramete // To keep things simple for the postgresql use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - // For now we bypass parameter work for DISTRIBUTED schemas because the plan - // is to make loading async for better ingestion performance + // For now we bypass parameter work for DISTRIBUTED or SHARDED schemas + // because the plan is to make loading async for better ingestion performance final String currentParameterHash = stmt.getString(oldParameterHashIndex); - if (getFlavor().getSchemaType() != SchemaType.DISTRIBUTED + if (getFlavor().getSchemaType() == SchemaType.PLAIN && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() || !parameterHashB64.equals(currentParameterHash))) { // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: @@ -219,12 +219,12 @@ public Resource read(String logicalId, String resourceType) throws FHIRPersisten logger.entering(CLASSNAME, METHODNAME); Resource resource = null; - if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { List resources; String stmtString = null; try { - stmtString = String.format(SQL_DISTRIBUTED_READ, resourceType, resourceType); + stmtString = String.format(SQL_SHARDED_READ, resourceType, resourceType); resources = this.runQuery(stmtString, shardKey, logicalId); if (!resources.isEmpty()) { resource = resources.get(0); @@ -244,11 +244,11 @@ public Resource versionRead(String logicalId, String resourceType, int versionId logger.entering(CLASSNAME, METHODNAME); Resource resource = null; - if (getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { String stmtString = null; try { - stmtString = String.format(SQL_DISTRIBUTED_VERSION_READ, resourceType, resourceType); + stmtString = String.format(SQL_SHARDED_VERSION_READ, resourceType, resourceType); List resources = this.runQuery(stmtString, shardKey, logicalId, versionId); if (!resources.isEmpty()) { resource = resources.get(0); @@ -263,23 +263,6 @@ public Resource versionRead(String logicalId, String resourceType, int versionId } - /** - * Delete all parameters for the given resourceId from the parameters table - * - * @param conn - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - private void deleteFromParameterTableX(Connection conn, String tableName, long logicalResourceId) throws SQLException { - final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - stmt.executeUpdate(); - } - } - /** * Read the id for the named type * @param resourceTypeName diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index cec3c1ffce1..3a883b6efb8 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -16,7 +16,6 @@ import static com.ibm.fhir.schema.app.menu.Menu.CREATE_SCHEMA_OAUTH; import static com.ibm.fhir.schema.app.menu.Menu.DB_TYPE; import static com.ibm.fhir.schema.app.menu.Menu.DELETE_TENANT_META; -import static com.ibm.fhir.schema.app.menu.Menu.DISTRIBUTED; import static com.ibm.fhir.schema.app.menu.Menu.DROP_ADMIN; import static com.ibm.fhir.schema.app.menu.Menu.DROP_DETACHED; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA; @@ -38,6 +37,7 @@ import static com.ibm.fhir.schema.app.menu.Menu.REVOKE_ALL_TENANT_KEYS; import static com.ibm.fhir.schema.app.menu.Menu.REVOKE_TENANT_KEY; import static com.ibm.fhir.schema.app.menu.Menu.SCHEMA_NAME; +import static com.ibm.fhir.schema.app.menu.Menu.SCHEMA_TYPE; import static com.ibm.fhir.schema.app.menu.Menu.SHOW_DB_SIZE; import static com.ibm.fhir.schema.app.menu.Menu.SHOW_DB_SIZE_DETAIL; import static com.ibm.fhir.schema.app.menu.Menu.SKIP_ALLOCATE_IF_TENANT_EXISTS; @@ -153,6 +153,7 @@ import com.ibm.fhir.schema.control.FhirSchemaGenerator; import com.ibm.fhir.schema.control.FhirSchemaVersion; import com.ibm.fhir.schema.control.GetLogicalResourceNeedsV0014Migration; +import com.ibm.fhir.schema.control.GetLogicalResourceNeedsV0027Migration; import com.ibm.fhir.schema.control.GetResourceChangeLogEmpty; import com.ibm.fhir.schema.control.GetResourceTypeList; import com.ibm.fhir.schema.control.GetTenantInfo; @@ -162,6 +163,7 @@ import com.ibm.fhir.schema.control.JavaBatchSchemaGenerator; import com.ibm.fhir.schema.control.MigrateV0014LogicalResourceIsDeletedLastUpdated; import com.ibm.fhir.schema.control.MigrateV0021AbstractTypeRemoval; +import com.ibm.fhir.schema.control.MigrateV0027LogicalResourceIdent; import com.ibm.fhir.schema.control.OAuthSchemaGenerator; import com.ibm.fhir.schema.control.PopulateParameterNames; import com.ibm.fhir.schema.control.PopulateResourceTypes; @@ -315,8 +317,8 @@ public class Main { // Configuration to control how the LeaseManager operates private ILeaseManagerConfig leaseManagerConfig; - // Do we want to build the distributed flavor of the FHIR data schema? - private boolean distributed = false; + // Which flavor of the FHIR data schema should we build? + private SchemaType dataSchemaType; // ----------------------------------------------------------------------------------------------------------------- // The following method is related to the common methods and functions @@ -362,8 +364,8 @@ protected void configureConnectionPool() { */ protected void buildAdminSchemaModel(PhysicalDataModel pdm) { // Add the tenant and tenant_keys tables and any other admin schema stuff - SchemaType schemaType = isMultitenant() ? SchemaType.MULTITENANT : SchemaType.PLAIN; - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), schemaType); + SchemaType adminSchemaType = isMultitenant() ? SchemaType.MULTITENANT : SchemaType.PLAIN; + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), adminSchemaType); gen.buildAdminSchema(pdm); } @@ -397,10 +399,11 @@ protected void buildJavaBatchSchemaModel(PhysicalDataModel pdm) { * @param collector * @param vhs */ - protected void applyModel(PhysicalDataModel pdm, ISchemaAdapter adapter, ITaskCollector collector, VersionHistoryService vhs) { + protected void applyModel(PhysicalDataModel pdm, ISchemaAdapter adapter, ITaskCollector collector, VersionHistoryService vhs, SchemaType schemaType) { logger.info("Collecting model update tasks"); - // If using a distributed RDBMS (Citus) then skip the initial FK creation - SchemaApplyContext context = SchemaApplyContext.builder().setIncludeForeignKeys(!isDistributed()).build(); + // If using a distributed RDBMS (like Citus) then skip the initial FK creation + final boolean includeForeignKeys = schemaType != SchemaType.DISTRIBUTED; + SchemaApplyContext context = SchemaApplyContext.builder().setIncludeForeignKeys(includeForeignKeys).build(); pdm.collect(collector, adapter, context, this.transactionProvider, vhs); // FHIR in the hole! @@ -512,9 +515,9 @@ protected void updateSchemas() { protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { FhirSchemaGenerator gen; if (resourceTypeSubset == null || resourceTypeSubset.isEmpty()) { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType()); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType()); } else { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType(), resourceTypeSubset); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType(), resourceTypeSubset); } gen.buildSchema(pdm); @@ -559,18 +562,18 @@ protected void updateFhirSchema() { FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { if (this.dbType == DbType.CITUS) { - // First version with Citus support is V0026 and we can't upgrade + // First version with Citus support is V0027 and we can't upgrade // from before that int currentSchemaVersion = svm.getVersionForSchema(); - if (currentSchemaVersion >= 0 && currentSchemaVersion < FhirSchemaVersion.V0026.vid()) { - throw new IllegalStateException("Cannot upgrade Citus databases with schema version < V0026"); + if (currentSchemaVersion >= 0 && currentSchemaVersion < FhirSchemaVersion.V0027.vid()) { + throw new IllegalStateException("Cannot upgrade Citus databases with schema version < V0027"); } } // Build/update the FHIR-related tables as well as the stored procedures PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); - boolean isNewDb = updateSchema(pdm, getSchemaType()); + boolean isNewDb = updateSchema(pdm, getDataSchemaType()); if (this.exitStatus == EXIT_OK) { // If the db is multi-tenant, we populate the resource types and parameter names in allocate-tenant. @@ -596,6 +599,9 @@ protected void updateFhirSchema() { // V0021 removes Abstract Type tables which are unused. applyTableRemovalForV0021(); + // V0027 populate the new LOGICAL_RESOURCE_IDENT table + applyDataMigrationForV0027(); + // Apply privileges if asked if (grantTo != null) { grantPrivilegesForFhirData(); @@ -744,10 +750,9 @@ protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) { boolean isNewDb = vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == null || vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == 0; - applyModel(pdm, schemaAdapter, collector, vhs); - if (isDistributed()) { - applyDistributionRules(pdm); - } + applyModel(pdm, schemaAdapter, collector, vhs, schemaType); + applyDistributionRules(pdm, schemaType); + // The physical database objects should now match what was defined in the PhysicalDataModel return isNewDb; @@ -758,31 +763,35 @@ protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) { * FK constraints that are needed * @param pdm */ - private void applyDistributionRules(PhysicalDataModel pdm) { - try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { - try { - ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); - pdm.applyDistributionRules(schemaAdapter); - } catch (RuntimeException x) { - tx.setRollbackOnly(); - throw x; + private void applyDistributionRules(PhysicalDataModel pdm, SchemaType schemaType) { + if (dbType == DbType.CITUS) { + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + pdm.applyDistributionRules(schemaAdapter); + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } } } - // Now that all the tables have been distributed, it should be safe - // to apply the FK constraints - try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { - try { - final String tenantColumnName = isMultitenant() ? "mt_id" : null; - ISchemaAdapter adapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); - AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName); - pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); - } catch (RuntimeException x) { - tx.setRollbackOnly(); - throw x; + final boolean includeForeignKeys = schemaType != SchemaType.DISTRIBUTED; + if (!includeForeignKeys) { + // Now that all the tables have been distributed, it should be safe + // to apply the FK constraints + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + final String tenantColumnName = isMultitenant() ? "mt_id" : null; + ISchemaAdapter adapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName); + pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } } - } - + } } /** @@ -869,7 +878,7 @@ protected void dropSchema() { try { JdbcTarget target = new JdbcTarget(c); IDatabaseAdapter adapter = getDbAdapter(dbType, target); - ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), adapter); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), adapter); ISchemaAdapter plainSchemaAdapter = getSchemaAdapter(SchemaType.PLAIN, adapter); VersionHistoryService vhs = new VersionHistoryService(schema.getAdminSchemaName(), schema.getSchemaName(), schema.getOauthSchemaName(), schema.getJavaBatchSchemaName()); @@ -882,7 +891,7 @@ protected void dropSchema() { if (this.dropSplitTransaction) { // important that we use an adapter connected with the connection pool // (which is connected to the transaction provider) - ISchemaAdapter poolSchemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); + ISchemaAdapter poolSchemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); pdm.dropSplitTransaction(poolSchemaAdapter, this.transactionProvider, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); } else { // old fashioned drop where we do everything in one (big) transaction @@ -988,7 +997,7 @@ protected void updateProcedures() { try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try (Connection c = connectionPool.getConnection();) { try { - ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); SchemaApplyContext context = SchemaApplyContext.getDefault(); pdm.applyProcedures(schemaAdapter, context); pdm.applyFunctions(schemaAdapter, context); @@ -1039,7 +1048,7 @@ protected void buildCommonModel(PhysicalDataModel pdm, boolean addFhirDataSchema */ protected void grantPrivilegesForFhirData() { - final ISchemaAdapter schemaAdapter = getSchemaAdapter(getSchemaType(), dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); @@ -1141,17 +1150,11 @@ protected boolean isMultitenant() { } /** - * What type of schema do we want to build? + * What type of FHIR data schema do we want to build? * @return */ - protected SchemaType getSchemaType() { - if (isMultitenant()) { - return SchemaType.MULTITENANT; - } else if (isDistributed()) { - return SchemaType.DISTRIBUTED; - } else { - return SchemaType.PLAIN; - } + protected SchemaType getDataSchemaType() { + return this.dataSchemaType; } // ----------------------------------------------------------------------------------------------------------------- @@ -1335,7 +1338,7 @@ protected void allocateTenant() { } // Build/update the tables as well as the stored procedures - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getSchemaType()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1468,7 +1471,7 @@ protected void refreshTenants() { if (ti.getTenantSchema() != null && (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema()))) { // It's crucial we use the correct schema for each particular tenant, which // is why we have to build the PhysicalDataModel separately for each tenant - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), getSchemaType()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), getDataSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1686,7 +1689,7 @@ protected void dropTenant() { TenantInfo tenantInfo = freezeTenant(); // Build the model of the data (FHIRDATA) schema which is then used to drive the drop - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getSchemaType()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getDataSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -1705,7 +1708,7 @@ protected void dropTenant() { protected void dropDetachedPartitionTables() { TenantInfo tenantInfo = getTenantInfo(); - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getSchemaType()); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getDataSchemaType()); PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); @@ -2158,8 +2161,12 @@ protected void parseArgs(String[] args) { case CONFIRM_DROP: this.confirmDrop = true; break; - case DISTRIBUTED: - this.distributed = true; + case SCHEMA_TYPE: + if (++i < args.length) { + this.dataSchemaType = SchemaType.valueOf(args[i]); + } else { + throw new IllegalArgumentException("Missing value for argument at posn: " + i); + } break; case ALLOCATE_TENANT: if (++i < args.length) { @@ -2227,7 +2234,10 @@ protected void parseArgs(String[] args) { break; case CITUS: translator = new CitusTranslator(); + break; case DB2: + dataSchemaType = SchemaType.MULTITENANT; + break; default: break; } @@ -2350,6 +2360,23 @@ protected void applyDataMigrationForV0014() { } } + protected void applyDataMigrationForV0027() { + if (MULTITENANT_FEATURE_ENABLED.contains(dbType)) { + // Process each tenant one-by-one + List tenants = getTenantList(); + for (TenantInfo ti : tenants) { + + // If no --schema-name override was specified, we process all tenants, otherwise we + // process only tenants which belong to the override schema name + if (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema())) { + dataMigrationForV0027(ti); + } + } + } else { + dataMigrationForV0027(); + } + } + /** * Get the list of resource types to drive resource-by-resource operations * @@ -2507,6 +2534,44 @@ private void dataMigrationForV0014() { } } + private void dataMigrationForV0027(TenantInfo ti) { + // Multi-tenant schema so we know this is Db2: + Db2Adapter adapter = new Db2Adapter(connectionPool); + + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + SetTenantIdDb2 setTenantId = new SetTenantIdDb2(schema.getAdminSchemaName(), ti.getTenantId()); + adapter.runStatement(setTenantId); + + logger.info("V0027 Migration: Populating LOGICAL_RESOURCE_IDENT for tenant['" + + ti.getTenantName() + "' in schema '" + ti.getTenantSchema() + "']"); + + dataMigrationForV0027(adapter, ti.getTenantSchema()); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + + /** + * V0027 migration. Populate LOGICAL_RESOURCE_IDENT from LOGICAL_RESOURCES + */ + private void dataMigrationForV0027() { + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + dataMigrationForV0027(adapter, schema.getSchemaName()); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + /** * only process tables which have not yet had their data migrated. The migration can't be * done as part of the schema change because some tables need a REORG which @@ -2527,6 +2592,20 @@ private void dataMigrationForV0014(IDatabaseAdapter adapter, String schemaName, } } + /** + * If the LOGICAL_RESOURCE_IDENT table is empty, fill it using values from + * LOGICAL_RESOURCES + * @param adapter + * @param schemaName + */ + private void dataMigrationForV0027(IDatabaseAdapter adapter, String schemaName) { + GetLogicalResourceNeedsV0027Migration needsMigrating = new GetLogicalResourceNeedsV0027Migration(schemaName); + if (adapter.runStatement(needsMigrating)) { + MigrateV0027LogicalResourceIdent cmd = new MigrateV0027LogicalResourceIdent(schemaName); + adapter.runStatement(cmd); + } + } + /** * Backfill the RESOURCE_CHANGE_LOG table if it is empty */ @@ -2646,11 +2725,13 @@ public void updateVacuumSettings() { } /** - * Should we build the distributed variant of the FHIR data schema + * Should we build the distributed variant of the FHIR data schema. This + * changes how we need to handle certain unique indexes and foreign key + * constraints. * @return */ private boolean isDistributed() { - return this.distributed; + return dataSchemaType == SchemaType.DISTRIBUTED; } /** diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java index 2390b165898..4dbd5029a71 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java @@ -62,7 +62,7 @@ public class Menu { public static final String HELP = "--help"; public static final String SHOW_DB_SIZE = "--show-db-size"; public static final String SHOW_DB_SIZE_DETAIL = "--show-db-size-detail"; - public static final String DISTRIBUTED = "--distributed"; + public static final String SCHEMA_TYPE = "--schema-type"; public Menu() { // NOP @@ -117,7 +117,7 @@ public enum HelpMenu { MI_CREATE_SCHEMA_FHIR(CREATE_SCHEMA_FHIR, "schemaName", "Create the FHIR Data Schema"), MI_CREATE_SCHEMA_BATCH(CREATE_SCHEMA_BATCH, "schemaName", "Create the Batch Schema"), MI_CREATE_SCHEMA_OAUTH(CREATE_SCHEMA_OAUTH, "schemaName", "Create the OAuth Schema"), - MI_DISTRIBUTED(DISTRIBUTED, "", "Build the distributed variant of the FHIR data schema"), + MI_SCHEMA_TYPE(SCHEMA_TYPE, "", "Which variant of the FHIR data schema to use"), MI_SHOW_DB_SIZE(SHOW_DB_SIZE, "", "Generate report with a breakdown of database size"), MI_SHOW_DB_SIZE_DETAIL(SHOW_DB_SIZE_DETAIL, "", "Include detailed table and index info in size report"); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java index 9042fbf1cde..1da1409fae0 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java @@ -34,6 +34,7 @@ import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; import com.ibm.fhir.schema.build.DistributedSchemaAdapter; import com.ibm.fhir.schema.build.FhirSchemaAdapter; +import com.ibm.fhir.schema.build.ShardedSchemaAdapter; import com.ibm.fhir.schema.control.FhirSchemaConstants; /** @@ -148,7 +149,9 @@ public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, DbType dbTy case MULTITENANT: return new FhirSchemaAdapter(dbAdapter); case DISTRIBUTED: - return new DistributedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + return new DistributedSchemaAdapter(dbAdapter); + case SHARDED: + return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); default: throw new IllegalArgumentException("Unsupported schema type: " + schemaType); } @@ -168,7 +171,9 @@ public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, IDatabaseAd case MULTITENANT: return new FhirSchemaAdapter(dbAdapter); case DISTRIBUTED: - return new DistributedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + return new DistributedSchemaAdapter(dbAdapter); + case SHARDED: + return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); default: throw new IllegalArgumentException("Unsupported schema type: " + schemaType); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java index 6a637d46e33..c59ca84acd1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java @@ -6,120 +6,21 @@ package com.ibm.fhir.schema.build; -import java.util.ArrayList; -import java.util.List; - -import com.ibm.fhir.database.utils.api.DistributionContext; -import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; -import com.ibm.fhir.database.utils.model.CheckConstraint; -import com.ibm.fhir.database.utils.model.ColumnBase; -import com.ibm.fhir.database.utils.model.IdentityDef; -import com.ibm.fhir.database.utils.model.OrderedColumnDef; -import com.ibm.fhir.database.utils.model.PrimaryKeyDef; -import com.ibm.fhir.database.utils.model.SmallIntColumn; -import com.ibm.fhir.database.utils.model.With; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; /** - * Adapter implementation used to build the distributed variant of - * the IBM FHIR Server RDBMS schema. - * - * This schema adds a distribution key column to every table identified as - * distributed. This column is also added to every index and FK relationship - * as needed. We use a smallint (2 bytes) which represents a signed integer - * holding values in the range [-32768, 32767]. This provides sufficient spread, - * assuming we won't be using a database with thousands of nodes. + * Represents an adapter used to build the FHIR schema when + * used with a distributed like Citus */ -public class DistributedSchemaAdapter extends FhirSchemaAdapter { - - // The distribution column to add to each table marked as distributed - final String distributionColumnName; +public class DistributedSchemaAdapter extends PlainSchemaAdapter { /** + * Public constructor + * * @param databaseAdapter */ - public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter, String distributionColumnName) { + public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter) { super(databaseAdapter); - this.distributionColumnName = distributionColumnName; - } - - @Override - public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, - String tablespaceName, List withs, List checkConstraints, DistributionType distributionType) { - // If the table is distributed, we need to inject the distribution column into the columns list. This same - // column will need to be injected into each of the index definitions - List actualColumns = new ArrayList<>(); - if (distributionType == DistributionType.DISTRIBUTED) { - ColumnBase distributionColumn = new SmallIntColumn(distributionColumnName, false, null); - actualColumns.add(distributionColumn); - if (primaryKey != null) { - // we need to alter the primary so it includes the distribution column - // as the last member - List newCols = new ArrayList<>(primaryKey.getColumns()); - newCols.add(distributionColumnName); - primaryKey = new PrimaryKeyDef(primaryKey.getConstraintName(), newCols); - } - } - - actualColumns.addAll(columns); - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); - databaseAdapter.createTable(schemaName, name, tenantColumnName, actualColumns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); - } - - @Override - public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns, DistributionType distributionType) { - - List actualColumns = new ArrayList<>(indexColumns); - if (distributionType == DistributionType.DISTRIBUTED) { - // inject the distribution column into the index definition - actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); - } - - // Create the index using the modified set of index columns - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); - databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, includeColumns, dc); - } - - @Override - public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - DistributionType distributionType) { - - List actualColumns = new ArrayList<>(indexColumns); - if (distributionType == DistributionType.DISTRIBUTED) { - // inject the distribution column into the index definition - actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); - } - - // Create the index using the modified set of index columns - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); - databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, dc); - } - - @Override - public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { - // for non-unique indexes, we don't need to include the distribution column -// List actualColumns = new ArrayList<>(indexColumns); -// if (distributionType == DistributionType.DISTRIBUTED) { -// // inject the distribution column into the index definition -// actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); -// } - - // Create the index using the modified set of index columns - databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); - } - - @Override - public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, - String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { - // If both this and the target table are distributed, we need to add the distributionColumnName - // to the FK relationship definition. If the target is a reference, it won't have the shard_key - // column because the table is fully replicated across all nodes and therefore any FK relationship - // can be based on the original PK definition without the extra sharding column. - List newCols = new ArrayList<>(columns); - if (distributionType == DistributionType.DISTRIBUTED && !targetIsReference) { - newCols.add(distributionColumnName); - } - databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, newCols, enforced); } -} \ No newline at end of file +} diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java new file mode 100644 index 00000000000..cbfedd1b2f6 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java @@ -0,0 +1,125 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.SmallIntColumn; +import com.ibm.fhir.database.utils.model.With; + +/** + * Adapter implementation used to build the distributed variant of + * the IBM FHIR Server RDBMS schema. + * + * This schema adds a distribution key column to every table identified as + * distributed. This column is also added to every index and FK relationship + * as needed. We use a smallint (2 bytes) which represents a signed integer + * holding values in the range [-32768, 32767]. This provides sufficient spread, + * assuming we won't be using a database with thousands of nodes. + */ +public class ShardedSchemaAdapter extends FhirSchemaAdapter { + + // The distribution column to add to each table marked as distributed + final String distributionColumnName; + + /** + * @param databaseAdapter + */ + public ShardedSchemaAdapter(IDatabaseAdapter databaseAdapter, String distributionColumnName) { + super(databaseAdapter); + this.distributionColumnName = distributionColumnName; + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType) { + // If the table is distributed, we need to inject the distribution column into the columns list. This same + // column will need to be injected into each of the index definitions + List actualColumns = new ArrayList<>(); + if (distributionType == DistributionType.DISTRIBUTED) { + ColumnBase distributionColumn = new SmallIntColumn(distributionColumnName, false, null); + actualColumns.add(distributionColumn); + if (primaryKey != null) { + // we need to alter the primary so it includes the distribution column + // as the last member + List newCols = new ArrayList<>(primaryKey.getColumns()); + newCols.add(distributionColumnName); + primaryKey = new PrimaryKeyDef(primaryKey.getConstraintName(), newCols); + } + } + + actualColumns.addAll(columns); + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createTable(schemaName, name, tenantColumnName, actualColumns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, includeColumns, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, dc); + } + + @Override + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { + // for non-unique indexes, we don't need to include the distribution column +// List actualColumns = new ArrayList<>(indexColumns); +// if (distributionType == DistributionType.DISTRIBUTED) { +// // inject the distribution column into the index definition +// actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); +// } + + // Create the index using the modified set of index columns + databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + } + + @Override + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, + String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { + // If both this and the target table are distributed, we need to add the distributionColumnName + // to the FK relationship definition. If the target is a reference, it won't have the shard_key + // column because the table is fully replicated across all nodes and therefore any FK relationship + // can be based on the original PK definition without the extra sharding column. + List newCols = new ArrayList<>(columns); + if (distributionType == DistributionType.DISTRIBUTED && !targetIsReference) { + newCols.add(distributionColumnName); + } + databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, newCols, enforced); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index ecae01396d2..27acf93b74f 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -51,6 +51,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_HIGH; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_LOW; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_LOGICAL_RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VERSION_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_PAYLOAD_KEY; @@ -177,6 +178,7 @@ public ObjectGroup addResourceType(String resourceTypeName) { addQuantityValues(group, tablePrefix); // composites table removed by issue-1683 addResourceTokenRefs(group, tablePrefix); + addRefValues(group, tablePrefix); addTokenValuesView(group, tablePrefix); addProfiles(group, tablePrefix); addTags(group, tablePrefix); @@ -201,9 +203,9 @@ public void addLogicalResources(List group, String prefix) { // We also have a FK constraint pointing back to that table to try and keep // things sensible. Table.Builder builder = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -338,9 +340,9 @@ public void addResources(List group, String prefix) { final String tableName = prefix + _RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( RESOURCE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -423,9 +425,9 @@ public void addStrValues(List group, String prefix) { // Parameters are tied to the logical resource Table tbl = Table.builder(schemaName, tableName) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding // .addBigIntColumn( ROW_ID, false) // Removed by issue-1683 - composites refactor .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( STR_VALUE, msb, true) @@ -513,9 +515,9 @@ public Table addResourceTokenRefs(List group, String prefix) { // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -580,6 +582,48 @@ public Table addResourceTokenRefs(List group, String prefix) { return tbl; } + /** + * Schema V0027 adds a dedicated table for supporting reference values instead of using + * token values. + * @param pdm + * @return + */ + public Table addRefValues(List group, String prefix) { + + final String tableName = prefix + "_REF_VALUES"; + + // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding + .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding + .addIntColumn( PARAMETER_NAME_ID, false) + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addBigIntColumn( REF_LOGICAL_RESOURCE_ID, true) + .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version + .addIntColumn( COMPOSITE_ID, true) + .addIndex(IDX + tableName + "_RFPN", REF_LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID) + .addIndex(IDX + tableName + "_LRPN", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + tableName + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(withs) + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + return statements; + }) + .build(model); + + tbl.addTag(FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + + group.add(tbl); + model.addTable(tbl); + + return tbl; + } + /** * Add the resource-specific profiles table which maps to the normalized URI * values stored in COMMON_CANONICAL_VALUES @@ -593,9 +637,9 @@ public Table addProfiles(List group, String prefix) { // logical_resources (1) ---- (*) patient_profiles (*) ---- (0|1) common_canonical_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn( CANONICAL_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addVarcharColumn( VERSION, VERSION_BYTES, true) @@ -642,9 +686,9 @@ public Table addTags(List group, String prefix) { // logical_resources (1) ---- (*) patient_tags (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -686,9 +730,9 @@ public Table addSecurity(List group, String prefix) { // logical_resources (1) ---- (*) patient_security (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -792,9 +836,9 @@ public void addDateValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addTimestampColumn( DATE_START, true) @@ -859,9 +903,9 @@ public void addNumberValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addDoubleColumn( NUMBER_VALUE, true) @@ -933,9 +977,9 @@ public void addLatLngValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addDoubleColumn( LATITUDE_VALUE, true) @@ -1006,9 +1050,9 @@ public void addQuantityValues(List group, String prefix) { final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( CODE, 255, false) @@ -1065,9 +1109,9 @@ public void addListLogicalResourceItems(List group, String pref final int lib = LOGICAL_ID_BYTES; Table tbl = Table.builder(schemaName, LIST_LOGICAL_RESOURCE_ITEMS) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIntColumn( RESOURCE_TYPE_ID, false) @@ -1109,9 +1153,9 @@ public void addPatientCurrentRefs(List group, String prefix) { // model with a foreign key to avoid order of insertion issues Table tbl = Table.builder(schemaName, PATIENT_CURRENT_REFS) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addVarcharColumn( CURRENT_PROBLEMS_LIST, lib, true) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java index 61e5c33ba17..7bb95967a00 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java @@ -39,6 +39,7 @@ public class FhirSchemaConstants { public static final String FHIR_SEQUENCE = "FHIR_SEQUENCE"; public static final String FHIR_REF_SEQUENCE = "FHIR_REF_SEQUENCE"; + public static final String FHIR_CHANGE_SEQUENCE = "FHIR_CHANGE_SEQUENCE"; public static final String TENANT_SEQUENCE = "TENANT_SEQUENCE"; public static final long FHIR_REF_SEQUENCE_START = 20000; public static final int FHIR_REF_SEQUENCE_CACHE = 1000; @@ -68,7 +69,7 @@ public class FhirSchemaConstants { public static final int FRAGMENT_BYTES = 16; // R4 Logical Resources - public static final String LOGICAL_RESOURCE_SHARDS = "LOGICAL_RESOURCE_SHARDS"; + public static final String LOGICAL_RESOURCE_IDENT = "LOGICAL_RESOURCE_IDENT"; public static final String LOGICAL_RESOURCES = "LOGICAL_RESOURCES"; public static final String REINDEX_TSTAMP = "REINDEX_TSTAMP"; public static final String REINDEX_TXID = "REINDEX_TXID"; @@ -161,7 +162,8 @@ public class FhirSchemaConstants { // Table for Normalization of References (Internal and External) public static final String LOCAL_REFERENCES = "LOCAL_REFERENCES"; - public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID"; + public static final String COMMON_REFERENCE_VALUES = "COMMON_REFERENCE_VALUES"; + public static final String COMMON_REFERENCE_VALUE_ID = "COMMON_REFERENCE_VALUE_ID"; // public static final String EXTERNAL_SYSTEMS = "EXTERNAL_SYSTEMS"; // public static final String EXTERNAL_SYSTEM_ID = "EXTERNAL_SYSTEM_ID"; // public static final String EXTERNAL_SYSTEM_NAME = "EXTERNAL_SYSTEM_NAME"; @@ -181,6 +183,9 @@ public class FhirSchemaConstants { public static final String REF_RESOURCE_TYPE_ID = "REF_RESOURCE_TYPE_ID"; public static final String REF_VERSION_ID = "REF_VERSION_ID"; + // logical_resource_id value used to point to a local reference + public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID"; + // View suffix to overlay the new common_token_values and resource_token_refs tables public static final String TOKEN_VALUES_V = "TOKEN_VALUES_V"; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index d149072491e..5687302faf6 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -24,6 +24,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN; import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_GROUP_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_CHANGE_SEQUENCE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK; @@ -37,9 +38,9 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_COMPARTMENTS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_IDENT; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY; -import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SHARDS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_TOKEN_VALUE_BYTES; @@ -178,6 +179,9 @@ public class FhirSchemaGenerator { // The common sequence used for allocated resource ids private Sequence fhirSequence; + // The sequence for tracking change history in distributed schemas like Citus + private Sequence fhirChangeSequence; + // The sequence used for the reference tables (parameter_names, code_systems etc) private Sequence fhirRefSequence; @@ -392,7 +396,7 @@ public void buildSchema(PhysicalDataModel model) { addCodeSystems(model); addCommonTokenValues(model); addResourceTypes(model); - addLogicalResourceShards(model); + addLogicalResourceIdent(model); addLogicalResources(model); // for system-level parameter search addReferencesSequence(model); addLogicalResourceCompartments(model); @@ -510,15 +514,11 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { final String deleteResourceParametersScript; final String addAnyResourceScript; final String eraseResourceScript; - if (model.isDistributed()) { - deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + "_distributed.sql"; - addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + "_distributed.sql"; - eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + "_distributed.sql"; - } else { - deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql"; - addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + ".sql"; - eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql"; - } + final String schemaTypeSuffix = getSchemaTypeSuffix(); + addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + schemaTypeSuffix; + deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql"; + eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql"; + FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, DELETE_RESOURCE_PARAMETERS, FhirSchemaVersion.V0020.vid(), @@ -542,6 +542,22 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); } + /** + * Get the suffix to select the appropriate procedure/function script + * for the schema type + * @return + */ + private String getSchemaTypeSuffix() { + switch (this.schemaType) { + case DISTRIBUTED: + return "_distributed.sql"; + case SHARDED: + return "_sharded.sql"; + default: + return ".sql"; + } + } + /** * @implNote following the current pattern, which is why all this stuff is replicated * @param model @@ -633,9 +649,9 @@ public void addLogicalResources(PhysicalDataModel pdm) { final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD"; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -721,24 +737,42 @@ public void addLogicalResources(PhysicalDataModel pdm) { } /** - * Adds a table to support sharding of the logical_id when running - * in a distributed RDBMS such as Citus. This table is sharded by - * LOGICAL_ID, which means we can use a primary key of - * {RESOURCE_TYPE_ID, LOGICAL_ID} which is required to ensure - * that we can lock the logical resource to avoid any concurrency - * issues. This is only used for distributed implementations. For - * the standard non-distributed solution, the locking is done - * using LOGICAL_RESOURCES. + * Adds a table to support logical identity management of resources + * when using a distributed RDBMS such as Citus. It represents the + * mapping: + * + * (RESOURCE_TYPE_ID, LOGICAL_ID) -> (LOGICAL_RESOURCE_ID) + * + * LOGICAL_RESOURCE_ID values are assigned from the sequence FHIR_SEQUENCE. + * + * When using Citus (or similar), this table is distributed by LOGICAL_ID, + * which means we can use a primary key of {RESOURCE_TYPE_ID, LOGICAL_ID}. + * This is required to ensure that we can lock the logical resource to + * avoid any concurrency issues. + * + * LOGICAL_RESOURCE_IDENT records are also generated when the tuple + * (RESOURCE_TYPE_ID. LOGICAL_ID) is used as a local resource reference + * value. For example: + * "reference": "Patient/aPatientId" + * will create a new LOGICAL_RESOURCE_IDENT record if the Patient resource + * "aPatientId" has not yet been created. The LOGICAL_RESOURCES record is + * not created until the actual resource is created. + * + * Note that there's no index on LOGICAL_RESOURCE_ID. This is intentional. + * An index is not required because LOGICAL_RESOURCE_ID is never used as an + * access path for this table. + * * @param pdm */ - public void addLogicalResourceShards(PhysicalDataModel pdm) { - final String tableName = LOGICAL_RESOURCE_SHARDS; + private void addLogicalResourceIdent(PhysicalDataModel pdm) { + final String tableName = LOGICAL_RESOURCE_IDENT; final String mtId = isMultitenant() ? MT_ID : null; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // add support for distribution/sharding .setTenantColumnName(mtId) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) + .setDistributionColumnName(LOGICAL_ID) // override distribution column for this table .addIntColumn(RESOURCE_TYPE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -777,9 +811,9 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) { final String tableName = COMMON_CANONICAL_VALUES; final String unqCanonicalUrl = "UNQ_" + tableName + "_URL"; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.REFERENCE) // V0027 support for sharding .addBigIntColumn(CANONICAL_ID, false) .addVarcharColumn(URL, CANONICAL_URL_BYTES, false) .addPrimaryKey(tableName + "_PK", CANONICAL_ID) @@ -816,9 +850,9 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn( CANONICAL_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addVarcharColumn( VERSION, VERSION_BYTES, true) @@ -865,9 +899,9 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -909,9 +943,9 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) { // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) // FK referencing COMMON_CANONICAL_VALUES .addBigIntColumn( LOGICAL_RESOURCE_ID, false) // FK referencing LOGICAL_RESOURCES .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -963,7 +997,7 @@ public void addResourceChangeLog(PhysicalDataModel pdm) { Table tbl = Table.builder(schemaName, tableName) .setTenantColumnName(MT_ID) .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes - .setDistributionType(DistributionType.DISTRIBUTED) + .setDistributionType(DistributionType.NONE) // don't distribute the history log .addBigIntColumn(RESOURCE_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) @@ -1009,9 +1043,9 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { // because it makes it very easy to find the most recent changes to resources associated with // a given patient (for example). Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( COMPARTMENT_NAME_ID, false) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addTimestampColumn(LAST_UPDATED, false) @@ -1056,7 +1090,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) { final int msb = MAX_SEARCH_STRING_BYTES; Table tbl = Table.builder(schemaName, STR_VALUES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( STR_VALUE, msb, true) @@ -1072,7 +1106,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1103,7 +1137,7 @@ public Table addResourceDateValues(PhysicalDataModel model) { final String logicalResourcesTable = LOGICAL_RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addTimestampColumn( DATE_START,6, true) @@ -1118,7 +1152,7 @@ public Table addResourceDateValues(PhysicalDataModel model) { .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) .addWiths(addWiths()) // New Column for V0017 - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion == 1) { @@ -1160,7 +1194,7 @@ resource_type VARCHAR(64) NOT NULL */ protected void addResourceTypes(PhysicalDataModel model) { resourceTypesTable = Table.builder(schemaName, RESOURCE_TYPES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( RESOURCE_TYPE_ID, false) .addVarcharColumn( RESOURCE_TYPE, 64, false) @@ -1169,7 +1203,7 @@ protected void addResourceTypes(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding + .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1239,7 +1273,7 @@ protected void addParameterNames(PhysicalDataModel model) { String[] prfIncludeCols = {PARAMETER_NAME_ID}; parameterNamesTable = Table.builder(schemaName, PARAMETER_NAMES) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( PARAMETER_NAME, 255, false) @@ -1248,7 +1282,7 @@ protected void addParameterNames(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding + .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1277,7 +1311,7 @@ code_system_name VARCHAR(255 OCTETS) NOT NULL */ protected void addCodeSystems(PhysicalDataModel model) { codeSystemsTable = Table.builder(schemaName, CODE_SYSTEMS) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addIntColumn( CODE_SYSTEM_ID, false) .addVarcharColumn(CODE_SYSTEM_NAME, 255, false) @@ -1286,7 +1320,7 @@ protected void addCodeSystems(PhysicalDataModel model) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionType(DistributionType.REFERENCE) // V0026 supporting for sharding + .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding .addMigration(priorVersion -> { List statements = new ArrayList<>(); if (priorVersion < FhirSchemaVersion.V0019.vid()) { @@ -1333,7 +1367,7 @@ protected void addCodeSystems(PhysicalDataModel model) { public void addCommonTokenValues(PhysicalDataModel pdm) { final String tableName = COMMON_TOKEN_VALUES; commonTokenValuesTable = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .addBigIntColumn( COMMON_TOKEN_VALUE_ID, false) .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) @@ -1345,7 +1379,7 @@ public void addCommonTokenValues(PhysicalDataModel pdm) { .setTablespace(fhirTablespace) .addPrivileges(resourceTablePrivileges) .enableAccessControl(this.sessionVariable) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 shard using token_value + .setDistributionType(DistributionType.REFERENCE) // V0027 shard using token_value .addMigration(priorVersion -> { List statements = new ArrayList<>(); // Intentionally a NOP @@ -1376,9 +1410,9 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { // logical_resources (0|1) ---- (*) resource_token_refs Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) // V0026: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setDistributionType(DistributionType.DISTRIBUTED) // V0026 support for sharding + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -1458,7 +1492,7 @@ public void addErasedResources(PhysicalDataModel pdm) { // or resource_id values here, because those records may have // already been deleted by $erase. Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0026.vid()) + .setVersion(FhirSchemaVersion.V0027.vid()) .setTenantColumnName(mtId) .addBigIntColumn(ERASED_RESOURCE_GROUP_ID, false) .addIntColumn(RESOURCE_TYPE_ID, false) @@ -1506,6 +1540,19 @@ protected void addFhirSequence(PhysicalDataModel pdm) { pdm.addObject(fhirSequence); } + /** + * Adds a new sequence required for distributed databases like Citus + * @param pdm + */ + protected void addFhirChangeSequence(PhysicalDataModel pdm) { + this.fhirChangeSequence = new Sequence(schemaName, FHIR_CHANGE_SEQUENCE, FhirSchemaVersion.V0027.vid(), 1, 1000); + this.fhirChangeSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + procedureDependencies.add(fhirChangeSequence); + sequencePrivileges.forEach(p -> p.addToObject(fhirChangeSequence)); + + pdm.addObject(fhirChangeSequence); + } + protected void addFhirRefSequence(PhysicalDataModel pdm) { this.fhirRefSequence = new Sequence(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE); this.fhirRefSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java index 151dc661225..14f5e4036df 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java @@ -45,7 +45,8 @@ public enum FhirSchemaVersion { ,V0023(23, "issue-2900 erased_resources to support $erase when offloading payloads", false) ,V0024(24, "issue-2900 for offloading add resource_payload_key to xx_resources", false) ,V0025(25, "issue-3158 stored proc updates to prevent deleting currently deleted resources", false) - ,V0026(26, "issue-nnnn extensions to support distribution/sharding", false) + ,V0026(26, "issue-nnnn R4B placeholder", true) + ,V0027(27, "issue-nnnn extensions to support distribution/sharding", true) ; // The version number recorded in the VERSION_HISTORY diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java new file mode 100644 index 00000000000..8e7b7945206 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java @@ -0,0 +1,53 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Check to see if we have any data in LOGICAL_RESOURCE_IDENT. If it is empty, + * we assume we need to perform a migration + */ +public class GetLogicalResourceNeedsV0027Migration implements IDatabaseSupplier { + + // The FHIR data schema + private final String schemaName; + + /** + * Public constructor + * + * @param schemaName + */ + public GetLogicalResourceNeedsV0027Migration(String schemaName) { + this.schemaName = schemaName; + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result = false; + final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES"); + final String SQL = "SELECT 1 FROM " + tableName + " " + translator.limit("1"); + + try (Statement s = c.createStatement()) { + ResultSet rs = s.executeQuery(SQL); + if (rs.next()) { + result = true; + } + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java new file mode 100644 index 00000000000..fa87521167d --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java @@ -0,0 +1,48 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Populate LOGICAL_RESOURCE_IDENT with records from LOGICAL_RESOURCES + */ +public class MigrateV0027LogicalResourceIdent implements IDatabaseStatement { + + // The FHIR data schema + private final String schemaName; + + /** + * Public constructor + * @param schemaName + */ + public MigrateV0027LogicalResourceIdent(String schemaName) { + this.schemaName = schemaName; + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES"); + final String logicalResourceIdent = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT"); + final String DML = "" + + "INSERT INTO " + logicalResourceIdent +"(resource_type_id, logical_id, logical_resource_id) " + + " SELECT resource_type_id, logical_id, logical_resource_id " + + " FROM " + logicalResources; + + try (PreparedStatement ps = c.prepareStatement(DML)) { + ps.executeUpdate(); + } catch (SQLException x) { + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql index 94ecff46ab6..d82815033c2 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql @@ -5,24 +5,18 @@ ------------------------------------------------------------------------------- -- ---------------------------------------------------------------------------- --- Provides support for the distributed database schema variant (e.g. Citus) --- which uses a shard_key value to distribute the data across multiple --- database nodes. --- --- For Citus, this function can also be tagged as distributed because --- all the SQL/DML uses either a reference table, or a table distributed --- by the p_shard_key parameter value. --- -- Procedure to add a resource version and its associated parameters. These -- parameters only ever point to the latest version of a resource, never to -- previous versions, which are kept to support history queries. +-- From V0027, we now use a logical_resource_ident table for locking. Records +-- can be created in this table either by this procedure, or as part of +-- reference parameter processing. -- implNote - Conventions: -- p_... prefix used to represent input parameters -- v_... prefix used to represent declared variables -- t_... prefix used to represent temp variables -- o_... prefix used to represent output parameters -- Parameters: --- p_shard_key: the key used to distribute resources by sharding -- p_logical_id: the logical id given to the resource by the FHIR server -- p_payload: the BLOB (of JSON) which is the resource content -- p_last_updated the last_updated time given by the FHIR server @@ -39,8 +33,7 @@ -- SQLSTATE 99002: missing expected row (data integrity) -- SQLSTATE 99004: delete a currently deleted resource (data integrity) -- ---------------------------------------------------------------------------- - ( IN p_shard_key SMALLINT, - IN p_resource_type VARCHAR( 36), + ( IN p_resource_type VARCHAR( 36), IN p_logical_id VARCHAR(255), IN p_payload BYTEA, IN p_last_updated TIMESTAMP, @@ -68,10 +61,11 @@ v_new_resource INT := 0; v_duplicate INT := 0; v_current_version INT := 0; + v_ghost_resource INT := 0; v_change_type CHAR(1) := NULL; -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. - lock_cur CURSOR (t_shard_key SMALLINT, t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id, parameter_hash, is_deleted FROM {{SCHEMA_NAME}}.logical_resources WHERE shard_key = t_shard_key AND resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; BEGIN -- default value unless we hit If-None-Match @@ -85,50 +79,83 @@ BEGIN -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; - -- Get a lock at the system-wide logical resource level - OPEN lock_cur(t_shard_key := p_shard_key, t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted; + -- Get a lock on the logical resource identity record + OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO v_logical_resource_id; CLOSE lock_cur; - -- Create the resource if we don't have it already + -- Create the resource ident record if we don't have it already IF v_logical_resource_id IS NULL THEN SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; -- remember that we have a concurrent system...so there is a possibility - -- that another thread snuck in before us and created the logical resource. This - -- is easy to handle, just turn around and read it - INSERT INTO {{SCHEMA_NAME}}.logical_resources (shard_key, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) - VALUES (p_shard_key, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; - - -- if row existed, we still need to obtain a lock on it. Because logical resource records are - -- never deleted, we don't need to worry about it disappearing again before we grab the row lock - OPEN lock_cur (t_shard_key := p_shard_key, t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted; - CLOSE lock_cur; - - -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL - o_current_parameter_hash := NULL; - + -- that another thread snuck in before us and created the ident record. To + -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn + -- around and read again to check that the logical_resource_id in the table + -- matches the value we tried to insert. + INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id) + VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; + + -- Do a read so that we can verify that *we* did the insert + OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO t_logical_resource_id; + CLOSE lock_cur; + IF v_logical_resource_id = t_logical_resource_id THEN - -- we created the logical resource and therefore we already own the lock. So now we can - -- safely create the corresponding record in the resource-type-specific logical_resources table - EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (shard_key, logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' - || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' USING p_shard_key, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - v_new_resource := 1; + -- we did the insert, so we know this is a new record + v_new_resource := 1; ELSE - v_logical_resource_id := t_logical_resource_id; + -- another thread created the resource. + -- New for V0027. Records in logical_resource_ident may be created because they + -- are the target of a reference. We therefore need to handle the case where + -- no logical_resources record exists. + SELECT logical_resource_id, parameter_hash, is_deleted + INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = t_logical_resource_id; + + IF (v_logical_resource_id IS NULL) + THEN + -- other thread only created the ident record, so we still need to treat + -- this as a new resource + v_logical_resource_id := t_logical_resource_id; + v_new_resource := 1; + END IF; + END IF; + ELSE + -- we have an ident record, but we still need to check if we have a logical_resources + -- record + SELECT logical_resource_id, parameter_hash, is_deleted + INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; + IF (t_logical_resource_id IS NULL) + THEN + v_new_resource := 1; END IF; END IF; - -- Remember everying is locked at the logical resource level, so we are thread-safe here - IF v_new_resource = 0 THEN + IF v_new_resource = 1 + THEN + -- we already own the lock on the ident record, so we can safely create + -- the corresponding records in the logical_resources and resource-type-specific + -- xx_logical_resources tables + INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; + + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + + -- Since the resource did not previously exist, make sure o_current_parameter_hash is null + o_current_parameter_hash := NULL; + ELSE -- as this is an existing resource, we need to know the current resource id. -- This is only available at the resource-specific logical_resources level EXECUTE 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' - || ' WHERE shard_key = $1 AND logical_resource_id = $2 ' - INTO v_current_resource_id, v_current_version USING p_shard_key, v_logical_resource_id; + || ' WHERE logical_resource_id = $1 ' + INTO v_current_resource_id, v_current_version USING v_logical_resource_id; IF v_current_resource_id IS NULL OR v_current_version IS NULL THEN @@ -165,27 +192,27 @@ BEGIN THEN -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) -- TODO patch parameter sets instead of all delete/all insert. - EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2, $3)' - USING p_shard_key, p_resource_type, v_logical_resource_id; + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' + USING p_resource_type, v_logical_resource_id; END IF; -- end if check parameter hash END IF; -- end if existing resource + -- create the new resource version entry in xx_resources EXECUTE - 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (shard_key, resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' - || ' VALUES ($1, $2, $3, $4, $5, $6, $7, $8)' - USING p_shard_key, v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; - + 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' + USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; IF v_new_resource = 0 THEN -- As this is an existing logical resource, we need to update the xx_logical_resource values to match -- the values of the current resource. For new resources, these are added by the insert so we don't -- need to update them here. - EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE shard_key = $5 AND logical_resource_id = $6' - USING v_resource_id, p_is_deleted, p_last_updated, p_version, p_shard_key, v_logical_resource_id; + EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' + USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level - EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE shard_key = $4 AND logical_resource_id = $5' - USING p_is_deleted, p_last_updated, p_parameter_hash_b64, p_shard_key, v_logical_resource_id; + EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' + USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; END IF; -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event @@ -202,8 +229,8 @@ BEGIN END IF; END IF; - INSERT INTO {{SCHEMA_NAME}}.resource_change_log(shard_key, resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) - VALUES (p_shard_key, v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); + INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) + VALUES (v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); -- Hand back the id of the logical resource we created earlier. In the new R4 schema -- only the logical_resource_id is the target of any FK, so there's no need to return diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql index abf1f6b50ac..496ade054a1 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql @@ -6,15 +6,10 @@ -- ---------------------------------------------------------------------------- -- Procedure to delete all search parameters values for a given resource. --- This variant is for use with the distributed schema variant typically --- deployed to a distributed database service like Citus. --- --- p_shard_key: the key used for distribution (sharding) -- p_resource_type: the resource type name -- p_logical_resource_id: the database id of the resource for which the parameters are to be deleted -- ---------------------------------------------------------------------------- - ( IN p_shard_key SMALLINT, - IN p_resource_type VARCHAR( 36), + ( IN p_resource_type VARCHAR( 36), IN p_logical_resource_id BIGINT, OUT o_logical_resource_id BIGINT) RETURNS BIGINT @@ -27,37 +22,37 @@ BEGIN v_schema_name := '{{SCHEMA_NAME}}'; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE logical_resource_id = $1' + USING p_logical_resource_id; -- because we're a function, pass back a result o_logical_resource_id := p_logical_resource_id; -END $$; +END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql new file mode 100644 index 00000000000..abf1f6b50ac --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql @@ -0,0 +1,63 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2021 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to delete all search parameters values for a given resource. +-- This variant is for use with the distributed schema variant typically +-- deployed to a distributed database service like Citus. +-- +-- p_shard_key: the key used for distribution (sharding) +-- p_resource_type: the resource type name +-- p_logical_resource_id: the database id of the resource for which the parameters are to be deleted +-- ---------------------------------------------------------------------------- + ( IN p_shard_key SMALLINT, + IN p_resource_type VARCHAR( 36), + IN p_logical_resource_id BIGINT, + OUT o_logical_resource_id BIGINT) + RETURNS BIGINT + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + +BEGIN + v_schema_name := '{{SCHEMA_NAME}}'; + + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + + -- because we're a function, pass back a result + o_logical_resource_id := p_logical_resource_id; +END $$; diff --git a/fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/erase_resource_sharded.sql similarity index 100% rename from fhir-persistence-schema/src/main/resources/postgres/erase_resource_distributed.sql rename to fhir-persistence-schema/src/main/resources/postgres/erase_resource_sharded.sql diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java index d72035c5b59..c711f273c90 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java @@ -311,8 +311,15 @@ public Statement createStatement() throws SQLException { @Override public PreparedStatement prepareStatement(String sql) throws SQLException { + + boolean hasRow = true; + if (sql.toUpperCase().startsWith("SELECT 1 FROM")) { + // this is one of our checks for the existing of a FK...which we want to + // say doesn't exist + hasRow = false; + } addCommand(sql); - return new PrintPreparedStatement(); + return new PrintPreparedStatement(hasRow); } @Override @@ -410,7 +417,7 @@ public Statement createStatement(int resultSetType, int resultSetConcurrency) th @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - return new PrintPreparedStatement(); + return new PrintPreparedStatement(true); } @Override @@ -593,11 +600,15 @@ public int getNetworkTimeout() throws SQLException { } class PrintPreparedStatement implements java.sql.PreparedStatement { + private final boolean hasRow; + public PrintPreparedStatement(boolean hasRow) { + this.hasRow = hasRow; + } @Override public ResultSet executeQuery(String sql) throws SQLException { addCommand(sql); - return new PrintResultSet(); + return new PrintResultSet(true); } @Override @@ -847,7 +858,7 @@ public boolean isWrapperFor(Class iface) throws SQLException { @Override public ResultSet executeQuery() throws SQLException { - return new PrintResultSet(); + return new PrintResultSet(this.hasRow); } @Override @@ -1143,7 +1154,7 @@ public boolean isWrapperFor(Class iface) throws SQLException { @Override public ResultSet executeQuery(String sql) throws SQLException { addCommand(sql); - return new PrintResultSet(); + return new PrintResultSet(true); } @Override @@ -1381,7 +1392,11 @@ public boolean isCloseOnCompletion() throws SQLException { } class PrintResultSet implements java.sql.ResultSet { + private boolean hasRow; + public PrintResultSet(boolean hasRow) { + this.hasRow = hasRow; + } @Override public T unwrap(Class iface) throws SQLException { return null; @@ -1394,7 +1409,8 @@ public boolean isWrapperFor(Class iface) throws SQLException { @Override public boolean next() throws SQLException { - return true; + // pretend to have a row + return this.hasRow; } @Override @@ -2434,7 +2450,7 @@ class PrintCallableStatement implements java.sql.CallableStatement { @Override public ResultSet executeQuery() throws SQLException { - return new PrintResultSet(); + return new PrintResultSet(true); } @Override @@ -2713,13 +2729,11 @@ public void setNClob(int parameterIndex, Reader reader) throws SQLException { @Override public ResultSet executeQuery(String sql) throws SQLException { - - return new PrintResultSet(); + return new PrintResultSet(true); } @Override public int executeUpdate(String sql) throws SQLException { - return 0; } diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java index fe51963b68e..497c80aab0e 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java @@ -142,7 +142,7 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S // Check that we have the correct number of tables. This will need to be updated // whenever tables, views or sequences are added or removed - assertEquals(adapter.listSchemaObjects(schemaName).size(), 1919); + assertEquals(adapter.listSchemaObjects(schemaName).size(), 2065); c.commit(); } catch (Throwable t) { c.rollback(); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java index ecc13bb7a6c..172222218c1 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java @@ -52,7 +52,7 @@ public void test() throws Exception { // Make sure we can correctly determine the latest schema version value svm.updateSchemaVersion(); - assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0026.vid()); + assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0027.vid()); assertFalse(svm.isSchemaOld()); } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java new file mode 100644 index 00000000000..dbc2e45e202 --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java @@ -0,0 +1,76 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.index; + + +/** + * A local reference search parameter value + */ +public class ReferenceParameter extends SearchParameterValue { + private String resourceType; + private String logicalId; + + // for storing versioned references + private Integer refVersionId; + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + result.append("Reference["); + addDescription(result); + result.append(","); + result.append(resourceType); + result.append(","); + result.append(logicalId); + result.append(","); + result.append(refVersionId); + result.append("]"); + return result.toString(); + } + + /** + * @return the refVersionId + */ + public Integer getRefVersionId() { + return refVersionId; + } + + /** + * @param refVersionId the refVersionId to set + */ + public void setRefVersionId(Integer refVersionId) { + this.refVersionId = refVersionId; + } + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + /** + * @param resourceType the resourceType to set + */ + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } + + /** + * @param logicalId the logicalId to set + */ + public void setLogicalId(String logicalId) { + this.logicalId = logicalId; + } +} \ No newline at end of file diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java index d4fb995daa3..2bed02ddef0 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java @@ -49,6 +49,7 @@ public class SearchParametersTransport { private List tagValues = new ArrayList<>(); private List profileValues = new ArrayList<>(); private List securityValues = new ArrayList<>(); + private List refValues = new ArrayList<>(); /** * Factory method to create a {@link Builder} instance @@ -70,12 +71,12 @@ public String toString() { result.append("versionId["); result.append(versionId); result.append("] "); -// result.append("parameterHash["); -// result.append(parameterHash); -// result.append("] "); -// result.append("lastUpdated["); -// result.append(lastUpdated); -// result.append("] "); + result.append("parameterHash["); + result.append(parameterHash); + result.append("] "); + result.append("lastUpdated["); + result.append(lastUpdated); + result.append("] "); return result.toString(); } @@ -92,6 +93,7 @@ public static class Builder { private List tagValues = new ArrayList<>(); private List profileValues = new ArrayList<>(); private List securityValues = new ArrayList<>(); + private List refValues = new ArrayList<>(); private String resourceType; private String logicalId; @@ -206,6 +208,16 @@ public Builder addTokenValue(TokenParameter value) { return this; } + /** + * Add a reference parameter value + * @param value + * @return + */ + public Builder addReferenceValue(ReferenceParameter value) { + refValues.add(value); + return this; + } + /** * Add a tag parameter value * @param value @@ -308,6 +320,9 @@ public SearchParametersTransport build() { if (this.securityValues.size() > 0) { result.setSecurityValues(new ArrayList<>(this.securityValues)); } + if (this.refValues.size() > 0) { + result.setRefValues(new ArrayList<>(this.refValues)); + } return result; } } @@ -560,4 +575,18 @@ public String getLastUpdated() { public void setLastUpdated(String lastUpdated) { this.lastUpdated = lastUpdated; } + + /** + * @return the refValues + */ + public List getRefValues() { + return refValues; + } + + /** + * @param refValues the refValues to set + */ + public void setRefValues(List refValues) { + this.refValues = refValues; + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java similarity index 87% rename from fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java rename to fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java index c982a072c41..4c29c5640ae 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/SearchParametersTransportAdapter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java @@ -4,24 +4,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.persistence.jdbc.impl; +package com.ibm.fhir.persistence.index; import java.math.BigDecimal; import java.sql.Timestamp; import java.time.Instant; -import com.ibm.fhir.persistence.index.DateParameter; -import com.ibm.fhir.persistence.index.LocationParameter; -import com.ibm.fhir.persistence.index.NumberParameter; -import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; -import com.ibm.fhir.persistence.index.ProfileParameter; -import com.ibm.fhir.persistence.index.QuantityParameter; -import com.ibm.fhir.persistence.index.SearchParametersTransport; -import com.ibm.fhir.persistence.index.SecurityParameter; -import com.ibm.fhir.persistence.index.StringParameter; -import com.ibm.fhir.persistence.index.TagParameter; -import com.ibm.fhir.persistence.index.TokenParameter; - /** * Visitor adapter implementation to build an instance of {@link SearchParametersTransport} to @@ -164,12 +152,12 @@ public void locationValue(String name, Double valueLatitude, Double valueLongitu @Override public void referenceValue(String name, String refResourceType, String refLogicalId, Integer refVersion, Integer compositeId) { - TokenParameter value = new TokenParameter(); + ReferenceParameter value = new ReferenceParameter(); value.setName(name); - value.setValueSystem(refResourceType); - value.setValueCode(refLogicalId); + value.setResourceType(refResourceType); + value.setLogicalId(refLogicalId); value.setRefVersionId(refVersion); value.setCompositeId(compositeId); - builder.addTokenValue(value); + builder.addReferenceValue(value); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java index a3613203d5b..b93ec926e28 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; import com.ibm.fhir.persistence.index.TagParameter; @@ -19,6 +20,7 @@ import com.ibm.fhir.remote.index.database.CodeSystemValue; import com.ibm.fhir.remote.index.database.CommonCanonicalValue; import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue; import com.ibm.fhir.remote.index.database.ParameterNameValue; /** @@ -132,4 +134,16 @@ public interface BatchParameterProcessor { * @throws FHIRPersistenceException */ void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException; + + /** + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param refLogicalResourceId + */ + void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, + ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 891467bc8c8..2ff77dbdd4e 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -29,6 +29,7 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; @@ -38,7 +39,9 @@ import com.ibm.fhir.remote.index.cache.IdentityCacheImpl; import com.ibm.fhir.remote.index.database.CacheLoader; import com.ibm.fhir.remote.index.database.DistributedPostgresMessageHandler; +import com.ibm.fhir.remote.index.database.PlainPostgresMessageHandler; import com.ibm.fhir.remote.index.kafka.RemoteIndexConsumer; +import com.ibm.fhir.remote.index.sharded.ShardedPostgresMessageHandler; /** * Main class for the FHIR remote index service Kafka consumer @@ -78,6 +81,7 @@ public class Main { private IdentityCacheImpl identityCache; // Database Configuration + private SchemaType schemaType = SchemaType.PLAIN; private IDatabaseTranslator translator; private IConnectionProvider connectionProvider; @@ -132,6 +136,13 @@ public void parseArgs(String[] args) { throw new IllegalArgumentException("Missing value for --max-ready-time-ms"); } break; + case "--schema-type": + if (a < args.length && !args[a].startsWith("--")) { + schemaType = SchemaType.valueOf(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --schema-type"); + } + break; default: throw new IllegalArgumentException("Bad arg: '" + arg + "'"); } @@ -294,7 +305,16 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { try { // Each handler gets a dedicated database connection so we don't have // to deal with contention when grabbing connections from a pool - return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + switch (schemaType) { + case SHARDED: + return new ShardedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + case PLAIN: + return new PlainPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + case DISTRIBUTED: + return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + default: + throw new FHIRPersistenceException("Schema type not supported: " + schemaType.name()); + } } catch (SQLException x) { throw new FHIRPersistenceException("get connection failed", x); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java new file mode 100644 index 00000000000..2aef5d8158c --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java @@ -0,0 +1,44 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.batch; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.ReferenceParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; + +/** + * A reference parameter we are collecting to batch + */ +public class BatchReferenceParameter extends BatchParameterValue { + private final ReferenceParameter parameter; + private final LogicalResourceIdentValue refLogicalResourceId; + + /** + * Canonical constructor + * + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param parameterNameValue + * @param parameter + * @param refLogicalResourceId + */ + public BatchReferenceParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) { + super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue); + this.parameter = parameter; + this.refLogicalResourceId = refLogicalResourceId; + } + + @Override + public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException { + processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, refLogicalResourceId); + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index 7e0362e0bf4..6a08a8e53fe 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -21,6 +21,7 @@ import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; import com.ibm.fhir.persistence.index.RemoteIndexMessage; import com.ibm.fhir.persistence.index.SearchParametersTransport; import com.ibm.fhir.persistence.index.SecurityParameter; @@ -178,7 +179,7 @@ private void processMessages(List messages) throws FHIRPersi // Ask the handle to check which messages match the database // and are therefore ready to be processed - prepare(messages, okToProcess, notReady); + checkReady(messages, okToProcess, notReady); // If the ready check fails just sleep for a bit because we need // to wait until the upstream transaction commits. This means we @@ -187,7 +188,9 @@ private void processMessages(List messages) throws FHIRPersi if (notReady.size() > 0) { long snoozeMs = Math.min(1000l, (timeoutTime - System.nanoTime()) / 1000000); // short sleep to wait for the upstream transaction to complete - ThreadHandler.safeSleep(snoozeMs); + if (snoozeMs > 0) { + ThreadHandler.safeSleep(snoozeMs); + } } } while (notReady.size() > 0 && System.nanoTime() < timeoutTime); @@ -210,7 +213,7 @@ private void processMessages(List messages) throws FHIRPersi * @param OUT: okToMessages the messages matching the current database * @param OUT: notReady the messages for which the upstream transaction has yet to commit */ - protected abstract void prepare(List messages, List okToProcess, List notReady) throws FHIRPersistenceException; + protected abstract void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException; /** * Process the data @@ -271,6 +274,12 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); } } + + if (params.getRefValues() != null) { + for (ReferenceParameter p: params.getRefValues()) { + process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p); + } + } } /** @@ -359,6 +368,18 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException */ protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException; + /** + * + * @param tenantId + * @param requestShard + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param p + * @throws FHIRPersistenceException + */ + protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException; + /** * @param tenantId * @param requestShard diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index 7dcbc907100..ee14fc99a71 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -6,666 +6,54 @@ package com.ibm.fhir.remote.index.database; -import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Types; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.common.ResultSetReader; -import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.index.DateParameter; -import com.ibm.fhir.persistence.index.LocationParameter; -import com.ibm.fhir.persistence.index.NumberParameter; -import com.ibm.fhir.persistence.index.ProfileParameter; -import com.ibm.fhir.persistence.index.QuantityParameter; -import com.ibm.fhir.persistence.index.RemoteIndexMessage; -import com.ibm.fhir.persistence.index.SearchParameterValue; -import com.ibm.fhir.persistence.index.SecurityParameter; -import com.ibm.fhir.persistence.index.StringParameter; -import com.ibm.fhir.persistence.index.TagParameter; -import com.ibm.fhir.persistence.index.TokenParameter; -import com.ibm.fhir.remote.index.api.BatchParameterValue; import com.ibm.fhir.remote.index.api.IdentityCache; -import com.ibm.fhir.remote.index.batch.BatchDateParameter; -import com.ibm.fhir.remote.index.batch.BatchLocationParameter; -import com.ibm.fhir.remote.index.batch.BatchNumberParameter; -import com.ibm.fhir.remote.index.batch.BatchProfileParameter; -import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; -import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; -import com.ibm.fhir.remote.index.batch.BatchStringParameter; -import com.ibm.fhir.remote.index.batch.BatchTagParameter; -import com.ibm.fhir.remote.index.batch.BatchTokenParameter; /** - * Loads search parameter values into the target FHIR schema on - * a PostgreSQL database. + * For the DISTRIBUTED schema variant used on databases such as Citus, we + * can't use IDENTITY columns. Instead we have to use values generated + * by a sequence, which means a slightly different INSERT statement + * in certain cases */ -public class DistributedPostgresMessageHandler extends BaseMessageHandler { +public class DistributedPostgresMessageHandler extends PlainPostgresMessageHandler { private static final Logger logger = Logger.getLogger(DistributedPostgresMessageHandler.class.getName()); - // the connection to use for the inserts - private final Connection connection; - - // We're a PostgreSQL DAO, so we now which translator to use - private final IDatabaseTranslator translator = new PostgresTranslator(); - - // The FHIR data schema - private final String schemaName; - - // the cache we use for various lookups - private final IdentityCache identityCache; - - // All parameter names we've seen (cleared if there's a rollback) - private final Map parameterNameMap = new HashMap<>(); - - // A map of code system name to the value holding its codeSystemId from the database - private final Map codeSystemValueMap = new HashMap<>(); - - // A map to support lookup of CommonTokenValue records by key - private final Map commonTokenValueMap = new HashMap<>(); - - // A map to support lookup of CommonCanonicalValue records by key - private final Map commonCanonicalValueMap = new HashMap<>(); - - // All parameter names in the current transaction for which we don't yet know the parameter_name_id - private final List unresolvedParameterNames = new ArrayList<>(); - - // A list of all the CodeSystemValues for which we don't yet know the code_system_id - private final List unresolvedSystemValues = new ArrayList<>(); - - // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id - private final List unresolvedTokenValues = new ArrayList<>(); - - // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id - private final List unresolvedCanonicalValues = new ArrayList<>(); - - // The processed values we've collected - private final List batchedParameterValues = new ArrayList<>(); - - // The processor used to process the batched parameter values after all the reference values are created - private final JDBCBatchParameterProcessor batchProcessor; - - private final int maxCodeSystemsPerStatement = 512; - private final int maxCommonTokenValuesPerStatement = 256; - private final int maxCommonCanonicalValuesPerStatement = 256; - private boolean rollbackOnly; - /** * Public constructor - * * @param connection * @param schemaName * @param cache * @param maxReadyTimeMs */ public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(maxReadyTimeMs); - this.connection = connection; - this.schemaName = schemaName; - this.identityCache = cache; - this.batchProcessor = new JDBCBatchParameterProcessor(connection); - } - - @Override - protected void startBatch() { - // always start with a clean slate - batchedParameterValues.clear(); - unresolvedParameterNames.clear(); - unresolvedSystemValues.clear(); - unresolvedTokenValues.clear(); - unresolvedCanonicalValues.clear(); - batchProcessor.startBatch(); - } - - @Override - protected void setRollbackOnly() { - this.rollbackOnly = true; - } - - @Override - public void close() { - try { - batchProcessor.close(); - } catch (Throwable t) { - logger.log(Level.SEVERE, "close batchProcessor failed" , t); - } - } - - @Override - protected void endTransaction() throws FHIRPersistenceException { - boolean committed = false; - try { - if (!this.rollbackOnly) { - logger.fine("Committing transaction"); - connection.commit(); - committed = true; - - // any values from parameter_names, code_systems and common_token_values - // are now committed to the database, so we can publish their record ids - // to the shared cache which makes them accessible from other threads - publishCachedValues(); - } else { - // something went wrong...try to roll back the transaction before we close - // everything - try { - connection.rollback(); - } catch (SQLException x) { - // It could very well be that we've lost touch with the database in which case - // the rollback will also fail. Not much we can do, although we don't bother - // with a stack trace here because it's just more noise for the log file, and - // the exception that triggered the rollback is already going to be propagated - // and logged. - logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); - } - } - } catch (SQLException x) { - throw new FHIRPersistenceException("commit failed", x); - } finally { - if (!committed) { - // The maps may contain ids that were not committed to the database so - // we should clean them out in case we decide to reuse this consumer - this.parameterNameMap.clear(); - this.codeSystemValueMap.clear(); - this.commonTokenValueMap.clear(); - this.commonCanonicalValueMap.clear(); - } - } - } - - /** - * After the transaction has been committed, we can publish certain values to the - * shared identity caches - */ - public void publishCachedValues() { - // all the unresolvedParameterNames should be resolved at this point - for (ParameterNameValue pnv: this.unresolvedParameterNames) { - identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); - } - } - - @Override - protected void pushBatch() throws FHIRPersistenceException { - // Push any data we've accumulated so far. This may occur - // if we cross a volume threshold, and will always occur as - // the last step before the current transaction is committed, - // Process the token values so that we can establish - // any entries we need for common_token_values - resolveParameterNames(); - resolveCodeSystems(); - resolveCommonTokenValues(); - resolveCommonCanonicalValues(); - - // Now that all the lookup values should've been resolved, we can go ahead - // and push the parameters to the JDBC batch insert statements via the - // batchProcessor - for (BatchParameterValue v: this.batchedParameterValues) { - v.apply(batchProcessor); - } - batchProcessor.pushBatch(); - } - - /** - * Get the parameter name value for the given parameter value - * @param p - * @return - */ - private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { - if (logger.isLoggable(Level.FINEST)) { - logger.finest("get ParameterNameValue for [" + p.toString() + "]"); - } - ParameterNameValue result = parameterNameMap.get(p.getName()); - if (result == null) { - result = new ParameterNameValue(p.getName()); - parameterNameMap.put(p.getName(), result); - - // let's see if the id is available in the shared identity cache - Integer parameterNameId = identityCache.getParameterNameId(p.getName()); - if (parameterNameId != null) { - result.setParameterNameId(parameterNameId); - } else { - // ids will be created later (so that we can process them in order) - unresolvedParameterNames.add(result); - } - } - return result; - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { - Short shardKey = batchProcessor.encodeShardKey(requestShard); - CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { - Short shardKey = batchProcessor.encodeShardKey(requestShard); - CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { - Short shardKey = batchProcessor.encodeShardKey(requestShard); - CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { - Short shardKey = batchProcessor.encodeShardKey(requestShard); - CommonCanonicalValue ctv = lookupCommonCanonicalValue(shardKey, p.getUrl()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); - this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); + super(connection, schemaName, cache, maxReadyTimeMs); } @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - /** - * Get the CodeSystemValue we've assigned for the given codeSystem value. This - * may not yet have the actual code_system_id from the database yet - any values - * we don't have will be assigned in a later phase (so we can do things neatly - * in bulk). - * @param codeSystem - * @return - */ - private CodeSystemValue lookupCodeSystemValue(String codeSystem) { - CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); - if (result == null) { - result = new CodeSystemValue(codeSystem); - this.codeSystemValueMap.put(codeSystem, result); - - // Take this opportunity to see if we have a cached value for this codeSystem - Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); - if (codeSystemId != null) { - result.setCodeSystemId(codeSystemId); - } else { - // Stash for later resolution - this.unresolvedSystemValues.add(result); - } - } - return result; - } - - /** - * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. - * The returned value may not yet have the actual common_token_value_id yet - we fetch - * these values later and create new database records as necessary. - * @param codeSystem - * @param tokenValue - * @return - */ - private CommonTokenValue lookupCommonTokenValue(short shardKey, String codeSystem, String tokenValue) { - CommonTokenValueKey key = new CommonTokenValueKey(shardKey, codeSystem, tokenValue); - CommonTokenValue result = this.commonTokenValueMap.get(key); - if (result == null) { - CodeSystemValue csv = lookupCodeSystemValue(codeSystem); - result = new CommonTokenValue(shardKey, csv, tokenValue); - this.commonTokenValueMap.put(key, result); - - // Take this opportunity to see if we have a cached value for this common token value - Long commonTokenValueId = identityCache.getCommonTokenValueId(shardKey, codeSystem, tokenValue); - if (commonTokenValueId != null) { - result.setCommonTokenValueId(commonTokenValueId); - } else { - this.unresolvedTokenValues.add(result); - } - } - return result; - } - - private CommonCanonicalValue lookupCommonCanonicalValue(short shardKey, String url) { - CommonCanonicalValueKey key = new CommonCanonicalValueKey(shardKey, url); - CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); - if (result == null) { - result = new CommonCanonicalValue(shardKey, url); - this.commonCanonicalValueMap.put(key, result); - - // Take this opportunity to see if we have a cached value for this common token value - Long canonicalId = identityCache.getCommonCanonicalValueId(shardKey, url); - if (canonicalId != null) { - result.setCanonicalId(canonicalId); - } else { - this.unresolvedCanonicalValues.add(result); - } - } - return result; - } - - /** - * Make sure we have values for all the code_systems we have collected - * in the current - * batch - * @throws FHIRPersistenceException - */ - private void resolveCodeSystems() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCodeSystemIds(unresolvedSystemValues); - - if (!missing.isEmpty()) { - addMissingCodeSystems(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCodeSystemIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happend, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all code system values"); - } - } - - /** - * Build and prepare a statement to fetch the code_system_id and code_system_name - * from the code_systems table for all the given (unresolved) code system values - * @param values - * @return - * @throws SQLException - */ - private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); - for (int i=0; i 0) { - query.append(","); - } - query.append("?"); - } - query.append(")"); - PreparedStatement ps = connection.prepareStatement(query.toString()); - // bind the parameter values - int param = 1; - for (CodeSystemValue csv: values) { - ps.setString(param++, csv.getCodeSystem()); - } - return ps; - } - - /** - * These code systems weren't found in the database, so we need to try and add them. - * We have to deal with concurrency here - there's a chance another thread could also - * be trying to add them. To avoid deadlocks, it's important to do any inserts in a - * consistent order. At the end, we should be able to read back values for each entry - * @param missing - */ - private void addMissingCodeSystems(List missing) throws FHIRPersistenceException { - List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); - // Sort the code system values first to help avoid deadlocks - Collections.sort(values); // natural ordering for String is fine here - - final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); - StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); - insert.append(nextVal); // next sequence value - insert.append(",?) ON CONFLICT DO NOTHING"); - - try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { - int count = 0; - for (String codeSystem: values) { - ps.setString(1, codeSystem); - ps.addBatch(); - if (++count == this.maxCodeSystemsPerStatement) { - // not too many statements in a single batch - ps.executeBatch(); - count = 0; - } - } - if (count > 0) { - // final batch - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); - throw new FHIRPersistenceException("code systems fetch failed"); - } - } - - private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); - if (csv != null) { - csv.setCodeSystemId(rs.getInt(1)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("code systems query returned an unexpected value"); - } - } - - // Most of the time we'll get everything, so we can bypass the check for - // missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CodeSystemValue csv: sub) { - if (csv.getCodeSystemId() == null) { - missing.add(csv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "code systems fetch failed", x); - throw new FHIRPersistenceException("code systems fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Make sure we have values for all the common_token_value records we have collected - * in the current batch - * @throws FHIRPersistenceException - */ - private void resolveCommonTokenValues() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCommonTokenValueIds(unresolvedTokenValues); - - if (!missing.isEmpty()) { - // Sort first to minimize deadlocks - Collections.sort(missing, (a,b) -> { - int result = a.getTokenValue().compareTo(b.getTokenValue()); - if (result == 0) { - result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); - if (result == 0) { - result = Short.compare(a.getShardKey(), b.getShardKey()); - } - } - return result; - }); - addMissingCommonTokenValues(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCommonTokenValueIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happend, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all common token values"); - } - } - - /** - * Build and prepare a statement to fetch the common_token_value records - * for all the given (unresolved) code system values - * @param values - * @return SELECT shard_key, code_system, token_value, common_token_value_id - * @throws SQLException - */ - private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - // need the code_system name - so we join back to the code_systems table as well - query.append("SELECT c.shard_key, cs.code_system_name, c.token_value, c.common_token_value_id "); - query.append(" FROM common_token_values c"); - query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); - query.append(" JOIN (VALUES "); - - // Create a (codeSystem, shardKey, tokenValue) tuple for each of the CommonTokenValue records - boolean first = true; - for (CommonTokenValue ctv: values) { - if (first) { - first = false; - } else { - query.append(","); - } - query.append("("); - query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id - query.append(",").append(ctv.getShardKey()); // literal for shard_key - query.append(",?)"); // bind variable for the token-value - } - query.append(") AS v(code_system_id, shard_key, token_value) "); - query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value AND c.shard_key = v.shard_key)"); - - // Create the prepared statement and bind the values - final String statementText = query.toString(); - PreparedStatement ps = connection.prepareStatement(statementText); - - // bind the parameter values - int param = 1; - for (CommonTokenValue ctv: values) { - ps.setString(param++, ctv.getTokenValue()); - } - return new PreparedStatementWrapper(statementText, ps); - } - - private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - String sql = null; // the SQL text for logging when there's an error - try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { - sql = ps.getStatementText(); - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CommonTokenValueKey key = new CommonTokenValueKey(rs.getShort(1), rs.getString(2), rs.getString(3)); - CommonTokenValue ctv = this.commonTokenValueMap.get(key); - if (ctv != null) { - ctv.setCommonTokenValueId(rs.getLong(4)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("common token values query returned an unexpected value"); - } - } - - // Optimize the check for missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CommonTokenValue ctv: sub) { - if (ctv.getCommonTokenValueId() == null) { - missing.add(ctv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); - throw new FHIRPersistenceException("common token values fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Add the values we think are missing from the database. The given list should be - * sorted to reduce deadlocks - * @param missing - * @throws FHIRPersistenceException - */ - private void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + // Need to use our own sequence number because distributed databases don't + // like generated identity columns final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) "); + insert.append("INSERT INTO common_token_values (code_system_id, token_value, common_token_value_id) "); insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number - insert.append(" VALUES (?,?,?,"); + insert.append(" VALUES (?,?,"); insert.append(nextVal); // next sequence value insert.append(") ON CONFLICT DO NOTHING"); try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { int count = 0; for (CommonTokenValue ctv: missing) { - ps.setShort(1, ctv.getShardKey()); - ps.setInt(2, ctv.getCodeSystemValue().getCodeSystemId()); - ps.setString(3, ctv.getTokenValue()); + ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId()); + ps.setString(2, ctv.getTokenValue()); ps.addBatch(); if (++count == this.maxCommonTokenValuesPerStatement) { // not too many statements in a single batch @@ -683,142 +71,16 @@ private void addMissingCommonTokenValues(List missing) throws } } - /** - * Make sure we have values for all the common_canonical_value records we have collected - * in the current batch - * @throws FHIRPersistenceException - */ - private void resolveCommonCanonicalValues() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCanonicalIds(unresolvedCanonicalValues); - - if (!missing.isEmpty()) { - // Sort on (url, shard_key) to minimize deadlocks - Collections.sort(missing, (a,b) -> { - int result = a.getUrl().compareTo(b.getUrl()); - if (result == 0) { - result = Short.compare(a.getShardKey(), b.getShardKey()); - } - return result; - }); - addMissingCommonCanonicalValues(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCanonicalIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happen, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all canonical values"); - } - } - - private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - String sql = null; // the SQL text for logging when there's an error - try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { - sql = ps.getStatementText(); - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CommonCanonicalValueKey key = new CommonCanonicalValueKey(rs.getShort(1), rs.getString(2)); - CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); - if (ctv != null) { - ctv.setCanonicalId(rs.getLong(3)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); - } - } - - // Optimize the check for missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CommonCanonicalValue ctv: sub) { - if (ctv.getCanonicalId() == null) { - missing.add(ctv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); - throw new FHIRPersistenceException("common canonical values fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Build and prepare a statement to fetch the common_token_value records - * for all the given (unresolved) code system values - * @param values - * @return SELECT shard_key, code_system, token_value, common_token_value_id - * @throws SQLException - */ - private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - query.append("SELECT c.shard_key, c.url, c.canonical_id "); - query.append(" FROM common_canonical_values c "); - query.append(" JOIN (VALUES "); - - // Create a (shardKey, url) tuple for each of the CommonCanonicalValue records - boolean first = true; - for (CommonCanonicalValue ctv: values) { - if (first) { - first = false; - } else { - query.append(","); - } - query.append("("); - query.append(ctv.getShardKey()); // literal for shard_key - query.append(",?)"); // bind variable for the uri - } - query.append(") AS v(shard_key, url) "); - query.append(" ON (c.url = v.url AND c.shard_key = v.shard_key)"); - - // Create the prepared statement and bind the values - final String statementText = query.toString(); - logger.finer(() -> "fetch common canonical values [" + statementText + "]"); - PreparedStatement ps = connection.prepareStatement(statementText); - - // bind the parameter values - int param = 1; - for (CommonCanonicalValue ctv: values) { - ps.setString(param++, ctv.getUrl()); - } - return new PreparedStatementWrapper(statementText, ps); - } - - /** - * Add the values we think are missing from the database. The given list should be - * sorted to reduce deadlocks - * @param missing - * @throws FHIRPersistenceException - */ - private void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + @Override + protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + // Need to use our own sequence number because distributed databases don't + // like generated identity columns final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO common_canonical_values (shard_key, url, canonical_id) "); + insert.append("INSERT INTO common_canonical_values (url, canonical_id) "); insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number - insert.append(" VALUES (?,?,"); + insert.append(" VALUES (?,"); insert.append(nextVal); // next sequence value insert.append(") ON CONFLICT DO NOTHING"); @@ -830,8 +92,7 @@ private void addMissingCommonCanonicalValues(List missing) int count = 0; for (CommonCanonicalValue ctv: missing) { logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); - ps.setShort(1, ctv.getShardKey()); - ps.setString(2, ctv.getUrl()); + ps.setString(1, ctv.getUrl()); ps.addBatch(); if (++count == this.maxCommonCanonicalValuesPerStatement) { // not too many statements in a single batch @@ -848,189 +109,4 @@ private void addMissingCommonCanonicalValues(List missing) throw new FHIRPersistenceException("failed inserting new common canonical values"); } } - - /** - * Make sure all the parameter names we've seen in the batch exist - * in the database and have ids. - * @throws FHIRPersistenceException - */ - private void resolveParameterNames() throws FHIRPersistenceException { - // We expect parameter names to have a very high cache hit rate and - // so we simplify processing by simply iterating one-by-one for the - // values we still need to resolve. The most important point here is - // to do this in a sorted order to avoid deadlock issues because this - // could be happening across multiple consumer threads at the same time. - Collections.sort(this.unresolvedParameterNames, (a,b) -> { - return a.getParameterName().compareTo(b.getParameterName()); - }); - - try { - for (ParameterNameValue pnv: this.unresolvedParameterNames) { - Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); - if (parameterNameId == null) { - parameterNameId = createParameterName(pnv.getParameterName()); - } - pnv.setParameterNameId(parameterNameId); - } - } catch (SQLException x) { - throw new FHIRPersistenceException("error resolving parameter names", x); - } - } - - private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { - String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; - try (PreparedStatement ps = connection.prepareStatement(SQL)) { - ps.setString(1, parameterName); - ResultSet rs = ps.executeQuery(); - if (rs.next()) { - return rs.getInt(1); - } - } - - // no entry in parameter_names - return null; - } - - /** - * Create the parameter name using the stored procedure which handles any concurrency - * issue we may have - * @param parameterName - * @return - */ - private Integer createParameterName(String parameterName) throws SQLException { - final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; - Integer parameterNameId; - try (CallableStatement stmt = connection.prepareCall(CALL)) { - stmt.setString(1, parameterName); - stmt.registerOutParameter(2, Types.INTEGER); - stmt.execute(); - parameterNameId = stmt.getInt(2); - } - - return parameterNameId; - } - - @Override - protected void resetBatch() { - // Called when a transaction has been rolled back because of a deadlock - // or other retryable error and we want to try and process the batch again - batchProcessor.reset(); - } - - /** - * Build the check ready query - * @param messagesByResourceType - * @return - */ - private String buildCheckReadyQuery(Map> messagesByResourceType) { - StringBuilder select = new StringBuilder(); - // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash - // FROM logical_resources AS lr, - // patient_logical_resources AS xlr - // WHERE lr.logical_resource_id = xlr.logical_resource_id - // AND xlr.logical_resource_id IN (1,2,3,4) - // UNION ALL - // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash - // FROM logical_resources AS lr, - // observation_logical_resources AS xlr - // WHERE lr.logical_resource_id = xlr.logical_resource_id - // AND xlr.logical_resource_id IN (5,6,7) - boolean first = true; - for (Map.Entry> entry: messagesByResourceType.entrySet()) { - final String resourceType = entry.getKey(); - final List messages = entry.getValue(); - final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); - if (first) { - first = false; - } else { - select.append(" UNION ALL "); - } - select.append(" SELECT lr.shard_key, lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); - select.append(" FROM logical_resources AS lr, "); - select.append(resourceType).append("_logical_resources AS xlr "); - select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); - select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); - } - - return select.toString(); - } - - @Override - protected void prepare(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { - // Get a list of all the resources for which we can see the current logical resource data. - // If the resource doesn't yet exist or its version meta doesn't the message - // then we add to the notReady list. If the resource version meta already - // exceeds the message, then we'll skip processing altogether because it - // means that there should be another message in the queue with more - // up-to-date parameters - Map messageMap = new HashMap<>(); - Map> messagesByResourceType = new HashMap<>(); - for (RemoteIndexMessage msg: messages) { - Long logicalResourceId = msg.getData().getLogicalResourceId(); - messageMap.put(logicalResourceId, msg); - - // split out the messages per resource type because we need to read from xx_logical_resources - List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); - values.add(msg); - } - - Set found = new HashSet<>(); - final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); - logger.fine(() -> "check ready query: " + checkReadyQuery); - try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { - ResultSet rs = ps.executeQuery(); - // wrap the ResultSet in a reader for easier consumption - ResultSetReader rsReader = new ResultSetReader(rs); - while (rsReader.next()) { - LogicalResourceValue lrv = LogicalResourceValue.builder() - .withShardKey(rsReader.getShort()) - .withLogicalResourceId(rsReader.getLong()) - .withResourceType(rsReader.getString()) - .withLogicalId(rsReader.getString()) - .withVersionId(rsReader.getInt()) - .withLastUpdated(rsReader.getTimestamp()) - .withParameterHash(rsReader.getString()) - .build(); - RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); - if (m == null) { - throw new IllegalStateException("query returned a logical resource which we didn't request"); - } - - // Check the values from the database to see if they match - // the information in the message. - if (m.getData().getVersionId() == lrv.getVersionId()) { - // only process this message if the parameter hash and lastUpdated - // times match - which is a good check that we're storing parameters - // from the correct transaction. If these don't match, we can simply - // say we found the data but don't need to process the message. - final String lastUpdated = lrv.getLastUpdated().toString(); - if (lrv.getParameterHash().equals(m.getData().getParameterHash()) - && lastUpdated.equals(m.getData().getLastUpdated())) { - okToProcess.add(m); - } - found.add(lrv.getLogicalResourceId()); // won't be marked as missing - } else if (m.getData().getVersionId() > lrv.getVersionId()) { - // we can skip processing this record because the database has already - // been updated with a newer version. Identify the record as having been - // found so we don't keep waiting for it - found.add(lrv.getLogicalResourceId()); - } - // if the version in the database is prior to version in the message we - // received it means that the server transaction hasn't been committed... - // so we have to wait just as though it were missing altogether - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); - throw new FHIRPersistenceException("prepare query failed"); - } - - if (found.size() < messages.size()) { - // identify the missing records and add to the notReady list - for (RemoteIndexMessage m: messages) { - if (!found.contains(m.getData().getLogicalResourceId())) { - notReady.add(m); - } - } - } - } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java new file mode 100644 index 00000000000..5c8e205f4fd --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java @@ -0,0 +1,42 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.util.Objects; + +/** + * Key used to uniquely identify a logical resource + */ +public class LogicalResourceIdentKey { + private final String resourceType; + private final String logicalId; + + /** + * Canonical constructor + * @param resourceType + * @param logicalId + */ + public LogicalResourceIdentKey(String resourceType, String logicalId) { + this.resourceType = resourceType; + this.logicalId = logicalId; + } + + @Override + public int hashCode() { + return Objects.hash(resourceType, logicalId); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof LogicalResourceIdentKey) { + LogicalResourceIdentKey that = (LogicalResourceIdentKey)obj; + return this.resourceType.equals(that.resourceType) + && this.logicalId.equals(that.logicalId); + } + return false; + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java new file mode 100644 index 00000000000..702ab132a85 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java @@ -0,0 +1,118 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.util.Objects; + +/** + * A DTO representing a record from logical_resource_ident + */ +public class LogicalResourceIdentValue implements Comparable { + private final String resourceType; + private final String logicalId; + private Long logicalResourceId; + private Integer resourceTypeId; + + public static class Builder { + private String resourceType; + private String logicalId; + private Long logicalResourceId; + + public Builder withLogicalResourceId(long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + return this; + } + public Builder withResourceType(String resourceType) { + this.resourceType = resourceType; + return this; + } + public Builder withLogicalId(String logicalId) { + this.logicalId = logicalId; + return this; + } + + /** + * Create a new {@link LogicalResourceValue} using the current state of this {@link Builder} + * @return + */ + public LogicalResourceIdentValue build() { + return new LogicalResourceIdentValue(resourceType, logicalId, logicalResourceId); + } + } + + /** + * Factor function to create a fresh instance of a {@link Builder} + * @return + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Public constructor + * @param resourceType + * @param logicalId + * @param logicalResourceId + */ + public LogicalResourceIdentValue(String resourceType, String logicalId, Long logicalResourceId) { + this.resourceType = resourceType; + this.logicalId = Objects.requireNonNull(logicalId); + this.logicalResourceId = logicalResourceId; + } + + /** + * @return the logicalResourceId + */ + public Long getLogicalResourceId() { + return logicalResourceId; + } + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } + + /** + * Set the logicalResourceId value + * @param logicalResourceId + */ + public void setLogicalResourceId(Long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + } + + @Override + public int compareTo(LogicalResourceIdentValue that) { + int result = this.resourceType.compareTo(that.resourceType); + if (0 == result) { + result = this.logicalId.compareTo(that.logicalId); + } + return result; + } + + /** + * @return the resourceTypeId + */ + public Integer getResourceTypeId() { + return resourceTypeId; + } + + /** + * @param resourceTypeId the resourceTypeId to set + */ + public void setResourceTypeId(Integer resourceTypeId) { + this.resourceTypeId = resourceTypeId; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java new file mode 100644 index 00000000000..36b7865d90c --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java @@ -0,0 +1,312 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; +import com.ibm.fhir.persistence.index.SecurityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterProcessor; + + +/** + * Processes batched parameters by pushing the values to various + * JDBC statements based on the plain variant of the schema + */ +public class PlainBatchParameterProcessor implements BatchParameterProcessor { + private static final Logger logger = Logger.getLogger(PlainBatchParameterProcessor.class.getName()); + + // A cache of the resource-type specific DAOs we've created + private final Map daoMap = new HashMap<>(); + + // Encapculates the statements for inserting whole-system level search params + private final PlainPostgresSystemParameterBatch systemDao; + + // Resource types we've touched in the current batch + private final Set resourceTypesInBatch = new HashSet<>(); + + // The database connection this consumer thread is using + private final Connection connection; + + /** + * Public constructor + * @param connection + */ + public PlainBatchParameterProcessor(Connection connection) { + this.connection = connection; + this.systemDao = new PlainPostgresSystemParameterBatch(connection); + } + + /** + * Close any resources we're holding to support a cleaner exit + */ + public void close() { + for (Map.Entry entry: daoMap.entrySet()) { + entry.getValue().close(); + } + systemDao.close(); + } + + /** + * Start processing a new batch + */ + public void startBatch() { + resourceTypesInBatch.clear(); + } + + /** + * Make sure that each statement that may contain data is cleared before we + * retry a batch + */ + public void reset() { + for (String resourceType: resourceTypesInBatch) { + PlainPostgresParameterBatch dao = daoMap.get(resourceType); + dao.close(); + } + systemDao.close(); + } + + @Override + public Short encodeShardKey(String requestShard) { + // This implementation doesn't get involved in application-based sharding + return null; + } + + /** + * Push any statements that have been batched but not yet executed + * @throws FHIRPersistenceException + */ + public void pushBatch() throws FHIRPersistenceException { + try { + for (String resourceType: resourceTypesInBatch) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Pushing batch for [" + resourceType + "]"); + } + PlainPostgresParameterBatch dao = daoMap.get(resourceType); + try { + dao.pushBatch(); + } catch (SQLException x) { + throw new FHIRPersistenceException("pushBatch failed for '" + resourceType + "'"); + } + } + + try { + logger.fine("Pushing batch for whole-system parameters"); + systemDao.pushBatch(); + } catch (SQLException x) { + throw new FHIRPersistenceException("batch insert for whole-system parameters", x); + } + } finally { + // Reset the set of active resource-types ready for the next batch + resourceTypesInBatch.clear(); + } + } + + private PlainPostgresParameterBatch getParameterBatchDao(String resourceType) { + resourceTypesInBatch.add(resourceType); + PlainPostgresParameterBatch dao = daoMap.get(resourceType); + if (dao == null) { + dao = new PlainPostgresParameterBatch(connection, resourceType); + daoMap.put(resourceType, dao); + } + return dao; + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process string parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + parameter.toString() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId()); + + if (parameter.isSystemParam()) { + systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId()); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process number parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId()); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process quantity parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId()); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting quantity params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process location parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId()); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting location params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process date parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId()); + if (p.isSystemParam()) { + systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId()); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process token parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId()); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process tag parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId()); + + if (p.isSystemParam()) { + systemDao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId()); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting tag params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter p, + CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process profile parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonCanonicalValue.getCanonicalId() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment()); + if (p.isSystemParam()) { + systemDao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment()); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting profile params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter p, + CommonTokenValue commonTokenValue) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process security parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId()); + + if (p.isSystemParam()) { + systemDao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId()); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'"); + } + } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, + ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException { + + if (logger.isLoggable(Level.FINE)) { + logger.fine("process reference parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + parameter.toString() + "] [" + refLogicalResourceId.getLogicalResourceId() + "]"); + } + + try { + PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); + dao.addReference(logicalResourceId, parameterNameValue.getParameterNameId(), refLogicalResourceId.getLogicalResourceId(), parameter.getRefVersionId()); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'"); + } + } +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java new file mode 100644 index 00000000000..006f298df55 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java @@ -0,0 +1,1218 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.ResultSetReader; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParameterValue; +import com.ibm.fhir.persistence.index.SecurityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.batch.BatchDateParameter; +import com.ibm.fhir.remote.index.batch.BatchLocationParameter; +import com.ibm.fhir.remote.index.batch.BatchNumberParameter; +import com.ibm.fhir.remote.index.batch.BatchProfileParameter; +import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; +import com.ibm.fhir.remote.index.batch.BatchReferenceParameter; +import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; +import com.ibm.fhir.remote.index.batch.BatchStringParameter; +import com.ibm.fhir.remote.index.batch.BatchTagParameter; +import com.ibm.fhir.remote.index.batch.BatchTokenParameter; + +/** + * Loads search parameter values into the target PostgreSQL database using + * the plain (non-sharded) schema variant. + */ +public class PlainPostgresMessageHandler extends BaseMessageHandler { + private static final Logger logger = Logger.getLogger(PlainPostgresMessageHandler.class.getName()); + private static final short FIXED_SHARD = 0; + + // the connection to use for the inserts + protected final Connection connection; + + // We're a PostgreSQL DAO, so we now which translator to use + protected final IDatabaseTranslator translator = new PostgresTranslator(); + + // The FHIR data schema + protected final String schemaName; + + // the cache we use for various lookups + protected final IdentityCache identityCache; + + // All logical_resource_ident values we've seen + private final Map logicalResourceIdentMap = new HashMap<>(); + + // All parameter names we've seen (cleared if there's a rollback) + private final Map parameterNameMap = new HashMap<>(); + + // A map of code system name to the value holding its codeSystemId from the database + private final Map codeSystemValueMap = new HashMap<>(); + + // A map to support lookup of CommonTokenValue records by key + private final Map commonTokenValueMap = new HashMap<>(); + + // A map to support lookup of CommonCanonicalValue records by key + private final Map commonCanonicalValueMap = new HashMap<>(); + + // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id + private final List unresolvedLogicalResourceIdents = new ArrayList<>(); + + // All parameter names in the current transaction for which we don't yet know the parameter_name_id + private final List unresolvedParameterNames = new ArrayList<>(); + + // A list of all the CodeSystemValues for which we don't yet know the code_system_id + private final List unresolvedSystemValues = new ArrayList<>(); + + // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id + private final List unresolvedTokenValues = new ArrayList<>(); + + // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id + private final List unresolvedCanonicalValues = new ArrayList<>(); + + // The processed values we've collected + private final List batchedParameterValues = new ArrayList<>(); + + // The processor used to process the batched parameter values after all the reference values are created + private final PlainBatchParameterProcessor batchProcessor; + + protected final int maxLogicalResourcesPerStatement = 256; + protected final int maxCodeSystemsPerStatement = 512; + protected final int maxCommonTokenValuesPerStatement = 256; + protected final int maxCommonCanonicalValuesPerStatement = 256; + private boolean rollbackOnly; + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param cache + * @param maxReadyTimeMs + */ + public PlainPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(maxReadyTimeMs); + this.connection = connection; + this.schemaName = schemaName; + this.identityCache = cache; + this.batchProcessor = new PlainBatchParameterProcessor(connection); + } + + @Override + protected void startBatch() { + // always start with a clean slate + batchedParameterValues.clear(); + unresolvedLogicalResourceIdents.clear(); + unresolvedParameterNames.clear(); + unresolvedSystemValues.clear(); + unresolvedTokenValues.clear(); + unresolvedCanonicalValues.clear(); + batchProcessor.startBatch(); + } + + @Override + protected void setRollbackOnly() { + this.rollbackOnly = true; + } + + @Override + public void close() { + try { + batchProcessor.close(); + } catch (Throwable t) { + logger.log(Level.SEVERE, "close batchProcessor failed" , t); + } + } + + @Override + protected void endTransaction() throws FHIRPersistenceException { + boolean committed = false; + try { + if (!this.rollbackOnly) { + logger.fine("Committing transaction"); + connection.commit(); + committed = true; + + // any values from parameter_names, code_systems and common_token_values + // are now committed to the database, so we can publish their record ids + // to the shared cache which makes them accessible from other threads + publishCachedValues(); + } else { + // something went wrong...try to roll back the transaction before we close + // everything + try { + connection.rollback(); + } catch (SQLException x) { + // It could very well be that we've lost touch with the database in which case + // the rollback will also fail. Not much we can do, although we don't bother + // with a stack trace here because it's just more noise for the log file, and + // the exception that triggered the rollback is already going to be propagated + // and logged. + logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); + } + } + } catch (SQLException x) { + throw new FHIRPersistenceException("commit failed", x); + } finally { + if (!committed) { + // The maps may contain ids that were not committed to the database so + // we should clean them out in case we decide to reuse this consumer + this.logicalResourceIdentMap.clear(); + this.parameterNameMap.clear(); + this.codeSystemValueMap.clear(); + this.commonTokenValueMap.clear(); + this.commonCanonicalValueMap.clear(); + } + } + } + + /** + * After the transaction has been committed, we can publish certain values to the + * shared identity caches + */ + private void publishCachedValues() { + // all the unresolvedParameterNames should be resolved at this point + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); + } + } + + @Override + protected void pushBatch() throws FHIRPersistenceException { + // Push any data we've accumulated so far. This may occur + // if we cross a volume threshold, and will always occur as + // the last step before the current transaction is committed, + // Process the token values so that we can establish + // any entries we need for common_token_values + resolveLogicalResourceIdents(); + resolveParameterNames(); + resolveCodeSystems(); + resolveCommonTokenValues(); + resolveCommonCanonicalValues(); + + // Now that all the lookup values should've been resolved, we can go ahead + // and push the parameters to the JDBC batch insert statements via the + // batchProcessor + for (BatchParameterValue v: this.batchedParameterValues) { + v.apply(batchProcessor); + } + batchProcessor.pushBatch(); + } + + /** + * Get the parameter name value for the given parameter value + * @param p + * @return + */ + private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("get ParameterNameValue for [" + p.toString() + "]"); + } + ParameterNameValue result = parameterNameMap.get(p.getName()); + if (result == null) { + result = new ParameterNameValue(p.getName()); + parameterNameMap.put(p.getName(), result); + + // let's see if the id is available in the shared identity cache + Integer parameterNameId = identityCache.getParameterNameId(p.getName()); + if (parameterNameId != null) { + result.setParameterNameId(parameterNameId); + } else { + // ids will be created later (so that we can process them in order) + unresolvedParameterNames.add(result); + } + } + return result; + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { + CommonCanonicalValue ctv = lookupCommonCanonicalValue(p.getUrl()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); + this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException { + logger.info("Processing reference parameter value:" + p.toString()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId()); + this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv)); + } + + /** + * Get the CodeSystemValue we've assigned for the given codeSystem value. This + * may not yet have the actual code_system_id from the database yet - any values + * we don't have will be assigned in a later phase (so we can do things neatly + * in bulk). + * @param codeSystem + * @return + */ + private CodeSystemValue lookupCodeSystemValue(String codeSystem) { + CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); + if (result == null) { + result = new CodeSystemValue(codeSystem); + this.codeSystemValueMap.put(codeSystem, result); + + // Take this opportunity to see if we have a cached value for this codeSystem + Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); + if (codeSystemId != null) { + result.setCodeSystemId(codeSystemId); + } else { + // Stash for later resolution + this.unresolvedSystemValues.add(result); + } + } + return result; + } + + /** + * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. + * The returned value may not yet have the actual common_token_value_id yet - we fetch + * these values later and create new database records as necessary. + * @param codeSystem + * @param tokenValue + * @return + */ + private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenValue) { + CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, codeSystem, tokenValue); + CommonTokenValue result = this.commonTokenValueMap.get(key); + if (result == null) { + CodeSystemValue csv = lookupCodeSystemValue(codeSystem); + result = new CommonTokenValue(FIXED_SHARD, csv, tokenValue); + this.commonTokenValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long commonTokenValueId = identityCache.getCommonTokenValueId(FIXED_SHARD, codeSystem, tokenValue); + if (commonTokenValueId != null) { + result.setCommonTokenValueId(commonTokenValueId); + } else { + this.unresolvedTokenValues.add(result); + } + } + return result; + } + + private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId); + LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key); + if (result == null) { + result = LogicalResourceIdentValue.builder() + .withResourceType(resourceType) + .withLogicalId(logicalId) + .build(); + this.logicalResourceIdentMap.put(key, result); + this.unresolvedLogicalResourceIdents.add(result); + } + return result; + } + + private CommonCanonicalValue lookupCommonCanonicalValue(String url) { + CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, url); + CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); + if (result == null) { + result = new CommonCanonicalValue(FIXED_SHARD, url); + this.commonCanonicalValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long canonicalId = identityCache.getCommonCanonicalValueId(FIXED_SHARD, url); + if (canonicalId != null) { + result.setCanonicalId(canonicalId); + } else { + this.unresolvedCanonicalValues.add(result); + } + } + return result; + } + + /** + * Make sure we have values for all the code_systems we have collected + * in the current + * batch + * @throws FHIRPersistenceException + */ + private void resolveCodeSystems() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCodeSystemIds(unresolvedSystemValues); + + if (!missing.isEmpty()) { + addMissingCodeSystems(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCodeSystemIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all code system values"); + } + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); + for (int i=0; i 0) { + query.append(","); + } + query.append("?"); + } + query.append(")"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (CodeSystemValue csv: values) { + ps.setString(param++, csv.getCodeSystem()); + } + return ps; + } + + /** + * These code systems weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingCodeSystems(List missing) throws FHIRPersistenceException { + List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); + // Sort the code system values first to help avoid deadlocks + Collections.sort(values); // natural ordering for String is fine here + + final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); + insert.append(nextVal); // next sequence value + insert.append(",?) ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (String codeSystem: values) { + ps.setString(1, codeSystem); + ps.addBatch(); + if (++count == this.maxCodeSystemsPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); + if (csv != null) { + csv.setCodeSystemId(rs.getInt(1)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("code systems query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CodeSystemValue csv: sub) { + if (csv.getCodeSystemId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed", x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Make sure we have values for all the common_token_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonTokenValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCommonTokenValueIds(unresolvedTokenValues); + + if (!missing.isEmpty()) { + // Sort first to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getTokenValue().compareTo(b.getTokenValue()); + if (result == 0) { + result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + } + return result; + }); + addMissingCommonTokenValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCommonTokenValueIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all common token values"); + } + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + // need the code_system name - so we join back to the code_systems table as well + query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id "); + query.append(" FROM common_token_values c"); + query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); + query.append(" JOIN (VALUES "); + + // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records + boolean first = true; + for (CommonTokenValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id + query.append(",?)"); // bind variable for the token-value + } + query.append(") AS v(code_system_id, token_value) "); + query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value)"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonTokenValue ctv: values) { + ps.setString(param++, ctv.getTokenValue()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, rs.getString(1), rs.getString(2)); + CommonTokenValue ctv = this.commonTokenValueMap.get(key); + if (ctv != null) { + ctv.setCommonTokenValueId(rs.getLong(3)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common token values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonTokenValue ctv: sub) { + if (ctv.getCommonTokenValueId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common token values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_token_values (code_system_id, token_value) "); + insert.append(" VALUES (?,?) "); + insert.append("ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (CommonTokenValue ctv: missing) { + ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId()); + ps.setString(2, ctv.getTokenValue()); + ps.addBatch(); + if (++count == this.maxCommonTokenValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common token values"); + } + } + + /** + * Make sure we have values for all the common_canonical_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonCanonicalValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCanonicalIds(unresolvedCanonicalValues); + + if (!missing.isEmpty()) { + // Sort on url to minimize deadlocks + Collections.sort(missing, (a,b) -> { + return a.getUrl().compareTo(b.getUrl()); + }); + addMissingCommonCanonicalValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCanonicalIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all canonical values"); + } + } + + private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, rs.getString(1)); + CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); + if (ctv != null) { + ctv.setCanonicalId(rs.getLong(2)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonCanonicalValue ctv: sub) { + if (ctv.getCanonicalId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common canonical values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT c.url, c.canonical_id "); + query.append(" FROM common_canonical_values c "); + query.append(" WHERE c.url IN ("); + + // add bind variables for each url we need to fetch + boolean first = true; + for (CommonCanonicalValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("?"); // bind variable for the url + } + query.append(")"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + logger.finer(() -> "fetch common canonical values [" + statementText + "]"); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonCanonicalValue ctv: values) { + ps.setString(param++, ctv.getUrl()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (url) VALUES (?) "); + insert.append("ON CONFLICT DO NOTHING"); + + final String DML = insert.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("addMissingCanonicalIds: " + DML); + } + try (PreparedStatement ps = connection.prepareStatement(DML)) { + int count = 0; + for (CommonCanonicalValue ctv: missing) { + logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); + ps.setString(1, ctv.getUrl()); + ps.addBatch(); + if (++count == this.maxCommonCanonicalValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common canonical values"); + } + } + + /** + * Make sure all the parameter names we've seen in the batch exist + * in the database and have ids. + * @throws FHIRPersistenceException + */ + private void resolveParameterNames() throws FHIRPersistenceException { + // We expect parameter names to have a very high cache hit rate and + // so we simplify processing by simply iterating one-by-one for the + // values we still need to resolve. The most important point here is + // to do this in a sorted order to avoid deadlock issues because this + // could be happening across multiple consumer threads at the same time. + Collections.sort(this.unresolvedParameterNames, (a,b) -> { + return a.getParameterName().compareTo(b.getParameterName()); + }); + + try { + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); + if (parameterNameId == null) { + parameterNameId = createParameterName(pnv.getParameterName()); + } + pnv.setParameterNameId(parameterNameId); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("error resolving parameter names", x); + } + } + + private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { + String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ps.setString(1, parameterName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return rs.getInt(1); + } + } + + // no entry in parameter_names + return null; + } + + /** + * Create the parameter name using the stored procedure which handles any concurrency + * issue we may have + * @param parameterName + * @return + */ + private Integer createParameterName(String parameterName) throws SQLException { + final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; + Integer parameterNameId; + try (CallableStatement stmt = connection.prepareCall(CALL)) { + stmt.setString(1, parameterName); + stmt.registerOutParameter(2, Types.INTEGER); + stmt.execute(); + parameterNameId = stmt.getInt(2); + } + + return parameterNameId; + } + + @Override + protected void resetBatch() { + // Called when a transaction has been rolled back because of a deadlock + // or other retryable error and we want to try and process the batch again + batchProcessor.reset(); + } + + /** + * Build the check ready query + * @param messagesByResourceType + * @return + */ + private String buildCheckReadyQuery(Map> messagesByResourceType) { + // The trouble here is that we'll end up with a unique query for every single + // batch of messages we process (which the database then need to parse etc). + // This may introduce scaling issues, in which case we should consider + // individual queries for each resource type using bind variables, perhaps + // going so far as using multiple statements with a power-of-2 number of bind + // variables. But JDBC doesn't support batching of select statements, so + // the alternative there would be to insert-as-select into a global temp table + // and then simply select from that. Fairly straightforward, but a lot more + // work so only worth doing if we identify contention here. + + StringBuilder select = new StringBuilder(); + // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // patient_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (1,2,3,4) + // UNION ALL + // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // observation_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (5,6,7) + boolean first = true; + for (Map.Entry> entry: messagesByResourceType.entrySet()) { + final String resourceType = entry.getKey(); + final List messages = entry.getValue(); + final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); + if (first) { + first = false; + } else { + select.append(" UNION ALL "); + } + select.append(" SELECT lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); + select.append(" FROM logical_resources AS lr, "); + select.append(resourceType).append("_logical_resources AS xlr "); + select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); + select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); + } + + return select.toString(); + } + + @Override + protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { + // Get a list of all the resources for which we can see the current logical resource data. + // If the resource doesn't yet exist or its version meta doesn't the message + // then we add to the notReady list. If the resource version meta already + // exceeds the message, then we'll skip processing altogether because it + // means that there should be another message in the queue with more + // up-to-date parameters + Map messageMap = new HashMap<>(); + Map> messagesByResourceType = new HashMap<>(); + for (RemoteIndexMessage msg: messages) { + Long logicalResourceId = msg.getData().getLogicalResourceId(); + messageMap.put(logicalResourceId, msg); + + // split out the messages per resource type because we need to read from xx_logical_resources + List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); + values.add(msg); + } + + Set found = new HashSet<>(); + final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); + logger.fine(() -> "check ready query: " + checkReadyQuery); + try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { + ResultSet rs = ps.executeQuery(); + // wrap the ResultSet in a reader for easier consumption + ResultSetReader rsReader = new ResultSetReader(rs); + while (rsReader.next()) { + LogicalResourceValue lrv = LogicalResourceValue.builder() + .withLogicalResourceId(rsReader.getLong()) + .withResourceType(rsReader.getString()) + .withLogicalId(rsReader.getString()) + .withVersionId(rsReader.getInt()) + .withLastUpdated(rsReader.getTimestamp()) + .withParameterHash(rsReader.getString()) + .build(); + RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); + if (m == null) { + throw new IllegalStateException("query returned a logical resource which we didn't request"); + } + + // Check the values from the database to see if they match + // the information in the message. + if (m.getData().getVersionId() == lrv.getVersionId()) { + // only process this message if the parameter hash and lastUpdated + // times match - which is a good check that we're storing parameters + // from the correct transaction. If these don't match, we can simply + // say we found the data but don't need to process the message. + final Instant dbLastUpdated = lrv.getLastUpdated().toInstant(); + final Instant msgLastUpdated = Instant.parse(m.getData().getLastUpdated()); + if (lrv.getParameterHash().equals(m.getData().getParameterHash()) + && dbLastUpdated.equals(msgLastUpdated)) { + okToProcess.add(m); + } else { + logger.warning("Parameter message must match both parameter_hash and last_updated. Must be from an uncommitted transaction so ignoring: " + m.toString()); + } + found.add(lrv.getLogicalResourceId()); // won't be marked as missing + } else if (m.getData().getVersionId() > lrv.getVersionId()) { + // we can skip processing this record because the database has already + // been updated with a newer version. Identify the record as having been + // found so we don't keep waiting for it + found.add(lrv.getLogicalResourceId()); + } + // if the version in the database is prior to version in the message we + // received it means that the server transaction hasn't been committed... + // so we have to wait just as though it were missing altogether + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); + throw new FHIRPersistenceException("prepare query failed"); + } + + if (found.size() < messages.size()) { + // identify the missing records and add to the notReady list + for (RemoteIndexMessage m: messages) { + if (!found.contains(m.getData().getLogicalResourceId())) { + notReady.add(m); + } + } + } + } + + + + + /** + * Make sure we have values for all the logical_resource_ident values + * we have collected in the current batch. Need to make sure these are + * added in order to minimize deadlocks. Note that because we may create + * new logical_resource_ident records, we could be blocked by the main + * add_any_resource procedure run within the server CREATE/UPDATE + * transaction. + * @throws FHIRPersistenceException + */ + private void resolveLogicalResourceIdents() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchLogicalResourceIdentIds(unresolvedLogicalResourceIdents); + + if (!missing.isEmpty()) { + addMissingLogicalResourceIdents(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchLogicalResourceIdentIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all logical_resource_ident values"); + } + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + private PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT rt.resource_type, lri.logical_id, rt.resource_type_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" JOIN resource_types AS rt ON (lri.resource_type_id = rt.resource_type_id) "); + query.append(" JOIN (VALUES "); + for (int i=0; i 0) { + query.append(","); + } + query.append("(?,?)"); + } + query.append(") AS v(resource_type, logical_id) "); + query.append(" ON (rt.resource_type = v.resource_type AND lri.logical_id = v.logical_id)"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setString(param++, val.getResourceType()); + ps.setString(param++, val.getLogicalId()); + } + return ps; + } + + /** + * These logical_resource_ident values weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { + // Sort the values first to help avoid deadlocks + Collections.sort(missing, (a,b) -> { + return a.compareTo(b); + }); + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (LogicalResourceIdentValue value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ps.addBatch(); + if (++count == this.maxLogicalResourcesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + private List fetchLogicalResourceIdentIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each LogicalResourceIdentValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + LogicalResourceIdentKey key = new LogicalResourceIdentKey(rs.getString(1), rs.getString(2)); + LogicalResourceIdentValue csv = this.logicalResourceIdentMap.get(key); + if (csv != null) { + csv.setResourceTypeId(rs.getInt(3)); + csv.setLogicalResourceId(rs.getLong(4)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("logical resource ident query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (LogicalResourceIdentValue csv: sub) { + if (csv.getLogicalResourceId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java new file mode 100644 index 00000000000..21c19b58636 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java @@ -0,0 +1,416 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; + +/** + * Parameter batch statements configured for a given resource type + * using the plain schema variant + */ +public class PlainPostgresParameterBatch { + private static final Logger logger = Logger.getLogger(PlainPostgresParameterBatch.class.getName()); + + private final Connection connection; + private final String resourceType; + + private PreparedStatement strings; + private int stringCount; + + private PreparedStatement numbers; + private int numberCount; + + private PreparedStatement dates; + private int dateCount; + + private PreparedStatement quantities; + private int quantityCount; + + private PreparedStatement locations; + private int locationCount; + + private PreparedStatement resourceTokenRefs; + private int resourceTokenRefCount; + + private PreparedStatement tags; + private int tagCount; + + private PreparedStatement profiles; + private int profileCount; + + private PreparedStatement security; + private int securityCount; + + private PreparedStatement refs; + private int refCount; + + /** + * Public constructor + * @param c + * @param resourceType + */ + public PlainPostgresParameterBatch(Connection c, String resourceType) { + this.connection = c; + this.resourceType = resourceType; + } + + /** + * Push the current batch + */ + public void pushBatch() throws SQLException { + if (stringCount > 0) { + strings.executeBatch(); + stringCount = 0; + } + if (numberCount > 0) { + numbers.executeBatch(); + numberCount = 0; + } + if (dateCount > 0) { + dates.executeBatch(); + dateCount = 0; + } + if (quantityCount > 0) { + quantities.executeBatch(); + quantityCount = 0; + } + if (locationCount > 0) { + locations.executeBatch(); + locationCount = 0; + } + if (resourceTokenRefCount > 0) { + resourceTokenRefs.executeBatch(); + resourceTokenRefCount = 0; + } + if (tagCount > 0) { + tags.executeBatch(); + tagCount = 0; + } + if (profileCount > 0) { + profiles.executeBatch(); + profileCount = 0; + } + if (securityCount > 0) { + security.executeBatch(); + securityCount = 0; + } + if (refCount > 0) { + refs.executeBatch(); + refCount = 0; + } + } + + /** + * Resets the state of the DAO by closing all statements and + * setting any batch counts to 0 + */ + public void close() { + if (strings != null) { + try { + strings.close(); + } catch (SQLException x) { + // NOP + } finally { + strings = null; + stringCount = 0; + } + } + + if (numbers != null) { + try { + numbers.close(); + } catch (SQLException x) { + // NOP + } finally { + numbers = null; + numberCount = 0; + } + } + + if (dates != null) { + try { + dates.close(); + } catch (SQLException x) { + // NOP + } finally { + dates = null; + dateCount = 0; + } + } + + if (quantities != null) { + try { + quantities.close(); + } catch (SQLException x) { + // NOP + } finally { + quantities = null; + quantityCount = 0; + } + } + + if (locations != null) { + try { + locations.close(); + } catch (SQLException x) { + // NOP + } finally { + locations = null; + locationCount = 0; + } + } + + if (resourceTokenRefs != null) { + try { + resourceTokenRefs.close(); + } catch (SQLException x) { + // NOP + } finally { + resourceTokenRefs = null; + resourceTokenRefCount = 0; + } + } + if (tags != null) { + try { + tags.close(); + } catch (SQLException x) { + // NOP + } finally { + tags = null; + tagCount = 0; + } + } + if (profiles != null) { + try { + profiles.close(); + } catch (SQLException x) { + // NOP + } finally { + profiles = null; + profileCount = 0; + } + } + if (security != null) { + try { + security.close(); + } catch (SQLException x) { + // NOP + } finally { + security = null; + securityCount = 0; + } + } + if (refs != null) { + try { + refs.close(); + } catch (SQLException x) { + // NOP + } finally { + refs = null; + refCount = 0; + } + } + } + + /** + * Set the compositeId on the given PreparedStatement, handling a value if necessary + * @param ps + * @param index + * @param compositeId + * @throws SQLException + */ + private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException { + if (compositeId != null) { + ps.setInt(index, compositeId); + } else { + ps.setNull(index, Types.INTEGER); + } + } + /** + * Utility method to set a string value and handle null + * @param ps + * @param index + * @param value + * @throws SQLException + */ + private void setString(PreparedStatement ps, int index, String value) throws SQLException { + if (value == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, value); + } + } + + public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId) throws SQLException { + if (strings == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + strings = connection.prepareStatement(insertString); + } + + strings.setInt(1, parameterNameId); + strings.setString(2, strValue); + strings.setString(3, strValueLower); + strings.setLong(4, logicalResourceId); + setComposite(strings, 5, compositeId); + strings.addBatch(); + stringCount++; + } + + public void addNumber(long logicalResourceId, int parameterNameId, BigDecimal value, BigDecimal valueLow, BigDecimal valueHigh, Integer compositeId) throws SQLException { + if (numbers == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertNumber = "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?)"; + numbers = connection.prepareStatement(insertNumber); + } + numbers.setInt(1, parameterNameId); + numbers.setBigDecimal(2, value); + numbers.setBigDecimal(3, valueLow); + numbers.setBigDecimal(4, valueHigh); + numbers.setLong(5, logicalResourceId); + setComposite(numbers, 6, compositeId); + numbers.addBatch(); + numberCount++; + } + + public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId) throws SQLException { + if (dates == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertDate = "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + dates = connection.prepareStatement(insertDate); + } + + final Calendar UTC = CalendarHelper.getCalendarForUTC(); + dates.setInt(1, parameterNameId); + dates.setTimestamp(2, dateStart, UTC); + dates.setTimestamp(3, dateEnd, UTC); + dates.setLong(4, logicalResourceId); + setComposite(dates, 5, compositeId); + dates.addBatch(); + dateCount++; + } + + public void addQuantity(long logicalResourceId, int parameterNameId, Integer codeSystemId, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId) throws SQLException { + if (quantities == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertQuantity = "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?,?,?)"; + quantities = connection.prepareStatement(insertQuantity); + } + + quantities.setInt(1, parameterNameId); + quantities.setInt(2, codeSystemId); + quantities.setString(3, valueCode); + quantities.setBigDecimal(4, valueNumber); + quantities.setBigDecimal(5, valueNumberLow); + quantities.setBigDecimal(6, valueNumberHigh); + quantities.setLong(7, logicalResourceId); + setComposite(quantities, 8, compositeId); + quantities.addBatch(); + quantityCount++; + } + + public void addLocation(long logicalResourceId, int parameterNameId, Double lat, Double lng, Integer compositeId) throws SQLException { + if (locations == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertLocation = "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + locations = connection.prepareStatement(insertLocation); + } + + locations.setInt(1, parameterNameId); + locations.setDouble(2, lat); + locations.setDouble(3, lng); + locations.setLong(4, logicalResourceId); + setComposite(locations, 5, compositeId); + locations.addBatch(); + locationCount++; + } + + public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId, Integer refVersionId, Integer compositeId) throws SQLException { + if (resourceTokenRefs == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, ref_version_id, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + resourceTokenRefs = connection.prepareStatement(tokenString); + } + resourceTokenRefs.setInt(1, parameterNameId); + resourceTokenRefs.setLong(2, commonTokenValueId); + setComposite(resourceTokenRefs, 3, refVersionId); + resourceTokenRefs.setLong(4, logicalResourceId); + setComposite(resourceTokenRefs, 5, compositeId); + resourceTokenRefs.addBatch(); + resourceTokenRefCount++; + } + + public void addTag(long logicalResourceId, long commonTokenValueId) throws SQLException { + if (tags == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_tags (common_token_value_id, logical_resource_id) VALUES (?,?)"; + tags = connection.prepareStatement(tokenString); + } + tags.setLong(1, commonTokenValueId); + tags.setLong(2, logicalResourceId); + tags.addBatch(); + tagCount++; + } + + public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment) throws SQLException { + if (profiles == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String tokenString = "INSERT INTO " + tablePrefix + "_profiles (canonical_id, logical_resource_id, version, fragment) VALUES (?,?,?,?)"; + profiles = connection.prepareStatement(tokenString); + } + profiles.setLong(1, canonicalId); + profiles.setLong(2, logicalResourceId); + setString(profiles, 3, version); + setString(profiles, 4, fragment); + profiles.addBatch(); + profileCount++; + } + + public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException { + if (tags == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String INS = "INSERT INTO " + tablePrefix + "_security (common_token_value_id, logical_resource_id) VALUES (?,?)"; + security = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(security); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .addBatch(); + securityCount++; + } + + /** + * @param logicalResourceId + * @param refLogicalResourceId + * @param refVersionId + */ + public void addReference(long logicalResourceId, int parameterNameId, long refLogicalResourceId, Integer refVersionId) throws SQLException { + logger.info("Adding reference: parameterNameId:" + parameterNameId + " refLogicalResourceId:" + refLogicalResourceId + " refVersionId:" + refVersionId); + if (refs == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertString = "INSERT INTO " + tablePrefix + "_ref_values (parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id) VALUES (?,?,?,?)"; + refs = connection.prepareStatement(insertString); + } + PreparedStatementHelper psh = new PreparedStatementHelper(refs); + psh.setInt(parameterNameId) + .setLong(logicalResourceId) + .setLong(refLogicalResourceId) + .setInt(refVersionId) + .addBatch(); + refCount++; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java new file mode 100644 index 00000000000..3967ff5bdae --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java @@ -0,0 +1,216 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; + +import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; + +/** + * Batch insert statements for system-level parameters + * @implNote targets the plain variant of the schema + * without an explicit sharding column + */ +public class PlainPostgresSystemParameterBatch { + private final Connection connection; + + private PreparedStatement systemStrings; + private int systemStringCount; + + private PreparedStatement systemDates; + private int systemDateCount; + + private PreparedStatement systemProfiles; + private int systemProfileCount; + + private PreparedStatement systemTags; + private int systemTagCount; + + private PreparedStatement systemSecurity; + private int systemSecurityCount; + + /** + * Public constructor + * @param c + */ + public PlainPostgresSystemParameterBatch(Connection c) { + this.connection = c; + } + + /** + * Push the current batch + */ + public void pushBatch() throws SQLException { + if (systemStringCount > 0) { + systemStrings.executeBatch(); + systemStringCount = 0; + } + if (systemDateCount > 0) { + systemDates.executeBatch(); + systemDateCount = 0; + } + if (systemTagCount > 0) { + systemTags.executeBatch(); + systemTagCount = 0; + } + if (systemProfileCount > 0) { + systemProfiles.executeBatch(); + systemProfileCount = 0; + } + if (systemSecurityCount > 0) { + systemSecurity.executeBatch(); + systemSecurityCount = 0; + } + } + + /** + * Closes all the statements currently open + */ + public void close() { + + if (systemStrings != null) { + try { + systemStrings.close(); + } catch (SQLException x) { + // NOP + } finally { + systemStrings = null; + systemStringCount = 0; + } + } + + if (systemDates != null) { + try { + systemDates.close(); + } catch (SQLException x) { + // NOP + } finally { + systemDates = null; + systemDateCount = 0; + } + } + if (systemTags != null) { + try { + systemTags.close(); + } catch (SQLException x) { + // NOP + } finally { + systemTags = null; + systemTagCount = 0; + } + } + if (systemProfiles != null) { + try { + systemProfiles.close(); + } catch (SQLException x) { + // NOP + } finally { + systemProfiles = null; + systemProfileCount = 0; + } + } + if (systemSecurity != null) { + try { + systemSecurity.close(); + } catch (SQLException x) { + // NOP + } finally { + systemSecurity = null; + systemProfileCount = 0; + } + } + } + + /** + * Set the compositeId on the given PreparedStatement, handling a value if necessary + * @param ps + * @param index + * @param compositeId + * @throws SQLException + */ + private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException { + if (compositeId != null) { + ps.setInt(index, compositeId); + } else { + ps.setNull(index, Types.INTEGER); + } + } + + public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId) throws SQLException { + // System level string attributes + if (systemStrings == null) { + final String insertSystemString = "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; + systemStrings = connection.prepareStatement(insertSystemString); + } + systemStrings.setInt(1, parameterNameId); + systemStrings.setString(2, strValue); + systemStrings.setString(3, strValueLower); + systemStrings.setLong(4, logicalResourceId); + setComposite(systemStrings, 5, compositeId); + systemStrings.addBatch(); + systemStringCount++; + } + + public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId) throws SQLException { + if (systemDates == null) { + final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?,?)"; + systemDates = connection.prepareStatement(insertSystemDate); + } + final Calendar UTC = CalendarHelper.getCalendarForUTC(); + systemDates.setInt(1, parameterNameId); + systemDates.setTimestamp(2, dateStart, UTC); + systemDates.setTimestamp(3, dateEnd, UTC); + systemDates.setLong(4, logicalResourceId); + setComposite(systemDates, 5, compositeId); + systemDates.addBatch(); + systemDateCount++; + } + + public void addTag(long logicalResourceId, long commonTokenValueId) throws SQLException { + if (systemTags == null) { + final String INS = "INSERT INTO logical_resource_tags(common_token_value_id, logical_resource_id) VALUES (?,?)"; + systemTags = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemTags); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .addBatch(); + systemTagCount++; + } + + public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment) throws SQLException { + if (systemProfiles == null) { + final String INS = "INSERT INTO logical_resource_profiles(canonical_id, logical_resource_id, version, fragment) VALUES (?,?,?,?)"; + systemProfiles = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemProfiles); + psh.setLong(canonicalId) + .setLong(logicalResourceId) + .setString(version) + .setString(fragment) + .addBatch(); + systemProfileCount++; + } + + public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException { + if (systemTags == null) { + final String INS = "INSERT INTO logical_resource_security(common_token_value_id, logical_resource_id) VALUES (?,?)"; + systemSecurity = connection.prepareStatement(INS); + } + PreparedStatementHelper psh = new PreparedStatementHelper(systemSecurity); + psh.setLong(commonTokenValueId) + .setLong(logicalResourceId) + .addBatch(); + systemSecurityCount++; + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java similarity index 79% rename from fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java rename to fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java index de559a51db4..50cc05df8f0 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/JDBCBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.remote.index.database; +package com.ibm.fhir.remote.index.sharded; import java.sql.Connection; import java.sql.SQLException; @@ -21,25 +21,32 @@ import com.ibm.fhir.persistence.index.NumberParameter; import com.ibm.fhir.persistence.index.ProfileParameter; import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; import com.ibm.fhir.persistence.index.SecurityParameter; import com.ibm.fhir.persistence.index.StringParameter; import com.ibm.fhir.persistence.index.TagParameter; import com.ibm.fhir.persistence.index.TokenParameter; import com.ibm.fhir.remote.index.api.BatchParameterProcessor; +import com.ibm.fhir.remote.index.database.CodeSystemValue; +import com.ibm.fhir.remote.index.database.CommonCanonicalValue; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; /** * Processes batched parameters by pushing the values to various - * JDBC statements + * JDBC statements based on the distributed (shard_key) variant + * of the schema */ -public class JDBCBatchParameterProcessor implements BatchParameterProcessor { - private static final Logger logger = Logger.getLogger(JDBCBatchParameterProcessor.class.getName()); +public class ShardedBatchParameterProcessor implements BatchParameterProcessor { + private static final Logger logger = Logger.getLogger(ShardedBatchParameterProcessor.class.getName()); // A cache of the resource-type specific DAOs we've created - private final Map daoMap = new HashMap<>(); + private final Map daoMap = new HashMap<>(); // Encapculates the statements for inserting whole-system level search params - private final DistributedPostgresSystemParameterBatch systemDao; + private final ShardedPostgresSystemParameterBatch systemDao; // Resource types we've touched in the current batch private final Set resourceTypesInBatch = new HashSet<>(); @@ -51,16 +58,16 @@ public class JDBCBatchParameterProcessor implements BatchParameterProcessor { * Public constructor * @param connection */ - public JDBCBatchParameterProcessor(Connection connection) { + public ShardedBatchParameterProcessor(Connection connection) { this.connection = connection; - this.systemDao = new DistributedPostgresSystemParameterBatch(connection); + this.systemDao = new ShardedPostgresSystemParameterBatch(connection); } /** * Close any resources we're holding to support a cleaner exit */ public void close() { - for (Map.Entry entry: daoMap.entrySet()) { + for (Map.Entry entry: daoMap.entrySet()) { entry.getValue().close(); } systemDao.close(); @@ -79,7 +86,7 @@ public void startBatch() { */ public void reset() { for (String resourceType: resourceTypesInBatch) { - DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + ShardedPostgresParameterBatch dao = daoMap.get(resourceType); dao.close(); } systemDao.close(); @@ -94,7 +101,7 @@ public void pushBatch() throws FHIRPersistenceException { if (logger.isLoggable(Level.FINE)) { logger.fine("Pushing batch for [" + resourceType + "]"); } - DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + ShardedPostgresParameterBatch dao = daoMap.get(resourceType); try { dao.pushBatch(); } catch (SQLException x) { @@ -114,11 +121,11 @@ public void pushBatch() throws FHIRPersistenceException { } } - private DistributedPostgresParameterBatch getParameterBatchDao(String resourceType) { + private ShardedPostgresParameterBatch getParameterBatchDao(String resourceType) { resourceTypesInBatch.add(resourceType); - DistributedPostgresParameterBatch dao = daoMap.get(resourceType); + ShardedPostgresParameterBatch dao = daoMap.get(resourceType); if (dao == null) { - dao = new DistributedPostgresParameterBatch(connection, resourceType); + dao = new ShardedPostgresParameterBatch(connection, resourceType); daoMap.put(resourceType, dao); } return dao; @@ -141,7 +148,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey); @@ -161,7 +168,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId(), shardKey); } catch (SQLException x) { @@ -177,7 +184,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId(), shardKey); } catch (SQLException x) { @@ -193,7 +200,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId(), shardKey); } catch (SQLException x) { @@ -209,7 +216,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId(), shardKey); if (p.isSystemParam()) { @@ -230,7 +237,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId(), shardKey); } catch (SQLException x) { @@ -248,7 +255,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); @@ -270,7 +277,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment(), shardKey); if (p.isSystemParam()) { @@ -291,7 +298,7 @@ public void process(String requestShard, String resourceType, String logicalId, } try { - DistributedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); final Short shardKey = encodeShardKey(requestShard); dao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey); @@ -302,4 +309,21 @@ public void process(String requestShard, String resourceType, String logicalId, throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'"); } } + + @Override + public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, + ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINE)) { + logger.fine("process ref parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] [" + + parameter.toString() + "] [" + refLogicalResourceId.getLogicalResourceId() + "]"); + } + + try { + ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType); + final Short shardKey = encodeShardKey(requestShard); + dao.addReference(logicalResourceId, parameterNameValue.getParameterNameId(), refLogicalResourceId.getLogicalResourceId(), parameter.getRefVersionId(), shardKey); + } catch (SQLException x) { + throw new FHIRPersistenceException("Failed inserting ref param for '" + resourceType + "'"); + } + } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java new file mode 100644 index 00000000000..6953a73f0a6 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java @@ -0,0 +1,1088 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.sharded; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.ResultSetReader; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParameterValue; +import com.ibm.fhir.persistence.index.SecurityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.batch.BatchDateParameter; +import com.ibm.fhir.remote.index.batch.BatchLocationParameter; +import com.ibm.fhir.remote.index.batch.BatchNumberParameter; +import com.ibm.fhir.remote.index.batch.BatchProfileParameter; +import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; +import com.ibm.fhir.remote.index.batch.BatchReferenceParameter; +import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; +import com.ibm.fhir.remote.index.batch.BatchStringParameter; +import com.ibm.fhir.remote.index.batch.BatchTagParameter; +import com.ibm.fhir.remote.index.batch.BatchTokenParameter; +import com.ibm.fhir.remote.index.database.BaseMessageHandler; +import com.ibm.fhir.remote.index.database.CodeSystemValue; +import com.ibm.fhir.remote.index.database.CommonCanonicalValue; +import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey; +import com.ibm.fhir.remote.index.database.CommonTokenValue; +import com.ibm.fhir.remote.index.database.CommonTokenValueKey; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentKey; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue; +import com.ibm.fhir.remote.index.database.LogicalResourceValue; +import com.ibm.fhir.remote.index.database.ParameterNameValue; +import com.ibm.fhir.remote.index.database.PlainPostgresMessageHandler; +import com.ibm.fhir.remote.index.database.PreparedStatementWrapper; + +/** + * Loads search parameter values into the target FHIR schema on + * a PostgreSQL database. + * TODO refactor to try and share more processing with the {@link PlainPostgresMessageHandler} + */ +public class ShardedPostgresMessageHandler extends BaseMessageHandler { + private static final Logger logger = Logger.getLogger(ShardedPostgresMessageHandler.class.getName()); + + // the connection to use for the inserts + private final Connection connection; + + // We're a PostgreSQL DAO, so we now which translator to use + private final IDatabaseTranslator translator = new PostgresTranslator(); + + // The FHIR data schema + private final String schemaName; + + // the cache we use for various lookups + private final IdentityCache identityCache; + + // All logical_resource_ident values we've seen + private final Map logicalResourceIdentMap = new HashMap<>(); + + // All parameter names we've seen (cleared if there's a rollback) + private final Map parameterNameMap = new HashMap<>(); + + // A map of code system name to the value holding its codeSystemId from the database + private final Map codeSystemValueMap = new HashMap<>(); + + // A map to support lookup of CommonTokenValue records by key + private final Map commonTokenValueMap = new HashMap<>(); + + // A map to support lookup of CommonCanonicalValue records by key + private final Map commonCanonicalValueMap = new HashMap<>(); + + // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id + private final List unresolvedLogicalResourceIdents = new ArrayList<>(); + + // All parameter names in the current transaction for which we don't yet know the parameter_name_id + private final List unresolvedParameterNames = new ArrayList<>(); + + // A list of all the CodeSystemValues for which we don't yet know the code_system_id + private final List unresolvedSystemValues = new ArrayList<>(); + + // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id + private final List unresolvedTokenValues = new ArrayList<>(); + + // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id + private final List unresolvedCanonicalValues = new ArrayList<>(); + + // The processed values we've collected + private final List batchedParameterValues = new ArrayList<>(); + + // The processor used to process the batched parameter values after all the reference values are created + private final ShardedBatchParameterProcessor batchProcessor; + + private final int maxCodeSystemsPerStatement = 512; + private final int maxCommonTokenValuesPerStatement = 256; + private final int maxCommonCanonicalValuesPerStatement = 256; + private boolean rollbackOnly; + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param cache + * @param maxReadyTimeMs + */ + public ShardedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(maxReadyTimeMs); + this.connection = connection; + this.schemaName = schemaName; + this.identityCache = cache; + this.batchProcessor = new ShardedBatchParameterProcessor(connection); + } + + @Override + protected void startBatch() { + // always start with a clean slate + batchedParameterValues.clear(); + unresolvedParameterNames.clear(); + unresolvedSystemValues.clear(); + unresolvedTokenValues.clear(); + unresolvedCanonicalValues.clear(); + batchProcessor.startBatch(); + } + + @Override + protected void setRollbackOnly() { + this.rollbackOnly = true; + } + + @Override + public void close() { + try { + batchProcessor.close(); + } catch (Throwable t) { + logger.log(Level.SEVERE, "close batchProcessor failed" , t); + } + } + + @Override + protected void endTransaction() throws FHIRPersistenceException { + boolean committed = false; + try { + if (!this.rollbackOnly) { + logger.fine("Committing transaction"); + connection.commit(); + committed = true; + + // any values from parameter_names, code_systems and common_token_values + // are now committed to the database, so we can publish their record ids + // to the shared cache which makes them accessible from other threads + publishCachedValues(); + } else { + // something went wrong...try to roll back the transaction before we close + // everything + try { + connection.rollback(); + } catch (SQLException x) { + // It could very well be that we've lost touch with the database in which case + // the rollback will also fail. Not much we can do, although we don't bother + // with a stack trace here because it's just more noise for the log file, and + // the exception that triggered the rollback is already going to be propagated + // and logged. + logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); + } + } + } catch (SQLException x) { + throw new FHIRPersistenceException("commit failed", x); + } finally { + if (!committed) { + // The maps may contain ids that were not committed to the database so + // we should clean them out in case we decide to reuse this consumer + this.parameterNameMap.clear(); + this.codeSystemValueMap.clear(); + this.commonTokenValueMap.clear(); + this.commonCanonicalValueMap.clear(); + } + } + } + + /** + * After the transaction has been committed, we can publish certain values to the + * shared identity caches + */ + public void publishCachedValues() { + // all the unresolvedParameterNames should be resolved at this point + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); + } + } + + @Override + protected void pushBatch() throws FHIRPersistenceException { + // Push any data we've accumulated so far. This may occur + // if we cross a volume threshold, and will always occur as + // the last step before the current transaction is committed, + // Process the token values so that we can establish + // any entries we need for common_token_values + resolveParameterNames(); + resolveCodeSystems(); + resolveCommonTokenValues(); + resolveCommonCanonicalValues(); + + // Now that all the lookup values should've been resolved, we can go ahead + // and push the parameters to the JDBC batch insert statements via the + // batchProcessor + for (BatchParameterValue v: this.batchedParameterValues) { + v.apply(batchProcessor); + } + batchProcessor.pushBatch(); + } + + /** + * Get the parameter name value for the given parameter value + * @param p + * @return + */ + private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("get ParameterNameValue for [" + p.toString() + "]"); + } + ParameterNameValue result = parameterNameMap.get(p.getName()); + if (result == null) { + result = new ParameterNameValue(p.getName()); + parameterNameMap.put(p.getName(), result); + + // let's see if the id is available in the shared identity cache + Integer parameterNameId = identityCache.getParameterNameId(p.getName()); + if (parameterNameId != null) { + result.setParameterNameId(parameterNameId); + } else { + // ids will be created later (so that we can process them in order) + unresolvedParameterNames.add(result); + } + } + return result; + } + + private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId); + LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key); + if (result == null) { + result = LogicalResourceIdentValue.builder() + .withResourceType(resourceType) + .withLogicalId(logicalId) + .build(); + this.logicalResourceIdentMap.put(key, result); + this.unresolvedLogicalResourceIdents.add(result); + } + return result; + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { + Short shardKey = batchProcessor.encodeShardKey(requestShard); + CommonCanonicalValue ctv = lookupCommonCanonicalValue(shardKey, p.getUrl()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); + this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId()); + this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv)); + } + + /** + * Get the CodeSystemValue we've assigned for the given codeSystem value. This + * may not yet have the actual code_system_id from the database yet - any values + * we don't have will be assigned in a later phase (so we can do things neatly + * in bulk). + * @param codeSystem + * @return + */ + private CodeSystemValue lookupCodeSystemValue(String codeSystem) { + CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); + if (result == null) { + result = new CodeSystemValue(codeSystem); + this.codeSystemValueMap.put(codeSystem, result); + + // Take this opportunity to see if we have a cached value for this codeSystem + Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); + if (codeSystemId != null) { + result.setCodeSystemId(codeSystemId); + } else { + // Stash for later resolution + this.unresolvedSystemValues.add(result); + } + } + return result; + } + + /** + * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. + * The returned value may not yet have the actual common_token_value_id yet - we fetch + * these values later and create new database records as necessary. + * @param codeSystem + * @param tokenValue + * @return + */ + private CommonTokenValue lookupCommonTokenValue(short shardKey, String codeSystem, String tokenValue) { + CommonTokenValueKey key = new CommonTokenValueKey(shardKey, codeSystem, tokenValue); + CommonTokenValue result = this.commonTokenValueMap.get(key); + if (result == null) { + CodeSystemValue csv = lookupCodeSystemValue(codeSystem); + result = new CommonTokenValue(shardKey, csv, tokenValue); + this.commonTokenValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long commonTokenValueId = identityCache.getCommonTokenValueId(shardKey, codeSystem, tokenValue); + if (commonTokenValueId != null) { + result.setCommonTokenValueId(commonTokenValueId); + } else { + this.unresolvedTokenValues.add(result); + } + } + return result; + } + + private CommonCanonicalValue lookupCommonCanonicalValue(short shardKey, String url) { + CommonCanonicalValueKey key = new CommonCanonicalValueKey(shardKey, url); + CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); + if (result == null) { + result = new CommonCanonicalValue(shardKey, url); + this.commonCanonicalValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long canonicalId = identityCache.getCommonCanonicalValueId(shardKey, url); + if (canonicalId != null) { + result.setCanonicalId(canonicalId); + } else { + this.unresolvedCanonicalValues.add(result); + } + } + return result; + } + + /** + * Make sure we have values for all the code_systems we have collected + * in the current + * batch + * @throws FHIRPersistenceException + */ + private void resolveCodeSystems() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCodeSystemIds(unresolvedSystemValues); + + if (!missing.isEmpty()) { + addMissingCodeSystems(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCodeSystemIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all code system values"); + } + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); + for (int i=0; i 0) { + query.append(","); + } + query.append("?"); + } + query.append(")"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (CodeSystemValue csv: values) { + ps.setString(param++, csv.getCodeSystem()); + } + return ps; + } + + /** + * These code systems weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + private void addMissingCodeSystems(List missing) throws FHIRPersistenceException { + List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); + // Sort the code system values first to help avoid deadlocks + Collections.sort(values); // natural ordering for String is fine here + + final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); + insert.append(nextVal); // next sequence value + insert.append(",?) ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (String codeSystem: values) { + ps.setString(1, codeSystem); + ps.addBatch(); + if (++count == this.maxCodeSystemsPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); + if (csv != null) { + csv.setCodeSystemId(rs.getInt(1)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("code systems query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CodeSystemValue csv: sub) { + if (csv.getCodeSystemId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed", x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Make sure we have values for all the common_token_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonTokenValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCommonTokenValueIds(unresolvedTokenValues); + + if (!missing.isEmpty()) { + // Sort first to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getTokenValue().compareTo(b.getTokenValue()); + if (result == 0) { + result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + } + return result; + }); + addMissingCommonTokenValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCommonTokenValueIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all common token values"); + } + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT shard_key, code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + // need the code_system name - so we join back to the code_systems table as well + query.append("SELECT c.shard_key, cs.code_system_name, c.token_value, c.common_token_value_id "); + query.append(" FROM common_token_values c"); + query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); + query.append(" JOIN (VALUES "); + + // Create a (codeSystem, shardKey, tokenValue) tuple for each of the CommonTokenValue records + boolean first = true; + for (CommonTokenValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id + query.append(",").append(ctv.getShardKey()); // literal for shard_key + query.append(",?)"); // bind variable for the token-value + } + query.append(") AS v(code_system_id, shard_key, token_value) "); + query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value AND c.shard_key = v.shard_key)"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonTokenValue ctv: values) { + ps.setString(param++, ctv.getTokenValue()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonTokenValueKey key = new CommonTokenValueKey(rs.getShort(1), rs.getString(2), rs.getString(3)); + CommonTokenValue ctv = this.commonTokenValueMap.get(key); + if (ctv != null) { + ctv.setCommonTokenValueId(rs.getLong(4)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common token values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonTokenValue ctv: sub) { + if (ctv.getCommonTokenValueId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common token values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + private void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) "); + insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number + insert.append(" VALUES (?,?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (CommonTokenValue ctv: missing) { + ps.setShort(1, ctv.getShardKey()); + ps.setInt(2, ctv.getCodeSystemValue().getCodeSystemId()); + ps.setString(3, ctv.getTokenValue()); + ps.addBatch(); + if (++count == this.maxCommonTokenValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common token values"); + } + } + + /** + * Make sure we have values for all the common_canonical_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonCanonicalValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCanonicalIds(unresolvedCanonicalValues); + + if (!missing.isEmpty()) { + // Sort on (url, shard_key) to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getUrl().compareTo(b.getUrl()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + return result; + }); + addMissingCommonCanonicalValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCanonicalIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all canonical values"); + } + } + + private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonCanonicalValueKey key = new CommonCanonicalValueKey(rs.getShort(1), rs.getString(2)); + CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); + if (ctv != null) { + ctv.setCanonicalId(rs.getLong(3)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonCanonicalValue ctv: sub) { + if (ctv.getCanonicalId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common canonical values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT shard_key, code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT c.shard_key, c.url, c.canonical_id "); + query.append(" FROM common_canonical_values c "); + query.append(" JOIN (VALUES "); + + // Create a (shardKey, url) tuple for each of the CommonCanonicalValue records + boolean first = true; + for (CommonCanonicalValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getShardKey()); // literal for shard_key + query.append(",?)"); // bind variable for the uri + } + query.append(") AS v(shard_key, url) "); + query.append(" ON (c.url = v.url AND c.shard_key = v.shard_key)"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + logger.finer(() -> "fetch common canonical values [" + statementText + "]"); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonCanonicalValue ctv: values) { + ps.setString(param++, ctv.getUrl()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + private void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (shard_key, url, canonical_id) "); + insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number + insert.append(" VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + final String DML = insert.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("addMissingCanonicalIds: " + DML); + } + try (PreparedStatement ps = connection.prepareStatement(DML)) { + int count = 0; + for (CommonCanonicalValue ctv: missing) { + logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); + ps.setShort(1, ctv.getShardKey()); + ps.setString(2, ctv.getUrl()); + ps.addBatch(); + if (++count == this.maxCommonCanonicalValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common canonical values"); + } + } + + /** + * Make sure all the parameter names we've seen in the batch exist + * in the database and have ids. + * @throws FHIRPersistenceException + */ + private void resolveParameterNames() throws FHIRPersistenceException { + // We expect parameter names to have a very high cache hit rate and + // so we simplify processing by simply iterating one-by-one for the + // values we still need to resolve. The most important point here is + // to do this in a sorted order to avoid deadlock issues because this + // could be happening across multiple consumer threads at the same time. + Collections.sort(this.unresolvedParameterNames, (a,b) -> { + return a.getParameterName().compareTo(b.getParameterName()); + }); + + try { + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); + if (parameterNameId == null) { + parameterNameId = createParameterName(pnv.getParameterName()); + } + pnv.setParameterNameId(parameterNameId); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("error resolving parameter names", x); + } + } + + private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { + String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ps.setString(1, parameterName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return rs.getInt(1); + } + } + + // no entry in parameter_names + return null; + } + + /** + * Create the parameter name using the stored procedure which handles any concurrency + * issue we may have + * @param parameterName + * @return + */ + private Integer createParameterName(String parameterName) throws SQLException { + final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; + Integer parameterNameId; + try (CallableStatement stmt = connection.prepareCall(CALL)) { + stmt.setString(1, parameterName); + stmt.registerOutParameter(2, Types.INTEGER); + stmt.execute(); + parameterNameId = stmt.getInt(2); + } + + return parameterNameId; + } + + @Override + protected void resetBatch() { + // Called when a transaction has been rolled back because of a deadlock + // or other retryable error and we want to try and process the batch again + batchProcessor.reset(); + } + + /** + * Build the check ready query + * @param messagesByResourceType + * @return + */ + private String buildCheckReadyQuery(Map> messagesByResourceType) { + // The trouble here is that we'll end up with a unique query for every single + // batch of messages we process (which the database then need to parse etc). + // This may introduce scaling issues, in which case we should consider + // individual queries for each resource type using bind variables, perhaps + // going so far as using multiple statements with a power-of-2 number of bind + // variables. But JDBC doesn't support batching of select statements, so + // the alternative there would be to insert-as-select into a global temp table + // and then simply select from that. Fairly straightforward, but a lot more + // work so only worth doing if we identify contention here. + + StringBuilder select = new StringBuilder(); + // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // patient_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (1,2,3,4) + // UNION ALL + // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // observation_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (5,6,7) + boolean first = true; + for (Map.Entry> entry: messagesByResourceType.entrySet()) { + final String resourceType = entry.getKey(); + final List messages = entry.getValue(); + final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); + if (first) { + first = false; + } else { + select.append(" UNION ALL "); + } + select.append(" SELECT lr.shard_key, lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); + select.append(" FROM logical_resources AS lr, "); + select.append(resourceType).append("_logical_resources AS xlr "); + select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); + select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); + } + + return select.toString(); + } + + @Override + protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { + // Get a list of all the resources for which we can see the current logical resource data. + // If the resource doesn't yet exist or its version meta doesn't the message + // then we add to the notReady list. If the resource version meta already + // exceeds the message, then we'll skip processing altogether because it + // means that there should be another message in the queue with more + // up-to-date parameters + Map messageMap = new HashMap<>(); + Map> messagesByResourceType = new HashMap<>(); + for (RemoteIndexMessage msg: messages) { + Long logicalResourceId = msg.getData().getLogicalResourceId(); + messageMap.put(logicalResourceId, msg); + + // split out the messages per resource type because we need to read from xx_logical_resources + List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); + values.add(msg); + } + + Set found = new HashSet<>(); + final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); + logger.fine(() -> "check ready query: " + checkReadyQuery); + try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { + ResultSet rs = ps.executeQuery(); + // wrap the ResultSet in a reader for easier consumption + ResultSetReader rsReader = new ResultSetReader(rs); + while (rsReader.next()) { + LogicalResourceValue lrv = LogicalResourceValue.builder() + .withShardKey(rsReader.getShort()) + .withLogicalResourceId(rsReader.getLong()) + .withResourceType(rsReader.getString()) + .withLogicalId(rsReader.getString()) + .withVersionId(rsReader.getInt()) + .withLastUpdated(rsReader.getTimestamp()) + .withParameterHash(rsReader.getString()) + .build(); + RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); + if (m == null) { + throw new IllegalStateException("query returned a logical resource which we didn't request"); + } + + // Check the values from the database to see if they match + // the information in the message. + if (m.getData().getVersionId() == lrv.getVersionId()) { + // only process this message if the parameter hash and lastUpdated + // times match - which is a good check that we're storing parameters + // from the correct transaction. If these don't match, we can simply + // say we found the data but don't need to process the message. + final String lastUpdated = lrv.getLastUpdated().toString(); + if (lrv.getParameterHash().equals(m.getData().getParameterHash()) + && lastUpdated.equals(m.getData().getLastUpdated())) { + okToProcess.add(m); + } + found.add(lrv.getLogicalResourceId()); // won't be marked as missing + } else if (m.getData().getVersionId() > lrv.getVersionId()) { + // we can skip processing this record because the database has already + // been updated with a newer version. Identify the record as having been + // found so we don't keep waiting for it + found.add(lrv.getLogicalResourceId()); + } + // if the version in the database is prior to version in the message we + // received it means that the server transaction hasn't been committed... + // so we have to wait just as though it were missing altogether + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); + throw new FHIRPersistenceException("prepare query failed"); + } + + if (found.size() < messages.size()) { + // identify the missing records and add to the notReady list + for (RemoteIndexMessage m: messages) { + if (!found.contains(m.getData().getLogicalResourceId())) { + notReady.add(m); + } + } + } + } +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java similarity index 89% rename from fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java rename to fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java index feebe7fb947..54dbfe7252e 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.remote.index.database; +package com.ibm.fhir.remote.index.sharded; import java.math.BigDecimal; import java.sql.Connection; @@ -20,7 +20,7 @@ /** * Parameter batch statements configured for a given resource type */ -public class DistributedPostgresParameterBatch { +public class ShardedPostgresParameterBatch { private final Connection connection; private final String resourceType; @@ -51,12 +51,15 @@ public class DistributedPostgresParameterBatch { private PreparedStatement security; private int securityCount; + private PreparedStatement refs; + private int refCount; + /** * Public constructor * @param c * @param resourceType */ - public DistributedPostgresParameterBatch(Connection c, String resourceType) { + public ShardedPostgresParameterBatch(Connection c, String resourceType) { this.connection = c; this.resourceType = resourceType; } @@ -101,6 +104,10 @@ public void pushBatch() throws SQLException { security.executeBatch(); securityCount = 0; } + if (refCount > 0) { + refs.executeBatch(); + refCount = 0; + } } /** @@ -203,6 +210,16 @@ public void close() { securityCount = 0; } } + if (refs != null) { + try { + refs.close(); + } catch (SQLException x) { + // NOP + } finally { + refs = null; + refCount = 0; + } + } } /** @@ -380,4 +397,27 @@ public void addSecurity(long logicalResourceId, long commonTokenValueId, short s .addBatch(); securityCount++; } + + /** + * @param logicalResourceId + * @param parameterNameId + * @param refLogicalResourceId + * @param refVersionId + * @param shardKey + */ + public void addReference(long logicalResourceId, int parameterNameId, long refLogicalResourceId, Integer refVersionId, short shardKey) throws SQLException { + if (refs == null) { + final String tablePrefix = resourceType.toLowerCase(); + final String insertString = "INSERT INTO " + tablePrefix + "_ref_values (parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, shard_key) VALUES (?,?,?,?,?)"; + refs = connection.prepareStatement(insertString); + } + PreparedStatementHelper psh = new PreparedStatementHelper(security); + psh.setInt(parameterNameId) + .setLong(logicalResourceId) + .setLong(refLogicalResourceId) + .setInt(refVersionId) + .setShort(shardKey) + .addBatch(); + refCount++; + } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java similarity index 97% rename from fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java rename to fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java index 8effe2e2afb..7f03b9cdc95 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresSystemParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.remote.index.database; +package com.ibm.fhir.remote.index.sharded; import java.sql.Connection; import java.sql.PreparedStatement; @@ -21,7 +21,7 @@ * @implNote targets the distributed variant of the schema * where each table includes a shard_key column */ -public class DistributedPostgresSystemParameterBatch { +public class ShardedPostgresSystemParameterBatch { private final Connection connection; private PreparedStatement systemStrings; @@ -43,7 +43,7 @@ public class DistributedPostgresSystemParameterBatch { * Public constructor * @param c */ - public DistributedPostgresSystemParameterBatch(Connection c) { + public ShardedPostgresSystemParameterBatch(Connection c) { this.connection = c; } diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java new file mode 100644 index 00000000000..b2ac1c518b2 --- /dev/null +++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java @@ -0,0 +1,65 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import java.time.Instant; + +import org.testng.annotations.Test; + +import com.google.gson.Gson; +import com.ibm.fhir.persistence.index.RemoteIndexConstants; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter; + +/** + * Unit test for message serialization (for the payload sent over Kafka as a string) + */ +public class MessageSerializationTest { + + @Test + public void testRoundtrip() throws Exception { + RemoteIndexMessage sent = new RemoteIndexMessage(); + sent.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); + + final String resourceType = "Observation"; + final String logicalId = "patientOne"; + final long logicalResourceId = 1; + final int versionId = 1; + final Instant lastUpdated = Instant.now(); + final String requestShard = null; + final String parameterHash = "1Z+NWYZb739Ava9Pd/d7wt2xecKmC2FkfLlCCml0I5M="; + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, + versionId, lastUpdated, requestShard, parameterHash); + sent.setData(adapter.build()); + final String payload = marshallToString(sent); + System.out.println("payload: " + payload); + // Now unmarshall the payload and check everything matches + RemoteIndexMessage rcvd = unmarshallPayload(payload); + assertNotNull(rcvd); + assertNotNull(rcvd.getData()); + assertEquals(rcvd.getMessageVersion(), RemoteIndexConstants.MESSAGE_VERSION); + assertEquals(rcvd.getData().getParameterHash(), parameterHash); + assertEquals(rcvd.getData().getLastUpdated(), lastUpdated.toString()); + } + + /** + * Marshall the message to a string + * @param message + * @return + */ + private String marshallToString(RemoteIndexMessage message) { + final Gson gson = new Gson(); + return gson.toJson(message); + } + private RemoteIndexMessage unmarshallPayload(String jsonPayload) throws Exception { + Gson gson = new Gson(); + return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + } +} diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java index c7be5d311cd..db57c84c9ed 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java @@ -775,6 +775,7 @@ public FHIRRestOperationResponse doPatchOrUpdatePersist(FHIRPersistenceEvent eve FHIRPersistenceContextImpl.builder(event) .withIfNoneMatch(ifNoneMatch) .withOffloadResponse(offloadResponse) + .withRequestShard(requestContext.getRequestShardKey()) .build(); boolean createOnUpdate = (prevResource == null); From f705e5aaf44ba60f8250cd70a660cffda402d107 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Fri, 27 May 2022 09:30:37 +0100 Subject: [PATCH 10/40] issue #3437 fix reference unit tests for new ref_values table Signed-off-by: Robin Arnold --- .../utils/common/PreparedStatementHelper.java | 1 + .../database/utils/query/FromAdapter.java | 12 + .../fhir/database/utils/query/FromClause.java | 11 + .../ibm/fhir/database/utils/query/Select.java | 10 + .../jdbc/FHIRPersistenceJDBCCache.java | 7 + .../jdbc/FHIRResourceDAOFactory.java | 8 +- .../fhir/persistence/jdbc/JDBCConstants.java | 4 +- .../cache/FHIRPersistenceJDBCCacheImpl.java | 16 +- .../cache/FHIRPersistenceJDBCCacheUtil.java | 8 +- .../cache/FHIRPersistenceJDBCTenantCache.java | 5 +- .../cache/LogicalResourceIdentCacheImpl.java | 172 ++++++++ .../jdbc/citus/CitusResourceReferenceDAO.java | 8 +- .../dao/api/ILogicalResourceIdentCache.java | 63 +++ .../jdbc/dao/api/IResourceReferenceDAO.java | 7 +- .../jdbc/dao/api/JDBCIdentityCache.java | 32 ++ .../jdbc/dao/api/LogicalResourceIdentKey.java | 64 +++ .../dao/api/LogicalResourceIdentValue.java | 38 ++ .../persistence/jdbc/dao/api/ResourceDAO.java | 17 + .../jdbc/dao/impl/JDBCIdentityCacheImpl.java | 63 +++ .../dao/impl/ParameterTransportVisitor.java | 5 +- .../dao/impl/ParameterVisitorBatchDAO.java | 40 +- .../jdbc/dao/impl/ResourceDAOImpl.java | 47 ++ .../jdbc/dao/impl/ResourceReferenceDAO.java | 229 +++++++++- .../dao/impl/ResourceReferenceValueRec.java | 100 +++++ .../jdbc/db2/Db2ResourceReferenceDAO.java | 9 +- .../jdbc/derby/DerbyResourceDAO.java | 119 ++++-- .../jdbc/derby/DerbyResourceReferenceDAO.java | 40 +- .../jdbc/domain/SearchQueryRenderer.java | 400 +++++++++++++++--- .../jdbc/dto/ResourceReferenceValue.java | 100 +++++ .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 24 +- .../impl/ParameterTransactionDataImpl.java | 14 +- .../jdbc/postgres/PostgresResourceDAO.java | 7 +- .../PostgresResourceReferenceDAO.java | 47 +- .../test/JDBCSearchCompartmentTest.java | 48 +-- .../search/test/JDBCSearchCompositeTest.java | 47 +- .../jdbc/search/test/JDBCSearchDateTest.java | 48 +-- .../test/JDBCSearchIdLastUpdatedTest.java | 46 +- .../jdbc/search/test/JDBCSearchNearTest.java | 36 +- .../search/test/JDBCSearchNumberTest.java | 46 +- .../search/test/JDBCSearchQuantityTest.java | 49 +-- .../search/test/JDBCSearchReferenceTest.java | 46 +- .../search/test/JDBCSearchStringTest.java | 46 +- .../jdbc/search/test/JDBCSearchTokenTest.java | 46 +- .../jdbc/search/test/JDBCSearchURITest.java | 46 +- .../test/JDBCWholeSystemSearchTest.java | 48 +-- .../jdbc/test/JDBCChangesTest.java | 57 +-- .../jdbc/test/JDBCCompartmentTest.java | 46 +- .../persistence/jdbc/test/JDBCDeleteTest.java | 48 +-- .../persistence/jdbc/test/JDBCEraseTest.java | 41 +- .../persistence/jdbc/test/JDBCExportTest.java | 57 +-- .../jdbc/test/JDBCIfNoneMatchTest.java | 48 +-- .../jdbc/test/JDBCIncludeRevincludeTest.java | 47 +- .../jdbc/test/JDBCMultiResourceTest.java | 47 +- .../persistence/jdbc/test/JDBCPagingTest.java | 59 +-- .../jdbc/test/JDBCReverseChainTest.java | 47 +- .../persistence/jdbc/test/JDBCSortTest.java | 46 +- .../jdbc/test/JDBCanonicalTest.java | 50 +-- .../jdbc/test/erase/EraseTestMain.java | 7 + .../fhir/persistence/jdbc/test/spec/Main.java | 14 +- .../jdbc/test/spec/R4JDBCExamplesTest.java | 5 +- .../test/util/PersistenceTestSupport.java | 98 +++++ .../util/QuantityParmBehaviorUtilTest.java | 18 +- .../java/com/ibm/fhir/schema/app/Main.java | 2 +- .../control/FhirResourceTableGroup.java | 48 +++ .../schema/control/FhirSchemaConstants.java | 2 + .../schema/control/FhirSchemaGenerator.java | 2 +- ...GetLogicalResourceNeedsV0027Migration.java | 7 +- .../schema/derby/DerbyFhirDatabaseTest.java | 2 +- .../fhir/persistence/index/DateParameter.java | 14 +- .../index/ParameterValueVisitorAdapter.java | 4 +- .../index/SearchParametersTransport.java | 20 +- .../SearchParametersTransportAdapter.java | 3 +- .../PlainBatchParameterProcessor.java | 7 +- .../database/PlainPostgresMessageHandler.java | 2 +- .../ShardedBatchParameterProcessor.java | 7 +- .../index/MessageSerializationTest.java | 74 +++- .../com/ibm/fhir/search/SearchConstants.java | 3 + .../ibm/fhir/search/util/ReferenceUtil.java | 11 +- 78 files changed, 2063 insertions(+), 1159 deletions(-) create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java create mode 100644 fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java index 86297e2ebcf..e2fd1de3f8f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java @@ -122,5 +122,6 @@ public PreparedStatementHelper setTimestamp(Timestamp value) throws SQLException */ public void addBatch() throws SQLException { ps.addBatch(); + index = 1; } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java index 0e2b050675d..dc61f7c3ae1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java @@ -59,6 +59,18 @@ public FromAdapter innerJoin(String tableName, Alias alias, WhereFragment joinOn return this; } + /** + * Add an INNER JOIN for the given sub select + * @param sub + * @param alias + * @param joinOnPredicate + * @return + */ + public FromAdapter innerJoin(Select sub, Alias alias, WhereFragment joinOnPredicate) { + this.select.addInnerJoin(sub, alias, joinOnPredicate.getExpression()); + return this; + } + /** * Add a LEFT OUTER JOIN for the given table * @param tableName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java index d2f2ece3bf9..631b9a89616 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java @@ -121,6 +121,17 @@ public void addInnerJoin(String tableName, Alias alias, ExpNode joinOnPredicate) items.add(new FromJoin(JoinType.INNER_JOIN, trs, alias, joinOnPredicate)); } + /** + * Add an inner join clause to the FROM items list + * @param sub + * @param alias + * @param joinOnPredicate + */ + public void addInnerJoin(Select sub, Alias alias, ExpNode joinOnPredicate) { + SelectRowSource srs = new SelectRowSource(sub); + items.add(new FromJoin(JoinType.INNER_JOIN, srs, alias, joinOnPredicate)); + } + /** * Add a left outer join clause to the FROM items list * @param tableName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java index 6397c514cc7..c1de626d076 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java @@ -288,6 +288,16 @@ public T render(StatementRenderer renderer) { public void addInnerJoin(String tableName, Alias alias, ExpNode joinOnPredicate) { fromClause.addInnerJoin(tableName, alias, joinOnPredicate); } + /** + * Add an inner join to the from clause for this select statement + * where the joining row source is a sub-query + * @param sub + * @param alias + * @param joinOnPredicate + */ + public void addInnerJoin(Select sub, Alias alias, ExpNode joinOnPredicate) { + fromClause.addInnerJoin(sub, alias, joinOnPredicate); + } /** * Add a left outer join to the from clause for this select statement. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java index 09c23f0ab8c..613310ebf09 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java @@ -8,6 +8,7 @@ import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -34,6 +35,12 @@ public interface FHIRPersistenceJDBCCache { */ ICommonTokenValuesCache getResourceReferenceCache(); + /** + * Getter for the cache handling lookups for logical_resource_id values + * @return + */ + ILogicalResourceIdentCache getLogicalResourceIdentCache(); + /** * Getter for the cache of resource types used to look up resource type id * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java index 9f15e820797..1a63a2fabb3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java @@ -178,16 +178,16 @@ public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection final ResourceReferenceDAO rrd; switch (flavor.getType()) { case DB2: - rrd = new Db2ResourceReferenceDAO(new Db2Translator(), connection, schemaName, cache.getResourceReferenceCache(), adminSchemaName, cache.getParameterNameCache()); + rrd = new Db2ResourceReferenceDAO(new Db2Translator(), connection, schemaName, cache.getResourceReferenceCache(), adminSchemaName, cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; case DERBY: - rrd = new DerbyResourceReferenceDAO(new DerbyTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new DerbyResourceReferenceDAO(new DerbyTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; case POSTGRESQL: - rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; case CITUS: - rrd = new CitusResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new CitusResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; default: throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java index b700b42e14f..334e0c7eca2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java @@ -6,11 +6,9 @@ package com.ibm.fhir.persistence.jdbc; import java.util.Arrays; -import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TimeZone; import com.ibm.fhir.search.SearchConstants.Modifier; import com.ibm.fhir.search.SearchConstants.Type; @@ -40,6 +38,8 @@ public class JDBCConstants { public static final String _RESOURCES = "_RESOURCES"; public static final String _LOGICAL_RESOURCES = "_LOGICAL_RESOURCES"; public static final String RESOURCE_ID = "RESOURCE_ID"; + public static final String RESOURCE_TYPE_ID = "RESOURCE_TYPE_ID"; + public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID"; public static final String LOGICAL_ID = "LOGICAL_ID"; public static final String LOGICAL_RESOURCE_ID = "LOGICAL_RESOURCE_ID"; public static final String CURRENT_RESOURCE_ID = "CURRENT_RESOURCE_ID"; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java index 5ae9580e1d7..5dfe120aa76 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -28,6 +29,8 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { private final ICommonTokenValuesCache resourceReferenceCache; + private final ILogicalResourceIdentCache logicalResourceIdentCache; + // flag to allow one lucky caller to get the opportunity to prefill private final AtomicBoolean needToPrefillFlag = new AtomicBoolean(true); @@ -37,13 +40,15 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { * @param resourceTypeNameCache * @param parameterNameCache * @param resourceReferenceCache + * @param logicalResourceIdentCache */ public FHIRPersistenceJDBCCacheImpl(INameIdCache resourceTypeCache, IIdNameCache resourceTypeNameCache, - INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) { + INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache, ILogicalResourceIdentCache logicalResourceIdentCache) { this.resourceTypeCache = resourceTypeCache; this.resourceTypeNameCache = resourceTypeNameCache; this.parameterNameCache = parameterNameCache; this.resourceReferenceCache = resourceReferenceCache; + this.logicalResourceIdentCache = logicalResourceIdentCache; } /** @@ -76,6 +81,11 @@ public INameIdCache getParameterNameCache() { return parameterNameCache; } + @Override + public ILogicalResourceIdentCache getLogicalResourceIdentCache() { + return logicalResourceIdentCache; + } + @Override public void transactionCommitted() { logger.fine("Transaction committed - updating cache shared maps"); @@ -83,6 +93,7 @@ public void transactionCommitted() { resourceTypeNameCache.updateSharedMaps(); parameterNameCache.updateSharedMaps(); resourceReferenceCache.updateSharedMaps(); + logicalResourceIdentCache.updateSharedMaps(); } @Override @@ -92,6 +103,7 @@ public void transactionRolledBack() { resourceTypeNameCache.clearLocalMaps(); parameterNameCache.clearLocalMaps(); resourceReferenceCache.clearLocalMaps(); + logicalResourceIdentCache.clearLocalMaps(); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java index d7d43a6a00f..111006639d3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; @@ -24,9 +25,10 @@ public class FHIRPersistenceJDBCCacheUtil { * Factory function to create a new cache instance * @return */ - public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize) { + public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize, int logicalResourceIdentCacheSize) { ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(codeSystemCacheSize, tokenValueCacheSize, canonicalCacheSize); - return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(logicalResourceIdentCacheSize); + return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java index fd378abddd5..853a3616ab2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -74,7 +74,8 @@ protected FHIRPersistenceJDBCCache createCache(String cacheKey) { int externalSystemCacheSize = pg.getIntProperty("externalSystemCacheSize", 1000); int externalValueCacheSize = pg.getIntProperty("externalValueCacheSize", 100000); int canonicalCacheSize = pg.getIntProperty("canonicalCacheSize", 1000); - return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize, canonicalCacheSize); + int logicalResourceIdentCacheSize = pg.getIntProperty("logicalResourceIdentCacheSize", 100000); + return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize, canonicalCacheSize, logicalResourceIdentCacheSize); } } catch (IllegalStateException ise) { throw ise; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java new file mode 100644 index 00000000000..84f240b5c50 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java @@ -0,0 +1,172 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; + + +/** + * Implementation of a cache used for lookups of entities related + * to local and external resource references + */ +public class LogicalResourceIdentCacheImpl implements ILogicalResourceIdentCache { + + // We use LinkedHashMap for the local map because we also need to maintain order + // of insertion to make sure we have correct LRU behavior when updating the shared cache + private final ThreadLocal> localLogicalResourceIdents = new ThreadLocal<>(); + + // The lru token values cache shared at the server level + private final LRUCache cache; + + /** + * Public constructor + * @param logicalResourceCacheSize + */ + public LogicalResourceIdentCacheImpl(int logicalResourceCacheSize) { + + // LRU cache for quick lookup of code-systems and token-values + cache = new LRUCache<>(logicalResourceCacheSize); + } + + /** + * Called after a transaction commit() to transfer all the staged (thread-local) data + * over to the shared LRU cache. + */ + @Override + public void updateSharedMaps() { + + LinkedHashMap localMap = localLogicalResourceIdents.get(); + if (localMap != null) { + synchronized(this.cache) { + cache.update(localMap); + } + + // clear the thread-local cache + localMap.clear(); + } + } + + @Override + public Long getLogicalResourceId(int resourceTypeId, String logicalId) { + // check the thread-local map first + Long result = null; + + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceTypeId, logicalId); + Map localMap = this.localLogicalResourceIdents.get(); + if (localMap != null) { + result = localMap.get(key); + + if (result != null) { + return result; + } + } + + // See if it's in the shared cache + synchronized (this.cache) { + result = cache.get(key); + } + + if (result != null) { + // We found it in the shared cache, so update our thread-local + // cache. + addRecord(key, result); + } + + return result; + } + + @Override + public void resolveReferenceValues(Collection values, List misses) { + // Make one pass over the collection and resolve as much as we can in one go. Anything + // we can't resolve gets put into the corresponding missing lists. Worst case is two passes, when + // there's nothing in the local cache and we have to then look up everything in the shared cache + + // See what we have currently in our thread-local cache + LinkedHashMap valMap = localLogicalResourceIdents.get(); + + List foundKeys = new ArrayList<>(values.size()); // for updating LRU + List needToFindValues = new ArrayList<>(values.size()); // for the ref values we haven't yet found + for (ResourceReferenceValueRec tv: values) { + if (valMap != null) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(tv.getRefResourceTypeId(), tv.getRefLogicalId()); + Long id = valMap.get(key); + if (id != null) { + foundKeys.add(tv); + tv.setRefLogicalResourceId(id); + } else { + // not found, so add to the cache miss list + needToFindValues.add(tv); + } + } else { + needToFindValues.add(tv); + } + } + + // If we still have keys to find, look them up in the shared cache (which we need to lock first) + if (needToFindValues.size() > 0) { + synchronized (this.cache) { + for (ResourceReferenceValueRec tv: needToFindValues) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(tv.getRefResourceTypeId(), tv.getRefLogicalId()); + Long id = cache.get(key); + if (id != null) { + tv.setRefLogicalResourceId(id); + + // Update the local cache with this value + addRecord(key, id); + } else { + // cache miss so add this record to the miss list for further processing + misses.add(tv); + } + } + } + } + } + + + @Override + public void addRecord(LogicalResourceIdentKey key, long id) { + LinkedHashMap map = localLogicalResourceIdents.get(); + + if (map == null) { + map = new LinkedHashMap<>(); + localLogicalResourceIdents.set(map); + } + + // add the id to the thread-local cache. The shared cache is updated + // only if a call is made to #updateSharedMaps() + map.put(key, id); + } + + @Override + public void reset() { + localLogicalResourceIdents.remove(); + + // clear the shared caches too + synchronized (this.cache) { + this.cache.clear(); + } + } + + @Override + public void clearLocalMaps() { + // clear the maps, but keep the maps in place because they'll be used again + // the next time this thread is picked from the pool + LinkedHashMap map = localLogicalResourceIdents.get(); + + if (map != null) { + map.clear(); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java index 8743ad2532f..903749419db 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java @@ -18,6 +18,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; @@ -37,9 +38,12 @@ public class CitusResourceReferenceDAO extends PostgresResourceReferenceDAO { * @param c * @param schemaName * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public CitusResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public CitusResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java new file mode 100644 index 00000000000..19df4d9fda0 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java @@ -0,0 +1,63 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + +import java.util.Collection; +import java.util.List; + +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; + +/** + * An interface for a cache of logical_resource_ident records. The + * cache is specialized in that it supports some specific operations to + * process list of objects with minimal locking. + * + */ +public interface ILogicalResourceIdentCache { + + /** + * Take the records we've touched in the current thread and update the + * shared LRU maps. + */ + void updateSharedMaps(); + + /** + * Lookup all the database values we have cached for the given collection. + * Put any objects with cache misses into the corresponding + * miss lists (so that we know which records we need to generate inserts for) + * @param referenceValues + * @param misses the objects we couldn't find in the cache + */ + void resolveReferenceValues(Collection referenceValues, + List misses); + + /** + * Add the LogicalResourceIdent key and id to the local cache + * @param key + * @param id + */ + public void addRecord(LogicalResourceIdentKey key, long id); + + /** + * Clear any thread-local cache maps (probably because a transaction was rolled back) + */ + void clearLocalMaps(); + + /** + * Clear the thread-local and shared caches (for test purposes) + */ + void reset(); + + /** + * Get the database logical_resourc_id for the given resource type + * and logicalId + * @param resourceTypeId + * @param logicalId + * @return + */ + Long getLogicalResourceId(int resourceTypeId, String logicalId); +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java index fb9eae123f9..2c0ea73047e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; @@ -40,20 +41,22 @@ public interface IResourceReferenceDAO { * as necessary * @param resourceType * @param xrefs + * @param refValues * @param profileRecs * @param tagRecs * @param securityRecs */ - void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; + void addNormalizedValues(String resourceType, Collection xrefs, Collection refValues, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; /** * Persist the records, which may span multiple resource types * @param records + * @param referenceRecords * @param profileRecs * @param tagRecs * @param securityRecs */ - void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; + void persist(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; /** * Find the database id for the given token value and system diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java index 834f41a6f58..64fe5b71e2f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; /** * Provides access to all the identity information we need when processing @@ -54,6 +55,16 @@ public interface JDBCIdentityCache { */ Integer getCanonicalId(String canonicalValue) throws FHIRPersistenceException; + /** + * Get the database id for the given (resourceType, logicalId) tuple. This + * represents records in logical_resource_ident which may be created before + * the actual resource is created. + * @param resourceType + * @param logicalId + * @return + */ + Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException; + /** * Get the database id for the given parameter name. Creates new records if necessary. * @param parameterName @@ -81,6 +92,15 @@ public interface JDBCIdentityCache { */ Set getCommonTokenValueIds(Collection tokenValues); + /** + * Get the logical_resource_ids for the given referenceValues. Reads from + * a cache, or the database if not found in the cache. Values with no + * corresponding record in the database will be omitted from the result set. + * @param referenceValues + * @return a non-null, possibly empty set of logical_resource_ids. + */ + Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException; + /** * Get a list of matching common_token_value_id values. Implementations may decide * to cache, but only if the cache can be invalidated when the list changes due to @@ -93,6 +113,18 @@ public interface JDBCIdentityCache { */ List getCommonTokenValueIdList(String tokenValue); + /** + * Get a list of logical_resource_id values matching the given logicalId without + * knowing the resource type. This means we could get back multiple ids, one per + * resource type, such as: + * Claim/foo + * Observation/foo + * Patient/foo + * @param tokenValue + * @return + */ + List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException; + /** * Get the list of all resource type names. * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java new file mode 100644 index 00000000000..bfe96976036 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java @@ -0,0 +1,64 @@ +/* + * (C) Copyright IBM Corp. 2020, 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + +import java.util.Objects; + +/** + * A DTO representing a mapping of a logical_resource identity to its database + * logical_resource_id value. + */ +public class LogicalResourceIdentKey { + + private final int resourceTypeId; + private final String logicalId; + + /** + * Public constructor + * @param resourceTypeId + * @param logicalId + */ + public LogicalResourceIdentKey(int resourceTypeId, String logicalId) { + this.resourceTypeId = resourceTypeId; + this.logicalId = logicalId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof LogicalResourceIdentKey) { + LogicalResourceIdentKey that = (LogicalResourceIdentKey)obj; + return this.resourceTypeId == that.resourceTypeId + && this.logicalId.equals(that.logicalId); + } + + throw new IllegalArgumentException("invalid type"); + } + + @Override + public int hashCode() { + return Objects.hash(this.resourceTypeId, this.logicalId); + } + + /** + * @return the resourceTypeId + */ + public int getResourceTypeId() { + return resourceTypeId; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java new file mode 100644 index 00000000000..b7843676c07 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java @@ -0,0 +1,38 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + + +/** + * Represents a record in logical_resource_ident + */ +public class LogicalResourceIdentValue extends LogicalResourceIdentKey { + private Long logicalResourceId; + + /** + * Public constructor + * @param resourceTypeId + * @param logicalId + */ + public LogicalResourceIdentValue(int resourceTypeId, String logicalId) { + super(resourceTypeId, logicalId); + } + + /** + * @return the logicalResourceId + */ + public Long getLogicalResourceId() { + return logicalResourceId; + } + + /** + * @param logicalResourceId the logicalResourceId to set + */ + public void setLogicalResourceId(Long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java index f9b3b8fd72d..8e777957c44 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java @@ -187,4 +187,21 @@ List search(String sqlSelect) Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao, Integer ifNoneMatch) throws FHIRPersistenceException; + + /** + * Look up the value of the logical_resource_id from the logical_resource_ident table + * @param resourceTypeId + * @param logicalId + * @return + */ + Long readLogicalResourceId(int resourceTypeId, String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; + + /** + * Read all the matching logical_resource_id values for the given logicalId + * @param logicalId + * @return + * @throws FHIRPersistenceDBConnectException + * @throws FHIRPersistenceDataAccessException + */ + List readLogicalResourceIdList(String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java index e81f8974b2a..6517e0882cc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java @@ -19,12 +19,15 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; /** @@ -212,4 +215,64 @@ public List getResourceTypeNames() throws FHIRPersistenceException { public List getResourceTypeIds() throws FHIRPersistenceException { return new ArrayList<>(cache.getResourceTypeCache().getAllIds()); } + + @Override + public Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException { + Integer resourceTypeId = cache.getResourceTypeCache().getId(resourceType); + if (resourceTypeId == null) { + throw new IllegalArgumentException("Invalid resource type: " + resourceType); + } + Long result = cache.getLogicalResourceIdentCache().getLogicalResourceId(resourceTypeId, logicalId); + if (result == null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Cache miss. Fetching logical_resource_id from database: '" + resourceType + "/" + logicalId + "'"); + } + result = resourceDAO.readLogicalResourceId(resourceTypeId, logicalId); + if (result != null) { + // Value exists in the database, so we can add this to our cache. Note that we still + // choose to add it the thread-local cache - this avoids any locking. The values will + // be promoted to the shared cache at the end of the transaction. This avoids unnecessary + // contention. + if (logger.isLoggable(Level.FINE)) { + logger.fine("Adding logical_resource_id to cache: '" + resourceType + "/" + logicalId + "' = " + result); + } + cache.getLogicalResourceIdentCache().addRecord(new LogicalResourceIdentKey(resourceTypeId, logicalId), result); + } + } + return result; + } + + @Override + public Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + // Pull values from the cache where we can. For anything unresolved, need to hit + // the database. It may be more efficient to collect all the misses and read in + // a single query, if the referenceValues collections are typically small, this + // won't make much difference. + ILogicalResourceIdentCache idCache = cache.getLogicalResourceIdentCache(); + Set result = new HashSet<>(referenceValues.size()); + for (ResourceReferenceValue rrv: referenceValues) { + Long logicalResourceId = idCache.getLogicalResourceId(rrv.getResourceTypeId(), rrv.getLogicalId()); + if (logicalResourceId != null) { + result.add(logicalResourceId); + } else { + // Just read directly from the database + logicalResourceId = resourceDAO.readLogicalResourceId(rrv.getResourceTypeId(), rrv.getLogicalId()); + if (logicalResourceId != null) { + result.add(logicalResourceId); + idCache.addRecord(new LogicalResourceIdentKey(rrv.getResourceTypeId(), rrv.getLogicalId()), logicalResourceId); + } + } + } + return result; + } + + @Override + public List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + // The implementation for this one is a little more interesting. We can + // leverage the fact that the PK on logical_resource_ident is + // logical_id, resource_type_id. This means that we get a simple + // index range scan (although where logical ids are derived from UUIDs, + // they are unique so we'll be getting just one row back) + return resourceDAO.readLogicalResourceIdList(logicalId); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java index f85586a1b1e..4e0b0d6548b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java @@ -6,6 +6,7 @@ package com.ibm.fhir.persistence.jdbc.dao.impl; +import java.time.Instant; import java.util.logging.Logger; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; @@ -79,7 +80,9 @@ public void visit(NumberParmVal numberParameter) throws FHIRPersistenceException @Override public void visit(DateParmVal dateParameter) throws FHIRPersistenceException { Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; - adapter.dateValue(dateParameter.getName(), dateParameter.getValueDateStart(), dateParameter.getValueDateEnd(), compositeId, dateParameter.isWholeSystem()); + Instant dateStart = dateParameter.getValueDateStart().toInstant(); + Instant dateEnd = dateParameter.getValueDateEnd().toInstant(); + adapter.dateValue(dateParameter.getName(), dateStart, dateEnd, compositeId, dateParameter.isWholeSystem()); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index 5647e32ab51..828e8b4f389 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -27,7 +27,6 @@ import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; @@ -102,6 +101,7 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, // Collect a list of token values to process in one go private final List tokenValueRecs = new ArrayList<>(); + private final List referenceValueRecs = new ArrayList<>(); // Tags are now stored in their own tables private final List tagTokenRecs = new ArrayList<>(); @@ -647,7 +647,7 @@ public void close() throws Exception { if (this.transactionData == null) { // Not using transaction data, so we need to process collected values right here - this.resourceReferenceDAO.addNormalizedValues(this.tablePrefix, tokenValueRecs, profileRecs, tagTokenRecs, securityTokenRecs); + this.resourceReferenceDAO.addNormalizedValues(this.tablePrefix, tokenValueRecs, referenceValueRecs, profileRecs, tagTokenRecs, securityTokenRecs); } closeStatement(strings); @@ -703,7 +703,6 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { String refResourceType = refValue.getTargetResourceType(); String refLogicalId = refValue.getValue(); Integer refVersion = refValue.getVersion(); - ResourceTokenValueRec rec; if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { // protect against code regression. Invalid/improper references should be @@ -713,19 +712,32 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { } // reference params are never system-level - final boolean isSystemParam = false; if (refResourceType != null) { - // Store a token value configured as a reference to another resource - rec = new ResourceTokenValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, refResourceType, refLogicalId, refVersion, this.currentCompositeId, isSystemParam); - } else { - // stored as a token with the default system - rec = new ResourceTokenValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, JDBCConstants.DEFAULT_TOKEN_SYSTEM, refLogicalId, this.currentCompositeId, isSystemParam); - } - - if (this.transactionData != null) { - this.transactionData.addValue(rec); + // Store a reference value configured as a reference to another resource + int refResourceTypeId = identityCache.getResourceTypeId(refResourceType); + ResourceReferenceValueRec rec = new ResourceReferenceValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, + refResourceType, refResourceTypeId, + refLogicalId, refVersion, this.currentCompositeId); + if (this.transactionData != null) { + this.transactionData.addReferenceValue(rec); + } else { + this.referenceValueRecs.add(rec); + } } else { - this.tokenValueRecs.add(rec); + // uri so we go back to store as a string instead + logger.info("reference param[" + parameterName + "] value[" + refLogicalId + "] => xx_str_values"); + try { + int parameterNameId = getParameterNameId(parameterName); + setStringParms(strings, parameterNameId, refValue.getValue()); + strings.addBatch(); + + if (++stringCount == this.batchSize) { + strings.executeBatch(); + stringCount = 0; + } + } catch (SQLException x) { + throw new FHIRPersistenceDataAccessException(parameterName + "='" + refValue.getValue() + "'", x); + } } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index 65dde69f7de..5b6d80e8f41 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -131,6 +131,18 @@ public class ResourceDAOImpl extends FHIRDbDAOImpl implements ResourceDAO { "FROM %s_RESOURCES R, %s_LOGICAL_RESOURCES LR WHERE R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID AND " + "R.RESOURCE_ID IN "; + private static final String SQL_GET_LOGICAL_RESOURCE_IDENT = "" + + "SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE resource_type_id = ? " + + " AND logical_id = ?"; + + // Get all records matching the given logical_id (multiple resource types) + private static final String SQL_GET_LOGICAL_RESOURCE_IDENT_LIST = "" + + " SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE logical_id = ?"; + private static final String SQL_ORDER_BY_IDS = "ORDER BY CASE R.RESOURCE_ID "; private static final String DERBY_PAGINATION_PARMS = "OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"; @@ -837,4 +849,39 @@ protected void setString(PreparedStatement ps, int index, String value) throws S ps.setString(index, value); } } + + @Override + public Long readLogicalResourceId(int resourceTypeId, String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { + Long result = null; + try (PreparedStatement ps = getConnection().prepareStatement(SQL_GET_LOGICAL_RESOURCE_IDENT)) { + ps.setInt(1, resourceTypeId); + ps.setString(2, logicalId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + result = rs.getLong(1); + } + } catch (Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving logical_resource_id"); + final String errMsg = "Failure retrieving logical_resource_id from logical_resource_ident for '" + resourceTypeId + "/" + logicalId + "'"; + throw severe(log, fx, errMsg, e); + } + return result; + } + + @Override + public List readLogicalResourceIdList(String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { + List result = new ArrayList<>(); + try (PreparedStatement ps = getConnection().prepareStatement(SQL_GET_LOGICAL_RESOURCE_IDENT_LIST)) { + ps.setString(1, logicalId); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(rs.getLong(1)); + } + } catch (Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving logical_resource_id"); + final String errMsg = "Failure retrieving logical_resource_id list from logical_resource_ident for '" + logicalId + "'"; + throw severe(log, fx, errMsg, e); + } + return result; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 475df62f6f7..717d52e4258 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -27,11 +27,15 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; @@ -70,6 +74,9 @@ public abstract class ResourceReferenceDAO implements IResourceReferenceDAO, Aut // Cache of parameter names to id private final INameIdCache parameterNameCache; + // Cache of the logical resource id values from logical_resource_ident + private final ILogicalResourceIdentCache logicalResourceIdentCache; + // The translator for the type of database we are connected to private final IDatabaseTranslator translator; @@ -78,14 +85,21 @@ public abstract class ResourceReferenceDAO implements IResourceReferenceDAO, Aut /** * Public constructor + * + * @param t * @param c + * @param schemaName + * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { + public ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, ILogicalResourceIdentCache logicalResourceIdentCache) { this.translator = t; + this.schemaName = schemaName; this.connection = c; this.cache = cache; this.parameterNameCache = parameterNameCache; - this.schemaName = schemaName; + this.logicalResourceIdentCache = logicalResourceIdentCache; } /** @@ -231,10 +245,10 @@ public Integer readCanonicalId(String canonicalValue) { } @Override - public void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void addNormalizedValues(String resourceType, Collection xrefs, Collection resourceRefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { // This method is only called when we're not using transaction data logger.fine("Persist parameters for this resource - no transaction data available"); - persist(xrefs, profileRecs, tagRecs, securityRecs); + persist(xrefs, resourceRefs, profileRecs, tagRecs, securityRecs); } /** @@ -294,6 +308,47 @@ protected void insertResourceTokenRefs(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = resourceType + "_REF_VALUES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, composite_id) " + + "VALUES (?, ?, ?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + for (ResourceReferenceValueRec xr: xrefs) { + psh.setInt(xr.getParameterNameId()) + .setLong(xr.getLogicalResourceId()) + .setLong(xr.getRefLogicalResourceId()) + .setInt(xr.getRefVersionId()) + .setInt(xr.getCompositeId()) + .addBatch(); + + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + /** * Insert any whole-system parameters to the token_refs table * @param resourceType @@ -868,9 +923,9 @@ public void upsertCommonTokenValues(List values) { protected abstract void doCommonTokenValuesUpsert(String paramList, Collection sortedTokenValues); @Override - public void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void persist(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { - collectAndResolveParameterNames(records, profileRecs, tagRecs, securityRecs); + collectAndResolveParameterNames(records, referenceRecords, profileRecs, tagRecs, securityRecs); // Grab the ids for all the code-systems, and upsert any misses List systemMisses = new ArrayList<>(); @@ -887,6 +942,11 @@ public void persist(Collection records, Collection referenceMisses = new ArrayList<>(); + logicalResourceIdentCache.resolveReferenceValues(referenceRecords, referenceMisses); + upsertLogicalResourceIdents(referenceMisses); + // Process all the common canonical values List canonicalMisses = new ArrayList<>(); cache.resolveCanonicalValues(profileRecs, canonicalMisses); @@ -904,6 +964,18 @@ public void persist(Collection records, Collection> referenceRecordMap = new HashMap<>(); + for (ResourceReferenceValueRec rtv: referenceRecords) { + List list = referenceRecordMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); + list.add(rtv); + } + + // process each list of reference values by resource type + for (Map.Entry> entry: referenceRecordMap.entrySet()) { + insertRefValues(entry.getKey(), entry.getValue()); + } + // Split profile values by resource type Map> profileMap = new HashMap<>(); for (ResourceProfileRec rtv: profileRecs) { @@ -945,15 +1017,17 @@ public void persist(Collection records, Collection records, Collection profileRecs, + private void collectAndResolveParameterNames(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { List recList = new ArrayList<>(); recList.addAll(records); + recList.addAll(referenceRecords); recList.addAll(profileRecs); recList.addAll(tagRecs); recList.addAll(securityRecs); @@ -1021,4 +1095,145 @@ protected int getParameterNameId(String parameterName) throws FHIRPersistenceDBC * @throws FHIRPersistenceDataAccessException */ protected abstract int readOrAddParameterNameId(String parameterName) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; + + protected void upsertLogicalResourceIdents(List unresolved) throws FHIRPersistenceException { + // Build a unique set of logical_resource_ident keys + Set keys = unresolved.stream().map(v -> new LogicalResourceIdentValue(v.getRefResourceTypeId(), v.getRefLogicalId())).collect(Collectors.toSet()); + List missing = new ArrayList<>(keys); + // Sort the list in logicalId,resourceTypeId order + missing.sort((a,b) -> { + int result = a.getLogicalId().compareTo(b.getLogicalId()); + if (result == 0) { + result = Integer.compare(a.getResourceTypeId(), b.getResourceTypeId()); + } + return result; + }); + addMissingLogicalResourceIdents(missing); + + // Now fetch all the identity records we just created so that we can + // process the unresolved list of ResourceReferenceValueRec records + Map lrIdentMap = new HashMap<>(); + fetchLogicalResourceIdentIds(lrIdentMap, missing); + + // Now we can use the map to find the logical_resource_id for each of the unresolved + // ResourceReferenceValueRec records + for (ResourceReferenceValueRec rec: unresolved) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(rec.getRefResourceTypeId(), rec.getRefLogicalId()); + LogicalResourceIdentValue val = lrIdentMap.get(key); + if (val != null) { + rec.setRefLogicalResourceId(val.getLogicalResourceId()); + } else { + // Shouldn't happen, but be defensive in case someone breaks something + throw new FHIRPersistenceException("logical_resource_idents still missing after upsert"); + } + } + } + + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.resource_type_id, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" JOIN (VALUES "); + for (int i=0; i 0) { + query.append(","); + } + query.append("(?,?)"); + } + query.append(") AS v(resource_type_id, logical_id) "); + query.append(" ON (lri.resource_type_id = v.resource_type_id AND lri.logical_id = v.logical_id)"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + + if (logger.isLoggable(Level.FINE)) { + String params = String.join(",", values.stream().map(v -> "(" + v.getResourceTypeId() + "," + v.getLogicalId() + ")").collect(Collectors.toList())); + logger.fine("ident fetch: " + query.toString() + "; params: " + params); + } + + return ps; + } + + /** + * These logical_resource_ident values weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { + + // simplified implementation which handles inserts individually + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(")"); + + logger.fine(() -> "ident insert: " + insert.toString()); + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + try { + ps.executeUpdate(); + } catch (SQLException x) { + if (getTranslator().isDuplicate(x)) { + // do nothing + } else { + throw x; + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + private void fetchLogicalResourceIdentIds(Map lrIdentMap, List unresolved) throws FHIRPersistenceException { + + int resultCount = 0; + final int maxValuesPerStatement = 512; + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, maxValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each LogicalResourceIdentValue + while (rs.next()) { + resultCount++; + final int resourceTypeId = rs.getInt(1); + final String logicalId = rs.getString(2); + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceTypeId, logicalId); + LogicalResourceIdentValue identValue = new LogicalResourceIdentValue(resourceTypeId, logicalId); + identValue.setLogicalResourceId(rs.getLong(3)); + lrIdentMap.put(key, identValue); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } + // quick check to make sure we got everything we expected + if (resultCount < unresolved.size()) { + throw new FHIRPersistenceException("logical_resource_ident fetch did not fetch everything expected"); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java new file mode 100644 index 00000000000..08cac7115b1 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java @@ -0,0 +1,100 @@ +/* + * (C) Copyright IBM Corp. 2020, 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.impl; + + +/** + * A DTO representing a mapping of a resource and reference value. The + * record is used to drive the population of the xx_ref_values tables + */ +public class ResourceReferenceValueRec extends ResourceRefRec { + + private final String refResourceType; + private final int refResourceTypeId; + + // The external ref value and its normalized database id (when we have it) + private final String refLogicalId; + private Long refLogicalResourceId; + private final Integer refVersionId; + + // Issue 1683 - optional composite id used to correlate parameters + private final Integer compositeId; + + /** + * Public constructor. Used to create a versioned reference record + * @param parameterName + * @param resourceType + * @param resourceTypeId + * @param logicalResourceId + * @param refResourceType + * @param refResourceTypeId + * @param refLogicalId + * @param refVersionId + * @param compositeId + */ + public ResourceReferenceValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, + String refResourceType, int refResourceTypeId, String refLogicalId, Integer refVersionId, Integer compositeId) { + super(parameterName, resourceType, resourceTypeId, logicalResourceId); + this.refResourceType = refResourceType; + this.refResourceTypeId = refResourceTypeId; + this.refLogicalId = refLogicalId; + this.refVersionId = refVersionId; + this.compositeId = compositeId; + } + + /** + * Get the refLogicalResourceId + * @return + */ + public Long getRefLogicalResourceId() { + return refLogicalResourceId; + } + + /** + * Sets the database id for the referenced logical resource + * @param refLogicalResourceId to set + */ + public void setRefLogicalResourceId(long refLogicalResourceId) { + // because we're setting this, it can no longer be null + this.refLogicalResourceId = refLogicalResourceId; + } + + /** + * @return the refVersionId + */ + public Integer getRefVersionId() { + return refVersionId; + } + + /** + * @return the compositeId + */ + public Integer getCompositeId() { + return compositeId; + } + /** + * @return the refResourceType + */ + public String getRefResourceType() { + return refResourceType; + } + + + /** + * @return the refLogicalId + */ + public String getRefLogicalId() { + return refLogicalId; + } + + /** + * @return the refResourceTypeId + */ + public int getRefResourceTypeId() { + return refResourceTypeId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index ecb16929ec6..104553fed4a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -10,9 +10,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -20,8 +18,8 @@ import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; @@ -48,8 +46,9 @@ public class Db2ResourceReferenceDAO extends ResourceReferenceDAO { * @param schemaName * @param cache */ - public Db2ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, String adminSchemaName, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public Db2ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, String adminSchemaName, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); this.adminSchemaName = adminSchemaName; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index 980ca60195a..6aababc3a96 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -260,10 +260,10 @@ public long storeResource(String tablePrefix, List para // Get a lock at the system-wide logical resource level. Note the Derby-specific syntax if (logger.isLoggable(Level.FINEST)) { - logger.finest("Getting LOGICAL_RESOURCES row lock for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Getting LOGICAL_RESOURCE_IDENT row lock for: " + v_resource_type + "/" + p_logical_id); } - final String SELECT_FOR_UPDATE = "SELECT logical_resource_id, parameter_hash, is_deleted" - + " FROM logical_resources" + final String SELECT_FOR_UPDATE = "SELECT logical_resource_id" + + " FROM logical_resource_ident " + " WHERE resource_type_id = ? AND logical_id = ?" + " FOR UPDATE WITH RS"; try (PreparedStatement stmt = conn.prepareStatement(SELECT_FOR_UPDATE)) { @@ -275,8 +275,6 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = rs.getLong(1); - currentParameterHash = rs.getString(2); - v_currently_deleted = "Y".equals(rs.getString(3)); } else { if (logger.isLoggable(Level.FINEST)) { @@ -302,25 +300,21 @@ public long storeResource(String tablePrefix, List para } } } - - // insert the system-wide logical resource record - final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (?, ?, ?, ?, ?, ?, ?)"; + // insert the logical_resource_ident record (which we now do our locking on) + final String INS_IDENT = "INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?, ?, ?)"; if (logger.isLoggable(Level.FINEST)) { - logger.finest("Creating new logical_resources row for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Creating new logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id); } - try (PreparedStatement stmt = conn.prepareStatement(sql4)) { + + try (PreparedStatement stmt = conn.prepareStatement(INS_IDENT)) { // bind parameters - stmt.setLong(1, v_logical_resource_id); - stmt.setInt(2, v_resource_type_id); - stmt.setString(3, p_logical_id); - stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); - stmt.setString(5, p_is_deleted ? "Y" : "N"); // from V0014 - stmt.setTimestamp(6, p_last_updated, UTC); // from V0014 - stmt.setString(7, p_parameterHashB64); // from V0015 + stmt.setInt(1, v_resource_type_id); + stmt.setString(2, p_logical_id); + stmt.setLong(3, v_logical_resource_id); stmt.executeUpdate(); if (logger.isLoggable(Level.FINEST)) { - logger.finest("Created logical_resources row for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Created logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id); } } catch (SQLException e) { if (translator.isDuplicate(e)) { @@ -353,41 +347,88 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = res.getLong(1); - currentParameterHash = res.getString(2); - v_currently_deleted = "Y".equals(res.getString(3)); } else { // Extremely unlikely as we should never delete logical resource records throw new IllegalStateException("Logical resource was deleted: " + tablePrefix + "/" + p_logical_id); } } } - } else { - v_new_resource = true; + } + } - // Insert the resource-specific logical resource record. Remember that logical_id is denormalized - // so it gets stored again here for convenience + // At this point we have an exclusive lock at the logical resource level, so we + // no longer have to worry about concurrency issues. Let's see if we have a + // logical_resources entry: + final String SELECT_LOGICAL_RESOURCE = "" + + "SELECT parameter_hash, is_deleted" + + " FROM logical_resources " + + " WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(SELECT_LOGICAL_RESOURCE)) { + stmt.setLong(1, v_logical_resource_id); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { if (logger.isLoggable(Level.FINEST)) { - logger.finest("Creating " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + logger.finest("Found logical_resources record for: " + v_resource_type + "/" + p_logical_id); } - final String sql5 = "INSERT INTO " + tablePrefix + "_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) VALUES (?, ?, ?, ?, ?, ?)"; - try (PreparedStatement stmt = conn.prepareStatement(sql5)) { - // bind parameters - stmt.setLong(1, v_logical_resource_id); - stmt.setString(2, p_logical_id); - stmt.setString(3, p_is_deleted ? "Y" : "N"); - stmt.setTimestamp(4, p_last_updated, UTC); - stmt.setInt(5, p_version); // initial version - stmt.setLong(6, v_resource_id); - stmt.executeUpdate(); - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Created " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); - } + currentParameterHash = rs.getString(1); + v_currently_deleted = "Y".equals(rs.getString(2)); + v_not_found = false; + } + else { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Logical_resources record not found for: " + v_resource_type + "/" + p_logical_id); } + v_not_found = true; } } - // We have a lock at the logical resource level so no concurrency issues here + if (v_not_found) { + // insert the system-wide logical resource record + final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (?, ?, ?, ?, ?, ?, ?)"; + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Creating new logical_resources row for: " + v_resource_type + "/" + p_logical_id); + } + try (PreparedStatement stmt = conn.prepareStatement(sql4)) { + // bind parameters + stmt.setLong(1, v_logical_resource_id); + stmt.setInt(2, v_resource_type_id); + stmt.setString(3, p_logical_id); + stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); + stmt.setString(5, p_is_deleted ? "Y" : "N"); // from V0014 + stmt.setTimestamp(6, p_last_updated, UTC); // from V0014 + stmt.setString(7, p_parameterHashB64); // from V0015 + stmt.executeUpdate(); + + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Created logical_resources row for: " + v_resource_type + "/" + p_logical_id); + } + + v_new_resource = true; + } + + // Insert the resource-specific logical resource record. Remember that logical_id is denormalized + // so it gets stored again here for convenience + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Creating " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + } + final String sql5 = "INSERT INTO " + tablePrefix + "_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) VALUES (?, ?, ?, ?, ?, ?)"; + try (PreparedStatement stmt = conn.prepareStatement(sql5)) { + // bind parameters + stmt.setLong(1, v_logical_resource_id); + stmt.setString(2, p_logical_id); + stmt.setString(3, p_is_deleted ? "Y" : "N"); + stmt.setTimestamp(4, p_last_updated, UTC); + stmt.setInt(5, p_version); // initial version + stmt.setLong(6, v_resource_id); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Created " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + } + } + + } + // For existing resources, we need to know the current resource version_id if (!v_new_resource) { // existing resource. We need to know the current version from the // resource-specific logical resources table. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java index 9a256dfc2a8..153958ff210 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java @@ -27,14 +27,15 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceReferenceDAO; /** @@ -52,9 +53,12 @@ public class DerbyResourceReferenceDAO extends ResourceReferenceDAO { * @param c * @param schemaName * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public DerbyResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public DerbyResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); } @Override @@ -295,4 +299,34 @@ protected int readOrAddParameterNameId(String parameterName) throws FHIRPersiste final ParameterNameDAO pnd = new DerbyParameterNamesDAO(getConnection(), getSchemaName()); return pnd.readOrAddParameterNameId(parameterName); } + + @Override + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + // Derby doesn't support a VALUES table list, so instead we simply build a big + // OR predicate + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.resource_type_id, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" WHERE "); + for (int i=0; i 0) { + query.append(" OR "); + } + query.append("(resource_type_id = ? AND logical_id = ?)"); + } + PreparedStatement ps = getConnection().prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + + if (logger.isLoggable(Level.FINE)) { + String params = String.join(",", values.stream().map(v -> "(" + v.getResourceTypeId() + "," + v.getLogicalId() + ")").collect(Collectors.toList())); + logger.fine("ident fetch: " + query.toString() + "; params: " + params); + } + + return ps; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java index 48267248e07..d6b3cd19bfe 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java @@ -21,12 +21,15 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ESCAPE_UNDERSCORE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.IS_DELETED; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LEFT_PAREN; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LOGICAL_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MAX; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MIN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.NUMBER_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PARAMETER_NAME_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PERCENT_WILDCARD; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.QUANTITY_VALUE; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.REF_LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RESOURCE_TYPE_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RIGHT_PAREN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.TOKEN_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UNDERSCORE_WILDCARD; @@ -69,14 +72,18 @@ import com.ibm.fhir.database.utils.query.expression.StringExpNodeVisitor; import com.ibm.fhir.database.utils.query.node.ExpNode; import com.ibm.fhir.model.resource.CodeSystem; +import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.resource.Resource; import com.ibm.fhir.model.type.Code; +import com.ibm.fhir.model.type.code.IssueSeverity; +import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.persistence.jdbc.util.CanonicalValue; import com.ibm.fhir.persistence.jdbc.util.NewUriModifierUtil; @@ -98,6 +105,7 @@ import com.ibm.fhir.search.parameters.QueryParameter; import com.ibm.fhir.search.parameters.QueryParameterValue; import com.ibm.fhir.search.sort.Sort.Direction; +import com.ibm.fhir.search.util.ReferenceUtil; import com.ibm.fhir.search.util.SearchHelper; import com.ibm.fhir.term.util.CodeSystemSupport; import com.ibm.fhir.term.util.ValueSetSupport; @@ -229,6 +237,25 @@ protected Set getCommonTokenValueIds(Collection tokenVal return this.identityCache.getCommonTokenValueIds(tokenValues); } + /** + * Obtain the logical_resource_id values for each of the given ResourceReferenceValues. + * @param referenceValues + * @return + * @throws FHIRPersistenceException + */ + protected Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + return this.identityCache.getLogicalResourceIds(referenceValues); + } + + /** + * Obtain the list of logical_resource_id values that match the given logicalId. + * @param logicalId + * @return + */ + protected List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + return this.identityCache.getLogicalResourceIdList(logicalId); + } + /** * Get a list of common token values matching the given code * @param code @@ -713,6 +740,25 @@ private void addCommonTokenValueIdFilter(WhereFragment where, String paramAlias, addCommonTokenValueIdFilter(where, paramAlias, ctvs); } + /** + * Adds a filter predicate for ref_logical_resource_id. Fetches the list of posible matches (there's no resourceType, + * so there could be multiple matches. If no match, then -1 is used to make sure the row isn't produced. If there is + * single match, the predicate uses an equality, otherwise an IN-LIST. + * The query uses literal values not bind variables on purpose (better performance). + * @param where + * @param paramAlias + * @param searchValue + * @throws FHIRPersistenceException + */ + private void addLogicalResourceIdFilter(WhereFragment where, String paramAlias, String searchValue) throws FHIRPersistenceException { + // grab the list of all matching common_token_value_id values + Set ctvs = new HashSet<>(); + fetchLogicalResourceValues(ctvs, searchValue); + + // and add a filter expression paramAlias IN (...) for the values + addRefLogicalResourceIdFilter(where, paramAlias, ctvs); + } + /** * Add all common_token_value_id matching the given searchValue to the ctvs set. * @param ctvs @@ -724,6 +770,17 @@ private void fetchCommonTokenValues(Set ctvs, String searchValue) throws F ctvs.addAll(ctvList); } + /** + * All all matching logical_resource_id values to the given set + * @param lrids + * @param searchValue + * @throws FHIRPersistenceException + */ + private void fetchLogicalResourceValues(Set lrids, String searchValue) throws FHIRPersistenceException { + List tmpList = this.identityCache.getLogicalResourceIdList(searchValue); + lrids.addAll(tmpList); + } + /** * Adds a filter predicate for COMMON_TOKEN_VALUE_ID. If the ctvs list is empty, then -1 is used to make * sure the row isn't produced. If there is a single match, the predicate is COMMON_TOKEN_VALUE_ID = {n}. @@ -746,6 +803,28 @@ private void addCommonTokenValueIdFilter(WhereFragment where, String paramAlias, } } + /** + * Adds a filter predicate for REF_LOGICAL_RESOURCE_ID. If the ctvs list is empty, then -1 is used to make + * sure the row isn't produced. If there is a single match, the predicate is REF_LOGICAL_RESOURCE_ID = {n}. + * If there are multiple matches, the predicate is REF_LOGICAL_RESOURCE_ID IN (1, 2, 3, ...). + * The query uses literal values not bind variables on purpose (better performance). + * @param where + * @param paramAlias + * @param ctvs + * @throws FHIRPersistenceException + */ + private void addRefLogicalResourceIdFilter(WhereFragment where, String paramAlias, Collection ctvs) throws FHIRPersistenceException { + final List ctvList = new ArrayList<>(ctvs); + if (ctvList.isEmpty()) { + // use -1...resulting in no data + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).eq(-1L); + } else if (ctvList.size() == 1) { + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).eq(ctvList.get(0)); + } else { + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).inLiteralLong(ctvList); + } + } + /** * Builds an SQL segment which populates an IN clause with codes for a token search parameter * specifying the :in, :not-in, :above, or :below modifier. @@ -1008,6 +1087,8 @@ public String paramValuesTableName(String resourceType, QueryParameter queryParm name.append("LATLNG_VALUES"); break; case REFERENCE: + name.append("REF_VALUES"); + break; case TOKEN: if (!this.legacyWholeSystemSearchParamsEnabled && TAG.equals(queryParm.getCode())) { name.append(wholeSystemSearch ? "LOGICAL_RESOURCE_TAGS" : "TAGS"); @@ -1049,6 +1130,8 @@ public String paramValuesColumnName(Type paramType) { result = "LATLNG_VALUES"; break; case REFERENCE: + result = "REF_LOGICAL_RESOURCE_ID"; + break; case TOKEN: result = "TOKEN_VALUE"; break; @@ -1274,6 +1357,43 @@ protected String getTokenParamTable(ExpNode filter, String resourceType, String return xxTokenValues; } + /** + * Compute the reference parameter table name we want to use to join with. This method + * inspects the content of the given filter {@link ExpNode}. If the filter contains + * a reference to the LOGICAL_ID column, the returned table name will be based + * on xx_REF_VALUES_V, otherwise it will be based on xx_REF_VALUES. The + * latter is preferable because it eliminates an unnecessary join, improves cardinality + * estimation and (usually) results in a better execution plan. + * @param filter + * @param resourceType + * @param paramAlias + * @return + */ + protected String getRefParamTable(ExpNode filter, String resourceType, String paramAlias) { + ColumnExpNodeVisitor visitor = new ColumnExpNodeVisitor(); // gathers all columns used in the filter expression + Set columns = filter.visit(visitor); + boolean usesLogicalIdValue = columns.contains(DataDefinitionUtil.getQualifiedName(paramAlias, LOGICAL_ID)) || + columns.contains(DataDefinitionUtil.getQualifiedName(paramAlias, RESOURCE_TYPE_ID)); + + final String xxRefValues; + if (usesLogicalIdValue) { + // can't optimize because we filter on LOGICAL_ID + xxRefValues = resourceType + "_REF_VALUES_V"; + } else { + // only filters on REF_LOGICAL_RESOURCE_ID so we can optimize + xxRefValues = resourceType + "_REF_VALUES"; + } + return xxRefValues; + } + + protected WhereFragment getIdentifierFilter(QueryParameter queryParm, String paramAlias) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + handleIdentifier(queryParm, paramAlias, whereClause); + whereClause.rightParen(); + return whereClause; + } + /** * Create a filter predicate for the given reference query parameter * @param queryParm @@ -1295,36 +1415,62 @@ protected WhereFragment getReferenceFilter(QueryParameter queryParm, String para resourceTypesAndIds.add(getResourceTypeAndId(queryParm, value)); } - List resourceReferenceTokenValues = new ArrayList<>(queryParm.getValues().size()); + List refValues = new ArrayList<>(queryParm.getValues().size()); List ambiguousResourceReferenceTokenValues = new ArrayList<>(); for (Pair resourceTypeAndId : resourceTypesAndIds) { String targetResourceType = resourceTypeAndId.getLeft(); String targetResourceId = resourceTypeAndId.getRight(); if (targetResourceType != null) { - Integer codeSystemIdForResourceType = getCodeSystemId(targetResourceType); + Integer resourceTypeId = identityCache.getResourceTypeId(targetResourceType); // targetResourceType is treated as the code-system for references - resourceReferenceTokenValues.add(new CommonTokenValue(targetResourceType, nullCheck(codeSystemIdForResourceType), targetResourceId)); + refValues.add(new ResourceReferenceValue(targetResourceType, resourceTypeId, targetResourceId)); } else { ambiguousResourceReferenceTokenValues.add(targetResourceId); } } - // For unambiguous resource references, look up the common token value ids - Set resourceReferenceTokenIds = getCommonTokenValueIds(resourceReferenceTokenValues); - addCommonTokenValueIdFilter(whereClause, paramAlias, resourceReferenceTokenIds); + // For unambiguous resource references, look up the logical_resource_ids + Set resourceReferenceTokenIds = getLogicalResourceIds(refValues); + addRefLogicalResourceIdFilter(whereClause, paramAlias, resourceReferenceTokenIds); for (String targetResourceId : ambiguousResourceReferenceTokenValues) { whereClause.or(); // grab the list of all matching common_token_value_id values - addCommonTokenValueIdFilter(whereClause, paramAlias, targetResourceId); + addLogicalResourceIdFilter(whereClause, paramAlias, targetResourceId); } whereClause.rightParen(); return whereClause; } + protected WhereFragment getReferenceFilter(QueryParameter queryParm, String paramAlias, List logicalResourceIdList) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + + // For unambiguous resource references, look up the logical_resource_ids + addRefLogicalResourceIdFilter(whereClause, paramAlias, logicalResourceIdList); + + whereClause.rightParen(); + return whereClause; + } + + /** + * Create a filter predicate for the given reference query parameter using + * the ambiguous + * @param queryParm + * @param paramAlias + * @throws FHIRPersistenceException + */ + protected WhereFragment getReferenceStrFilter(QueryParameter queryParm, String paramAlias, List ambiguousResourceReferenceTokenValues) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + whereClause.col(paramAlias, "str_value").in(ambiguousResourceReferenceTokenValues); + whereClause.rightParen(); + return whereClause; + } + private Pair getResourceTypeAndId(QueryParameter queryParm, QueryParameterValue value) { String targetResourceType = null; String searchValue = SqlParameterEncoder.encode(value.getValueString()); @@ -1535,11 +1681,10 @@ public QueryData addIncludeFilter(QueryData queryData, InclusionParameter inclus // > the specified version SHOULD be provided. /* SELECT R0.RESOURCE_ID, R0.LOGICAL_RESOURCE_ID, R0.VERSION_ID, R0.LAST_UPDATED, R0.IS_DELETED, R0.DATA, R0.RESOURCE_PAYLOAD_KEY, LR0.LOGICAL_ID - FROM fhirdata.ExplanationOfBenefit_TOKEN_VALUES_V AS P1 + FROM fhirdata.ExplanationOfBenefit_REF_VALUES AS P1 INNER JOIN fhirdata.Claim_LOGICAL_RESOURCES AS LR0 - ON LR0.LOGICAL_ID = P1.TOKEN_VALUE + ON LR0.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID AND P1.PARAMETER_NAME_ID = 9263 - AND P1.CODE_SYSTEM_ID = 341729359 AND P1.LOGICAL_RESOURCE_ID IN (135010606,135010540,135010498,135010412,135010428) INNER JOIN fhirdata.Claim_RESOURCES AS R0 ON LR0.LOGICAL_RESOURCE_ID = R0.LOGICAL_RESOURCE_ID @@ -1597,12 +1742,11 @@ AND COALESCE(P1.REF_VERSION_ID,LR0.VERSION_ID) = R0.VERSION_ID .and(lrAlias, "VERSION_ID").eq(rAlias, "VERSION_ID") .and(rAlias, IS_DELETED).eq().literal("N")); } else { - final String tokenValues = joinResourceType + "_TOKEN_VALUES_V"; + final String tokenValues = joinResourceType + "_REF_VALUES"; select.from(tokenValues, alias(paramAlias)) .innerJoin(xxLogicalResources, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(getCodeSystemId(targetResourceType)) .and(paramAlias, "LOGICAL_RESOURCE_ID").inLiteralLong(logicalResourceIds)) .innerJoin(xxResources, alias(rAlias), on(lrAlias, "LOGICAL_RESOURCE_ID").eq(rAlias, "LOGICAL_RESOURCE_ID") @@ -1683,14 +1827,13 @@ public QueryData addRevIncludeFilter(QueryData queryData, InclusionParameter inc .or().col(nextPlus2ParamAlias, "STR_VALUE").eq().col(nextPlus1ParamAlias, "STR_VALUE") .rightParen()); } else { - final String tokenValues = joinResourceType + "_TOKEN_VALUES_V"; + final String tokenValues = joinResourceType + "_REF_VALUES"; query.from() .innerJoin(tokenValues, alias(paramAlias), on(parentLRAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "LOGICAL_RESOURCE_ID") - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(getCodeSystemId(targetResourceType))) + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter()))) .innerJoin(targetLR, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and().coalesce(col(paramAlias, "REF_VERSION_ID"), col(lrAlias, "VERSION_ID")).eq(lrAlias, "VERSION_ID") .and(lrAlias, "LOGICAL_RESOURCE_ID").inLiteralLong(logicalResourceIds)); } @@ -1988,6 +2131,7 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, // note that there's no filter here to look for a specific value. We simply want to know // whether or not the parameter exists for a given resource final String parameterName = queryParm.getCode(); + final int parameterNameId = getParameterNameId(parameterName); final int aliasIndex = getNextAliasIndex(); final String resourceType = queryData.getResourceType(); final String paramTableName = paramValuesTableName(resourceType, queryParm); @@ -2002,7 +2146,18 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, // their own tables. if (this.legacyWholeSystemSearchParamsEnabled || (!PROFILE.equals(parameterName) && !SECURITY.equals(parameterName) && !TAG.equals(parameterName))) { - exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(getParameterNameId(parameterName)); + exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(parameterNameId); + } + + if (queryParm.getType() == Type.REFERENCE) { + // From V0027 we store absolute references in xx_str_values, so need to check there too + final String strParamAlias = getParamAlias(getNextAliasIndex()); + final String strParamTableName = resourceType + "_STR_VALUES"; + SelectAdapter strExists = Select.select("1"); + strExists.from(strParamTableName, alias(strParamAlias)) + .where(strParamAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query + .and(strParamAlias, PARAMETER_NAME_ID).eq(parameterNameId); + exists.unionAll(strExists.build()); } // Add the exists to the where clause of the main query which already has a predicate @@ -2024,14 +2179,15 @@ public QueryData addChained(QueryData queryData, QueryParameter currentParm) thr // In this variant, each chained element is added as join to the current statement. We still need // to add the EXISTS clause when depth == 0 (the first element in the chain) + // Because logical_resource_id is already unique across all resources, we don't need to constrain + // with resource_type_id. // AND EXISTS (SELECT 1 - // FROM fhirdata.Observation_TOKEN_VALUES_V AS P1 -- Observation references to - // INNER JOIN fhirdata.Device_LOGICAL_RESOURCES AS LR1 -- Device - // ON LR1.LOGICAL_ID = P1.TOKEN_VALUE -- Device.LOGICAL_ID = Observation.device - // AND P1.PARAMETER_NAME_ID = 1234 -- Observation.device reference param - // AND P1.CODE_SYSTEM_ID = 4321 -- code-system for Device - // AND LR1.IS_DELETED = 'N' -- referenced Device is not deleted - // WHERE P1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID -- correlate parameter to parent + // FROM fhirdata.Observation_REF_VALUES AS P1 -- Observation references to + // INNER JOIN fhirdata.Device_LOGICAL_RESOURCES AS LR1 -- Device + // ON LR1.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID -- Device.LOGICAL_RESOURCE_ID = Observation.device + // AND P1.PARAMETER_NAME_ID = 1234 -- Observation.device reference param + // AND LR1.IS_DELETED = 'N' -- referenced Device is not deleted + // WHERE P1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID -- correlate parameter to parent final String sourceResourceType = queryData.getResourceType(); final SelectAdapter currentSubQuery = queryData.getQuery(); @@ -2083,14 +2239,13 @@ public QueryData addChained(QueryData queryData, QueryParameter currentParm) thr paramAlias = nextPlus2ParamAlias; } else { // Chain via the logical ID - final String tokenValues = sourceResourceType + "_TOKEN_VALUES_V"; // because we need TOKEN_VALUE + final String refValues = sourceResourceType + "_REF_VALUES"; currentSubQuery.from() - .innerJoin(tokenValues, alias(paramAlias), + .innerJoin(refValues, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(queryData.getLRAlias(), "LOGICAL_RESOURCE_ID")) .innerJoin(xxLogicalResources, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(nullCheck(getCodeSystemId(targetResourceType))) .and(lrAlias, "IS_DELETED").eq().literal("N")); } @@ -2174,13 +2329,14 @@ public QueryData addReverseChained(QueryData queryData, QueryParameter currentPa // For reverse chaining, we connect the token-value (reference) // back to the parent query LOGICAL_ID and an xx_LOGICAL_RESOURCES // to provide the LOGICAL_ID as the target for future chain elements - // INNER JOIN fhirdata.Observation_TOKEN_VALUES_V AS P1 - // AND LR0.LOGICAL_ID = P1.TOKEN_VALUE -- 'Patient.LOGICAL_ID = Observation.patient' + + // INNER JOIN fhirdata.Observation_REF_VALUES AS P1 + // AND LR0.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID -- 'Patient.LOGICAL_ID = Observation.patient' // AND LR0.VERSION_ID = COALESCE(P1.REF_VERSION_ID, LR0.VERSION_ID) // AND P1.PARAMETER_NAME_ID = 1246 -- 'Observation.patient' - // AND P1.CODE_SYSTEM_ID = 6 -- 'code system for Patient references' // INNER JOIN fhirdata.Observation_LOGICAL_RESOURCES LR1 // ON LR1.LOGICAL_RESOURCE_ID = P1.LOGICAL_RESOURCE_ID + final String refResourceType = queryData.getResourceType(); final SelectAdapter currentSubQuery = queryData.getQuery(); final int aliasIndex = getNextAliasIndex(); @@ -2225,13 +2381,12 @@ public QueryData addReverseChained(QueryData queryData, QueryParameter currentPa .rightParen()); paramAlias = nextPlus2ParamAlias; } else { - final String tokenValues = resourceTypeName + "_TOKEN_VALUES_V"; + final String refValues = resourceTypeName + "_REF_VALUES"; currentSubQuery.from() - .innerJoin(tokenValues, alias(paramAlias), - on(lrPrevAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") // correlate with the main query + .innerJoin(refValues, alias(paramAlias), + on(lrPrevAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") // correlate with the main query .and(lrPrevAlias, "VERSION_ID").eq().coalesce(col(paramAlias, "REF_VERSION_ID"), col(lrPrevAlias, "VERSION_ID")) - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(nullCheck(getCodeSystemId(refResourceType)))); + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode()))); } currentSubQuery.from() .innerJoin(xxLogicalResources, alias(lrAlias), @@ -2315,27 +2470,172 @@ public QueryData addLocationParam(QueryData queryData, String resourceType, Quer @Override public QueryData addReferenceParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); + final boolean isIdentifier = Modifier.IDENTIFIER.equals(queryParm.getModifier()); + final ExpNode filter; + final String paramTableName; + if (isIdentifier) { + // Identifiers are tokens so we need to join with token_values. + // Grab the filter expression first. We can then inspect the expression to + // look for use of the TOKEN_VALUE column. If use of this column isn't found, + // we can apply an optimization by joining against the RESOURCE_TOKEN_REFS + // table directly. + filter = getIdentifierFilter(queryParm, paramAlias).getExpression(); + paramTableName = getTokenParamTable(filter, resourceType, paramAlias); + String queryParmCode = queryParm.getCode(); + queryParmCode += SearchConstants.IDENTIFIER_MODIFIER_SUFFIX; + query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) + .and(filter)); + } else { + // For V0027 we need to handle parameters that may come from xx_ref_values or xx_str_values + return processRealReferenceParam(queryData, resourceType, queryParm); + } - // Grab the filter expression first. We can then inspect the expression to - // look for use of the TOKEN_VALUE column. If use of this column isn't found, - // we can apply an optimization by joining against the RESOURCE_TOKEN_REFS - // table directly. - ExpNode filter = getReferenceFilter(queryParm, paramAlias).getExpression(); - final String paramTableName = getTokenParamTable(filter, resourceType, paramAlias); + return queryData; + } - // Append the suffix for :identifier modifier - String queryParmCode = queryParm.getCode(); - if (Modifier.IDENTIFIER.equals(queryParm.getModifier())) { - queryParmCode += SearchConstants.IDENTIFIER_MODIFIER_SUFFIX; + /** + * FHIR Specification: + * A reference parameter refers to references between resources. For + * example, find all Conditions where the subject reference is a + * particular patient, where the patient is selected by name or + * identifier. The interpretation of a reference parameter is either: + * [1] [parameter]=[id] the logical [id] of a resource using a local reference (i.e. a relative reference) + * [2] [parameter]=[type]/[id] the logical [id] of a resource of a specified type using a local reference (i.e. a relative reference), for when the reference can point to different types of resources (e.g. Observation.subject) + * [3] [parameter]=[url] where the [url] is an absolute URL - a reference to a resource by its absolute location, or by it's canonical URL + * + * For [1], the target resource type isn't known. This shouldn't matter, because + * we still look up the logical_resource_id by its logical_id. If there are + * multiple matches, they are by definition of different type, so this would be + * an error. Therefore the query still only needs to deal with a single logical_resource_id. + * For [2], we are guaranteed a single logical_resource_id because resourceType/logicalId + * is unique. + * For [3], we need to identify the value string as a url and not a local reference. + * + * @param queryData + * @param resourceType + * @param queryParm + * @return + * @throws FHIRPersistenceException + */ + private QueryData processRealReferenceParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + final int aliasIndex = getNextAliasIndex(); + final SelectAdapter query = queryData.getQuery(); + final String paramAlias = getParamAlias(aliasIndex); + final String lrAlias = queryData.getLRAlias(); + + // For V0027 we split reference parameters into two tables: xx_ref_values and xx_str_values + // Firstly we need to split the query parm values into separate lists + List> resourceTypesAndIds = new ArrayList<>(queryParm.getValues().size()); + for (QueryParameterValue value : queryParm.getValues()) { + resourceTypesAndIds.add(getResourceTypeAndId(queryParm, value)); } - query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) - .and(filter)); + List logicalResourceIdList = new ArrayList<>(); + List refValues = new ArrayList<>(queryParm.getValues().size()); + List absoluteReferenceValues = new ArrayList<>(); + for (Pair resourceTypeAndId : resourceTypesAndIds) { + String targetResourceType = resourceTypeAndId.getLeft(); + String referenceValue = resourceTypeAndId.getRight(); + + if (targetResourceType != null) { + Integer resourceTypeId = identityCache.getResourceTypeId(targetResourceType); + if (resourceTypeId != null) { + // It's a valid resource type, so we treat as a local reference + logger.info(() -> "reference search value: type[local] value[" + targetResourceType + "/" + referenceValue + "]"); + Long logicalResourceId = identityCache.getLogicalResourceId(targetResourceType, referenceValue); + logicalResourceIdList.add(logicalResourceId != null ? logicalResourceId : -1); + } else { + // Treat this as an error because it's not a valid local reference + throw new FHIRPersistenceException("Local reference specified with invalid resource type").withIssue( + Issue.builder() + .code(IssueType.INVALID) + .diagnostics("Local reference specified with invalid resource type") + .severity(IssueSeverity.ERROR) + .build()); + } + } else { + // Determine if the target value is an absolute or local reference + if (ReferenceUtil.isAbsolute(referenceValue)) { + logger.info(() -> "reference search value: type[absolute] value[" + referenceValue + "]"); + absoluteReferenceValues.add(referenceValue); + } else { + // treat as a local reference where we don't know the type. + List localLogicalResourceIds = getLogicalResourceIdList(referenceValue); + if (localLogicalResourceIds.size() == 1) { + logger.info(() -> "reference search value: type[local] value[" + referenceValue + "]"); + logicalResourceIdList.add(localLogicalResourceIds.get(0)); + } else if (localLogicalResourceIds.size() == 0) { + logger.info(() -> "reference search value: type[local] value[" + referenceValue + "] notFound[true]"); + logicalResourceIdList.add(-1L); + } else { + // We may match multiple resource types here, but it's only an error + // if we join with the xx_ref_value table and still get multiple rows + logicalResourceIdList.addAll(localLogicalResourceIds); + } + } + } + } + + final String queryParmCode = queryParm.getCode(); + if (absoluteReferenceValues.isEmpty()) { + // Only need to join with xx_ref_values + final ExpNode filter = getReferenceFilter(queryParm, paramAlias, logicalResourceIdList).getExpression(); + final String paramTableName = getRefParamTable(filter, resourceType, paramAlias); + query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) + .and(filter)); + } else if (refValues.isEmpty()) { + // Only need to join with xx_str_values + final ExpNode filter = getReferenceStrFilter(queryParm, paramAlias, absoluteReferenceValues).getExpression(); + final String paramTableName = resourceType + "_str_values"; + query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) + .and(filter)); + } else { + // The more complicated scenario where we need to filter on xx_ref_values + // and bolt on a union all with a filter on the xx_str_values. It's an + // edge-case, which is lucky because the query plan won't be as clean as + // the prior two cases. But this form is required for the correct semantics. + // SELECT P2.LOGICAL_RESOURCE_ID + // FROM ibmfhirpg3.Basic_REF_VALUES AS P2 + // WHERE P2.PARAMETER_NAME_ID = 25585 + // AND (P2.REF_LOGICAL_RESOURCE_ID = 1 OR P2.REF_LOGICAL_RESOURCE_ID = 2 OR ...) + // UNION ALL + // SELECT P3.LOGICAL_RESOURCE_ID + // FROM ibmfhirpg3.Basic_STR_VALUES AS P3 + // WHERE P3.PARAMETER_NAME_ID = 25585 + // AND (P3.STR_VALUE = 'abc') + + final int parameterNameId = getParameterNameId(queryParmCode); + final String refParamAlias = getParamAlias(getNextAliasIndex()); + final ExpNode refFilter = getReferenceFilter(queryParm, refParamAlias, logicalResourceIdList).getExpression(); + final String refParamTableName = getRefParamTable(refFilter, resourceType, refParamAlias); + final String strParamAlias = getParamAlias(getNextAliasIndex()); + final ExpNode strFilter = getReferenceStrFilter(queryParm, strParamAlias, absoluteReferenceValues).getExpression(); + final String strParamTableName = resourceType + "_str_values"; + + SelectAdapter strSelect = Select.select("LOGICAL_RESOURCE_ID"); + strSelect.from(strParamTableName, alias(strParamAlias)) + .where(strParamAlias, "PARAMETER_NAME_ID").eq(parameterNameId) + .and(strFilter); + + SelectAdapter refSelect = Select.select("LOGICAL_RESOURCE_ID"); + refSelect.from(refParamTableName, alias(refParamAlias)) + .where(refParamAlias, "PARAMETER_NAME_ID").eq(parameterNameId) + .and(refFilter); + refSelect.unionAll(strSelect.build()); + + // add everything to the main query + query.from().innerJoin(refSelect.build(), alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) + ); + } return queryData; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java new file mode 100644 index 00000000000..88653e5ec7e --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java @@ -0,0 +1,100 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dto; + +import java.util.Comparator; + +/** + * DTO representing a resource reference record. + */ +public class ResourceReferenceValue implements Comparable { + private static final Comparator NULL_SAFE_COMPARATOR = Comparator.nullsFirst(String::compareTo); + + private final String resourceType; + private final int resourceTypeId; + + // the target logicalId...can be null + private final String logicalId; + + /** + * Canonical constructor + * + * @param resourceType + * @param resourceTypeId + * @param logicalId + */ + public ResourceReferenceValue(String resourceType, int resourceTypeId, String logicalId) { + if (resourceTypeId < 0) { + throw new IllegalArgumentException("Invalid resourceTypeId argument"); + } + + this.resourceType = resourceType; + this.resourceTypeId = resourceTypeId; + this.logicalId = logicalId; + } + + @Override + public int hashCode() { + // We don't need to include codeSystem in the hash because codeSystemId is synonymous + // with codeSystem as far as identity is concerned + return Integer.hashCode(resourceTypeId) * 37 + (logicalId == null ? 7 : logicalId.hashCode()); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ResourceReferenceValue) { + ResourceReferenceValue that = (ResourceReferenceValue)other; + return this.resourceTypeId == that.resourceTypeId + && ( this.logicalId == null && that.logicalId == null + || this.logicalId != null && this.logicalId.equals(that.logicalId) + ); + } else { + return false; + } + } + + @Override + public String toString() { + return "[resourceTypeId=" + resourceTypeId + ", logicalId=" + logicalId + "]"; + } + + @Override + public int compareTo(ResourceReferenceValue other) { + // allow ResourceReferenceValue objects to be sorted in a deterministic way. Note that + // we sort on resourceType not resourceTypeId. This is to help avoid deadlocks with + // Derby + int result = NULL_SAFE_COMPARATOR.compare(resourceType, other.resourceType); + if (result == 0) { + result = NULL_SAFE_COMPARATOR.compare(logicalId, other.logicalId); + } + return result; + } + + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + + /** + * @return the resourceTypeId + */ + public int getResourceTypeId() { + return resourceTypeId; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index d8f00cae064..be2e606586d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -141,6 +141,7 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterTransportVisitor; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.RetrieveIndexDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.TransactionDataImpl; @@ -422,8 +423,10 @@ public SingleResourceResult create(FHIRPersistenceContex + ", version=" + resourceDTO.getVersionId()); } - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), - resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + } SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -581,13 +584,13 @@ private Short encodeRequestShard(String requestShard) { private ResourceDAO makeResourceDAO(FHIRPersistenceContext persistenceContext, Connection connection) throws FHIRPersistenceDataAccessException, FHIRPersistenceException, IllegalArgumentException { Short shardKey = null; - if (connectionStrategy.getFlavor().getSchemaType() == SchemaType.DISTRIBUTED) { + if (connectionStrategy.getFlavor().getSchemaType() == SchemaType.SHARDED) { if (persistenceContext == null) { - throw new FHIRPersistenceException("persistenceContext is always required for DISTRIBUTED schemas"); + throw new FHIRPersistenceException("persistenceContext is always required for SHARDED schemas"); } shardKey = encodeRequestShard(persistenceContext.getRequestShard()); if (shardKey == null) { - throw new FHIRPersistenceException("shardKey value is required for DISTRIBUTED schemas"); + throw new FHIRPersistenceException("shardKey value is required for SHARDED schemas"); } } @@ -678,8 +681,10 @@ public SingleResourceResult update(FHIRPersistenceContex } // If configured, send the extracted parameters to the remote indexing service - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), - resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + } SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) @@ -2889,15 +2894,16 @@ private ParameterTransactionDataImpl createTransactionData(String datasourceId) * that have been accumulated during the transaction. This collection therefore * contains multiple resource types, which have to be processed separately. * @param records + * @param referenceRecords * @param profileRecs * @param tagRecs * @param securityRecs * @throws FHIRPersistenceException */ - public void onCommit(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void onCommit(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { try (Connection connection = openConnection()) { IResourceReferenceDAO rrd = makeResourceReferenceDAO(connection); - rrd.persist(records, profileRecs, tagRecs, securityRecs); + rrd.persist(records, referenceRecords, profileRecs, tagRecs, securityRecs); } catch(FHIRPersistenceFKVException e) { log.log(Level.SEVERE, "FK violation", e); throw e; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java index a337ffb3db9..4a6138554a4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java @@ -16,6 +16,7 @@ import com.ibm.fhir.persistence.jdbc.TransactionData; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; /** @@ -38,6 +39,9 @@ public class ParameterTransactionDataImpl implements TransactionData { // Collect all the token values so we can submit once per transaction private final List tokenValueRecs = new ArrayList<>(); + // Collect all the reference values so we can submit once per transaction + private final List referenceValueRecs = new ArrayList<>(); + // Collect all the profile values so we can submit once per transaction private final List profileRecs = new ArrayList<>(); @@ -63,7 +67,7 @@ public ParameterTransactionDataImpl(String datasourceId, FHIRPersistenceJDBCImpl public void persist() { try { - impl.onCommit(tokenValueRecs, profileRecs, tagRecs, securityRecs); + impl.onCommit(tokenValueRecs, referenceValueRecs, profileRecs, tagRecs, securityRecs); } catch (Throwable t) { logger.log(Level.SEVERE, "Failed persisting parameter transaction data. Marking transaction for rollback", t); try { @@ -82,6 +86,14 @@ public void addValue(ResourceTokenValueRec rec) { tokenValueRecs.add(rec); } + /** + * Add the record to the list of reference values being accumulated in this transaction + * @param rec + */ + public void addReferenceValue(ResourceReferenceValueRec rec) { + referenceValueRecs.add(rec); + } + /** * Add the given profile parameter record to the list of records being accumulated in * this transaction data. The records will be inserted to the database together at the diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 114ae79728e..bf39a82b4d4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -28,6 +28,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; @@ -171,10 +172,10 @@ public Resource insert(Resource resource, List paramete // To keep things simple for the postgresql use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - // For now we bypass parameter work for DISTRIBUTED or SHARDED schemas - // because the plan is to make loading async for better ingestion performance + // Bypass the parameter insert here if we have the remoteIndexService configured + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); final String currentParameterHash = stmt.getString(oldParameterHashIndex); - if (getFlavor().getSchemaType() == SchemaType.PLAIN + if (remoteIndexService == null && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() || !parameterHashB64.equals(currentParameterHash))) { // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java index c5c578fb614..4a15aefd001 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java @@ -10,13 +10,18 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Collection; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; @@ -31,13 +36,17 @@ public class PostgresResourceReferenceDAO extends ResourceReferenceDAO { /** * Public constructor + * * @param t * @param c * @param schemaName * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public PostgresResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public PostgresResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); } @Override @@ -138,6 +147,40 @@ protected void doCommonTokenValuesUpsert(String paramList, Collection missing) throws FHIRPersistenceException { + // For PostgreSQL we can handle concurrency issues using ON CONFLICT DO NOTHING + // to skip inserts for records that already exist + final int batchSize = 256; + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + logger.fine(() -> "ident insert: " + insert.toString()); + try (PreparedStatement ps = getConnection().prepareStatement(insert.toString())) { + int count = 0; + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ps.addBatch(); + if (++count == batchSize) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } @Override protected int readOrAddParameterNameId(String parameterName) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java index 6851fd931bd..c73eee34a42 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java @@ -1,25 +1,13 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchCompartmentTest; /** @@ -27,43 +15,23 @@ */ public class JDBCSearchCompartmentTest extends AbstractSearchCompartmentTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchCompartmentTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java index 0dd3ad82480..d4668fecd09 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java @@ -6,63 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchCompositeTest; public class JDBCSearchCompositeTest extends AbstractSearchCompositeTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchCompositeTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java index 97545c59295..d4e95e1a7f3 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java @@ -6,60 +6,28 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchDateTest; public class JDBCSearchDateTest extends AbstractSearchDateTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchDateTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } + @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java index 1d1e06e24eb..960bc37001c 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java @@ -6,60 +6,28 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchIdAndLastUpdatedTest; public class JDBCSearchIdLastUpdatedTest extends AbstractSearchIdAndLastUpdatedTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchIdLastUpdatedTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java index 496b60a83dc..22858052857 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Properties; import java.util.UUID; import java.util.logging.LogManager; @@ -32,8 +31,6 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; import com.ibm.fhir.model.resource.Location; import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.model.type.Id; @@ -45,14 +42,7 @@ import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.context.FHIRPersistenceContextFactory; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.util.FHIRPersistenceUtil; import com.ibm.fhir.search.context.FHIRSearchContext; import com.ibm.fhir.search.util.SearchHelper; @@ -72,13 +62,15 @@ * */ public class JDBCSearchNearTest { - private Properties testProps; protected Location savedResource; protected static FHIRPersistence persistence; protected static SearchHelper searchHelper; + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; + @BeforeClass public void startup() throws Exception { LogManager.getLogManager().readConfiguration( @@ -88,18 +80,7 @@ public void startup() throws Exception { searchHelper = new SearchHelper(); FHIRRequestContext.get().setTenantId("default"); - testProps = TestUtil.readTestProperties("test.jdbc.properties"); - - DerbyInitializer derbyInit; - PoolConnectionProvider connectionPool; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - connectionPool = new PoolConnectionProvider(cp, 1); - } else { - throw new IllegalStateException("dbDriverName must be set in test.jdbc.properties"); - } + testSupport = new PersistenceTestSupport(); savedResource = TestUtil.readExampleResource("json/spec/location-example.json"); savedResource = savedResource.toBuilder() @@ -110,9 +91,7 @@ public void startup() throws Exception { .build()) .build(); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - persistence = new FHIRPersistenceJDBCImpl(this.testProps, connectionPool, cache); + persistence = testSupport.getPersistenceImpl(); SingleResourceResult result = persistence.create(FHIRPersistenceContextFactory.createPersistenceContext(null), savedResource); @@ -140,6 +119,9 @@ public void teardown() throws Exception { } } FHIRRequestContext.get().setTenantId("default"); + if (testSupport != null) { + testSupport.shutdown(); + } } public MultiResourceResult runQueryTest(String searchParamCode, String queryValue) throws Exception { diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java index 7375c5ac40d..b4cde5ebca4 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java @@ -6,60 +6,28 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchNumberTest; public class JDBCSearchNumberTest extends AbstractSearchNumberTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchNumberTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java index 5dcbacc7200..5a73e5051f2 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java @@ -6,61 +6,28 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchQuantityTest; public class JDBCSearchQuantityTest extends AbstractSearchQuantityTest { - - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchQuantityTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java index 561a8bef57d..9d6c06afeb4 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java @@ -6,61 +6,29 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchReferenceTest; public class JDBCSearchReferenceTest extends AbstractSearchReferenceTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchReferenceTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java index ff20f9f60d6..216a0c54163 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchStringTest; public class JDBCSearchStringTest extends AbstractSearchStringTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchStringTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java index 6d4ff1c3e4b..9f97262406a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchTokenTest; public class JDBCSearchTokenTest extends AbstractSearchTokenTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchTokenTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, this.cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java index f8baa21d3bc..394293f9248 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java @@ -6,60 +6,28 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchURITest; public class JDBCSearchURITest extends AbstractSearchURITest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchURITest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java index 5db0249970c..cef729fc9d5 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java @@ -1,66 +1,34 @@ /* - * (C) Copyright IBM Corp. 2017, 2021 + * (C) Copyright IBM Corp. 2017, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractWholeSystemSearchTest; public class JDBCWholeSystemSearchTest extends AbstractWholeSystemSearchTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCWholeSystemSearchTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java index 1c4fbd0d316..904626fddfd 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java @@ -6,23 +6,8 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractChangesTest; /** @@ -30,54 +15,28 @@ */ public class JDBCChangesTest extends AbstractChangesTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCChangesTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java index 32ce6244c31..2231c9096b2 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractCompartmentTest; public class JDBCCompartmentTest extends AbstractCompartmentTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCCompartmentTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java index 55fe6a61af6..548e9b0c0f6 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java @@ -6,20 +6,8 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractDeleteTest; /** @@ -27,45 +15,23 @@ */ public class JDBCDeleteTest extends AbstractDeleteTest { - // test properties - private Properties testProps; - - // Connection pool used to provide connections for the FHIRPersistenceJDBCImpl - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCDeleteTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java index f787df5c411..3adc4029112 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java @@ -23,6 +23,7 @@ import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractEraseTest; /** @@ -30,54 +31,28 @@ */ public class JDBCEraseTest extends AbstractEraseTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCEraseTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java index 21188a3e259..352451b9cd4 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java @@ -6,23 +6,8 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractExportTest; /** @@ -30,54 +15,28 @@ */ public class JDBCExportTest extends AbstractExportTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCExportTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java index 9d4601292a9..8fd1de71a77 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java @@ -6,20 +6,8 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractIfNoneMatchTest; /** @@ -27,45 +15,23 @@ */ public class JDBCIfNoneMatchTest extends AbstractIfNoneMatchTest { - // test properties - private Properties testProps; - - // Connection pool used to provide connections for the FHIRPersistenceJDBCImpl - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCIfNoneMatchTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java index b4b81324015..dfcae44fb7a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java @@ -6,61 +6,28 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractIncludeRevincludeTest; public class JDBCIncludeRevincludeTest extends AbstractIncludeRevincludeTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCIncludeRevincludeTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java index 34eeb477f28..2856be1272a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java @@ -6,63 +6,30 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractMultiResourceTest; public class JDBCMultiResourceTest extends AbstractMultiResourceTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCMultiResourceTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java index b0b8d8f8003..bbc33899958 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java @@ -6,76 +6,35 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractPagingTest; public class JDBCPagingTest extends AbstractPagingTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCPagingTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java index 8ae2a2781fc..0695002c370 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java @@ -6,61 +6,28 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractReverseChainTest; public class JDBCReverseChainTest extends AbstractReverseChainTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCReverseChainTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java index 5f332bc3364..d2436106568 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractSortTest; public class JDBCSortTest extends AbstractSortTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSortTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java index d2960765698..53b87f39b14 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java @@ -1,67 +1,33 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractCanonicalTest; public class JDBCanonicalTest extends AbstractCanonicalTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCanonicalTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl() throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + return testSupport.getPersistenceImpl(); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } - } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java index e1a637dec0f..61d783509fe 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java @@ -32,6 +32,7 @@ import com.ibm.fhir.persistence.jdbc.dao.EraseResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.schema.app.util.CommonUtil; import com.ibm.fhir.schema.control.FhirSchemaConstants; @@ -256,6 +257,12 @@ public void transactionCommitted() { public void transactionRolledBack() { // No Operation } + + @Override + public ILogicalResourceIdentCache getLogicalResourceIdentCache() { + // TODO Auto-generated method stub + return null; + } } /** diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java index b80d3ba8ca4..c8b4de33845 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java @@ -49,8 +49,10 @@ import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; import com.ibm.fhir.schema.derby.DerbyFhirDatabase; import com.ibm.fhir.validation.test.ValidationProcessor; @@ -356,7 +358,8 @@ protected void processDB2() throws Exception { TestFHIRConfigProvider configProvider = new TestFHIRConfigProvider(new DefaultFHIRConfigProvider()); configure(configProvider); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // Provide the credentials we need for accessing a multi-tenant schema (if enabled) // Must set this BEFORE we create our persistence object @@ -438,7 +441,8 @@ protected void processDerby() throws Exception { // layer to obtain connections. try (DerbyFhirDatabase database = new DerbyFhirDatabase()) { ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); persistence = new FHIRPersistenceJDBCImpl(this.configProps, database, cache); // create a custom list of operations to apply in order to each resource @@ -490,7 +494,8 @@ protected void processDerbyNetwork() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // create a custom list of operations to apply in order to each resource DriverMetrics dm = new DriverMetrics(); @@ -545,7 +550,8 @@ protected void processPostgreSql() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // create a custom list of operations to apply in order to each resource diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java index d6f4563a82f..d69a9da47da 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java @@ -29,8 +29,10 @@ import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; import com.ibm.fhir.persistence.test.common.AbstractPersistenceTest; import com.ibm.fhir.validation.test.ValidationProcessor; @@ -60,7 +62,8 @@ public void perform() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); List operations = new ArrayList<>(); operations.add(new CreateOperation()); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java new file mode 100644 index 00000000000..6fdcc39827a --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java @@ -0,0 +1,98 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.derby.DerbyMaster; +import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; +import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.persistence.FHIRPersistence; +import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; +import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; + +/** + * Encapsulates the instantiation of objects needed to support the JDBC persistence tests. + * If the constructors for these objects change, we only need to modify thir instantiation + * here instead of every for every concrete test class + */ +public class PersistenceTestSupport { + private static final Logger logger = Logger.getLogger(PersistenceTestSupport.class.getName()); + private Properties testProps; + + private PoolConnectionProvider connectionPool; + + private FHIRPersistenceJDBCCache cache; + + /** + * Public constructor + * @throws Exception + */ + public PersistenceTestSupport() throws Exception { + this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); + DerbyInitializer derbyInit; + String dbDriverName = this.testProps.getProperty("dbDriverName"); + if (dbDriverName != null && dbDriverName.contains("derby")) { + derbyInit = new DerbyInitializer(this.testProps); + IConnectionProvider cp = derbyInit.getConnectionProvider(false); + this.connectionPool = new PoolConnectionProvider(cp, 1); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); + } + } + + /** + * Return a new FHIRPersistence implementation configured using the connection pool + * and cache from this object + * @return + * @throws Exception + */ + public FHIRPersistence getPersistenceImpl() throws Exception { + + if (this.connectionPool == null) { + throw new IllegalStateException("Database not bootstrapped"); + } + + return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + + } + + /** + * Close any resources we may still have open + */ + public void shutdown() { + if (this.connectionPool != null) { + this.connectionPool.close(); + } + } + + /** + * Debug locks in the Derby database we're using + */ + public void debugLocks() { + // Exception running a query. Let's dump the lock table + try (Connection c = connectionPool.getConnection()) { + DerbyMaster.dumpLockInfo(c); + } catch (SQLException x) { + // just log the error...things are already bad if this method has been called + logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); + } + + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java index a2e0b8dc31e..f1a4f5f3450 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -25,6 +25,7 @@ import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; import com.ibm.fhir.persistence.jdbc.util.type.NewNumberParmBehaviorUtil; import com.ibm.fhir.persistence.jdbc.util.type.NewQuantityParmBehaviorUtil; import com.ibm.fhir.search.SearchConstants; @@ -169,6 +170,21 @@ public List getResourceTypeNames() throws FHIRPersistenceException { public List getResourceTypeIds() throws FHIRPersistenceException { return null; } + + @Override + public Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException { + return null; + } + + @Override + public Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + return null; + } + + @Override + public List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + return null; + } }; } //--------------------------------------------------------------------------------------------------------- diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 3a883b6efb8..ba53fc53d6e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -318,7 +318,7 @@ public class Main { private ILeaseManagerConfig leaseManagerConfig; // Which flavor of the FHIR data schema should we build? - private SchemaType dataSchemaType; + private SchemaType dataSchemaType = SchemaType.PLAIN; // ----------------------------------------------------------------------------------------------------------------- // The following method is related to the common methods and functions diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index 27acf93b74f..00b6b66fd94 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -36,6 +36,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_IDENT; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LONGITUDE_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MT_ID; @@ -52,6 +53,8 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_HIGH; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_LOW; import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VALUES_V; import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VERSION_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_PAYLOAD_KEY; @@ -180,6 +183,7 @@ public ObjectGroup addResourceType(String resourceTypeName) { addResourceTokenRefs(group, tablePrefix); addRefValues(group, tablePrefix); addTokenValuesView(group, tablePrefix); + addRefValuesView(group, tablePrefix); addProfiles(group, tablePrefix); addTags(group, tablePrefix); addSecurity(group, tablePrefix); @@ -811,6 +815,50 @@ public void addTokenValuesView(List group, String prefix) { group.add(view); } + /** + * View to encapsulate the join between xx_ref_values and logical_resource_ident + * tables, which makes it easier for the search query builder to compose search + * queries using reference parameters. + * @param group + * @param prefix + */ + public void addRefValuesView(List group, String prefix) { + + final String viewName = prefix + "_" + REF_VALUES_V; + + // Find the two dependencies we need for this view + IDatabaseObject logicalResourceIdent = model.findTable(schemaName, LOGICAL_RESOURCE_IDENT); + IDatabaseObject refValues = model.findTable(schemaName, prefix + "_" + REF_VALUES); + + StringBuilder select = new StringBuilder(); + if (this.multitenant) { + // Make sure we include MT_ID in both the select list and join condition. It's needed + // in the join condition to give the optimizer the best chance at finding a good nested + // loop strategy + select.append("SELECT ref.").append(MT_ID); + select.append(" , ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id"); + select.append(" FROM ").append(logicalResourceIdent.getName()).append(" AS lri, "); + select.append(refValues.getName()).append(" AS ref "); + select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id "); + select.append(" AND lri.").append(MT_ID).append(" = ").append("ref.").append(MT_ID); + } else { + select.append("SELECT ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id"); + select.append(" FROM ").append(logicalResourceIdent.getName()).append(" AS lri, "); + select.append(refValues.getName()).append(" AS ref "); + select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id "); + } + + View view = View.builder(schemaName, viewName) + .setVersion(FhirSchemaVersion.V0027.vid()) + .setSelectClause(select.toString()) + .addPrivileges(resourceTablePrivileges) + .addDependency(logicalResourceIdent) + .addDependency(refValues) + .build(); + + group.add(view); + } + /** * 
 CREATE TABLE device_date_values  (
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
index 7bb95967a00..4a4cceeee17 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
@@ -188,6 +188,8 @@ public class FhirSchemaConstants {
 
     // View suffix to overlay the new common_token_values and resource_token_refs tables
     public static final String TOKEN_VALUES_V = "TOKEN_VALUES_V";
+    public static final String REF_VALUES = "REF_VALUES";
+    public static final String REF_VALUES_V = "REF_VALUES_V";
 
     public static final String LOGICAL_RESOURCE_COMPARTMENTS = "LOGICAL_RESOURCE_COMPARTMENTS";
     public static final String COMPARTMENT_LOGICAL_RESOURCE_ID = "COMPARTMENT_LOGICAL_RESOURCE_ID";
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
index 5687302faf6..80b1f3e3b4f 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
@@ -776,7 +776,7 @@ private void addLogicalResourceIdent(PhysicalDataModel pdm) {
                 .addIntColumn(RESOURCE_TYPE_ID, false)
                 .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false)
                 .addBigIntColumn(LOGICAL_RESOURCE_ID, false)
-                .addPrimaryKey(tableName + "_PK", RESOURCE_TYPE_ID, LOGICAL_ID)
+                .addPrimaryKey(tableName + "_PK", LOGICAL_ID, RESOURCE_TYPE_ID) // we need this order for a specific index
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID)
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java
index 8e7b7945206..3a3faa8e266 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java
@@ -35,14 +35,15 @@ public GetLogicalResourceNeedsV0027Migration(String schemaName) {
 
     @Override
     public Boolean run(IDatabaseTranslator translator, Connection c) {
-        Boolean result = false;
-        final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES");
+        Boolean result = true;
+        final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT");
         final String SQL = "SELECT 1 FROM " + tableName + " " + translator.limit("1");
 
         try (Statement s = c.createStatement()) {
             ResultSet rs = s.executeQuery(SQL);
             if (rs.next()) {
-                result = true;
+                // logical_resource_ident already contains data, so no need to migrate
+                result = false;
             }
         } catch (SQLException x) {
             throw translator.translate(x);
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
index 497c80aab0e..56d6e6a439c 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
@@ -142,7 +142,7 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S
 
                 // Check that we have the correct number of tables. This will need to be updated
                 // whenever tables, views or sequences are added or removed
-                assertEquals(adapter.listSchemaObjects(schemaName).size(), 2065);
+                assertEquals(adapter.listSchemaObjects(schemaName).size(), 2357);
                 c.commit();
             } catch (Throwable t) {
                 c.rollback();
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java
index ddb50e5b147..ad5d5d73a3a 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java
@@ -6,14 +6,14 @@
  
 package com.ibm.fhir.persistence.index;
 
-import java.sql.Timestamp;
+import java.time.Instant;
 
 /**
  * A date search parameter value
  */
 public class DateParameter extends SearchParameterValue {
-    private Timestamp valueDateStart;
-    private Timestamp valueDateEnd;
+    private Instant valueDateStart;
+    private Instant valueDateEnd;
 
     @Override
     public String toString() {
@@ -31,28 +31,28 @@ public String toString() {
     /**
      * @return the valueDateStart
      */
-    public Timestamp getValueDateStart() {
+    public Instant getValueDateStart() {
         return valueDateStart;
     }
     
     /**
      * @param valueDateStart the valueDateStart to set
      */
-    public void setValueDateStart(Timestamp valueDateStart) {
+    public void setValueDateStart(Instant valueDateStart) {
         this.valueDateStart = valueDateStart;
     }
     
     /**
      * @return the valueDateEnd
      */
-    public Timestamp getValueDateEnd() {
+    public Instant getValueDateEnd() {
         return valueDateEnd;
     }
     
     /**
      * @param valueDateEnd the valueDateEnd to set
      */
-    public void setValueDateEnd(Timestamp valueDateEnd) {
+    public void setValueDateEnd(Instant valueDateEnd) {
         this.valueDateEnd = valueDateEnd;
     }
 
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
index 173a7334ab8..32a45fc304b 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
@@ -7,7 +7,7 @@
 package com.ibm.fhir.persistence.index;
 
 import java.math.BigDecimal;
-import java.sql.Timestamp;
+import java.time.Instant;
 
 /**
  * Used by a parameter value visitor to translate the parameter values
@@ -39,7 +39,7 @@ public interface ParameterValueVisitorAdapter {
      * @param compositeId
      * @param wholeSystem
      */
-    void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId, boolean wholeSystem);
+    void dateValue(String name, Instant valueDateStart, Instant valueDateEnd, Integer compositeId, boolean wholeSystem);
 
     /**
      * @param name
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java
index 2bed02ddef0..cd5954b151a 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java
@@ -34,8 +34,8 @@ public class SearchParametersTransport {
     // The parameter hash computed for this set of parameters
     private String parameterHash;
 
-    // The last_updated time in a fixed format for transport
-    private String lastUpdated;
+    // The last_updated time
+    private Instant lastUpdated;
 
     // The key value used for sharding the data when using a distributed database
     private String requestShard;
@@ -46,10 +46,10 @@ public class SearchParametersTransport {
     private List tokenValues;
     private List dateValues;
     private List locationValues;
-    private List tagValues = new ArrayList<>();
-    private List profileValues = new ArrayList<>();
-    private List securityValues = new ArrayList<>();
-    private List refValues = new ArrayList<>();
+    private List tagValues;
+    private List profileValues;
+    private List securityValues;
+    private List refValues;
 
     /**
      * Factory method to create a {@link Builder} instance
@@ -101,7 +101,7 @@ public static class Builder {
         private String requestShard;
         private int versionId;
         private String parameterHash;
-        private String lastUpdated;
+        private Instant lastUpdated;
 
         /**
          * Set the resourceType
@@ -124,7 +124,7 @@ public Builder withParameterHash(String hash) {
         }
 
         public Builder withLastUpdated(Instant lastUpdated) {
-            this.lastUpdated = lastUpdated.toString();
+            this.lastUpdated = lastUpdated;
             return this;
         }
 
@@ -565,14 +565,14 @@ public void setParameterHash(String parameterHash) {
     /**
      * @return the lastUpdated
      */
-    public String getLastUpdated() {
+    public Instant getLastUpdated() {
         return lastUpdated;
     }
 
     /**
      * @param lastUpdated the lastUpdated to set
      */
-    public void setLastUpdated(String lastUpdated) {
+    public void setLastUpdated(Instant lastUpdated) {
         this.lastUpdated = lastUpdated;
     }
 
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java
index 4c29c5640ae..c156a994df2 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java
@@ -7,7 +7,6 @@
 package com.ibm.fhir.persistence.index;
 
 import java.math.BigDecimal;
-import java.sql.Timestamp;
 import java.time.Instant;
 
 
@@ -75,7 +74,7 @@ public void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNum
     }
 
     @Override
-    public void dateValue(String name, Timestamp valueDateStart, Timestamp valueDateEnd, Integer compositeId, boolean wholeSystem) {
+    public void dateValue(String name, Instant valueDateStart, Instant valueDateEnd, Integer compositeId, boolean wholeSystem) {
         DateParameter value = new DateParameter();
         value.setName(name);
         value.setValueDateStart(valueDateStart);
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java
index 36b7865d90c..da4751977d2 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java
@@ -8,6 +8,7 @@
 
 import java.sql.Connection;
 import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -205,9 +206,11 @@ public void process(String requestShard, String resourceType, String logicalId,
 
         try {
             PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
-            dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId());
+            final Timestamp valueDateStart = Timestamp.from(p.getValueDateStart());
+            final Timestamp valueDateEnd = Timestamp.from(p.getValueDateEnd());
+            dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId());
             if (p.isSystemParam()) {
-                systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueDateStart(), p.getValueDateEnd(), p.getCompositeId());
+                systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId());
             }
         } catch (SQLException x) {
             throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'");
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
index 006f298df55..e24fba20d99 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
@@ -1028,7 +1028,7 @@ protected void checkReady(List messages, List
Date: Fri, 27 May 2022 15:34:26 +0100
Subject: [PATCH 11/40] issue #3437 use logical_resource_ident for absolute
 references

Signed-off-by: Robin Arnold 
---
 .../fhir/persistence/jdbc/JDBCConstants.java  |   1 +
 .../dao/impl/ParameterVisitorBatchDAO.java    |  41 +++----
 .../jdbc/domain/SearchQueryRenderer.java      | 115 +++++++-----------
 .../control/FhirResourceTableGroup.java       |   8 +-
 .../schema/control/FhirSchemaGenerator.java   |   5 +-
 5 files changed, 71 insertions(+), 99 deletions(-)

diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
index 334e0c7eca2..9e3e8d041fe 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
@@ -39,6 +39,7 @@ public class JDBCConstants {
     public static final String _LOGICAL_RESOURCES = "_LOGICAL_RESOURCES";
     public static final String RESOURCE_ID = "RESOURCE_ID";
     public static final String RESOURCE_TYPE_ID = "RESOURCE_TYPE_ID";
+    public static final String REF_VALUE = "REF_VALUE";
     public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID";
     public static final String LOGICAL_ID = "LOGICAL_ID";
     public static final String LOGICAL_RESOURCE_ID = "LOGICAL_RESOURCE_ID";
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java
index 828e8b4f389..f32b1be9981 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java
@@ -711,33 +711,22 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException {
             throw new IllegalArgumentException("Invalid reference parameter value. See server log for details.");
         }
 
-        // reference params are never system-level
-        if (refResourceType != null) {
-            // Store a reference value configured as a reference to another resource
-            int refResourceTypeId = identityCache.getResourceTypeId(refResourceType);
-            ResourceReferenceValueRec rec = new ResourceReferenceValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, 
-                refResourceType, refResourceTypeId, 
-                refLogicalId, refVersion, this.currentCompositeId);
-            if (this.transactionData != null) {
-                this.transactionData.addReferenceValue(rec);
-            } else {
-                this.referenceValueRecs.add(rec);
-            }
+        // V0027. Absolute references won't have a resource type, but in order to store them
+        // in the LOGICAL_RESOURCE_IDENT table we need to have a valid LOGICAL_RESOURCE_ID. For
+        // that we use "Resource"
+        if (refResourceType == null) {
+            refResourceType = "Resource";
+        }
+        // Store a reference value configured as a reference to another resource (reference params
+        // are never system-level).
+        int refResourceTypeId = identityCache.getResourceTypeId(refResourceType);
+        ResourceReferenceValueRec rec = new ResourceReferenceValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, 
+            refResourceType, refResourceTypeId, 
+            refLogicalId, refVersion, this.currentCompositeId);
+        if (this.transactionData != null) {
+            this.transactionData.addReferenceValue(rec);
         } else {
-            // uri so we go back to store as a string instead
-            logger.info("reference param[" + parameterName + "] value[" + refLogicalId + "] => xx_str_values");
-            try {
-                int parameterNameId = getParameterNameId(parameterName);
-                setStringParms(strings, parameterNameId, refValue.getValue());
-                strings.addBatch();
-    
-                if (++stringCount == this.batchSize) {
-                    strings.executeBatch();
-                    stringCount = 0;
-                }
-            } catch (SQLException x) {
-                throw new FHIRPersistenceDataAccessException(parameterName + "='" + refValue.getValue() + "'", x);
-            }
+            this.referenceValueRecs.add(rec);
         }
     }
 }
\ No newline at end of file
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java
index d6b3cd19bfe..1949d94ea83 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java
@@ -29,6 +29,7 @@
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PERCENT_WILDCARD;
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.QUANTITY_VALUE;
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.REF_LOGICAL_RESOURCE_ID;
+import static com.ibm.fhir.persistence.jdbc.JDBCConstants.REF_VALUE;
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RESOURCE_TYPE_ID;
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RIGHT_PAREN;
 import static com.ibm.fhir.persistence.jdbc.JDBCConstants.TOKEN_VALUE;
@@ -2149,16 +2150,16 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm,
             exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(parameterNameId);
         }
 
-        if (queryParm.getType() == Type.REFERENCE) {
-            // From V0027 we store absolute references in xx_str_values, so need to check there too
-            final String strParamAlias = getParamAlias(getNextAliasIndex());
-            final String strParamTableName = resourceType + "_STR_VALUES";
-            SelectAdapter strExists = Select.select("1");
-            strExists.from(strParamTableName, alias(strParamAlias))
-                    .where(strParamAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query
-                    .and(strParamAlias, PARAMETER_NAME_ID).eq(parameterNameId);
-            exists.unionAll(strExists.build());
-        }
+//        if (queryParm.getType() == Type.REFERENCE) {
+//            // From V0027 we store absolute references in xx_str_values, so need to check there too
+//            final String strParamAlias = getParamAlias(getNextAliasIndex());
+//            final String strParamTableName = resourceType + "_STR_VALUES";
+//            SelectAdapter strExists = Select.select("1");
+//            strExists.from(strParamTableName, alias(strParamAlias))
+//                    .where(strParamAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") // correlate with the main query
+//                    .and(strParamAlias, PARAMETER_NAME_ID).eq(parameterNameId);
+//            exists.unionAll(strExists.build());
+//        }
 
         // Add the exists to the where clause of the main query which already has a predicate
         // so we need to AND the exists
@@ -2291,7 +2292,21 @@ public void addFilter(QueryData queryData, String resourceType, QueryParameter c
                 paramTable = paramValuesTableName(queryData.getResourceType(), currentParm);
             }
 
-            if (currentParm.getModifier() == Modifier.NOT) {
+            if (Type.REFERENCE.equals(currentParm.getType())) {
+                // V0027, reference filters now need to look at both xx_ref_values and xx_str_values
+                // so we use a full correlated sub-query for exists/not-exists
+                final String anchorAlias = "LR" + getNextAliasIndex();
+                SelectAdapter exists = Select.select("1");
+                exists.from("LOGICAL_RESOURCES", alias(anchorAlias))
+                    .where(anchorAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID"); // correlate to parent query
+                QueryData subQuery = new QueryData(exists, anchorAlias, null, resourceType, 0);
+                addReferenceParam(subQuery, queryData.getResourceType(), currentParm);
+                if (currentParm.getModifier() == Modifier.NOT) {
+                    currentSubQuery.from().where().and().notExists(exists.build());
+                } else {
+                    currentSubQuery.from().where().and().exists(exists.build());
+                }
+            } else if (currentParm.getModifier() == Modifier.NOT) {
                 // Needs to be handled as a NOT EXISTS correlated subquery
                 SelectAdapter exists = Select.select("1");
                 exists.from(paramTable, alias(paramAlias))
@@ -2529,7 +2544,12 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource
         final String paramAlias = getParamAlias(aliasIndex);
         final String lrAlias = queryData.getLRAlias();
 
-        // For V0027 we split reference parameters into two tables: xx_ref_values and xx_str_values
+        // For V0027 reference parameters are stored in xx_ref_values using the
+        // logical_id values stored in logical_resource_ident. Absolute references
+        // are stored using a resource_type of "Resource" (similar to the default
+        // code-system we used to use with common_token_values).
+        final int resourceTypeIdForResource = identityCache.getResourceTypeId("Resource");
+
         // Firstly we need to split the query parm values into separate lists
         List> resourceTypesAndIds = new ArrayList<>(queryParm.getValues().size());
         for (QueryParameterValue value : queryParm.getValues()) {
@@ -2538,7 +2558,6 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource
 
         List logicalResourceIdList = new ArrayList<>();
         List refValues = new ArrayList<>(queryParm.getValues().size());
-        List absoluteReferenceValues = new ArrayList<>();
         for (Pair resourceTypeAndId : resourceTypesAndIds) {
             String targetResourceType = resourceTypeAndId.getLeft();
             String referenceValue = resourceTypeAndId.getRight();
@@ -2563,7 +2582,8 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource
                 // Determine if the target value is an absolute or local reference
                 if (ReferenceUtil.isAbsolute(referenceValue)) {
                     logger.info(() -> "reference search value: type[absolute] value[" + referenceValue + "]");
-                    absoluteReferenceValues.add(referenceValue);
+                    Long logicalResourceId = identityCache.getLogicalResourceId("Resource", referenceValue);
+                    logicalResourceIdList.add(logicalResourceId != null ? logicalResourceId : -1);
                 } else {
                     // treat as a local reference where we don't know the type.
                     List localLogicalResourceIds = getLogicalResourceIdList(referenceValue);
@@ -2572,7 +2592,9 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource
                         logicalResourceIdList.add(localLogicalResourceIds.get(0));
                     } else if (localLogicalResourceIds.size() == 0) {
                         logger.info(() -> "reference search value: type[local] value[" + referenceValue + "] notFound[true]");
-                        logicalResourceIdList.add(-1L);
+                        if (logicalResourceIdList.isEmpty()) {
+                            logicalResourceIdList.add(-1L); // need at least one value
+                        }
                     } else {
                         // We may match multiple resource types here, but it's only an error
                         // if we join with the xx_ref_value table and still get multiple rows
@@ -2582,60 +2604,13 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource
             }
         }
 
+        // Only need to join with xx_ref_values
         final String queryParmCode = queryParm.getCode();
-        if (absoluteReferenceValues.isEmpty()) {
-            // Only need to join with xx_ref_values
-            final ExpNode filter = getReferenceFilter(queryParm, paramAlias, logicalResourceIdList).getExpression();
-            final String paramTableName = getRefParamTable(filter, resourceType, paramAlias);
-            query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID")
-                .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode))
-                .and(filter));
-        } else if (refValues.isEmpty()) {
-            // Only need to join with xx_str_values
-            final ExpNode filter = getReferenceStrFilter(queryParm, paramAlias, absoluteReferenceValues).getExpression();
-            final String paramTableName = resourceType + "_str_values";
-            query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID")
-                .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode))
-                .and(filter));
-        } else {
-            // The more complicated scenario where we need to filter on xx_ref_values
-            // and bolt on a union all with a filter on the xx_str_values. It's an
-            // edge-case, which is lucky because the query plan won't be as clean as
-            // the prior two cases. But this form is required for the correct semantics.
-            // SELECT P2.LOGICAL_RESOURCE_ID 
-            //   FROM ibmfhirpg3.Basic_REF_VALUES AS P2 
-            //  WHERE P2.PARAMETER_NAME_ID = 25585  
-            //    AND (P2.REF_LOGICAL_RESOURCE_ID = 1 OR P2.REF_LOGICAL_RESOURCE_ID = 2 OR ...)
-            //  UNION ALL
-            // SELECT P3.LOGICAL_RESOURCE_ID
-            //   FROM ibmfhirpg3.Basic_STR_VALUES AS P3
-            //  WHERE P3.PARAMETER_NAME_ID = 25585
-            //    AND (P3.STR_VALUE = 'abc')
-
-            final int parameterNameId = getParameterNameId(queryParmCode);
-            final String refParamAlias = getParamAlias(getNextAliasIndex());
-            final ExpNode refFilter = getReferenceFilter(queryParm, refParamAlias, logicalResourceIdList).getExpression();
-            final String refParamTableName = getRefParamTable(refFilter, resourceType, refParamAlias);
-            final String strParamAlias = getParamAlias(getNextAliasIndex());
-            final ExpNode strFilter = getReferenceStrFilter(queryParm, strParamAlias, absoluteReferenceValues).getExpression();
-            final String strParamTableName = resourceType + "_str_values";
-
-            SelectAdapter strSelect = Select.select("LOGICAL_RESOURCE_ID");
-            strSelect.from(strParamTableName, alias(strParamAlias))
-                .where(strParamAlias, "PARAMETER_NAME_ID").eq(parameterNameId)
-                .and(strFilter);
-
-            SelectAdapter refSelect = Select.select("LOGICAL_RESOURCE_ID");
-            refSelect.from(refParamTableName, alias(refParamAlias))
-                .where(refParamAlias, "PARAMETER_NAME_ID").eq(parameterNameId)
-                .and(refFilter);
-            refSelect.unionAll(strSelect.build());
-            
-            // add everything to the main query
-            query.from().innerJoin(refSelect.build(), alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID")
-                .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode))
-                );
-        }
+        final ExpNode filter = getReferenceFilter(queryParm, paramAlias, logicalResourceIdList).getExpression();
+        final String paramTableName = getRefParamTable(filter, resourceType, paramAlias);
+        query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID")
+            .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode))
+            .and(filter));
 
         return queryData;
     }
@@ -2910,6 +2885,8 @@ protected String getSortParameterTableName(String resourceType, String code, Typ
             sortParameterTableName.append("DATE_VALUES");
             break;
         case REFERENCE:
+            sortParameterTableName.append("REF_VALUES_V");
+            break;
         case TOKEN:
             if (!this.legacyWholeSystemSearchParamsEnabled && TAG.equals(code)) {
                 sortParameterTableName.append("TAGS");
@@ -3004,7 +2981,7 @@ private List getValueAttributeNames(Type type) throws FHIRPersistenceExc
             attributeNames.add(STR_VALUE);
             break;
         case REFERENCE:
-            attributeNames.add(TOKEN_VALUE);
+            attributeNames.add(REF_VALUE); // V0027 using xx_REF_VALUES_V
             break;
         case DATE:
             attributeNames.add(DATE_START);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java
index 00b6b66fd94..9ab4e67907f 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java
@@ -836,13 +836,17 @@ public void addRefValuesView(List group, String prefix) {
             // in the join condition to give the optimizer the best chance at finding a good nested
             // loop strategy
             select.append("SELECT ref.").append(MT_ID);
-            select.append("     , ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id");
+            select.append("       ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ");
+            select.append("       ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id, ");
+            select.append("       lri.logical_id AS ref_value ");
             select.append("  FROM ").append(logicalResourceIdent.getName()).append(" AS lri, ");
             select.append(refValues.getName()).append(" AS ref ");
             select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id ");
             select.append("   AND lri.").append(MT_ID).append(" = ").append("ref.").append(MT_ID);
         } else {
-            select.append("SELECT ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id");
+            select.append("SELECT ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, ");
+            select.append("       ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id, ");
+            select.append("       lri.logical_id AS ref_value ");
             select.append("  FROM ").append(logicalResourceIdent.getName()).append(" AS lri, ");
             select.append(refValues.getName()).append(" AS ref ");
             select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id ");
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
index 80b1f3e3b4f..1baf4283e40 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
@@ -774,9 +774,10 @@ private void addLogicalResourceIdent(PhysicalDataModel pdm) {
                 .setDistributionType(DistributionType.DISTRIBUTED)
                 .setDistributionColumnName(LOGICAL_ID)             // override distribution column for this table
                 .addIntColumn(RESOURCE_TYPE_ID, false)
-                .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false)
+                .addVarcharColumn(LOGICAL_ID, MAX_SEARCH_STRING_BYTES, false) // used to also store absolute reference values
                 .addBigIntColumn(LOGICAL_RESOURCE_ID, false)
-                .addPrimaryKey(tableName + "_PK", LOGICAL_ID, RESOURCE_TYPE_ID) // we need this order for a specific index
+                .addPrimaryKey(tableName + "_PK", LOGICAL_ID, RESOURCE_TYPE_ID) // do not change this order
+                .addIndex("IDX_" + LOGICAL_RESOURCE_IDENT + "_LRID", LOGICAL_RESOURCE_ID) // non-unique to allow easy distribution
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID)

From 8c5cc75e3830c30ba8165883465bb1b18001e020 Mon Sep 17 00:00:00 2001
From: Robin Arnold 
Date: Fri, 27 May 2022 16:08:12 +0100
Subject: [PATCH 12/40] issue #3437 compilation issues from shard update

Signed-off-by: Robin Arnold 
---
 .../export/patient/resource/PatientResourceHandler.java         | 2 +-
 .../ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java    | 2 +-
 .../com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java | 2 +-
 .../fhir/persistence/cassandra/payload/CqlDeletePayload.java    | 2 +-
 .../ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java
index 2d1f91894ca..68542ceccc6 100644
--- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java
+++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java
@@ -125,7 +125,7 @@ public List executeSearch(Set patientIds) throws Exception {
 
             do {
                 searchContext.setPageNumber(compartmentPageNum);
-                FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext);
+                FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null);
 
                 Date startTime = new Date(System.currentTimeMillis());
                 List> resourceResults = fhirPersistence.search(persistenceContext, resourceType).getResourceResults();
diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java
index 7807463beeb..5b0cca0ba30 100644
--- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java
+++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java
@@ -201,7 +201,7 @@ public Object readItem() throws Exception {
         FHIRTransactionHelper txn = new FHIRTransactionHelper(fhirPersistence.getTransaction());
         txn.begin();
         try {
-            FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext);
+            FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null);
             Date startTime = new Date(System.currentTimeMillis());
             List> resourceResults = fhirPersistence.search(persistenceContext, Patient.class).getResourceResults();
             List patientResources = ResourceResult.toResourceList(resourceResults);
diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java
index dd9991ac925..f803038a714 100644
--- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java
+++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java
@@ -222,7 +222,7 @@ public Object readItem() throws Exception {
         Date startTime = new Date(System.currentTimeMillis());
         try {
             // Execute the search query to obtain the page of resources
-            persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext);
+            persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null);
             List> resourceResults = fhirPersistence.search(persistenceContext, resourceType).getResourceResults();
             resources = ResourceResult.toResourceList(resourceResults);
 
diff --git a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java
index 90a7d619c60..458b5919040 100644
--- a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java
+++ b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java
@@ -25,8 +25,8 @@
 import com.datastax.oss.driver.api.core.cql.Row;
 import com.datastax.oss.driver.api.core.cql.SimpleStatement;
 import com.datastax.oss.driver.api.querybuilder.select.Select;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
-import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException;
 
 /**
  * DAO to delete all the records associated with this resource payload
diff --git a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java
index e8837783779..060384e1ffb 100644
--- a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java
+++ b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java
@@ -28,8 +28,8 @@
 import com.datastax.oss.driver.api.core.cql.PreparedStatement;
 import com.datastax.oss.driver.api.core.cql.SimpleStatement;
 import com.datastax.oss.driver.api.querybuilder.insert.RegularInsert;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
-import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException;
 import com.ibm.fhir.persistence.util.InputOutputByteStream;
 
 /**

From ca0bfa05d3c90b852bb03addbe08bfec14fe02d2 Mon Sep 17 00:00:00 2001
From: Robin Arnold 
Date: Mon, 30 May 2022 11:13:46 +0100
Subject: [PATCH 13/40] issue #3437 fix remote index processing for new
 logical_resource_ident recs

Signed-off-by: Robin Arnold 
---
 .../fhir/persistence/jdbc/JDBCConstants.java  |  3 +
 .../dao/impl/ParameterTransportVisitor.java   |  7 ++-
 .../fhir/remote/index/api/IdentityCache.java  |  8 +++
 .../remote/index/cache/IdentityCacheImpl.java | 22 +++++++
 .../remote/index/database/CacheLoader.java    | 22 ++++++-
 .../database/LogicalResourceIdentValue.java   | 61 ++++++++++++++++---
 .../database/PlainPostgresMessageHandler.java | 20 +++---
 .../database/PlainPostgresParameterBatch.java |  2 +-
 .../index/database/ResourceTypeValue.java     | 42 +++++++++++++
 9 files changed, 164 insertions(+), 23 deletions(-)
 create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java

diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
index 9e3e8d041fe..58c37a91177 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java
@@ -119,6 +119,9 @@ public class JDBCConstants {
     // Default code_system_id value
     public static final String DEFAULT_TOKEN_SYSTEM = "default-token-system";
 
+    // Default resource type for references without a resource type
+    public static final String RESOURCE = "Resource";
+
     /**
      * This Calendar object is not thread-safe! Use CalendarHelper#getCalendarForUTC() instead.
      */
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java
index 4e0b0d6548b..595fa1b344a 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java
@@ -134,8 +134,13 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException {
             logger.warning("Invalid reference parameter type: '" + resourceType + "." + rpv.getName() + "' type=" + refValue.getType().name());
             throw new IllegalArgumentException("Invalid reference parameter value. See server log for details.");
         }
+
         if (refResourceType == null) {
-            refResourceType = JDBCConstants.DEFAULT_TOKEN_SYSTEM;
+            // Prior to V0027, references without a target resource type would be assigned the
+            // DEFAULT_TOKEN_SYSTEM (having a valid system makes queries faster). For V0027,
+            // all reference values get an entry in logical_resource_ident so in order to use
+            // a valid resource type we use "Resource" instead.
+            refResourceType = JDBCConstants.RESOURCE;
         }
         adapter.referenceValue(rpv.getName(), refResourceType, refLogicalId, refVersion, compositeId);
     }
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java
index cb3e5679f78..1f731792db3 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java
@@ -48,4 +48,12 @@ public interface IdentityCache {
      * @return
      */
     Long getCommonCanonicalValueId(short shardKey, String url);
+
+    /**
+     * Get the database resource_type_id value for the given resourceType value
+     * @param resourceType
+     * @return
+     * @throws IllegalArgumentException if resourceType is not a valid resource type name
+     */
+    int getResourceTypeId(String resourceType);
 }
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java
index 0b47e4d8cee..a0a43164daf 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java
@@ -7,6 +7,7 @@
 package com.ibm.fhir.remote.index.cache;
 
 import java.time.Duration;
+import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.github.benmanes.caffeine.cache.Cache;
@@ -14,6 +15,7 @@
 import com.ibm.fhir.remote.index.api.IdentityCache;
 import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey;
 import com.ibm.fhir.remote.index.database.CommonTokenValueKey;
+import com.ibm.fhir.remote.index.database.ResourceTypeValue;
 
 /**
  * Implementation of a cache we use to reduce the number of databases accesses
@@ -21,6 +23,7 @@
  */
 public class IdentityCacheImpl implements IdentityCache {
     private final ConcurrentHashMap parameterNames = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap resourceTypes = new ConcurrentHashMap<>();
     private final Cache codeSystemCache;
     private final Cache commonTokenValueCache;
     private final Cache commonCanonicalValueCache;
@@ -47,6 +50,16 @@ public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDur
                 .build();
     }
 
+    /**
+     * Initialize the cache
+     * @param resourceTypeValues the complete list of resource types
+     */
+    public void init(Collection resourceTypeValues) {
+        for (ResourceTypeValue rtv: resourceTypeValues) {
+            resourceTypes.put(rtv.getResourceType(), rtv.getResourceTypeId());
+        }
+    }
+
     @Override
     public Integer getParameterNameId(String parameterName) {
         // This should only miss if the parameter name value doesn't actually
@@ -73,4 +86,13 @@ public void addParameterName(String parameterName, int parameterNameId) {
     public Long getCommonCanonicalValueId(short shardKey, String url) {
         return commonCanonicalValueCache.get(new CommonCanonicalValueKey(shardKey, url), k -> NULL_LONG);
     }
+
+    @Override
+    public int getResourceTypeId(String resourceType) {
+        Integer resourceTypeId = resourceTypes.get(resourceType);
+        if (resourceTypeId == null) {
+            throw new IllegalArgumentException("Not a valid resource type: " + resourceType);
+        }
+        return resourceTypeId;
+    }
 }
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
index aa4e67fa46a..9e4d08a194c 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
@@ -10,25 +10,41 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
-import com.ibm.fhir.remote.index.api.IdentityCache;
+import com.ibm.fhir.remote.index.cache.IdentityCacheImpl;
 
 /**
  * Preload the cache
  */
 public class CacheLoader {
-    private final IdentityCache cache;
+    private final IdentityCacheImpl cache;
 
     /**
      * Public constructor
      * @param cache
      */
-    public CacheLoader(IdentityCache cache) {
+    public CacheLoader(IdentityCacheImpl cache) {
         this.cache = cache;
     }
 
     public void apply(Connection connection) throws FHIRPersistenceException {
+        // load the static list of resource types
+        List resourceTypes = new ArrayList<>();
+        final String SELECT_RESOURCE_TYPES = "SELECT resource_type, resource_type_id FROM resource_types";
+        try (PreparedStatement ps = connection.prepareStatement(SELECT_RESOURCE_TYPES)) {
+            ResultSet rs = ps.executeQuery();
+            while (rs.next()) {
+                resourceTypes.add(new ResourceTypeValue(rs.getString(1), rs.getInt(2)));
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("fetch parameter names failed", x);
+        }
+        cache.init(resourceTypes);
+
+        // also seed the cache with all the parameter_names we know so far
         final String SQL = "SELECT parameter_name, parameter_name_id FROM parameter_names";
         try (PreparedStatement ps = connection.prepareStatement(SQL)) {
             ResultSet rs = ps.executeQuery();
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java
index 702ab132a85..7e882366599 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java
@@ -12,24 +12,70 @@
  * A DTO representing a record from logical_resource_ident
  */
 public class LogicalResourceIdentValue implements Comparable {
+    private final int resourceTypeId;
     private final String resourceType;
     private final String logicalId;
     private Long logicalResourceId;
-    private Integer resourceTypeId;
 
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("'");
+        result.append(resourceType);
+        result.append("/");
+        result.append(logicalId);
+        result.append("'");
+        result.append(" => ");
+        result.append(resourceTypeId);
+        result.append("/");
+        result.append(logicalResourceId);
+        return result.toString();
+    }
+
+    /**
+     * Builder for fluent creation of LogicalResourceIdentValue objects
+     */
     public static class Builder {
+        private int resourceTypeId;
         private String resourceType;
         private String logicalId;
         private Long logicalResourceId;
 
+        /**
+         * Set the resourceTypeId
+         * @param resourceTypeId
+         * @return
+         */
+        public Builder withResourceTypeId(int resourceTypeId) {
+            this.resourceTypeId = resourceTypeId;
+            return this;
+        }
+
+        /**
+         * Set the logicalResourceId
+         * @param logicalResourceId
+         * @return
+         */
         public Builder withLogicalResourceId(long logicalResourceId) {
             this.logicalResourceId = logicalResourceId;
             return this;
         }
+
+        /**
+         * Set the resourceType
+         * @param resourceType
+         * @return
+         */
         public Builder withResourceType(String resourceType) {
             this.resourceType = resourceType;
             return this;
         }
+
+        /**
+         * Set the logicalId
+         * @param logicalId
+         * @return
+         */
         public Builder withLogicalId(String logicalId) {
             this.logicalId = logicalId;
             return this;
@@ -40,7 +86,7 @@ public Builder withLogicalId(String logicalId) {
          * @return
          */
         public LogicalResourceIdentValue build() {
-            return new LogicalResourceIdentValue(resourceType, logicalId, logicalResourceId);
+            return new LogicalResourceIdentValue(resourceTypeId, resourceType, logicalId, logicalResourceId);
         }
     }
 
@@ -54,11 +100,13 @@ public static Builder builder() {
 
     /**
      * Public constructor
+     * @param resourceTypeId
      * @param resourceType
      * @param logicalId
      * @param logicalResourceId
      */
-    public LogicalResourceIdentValue(String resourceType, String logicalId, Long logicalResourceId) {
+    public LogicalResourceIdentValue(int resourceTypeId, String resourceType, String logicalId, Long logicalResourceId) {
+        this.resourceTypeId = resourceTypeId;
         this.resourceType = resourceType;
         this.logicalId = Objects.requireNonNull(logicalId);
         this.logicalResourceId = logicalResourceId;
@@ -108,11 +156,4 @@ public int compareTo(LogicalResourceIdentValue that) {
     public Integer getResourceTypeId() {
         return resourceTypeId;
     }
-
-    /**
-     * @param resourceTypeId the resourceTypeId to set
-     */
-    public void setResourceTypeId(Integer resourceTypeId) {
-        this.resourceTypeId = resourceTypeId;
-    }
 }
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
index e24fba20d99..6749e082ed2 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
@@ -319,7 +319,7 @@ protected void process(String tenantId, String requestShard, String resourceType
 
     @Override
     protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException {
-        logger.info("Processing reference parameter value:" + p.toString());
+        logger.fine(() -> "Processing reference parameter value:" + p.toString());
         ParameterNameValue parameterNameValue = getParameterNameId(p);
         LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId());
         this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv));
@@ -383,6 +383,7 @@ private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourc
         LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key);
         if (result == null) {
             result = LogicalResourceIdentValue.builder()
+                    .withResourceTypeId(identityCache.getResourceTypeId(resourceType))
                     .withResourceType(resourceType)
                     .withLogicalId(logicalId)
                     .build();
@@ -1100,9 +1101,8 @@ private void resolveLogicalResourceIdents() throws FHIRPersistenceException {
      */
     private PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException {
         StringBuilder query = new StringBuilder();
-        query.append("SELECT rt.resource_type, lri.logical_id, rt.resource_type_id, lri.logical_resource_id ");
+        query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id ");
         query.append("  FROM logical_resource_ident AS lri ");
-        query.append("  JOIN resource_types AS rt ON (lri.resource_type_id = rt.resource_type_id) ");
         query.append("  JOIN (VALUES ");
         for (int i=0; i 0) {
@@ -1110,15 +1110,17 @@ private PreparedStatement buildLogicalResourceIdentSelectStatement(List "logicalResourceIdents: " + query.toString());
         return ps;
     }
 
@@ -1144,6 +1146,9 @@ protected void addMissingLogicalResourceIdents(List m
         try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
             int count = 0;
             for (LogicalResourceIdentValue value: missing) {
+                if (value.getResourceTypeId() == null) {
+                    logger.severe("bad value: " + value);
+                }
                 ps.setInt(1, value.getResourceTypeId());
                 ps.setString(2, value.getLogicalId());
                 ps.addBatch();
@@ -1183,8 +1188,7 @@ private List fetchLogicalResourceIdentIds(List "Adding reference: parameterNameId:" + parameterNameId + " refLogicalResourceId:" + refLogicalResourceId + " refVersionId:" + refVersionId);
         if (refs == null) {
             final String tablePrefix = resourceType.toLowerCase();
             final String insertString = "INSERT INTO " + tablePrefix + "_ref_values (parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id) VALUES (?,?,?,?)";
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java
new file mode 100644
index 00000000000..ae640d61f87
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java
@@ -0,0 +1,42 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+
+/**
+ * A DTO representing a record from the resource_types table
+ */
+public class ResourceTypeValue {
+    private final String resourceType;
+    private final int resourceTypeId;
+
+    /**
+     * Canonical constructor
+     * @param resourceType
+     * @param resourceTypeId
+     */
+    public ResourceTypeValue(String resourceType, int resourceTypeId) {
+        this.resourceType = resourceType;
+        this.resourceTypeId = resourceTypeId;
+    }
+
+    
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    
+    /**
+     * @return the resourceTypeId
+     */
+    public int getResourceTypeId() {
+        return resourceTypeId;
+    }
+}

From 246897fba7dcbc04d7e182b724890538296a27eb Mon Sep 17 00:00:00 2001
From: Robin Arnold 
Date: Mon, 30 May 2022 12:02:05 +0100
Subject: [PATCH 14/40] issue #3437 tidied javadoc

Signed-off-by: Robin Arnold 
---
 .../index/ParameterValueVisitorAdapter.java   | 20 +++++
 .../persistence/index/TokenParameter.java     |  3 +-
 .../remote/index/api/IMessageHandler.java     |  5 ++
 .../com/ibm/fhir/remote/index/app/Main.java   |  4 +
 .../index/database/BaseMessageHandler.java    | 18 +++-
 .../remote/index/database/CacheLoader.java    |  6 ++
 .../index/database/LogicalResourceValue.java  | 50 ++++++++++-
 .../database/PlainPostgresMessageHandler.java | 49 ++++++++++-
 .../database/PlainPostgresParameterBatch.java | 84 +++++++++++++++++++
 .../PlainPostgresSystemParameterBatch.java    | 40 +++++++++
 10 files changed, 271 insertions(+), 8 deletions(-)

diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
index 32a45fc304b..984d03e8d49 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
@@ -16,6 +16,8 @@
 public interface ParameterValueVisitorAdapter {
 
     /**
+     * Process a string parameter
+     * 
      * @param name
      * @param valueString
      * @param compositeId
@@ -24,6 +26,8 @@ public interface ParameterValueVisitorAdapter {
     void stringValue(String name, String valueString, Integer compositeId, boolean wholeSystem);
 
     /**
+     * Process a number parameter
+     * 
      * @param name
      * @param valueNumber
      * @param valueNumberLow
@@ -33,6 +37,8 @@ public interface ParameterValueVisitorAdapter {
     void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId);
 
     /**
+     * Process a date parameter
+     * 
      * @param name
      * @param valueDateStart
      * @param valueDateEnd
@@ -42,6 +48,8 @@ public interface ParameterValueVisitorAdapter {
     void dateValue(String name, Instant valueDateStart, Instant valueDateEnd, Integer compositeId, boolean wholeSystem);
 
     /**
+     * Process a token parameter
+     * 
      * @param name
      * @param valueSystem
      * @param valueCode
@@ -50,6 +58,8 @@ public interface ParameterValueVisitorAdapter {
     void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId);
 
     /**
+     * Process a tag parameter
+     * 
      * @param name
      * @param valueSystem
      * @param valueCode
@@ -59,6 +69,8 @@ public interface ParameterValueVisitorAdapter {
     void tagValue(String name, String valueSystem, String valueCode, boolean wholeSystem);
 
     /**
+     * Process a profile parameter
+     * 
      * @param name
      * @param url
      * @param version
@@ -68,6 +80,8 @@ public interface ParameterValueVisitorAdapter {
     void profileValue(String name, String url, String version, String fragment, boolean wholeSystem);
 
     /**
+     * Process a security parameter
+     * 
      * @param name
      * @param valueSystem
      * @param valueCode
@@ -76,6 +90,8 @@ public interface ParameterValueVisitorAdapter {
     void securityValue(String name, String valueSystem, String valueCode, boolean wholeSystem);
     
     /**
+     * Process a quantity parameter
+     * 
      * @param name
      * @param valueSystem
      * @param valueCode
@@ -88,6 +104,8 @@ void quantityValue(String name, String valueSystem, String valueCode, BigDecimal
         Integer compositeId);
 
     /**
+     * Process a location parameter
+     * 
      * @param name
      * @param valueLatitude
      * @param valueLongitude
@@ -96,6 +114,8 @@ void quantityValue(String name, String valueSystem, String valueCode, BigDecimal
     void locationValue(String name, Double valueLatitude, Double valueLongitude, Integer compositeId);
 
     /**
+     * Process a reference parameter
+     * 
      * @param name
      * @param refResourceType
      * @param refLogicalId
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java
index 0d12749f5af..5e575784b71 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java
@@ -73,5 +73,4 @@ public Integer getRefVersionId() {
     public void setRefVersionId(Integer refVersionId) {
         this.refVersionId = refVersionId;
     }
-
-}
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java
index b503b5bcb75..4136b1355ee 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java
@@ -16,6 +16,11 @@
  */
 public interface IMessageHandler {
 
+    /**
+     * Ask the handler to process the list of messages.
+     * @param messages
+     * @throws FHIRPersistenceException
+     */
     void process(List messages) throws FHIRPersistenceException;
 
     /**
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java
index 2ff77dbdd4e..d2f8b9b15d1 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java
@@ -262,6 +262,7 @@ private void initIdentityCache() throws FHIRPersistenceException {
             throw new FHIRPersistenceException("cache init failed", x);
         }
     }
+
     /**
      * Create a new consumer
      * @return
@@ -282,6 +283,9 @@ private KafkaConsumer buildConsumer() {
         return consumer;
     }
 
+    /**
+     * Set things up to talk to a PostgreSQL database
+     */
     private void configureForPostgres() {
         this.translator = new PostgresTranslator();
         try {
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java
index 6a08a8e53fe..a665c6d6f04 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java
@@ -44,6 +44,7 @@ public abstract class BaseMessageHandler implements IMessageHandler {
     private SecureRandom random = new SecureRandom();
 
     private final long maxReadyWaitMs;
+
     /**
      * Protected constructor
      * @param maxReadyWaitMs the max time in ms to wait for the upstream transaction to make the data ready
@@ -283,6 +284,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     }
 
     /**
+     * Process the given LocationParameter p
+     * 
      * @param tenantId
      * @param requestShard
      * @param resourceType
@@ -293,6 +296,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given TokenParameter p
+     * 
      * @param tenantId
      * @param requestShard
      * @param resourceType
@@ -303,6 +308,7 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given TagParameter p
      * 
      * @param tenantId
      * @param requestShard
@@ -315,6 +321,7 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given ProfileParameter p
      * 
      * @param tenantId
      * @param requestShard
@@ -327,6 +334,7 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException;
 
     /**
+     * Proces the given SecurityParameter p
      * 
      * @param tenantId
      * @param requestShard
@@ -339,6 +347,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given QuantityParameter p
+     * 
      * @param tenantId
      * @param requestShard
      * @param resourceType
@@ -349,6 +359,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given NumberParameter p
+     * 
      * @param tenantId
      * @param requestShard
      * @param resourceType
@@ -359,6 +371,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given DateParameter p
+     * 
      * @param tenantId
      * @param requestShard
      * @param resourceType
@@ -369,6 +383,7 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
     protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException;
 
     /**
+     * Process the given ReferenceParameter p
      * 
      * @param tenantId
      * @param requestShard
@@ -392,7 +407,8 @@ private void process(RemoteIndexMessage message) throws FHIRPersistenceException
 
     /**
      * Tell the persistence layer to commit the current transaction, or perform a rollback
-     * if setRollbackOnly() has been called
+     * if setRollbackOnly() has been called.
+     * 
      * @throws FHIRPersistenceException
      */
     protected abstract void endTransaction() throws FHIRPersistenceException;
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
index 9e4d08a194c..7d66a6bcae8 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
@@ -30,6 +30,12 @@ public CacheLoader(IdentityCacheImpl cache) {
         this.cache = cache;
     }
 
+    /**
+     * Read records from the database using the given connection and apply the
+     * values to the configured cache object.
+     * @param connection
+     * @throws FHIRPersistenceException
+     */
     public void apply(Connection connection) throws FHIRPersistenceException {
         // load the static list of resource types
         List resourceTypes = new ArrayList<>();
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java
index 7491e2f8d79..1c6dc14cc7d 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java
@@ -19,6 +19,10 @@ public class LogicalResourceValue {
     private final int versionId;
     private final Timestamp lastUpdated;
     private final String parameterHash;
+
+    /**
+     * Builder for fluent creation of LogicalResourceValue objects
+     */
     public static class Builder {
         private short shardKey;
         private long logicalResourceId;
@@ -28,30 +32,71 @@ public static class Builder {
         private Timestamp lastUpdated;
         private String parameterHash;
 
+        /**
+         * Set the shardKey
+         * @param shardKey
+         * @return
+         */
         public Builder withShardKey(short shardKey) {
             this.shardKey = shardKey;
             return this;
         }
+
+        /**
+         * Set the logicalResourceId value
+         * @param logicalResourceId
+         * @return
+         */
         public Builder withLogicalResourceId(long logicalResourceId) {
             this.logicalResourceId = logicalResourceId;
             return this;
         }
+
+        /**
+         * Set the resourceType value
+         * @param resourceType
+         * @return
+         */
         public Builder withResourceType(String resourceType) {
             this.resourceType = resourceType;
             return this;
         }
+
+        /**
+         * Set the logicalId value
+         * @param logicalId
+         * @return
+         */
         public Builder withLogicalId(String logicalId) {
             this.logicalId = logicalId;
             return this;
         }
+
+        /**
+         * Set the versionId value
+         * @param versionId
+         * @return
+         */
         public Builder withVersionId(int versionId) {
             this.versionId = versionId;
             return this;
         }
+
+        /**
+         * Set the lastUpdated value
+         * @param lastUpdated
+         * @return
+         */
         public Builder withLastUpdated(Timestamp lastUpdated) {
             this.lastUpdated = lastUpdated;
             return this;
         }
+
+        /**
+         * Set the parameterHash value
+         * @param parameterHash
+         * @return
+         */
         public Builder withParameterHash(String parameterHash) {
             this.parameterHash = parameterHash;
             return this;
@@ -67,7 +112,7 @@ public LogicalResourceValue build() {
     }
 
     /**
-     * Factor function to create a fresh instance of a {@link Builder}
+     * Factory function to create a fresh instance of a {@link Builder}
      * @return
      */
     public static Builder builder() {
@@ -75,7 +120,8 @@ public static Builder builder() {
     }
 
     /**
-     * Public constructor
+     * Canonical constructor
+     * 
      * @param shardKey
      * @param logicalResourceId
      * @param resourceType
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
index 6749e082ed2..0f6e268f518 100644
--- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
@@ -378,6 +378,14 @@ private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenV
         return result;
     }
 
+    /**
+     * Get the LogicalReosurceIdentValue we've assigned for the given (resourceType, logicalId)
+     * tuple. The returned value may not yet have the actual logical_resource_id yet - we fetch
+     * these values later and create new database records as necessary
+     * @param resourceType
+     * @param logicalId
+     * @return
+     */
     private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) {
         LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId);
         LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key);
@@ -393,6 +401,13 @@ private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourc
         return result;
     }
 
+    /**
+     * Get the CommonCanonicalValue we've assigned for the given url value.
+     * The returned value may not yet have the actual canonical_id yet - we fetch
+     * these values later and create new database records as necessary.
+     * @param url
+     * @return
+     */
     private CommonCanonicalValue lookupCommonCanonicalValue(String url) {
         CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, url);
         CommonCanonicalValue result = this.commonCanonicalValueMap.get(key);
@@ -500,6 +515,12 @@ protected void addMissingCodeSystems(List missing) throws FHIRP
         }
     }
 
+    /**
+     * Fetch all the code_system_id values for the given list of CodeSystemValue objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
     private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException {
         // track which values aren't yet in the database
         List missing = new ArrayList<>();
@@ -624,7 +645,13 @@ private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException {
         // track which values aren't yet in the database
         List missing = new ArrayList<>();
@@ -738,6 +765,12 @@ private void resolveCommonCanonicalValues() throws FHIRPersistenceException {
         }
     }
 
+    /**
+     * Fetch the common_canonical_id values for the given list of CommonCanonicalValue objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
     private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException {
         // track which values aren't yet in the database
         List missing = new ArrayList<>();
@@ -893,6 +926,12 @@ private void resolveParameterNames() throws FHIRPersistenceException {
         }
     }
 
+    /**
+     * Fetch the parameter_name_id for the given parameterName value
+     * @param parameterName
+     * @return
+     * @throws SQLException
+     */
     private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException {
         String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?";
         try (PreparedStatement ps = connection.prepareStatement(SQL)) {
@@ -1168,6 +1207,12 @@ protected void addMissingLogicalResourceIdents(List m
         }
     }
 
+    /**
+     * Fetch logical_resource_id values for the given list of LogicalResourceIdent objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
     private List fetchLogicalResourceIdentIds(List unresolved) throws FHIRPersistenceException {
         // track which values aren't yet in the database
         List missing = new ArrayList<>();
@@ -1217,6 +1262,4 @@ private List fetchLogicalResourceIdentIds(List
Date: Wed, 1 Jun 2022 09:48:23 +0100
Subject: [PATCH 15/40] issue #3437 fixed issues using Citus

Signed-off-by: Robin Arnold 
---
 .../database/utils/api/ISchemaAdapter.java    | 19 +++--
 .../database/utils/citus/CitusAdapter.java    | 14 ++--
 .../citus/CitusDistributionCheckDAO.java      | 62 ++++++++++++++
 .../utils/common/PlainSchemaAdapter.java      |  8 +-
 .../database/utils/model/CreateIndex.java     | 20 ++++-
 .../fhir/database/utils/model/IndexDef.java   |  6 +-
 .../utils/model/PhysicalDataModel.java        | 83 ++++++++++++++++---
 .../ibm/fhir/database/utils/model/Table.java  | 50 ++++++++---
 .../jdbc/dao/impl/ResourceReferenceDAO.java   | 14 +++-
 .../java/com/ibm/fhir/schema/app/Main.java    | 42 ++++------
 .../ibm/fhir/schema/app/util/CommonUtil.java  |  5 +-
 .../build/DistributedSchemaAdapter.java       | 65 ++++++++++++++-
 .../schema/build/ShardedSchemaAdapter.java    | 26 +++---
 .../schema/control/FhirSchemaGenerator.java   |  6 ++
 .../schema/derby/DerbyFhirDatabaseTest.java   |  2 +-
 .../com/ibm/fhir/remote/index/app/Main.java   | 32 ++++++-
 .../index/database/BaseMessageHandler.java    |  8 +-
 .../PlainBatchParameterProcessor.java         |  7 ++
 .../database/PlainPostgresMessageHandler.java | 25 ++++++
 19 files changed, 400 insertions(+), 94 deletions(-)
 create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusDistributionCheckDAO.java

diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java
index ef537a5f015..f6633952b37 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java
@@ -69,19 +69,21 @@ public interface ISchemaAdapter {
      * @param tablespaceName
      * @param withs
      * @param checkConstraints
-     * @param distributionRules
+     * @param distributionType
+     * @param distributionColumnName
      */
     public void createTable(String schemaName, String name, String tenantColumnName, List columns,
             PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints,
-            DistributionType distributionRules);
+            DistributionType distributionType, String distributionColumnName);
 
     /**
      * Apply any distribution rules configured for the named table
      * @param schemaName
      * @param tableName
-     * @param distributionRules
+     * @param distributionType
+     * @param distributionColumnName
      */
-    public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionRules);
+    public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName);
 
     /**
      * Add a new column to an existing table
@@ -157,10 +159,12 @@ public void createTable(String schemaName, String name, String tenantColumnName,
      * @param tenantColumnName
      * @param indexColumns
      * @param includeColumns
-     * @param distributionRules
+     * @param distributionType
+     * @param distributionColumnName
      */
     public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName,
-            List indexColumns, List includeColumns, DistributionType distributionRules);
+            List indexColumns, List includeColumns, 
+            DistributionType distributionType, String distributionColumnName);
 
     /**
      * Create a unique index
@@ -170,9 +174,10 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN
      * @param tenantColumnName
      * @param indexColumns
      * @param distributionRules
+     * @param distributionColumnName
      */
     public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName,
-            List indexColumns, DistributionType distributionRules);
+            List indexColumns, DistributionType distributionType, String distributionColumnName);
 
     /**
      * Create an index on the named schema.table object
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java
index e23fbaa4277..bda0eac3005 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java
@@ -91,7 +91,7 @@ private String buildCitusCreateTableStatement(String schema, String name, List {
+    private static final Logger logger = Logger.getLogger(CitusDistributionCheckDAO.class.getName());
+
+    private final String schemaName;
+    private final String tableName;
+
+    /**
+     * Public constructor
+     * 
+     * @param schemaName
+     * @param tableName
+     */
+    public CitusDistributionCheckDAO(String schemaName, String tableName) {
+        DataDefinitionUtil.assertValidName(schemaName);
+        DataDefinitionUtil.assertValidName(tableName);
+        this.schemaName = schemaName.toLowerCase();
+        this.tableName = tableName.toLowerCase();
+    }
+
+    @Override
+    public Boolean run(IDatabaseTranslator translator, Connection c) {
+        Boolean result = Boolean.FALSE;
+
+        final String relname = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName);
+        final String SQL = "SELECT 1 FROM pg_dist_partition WHERE logicalrelid = ?::regclass";
+
+        try (PreparedStatement ps = c.prepareStatement(SQL)) {
+            ps.setString(1, relname);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                result = Boolean.TRUE;
+            }
+        } catch (SQLException x) {
+            // Translate the exception into something a little more meaningful
+            // for this database type and application
+            logger.severe("select failed: " + SQL + " for logicalrelid = '" + relname + "'");
+            throw translator.translate(x);
+        }
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java
index 8bfdb632f12..b715be2e8c0 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java
@@ -66,12 +66,12 @@ public void detachPartition(String schemaName, String tableName, String partitio
 
     @Override
     public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity,
-        String tablespaceName, List withs, List checkConstraints, DistributionType distributionRules) {
+        String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) {
         databaseAdapter.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, null);
     }
 
     @Override
-    public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionRules) {
+    public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName) {
         databaseAdapter.applyDistributionRules(schemaName, tableName, null);
     }
 
@@ -112,13 +112,13 @@ public void dropProcedure(String schemaName, String procedureName) {
 
     @Override
     public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns,
-        List includeColumns, DistributionType distributionRules) {
+        List includeColumns, DistributionType distributionType, String distributionColumnName) {
         databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, null);
     }
 
     @Override
     public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns,
-        DistributionType distributionRules) {
+        DistributionType distributionType, String distributionColumnName) {
         databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, null);
     }
 
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java
index 98649f37d44..6140d6cd043 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java
@@ -34,6 +34,7 @@ public class CreateIndex extends BaseObject {
 
     // Distribution rules if the associated table is distributed
     private final DistributionType distributionType;
+    private final String distributionColumnName;
 
     /**
      * Protected constructor. Use the Builder to create instance.
@@ -43,12 +44,14 @@ public class CreateIndex extends BaseObject {
      * @param distributionType
      */
     protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName,
-            DistributionType distributionType) {
+            DistributionType distributionType, String distributionColumnName) {
         super(schemaName, versionTrackingName, DatabaseObjectType.INDEX, version);
         this.tableName = tableName;
         this.indexDef = indexDef;
         this.tenantColumnName = tenantColumnName;
         this.distributionType = distributionType;
+        this.distributionColumnName = distributionColumnName;
+        
     }
     
     /**
@@ -95,7 +98,7 @@ public String getTypeNameVersion() {
     @Override
     public void apply(ISchemaAdapter target, SchemaApplyContext context) {
         long start = System.nanoTime();
-        indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionType);
+        indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionType, distributionColumnName);
         
         if (logger.isLoggable(Level.FINE)) {
             long end = System.nanoTime();
@@ -162,6 +165,7 @@ public static class Builder {
 
         // Set if the table is distributed
         private DistributionType distributionType = DistributionType.NONE;
+        private String distributionColumnName;
 
         /**
          * @param schemaName the schemaName to set
@@ -204,6 +208,16 @@ public Builder setDistributionType(DistributionType dt) {
             return this;
         }
 
+        /**
+         * Setter for distributionColumnName
+         * @param distributionColumnName
+         * @return
+         */
+        public Builder setDistributionColumnName(String distributionColumnName) {
+            this.distributionColumnName = distributionColumnName;
+            return this;
+        }
+
         /**
          * @param version the version to set
          */
@@ -258,7 +272,7 @@ public CreateIndex build() {
             }
             
             return new CreateIndex(schemaName, versionTrackingName, tableName, version,
-                new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionType);
+                new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionType, distributionColumnName);
         }
         
         /**
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
index dd6f478fc8c..9867b98fcef 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
@@ -72,12 +72,12 @@ public boolean isUnique() {
      * @param distributionRules
      */
     public void apply(String schemaName, String tableName, String tenantColumnName, ISchemaAdapter target,
-            DistributionType distributionType) {
+            DistributionType distributionType, String distributionColumn) {
         if (includeColumns != null && includeColumns.size() > 0) {
-            target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionType);
+            target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionType, distributionColumn);
         }
         else if (unique) {
-            target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType);
+            target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType, distributionColumn);
         }
         else {
             target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType);
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java
index cec5dc379ab..dba497299fc 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java
@@ -141,19 +141,52 @@ public void apply(ISchemaAdapter target, SchemaApplyContext context) {
 
     /**
      * Make a pass over all the objects and apply any distribution rules they
-     * may have (e.g. for Citus)
+     * may have (e.g. for Citus). We have to process a large number of tables,
+     * which can cause shared memory issues for Citus if we try and do this in
+     * a single transaction, hence the need for a transactionSupplier
      * @param target
      */
-    public void applyDistributionRules(ISchemaAdapter target) {
+    public void applyDistributionRules(ISchemaAdapter target, Supplier transactionSupplier) {
 
+        // takes a long time, so track progress
+        int total = allObjects.size() * 2;
+        int count = 0;
+        int objectsPerMessage = total / 100; // 1% increments
+        int nextCount = objectsPerMessage;
         // make a first pass to apply reference rules
         for (IDatabaseObject obj: allObjects) {
-            obj.applyDistributionRules(target, 0);
+            try (ITransaction tx = transactionSupplier.get()) {
+                try {
+                    obj.applyDistributionRules(target, 0);
+                    
+                    if (++count >= nextCount) {
+                        int pc = 100 * nextCount / total;
+                        logger.info("Progress: [" + pc + "% complete]");
+                        nextCount += objectsPerMessage;
+                    }
+                } catch (RuntimeException x) {
+                    tx.setRollbackOnly();
+                    throw x;
+                }                    
+            }
         }
         
         // and another pass to apply sharding rules
         for (IDatabaseObject obj: allObjects) {
-            obj.applyDistributionRules(target, 1);
+            try (ITransaction tx = transactionSupplier.get()) {
+                try {
+                    obj.applyDistributionRules(target, 1);
+
+                    if (++count >= nextCount) {
+                        int pc = 100 * nextCount / total;
+                        logger.info("Progress: [" + pc + "% complete]");
+                        nextCount += objectsPerMessage;
+                    }
+                } catch (RuntimeException x) {
+                    tx.setRollbackOnly();
+                    throw x;
+                }                    
+            }
         }
     }
 
@@ -300,12 +333,42 @@ public void dropForeignKeyConstraints(ISchemaAdapter target, String tagGroup, St
      * @param v
      * @param tagGroup
      * @param tag
-     */
-    public void visit(DataModelVisitor v, final String tagGroup, final String tag) {
-        // visit just the matching subset of objects
-        this.allObjects.stream()
-            .filter(obj -> tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup)))
-            .forEach(obj -> obj.visit(v));
+     * @param transactionSupplier
+     */
+    public void visit(DataModelVisitor v, final String tagGroup, final String tag, Supplier transactionSupplier) {
+        // visit just the matching subset of objects. If a transactionSupplier has been provided, we break up the
+        // operation into multiple transactions to avoid transaction size limitations (e.g. with Citus FK creation)
+        if (transactionSupplier != null) {
+            ITransaction tx = transactionSupplier.get();
+            try {
+                int count = 0;
+                for (IDatabaseObject obj: allObjects) {
+                    if (tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) {
+                        if (++count == 10) {
+                            // commit the current transaction and start a fresh one
+                            tx.close();
+                            tx = transactionSupplier.get();
+                            count = 0;
+                        }
+
+                        try {
+                            obj.visit(v);
+                        } catch (RuntimeException x) {
+                            tx.setRollbackOnly();
+                            throw x;
+                        }
+                    }
+                    
+                }
+            } finally {
+                tx.close();
+            }
+        } else {
+            // the old way, which will visit everything in the scope of one transaction
+            this.allObjects.stream()
+                .filter(obj -> tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup)))
+                .forEach(obj -> obj.visit(v));
+        }
     }
 
     /**
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java
index 48b609eb965..bdbbc5e5dc7 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java
@@ -52,7 +52,7 @@ public class Table extends BaseObject {
     // The rules to distribute the table in a distributed RDBMS implementation (Citus)
     private final DistributionType distributionType;
 
-    // If set, overrides the column used to distribute the data in a sharded database
+    // If set, overrides the column used to distribute the data in a distributed database
     private final String distributionColumnName;
 
     // The With parameters on the table
@@ -60,6 +60,9 @@ public class Table extends BaseObject {
     
     private final List checkConstraints = new ArrayList<>();
 
+    // Do we still want to create this table?
+    private final boolean create;
+
     /**
      * Public constructor
      *
@@ -82,13 +85,14 @@ public class Table extends BaseObject {
      * @param checkConstraints
      * @param distributionType
      * @param distributionColumnName
+     * @param create
      */
     public Table(String schemaName, String name, int version, String tenantColumnName, 
             Collection columns, PrimaryKeyDef pk,
             IdentityDef identity, Collection indexes, Collection fkConstraints,
             SessionVariableDef accessControlVar, Tablespace tablespace, List dependencies, Map tags,
             Collection privileges, List migrations, List withs, List checkConstraints,
-            DistributionType distributionType, String distributionColumnName) {
+            DistributionType distributionType, String distributionColumnName, boolean create) {
         super(schemaName, name, DatabaseObjectType.TABLE, version, migrations);
         this.tenantColumnName = tenantColumnName;
         this.columns.addAll(columns);
@@ -102,6 +106,7 @@ public Table(String schemaName, String name, int version, String tenantColumnNam
         this.checkConstraints.addAll(checkConstraints);
         this.distributionType = distributionType;
         this.distributionColumnName = distributionColumnName;
+        this.create = create;
 
         // Adds all dependencies which aren't null.
         // The only circumstances where it is null is when it is self referencial (an FK on itself).
@@ -145,14 +150,19 @@ public DistributionType getDistributionType() {
 
     @Override
     public void apply(ISchemaAdapter target, SchemaApplyContext context) {
+        if (!create) {
+            // Skip creation for tables we no longer want
+            return;
+        }
+
         final String tsName = this.tablespace == null ? null : this.tablespace.getName();
         target.createTable(getSchemaName(), getObjectName(), this.tenantColumnName, this.columns, 
             this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints,
-            this.distributionType);
+            this.distributionType, this.distributionColumnName);
 
         // Now add any indexes associated with this table
         for (IndexDef idx: this.indexes) {
-            idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType);
+            idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType, this.distributionColumnName);
         }
 
         if (context.isIncludeForeignKeys()) {
@@ -272,6 +282,9 @@ public static class Builder extends VersionedSchemaObject {
         // Allows the standard distribution column to be overridden
         private String distributionColumnName;
 
+        // Do we still want to create this table
+        private boolean create = true;
+
         /**
          * Private constructor to force creation through factory method
          * @param schemaName
@@ -301,6 +314,16 @@ public Builder setTablespace(Tablespace ts) {
             return this;
         }
 
+        /**
+         * Setter for the create flag
+         * @param ts
+         * @return
+         */
+        public Builder setCreate(boolean flag) {
+            this.create = flag;
+            return this;
+        }
+
         /**
          * Setter for the distributionType
          * @param cn
@@ -826,7 +849,7 @@ public Table build(IDataModel dataModel) {
             // through the constructor
             return new Table(getSchemaName(), getObjectName(), this.version, this.tenantColumnName, buildColumns(), this.primaryKey, this.identity, this.indexes.values(),
                     enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionType,
-                    distributionColumnName);
+                    distributionColumnName, create);
         }
 
         /**
@@ -968,8 +991,10 @@ public boolean exists(ISchemaAdapter target) {
 
     @Override
     public void visit(DataModelVisitor v) {
-        v.visited(this);
-        this.fkConstraints.forEach(fk -> v.visited(this, fk));
+        if (this.create) {
+            v.visited(this);
+            this.fkConstraints.forEach(fk -> v.visited(this, fk));
+        }
     }
 
     @Override
@@ -981,12 +1006,17 @@ public void visitReverse(DataModelVisitor v) {
 
     @Override
     public void applyDistributionRules(ISchemaAdapter target, int pass) {
+        if (!this.create) {
+            // skip if we no longer create this table
+            return;
+        }
+
         // make sure all the reference tables are distributed first before
-        // we attempt to shard anything
+        // we attempt to distribute any of the sharded (DISTRIBUTED) tables
         if (pass == 0 && this.distributionType == DistributionType.REFERENCE) {
-            target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType);
+            target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType, null);
         } else if (pass == 1 && this.distributionType == DistributionType.DISTRIBUTED) {
-            target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType);
+            target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType, this.distributionColumnName);
         }
     }
 }
\ No newline at end of file
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java
index 717d52e4258..44a60ac2936 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java
@@ -925,7 +925,11 @@ public void upsertCommonTokenValues(List values) {
     @Override
     public void persist(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException {
 
-        collectAndResolveParameterNames(records, referenceRecords, profileRecs, tagRecs, securityRecs);
+        boolean gotSomething = collectAndResolveParameterNames(records, referenceRecords, profileRecs, tagRecs, securityRecs);
+        if (!gotSomething) {
+            // nothing to do
+            return;
+        }
 
         // Grab the ids for all the code-systems, and upsert any misses
         List systemMisses = new ArrayList<>();
@@ -1021,8 +1025,9 @@ public void persist(Collection records, Collection records, Collection referenceRecords, Collection profileRecs,
+    private boolean collectAndResolveParameterNames(Collection records, Collection referenceRecords, Collection profileRecs,
         Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException {
 
         List recList = new ArrayList<>();
@@ -1047,6 +1052,7 @@ private void collectAndResolveParameterNames(Collection r
         for (ResourceRefRec rec: recList) {
             rec.setParameterNameId(getParameterNameId(rec.getParameterName()));
         }
+        return recList.size() > 0;
     }
 
     @Override
@@ -1097,6 +1103,10 @@ protected int getParameterNameId(String parameterName) throws FHIRPersistenceDBC
     protected abstract int readOrAddParameterNameId(String parameterName) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException;
 
     protected void upsertLogicalResourceIdents(List unresolved) throws FHIRPersistenceException {
+        if (unresolved.isEmpty()) {
+            return;
+        }
+
         // Build a unique set of logical_resource_ident keys
         Set keys = unresolved.stream().map(v -> new LogicalResourceIdentValue(v.getRefResourceTypeId(), v.getRefLogicalId())).collect(Collectors.toSet());
         List missing = new ArrayList<>(keys);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java
index ba53fc53d6e..4e58d935ea3 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java
@@ -352,9 +352,9 @@ protected void configureConnectionPool() {
         this.connectionPool = new PoolConnectionProvider(cp, this.maxConnectionPoolSize);
         this.transactionProvider = new SimpleTransactionProvider(this.connectionPool);
 
-//        if (this.dbType == DbType.CITUS) {
-//            connectionPool.setNewConnectionHandler(Main::configureCitusConnection);
-//        }
+        if (this.dbType == DbType.CITUS) {
+            connectionPool.setNewConnectionHandler(Main::configureCitusConnection);
+        }
     }
 
     /**
@@ -532,7 +532,8 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) {
             gen.buildDatabaseSpecificArtifactsPostgres(pdm);
             break;
         case CITUS:
-            gen.buildDatabaseSpecificArtifactsCitus(pdm);
+            gen.buildDatabaseSpecificArtifactsPostgres(pdm);
+            // gen.buildDatabaseSpecificArtifactsCitus(pdm);
             break;
         default:
             throw new IllegalStateException("Unsupported db type: " + dbType);
@@ -765,32 +766,18 @@ protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) {
      */
     private void applyDistributionRules(PhysicalDataModel pdm, SchemaType schemaType) {
         if (dbType == DbType.CITUS) {
-            try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) {
-                try {
-                    ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool);
-                    pdm.applyDistributionRules(schemaAdapter);
-                } catch (RuntimeException x) {
-                    tx.setRollbackOnly();
-                    throw x;
-                }
-            }
+            ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool);
+            pdm.applyDistributionRules(schemaAdapter, () -> TransactionFactory.openTransaction(connectionPool));
         }
 
-        final boolean includeForeignKeys = schemaType != SchemaType.DISTRIBUTED;
-        if (!includeForeignKeys) {
+        final boolean includedForeignKeys = schemaType != SchemaType.DISTRIBUTED;
+        if (!includedForeignKeys) {
             // Now that all the tables have been distributed, it should be safe
             // to apply the FK constraints
-            try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) {
-                try {
-                    final String tenantColumnName = isMultitenant() ? "mt_id" : null;
-                    ISchemaAdapter adapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool);
-                    AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName);
-                    pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP);
-                } catch (RuntimeException x) {
-                    tx.setRollbackOnly();
-                    throw x;
-                }
-            }
+            final String tenantColumnName = isMultitenant() ? "mt_id" : null;
+            ISchemaAdapter adapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool);
+            AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName);
+            pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP, () -> TransactionFactory.openTransaction(connectionPool));
         }        
     }
 
@@ -963,7 +950,7 @@ private void dropForeignKeyConstraints(PhysicalDataModel pdm, String tagGroup, S
 
                 Set
referencedTables = new HashSet<>(); DropForeignKey dropper = new DropForeignKey(adapter, referencedTables); - pdm.visit(dropper, tagGroup, tag); + pdm.visit(dropper, tagGroup, tag, null); } catch (Exception x) { c.rollback(); throw x; @@ -2234,6 +2221,7 @@ protected void parseArgs(String[] args) { break; case CITUS: translator = new CitusTranslator(); + dataSchemaType = SchemaType.DISTRIBUTED; // by default break; case DB2: dataSchemaType = SchemaType.MULTITENANT; diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java index 1da1409fae0..73eb2332f7e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java @@ -43,6 +43,7 @@ public final class CommonUtil { // Random generator for new tenant keys and salts private static final SecureRandom random = new SecureRandom(); + private static final String DEFAULT_DISTRIBUTION_COLUMN = "LOGICAL_RESOURCE_ID"; /** * Set up the logger using the log.dir system property @@ -149,7 +150,7 @@ public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, DbType dbTy case MULTITENANT: return new FhirSchemaAdapter(dbAdapter); case DISTRIBUTED: - return new DistributedSchemaAdapter(dbAdapter); + return new DistributedSchemaAdapter(dbAdapter, DEFAULT_DISTRIBUTION_COLUMN); case SHARDED: return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); default: @@ -171,7 +172,7 @@ public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, IDatabaseAd case MULTITENANT: return new FhirSchemaAdapter(dbAdapter); case DISTRIBUTED: - return new DistributedSchemaAdapter(dbAdapter); + return new DistributedSchemaAdapter(dbAdapter, DEFAULT_DISTRIBUTION_COLUMN); case SHARDED: return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); default: diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java index c59ca84acd1..a56e0b41908 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java @@ -6,21 +6,82 @@ package com.ibm.fhir.schema.build; +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.With; /** * Represents an adapter used to build the FHIR schema when - * used with a distributed like Citus + * used with a distributed databse like Citus */ public class DistributedSchemaAdapter extends PlainSchemaAdapter { + // The distribution column to use by default for DISTRIBUTED tables + private final String defaultDistributionColumnName; /** * Public constructor * * @param databaseAdapter */ - public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter) { + public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter, String defaultDistributionColumnName) { super(databaseAdapter); + this.defaultDistributionColumnName = defaultDistributionColumnName; + } + + /** + * Create a DistributionContext using the given type and column. If the distribution column + * is null, then the default distribution column is used. + * @param distributionType + * @param distributionColumnName + * @return + */ + private DistributionContext createContext(DistributionType distributionType, String distributionColumnName) { + if (distributionType == DistributionType.DISTRIBUTED && distributionColumnName == null) { + distributionColumnName = defaultDistributionColumnName; + } + return new DistributionContext(distributionType, distributionColumnName); + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { + + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType, String distributionColumnName) { + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType, String distributionColumnName) { + + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, dc); + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName) { + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.applyDistributionRules(schemaName, tableName, dc); + } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + // NOP for now } } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java index cbfedd1b2f6..e1dcf6eeca1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java @@ -33,24 +33,24 @@ public class ShardedSchemaAdapter extends FhirSchemaAdapter { // The distribution column to add to each table marked as distributed - final String distributionColumnName; + final String shardColumnName; /** * @param databaseAdapter */ - public ShardedSchemaAdapter(IDatabaseAdapter databaseAdapter, String distributionColumnName) { + public ShardedSchemaAdapter(IDatabaseAdapter databaseAdapter, String shardColumnName) { super(databaseAdapter); - this.distributionColumnName = distributionColumnName; + this.shardColumnName = shardColumnName; } @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, - String tablespaceName, List withs, List checkConstraints, DistributionType distributionType) { + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { // If the table is distributed, we need to inject the distribution column into the columns list. This same // column will need to be injected into each of the index definitions List actualColumns = new ArrayList<>(); if (distributionType == DistributionType.DISTRIBUTED) { - ColumnBase distributionColumn = new SmallIntColumn(distributionColumnName, false, null); + ColumnBase distributionColumn = new SmallIntColumn(shardColumnName, false, null); actualColumns.add(distributionColumn); if (primaryKey != null) { // we need to alter the primary so it includes the distribution column @@ -62,37 +62,37 @@ public void createTable(String schemaName, String name, String tenantColumnName, } actualColumns.addAll(columns); - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); databaseAdapter.createTable(schemaName, name, tenantColumnName, actualColumns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); } @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns, DistributionType distributionType) { + List includeColumns, DistributionType distributionType, String distributionColumnName) { List actualColumns = new ArrayList<>(indexColumns); if (distributionType == DistributionType.DISTRIBUTED) { // inject the distribution column into the index definition - actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + actualColumns.add(new OrderedColumnDef(this.shardColumnName, null, null)); } // Create the index using the modified set of index columns - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, includeColumns, dc); } @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - DistributionType distributionType) { + DistributionType distributionType, String distributionColumnName) { List actualColumns = new ArrayList<>(indexColumns); if (distributionType == DistributionType.DISTRIBUTED) { // inject the distribution column into the index definition - actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); + actualColumns.add(new OrderedColumnDef(this.shardColumnName, null, null)); } // Create the index using the modified set of index columns - DistributionContext dc = new DistributionContext(distributionType, distributionColumnName); + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, dc); } @@ -118,7 +118,7 @@ public void createForeignKeyConstraint(String constraintName, String schemaName, // can be based on the original PK definition without the extra sharding column. List newCols = new ArrayList<>(columns); if (distributionType == DistributionType.DISTRIBUTED && !targetIsReference) { - newCols.add(distributionColumnName); + newCols.add(shardColumnName); } databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, newCols, enforced); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 1baf4283e40..e0eb5f351da 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -1044,6 +1044,7 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { // because it makes it very easy to find the most recent changes to resources associated with // a given patient (for example). Table tbl = Table.builder(schemaName, tableName) + .setCreate(false) // V0027 no longer used .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding @@ -1067,6 +1068,11 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) { if (priorVersion < FhirSchemaVersion.V0020.vid()) { statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); } + if (priorVersion < FhirSchemaVersion.V0027.vid()) { + // This table is never used and the FK_LOGICAL_RESOURCE_COMPARTMENTS_COMP FK + // causes issues with Citus distribution, so it's time for it to go + statements.add(new DropTable(schemaName, tableName)); + } return statements; }) .build(pdm); diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java index 56d6e6a439c..99288fbc7e7 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java @@ -142,7 +142,7 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S // Check that we have the correct number of tables. This will need to be updated // whenever tables, views or sequences are added or removed - assertEquals(adapter.listSchemaObjects(schemaName).size(), 2357); + assertEquals(adapter.listSchemaObjects(schemaName).size(), 2356); c.commit(); } catch (Throwable t) { c.rollback(); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index d2f8b9b15d1..6225ba5dedf 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -30,7 +30,10 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.database.utils.citus.CitusTranslator; +import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.database.utils.thread.ThreadHandler; @@ -84,6 +87,7 @@ public class Main { private SchemaType schemaType = SchemaType.PLAIN; private IDatabaseTranslator translator; private IConnectionProvider connectionProvider; + private DbType dbType = DbType.POSTGRESQL; /** * Parse the given command line arguments @@ -108,6 +112,13 @@ public void parseArgs(String[] args) { throw new IllegalArgumentException("Missing value for --database-properties"); } break; + case "--db-type": + if (a < args.length && !args[a].startsWith("--")) { + this.dbType = DbType.from(args[a++]); + } else { + throw new IllegalArgumentException("Missing value for --db-type"); + } + break; case "--topic-name": if (a < args.length && !args[a].startsWith("--")) { topicName = args[a++]; @@ -309,13 +320,18 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { try { // Each handler gets a dedicated database connection so we don't have // to deal with contention when grabbing connections from a pool + Connection c = connectionProvider.getConnection(); + if (this.dbType == DbType.CITUS) { + configureCitusConnection(c); + } + switch (schemaType) { case SHARDED: - return new ShardedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + return new ShardedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); case PLAIN: - return new PlainPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + return new PlainPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); case DISTRIBUTED: - return new DistributedPostgresMessageHandler(connectionProvider.getConnection(), getSchemaName(), identityCache, maxReadyTimeMs); + return new DistributedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); default: throw new FHIRPersistenceException("Schema type not supported: " + schemaType.name()); } @@ -324,6 +340,16 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { } } + /** + * Configure the connection by setting local properties required for Citus + * @param c + */ + private static void configureCitusConnection(Connection c) { + logger.info("Citus: Configuring new database connection"); + ConfigureConnectionDAO dao = new ConfigureConnectionDAO(); + dao.run(new CitusTranslator(), c); + } + /** * Get the partitions for the named topic to check if the topic actually exists */ diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index a665c6d6f04..4a7f54bc09e 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -105,6 +105,10 @@ private void processWithRetry(List messages) throws FHIRPers } else { throw x; } + } catch (Throwable t) { + setRollbackOnly(); + logger.log(Level.SEVERE, "batch failed", t); + throw t; } finally { endTransaction(); } @@ -157,8 +161,8 @@ private RemoteIndexMessage unmarshall(String jsonPayload) { */ private void processMessages(List messages) throws FHIRPersistenceException { // We need to do a quick scan of all the messages to make sure that - // the logical resource records for each already exist. If prepare - // returns false, it means one of two things: + // the logical resource records for each already exist. If the check + // returns anything in the notReady list, it means one of two things: // 1. we received the message before the server transaction committed // 2. the server transaction failed/rolled back, so we'll never be ready long timeoutTime = System.nanoTime() + this.maxReadyWaitMs * 1000000; diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java index da4751977d2..06f683ec2cd 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java @@ -123,6 +123,13 @@ public void pushBatch() throws FHIRPersistenceException { } } + /** + * Get the DAO used to batch parameter inserts for the given resourceType. + * This method also tracks the unique set of resource types seen for a + * collection of messages. + * @param resourceType + * @return + */ private PlainPostgresParameterBatch getParameterBatchDao(String resourceType) { resourceTypesInBatch.add(resourceType); PlainPostgresParameterBatch dao = daoMap.get(resourceType); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java index 0f6e268f518..1e13867e30b 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java @@ -58,6 +58,7 @@ * the plain (non-sharded) schema variant. */ public class PlainPostgresMessageHandler extends BaseMessageHandler { + private static final String CLASSNAME = PlainPostgresMessageHandler.class.getName(); private static final Logger logger = Logger.getLogger(PlainPostgresMessageHandler.class.getName()); private static final short FIXED_SHARD = 0; @@ -206,6 +207,7 @@ protected void endTransaction() throws FHIRPersistenceException { private void publishCachedValues() { // all the unresolvedParameterNames should be resolved at this point for (ParameterNameValue pnv: this.unresolvedParameterNames) { + logger.fine(() -> "Adding parameter-name to cache: '" + pnv.getParameterName() + "' -> " + pnv.getParameterNameId()); identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); } } @@ -217,6 +219,7 @@ protected void pushBatch() throws FHIRPersistenceException { // the last step before the current transaction is committed, // Process the token values so that we can establish // any entries we need for common_token_values + logger.fine("pushBatch: resolving all ids"); resolveLogicalResourceIdents(); resolveParameterNames(); resolveCodeSystems(); @@ -226,10 +229,14 @@ protected void pushBatch() throws FHIRPersistenceException { // Now that all the lookup values should've been resolved, we can go ahead // and push the parameters to the JDBC batch insert statements via the // batchProcessor + logger.fine("pushBatch: processing collected parameters"); for (BatchParameterValue v: this.batchedParameterValues) { v.apply(batchProcessor); } + + logger.fine("pushBatch: executing final batch statements"); batchProcessor.pushBatch(); + logger.fine("pushBatch completed"); } /** @@ -909,20 +916,34 @@ private void resolveParameterNames() throws FHIRPersistenceException { // values we still need to resolve. The most important point here is // to do this in a sorted order to avoid deadlock issues because this // could be happening across multiple consumer threads at the same time. + logger.fine("resolveParameterNames: sorting unresolved names"); Collections.sort(this.unresolvedParameterNames, (a,b) -> { return a.getParameterName().compareTo(b.getParameterName()); }); try { for (ParameterNameValue pnv: this.unresolvedParameterNames) { + logger.finer(() -> "fetching parameter_name_id for '" + pnv.getParameterName() + "'"); Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); if (parameterNameId == null) { parameterNameId = createParameterName(pnv.getParameterName()); + if (logger.isLoggable(Level.FINER)) { + logger.finer("assigned parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); + } + + if (parameterNameId == null) { + // be defensive + throw new FHIRPersistenceException("parameter_name_id not assigned for '" + pnv.getParameterName()); + } + } else if (logger.isLoggable(Level.FINER)) { + logger.finer("read parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); } pnv.setParameterNameId(parameterNameId); } } catch (SQLException x) { throw new FHIRPersistenceException("error resolving parameter names", x); + } finally { + logger.exiting(CLASSNAME, "resolveParameterNames"); } } @@ -1114,21 +1135,25 @@ protected void checkReady(List messages, List missing = fetchLogicalResourceIdentIds(unresolvedLogicalResourceIdents); if (!missing.isEmpty()) { + logger.fine("resolveLogicalResourceIdents: add missing LogicalResourceIdent records"); addMissingLogicalResourceIdents(missing); } // All the previously missing values should now be in the database. We need to fetch them again, // possibly having to use multiple queries + logger.fine("resolveLogicalResourceIdents: fetch ids for missing LogicalResourceIdent records"); List bad = fetchLogicalResourceIdentIds(missing); if (!bad.isEmpty()) { // shouldn't happen, but let's protected against it anyway throw new FHIRPersistenceException("Failed to create all logical_resource_ident values"); } + logger.fine("resolveLogicalResourceIdents: all resolved"); } /** From 3e6699ddf6a047f8230743e7a04565e5d64cc2fd Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 1 Jun 2022 14:02:10 +0100 Subject: [PATCH 16/40] issue #3437 use identity cache for more ids Signed-off-by: Robin Arnold --- .../utils/citus/ConfigureConnectionDAO.java | 4 +- .../fhir/remote/index/api/IdentityCache.java | 40 +++++++++++++++ .../com/ibm/fhir/remote/index/app/Main.java | 9 ++-- .../remote/index/cache/IdentityCacheImpl.java | 34 ++++++++++++- .../database/PlainPostgresMessageHandler.java | 51 +++++++++++++------ 5 files changed, 118 insertions(+), 20 deletions(-) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java index c578a9f7c06..039c07682b0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java @@ -35,7 +35,9 @@ public ConfigureConnectionDAO() { @Override public void run(IDatabaseTranslator translator, Connection c) { - final String SQL = "SET LOCAL citus.multi_shard_modify_mode TO 'sequential'"; + // we need this behavior for all transactions on this connection, so + // we use SET SESSION instead of SET LOCAL + final String SQL = "SET SESSION citus.multi_shard_modify_mode TO 'sequential'"; try (Statement s = c.createStatement()) { s.executeUpdate(SQL); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java index 1f731792db3..786a3e952d5 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java @@ -49,6 +49,30 @@ public interface IdentityCache { */ Long getCommonCanonicalValueId(short shardKey, String url); + /** + * Add the common canonical value to the cache + * @param shardKey + * @param url + * @param commonCanonicalValueId + */ + void addCommonCanonicalValue(short shardKey, String url, long commonCanonicalValueId); + + /** + * Add the common token value to the cache + * @param shardKey + * @param codeSystem + * @param tokenValue + * @param commonTokenValueId + */ + void addCommonTokenValue(short shardKey, String codeSystem, String tokenValue, long commonTokenValueId); + + /** + * Add the code system value to the cache + * @param codeSystem + * @param codeSystemId + */ + void addCodeSystem(String codeSystem, int codeSystemId); + /** * Get the database resource_type_id value for the given resourceType value * @param resourceType @@ -56,4 +80,20 @@ public interface IdentityCache { * @throws IllegalArgumentException if resourceType is not a valid resource type name */ int getResourceTypeId(String resourceType); + + /** + * Get the database logical_resource_id for the given resourceType/logicalId tuple. + * @param resourceType + * @param logicalId + * @return + */ + Long getLogicalResourceIdentId(String resourceType, String logicalId); + + /** + * Add the logical_resource_ident mapping to the cache + * @param resourceType + * @param logicalId + * @param logicalResourceId + */ + void addLogicalResourceIdent(String resourceType, String logicalId, long logicalResourceId); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 6225ba5dedf..6f66f037a2f 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -262,10 +262,13 @@ public void run() throws FHIRPersistenceException { private void initIdentityCache() throws FHIRPersistenceException { logger.info("Initializing identity cache"); identityCache = new IdentityCacheImpl( - 1000, Duration.ofSeconds(3600), - 10000, Duration.ofSeconds(3600), - 1000, Duration.ofSeconds(3600)); + 1000, Duration.ofSeconds(86400), // code systems + 10000, Duration.ofSeconds(86400), // common token values + 1000, Duration.ofSeconds(86400), // common canonical values + 100000, Duration.ofSeconds(86400)); // logical resource idents CacheLoader loader = new CacheLoader(identityCache); + + // prefill the cache try (Connection connection = connectionProvider.getConnection()) { loader.apply(connection); connection.commit(); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java index a0a43164daf..9b73330e0f8 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java @@ -15,6 +15,7 @@ import com.ibm.fhir.remote.index.api.IdentityCache; import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey; import com.ibm.fhir.remote.index.database.CommonTokenValueKey; +import com.ibm.fhir.remote.index.database.LogicalResourceIdentKey; import com.ibm.fhir.remote.index.database.ResourceTypeValue; /** @@ -27,6 +28,7 @@ public class IdentityCacheImpl implements IdentityCache { private final Cache codeSystemCache; private final Cache commonTokenValueCache; private final Cache commonCanonicalValueCache; + private final Cache logicalResourceIdentCache; private static final Integer NULL_INT = null; private static final Long NULL_LONG = null; @@ -35,7 +37,8 @@ public class IdentityCacheImpl implements IdentityCache { */ public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDuration, long maxCommonTokenCacheSize, Duration commonTokenCacheDuration, - long maxCommonCanonicalCacheSize, Duration commonCanonicalCacheDuration) { + long maxCommonCanonicalCacheSize, Duration commonCanonicalCacheDuration, + long maxLogicalResourceIdentCacheSize, Duration logicalResourceIdentCacheDuration) { codeSystemCache = Caffeine.newBuilder() .maximumSize(maxCodeSystemCacheSize) .expireAfterWrite(codeSystemCacheDuration) @@ -48,6 +51,10 @@ public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDur .maximumSize(maxCommonCanonicalCacheSize) .expireAfterWrite(commonCanonicalCacheDuration) .build(); + logicalResourceIdentCache = Caffeine.newBuilder() + .maximumSize(maxLogicalResourceIdentCacheSize) + .expireAfterWrite(logicalResourceIdentCacheDuration) + .build(); } /** @@ -95,4 +102,29 @@ public int getResourceTypeId(String resourceType) { } return resourceTypeId; } + + @Override + public void addCommonCanonicalValue(short shardKey, String url, long commonCanonicalValueId) { + this.commonCanonicalValueCache.put(new CommonCanonicalValueKey(shardKey, url), commonCanonicalValueId); + } + + @Override + public void addCommonTokenValue(short shardKey, String codeSystem, String tokenValue, long commonTokenValueId) { + this.commonTokenValueCache.put(new CommonTokenValueKey(shardKey, codeSystem, tokenValue), commonTokenValueId); + } + + @Override + public void addCodeSystem(String codeSystem, int codeSystemId) { + this.codeSystemCache.put(codeSystem, codeSystemId); + } + + @Override + public Long getLogicalResourceIdentId(String resourceType, String logicalId) { + return logicalResourceIdentCache.get(new LogicalResourceIdentKey(resourceType, logicalId), k -> NULL_LONG); + } + + @Override + public void addLogicalResourceIdent(String resourceType, String logicalId, long logicalResourceId) { + logicalResourceIdentCache.put(new LogicalResourceIdentKey(resourceType, logicalId), logicalResourceId); + } } \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java index 1e13867e30b..ec707641708 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java @@ -170,7 +170,7 @@ protected void endTransaction() throws FHIRPersistenceException { // any values from parameter_names, code_systems and common_token_values // are now committed to the database, so we can publish their record ids // to the shared cache which makes them accessible from other threads - publishCachedValues(); + publishValuesToCache(); } else { // something went wrong...try to roll back the transaction before we close // everything @@ -188,28 +188,41 @@ protected void endTransaction() throws FHIRPersistenceException { } catch (SQLException x) { throw new FHIRPersistenceException("commit failed", x); } finally { - if (!committed) { - // The maps may contain ids that were not committed to the database so - // we should clean them out in case we decide to reuse this consumer - this.logicalResourceIdentMap.clear(); - this.parameterNameMap.clear(); - this.codeSystemValueMap.clear(); - this.commonTokenValueMap.clear(); - this.commonCanonicalValueMap.clear(); - } + // always clear these maps because otherwise they could grow unbounded. Values + // are cached by the identityCache + this.logicalResourceIdentMap.clear(); + this.parameterNameMap.clear(); + this.codeSystemValueMap.clear(); + this.commonTokenValueMap.clear(); + this.commonCanonicalValueMap.clear(); } } /** * After the transaction has been committed, we can publish certain values to the - * shared identity caches + * shared identity caches allowing them to be used by other threads */ - private void publishCachedValues() { - // all the unresolvedParameterNames should be resolved at this point + private void publishValuesToCache() { for (ParameterNameValue pnv: this.unresolvedParameterNames) { logger.fine(() -> "Adding parameter-name to cache: '" + pnv.getParameterName() + "' -> " + pnv.getParameterNameId()); identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); } + + for (CommonCanonicalValue value: this.unresolvedCanonicalValues) { + identityCache.addCommonCanonicalValue(FIXED_SHARD, value.getUrl(), value.getCanonicalId()); + } + + for (CodeSystemValue value: this.unresolvedSystemValues) { + identityCache.addCodeSystem(value.getCodeSystem(), value.getCodeSystemId()); + } + + for (CommonTokenValue value: this.unresolvedTokenValues) { + identityCache.addCommonTokenValue(FIXED_SHARD, value.getCodeSystemValue().getCodeSystem(), value.getTokenValue(), value.getCommonTokenValueId()); + } + + for (LogicalResourceIdentValue value: this.unresolvedLogicalResourceIdents) { + identityCache.addLogicalResourceIdent(value.getResourceType(), value.getLogicalId(), value.getLogicalResourceId()); + } } @Override @@ -386,7 +399,7 @@ private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenV } /** - * Get the LogicalReosurceIdentValue we've assigned for the given (resourceType, logicalId) + * Get the LogicalResourceIdentValue we've assigned for the given (resourceType, logicalId) * tuple. The returned value may not yet have the actual logical_resource_id yet - we fetch * these values later and create new database records as necessary * @param resourceType @@ -403,7 +416,15 @@ private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourc .withLogicalId(logicalId) .build(); this.logicalResourceIdentMap.put(key, result); - this.unresolvedLogicalResourceIdents.add(result); + + // see if we can find the logical_resource_id from the cache + Long logicalResourceId = identityCache.getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId != null) { + result.setLogicalResourceId(logicalResourceId); + } else { + // Add to the unresolved list to look up later + this.unresolvedLogicalResourceIdents.add(result); + } } return result; } From fcf2c1ee4bbd89c3dc6ee21a62a0e12c3999de2b Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 2 Jun 2022 10:59:44 +0100 Subject: [PATCH 17/40] issue #3437 do not apply permissions to tables not created Signed-off-by: Robin Arnold --- .../java/com/ibm/fhir/database/utils/model/Table.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index bdbbc5e5dc7..67d3943ac44 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -207,6 +207,14 @@ public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContex } } + @Override + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { + if (create) { + // only issue the grant if we have created this object + super.grantGroupPrivileges(target, group, toUser); + } + } + @Override public void drop(ISchemaAdapter target) { if (this.accessControlVar != null) { From f2687cc5d9af17c4482c59ddebab28a19bbb9f14 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Fri, 3 Jun 2022 11:25:13 +0100 Subject: [PATCH 18/40] issue #3437 initial derby unit test for fhir-remote-index Signed-off-by: Robin Arnold --- fhir-remote-index/pom.xml | 23 + .../com/ibm/fhir/remote/index/app/Main.java | 7 +- .../DistributedPostgresMessageHandler.java | 16 +- .../PlainBatchParameterProcessor.java | 1 + .../database/PlainDerbyMessageHandler.java | 151 ++ .../index/database/PlainMessageHandler.java | 1357 +++++++++++++++++ .../database/PlainPostgresMessageHandler.java | 1289 +--------------- .../database/PlainPostgresParameterBatch.java | 2 +- .../PlainPostgresSystemParameterBatch.java | 3 +- .../ShardedPostgresMessageHandler.java | 4 +- .../fhir/remote/index/DerbyFhirFactory.java | 151 ++ .../fhir/remote/index/RemoteIndexTest.java | 223 +++ .../resources/test-remote-index.properties | 31 + 13 files changed, 1965 insertions(+), 1293 deletions(-) create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java create mode 100644 fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java create mode 100644 fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java create mode 100644 fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java create mode 100644 fhir-remote-index/src/test/resources/test-remote-index.properties diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml index ecf07ad738e..f8271ea09bc 100644 --- a/fhir-remote-index/pom.xml +++ b/fhir-remote-index/pom.xml @@ -81,6 +81,29 @@ derbytools test + + ${project.groupId} + fhir-persistence-schema + ${project.version} + test + + + ${project.groupId} + fhir-model + ${project.version} + test-jar + test + + + org.eclipse.parsson + jakarta.json + test + + + org.skyscreamer + jsonassert + test + diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 6f66f037a2f..9ca6f29b6d1 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -42,6 +42,7 @@ import com.ibm.fhir.remote.index.cache.IdentityCacheImpl; import com.ibm.fhir.remote.index.database.CacheLoader; import com.ibm.fhir.remote.index.database.DistributedPostgresMessageHandler; +import com.ibm.fhir.remote.index.database.PlainDerbyMessageHandler; import com.ibm.fhir.remote.index.database.PlainPostgresMessageHandler; import com.ibm.fhir.remote.index.kafka.RemoteIndexConsumer; import com.ibm.fhir.remote.index.sharded.ShardedPostgresMessageHandler; @@ -332,7 +333,11 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { case SHARDED: return new ShardedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); case PLAIN: - return new PlainPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + if (dbType == DbType.DERBY) { + return new PlainDerbyMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + } else { + return new PlainPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + } case DISTRIBUTED: return new DistributedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); default: diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index ee14fc99a71..041082adc7c 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -13,6 +13,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.remote.index.api.IdentityCache; @@ -22,7 +23,7 @@ * by a sequence, which means a slightly different INSERT statement * in certain cases */ -public class DistributedPostgresMessageHandler extends PlainPostgresMessageHandler { +public class DistributedPostgresMessageHandler extends PlainMessageHandler { private static final Logger logger = Logger.getLogger(DistributedPostgresMessageHandler.class.getName()); /** @@ -33,7 +34,12 @@ public class DistributedPostgresMessageHandler extends PlainPostgresMessageHandl * @param maxReadyTimeMs */ public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(connection, schemaName, cache, maxReadyTimeMs); + super(new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs); + } + + @Override + protected String onConflict() { + return "ON CONFLICT DO NOTHING"; } @Override @@ -47,7 +53,8 @@ protected void addMissingCommonTokenValues(List missing) throw insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number insert.append(" VALUES (?,?,"); insert.append(nextVal); // next sequence value - insert.append(") ON CONFLICT DO NOTHING"); + insert.append(") "); + insert.append(onConflict()); try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { int count = 0; @@ -82,7 +89,8 @@ protected void addMissingCommonCanonicalValues(List missin insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number insert.append(" VALUES (?,"); insert.append(nextVal); // next sequence value - insert.append(") ON CONFLICT DO NOTHING"); + insert.append(") "); + insert.append(onConflict()); final String DML = insert.toString(); if (logger.isLoggable(Level.FINE)) { diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java index 06f683ec2cd..c1f65e0317a 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java @@ -220,6 +220,7 @@ public void process(String requestShard, String resourceType, String logicalId, systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId()); } } catch (SQLException x) { + logger.log(Level.SEVERE, "DateParameter", x); throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'"); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java new file mode 100644 index 00000000000..3c7ba2c9234 --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java @@ -0,0 +1,151 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.derby.DerbyTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.remote.index.api.IdentityCache; + +/** + * Derby variant of the plain schema message handler which is needed because Derby + * needs slightly different syntax for some queries + */ +public class PlainDerbyMessageHandler extends PlainMessageHandler { + private static final Logger logger = Logger.getLogger(PlainDerbyMessageHandler.class.getName()); + + /** + * Public constructor + * @param connection + * @param schemaName + * @param cache + * @param maxReadyTimeMs + */ + public PlainDerbyMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(new DerbyTranslator(), connection, schemaName, cache, maxReadyTimeMs); + } + + @Override + protected String onConflict() { + return ""; + } + + @Override + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" JOIN resource_types AS rt ON (rt.resource_type_id = lri.resource_type_id)"); + query.append(" WHERE "); + for (int i=0; i 0) { + query.append(" OR "); + } + query.append("(lri.resource_type_id = ? AND lri.logical_id = ?)"); + } + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + logger.fine(() -> "logicalResourceIdents: " + query.toString()); + return ps; + } + + @Override + protected Integer createParameterName(String parameterName) throws SQLException { + Integer parameterNameId = getNextRefId(); + final String insertParameterName = "" + + "INSERT INTO parameter_names (parameter_name_id, parameter_name) " + + " VALUES (?, ?)"; + try (PreparedStatement stmt = connection.prepareStatement(insertParameterName)) { + stmt.setInt(1, parameterNameId); + stmt.setString(2, parameterName); + stmt.execute(); + } + + return parameterNameId; + } + + @Override + protected PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + // need the code_system name - so we join back to the code_systems table as well + query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id "); + query.append(" FROM common_token_values c"); + query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); + query.append(" WHERE "); + + // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records + boolean first = true; + for (CommonTokenValue ctv: values) { + if (first) { + first = false; + } else { + query.append(" OR "); + } + query.append("(c.code_system_id = "); + query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id + query.append(" AND c.token_value = ?)"); + } + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonTokenValue ctv: values) { + ps.setString(param++, ctv.getTokenValue()); + } + return new PreparedStatementWrapper(statementText, ps); + } + @Override + protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (url, canonical_id) "); + insert.append(" VALUES (?,"); + insert.append(nextVal); // next sequence value + insert.append(") "); + + final String DML = insert.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("addMissingCanonicalIds: " + DML); + } + try (PreparedStatement ps = connection.prepareStatement(DML)) { + int count = 0; + for (CommonCanonicalValue ctv: missing) { + logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); + ps.setString(1, ctv.getUrl()); + ps.addBatch(); + if (++count == this.maxCommonCanonicalValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common canonical values"); + } + } + +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java new file mode 100644 index 00000000000..fbdf152c39d --- /dev/null +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java @@ -0,0 +1,1357 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index.database; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.ResultSetReader; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.DateParameter; +import com.ibm.fhir.persistence.index.LocationParameter; +import com.ibm.fhir.persistence.index.NumberParameter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.index.QuantityParameter; +import com.ibm.fhir.persistence.index.ReferenceParameter; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParameterValue; +import com.ibm.fhir.persistence.index.SecurityParameter; +import com.ibm.fhir.persistence.index.StringParameter; +import com.ibm.fhir.persistence.index.TagParameter; +import com.ibm.fhir.persistence.index.TokenParameter; +import com.ibm.fhir.remote.index.api.BatchParameterValue; +import com.ibm.fhir.remote.index.api.IdentityCache; +import com.ibm.fhir.remote.index.batch.BatchDateParameter; +import com.ibm.fhir.remote.index.batch.BatchLocationParameter; +import com.ibm.fhir.remote.index.batch.BatchNumberParameter; +import com.ibm.fhir.remote.index.batch.BatchProfileParameter; +import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; +import com.ibm.fhir.remote.index.batch.BatchReferenceParameter; +import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; +import com.ibm.fhir.remote.index.batch.BatchStringParameter; +import com.ibm.fhir.remote.index.batch.BatchTagParameter; +import com.ibm.fhir.remote.index.batch.BatchTokenParameter; + +/** + * Loads search parameter values into the target PostgreSQL database using + * the plain (non-sharded) schema variant. + */ +public abstract class PlainMessageHandler extends BaseMessageHandler { + private static final String CLASSNAME = PlainMessageHandler.class.getName(); + private static final Logger logger = Logger.getLogger(PlainMessageHandler.class.getName()); + private static final short FIXED_SHARD = 0; + + // the connection to use for the inserts + protected final Connection connection; + + // The translator to handle variations in SQL syntax + protected final IDatabaseTranslator translator; + + // The FHIR data schema + protected final String schemaName; + + // the cache we use for various lookups + protected final IdentityCache identityCache; + + // All logical_resource_ident values we've seen + private final Map logicalResourceIdentMap = new HashMap<>(); + + // All parameter names we've seen (cleared if there's a rollback) + private final Map parameterNameMap = new HashMap<>(); + + // A map of code system name to the value holding its codeSystemId from the database + private final Map codeSystemValueMap = new HashMap<>(); + + // A map to support lookup of CommonTokenValue records by key + private final Map commonTokenValueMap = new HashMap<>(); + + // A map to support lookup of CommonCanonicalValue records by key + private final Map commonCanonicalValueMap = new HashMap<>(); + + // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id + private final List unresolvedLogicalResourceIdents = new ArrayList<>(); + + // All parameter names in the current transaction for which we don't yet know the parameter_name_id + private final List unresolvedParameterNames = new ArrayList<>(); + + // A list of all the CodeSystemValues for which we don't yet know the code_system_id + private final List unresolvedSystemValues = new ArrayList<>(); + + // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id + private final List unresolvedTokenValues = new ArrayList<>(); + + // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id + private final List unresolvedCanonicalValues = new ArrayList<>(); + + // The processed values we've collected + private final List batchedParameterValues = new ArrayList<>(); + + // The processor used to process the batched parameter values after all the reference values are created + private final PlainBatchParameterProcessor batchProcessor; + + protected final int maxLogicalResourcesPerStatement = 256; + protected final int maxCodeSystemsPerStatement = 512; + protected final int maxCommonTokenValuesPerStatement = 256; + protected final int maxCommonCanonicalValuesPerStatement = 256; + private boolean rollbackOnly; + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param cache + * @param maxReadyTimeMs + */ + public PlainMessageHandler(IDatabaseTranslator translator, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(maxReadyTimeMs); + this.translator = translator; + this.connection = connection; + this.schemaName = schemaName; + this.identityCache = cache; + this.batchProcessor = new PlainBatchParameterProcessor(connection); + } + + @Override + protected void startBatch() { + // always start with a clean slate + batchedParameterValues.clear(); + unresolvedLogicalResourceIdents.clear(); + unresolvedParameterNames.clear(); + unresolvedSystemValues.clear(); + unresolvedTokenValues.clear(); + unresolvedCanonicalValues.clear(); + batchProcessor.startBatch(); + } + + @Override + protected void setRollbackOnly() { + this.rollbackOnly = true; + } + + @Override + public void close() { + try { + batchProcessor.close(); + } catch (Throwable t) { + logger.log(Level.SEVERE, "close batchProcessor failed" , t); + } + } + + @Override + protected void endTransaction() throws FHIRPersistenceException { + boolean committed = false; + try { + if (!this.rollbackOnly) { + logger.fine("Committing transaction"); + connection.commit(); + committed = true; + + // any values from parameter_names, code_systems and common_token_values + // are now committed to the database, so we can publish their record ids + // to the shared cache which makes them accessible from other threads + publishValuesToCache(); + } else { + // something went wrong...try to roll back the transaction before we close + // everything + try { + connection.rollback(); + } catch (SQLException x) { + // It could very well be that we've lost touch with the database in which case + // the rollback will also fail. Not much we can do, although we don't bother + // with a stack trace here because it's just more noise for the log file, and + // the exception that triggered the rollback is already going to be propagated + // and logged. + logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); + } + } + } catch (SQLException x) { + throw new FHIRPersistenceException("commit failed", x); + } finally { + // always clear these maps because otherwise they could grow unbounded. Values + // are cached by the identityCache + this.logicalResourceIdentMap.clear(); + this.parameterNameMap.clear(); + this.codeSystemValueMap.clear(); + this.commonTokenValueMap.clear(); + this.commonCanonicalValueMap.clear(); + } + } + + /** + * After the transaction has been committed, we can publish certain values to the + * shared identity caches allowing them to be used by other threads + */ + private void publishValuesToCache() { + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + logger.fine(() -> "Adding parameter-name to cache: '" + pnv.getParameterName() + "' -> " + pnv.getParameterNameId()); + identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); + } + + for (CommonCanonicalValue value: this.unresolvedCanonicalValues) { + identityCache.addCommonCanonicalValue(FIXED_SHARD, value.getUrl(), value.getCanonicalId()); + } + + for (CodeSystemValue value: this.unresolvedSystemValues) { + identityCache.addCodeSystem(value.getCodeSystem(), value.getCodeSystemId()); + } + + for (CommonTokenValue value: this.unresolvedTokenValues) { + identityCache.addCommonTokenValue(FIXED_SHARD, value.getCodeSystemValue().getCodeSystem(), value.getTokenValue(), value.getCommonTokenValueId()); + } + + for (LogicalResourceIdentValue value: this.unresolvedLogicalResourceIdents) { + identityCache.addLogicalResourceIdent(value.getResourceType(), value.getLogicalId(), value.getLogicalResourceId()); + } + } + + @Override + protected void pushBatch() throws FHIRPersistenceException { + // Push any data we've accumulated so far. This may occur + // if we cross a volume threshold, and will always occur as + // the last step before the current transaction is committed, + // Process the token values so that we can establish + // any entries we need for common_token_values + logger.fine("pushBatch: resolving all ids"); + resolveLogicalResourceIdents(); + resolveParameterNames(); + resolveCodeSystems(); + resolveCommonTokenValues(); + resolveCommonCanonicalValues(); + + // Now that all the lookup values should've been resolved, we can go ahead + // and push the parameters to the JDBC batch insert statements via the + // batchProcessor + logger.fine("pushBatch: processing collected parameters"); + for (BatchParameterValue v: this.batchedParameterValues) { + v.apply(batchProcessor); + } + + logger.fine("pushBatch: executing final batch statements"); + batchProcessor.pushBatch(); + logger.fine("pushBatch completed"); + } + + /** + * Get the parameter name value for the given parameter value + * @param p + * @return + */ + private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("get ParameterNameValue for [" + p.toString() + "]"); + } + ParameterNameValue result = parameterNameMap.get(p.getName()); + if (result == null) { + result = new ParameterNameValue(p.getName()); + parameterNameMap.put(p.getName(), result); + + // let's see if the id is available in the shared identity cache + Integer parameterNameId = identityCache.getParameterNameId(p.getName()); + if (parameterNameId != null) { + result.setParameterNameId(parameterNameId); + } else { + // ids will be created later (so that we can process them in order) + unresolvedParameterNames.add(result); + } + } + return result; + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { + CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { + CommonCanonicalValue ctv = lookupCommonCanonicalValue(p.getUrl()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); + this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { + ParameterNameValue parameterNameValue = getParameterNameId(p); + this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); + } + + @Override + protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException { + logger.fine(() -> "Processing reference parameter value:" + p.toString()); + ParameterNameValue parameterNameValue = getParameterNameId(p); + LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId()); + this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv)); + } + + /** + * Get the CodeSystemValue we've assigned for the given codeSystem value. This + * may not yet have the actual code_system_id from the database yet - any values + * we don't have will be assigned in a later phase (so we can do things neatly + * in bulk). + * @param codeSystem + * @return + */ + private CodeSystemValue lookupCodeSystemValue(String codeSystem) { + CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); + if (result == null) { + result = new CodeSystemValue(codeSystem); + this.codeSystemValueMap.put(codeSystem, result); + + // Take this opportunity to see if we have a cached value for this codeSystem + Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); + if (codeSystemId != null) { + result.setCodeSystemId(codeSystemId); + } else { + // Stash for later resolution + this.unresolvedSystemValues.add(result); + } + } + return result; + } + + /** + * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. + * The returned value may not yet have the actual common_token_value_id yet - we fetch + * these values later and create new database records as necessary. + * @param codeSystem + * @param tokenValue + * @return + */ + private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenValue) { + CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, codeSystem, tokenValue); + CommonTokenValue result = this.commonTokenValueMap.get(key); + if (result == null) { + CodeSystemValue csv = lookupCodeSystemValue(codeSystem); + result = new CommonTokenValue(FIXED_SHARD, csv, tokenValue); + this.commonTokenValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long commonTokenValueId = identityCache.getCommonTokenValueId(FIXED_SHARD, codeSystem, tokenValue); + if (commonTokenValueId != null) { + result.setCommonTokenValueId(commonTokenValueId); + } else { + this.unresolvedTokenValues.add(result); + } + } + return result; + } + + /** + * Get the LogicalResourceIdentValue we've assigned for the given (resourceType, logicalId) + * tuple. The returned value may not yet have the actual logical_resource_id yet - we fetch + * these values later and create new database records as necessary + * @param resourceType + * @param logicalId + * @return + */ + private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId); + LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key); + if (result == null) { + result = LogicalResourceIdentValue.builder() + .withResourceTypeId(identityCache.getResourceTypeId(resourceType)) + .withResourceType(resourceType) + .withLogicalId(logicalId) + .build(); + this.logicalResourceIdentMap.put(key, result); + + // see if we can find the logical_resource_id from the cache + Long logicalResourceId = identityCache.getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId != null) { + result.setLogicalResourceId(logicalResourceId); + } else { + // Add to the unresolved list to look up later + this.unresolvedLogicalResourceIdents.add(result); + } + } + return result; + } + + /** + * Get the CommonCanonicalValue we've assigned for the given url value. + * The returned value may not yet have the actual canonical_id yet - we fetch + * these values later and create new database records as necessary. + * @param url + * @return + */ + private CommonCanonicalValue lookupCommonCanonicalValue(String url) { + CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, url); + CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); + if (result == null) { + result = new CommonCanonicalValue(FIXED_SHARD, url); + this.commonCanonicalValueMap.put(key, result); + + // Take this opportunity to see if we have a cached value for this common token value + Long canonicalId = identityCache.getCommonCanonicalValueId(FIXED_SHARD, url); + if (canonicalId != null) { + result.setCanonicalId(canonicalId); + } else { + this.unresolvedCanonicalValues.add(result); + } + } + return result; + } + + /** + * Make sure we have values for all the code_systems we have collected + * in the current + * batch + * @throws FHIRPersistenceException + */ + private void resolveCodeSystems() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCodeSystemIds(unresolvedSystemValues); + + if (!missing.isEmpty()) { + addMissingCodeSystems(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCodeSystemIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all code system values"); + } + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); + for (int i=0; i 0) { + query.append(","); + } + query.append("?"); + } + query.append(")"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (CodeSystemValue csv: values) { + ps.setString(param++, csv.getCodeSystem()); + } + return ps; + } + + protected abstract String onConflict(); + + /** + * These code systems weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingCodeSystems(List missing) throws FHIRPersistenceException { + List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); + // Sort the code system values first to help avoid deadlocks + Collections.sort(values); // natural ordering for String is fine here + + final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); + insert.append(nextVal); // next sequence value + insert.append(",?) "); + insert.append(onConflict()); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (String codeSystem: values) { + ps.setString(1, codeSystem); + ps.addBatch(); + if (++count == this.maxCodeSystemsPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + /** + * Fetch all the code_system_id values for the given list of CodeSystemValue objects. + * @param unresolved + * @return + * @throws FHIRPersistenceException + */ + private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); + if (csv != null) { + csv.setCodeSystemId(rs.getInt(1)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("code systems query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CodeSystemValue csv: sub) { + if (csv.getCodeSystemId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "code systems fetch failed", x); + throw new FHIRPersistenceException("code systems fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Make sure we have values for all the common_token_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonTokenValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCommonTokenValueIds(unresolvedTokenValues); + + if (!missing.isEmpty()) { + // Sort first to minimize deadlocks + Collections.sort(missing, (a,b) -> { + int result = a.getTokenValue().compareTo(b.getTokenValue()); + if (result == 0) { + result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); + if (result == 0) { + result = Short.compare(a.getShardKey(), b.getShardKey()); + } + } + return result; + }); + addMissingCommonTokenValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCommonTokenValueIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happend, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all common token values"); + } + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT code_system, token_value, common_token_value_id + * @throws SQLException + */ + protected PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + // need the code_system name - so we join back to the code_systems table as well + query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id "); + query.append(" FROM common_token_values c"); + query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); + query.append(" JOIN (VALUES "); + + // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records + boolean first = true; + for (CommonTokenValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("("); + query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id + query.append(",?)"); // bind variable for the token-value + } + query.append(") AS v(code_system_id, token_value) "); + query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value)"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonTokenValue ctv: values) { + ps.setString(param++, ctv.getTokenValue()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + /** + * Fetch the common_token_value_id values for the given list of CommonTokenValue objects. + * @param unresolved + * @return + * @throws FHIRPersistenceException + */ + private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, rs.getString(1), rs.getString(2)); + CommonTokenValue ctv = this.commonTokenValueMap.get(key); + if (ctv != null) { + ctv.setCommonTokenValueId(rs.getLong(3)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common token values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonTokenValue ctv: sub) { + if (ctv.getCommonTokenValueId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common token values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_token_values (code_system_id, token_value) "); + insert.append(" VALUES (?,?) "); + insert.append(onConflict()); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (CommonTokenValue ctv: missing) { + ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId()); + ps.setString(2, ctv.getTokenValue()); + ps.addBatch(); + if (++count == this.maxCommonTokenValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common token values"); + } + } + + /** + * Make sure we have values for all the common_canonical_value records we have collected + * in the current batch + * @throws FHIRPersistenceException + */ + private void resolveCommonCanonicalValues() throws FHIRPersistenceException { + // identify which values aren't yet in the database + List missing = fetchCanonicalIds(unresolvedCanonicalValues); + + if (!missing.isEmpty()) { + // Sort on url to minimize deadlocks + Collections.sort(missing, (a,b) -> { + return a.getUrl().compareTo(b.getUrl()); + }); + addMissingCommonCanonicalValues(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + List bad = fetchCanonicalIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all canonical values"); + } + } + + /** + * Fetch the common_canonical_id values for the given list of CommonCanonicalValue objects. + * @param unresolved + * @return + * @throws FHIRPersistenceException + */ + private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + String sql = null; // the SQL text for logging when there's an error + try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { + sql = ps.getStatementText(); + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each CodeSystemValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, rs.getString(1)); + CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); + if (ctv != null) { + ctv.setCanonicalId(rs.getLong(2)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); + } + } + + // Optimize the check for missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (CommonCanonicalValue ctv: sub) { + if (ctv.getCanonicalId() == null) { + missing.add(ctv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); + throw new FHIRPersistenceException("common canonical values fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Build and prepare a statement to fetch the common_token_value records + * for all the given (unresolved) code system values + * @param values + * @return SELECT code_system, token_value, common_token_value_id + * @throws SQLException + */ + private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT c.url, c.canonical_id "); + query.append(" FROM common_canonical_values c "); + query.append(" WHERE c.url IN ("); + + // add bind variables for each url we need to fetch + boolean first = true; + for (CommonCanonicalValue ctv: values) { + if (first) { + first = false; + } else { + query.append(","); + } + query.append("?"); // bind variable for the url + } + query.append(")"); + + // Create the prepared statement and bind the values + final String statementText = query.toString(); + logger.finer(() -> "fetch common canonical values [" + statementText + "]"); + PreparedStatement ps = connection.prepareStatement(statementText); + + // bind the parameter values + int param = 1; + for (CommonCanonicalValue ctv: values) { + ps.setString(param++, ctv.getUrl()); + } + return new PreparedStatementWrapper(statementText, ps); + } + + /** + * Add the values we think are missing from the database. The given list should be + * sorted to reduce deadlocks + * @param missing + * @throws FHIRPersistenceException + */ + protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO common_canonical_values (url) VALUES (?) "); + insert.append(onConflict()); + + final String DML = insert.toString(); + if (logger.isLoggable(Level.FINE)) { + logger.fine("addMissingCanonicalIds: " + DML); + } + try (PreparedStatement ps = connection.prepareStatement(DML)) { + int count = 0; + for (CommonCanonicalValue ctv: missing) { + logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); + ps.setString(1, ctv.getUrl()); + ps.addBatch(); + if (++count == this.maxCommonCanonicalValuesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "failed: " + insert.toString(), x); + throw new FHIRPersistenceException("failed inserting new common canonical values"); + } + } + + /** + * Make sure all the parameter names we've seen in the batch exist + * in the database and have ids. + * @throws FHIRPersistenceException + */ + private void resolveParameterNames() throws FHIRPersistenceException { + // We expect parameter names to have a very high cache hit rate and + // so we simplify processing by simply iterating one-by-one for the + // values we still need to resolve. The most important point here is + // to do this in a sorted order to avoid deadlock issues because this + // could be happening across multiple consumer threads at the same time. + logger.fine("resolveParameterNames: sorting unresolved names"); + Collections.sort(this.unresolvedParameterNames, (a,b) -> { + return a.getParameterName().compareTo(b.getParameterName()); + }); + + try { + for (ParameterNameValue pnv: this.unresolvedParameterNames) { + logger.finer(() -> "fetching parameter_name_id for '" + pnv.getParameterName() + "'"); + Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); + if (parameterNameId == null) { + parameterNameId = createParameterName(pnv.getParameterName()); + if (logger.isLoggable(Level.FINER)) { + logger.finer("assigned parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); + } + + if (parameterNameId == null) { + // be defensive + throw new FHIRPersistenceException("parameter_name_id not assigned for '" + pnv.getParameterName()); + } + } else if (logger.isLoggable(Level.FINER)) { + logger.finer("read parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); + } + pnv.setParameterNameId(parameterNameId); + } + } catch (SQLException x) { + throw new FHIRPersistenceException("error resolving parameter names", x); + } finally { + logger.exiting(CLASSNAME, "resolveParameterNames"); + } + } + + /** + * Fetch the parameter_name_id for the given parameterName value + * @param parameterName + * @return + * @throws SQLException + */ + private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { + String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; + try (PreparedStatement ps = connection.prepareStatement(SQL)) { + ps.setString(1, parameterName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + return rs.getInt(1); + } + } + + // no entry in parameter_names + return null; + } + + /** + * Create the parameter name using the stored procedure which handles any concurrency + * issue we may have + * @param parameterName + * @return + */ + protected Integer createParameterName(String parameterName) throws SQLException { + final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; + Integer parameterNameId; + try (CallableStatement stmt = connection.prepareCall(CALL)) { + stmt.setString(1, parameterName); + stmt.registerOutParameter(2, Types.INTEGER); + stmt.execute(); + parameterNameId = stmt.getInt(2); + } + + return parameterNameId; + } + + @Override + protected void resetBatch() { + // Called when a transaction has been rolled back because of a deadlock + // or other retryable error and we want to try and process the batch again + batchProcessor.reset(); + } + + /** + * Build the check ready query + * @param messagesByResourceType + * @return + */ + private String buildCheckReadyQuery(Map> messagesByResourceType) { + // The trouble here is that we'll end up with a unique query for every single + // batch of messages we process (which the database then need to parse etc). + // This may introduce scaling issues, in which case we should consider + // individual queries for each resource type using bind variables, perhaps + // going so far as using multiple statements with a power-of-2 number of bind + // variables. But JDBC doesn't support batching of select statements, so + // the alternative there would be to insert-as-select into a global temp table + // and then simply select from that. Fairly straightforward, but a lot more + // work so only worth doing if we identify contention here. + + StringBuilder select = new StringBuilder(); + // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // patient_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (1,2,3,4) + // UNION ALL + // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash + // FROM logical_resources AS lr, + // observation_logical_resources AS xlr + // WHERE lr.logical_resource_id = xlr.logical_resource_id + // AND xlr.logical_resource_id IN (5,6,7) + boolean first = true; + for (Map.Entry> entry: messagesByResourceType.entrySet()) { + final String resourceType = entry.getKey(); + final List messages = entry.getValue(); + final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); + if (first) { + first = false; + } else { + select.append(" UNION ALL "); + } + select.append(" SELECT lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); + select.append(" FROM logical_resources AS lr, "); + select.append(resourceType).append("_logical_resources AS xlr "); + select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); + select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); + } + + return select.toString(); + } + + @Override + protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { + // Get a list of all the resources for which we can see the current logical resource data. + // If the resource doesn't yet exist or its version meta doesn't the message + // then we add to the notReady list. If the resource version meta already + // exceeds the message, then we'll skip processing altogether because it + // means that there should be another message in the queue with more + // up-to-date parameters + Map messageMap = new HashMap<>(); + Map> messagesByResourceType = new HashMap<>(); + for (RemoteIndexMessage msg: messages) { + Long logicalResourceId = msg.getData().getLogicalResourceId(); + messageMap.put(logicalResourceId, msg); + + // split out the messages per resource type because we need to read from xx_logical_resources + List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); + values.add(msg); + } + + Set found = new HashSet<>(); + final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); + logger.fine(() -> "check ready query: " + checkReadyQuery); + try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { + ResultSet rs = ps.executeQuery(); + // wrap the ResultSet in a reader for easier consumption + ResultSetReader rsReader = new ResultSetReader(rs); + while (rsReader.next()) { + LogicalResourceValue lrv = LogicalResourceValue.builder() + .withLogicalResourceId(rsReader.getLong()) + .withResourceType(rsReader.getString()) + .withLogicalId(rsReader.getString()) + .withVersionId(rsReader.getInt()) + .withLastUpdated(rsReader.getTimestamp()) + .withParameterHash(rsReader.getString()) + .build(); + RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); + if (m == null) { + throw new IllegalStateException("query returned a logical resource which we didn't request"); + } + + // Check the values from the database to see if they match + // the information in the message. + if (m.getData().getVersionId() == lrv.getVersionId()) { + // only process this message if the parameter hash and lastUpdated + // times match - which is a good check that we're storing parameters + // from the correct transaction. If these don't match, we can simply + // say we found the data but don't need to process the message. + final Instant dbLastUpdated = lrv.getLastUpdated().toInstant(); + final Instant msgLastUpdated = m.getData().getLastUpdated(); + if (lrv.getParameterHash().equals(m.getData().getParameterHash()) + && dbLastUpdated.equals(msgLastUpdated)) { + okToProcess.add(m); + } else { + logger.warning("Parameter message must match both parameter_hash and last_updated. Must be from an uncommitted transaction so ignoring: " + m.toString()); + } + found.add(lrv.getLogicalResourceId()); // won't be marked as missing + } else if (m.getData().getVersionId() > lrv.getVersionId()) { + // we can skip processing this record because the database has already + // been updated with a newer version. Identify the record as having been + // found so we don't keep waiting for it + found.add(lrv.getLogicalResourceId()); + } + // if the version in the database is prior to version in the message we + // received it means that the server transaction hasn't been committed... + // so we have to wait just as though it were missing altogether + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); + throw new FHIRPersistenceException("prepare query failed"); + } + + if (found.size() < messages.size()) { + // identify the missing records and add to the notReady list + for (RemoteIndexMessage m: messages) { + if (!found.contains(m.getData().getLogicalResourceId())) { + notReady.add(m); + } + } + } + } + + + + + /** + * Make sure we have values for all the logical_resource_ident values + * we have collected in the current batch. Need to make sure these are + * added in order to minimize deadlocks. Note that because we may create + * new logical_resource_ident records, we could be blocked by the main + * add_any_resource procedure run within the server CREATE/UPDATE + * transaction. + * @throws FHIRPersistenceException + */ + private void resolveLogicalResourceIdents() throws FHIRPersistenceException { + logger.fine("resolveLogicalResourceIdents: fetching ids for unresolved LogicalResourceIdent records"); + // identify which values aren't yet in the database + List missing = fetchLogicalResourceIdentIds(unresolvedLogicalResourceIdents); + + if (!missing.isEmpty()) { + logger.fine("resolveLogicalResourceIdents: add missing LogicalResourceIdent records"); + addMissingLogicalResourceIdents(missing); + } + + // All the previously missing values should now be in the database. We need to fetch them again, + // possibly having to use multiple queries + logger.fine("resolveLogicalResourceIdents: fetch ids for missing LogicalResourceIdent records"); + List bad = fetchLogicalResourceIdentIds(missing); + + if (!bad.isEmpty()) { + // shouldn't happen, but let's protected against it anyway + throw new FHIRPersistenceException("Failed to create all logical_resource_ident values"); + } + logger.fine("resolveLogicalResourceIdents: all resolved"); + } + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" JOIN (VALUES "); + for (int i=0; i 0) { + query.append(","); + } + query.append("(?,?)"); + } + query.append(") AS v(resource_type_id, logical_id) "); + query.append(" ON (lri.resource_type_id = v.resource_type_id AND lri.logical_id = v.logical_id)"); + query.append(" JOIN resource_types AS rt ON (rt.resource_type_id = v.resource_type_id)"); // convenient to get the resource type name here + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + logger.fine(() -> "logicalResourceIdents: " + query.toString()); + return ps; + } + + /** + * These logical_resource_ident values weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { + // Sort the values first to help avoid deadlocks + Collections.sort(missing, (a,b) -> { + return a.compareTo(b); + }); + + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") "); + insert.append(onConflict()); + + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + int count = 0; + for (LogicalResourceIdentValue value: missing) { + if (value.getResourceTypeId() == null) { + logger.severe("bad value: " + value); + } + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ps.addBatch(); + if (++count == this.maxLogicalResourcesPerStatement) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + /** + * Fetch logical_resource_id values for the given list of LogicalResourceIdent objects. + * @param unresolved + * @return + * @throws FHIRPersistenceException + */ + private List fetchLogicalResourceIdentIds(List unresolved) throws FHIRPersistenceException { + // track which values aren't yet in the database + List missing = new ArrayList<>(); + + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each LogicalResourceIdentValue + int resultCount = 0; + while (rs.next()) { + resultCount++; + LogicalResourceIdentKey key = new LogicalResourceIdentKey(rs.getString(1), rs.getString(2)); + LogicalResourceIdentValue csv = this.logicalResourceIdentMap.get(key); + if (csv != null) { + csv.setLogicalResourceId(rs.getLong(3)); + } else { + // can't really happen, but be defensive + throw new FHIRPersistenceException("logical resource ident query returned an unexpected value"); + } + } + + // Most of the time we'll get everything, so we can bypass the check for + // missing values + if (resultCount == 0) { + // 100% miss + missing.addAll(sub); + } else if (resultCount < subSize) { + // need to scan the sub list and see which values we don't yet have ids for + for (LogicalResourceIdentValue csv: sub) { + if (csv.getLogicalResourceId() == null) { + missing.add(csv); + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } + + // Return the list of CodeSystemValues which don't yet have a database entry + return missing; + } + + /** + * Get the next value from fhir_ref_sequence + * @param c + * @return + * @throws SQLException + */ + protected Integer getNextRefId() throws SQLException { + final String select = translator.selectSequenceNextValue(schemaName, "fhir_ref_sequence"); + Integer result; + try (Statement s = connection.createStatement()) { + ResultSet rs = s.executeQuery(select); + if (rs.next()) { + result = rs.getInt(1); + } else { + throw new IllegalStateException("no row from '" + select + "'"); + } + } + return result; + } + + /** + * Get the next value from fhir_sequence + * @param c + * @return + * @throws SQLException + */ + protected Long getNextId() throws SQLException { + final String select = translator.selectSequenceNextValue(schemaName, "fhir_sequence"); + Long result; + try (Statement s = connection.createStatement()) { + ResultSet rs = s.executeQuery(select); + if (rs.next()) { + result = rs.getLong(1); + } else { + throw new IllegalStateException("no row from '" + select + "'"); + } + } + return result; + } + +} \ No newline at end of file diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java index ec707641708..bc39fc0c15a 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java @@ -6,1306 +6,29 @@ package com.ibm.fhir.remote.index.database; -import java.sql.CallableStatement; import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Types; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.common.ResultSetReader; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; -import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.index.DateParameter; -import com.ibm.fhir.persistence.index.LocationParameter; -import com.ibm.fhir.persistence.index.NumberParameter; -import com.ibm.fhir.persistence.index.ProfileParameter; -import com.ibm.fhir.persistence.index.QuantityParameter; -import com.ibm.fhir.persistence.index.ReferenceParameter; -import com.ibm.fhir.persistence.index.RemoteIndexMessage; -import com.ibm.fhir.persistence.index.SearchParameterValue; -import com.ibm.fhir.persistence.index.SecurityParameter; -import com.ibm.fhir.persistence.index.StringParameter; -import com.ibm.fhir.persistence.index.TagParameter; -import com.ibm.fhir.persistence.index.TokenParameter; -import com.ibm.fhir.remote.index.api.BatchParameterValue; import com.ibm.fhir.remote.index.api.IdentityCache; -import com.ibm.fhir.remote.index.batch.BatchDateParameter; -import com.ibm.fhir.remote.index.batch.BatchLocationParameter; -import com.ibm.fhir.remote.index.batch.BatchNumberParameter; -import com.ibm.fhir.remote.index.batch.BatchProfileParameter; -import com.ibm.fhir.remote.index.batch.BatchQuantityParameter; -import com.ibm.fhir.remote.index.batch.BatchReferenceParameter; -import com.ibm.fhir.remote.index.batch.BatchSecurityParameter; -import com.ibm.fhir.remote.index.batch.BatchStringParameter; -import com.ibm.fhir.remote.index.batch.BatchTagParameter; -import com.ibm.fhir.remote.index.batch.BatchTokenParameter; /** - * Loads search parameter values into the target PostgreSQL database using - * the plain (non-sharded) schema variant. + * PostgreSQL variant of the remote index message handler */ -public class PlainPostgresMessageHandler extends BaseMessageHandler { - private static final String CLASSNAME = PlainPostgresMessageHandler.class.getName(); - private static final Logger logger = Logger.getLogger(PlainPostgresMessageHandler.class.getName()); - private static final short FIXED_SHARD = 0; - - // the connection to use for the inserts - protected final Connection connection; - - // We're a PostgreSQL DAO, so we now which translator to use - protected final IDatabaseTranslator translator = new PostgresTranslator(); - - // The FHIR data schema - protected final String schemaName; - - // the cache we use for various lookups - protected final IdentityCache identityCache; - - // All logical_resource_ident values we've seen - private final Map logicalResourceIdentMap = new HashMap<>(); - - // All parameter names we've seen (cleared if there's a rollback) - private final Map parameterNameMap = new HashMap<>(); - - // A map of code system name to the value holding its codeSystemId from the database - private final Map codeSystemValueMap = new HashMap<>(); - - // A map to support lookup of CommonTokenValue records by key - private final Map commonTokenValueMap = new HashMap<>(); - - // A map to support lookup of CommonCanonicalValue records by key - private final Map commonCanonicalValueMap = new HashMap<>(); - - // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id - private final List unresolvedLogicalResourceIdents = new ArrayList<>(); - - // All parameter names in the current transaction for which we don't yet know the parameter_name_id - private final List unresolvedParameterNames = new ArrayList<>(); - - // A list of all the CodeSystemValues for which we don't yet know the code_system_id - private final List unresolvedSystemValues = new ArrayList<>(); - - // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id - private final List unresolvedTokenValues = new ArrayList<>(); - - // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id - private final List unresolvedCanonicalValues = new ArrayList<>(); - - // The processed values we've collected - private final List batchedParameterValues = new ArrayList<>(); - - // The processor used to process the batched parameter values after all the reference values are created - private final PlainBatchParameterProcessor batchProcessor; - - protected final int maxLogicalResourcesPerStatement = 256; - protected final int maxCodeSystemsPerStatement = 512; - protected final int maxCommonTokenValuesPerStatement = 256; - protected final int maxCommonCanonicalValuesPerStatement = 256; - private boolean rollbackOnly; +public class PlainPostgresMessageHandler extends PlainMessageHandler { /** - * Public constructor - * * @param connection * @param schemaName * @param cache * @param maxReadyTimeMs */ public PlainPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(maxReadyTimeMs); - this.connection = connection; - this.schemaName = schemaName; - this.identityCache = cache; - this.batchProcessor = new PlainBatchParameterProcessor(connection); - } - - @Override - protected void startBatch() { - // always start with a clean slate - batchedParameterValues.clear(); - unresolvedLogicalResourceIdents.clear(); - unresolvedParameterNames.clear(); - unresolvedSystemValues.clear(); - unresolvedTokenValues.clear(); - unresolvedCanonicalValues.clear(); - batchProcessor.startBatch(); - } - - @Override - protected void setRollbackOnly() { - this.rollbackOnly = true; - } - - @Override - public void close() { - try { - batchProcessor.close(); - } catch (Throwable t) { - logger.log(Level.SEVERE, "close batchProcessor failed" , t); - } - } - - @Override - protected void endTransaction() throws FHIRPersistenceException { - boolean committed = false; - try { - if (!this.rollbackOnly) { - logger.fine("Committing transaction"); - connection.commit(); - committed = true; - - // any values from parameter_names, code_systems and common_token_values - // are now committed to the database, so we can publish their record ids - // to the shared cache which makes them accessible from other threads - publishValuesToCache(); - } else { - // something went wrong...try to roll back the transaction before we close - // everything - try { - connection.rollback(); - } catch (SQLException x) { - // It could very well be that we've lost touch with the database in which case - // the rollback will also fail. Not much we can do, although we don't bother - // with a stack trace here because it's just more noise for the log file, and - // the exception that triggered the rollback is already going to be propagated - // and logged. - logger.severe("Rollback failed; reason=[" + x.getMessage() + "]"); - } - } - } catch (SQLException x) { - throw new FHIRPersistenceException("commit failed", x); - } finally { - // always clear these maps because otherwise they could grow unbounded. Values - // are cached by the identityCache - this.logicalResourceIdentMap.clear(); - this.parameterNameMap.clear(); - this.codeSystemValueMap.clear(); - this.commonTokenValueMap.clear(); - this.commonCanonicalValueMap.clear(); - } - } - - /** - * After the transaction has been committed, we can publish certain values to the - * shared identity caches allowing them to be used by other threads - */ - private void publishValuesToCache() { - for (ParameterNameValue pnv: this.unresolvedParameterNames) { - logger.fine(() -> "Adding parameter-name to cache: '" + pnv.getParameterName() + "' -> " + pnv.getParameterNameId()); - identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId()); - } - - for (CommonCanonicalValue value: this.unresolvedCanonicalValues) { - identityCache.addCommonCanonicalValue(FIXED_SHARD, value.getUrl(), value.getCanonicalId()); - } - - for (CodeSystemValue value: this.unresolvedSystemValues) { - identityCache.addCodeSystem(value.getCodeSystem(), value.getCodeSystemId()); - } - - for (CommonTokenValue value: this.unresolvedTokenValues) { - identityCache.addCommonTokenValue(FIXED_SHARD, value.getCodeSystemValue().getCodeSystem(), value.getTokenValue(), value.getCommonTokenValueId()); - } - - for (LogicalResourceIdentValue value: this.unresolvedLogicalResourceIdents) { - identityCache.addLogicalResourceIdent(value.getResourceType(), value.getLogicalId(), value.getLogicalResourceId()); - } - } - - @Override - protected void pushBatch() throws FHIRPersistenceException { - // Push any data we've accumulated so far. This may occur - // if we cross a volume threshold, and will always occur as - // the last step before the current transaction is committed, - // Process the token values so that we can establish - // any entries we need for common_token_values - logger.fine("pushBatch: resolving all ids"); - resolveLogicalResourceIdents(); - resolveParameterNames(); - resolveCodeSystems(); - resolveCommonTokenValues(); - resolveCommonCanonicalValues(); - - // Now that all the lookup values should've been resolved, we can go ahead - // and push the parameters to the JDBC batch insert statements via the - // batchProcessor - logger.fine("pushBatch: processing collected parameters"); - for (BatchParameterValue v: this.batchedParameterValues) { - v.apply(batchProcessor); - } - - logger.fine("pushBatch: executing final batch statements"); - batchProcessor.pushBatch(); - logger.fine("pushBatch completed"); - } - - /** - * Get the parameter name value for the given parameter value - * @param p - * @return - */ - private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException { - if (logger.isLoggable(Level.FINEST)) { - logger.finest("get ParameterNameValue for [" + p.toString() + "]"); - } - ParameterNameValue result = parameterNameMap.get(p.getName()); - if (result == null) { - result = new ParameterNameValue(p.getName()); - parameterNameMap.put(p.getName(), result); - - // let's see if the id is available in the shared identity cache - Integer parameterNameId = identityCache.getParameterNameId(p.getName()); - if (parameterNameId != null) { - result.setParameterNameId(parameterNameId); - } else { - // ids will be created later (so that we can process them in order) - unresolvedParameterNames.add(result); - } - } - return result; - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException { - CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException { - CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException { - CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException { - CommonCanonicalValue ctv = lookupCommonCanonicalValue(p.getUrl()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem()); - this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException { - ParameterNameValue parameterNameValue = getParameterNameId(p); - this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p)); - } - - @Override - protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException { - logger.fine(() -> "Processing reference parameter value:" + p.toString()); - ParameterNameValue parameterNameValue = getParameterNameId(p); - LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId()); - this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv)); - } - - /** - * Get the CodeSystemValue we've assigned for the given codeSystem value. This - * may not yet have the actual code_system_id from the database yet - any values - * we don't have will be assigned in a later phase (so we can do things neatly - * in bulk). - * @param codeSystem - * @return - */ - private CodeSystemValue lookupCodeSystemValue(String codeSystem) { - CodeSystemValue result = this.codeSystemValueMap.get(codeSystem); - if (result == null) { - result = new CodeSystemValue(codeSystem); - this.codeSystemValueMap.put(codeSystem, result); - - // Take this opportunity to see if we have a cached value for this codeSystem - Integer codeSystemId = identityCache.getCodeSystemId(codeSystem); - if (codeSystemId != null) { - result.setCodeSystemId(codeSystemId); - } else { - // Stash for later resolution - this.unresolvedSystemValues.add(result); - } - } - return result; - } - - /** - * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple. - * The returned value may not yet have the actual common_token_value_id yet - we fetch - * these values later and create new database records as necessary. - * @param codeSystem - * @param tokenValue - * @return - */ - private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenValue) { - CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, codeSystem, tokenValue); - CommonTokenValue result = this.commonTokenValueMap.get(key); - if (result == null) { - CodeSystemValue csv = lookupCodeSystemValue(codeSystem); - result = new CommonTokenValue(FIXED_SHARD, csv, tokenValue); - this.commonTokenValueMap.put(key, result); - - // Take this opportunity to see if we have a cached value for this common token value - Long commonTokenValueId = identityCache.getCommonTokenValueId(FIXED_SHARD, codeSystem, tokenValue); - if (commonTokenValueId != null) { - result.setCommonTokenValueId(commonTokenValueId); - } else { - this.unresolvedTokenValues.add(result); - } - } - return result; - } - - /** - * Get the LogicalResourceIdentValue we've assigned for the given (resourceType, logicalId) - * tuple. The returned value may not yet have the actual logical_resource_id yet - we fetch - * these values later and create new database records as necessary - * @param resourceType - * @param logicalId - * @return - */ - private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) { - LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId); - LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key); - if (result == null) { - result = LogicalResourceIdentValue.builder() - .withResourceTypeId(identityCache.getResourceTypeId(resourceType)) - .withResourceType(resourceType) - .withLogicalId(logicalId) - .build(); - this.logicalResourceIdentMap.put(key, result); - - // see if we can find the logical_resource_id from the cache - Long logicalResourceId = identityCache.getLogicalResourceIdentId(resourceType, logicalId); - if (logicalResourceId != null) { - result.setLogicalResourceId(logicalResourceId); - } else { - // Add to the unresolved list to look up later - this.unresolvedLogicalResourceIdents.add(result); - } - } - return result; - } - - /** - * Get the CommonCanonicalValue we've assigned for the given url value. - * The returned value may not yet have the actual canonical_id yet - we fetch - * these values later and create new database records as necessary. - * @param url - * @return - */ - private CommonCanonicalValue lookupCommonCanonicalValue(String url) { - CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, url); - CommonCanonicalValue result = this.commonCanonicalValueMap.get(key); - if (result == null) { - result = new CommonCanonicalValue(FIXED_SHARD, url); - this.commonCanonicalValueMap.put(key, result); - - // Take this opportunity to see if we have a cached value for this common token value - Long canonicalId = identityCache.getCommonCanonicalValueId(FIXED_SHARD, url); - if (canonicalId != null) { - result.setCanonicalId(canonicalId); - } else { - this.unresolvedCanonicalValues.add(result); - } - } - return result; - } - - /** - * Make sure we have values for all the code_systems we have collected - * in the current - * batch - * @throws FHIRPersistenceException - */ - private void resolveCodeSystems() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCodeSystemIds(unresolvedSystemValues); - - if (!missing.isEmpty()) { - addMissingCodeSystems(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCodeSystemIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happend, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all code system values"); - } - } - - /** - * Build and prepare a statement to fetch the code_system_id and code_system_name - * from the code_systems table for all the given (unresolved) code system values - * @param values - * @return - * @throws SQLException - */ - private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN ("); - for (int i=0; i 0) { - query.append(","); - } - query.append("?"); - } - query.append(")"); - PreparedStatement ps = connection.prepareStatement(query.toString()); - // bind the parameter values - int param = 1; - for (CodeSystemValue csv: values) { - ps.setString(param++, csv.getCodeSystem()); - } - return ps; - } - - /** - * These code systems weren't found in the database, so we need to try and add them. - * We have to deal with concurrency here - there's a chance another thread could also - * be trying to add them. To avoid deadlocks, it's important to do any inserts in a - * consistent order. At the end, we should be able to read back values for each entry - * @param missing - */ - protected void addMissingCodeSystems(List missing) throws FHIRPersistenceException { - List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList()); - // Sort the code system values first to help avoid deadlocks - Collections.sort(values); // natural ordering for String is fine here - - final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence"); - StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES ("); - insert.append(nextVal); // next sequence value - insert.append(",?) ON CONFLICT DO NOTHING"); - - try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { - int count = 0; - for (String codeSystem: values) { - ps.setString(1, codeSystem); - ps.addBatch(); - if (++count == this.maxCodeSystemsPerStatement) { - // not too many statements in a single batch - ps.executeBatch(); - count = 0; - } - } - if (count > 0) { - // final batch - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x); - throw new FHIRPersistenceException("code systems fetch failed"); - } - } - - /** - * Fetch all the code_system_id values for the given list of CodeSystemValue objects. - * @param unresolved - * @return - * @throws FHIRPersistenceException - */ - private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) { - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2)); - if (csv != null) { - csv.setCodeSystemId(rs.getInt(1)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("code systems query returned an unexpected value"); - } - } - - // Most of the time we'll get everything, so we can bypass the check for - // missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CodeSystemValue csv: sub) { - if (csv.getCodeSystemId() == null) { - missing.add(csv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "code systems fetch failed", x); - throw new FHIRPersistenceException("code systems fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Make sure we have values for all the common_token_value records we have collected - * in the current batch - * @throws FHIRPersistenceException - */ - private void resolveCommonTokenValues() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCommonTokenValueIds(unresolvedTokenValues); - - if (!missing.isEmpty()) { - // Sort first to minimize deadlocks - Collections.sort(missing, (a,b) -> { - int result = a.getTokenValue().compareTo(b.getTokenValue()); - if (result == 0) { - result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId()); - if (result == 0) { - result = Short.compare(a.getShardKey(), b.getShardKey()); - } - } - return result; - }); - addMissingCommonTokenValues(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCommonTokenValueIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happend, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all common token values"); - } - } - - /** - * Build and prepare a statement to fetch the common_token_value records - * for all the given (unresolved) code system values - * @param values - * @return SELECT code_system, token_value, common_token_value_id - * @throws SQLException - */ - private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - // need the code_system name - so we join back to the code_systems table as well - query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id "); - query.append(" FROM common_token_values c"); - query.append(" JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)"); - query.append(" JOIN (VALUES "); - - // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records - boolean first = true; - for (CommonTokenValue ctv: values) { - if (first) { - first = false; - } else { - query.append(","); - } - query.append("("); - query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id - query.append(",?)"); // bind variable for the token-value - } - query.append(") AS v(code_system_id, token_value) "); - query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value)"); - - // Create the prepared statement and bind the values - final String statementText = query.toString(); - PreparedStatement ps = connection.prepareStatement(statementText); - - // bind the parameter values - int param = 1; - for (CommonTokenValue ctv: values) { - ps.setString(param++, ctv.getTokenValue()); - } - return new PreparedStatementWrapper(statementText, ps); - } - - /** - * Fetch the common_token_value_id values for the given list of CommonTokenValue objects. - * @param unresolved - * @return - * @throws FHIRPersistenceException - */ - private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - String sql = null; // the SQL text for logging when there's an error - try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) { - sql = ps.getStatementText(); - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, rs.getString(1), rs.getString(2)); - CommonTokenValue ctv = this.commonTokenValueMap.get(key); - if (ctv != null) { - ctv.setCommonTokenValueId(rs.getLong(3)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("common token values query returned an unexpected value"); - } - } - - // Optimize the check for missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CommonTokenValue ctv: sub) { - if (ctv.getCommonTokenValueId() == null) { - missing.add(ctv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x); - throw new FHIRPersistenceException("common token values fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Add the values we think are missing from the database. The given list should be - * sorted to reduce deadlocks - * @param missing - * @throws FHIRPersistenceException - */ - protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException { - - StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO common_token_values (code_system_id, token_value) "); - insert.append(" VALUES (?,?) "); - insert.append("ON CONFLICT DO NOTHING"); - - try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { - int count = 0; - for (CommonTokenValue ctv: missing) { - ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId()); - ps.setString(2, ctv.getTokenValue()); - ps.addBatch(); - if (++count == this.maxCommonTokenValuesPerStatement) { - // not too many statements in a single batch - ps.executeBatch(); - count = 0; - } - } - if (count > 0) { - // final batch - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "failed: " + insert.toString(), x); - throw new FHIRPersistenceException("failed inserting new common token values"); - } - } - - /** - * Make sure we have values for all the common_canonical_value records we have collected - * in the current batch - * @throws FHIRPersistenceException - */ - private void resolveCommonCanonicalValues() throws FHIRPersistenceException { - // identify which values aren't yet in the database - List missing = fetchCanonicalIds(unresolvedCanonicalValues); - - if (!missing.isEmpty()) { - // Sort on url to minimize deadlocks - Collections.sort(missing, (a,b) -> { - return a.getUrl().compareTo(b.getUrl()); - }); - addMissingCommonCanonicalValues(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - List bad = fetchCanonicalIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happen, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all canonical values"); - } - } - - /** - * Fetch the common_canonical_id values for the given list of CommonCanonicalValue objects. - * @param unresolved - * @return - * @throws FHIRPersistenceException - */ - private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - String sql = null; // the SQL text for logging when there's an error - try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) { - sql = ps.getStatementText(); - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each CodeSystemValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, rs.getString(1)); - CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key); - if (ctv != null) { - ctv.setCanonicalId(rs.getLong(2)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("common canonical values query returned an unexpected value"); - } - } - - // Optimize the check for missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (CommonCanonicalValue ctv: sub) { - if (ctv.getCanonicalId() == null) { - missing.add(ctv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x); - throw new FHIRPersistenceException("common canonical values fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } - - /** - * Build and prepare a statement to fetch the common_token_value records - * for all the given (unresolved) code system values - * @param values - * @return SELECT code_system, token_value, common_token_value_id - * @throws SQLException - */ - private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - query.append("SELECT c.url, c.canonical_id "); - query.append(" FROM common_canonical_values c "); - query.append(" WHERE c.url IN ("); - - // add bind variables for each url we need to fetch - boolean first = true; - for (CommonCanonicalValue ctv: values) { - if (first) { - first = false; - } else { - query.append(","); - } - query.append("?"); // bind variable for the url - } - query.append(")"); - - // Create the prepared statement and bind the values - final String statementText = query.toString(); - logger.finer(() -> "fetch common canonical values [" + statementText + "]"); - PreparedStatement ps = connection.prepareStatement(statementText); - - // bind the parameter values - int param = 1; - for (CommonCanonicalValue ctv: values) { - ps.setString(param++, ctv.getUrl()); - } - return new PreparedStatementWrapper(statementText, ps); - } - - /** - * Add the values we think are missing from the database. The given list should be - * sorted to reduce deadlocks - * @param missing - * @throws FHIRPersistenceException - */ - protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException { - - StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO common_canonical_values (url) VALUES (?) "); - insert.append("ON CONFLICT DO NOTHING"); - - final String DML = insert.toString(); - if (logger.isLoggable(Level.FINE)) { - logger.fine("addMissingCanonicalIds: " + DML); - } - try (PreparedStatement ps = connection.prepareStatement(DML)) { - int count = 0; - for (CommonCanonicalValue ctv: missing) { - logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]"); - ps.setString(1, ctv.getUrl()); - ps.addBatch(); - if (++count == this.maxCommonCanonicalValuesPerStatement) { - // not too many statements in a single batch - ps.executeBatch(); - count = 0; - } - } - if (count > 0) { - // final batch - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "failed: " + insert.toString(), x); - throw new FHIRPersistenceException("failed inserting new common canonical values"); - } - } - - /** - * Make sure all the parameter names we've seen in the batch exist - * in the database and have ids. - * @throws FHIRPersistenceException - */ - private void resolveParameterNames() throws FHIRPersistenceException { - // We expect parameter names to have a very high cache hit rate and - // so we simplify processing by simply iterating one-by-one for the - // values we still need to resolve. The most important point here is - // to do this in a sorted order to avoid deadlock issues because this - // could be happening across multiple consumer threads at the same time. - logger.fine("resolveParameterNames: sorting unresolved names"); - Collections.sort(this.unresolvedParameterNames, (a,b) -> { - return a.getParameterName().compareTo(b.getParameterName()); - }); - - try { - for (ParameterNameValue pnv: this.unresolvedParameterNames) { - logger.finer(() -> "fetching parameter_name_id for '" + pnv.getParameterName() + "'"); - Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName()); - if (parameterNameId == null) { - parameterNameId = createParameterName(pnv.getParameterName()); - if (logger.isLoggable(Level.FINER)) { - logger.finer("assigned parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); - } - - if (parameterNameId == null) { - // be defensive - throw new FHIRPersistenceException("parameter_name_id not assigned for '" + pnv.getParameterName()); - } - } else if (logger.isLoggable(Level.FINER)) { - logger.finer("read parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId); - } - pnv.setParameterNameId(parameterNameId); - } - } catch (SQLException x) { - throw new FHIRPersistenceException("error resolving parameter names", x); - } finally { - logger.exiting(CLASSNAME, "resolveParameterNames"); - } - } - - /** - * Fetch the parameter_name_id for the given parameterName value - * @param parameterName - * @return - * @throws SQLException - */ - private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException { - String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?"; - try (PreparedStatement ps = connection.prepareStatement(SQL)) { - ps.setString(1, parameterName); - ResultSet rs = ps.executeQuery(); - if (rs.next()) { - return rs.getInt(1); - } - } - - // no entry in parameter_names - return null; - } - - /** - * Create the parameter name using the stored procedure which handles any concurrency - * issue we may have - * @param parameterName - * @return - */ - private Integer createParameterName(String parameterName) throws SQLException { - final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}"; - Integer parameterNameId; - try (CallableStatement stmt = connection.prepareCall(CALL)) { - stmt.setString(1, parameterName); - stmt.registerOutParameter(2, Types.INTEGER); - stmt.execute(); - parameterNameId = stmt.getInt(2); - } - - return parameterNameId; - } - - @Override - protected void resetBatch() { - // Called when a transaction has been rolled back because of a deadlock - // or other retryable error and we want to try and process the batch again - batchProcessor.reset(); - } - - /** - * Build the check ready query - * @param messagesByResourceType - * @return - */ - private String buildCheckReadyQuery(Map> messagesByResourceType) { - // The trouble here is that we'll end up with a unique query for every single - // batch of messages we process (which the database then need to parse etc). - // This may introduce scaling issues, in which case we should consider - // individual queries for each resource type using bind variables, perhaps - // going so far as using multiple statements with a power-of-2 number of bind - // variables. But JDBC doesn't support batching of select statements, so - // the alternative there would be to insert-as-select into a global temp table - // and then simply select from that. Fairly straightforward, but a lot more - // work so only worth doing if we identify contention here. - - StringBuilder select = new StringBuilder(); - // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash - // FROM logical_resources AS lr, - // patient_logical_resources AS xlr - // WHERE lr.logical_resource_id = xlr.logical_resource_id - // AND xlr.logical_resource_id IN (1,2,3,4) - // UNION ALL - // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash - // FROM logical_resources AS lr, - // observation_logical_resources AS xlr - // WHERE lr.logical_resource_id = xlr.logical_resource_id - // AND xlr.logical_resource_id IN (5,6,7) - boolean first = true; - for (Map.Entry> entry: messagesByResourceType.entrySet()) { - final String resourceType = entry.getKey(); - final List messages = entry.getValue(); - final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(",")); - if (first) { - first = false; - } else { - select.append(" UNION ALL "); - } - select.append(" SELECT lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash "); - select.append(" FROM logical_resources AS lr, "); - select.append(resourceType).append("_logical_resources AS xlr "); - select.append(" WHERE lr.logical_resource_id = xlr.logical_resource_id "); - select.append(" AND xlr.logical_resource_id IN (").append(inlist).append(")"); - } - - return select.toString(); + super(new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs); } @Override - protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException { - // Get a list of all the resources for which we can see the current logical resource data. - // If the resource doesn't yet exist or its version meta doesn't the message - // then we add to the notReady list. If the resource version meta already - // exceeds the message, then we'll skip processing altogether because it - // means that there should be another message in the queue with more - // up-to-date parameters - Map messageMap = new HashMap<>(); - Map> messagesByResourceType = new HashMap<>(); - for (RemoteIndexMessage msg: messages) { - Long logicalResourceId = msg.getData().getLogicalResourceId(); - messageMap.put(logicalResourceId, msg); - - // split out the messages per resource type because we need to read from xx_logical_resources - List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>()); - values.add(msg); - } - - Set found = new HashSet<>(); - final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType); - logger.fine(() -> "check ready query: " + checkReadyQuery); - try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) { - ResultSet rs = ps.executeQuery(); - // wrap the ResultSet in a reader for easier consumption - ResultSetReader rsReader = new ResultSetReader(rs); - while (rsReader.next()) { - LogicalResourceValue lrv = LogicalResourceValue.builder() - .withLogicalResourceId(rsReader.getLong()) - .withResourceType(rsReader.getString()) - .withLogicalId(rsReader.getString()) - .withVersionId(rsReader.getInt()) - .withLastUpdated(rsReader.getTimestamp()) - .withParameterHash(rsReader.getString()) - .build(); - RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId()); - if (m == null) { - throw new IllegalStateException("query returned a logical resource which we didn't request"); - } - - // Check the values from the database to see if they match - // the information in the message. - if (m.getData().getVersionId() == lrv.getVersionId()) { - // only process this message if the parameter hash and lastUpdated - // times match - which is a good check that we're storing parameters - // from the correct transaction. If these don't match, we can simply - // say we found the data but don't need to process the message. - final Instant dbLastUpdated = lrv.getLastUpdated().toInstant(); - final Instant msgLastUpdated = m.getData().getLastUpdated(); - if (lrv.getParameterHash().equals(m.getData().getParameterHash()) - && dbLastUpdated.equals(msgLastUpdated)) { - okToProcess.add(m); - } else { - logger.warning("Parameter message must match both parameter_hash and last_updated. Must be from an uncommitted transaction so ignoring: " + m.toString()); - } - found.add(lrv.getLogicalResourceId()); // won't be marked as missing - } else if (m.getData().getVersionId() > lrv.getVersionId()) { - // we can skip processing this record because the database has already - // been updated with a newer version. Identify the record as having been - // found so we don't keep waiting for it - found.add(lrv.getLogicalResourceId()); - } - // if the version in the database is prior to version in the message we - // received it means that the server transaction hasn't been committed... - // so we have to wait just as though it were missing altogether - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x); - throw new FHIRPersistenceException("prepare query failed"); - } - - if (found.size() < messages.size()) { - // identify the missing records and add to the notReady list - for (RemoteIndexMessage m: messages) { - if (!found.contains(m.getData().getLogicalResourceId())) { - notReady.add(m); - } - } - } - } - - - - - /** - * Make sure we have values for all the logical_resource_ident values - * we have collected in the current batch. Need to make sure these are - * added in order to minimize deadlocks. Note that because we may create - * new logical_resource_ident records, we could be blocked by the main - * add_any_resource procedure run within the server CREATE/UPDATE - * transaction. - * @throws FHIRPersistenceException - */ - private void resolveLogicalResourceIdents() throws FHIRPersistenceException { - logger.fine("resolveLogicalResourceIdents: fetching ids for unresolved LogicalResourceIdent records"); - // identify which values aren't yet in the database - List missing = fetchLogicalResourceIdentIds(unresolvedLogicalResourceIdents); - - if (!missing.isEmpty()) { - logger.fine("resolveLogicalResourceIdents: add missing LogicalResourceIdent records"); - addMissingLogicalResourceIdents(missing); - } - - // All the previously missing values should now be in the database. We need to fetch them again, - // possibly having to use multiple queries - logger.fine("resolveLogicalResourceIdents: fetch ids for missing LogicalResourceIdent records"); - List bad = fetchLogicalResourceIdentIds(missing); - - if (!bad.isEmpty()) { - // shouldn't happen, but let's protected against it anyway - throw new FHIRPersistenceException("Failed to create all logical_resource_ident values"); - } - logger.fine("resolveLogicalResourceIdents: all resolved"); - } - - /** - * Build and prepare a statement to fetch the code_system_id and code_system_name - * from the code_systems table for all the given (unresolved) code system values - * @param values - * @return - * @throws SQLException - */ - private PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { - StringBuilder query = new StringBuilder(); - query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id "); - query.append(" FROM logical_resource_ident AS lri "); - query.append(" JOIN (VALUES "); - for (int i=0; i 0) { - query.append(","); - } - query.append("(?,?)"); - } - query.append(") AS v(resource_type_id, logical_id) "); - query.append(" ON (lri.resource_type_id = v.resource_type_id AND lri.logical_id = v.logical_id)"); - query.append(" JOIN resource_types AS rt ON (rt.resource_type_id = v.resource_type_id)"); // convenient to get the resource type name here - PreparedStatement ps = connection.prepareStatement(query.toString()); - // bind the parameter values - int param = 1; - for (LogicalResourceIdentValue val: values) { - ps.setInt(param++, val.getResourceTypeId()); - ps.setString(param++, val.getLogicalId()); - } - logger.fine(() -> "logicalResourceIdents: " + query.toString()); - return ps; - } - - /** - * These logical_resource_ident values weren't found in the database, so we need to try and add them. - * We have to deal with concurrency here - there's a chance another thread could also - * be trying to add them. To avoid deadlocks, it's important to do any inserts in a - * consistent order. At the end, we should be able to read back values for each entry - * @param missing - */ - protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { - // Sort the values first to help avoid deadlocks - Collections.sort(missing, (a,b) -> { - return a.compareTo(b); - }); - - final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); - StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); - insert.append(nextVal); // next sequence value - insert.append(") ON CONFLICT DO NOTHING"); - - try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { - int count = 0; - for (LogicalResourceIdentValue value: missing) { - if (value.getResourceTypeId() == null) { - logger.severe("bad value: " + value); - } - ps.setInt(1, value.getResourceTypeId()); - ps.setString(2, value.getLogicalId()); - ps.addBatch(); - if (++count == this.maxLogicalResourcesPerStatement) { - // not too many statements in a single batch - ps.executeBatch(); - count = 0; - } - } - if (count > 0) { - // final batch - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); - throw new FHIRPersistenceException("logical_resource_ident insert failed"); - } + protected String onConflict() { + return "ON CONFLICT DO NOTHING"; } - /** - * Fetch logical_resource_id values for the given list of LogicalResourceIdent objects. - * @param unresolved - * @return - * @throws FHIRPersistenceException - */ - private List fetchLogicalResourceIdentIds(List unresolved) throws FHIRPersistenceException { - // track which values aren't yet in the database - List missing = new ArrayList<>(); - - int offset = 0; - while (offset < unresolved.size()) { - int remaining = unresolved.size() - offset; - int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement); - List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive - offset += subSize; // set up for the next iteration - try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) { - ResultSet rs = ps.executeQuery(); - // We can't rely on the order of result rows matching the order of the in-list, - // so we have to go back to our map to look up each LogicalResourceIdentValue - int resultCount = 0; - while (rs.next()) { - resultCount++; - LogicalResourceIdentKey key = new LogicalResourceIdentKey(rs.getString(1), rs.getString(2)); - LogicalResourceIdentValue csv = this.logicalResourceIdentMap.get(key); - if (csv != null) { - csv.setLogicalResourceId(rs.getLong(3)); - } else { - // can't really happen, but be defensive - throw new FHIRPersistenceException("logical resource ident query returned an unexpected value"); - } - } - - // Most of the time we'll get everything, so we can bypass the check for - // missing values - if (resultCount == 0) { - // 100% miss - missing.addAll(sub); - } else if (resultCount < subSize) { - // need to scan the sub list and see which values we don't yet have ids for - for (LogicalResourceIdentValue csv: sub) { - if (csv.getLogicalResourceId() == null) { - missing.add(csv); - } - } - } - } catch (SQLException x) { - logger.log(Level.SEVERE, "logical resource ident fetch failed", x); - throw new FHIRPersistenceException("logical resource ident fetch failed"); - } - } - - // Return the list of CodeSystemValues which don't yet have a database entry - return missing; - } -} \ No newline at end of file +} diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java index 1d6f7e56b05..8b00326b21d 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java @@ -463,7 +463,7 @@ public void addProfile(long logicalResourceId, long canonicalId, String version, * @throws SQLException */ public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException { - if (tags == null) { + if (security == null) { final String tablePrefix = resourceType.toLowerCase(); final String INS = "INSERT INTO " + tablePrefix + "_security (common_token_value_id, logical_resource_id) VALUES (?,?)"; security = connection.prepareStatement(INS); diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java index 3e9757c1efb..8cbb53e8458 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java @@ -182,7 +182,7 @@ public void addString(long logicalResourceId, int parameterNameId, String strVal */ public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId) throws SQLException { if (systemDates == null) { - final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?,?)"; + final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)"; systemDates = connection.prepareStatement(insertSystemDate); } final Calendar UTC = CalendarHelper.getCalendarForUTC(); @@ -190,7 +190,6 @@ public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateS systemDates.setTimestamp(2, dateStart, UTC); systemDates.setTimestamp(3, dateEnd, UTC); systemDates.setLong(4, logicalResourceId); - setComposite(systemDates, 5, compositeId); systemDates.addBatch(); systemDateCount++; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java index 6953a73f0a6..25bfac4e4f9 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java @@ -61,13 +61,13 @@ import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue; import com.ibm.fhir.remote.index.database.LogicalResourceValue; import com.ibm.fhir.remote.index.database.ParameterNameValue; -import com.ibm.fhir.remote.index.database.PlainPostgresMessageHandler; +import com.ibm.fhir.remote.index.database.PlainMessageHandler; import com.ibm.fhir.remote.index.database.PreparedStatementWrapper; /** * Loads search parameter values into the target FHIR schema on * a PostgreSQL database. - * TODO refactor to try and share more processing with the {@link PlainPostgresMessageHandler} + * TODO refactor to try and share more processing with the {@link PlainMessageHandler} */ public class ShardedPostgresMessageHandler extends BaseMessageHandler { private static final Logger logger = Logger.getLogger(ShardedPostgresMessageHandler.class.getName()); diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java new file mode 100644 index 00000000000..602a328c118 --- /dev/null +++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java @@ -0,0 +1,151 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; +import com.ibm.fhir.database.utils.derby.DerbyMaster; +import com.ibm.fhir.database.utils.derby.DerbyPropertyAdapter; +import com.ibm.fhir.database.utils.derby.DerbyTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.schema.derby.DerbyFhirDatabase; + +/** + * Initialize a FHIR database schema in Derby for use by the remote index tests + */ +public class DerbyFhirFactory { + // All tests use this same database, which we only have to bootstrap once + public static final String DB_NAME = "target/derby/fhirDB"; + + // The translator to help us out with Derby urls/syntax/exceptions + private static final IDatabaseTranslator DERBY_TRANSLATOR = new DerbyTranslator(); + + private Properties dbProps; + + // When not null, restricts the set of resource types we create tables for + private Set resourceTypeNames; + + /** + * Constructs a new DerbyInitializer using default database properties. + */ + public DerbyFhirFactory(Set resourceTypeNames) { + this.dbProps = new Properties(); + if (resourceTypeNames != null) { + this.resourceTypeNames = new HashSet<>(resourceTypeNames); + } else { + this.resourceTypeNames = null; + } + } + + /** + * Constructs a new DerbyInitializer using the passed database properties. + */ + public DerbyFhirFactory(Properties props, Set resourceTypeNames) { + this.dbProps = props; + if (resourceTypeNames != null) { + this.resourceTypeNames = new HashSet<>(resourceTypeNames); + } else { + this.resourceTypeNames = null; + } + } + + /** + * Default bootstrap of the database. Does not drop/rebuild. + * + * @return a DerbyFhirDatabase instance that represents the create database if one was created or null if it already exists + * @throws FHIRPersistenceDBConnectException + * @throws SQLException + */ + public DerbyFhirDatabase bootstrapDb() throws FHIRPersistenceException, SQLException { + return bootstrapDb(false); + } + + /** + * Tests for the existence of fhirDB and creates the database if necessary, complete with tables and indices. + * + * @param reset + * Whether to "reset" the database by deleting the existing one before attempting the create + * @return a DerbyFhirDatabase object if one was created or null if it already exists and {@code reset} is false + * @throws FHIRPersistenceDBConnectException + * @throws SQLException + */ + public DerbyFhirDatabase bootstrapDb(boolean reset) throws FHIRPersistenceException, SQLException { + if (reset) { + // wipes the disk content of the database. Hopefully there aren't any + // open connections at this point + DerbyMaster.dropDatabase(DB_NAME); + } + + // Inject the DB_NAME into the dbProps + DerbyPropertyAdapter adapter = new DerbyPropertyAdapter(dbProps); + adapter.setDatabase(DB_NAME); + + // Only bootstrap the database if it is new + boolean exists; + try (Connection connection = getConnection()) { + exists = true; + } catch (SQLException x) { + exists = false; + } + + if (exists) { + System.out.println("Existing database: skipping bootstrap"); + return null; + } else { + System.out.println("Bootstrapping database"); + if (resourceTypeNames != null) { + return new DerbyFhirDatabase(DB_NAME, resourceTypeNames); + } else { + return new DerbyFhirDatabase(DB_NAME); + } + } + } + + /** + * Get the name of the schema holding all the FHIR resource tables. + */ + protected String getDataSchemaName() { + return dbProps.getProperty("schemaName", "FHIRDATA"); + } + + /** + * Get a connection to an established database. + * Autocommit is disabled (of course). + */ + public Connection getConnection() throws SQLException { + Connection connection = DriverManager.getConnection(DERBY_TRANSLATOR.getUrl(dbProps)); + connection.setAutoCommit(false); + return connection; + } + + /** + * Bootstrap the database if necessary, and get a connection provider for it + * @return an {@link IConnectionProvider} configured for the FHIR Derby database + * @param reset resets the database if true + * @throws SQLException + * @throws FHIRPersistenceDBConnectException + */ + public IConnectionProvider getConnectionProvider(boolean reset) throws FHIRPersistenceException, SQLException { + bootstrapDb(reset); + JdbcPropertyAdapter propAdapter = new JdbcPropertyAdapter(this.dbProps); + + // make sure the schema name is correctly set in the properties + String fhirDataSchema = getDataSchemaName(); + propAdapter.setDefaultSchema(fhirDataSchema); + + return new JdbcConnectionProvider(new DerbyTranslator(), propAdapter); + } +} diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java new file mode 100644 index 00000000000..e5edafb2248 --- /dev/null +++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java @@ -0,0 +1,223 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.remote.index; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import com.google.gson.Gson; +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.database.utils.derby.DerbyTranslator; +import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.persistence.index.RemoteIndexConstants; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; +import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter; +import com.ibm.fhir.remote.index.cache.IdentityCacheImpl; +import com.ibm.fhir.remote.index.database.CacheLoader; +import com.ibm.fhir.remote.index.database.PlainDerbyMessageHandler; + +/** + * + */ +public class RemoteIndexTest { + private Properties testProps; + + private IConnectionProvider connectionProvider; + private String[] TEST_RESOURCE_TYPES = {"Patient", "Observation" }; + private IdentityCacheImpl identityCache; + private static final String SCHEMA_NAME = "FHIRDATA"; + private static final IDatabaseTranslator translator = new DerbyTranslator(); + + private final String OBSERVATION = "Observation"; + private final String OBSERVATION_LOGICAL_ID = UUID.randomUUID().toString(); + private final int versionId = 1; + private final Instant lastUpdated = Instant.now(); + private final String requestShard = null; + private final String parameterHash = "1Z+NWYZb739Ava9Pd/d7wt2xecKmC2FkfLlCCml0I5M="; + private final Instant ts1 = lastUpdated.plusMillis(1000); + private final Instant ts2 = lastUpdated.plusMillis(2000); + private final BigDecimal valueNumber = BigDecimal.valueOf(1.0); + private final BigDecimal valueNumberLow = BigDecimal.valueOf(0.5); + private final BigDecimal valueNumberHigh = BigDecimal.valueOf(1.5); + private final String valueSystem = "system1"; + private final String valueCode = "code1"; + private final String refResourceType = "Patient"; + private final String refLogicalId = "pat1"; + private final Integer refVersion = 2; + private final boolean wholeSystem = false; + private final Integer compositeId = null; + private final String valueString = "str1"; + private final String url = "http://some.profile/location"; + private final String profileVersion = "1.0"; + + @BeforeClass + public void bootstrapDatabase() throws Exception { + final Set resourceTypeNames = Set.of(TEST_RESOURCE_TYPES); + this.testProps = TestUtil.readTestProperties("test-remote-index.properties"); + DerbyFhirFactory derbyInit; + String dbDriverName = this.testProps.getProperty("dbDriverName"); + if (dbDriverName == null || !dbDriverName.contains("derby")) { + throw new IllegalStateException("test properties missing derby driver configuration"); + } + + derbyInit = new DerbyFhirFactory(this.testProps, resourceTypeNames); + this.connectionProvider = derbyInit.getConnectionProvider(false); + Duration cacheDuration = Duration.ofDays(1); + this.identityCache = new IdentityCacheImpl( + 100, cacheDuration, // code systems + 100, cacheDuration, // common token values + 100, cacheDuration, // canonical values + 100, cacheDuration); // logical resource idents + + // Preload the cache so we have all the resource types available + try (Connection c = connectionProvider.getConnection()) { + CacheLoader cacheLoader = new CacheLoader(identityCache); + cacheLoader.apply(c); + c.commit(); + } + } + + /** + * Get a list of messages to process + * @return + */ + private List getMessages(long logicalResourceId) { + RemoteIndexMessage sent = new RemoteIndexMessage(); + sent.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); + + // Create an Observation resource with a few parameters + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(OBSERVATION, OBSERVATION_LOGICAL_ID, logicalResourceId, + versionId, lastUpdated, requestShard, parameterHash); + adapter.dateValue("date-param", ts1, ts2, null, true); + adapter.locationValue("location-param", 0.1, 0.2, null); + adapter.numberValue("number-param", valueNumber, valueNumberLow, valueNumberHigh, null); + adapter.profileValue("profile-param", url, profileVersion, null, true); + adapter.quantityValue("quantity-param", valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh, compositeId); + adapter.referenceValue("reference-param", refResourceType, refLogicalId, refVersion, compositeId); + adapter.securityValue("security-param", valueSystem, valueCode, wholeSystem); + adapter.stringValue("string-param", valueString, compositeId, wholeSystem); + adapter.tagValue("tag-param", valueSystem, valueCode, wholeSystem); + adapter.tokenValue("token-param", valueSystem, valueCode, compositeId); + + sent.setData(adapter.build()); + final String payload = marshallToString(sent); + + final List result = new ArrayList<>(); + result.add(payload); + return result; + } + + /** + * Marshall the message to a string + * @param message + * @return + */ + private String marshallToString(RemoteIndexMessage message) { + final Gson gson = new Gson(); + return gson.toJson(message); + } + + @Test + public void testFill() throws Exception { + final long logicalResourceId; + try (Connection c = connectionProvider.getConnection()) { + logicalResourceId = addObservationLogicalResource(c, OBSERVATION_LOGICAL_ID); + c.commit(); + } + + try (Connection c = connectionProvider.getConnection()) { + PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(c, SCHEMA_NAME, identityCache, 1000L); + handler.process(getMessages(logicalResourceId)); + checkData(c, logicalResourceId); + } + } + + /** + * Inject the logical_resource_ident, logical_resources and observation_logical_resources + * record as we would normally see added by the FHIR server. We're not dealing with any + * concurrency here, so we just use 3 simple inserts. + * @param c + * @throws SQLException + */ + private long addObservationLogicalResource(Connection c, String logicalId) throws SQLException { + final int resourceTypeId = identityCache.getResourceTypeId(OBSERVATION); + final String getNextLogicalId = translator.selectSequenceNextValue(SCHEMA_NAME, "fhir_sequence"); + long logicalResourceId; + try (Statement s = c.createStatement()) { + ResultSet rs = s.executeQuery(getNextLogicalId); + if (rs.next()) { + logicalResourceId = rs.getLong(1); + } else { + throw new IllegalStateException("no row from '" + getNextLogicalId + "'"); + } + } + + final String insertIdent = "INSERT INTO logical_resource_ident(logical_resource_id, resource_type_id, logical_id) VALUES (?, ?, ?)"; + try (PreparedStatement ps = c.prepareStatement(insertIdent)) { + ps.setLong(1, logicalResourceId); + ps.setInt(2, resourceTypeId); + ps.setString(3, logicalId); + ps.executeUpdate(); + } + + final Timestamp lastUpdated = Timestamp.from(this.lastUpdated); + final String insertLogicalResource = "INSERT INTO logical_resources(logical_resource_id, resource_type_id, logical_id, last_updated, is_deleted, parameter_hash)" + + " VALUES (?,?,?,?,?,?)"; + try (PreparedStatement ps = c.prepareStatement(insertLogicalResource)) { + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + + psh.setLong(logicalResourceId) + .setInt(resourceTypeId) + .setString(logicalId) + .setTimestamp(lastUpdated) + .setString("N") + .setString(parameterHash); + ps.executeUpdate(); + } + + final String insertObservationLogicalResource = "INSERT INTO observation_logical_resources(logical_resource_id, logical_id, is_deleted, last_updated, version_id)" + + " VALUES (?,?,?,?,?)"; + try (PreparedStatement ps = c.prepareStatement(insertObservationLogicalResource)) { + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + + psh.setLong(logicalResourceId) + .setString(logicalId) + .setString("N") + .setTimestamp(lastUpdated) + .setInt(this.versionId); + ps.executeUpdate(); + } + + return logicalResourceId; + } + /** + * Check that the data in the processed messages now exists in + * the database + * @param c + * @throws Exception + */ + private void checkData(Connection c, long logicalResourceId) throws Exception { + + } +} diff --git a/fhir-remote-index/src/test/resources/test-remote-index.properties b/fhir-remote-index/src/test/resources/test-remote-index.properties new file mode 100644 index 00000000000..1cd33b12b59 --- /dev/null +++ b/fhir-remote-index/src/test/resources/test-remote-index.properties @@ -0,0 +1,31 @@ +# Properties for TestNG tests in the fhir-remote-index project +# Default datastore is a Derby database using the embedded Derby driver. +# Note that the dbUser and dbPassword properties are not used for Derby. + +#Derby properties +dbDriverName = org.apache.derby.jdbc.EmbeddedDriver +dbUrl = jdbc:derby:target/derby/fhirDB +schemaName = FHIRDATA + +#Derby network properties +#dbDriverName = org.apache.derby.jdbc.ClientXADataSource +#dbUrl=jdbc:derby://localhost:1527/fhirdb +#schemaName = FHIRDATA + +#Db2 properties +#dbDriverName = com.ibm.db2.jcc.DB2Driver +#dbUrl = jdbc:db2://localhost:50000/fhirdb +#user = db2inst1 +#password = change-password +#schemaName = FHIRDATA + +#PostgreSql properties +#dbDriverName = org.postgresql.Driver +#dbUrl = jdbc:postgresql://localhost:5432/fhirdb +#user = postgre +#password = change-password +#PostgreSql use lower case by default +#schemaName = fhirdata + +#common properties +updateCreateEnabled = true From f44b4cb67fc662e9074c0fb29054df275dea018a Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Sat, 4 Jun 2022 11:21:14 +0100 Subject: [PATCH 19/40] issue #3437 more unit test coverage for fhir-remote-index Signed-off-by: Robin Arnold --- .../utils/common/ResultSetReader.java | 27 ++ .../PlainBatchParameterProcessor.java | 3 +- .../PlainPostgresSystemParameterBatch.java | 22 +- .../fhir/remote/index/RemoteIndexTest.java | 444 +++++++++++++++++- 4 files changed, 461 insertions(+), 35 deletions(-) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java index ec847754aa3..f74901ef08a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java @@ -6,6 +6,7 @@ package com.ibm.fhir.database.utils.common; +import java.math.BigDecimal; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -88,6 +89,32 @@ public Long getLong() throws SQLException { return result; } + /** + * Get a BigDecimal column value and increment the column index + * @return + * @throws SQLException + */ + public BigDecimal getBigDecimal() throws SQLException { + BigDecimal result = rs.getBigDecimal(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Double column value and increment the column index + * @return + * @throws SQLException + */ + public Double getDouble() throws SQLException { + Double result = rs.getDouble(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + /** * Get a Timestamp column value and increment the column index * @return diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java index c1f65e0317a..33724b67d5e 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java @@ -152,9 +152,10 @@ public void process(String requestShard, String resourceType, String logicalId, dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId()); if (parameter.isSystemParam()) { - systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId()); + systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase()); } } catch (SQLException x) { + logger.log(Level.SEVERE, "StringParameter", x); throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'"); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java index 8cbb53e8458..65e7f4b94eb 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java @@ -10,7 +10,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; -import java.sql.Types; import java.util.Calendar; import com.ibm.fhir.database.utils.common.CalendarHelper; @@ -131,21 +130,6 @@ public void close() { } } - /** - * Set the compositeId on the given PreparedStatement, handling a value if necessary - * @param ps - * @param index - * @param compositeId - * @throws SQLException - */ - private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException { - if (compositeId != null) { - ps.setInt(index, compositeId); - } else { - ps.setNull(index, Types.INTEGER); - } - } - /** * Add a string parameter value to the whole-system batch statement * @@ -153,10 +137,9 @@ private void setComposite(PreparedStatement ps, int index, Integer compositeId) * @param parameterNameId * @param strValue * @param strValueLower - * @param compositeId * @throws SQLException */ - public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId) throws SQLException { + public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower) throws SQLException { // System level string attributes if (systemStrings == null) { final String insertSystemString = "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)"; @@ -166,7 +149,6 @@ public void addString(long logicalResourceId, int parameterNameId, String strVal systemStrings.setString(2, strValue); systemStrings.setString(3, strValueLower); systemStrings.setLong(4, logicalResourceId); - setComposite(systemStrings, 5, compositeId); systemStrings.addBatch(); systemStringCount++; } @@ -242,7 +224,7 @@ public void addProfile(long logicalResourceId, long canonicalId, String version, * @throws SQLException */ public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException { - if (systemTags == null) { + if (systemSecurity == null) { final String INS = "INSERT INTO logical_resource_security(common_token_value_id, logical_resource_id) VALUES (?,?)"; systemSecurity = connection.prepareStatement(INS); } diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java index e5edafb2248..e9f5a73c788 100644 --- a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java +++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java @@ -6,6 +6,8 @@ package com.ibm.fhir.remote.index; +import static org.testng.Assert.assertEquals; + import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; @@ -20,6 +22,7 @@ import java.util.Properties; import java.util.Set; import java.util.UUID; +import java.util.logging.Logger; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -28,8 +31,10 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.database.utils.common.ResultSetReader; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.index.RemoteIndexConstants; import com.ibm.fhir.persistence.index.RemoteIndexMessage; import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter; @@ -38,18 +43,20 @@ import com.ibm.fhir.remote.index.database.PlainDerbyMessageHandler; /** - * + * Unit test for remote index message handling and database processing */ public class RemoteIndexTest { + private static final Logger logger = Logger.getLogger(RemoteIndexTest.class.getName()); private Properties testProps; private IConnectionProvider connectionProvider; private String[] TEST_RESOURCE_TYPES = {"Patient", "Observation" }; private IdentityCacheImpl identityCache; private static final String SCHEMA_NAME = "FHIRDATA"; + private static final boolean WHOLE_SYSTEM = true; private static final IDatabaseTranslator translator = new DerbyTranslator(); + private static final String OBSERVATION = "Observation"; - private final String OBSERVATION = "Observation"; private final String OBSERVATION_LOGICAL_ID = UUID.randomUUID().toString(); private final int versionId = 1; private final Instant lastUpdated = Instant.now(); @@ -65,7 +72,6 @@ public class RemoteIndexTest { private final String refResourceType = "Patient"; private final String refLogicalId = "pat1"; private final Integer refVersion = 2; - private final boolean wholeSystem = false; private final Integer compositeId = null; private final String valueString = "str1"; private final String url = "http://some.profile/location"; @@ -109,16 +115,16 @@ private List getMessages(long logicalResourceId) { // Create an Observation resource with a few parameters SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(OBSERVATION, OBSERVATION_LOGICAL_ID, logicalResourceId, versionId, lastUpdated, requestShard, parameterHash); - adapter.dateValue("date-param", ts1, ts2, null, true); - adapter.locationValue("location-param", 0.1, 0.2, null); + adapter.stringValue("string-param", valueString, compositeId, WHOLE_SYSTEM); + adapter.dateValue("date-param", ts1, ts2, null, WHOLE_SYSTEM); adapter.numberValue("number-param", valueNumber, valueNumberLow, valueNumberHigh, null); - adapter.profileValue("profile-param", url, profileVersion, null, true); adapter.quantityValue("quantity-param", valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh, compositeId); - adapter.referenceValue("reference-param", refResourceType, refLogicalId, refVersion, compositeId); - adapter.securityValue("security-param", valueSystem, valueCode, wholeSystem); - adapter.stringValue("string-param", valueString, compositeId, wholeSystem); - adapter.tagValue("tag-param", valueSystem, valueCode, wholeSystem); adapter.tokenValue("token-param", valueSystem, valueCode, compositeId); + adapter.locationValue("location-param", 0.1, 0.2, null); + adapter.referenceValue("reference-param", refResourceType, refLogicalId, refVersion, compositeId); + adapter.securityValue("security-param", valueSystem, valueCode, WHOLE_SYSTEM); + adapter.profileValue("profile-param", url, profileVersion, null, WHOLE_SYSTEM); + adapter.tagValue("tag-param", valueSystem, valueCode, WHOLE_SYSTEM); sent.setData(adapter.build()); final String payload = marshallToString(sent); @@ -147,12 +153,29 @@ public void testFill() throws Exception { } try (Connection c = connectionProvider.getConnection()) { - PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(c, SCHEMA_NAME, identityCache, 1000L); - handler.process(getMessages(logicalResourceId)); - checkData(c, logicalResourceId); + try { + PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(c, SCHEMA_NAME, identityCache, 1000L); + handler.process(getMessages(logicalResourceId)); + checkData(c, logicalResourceId); + c.commit(); + } catch (Throwable t) { + safeRollback(c); + throw t; + } } } + /** + * Try and rollback the transaction, squashing any exception + * @param c + */ + private void safeRollback(Connection c) { + try { + c.rollback(); + } catch (SQLException x) { + logger.warning("rollback failed: " + x.getMessage()); + } + } /** * Inject the logical_resource_ident, logical_resources and observation_logical_resources * record as we would normally see added by the FHIR server. We're not dealing with any @@ -218,6 +241,399 @@ private long addObservationLogicalResource(Connection c, String logicalId) throw * @throws Exception */ private void checkData(Connection c, long logicalResourceId) throws Exception { + // check the resource level parameters + checkStringParam(c, OBSERVATION, logicalResourceId, valueString); + checkDateParam(c, OBSERVATION, logicalResourceId, ts1, ts2); + checkNumberParam(c, OBSERVATION, logicalResourceId, valueNumber, valueNumberLow, valueNumberHigh); + checkLocationParam(c, OBSERVATION, logicalResourceId, 0.1, 0.2); + checkProfileParam(c, OBSERVATION, logicalResourceId, url, profileVersion); + checkQuantityParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh); + checkReferenceParam(c, OBSERVATION, logicalResourceId, refResourceType, refLogicalId); + checkTagParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode); + checkSecurityParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode); + checkTokenParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode); + + // check the whole-system level parameters + checkStringSystemParam(c, OBSERVATION, logicalResourceId, valueString); + checkDateSystemParam(c, OBSERVATION, logicalResourceId, ts1, ts2); + checkProfileSystemParam(c, OBSERVATION, logicalResourceId, url, profileVersion); + checkTagSystemParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode); + checkSecuritySystemParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode); + } + + /** + * @param c + * @param resourceType + * @param logicalResourceId + * @param valueSystem + * @param valueCode + * @param valueNumber + * @param valueNumberLow + * @param valueNumberHigh + */ + private void checkQuantityParam(Connection c, String resourceType, long logicalResourceId, String valueSystem, String valueCode, BigDecimal valueNumber, + BigDecimal valueNumberLow, BigDecimal valueNumberHigh) throws Exception { + final String select = "" + + "SELECT c.code_system_name, p.code, p.quantity_value, p.quantity_value_low, p.quantity_value_high " + + " FROM " + resourceType + "_quantity_values p " + + " JOIN code_systems c ON c.code_system_id = p.code_system_id " + + " WHERE p.logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getString(), valueSystem); + assertEquals(rsr.getString(), valueCode); + assertEquals(rsr.getBigDecimal(), valueNumber); + assertEquals(rsr.getBigDecimal(), valueNumberLow); + assertEquals(rsr.getBigDecimal(), valueNumberHigh); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one quantity parameter"); + } + } } -} + + private void checkStringParam(Connection c, String resourceType, long logicalResourceId, String valueString) throws Exception { + final String select = "SELECT str_value FROM " + resourceType + "_str_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + assertEquals(rs.getString(1), valueString); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one string parameter"); + } + } + } + + private void checkStringSystemParam(Connection c, String resourceType, long logicalResourceId, String valueString) throws Exception { + final String select = "SELECT str_value FROM str_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + assertEquals(rs.getString(1), valueString); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one string parameter"); + } + } + } + + private void checkNumberParam(Connection c, String resourceType, long logicalResourceId, BigDecimal numberValue, + BigDecimal numberValueLow, BigDecimal numberValueHigh) throws Exception { + final String select = "SELECT number_value, number_value_low, number_value_high FROM " + resourceType + "_number_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + assertEquals(rs.getBigDecimal(1), numberValue); + assertEquals(rs.getBigDecimal(2), numberValueLow); + assertEquals(rs.getBigDecimal(3), numberValueHigh); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one number parameter"); + } + } + } + + private void checkDateParam(Connection c, String resourceType, long logicalResourceId, Instant dateStart, + Instant dateEnd) throws Exception { + final String select = "SELECT date_start, date_end FROM " + resourceType + "_date_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getTimestamp(), Timestamp.from(dateStart)); + assertEquals(rsr.getTimestamp(), Timestamp.from(dateEnd)); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one date parameter"); + } + } + } + + private void checkDateSystemParam(Connection c, String resourceType, long logicalResourceId, Instant dateStart, + Instant dateEnd) throws Exception { + final String select = "SELECT date_start, date_end FROM date_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getTimestamp(), Timestamp.from(dateStart)); + assertEquals(rsr.getTimestamp(), Timestamp.from(dateEnd)); + } else { + throw new FHIRPersistenceException("missing date system value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one date system parameter"); + } + } + } + + private void checkLocationParam(Connection c, String resourceType, long logicalResourceId, double latitude, + double longitude) throws Exception { + final String select = "SELECT latitude_value, longitude_value FROM " + resourceType + "_latlng_values WHERE logical_resource_id = ?"; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getDouble(), latitude); + assertEquals(rsr.getDouble(), longitude); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one date parameter"); + } + } + } + + private void checkProfileParam(Connection c, String resourceType, long logicalResourceId, String profile, String version) throws Exception { + final String select = "" + + "SELECT c.url, p.version FROM " + resourceType + "_profiles p" + + " JOIN common_canonical_values c ON c.canonical_id = p.canonical_id " + + " WHERE logical_resource_id = ?"; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getString(), profile); + assertEquals(rsr.getString(), version); + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one profile parameter"); + } + } + } + + private void checkProfileSystemParam(Connection c, String resourceType, long logicalResourceId, String profile, String version) throws Exception { + final String select = "" + + "SELECT c.url, p.version FROM logical_resource_profiles p" + + " JOIN common_canonical_values c ON c.canonical_id = p.canonical_id " + + " WHERE logical_resource_id = ?"; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + assertEquals(rsr.getString(), profile); + assertEquals(rsr.getString(), version); + } else { + throw new FHIRPersistenceException("missing profile system value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one profile system parameter"); + } + } + } + + private void checkSecurityParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { + final String select = "" + + "SELECT 1 FROM " + resourceType + "_security p" + + " JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id " + + " JOIN code_systems s ON s.code_system_id = c.code_system_id " + + " WHERE logical_resource_id = ? " + + " AND s.code_system_name = ? " + + " AND c.token_value = ? "; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, codeSystem); + ps.setString(3, tokenValue); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // OK + } else { + throw new FHIRPersistenceException("missing security value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one security parameter"); + } + } + } + + private void checkSecuritySystemParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { + final String select = "" + + "SELECT 1 FROM logical_resource_security p" + + " JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id " + + " JOIN code_systems s ON s.code_system_id = c.code_system_id " + + " WHERE logical_resource_id = ? " + + " AND s.code_system_name = ? " + + " AND c.token_value = ? "; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, codeSystem); + ps.setString(3, tokenValue); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // OK + } else { + throw new FHIRPersistenceException("missing security value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one security parameter"); + } + } + } + + private void checkTagParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { + final String select = "" + + "SELECT 1 FROM " + resourceType + "_tags p" + + " JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id " + + " JOIN code_systems s ON s.code_system_id = c.code_system_id " + + " WHERE logical_resource_id = ? " + + " AND s.code_system_name = ? " + + " AND c.token_value = ? "; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, codeSystem); + ps.setString(3, tokenValue); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // OK + } else { + throw new FHIRPersistenceException("missing tag value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one tag parameter"); + } + } + } + + private void checkTagSystemParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { + final String select = "" + + "SELECT 1 FROM logical_resource_tags p" + + " JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id " + + " JOIN code_systems s ON s.code_system_id = c.code_system_id " + + " WHERE logical_resource_id = ? " + + " AND s.code_system_name = ? " + + " AND c.token_value = ? "; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, codeSystem); + ps.setString(3, tokenValue); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // OK + } else { + throw new FHIRPersistenceException("missing tag value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one tag parameter"); + } + } + } + + private void checkTokenParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { + final String select = "" + + "SELECT 1 FROM " + resourceType + "_resource_token_refs p" + + " JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id " + + " JOIN code_systems s ON s.code_system_id = c.code_system_id " + + " WHERE logical_resource_id = ? " + + " AND s.code_system_name = ? " + + " AND c.token_value = ? "; + + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, codeSystem); + ps.setString(3, tokenValue); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // OK + } else { + throw new FHIRPersistenceException("missing token value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one token parameter"); + } + } + } + + private void checkReferenceParam(Connection c, String resourceType, long logicalResourceId, String refResourceType, + String refLogicalId) throws Exception { + final String select = "" + + "SELECT 1 FROM " + resourceType + "_ref_values p " + + " JOIN logical_resource_ident i ON i.logical_resource_id = p.ref_logical_resource_id " + + " JOIN resource_types rrt ON rrt.resource_type_id = i.resource_type_id " + + " WHERE p.logical_resource_id = ?" + + " AND rrt.resource_type = ? " + + " AND i.logical_id = ? "; + try (PreparedStatement ps = c.prepareStatement(select)) { + ps.setLong(1, logicalResourceId); + ps.setString(2, refResourceType); + ps.setString(3, refLogicalId); + ResultSet rs = ps.executeQuery(); + ResultSetReader rsr = new ResultSetReader(rs); + if (rsr.next()) { + // ok + } else { + throw new FHIRPersistenceException("missing value: " + select); + } + + if (rs.next()) { + // there can be only one + throw new FHIRPersistenceException("more than one date parameter"); + } + } + } + +} \ No newline at end of file From 4de35a569b38370463d123c93a850c2e0009eee6 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 6 Jun 2022 17:14:28 +0100 Subject: [PATCH 20/40] issue #3437 delete parameters from new xx_ref_values Signed-off-by: Robin Arnold --- .../jdbc/dao/impl/ResourceReferenceDAO.java | 2 +- .../jdbc/derby/DerbyResourceReferenceDAO.java | 34 +++++++++++++++++++ .../jdbc/util/ParameterTableSupport.java | 1 + .../db2/delete_resource_parameters.sql | 3 ++ .../postgres/delete_resource_parameters.sql | 2 ++ ...delete_resource_parameters_distributed.sql | 2 ++ .../delete_resource_parameters_sharded.sql | 2 ++ 7 files changed, 45 insertions(+), 1 deletion(-) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 44a60ac2936..5fbb483a51c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -1213,7 +1213,7 @@ protected void addMissingLogicalResourceIdents(List m } } - private void fetchLogicalResourceIdentIds(Map lrIdentMap, List unresolved) throws FHIRPersistenceException { + protected void fetchLogicalResourceIdentIds(Map lrIdentMap, List unresolved) throws FHIRPersistenceException { int resultCount = 0; final int maxValuesPerStatement = 512; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java index 153958ff210..ac49d32151c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java @@ -26,9 +26,11 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; @@ -329,4 +331,36 @@ protected PreparedStatement buildLogicalResourceIdentSelectStatement(List lrIdentMap, List unresolved) throws FHIRPersistenceException { + // For Derby, we opt to do this row by row so that we can keep the selects in order which + // helps us to avoid deadlocks due to lock compatibility issues with Derby + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" WHERE lri.resource_type_id = ? AND lri.logical_id = ?"); + final String sql = query.toString(); + try (PreparedStatement ps = getConnection().prepareStatement(sql)) { + for (LogicalResourceIdentValue value: unresolved) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + final long logicalResourceId = rs.getLong(1); + LogicalResourceIdentKey key = new LogicalResourceIdentKey(value.getResourceTypeId(), value.getLogicalId()); + value.setLogicalResourceId(logicalResourceId); + lrIdentMap.put(key, value); + } else { + // something wrong with our data handling code because we should already have values + // for every logical resource at this point + throw new FHIRPersistenceException("logical_resource_ident record missing: resourceTypeId[" + + value.getResourceTypeId() + "] logicalId[" + value.getLogicalId() + "]"); + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java index 639c0ef15fd..5324a390628 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java @@ -33,6 +33,7 @@ public static void deleteFromParameterTables(Connection conn, String tablePrefix deleteFromParameterTable(conn, tablePrefix + "_profiles", v_logical_resource_id); deleteFromParameterTable(conn, tablePrefix + "_tags", v_logical_resource_id); deleteFromParameterTable(conn, tablePrefix + "_security", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_ref_values", v_logical_resource_id); // delete any system level parameters we have for this resource deleteFromParameterTable(conn, "str_values", v_logical_resource_id); diff --git a/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql index 2ae2c86f58e..472f78810ad 100644 --- a/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql +++ b/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql @@ -45,6 +45,9 @@ BEGIN PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = ?'; EXECUTE d_stmt USING p_logical_resource_id; + PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = ?'; + EXECUTE d_stmt USING p_logical_resource_id; + PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'str_values WHERE logical_resource_id = ?'; EXECUTE d_stmt USING p_logical_resource_id; diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql index 496ade054a1..01ea9279092 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql @@ -40,6 +40,8 @@ BEGIN USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql index 496ade054a1..3724e0bf1c1 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql @@ -40,6 +40,8 @@ BEGIN USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql index abf1f6b50ac..e87822b7f58 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql @@ -45,6 +45,8 @@ BEGIN USING p_shard_key, p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' USING p_shard_key, p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' From 40e2454d96f278797d9c2aef150061ac9f28604d Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 8 Jun 2022 11:05:12 +0100 Subject: [PATCH 21/40] issue #3437 allow reindex to be forced after database migration Signed-off-by: Robin Arnold --- build/migration/bin/6_current-reindex.sh | 6 +++--- docs/src/pages/guides/FHIRSearchConfiguration.md | 1 + .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 9 +++++---- .../ibm/fhir/persistence/FHIRPersistence.java | 3 ++- .../persistence/test/MockPersistenceImpl.java | 2 +- .../spi/operation/FHIRResourceHelpers.java | 3 ++- .../com/ibm/fhir/server/util/FHIRRestHelper.java | 16 +++++++++------- .../fhir/server/test/MockPersistenceImpl.java | 2 +- .../server/test/ServerResolveFunctionTest.java | 3 ++- .../ibm/fhir/smart/test/MockPersistenceImpl.java | 2 +- .../erase/mock/MockFHIRResourceHelpers.java | 2 +- .../davinci/hrex/test/MemberMatchTest.java | 2 +- .../fhir/operation/reindex/ReindexOperation.java | 14 ++++++++++++-- .../src/main/resources/reindex.json | 10 +++++++++- 14 files changed, 50 insertions(+), 25 deletions(-) diff --git a/build/migration/bin/6_current-reindex.sh b/build/migration/bin/6_current-reindex.sh index 296abdd0a5e..07f18104c06 100644 --- a/build/migration/bin/6_current-reindex.sh +++ b/build/migration/bin/6_current-reindex.sh @@ -23,7 +23,7 @@ run_reindex(){ DATE_ISO=$(date +%Y-%m-%dT%H:%M:%SZ) status=$(curl -k -X POST -o reindex.json -i -w '%{http_code}' -u 'fhiruser:change-password' 'https://localhost:9443/fhir-server/api/v4/$reindex' \ -H 'Content-Type: application/fhir+json' -H 'X-FHIR-TENANT-ID: default' \ - -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"}]}") + -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"},{\"name\":\"force\",\"valueBoolean\":true}]}") echo "Status: ${status}" while [ $status -ne 200 ] @@ -57,7 +57,7 @@ run_reindex(){ fi status=$(curl -k -X POST -o reindex.json -i -w '%{http_code}' -u 'fhiruser:change-password' 'https://localhost:9443/fhir-server/api/v4/$reindex' \ -H 'Content-Type: application/fhir+json' -H 'X-FHIR-TENANT-ID: default' \ - -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"}]}") + -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"},{\"name\":\"force\",\"valueBoolean\":true}]}") echo "Status: ${status}" done fi @@ -77,4 +77,4 @@ run_reindex "${1}" popd > /dev/null # EOF -############################################################################### \ No newline at end of file +############################################################################### diff --git a/docs/src/pages/guides/FHIRSearchConfiguration.md b/docs/src/pages/guides/FHIRSearchConfiguration.md index 7df11c00eb3..0ee87f36e56 100644 --- a/docs/src/pages/guides/FHIRSearchConfiguration.md +++ b/docs/src/pages/guides/FHIRSearchConfiguration.md @@ -219,6 +219,7 @@ By default, the operation will select 10 resources and re-extract their search p |----|----|-----------| |`tstamp`|string|Reindex only resources not previously reindexed since this timestamp. Format as a date YYYY-MM-DD or time YYYY-MM-DDTHH:MM:SSZ.| |`resourceCount`|integer|The maximum number of resources to reindex in this call. If this number is too large, the processing time might exceed the transaction timeout and fail.| +|`force`|boolean|Force the parameters to be replaced even if the parameter hash matches. This is only required following a schema migration which changes how the parameters are stored in the database.| The IBM FHIR Server tracks when a resource was last reindexed and only resources with a reindex_tstamp value less than the given tstamp parameter will be processed. When a resource is reindexed, its reindex_tstamp is set to the given tstamp value. In most cases, using the current date (for example "2020-10-27") is the best option for this value. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 137265be225..80be6df1200 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -2607,7 +2607,7 @@ public boolean isOffloadingSupported() { @Override public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, boolean force) throws FHIRPersistenceException { final String METHODNAME = "reindex"; log.entering(CLASSNAME, METHODNAME); @@ -2679,7 +2679,7 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper rir.setDeleted(false); // just to be clear Class resourceTypeClass = getResourceType(rir.getResourceType()); reindexDAO.setPersistenceContext(context); - updateParameters(rir, resourceTypeClass, existingResourceDTO, reindexDAO, operationOutcomeResult); + updateParameters(rir, resourceTypeClass, existingResourceDTO, reindexDAO, operationOutcomeResult, force); // result is only 0 if getResourceToReindex doesn't give us anything because this indicates // there's nothing left to do @@ -2738,10 +2738,11 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper * @param existingResourceDTO the existing resource DTO * @param reindexDAO the reindex resource DAO * @param operationOutcomeResult the operation outcome result + * @param force * @throws Exception */ public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, - ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult) throws Exception { + ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult, boolean force) throws Exception { if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); @@ -2751,7 +2752,7 @@ public void updateParameters(ResourceIndexRecord rir, Class // Compare the hash of the extracted parameters with the hash in the index record. // If hash in the index record is not null and it matches the hash of the extracted parameters, then no need to replace the // extracted search parameters in the database tables for this resource, which helps with performance during reindex. - if (rir.getParameterHash() == null || !rir.getParameterHash().equals(searchParameters.getParameterHashB64())) { + if (force || rir.getParameterHash() == null || !rir.getParameterHash().equals(searchParameters.getParameterHashB64())) { reindexDAO.updateParameters(rir.getResourceType(), searchParameters.getParameters(), searchParameters.getParameterHashB64(), rir.getLogicalId(), rir.getLogicalResourceId()); } else { log.fine(() -> "Skipping update of unchanged parameters for FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java index 793bbe3e3bb..9ee3113c1cd 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java @@ -211,11 +211,12 @@ default boolean isOffloadingSupported() { * @param indexIds list of index IDs of resources to reindex, or null * @param resourceLogicalId resourceType/logicalId value of a specific resource to reindex, or null; * this parameter is ignored if the indexIds parameter value is non-null + * @param force if true, always replace the stored parameters * @return count of the number of resources reindexed by this call * @throws FHIRPersistenceException */ int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException; + String resourceLogicalId, boolean force) throws FHIRPersistenceException; /** * Special function for high speed export of resource payloads. The process diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java index b4e70c185b8..d0e8f59f05d 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java @@ -75,7 +75,7 @@ public OperationOutcome getHealth() throws FHIRPersistenceException { @Override public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, boolean force) throws FHIRPersistenceException { return 0; } diff --git a/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java b/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java index 9604f67d724..f5860263bec 100644 --- a/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java +++ b/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java @@ -490,11 +490,12 @@ Resource doInvoke(FHIROperationContext operationContext, String resourceTypeName * @param indexIds list of index IDs of resources to reindex, or null * @param resourceLogicalId resourceType (e.g. "Patient"), or resourceType/logicalId a specific resource (e.g. "Patient/abc123"), to reindex, or null; * this parameter is ignored if the indexIds parameter value is non-null + * @param force if true, ignore parameter hash and always replace the parameters * @return count of the number of resources reindexed by this call * @throws Exception */ int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds, - String resourceLogicalId) throws Exception; + String resourceLogicalId, boolean force) throws Exception; /** * Invoke the FHIR Persistence erase operation for a specific instance of the erase. diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java index 363a4f94fcc..dd55ae31b9e 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java @@ -2611,13 +2611,13 @@ private void setOperationContextProperties(FHIROperationContext operationContext @Override public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, - List indexIds, String resourceLogicalId) throws Exception { + List indexIds, String resourceLogicalId, boolean force) throws Exception { int result = 0; // Since the try logic is slightly different in the code paths, we want to dispatch to separate methods to simplify the logic. if (indexIds == null) { - result = doReindexSingle(operationOutcomeResult, tstamp, resourceLogicalId); + result = doReindexSingle(operationOutcomeResult, tstamp, resourceLogicalId, force); } else { - result = doReindexList(operationOutcomeResult, tstamp, indexIds); + result = doReindexList(operationOutcomeResult, tstamp, indexIds, force); } return result; } @@ -2628,10 +2628,11 @@ public int doReindex(FHIROperationContext operationContext, OperationOutcome.Bui * @param operationOutcomeResult * @param tstamp * @param indexIds + * @param force * @return * @throws Exception */ - public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds) throws Exception { + public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds, boolean force) throws Exception { // If the indexIds are empty or null, then it's not properly formed. if (indexIds == null || indexIds.isEmpty()) { throw new IllegalArgumentException("No indexIds sent to the $reindex list method"); @@ -2684,7 +2685,7 @@ public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instan txn.begin(); try { FHIRPersistenceContext persistenceContext = null; - result += persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, subListIndexIds, null); + result += persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, subListIndexIds, null, force); } catch (FHIRPersistenceDataAccessException x) { // At this point, the transaction is marked for rollback if (x.isTransactionRetryable() && ++attempt <= TX_ATTEMPTS) { @@ -2722,10 +2723,11 @@ public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instan * @param operationOutcomeResult * @param tstamp * @param resourceLogicalId + * @param force * @return * @throws Exception */ - public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId) throws Exception { + public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId, boolean force) throws Exception { int result = 0; // handle some retries in case of deadlock exceptions final int TX_ATTEMPTS = 5; @@ -2735,7 +2737,7 @@ public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Inst txn.begin(); try { FHIRPersistenceContext persistenceContext = null; - result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, null, resourceLogicalId); + result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, null, resourceLogicalId, force); attempt = TX_ATTEMPTS; // end the retry loop } catch (FHIRPersistenceDataAccessException x) { if (x.isTransactionRetryable() && attempt < TX_ATTEMPTS) { diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java index 98df861ef73..6c798873207 100644 --- a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java +++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java @@ -144,7 +144,7 @@ public String generateResourceId() { @Override public int reindex(FHIRPersistenceContext context, Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, boolean force) throws FHIRPersistenceException { return 0; } diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java index 8b5818a677d..b30492c3b65 100644 --- a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java +++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java @@ -418,7 +418,8 @@ public int reindex( Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, + boolean force) throws FHIRPersistenceException { throw new UnsupportedOperationException(); } diff --git a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java index 52dbe4c6053..cbca679ba0c 100644 --- a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java +++ b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java @@ -169,7 +169,7 @@ public OperationOutcome getHealth() throws FHIRPersistenceException { @Override public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, boolean force) throws FHIRPersistenceException { return 0; } diff --git a/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java b/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java index e78252814f0..a9824d7a80d 100644 --- a/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java +++ b/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java @@ -74,7 +74,7 @@ public FHIRPersistenceTransaction getTransaction() throws Exception { @Override public int doReindex(FHIROperationContext operationContext, Builder operationOutcomeResult, Instant tstamp, List indexIds, - String resourceLogicalId) throws Exception { + String resourceLogicalId, boolean force) throws Exception { return 0; } diff --git a/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java b/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java index 52e1456c37e..9ee0ae3a2f8 100644 --- a/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java +++ b/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java @@ -1702,7 +1702,7 @@ public FHIRPersistenceTransaction getTransaction() throws Exception { @Override public int doReindex(FHIROperationContext operationContext, Builder operationOutcomeResult, Instant tstamp, List indexIds, - String resourceLogicalId) throws Exception { + String resourceLogicalId, boolean force) throws Exception { throw new AssertionError("Unused"); } diff --git a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java index 5806a177acb..dcba1a443b6 100644 --- a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java +++ b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java @@ -46,6 +46,7 @@ public class ReindexOperation extends AbstractOperation { private static final String PARAM_INDEX_IDS = "indexIds"; private static final String PARAM_RESOURCE_COUNT = "resourceCount"; private static final String PARAM_RESOURCE_LOGICAL_ID = "resourceLogicalId"; + private static final String PARAM_FORCE = "force"; // The max number of resources we allow to be processed by one request private static final int MAX_RESOURCE_COUNT = 1000; @@ -84,6 +85,7 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class indexIds = null; int resourceCount = 10; String resourceLogicalId = null; + boolean force = false; boolean hasSpecificResourceType = false; if (resourceType != null) { @@ -137,6 +139,14 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class 0; i++) { - processed = resourceHelper.doReindex(operationContext, result, tstamp, null, resourceLogicalId); + processed = resourceHelper.doReindex(operationContext, result, tstamp, null, resourceLogicalId, force); totalProcessed += processed; } } diff --git a/operation/fhir-operation-reindex/src/main/resources/reindex.json b/operation/fhir-operation-reindex/src/main/resources/reindex.json index 45abc9f7ce3..51f78138910 100644 --- a/operation/fhir-operation-reindex/src/main/resources/reindex.json +++ b/operation/fhir-operation-reindex/src/main/resources/reindex.json @@ -52,6 +52,14 @@ "max": "1", "documentation": "Reindex only the specified resource or resources of the given resource type when no id is provided. Format as Patient/abc123 or Patient. If indexIds is specified, this parameter is not used.", "type": "string" + }, + { + "name": "force", + "use": "in", + "min": 0, + "max": "1", + "documentation": "When true, always replace the parameters even if the parameter hash matches. This is typically used when a schema migration step changes structure used to stored parameters in the database.", + "type": "boolean" } ] -} \ No newline at end of file +} From c3ab57481dca291e0d04c652042ecef694d38b1d Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 8 Jun 2022 12:06:58 +0100 Subject: [PATCH 22/40] issue #3437 use logical_resource_ident for read and vread with citus Signed-off-by: Robin Arnold --- .../jdbc/citus/CitusResourceDAO.java | 92 ++++++++++++++++++- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java index 94f5e4330d2..fc2c352abe4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -7,7 +7,11 @@ package com.ibm.fhir.persistence.jdbc.citus; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import javax.transaction.TransactionSynchronizationRegistry; @@ -34,9 +38,18 @@ public class CitusResourceDAO extends PostgresResourceDAO { + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + " FROM %s_RESOURCES R, " + " %s_LOGICAL_RESOURCES LR " - + " WHERE LR.LOGICAL_ID = ? " - + " AND R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " - + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID"; // additional predicate using common Citus distribution column + + " WHERE R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " // join must use common Citus distribution column + + " AND LR.LOGICAL_RESOURCE_ID = ? "; // lookup using logical_resource_id + + // Read a specific version of the resource + private static final String SQL_VERSION_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.LOGICAL_RESOURCE_ID = ? " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " + + " AND R.VERSION_ID = ?"; /** * Public constructor @@ -67,18 +80,87 @@ public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor f super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); } + /** + * Read the logical_resource_id value from logical_resource_ident + * @param resourceType + * @param logicalId + * @return + */ + private Long getLogicalResourceIdentId(String resourceType, String logicalId) throws FHIRPersistenceDataAccessException { + final int resourceTypeId = getCache().getResourceTypeCache().getId(resourceType); + final Long logicalResourceId; + final String selectLogicalResourceIdent = "" + + "SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE resource_type_id = ? " + + " AND logical_id = ? "; // distribution key + try (PreparedStatement ps = getConnection().prepareStatement(selectLogicalResourceIdent)) { + ps.setInt(1, resourceTypeId); + ps.setString(2, logicalId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + logicalResourceId = rs.getLong(1); + } else { + logicalResourceId = null; + } + } catch (SQLException x) { + log.log(Level.SEVERE, "read '" + resourceType + "/" + logicalId + "'", x); + throw new FHIRPersistenceDataAccessException("read failed for logical resource ident record"); + } + return logicalResourceId; + } + @Override public Resource read(String logicalId, String resourceType) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { final String METHODNAME = "read"; log.entering(CLASSNAME, METHODNAME); + // For Citus we want to first query the logical_resource_ident table because it is + // distributed by the logicalId. This gets us the logical_resource_id value which + // we can then use to access the logical_resource tables which are distributed by + // logical_resource_id + Long logicalResourceId = getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId == null) { + return null; + } + Resource resource = null; List resources; String stmtString = null; try { stmtString = String.format(SQL_READ, resourceType, resourceType); - resources = this.runQuery(stmtString, logicalId); + resources = this.runQuery(stmtString, logicalResourceId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + return resource; + } + + @Override + public Resource versionRead(String logicalId, String resourceType, int versionId) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "versionRead"; + log.entering(CLASSNAME, METHODNAME); + + // For Citus we want to first query the logical_resource_ident table because it is + // distributed by the logicalId. This gets us the logical_resource_id value which + // we can then use to access the logical_resource tables which are distributed by + // logical_resource_id + Long logicalResourceId = getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId == null) { + return null; + } + + Resource resource = null; + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_VERSION_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, logicalResourceId, versionId); if (!resources.isEmpty()) { resource = resources.get(0); } @@ -86,5 +168,7 @@ public Resource read(String logicalId, String resourceType) throws FHIRPersisten log.exiting(CLASSNAME, METHODNAME); } return resource; + } + } \ No newline at end of file From fe036118328434863bd487ea7aaec780b7ffead3 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 8 Jun 2022 15:48:27 +0100 Subject: [PATCH 23/40] issue #3437 fixed postgres and db2 add_any_resource to use lr ident Signed-off-by: Robin Arnold --- .../jdbc/domain/SearchQueryRenderer.java | 6 +- .../main/resources/db2/add_any_resource.sql | 84 +++++++++++----- .../resources/postgres/add_any_resource.sql | 99 +++++++++++++------ 3 files changed, 130 insertions(+), 59 deletions(-) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java index 1949d94ea83..fde68236e7c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java @@ -2522,7 +2522,7 @@ public QueryData addReferenceParam(QueryData queryData, String resourceType, Que * identifier. The interpretation of a reference parameter is either: * [1] [parameter]=[id] the logical [id] of a resource using a local reference (i.e. a relative reference) * [2] [parameter]=[type]/[id] the logical [id] of a resource of a specified type using a local reference (i.e. a relative reference), for when the reference can point to different types of resources (e.g. Observation.subject) - * [3] [parameter]=[url] where the [url] is an absolute URL - a reference to a resource by its absolute location, or by it's canonical URL + * [3] [parameter]=[url] where the [url] is an absolute URL - a reference to a resource by its absolute location, or by its canonical URL * * For [1], the target resource type isn't known. This shouldn't matter, because * we still look up the logical_resource_id by its logical_id. If there are @@ -2548,7 +2548,6 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource // logical_id values stored in logical_resource_ident. Absolute references // are stored using a resource_type of "Resource" (similar to the default // code-system we used to use with common_token_values). - final int resourceTypeIdForResource = identityCache.getResourceTypeId("Resource"); // Firstly we need to split the query parm values into separate lists List> resourceTypesAndIds = new ArrayList<>(queryParm.getValues().size()); @@ -2557,7 +2556,6 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource } List logicalResourceIdList = new ArrayList<>(); - List refValues = new ArrayList<>(queryParm.getValues().size()); for (Pair resourceTypeAndId : resourceTypesAndIds) { String targetResourceType = resourceTypeAndId.getLeft(); String referenceValue = resourceTypeAndId.getRight(); @@ -2566,7 +2564,7 @@ private QueryData processRealReferenceParam(QueryData queryData, String resource Integer resourceTypeId = identityCache.getResourceTypeId(targetResourceType); if (resourceTypeId != null) { // It's a valid resource type, so we treat as a local reference - logger.info(() -> "reference search value: type[local] value[" + targetResourceType + "/" + referenceValue + "]"); + logger.fine(() -> "reference search value: type[local] value[" + targetResourceType + "/" + referenceValue + "]"); Long logicalResourceId = identityCache.getLogicalResourceId(targetResourceType, referenceValue); logicalResourceIdList.add(logicalResourceId != null ? logicalResourceId : -1); } else { diff --git a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql index 7f15ea4fa8f..a09673ea2ac 100644 --- a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql +++ b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql @@ -54,6 +54,7 @@ BEGIN DECLARE v_schema_name VARCHAR(128 OCTETS); DECLARE v_logical_resource_id BIGINT DEFAULT NULL; + DECLARE t_logical_resource_id BIGINT DEFAULT NULL; DECLARE v_current_resource_id BIGINT DEFAULT NULL; DECLARE v_resource_id BIGINT DEFAULT NULL; DECLARE v_resource_type_id INT DEFAULT NULL; @@ -83,9 +84,11 @@ BEGIN FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type; -- FOR UPDATE WITH RS does not appear to work using a prepared statement and - -- cursor, so we have to run this directly against the logical_resources table. - SELECT logical_resource_id, parameter_hash, is_deleted INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources + -- cursor, so we have to run as compiled SQL. For V0027 we now use + -- logical_resource_ident for managing the identity of a resource and the + -- associated locking + SELECT logical_resource_id INTO v_logical_resource_id + FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id FOR UPDATE WITH RS ; @@ -94,40 +97,73 @@ BEGIN IF v_logical_resource_id IS NULL THEN VALUES NEXT VALUE FOR {{SCHEMA_NAME}}.fhir_sequence INTO v_logical_resource_id; - PREPARE stmt FROM - 'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) ' - || ' VALUES (?, ?, ?, ?, ?, ?, ?, ?)'; - EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0', p_is_deleted, p_last_updated, p_parameter_hash_b64; + PREPARE stmt FROM + 'INSERT INTO ' || v_schema_name || '.logical_resource_ident (mt_id, resource_type_id, logical_id, logical_resource_id) ' + || ' VALUES (?, ?, ?, ?)'; + EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_resource_type_id, p_logical_id, v_logical_resource_id; -- remember that we have a concurrent system...so there is a possibility - -- that another thread snuck in before us and created the logical resource. This + -- that another thread snuck in before us and created the logical resource ident. This -- is easy to handle, just turn around and read it IF v_duplicate = 1 THEN - -- row exists, so we just need to obtain a lock on it. Because logical resource records are - -- never deleted, we don't need to worry about it disappearing again before we grab the row lock - SELECT logical_resource_id, parameter_hash, is_deleted INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources + SELECT logical_resource_id INTO v_logical_resource_id + FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id FOR UPDATE WITH RS ; + + -- Because someone else created the logical_resoure_ident record, we need to see if + -- they also created the corresponding logical_resources record + SELECT logical_resource_id, parameter_hash, is_deleted + INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; + + IF (t_logical_resource_id IS NULL) + THEN + -- other thread only created the ident record, so we still need to treat + -- this as a new resource + SET v_new_resource = 1; + END IF; + ELSE + -- we created the logical_resource_ident, so we know this is a new resource + SET v_new_resource = 1; + END IF; + ELSE + -- the logical_resource_ident record exists, so now we need to find out + -- if the corresponding logical_resources record exists + SELECT logical_resource_id, parameter_hash, is_deleted + INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; - -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL - SET o_current_parameter_hash = NULL; + IF (t_logical_resource_id IS NULL) + THEN + -- the ident record was created as a reference, but because there's no logical_resources + -- record, we treat this as a new resource + SET v_new_resource = 1; + END IF; + END IF; - ELSE - -- we created the logical resource and therefore we already own the lock. So now we can - -- safely create the corresponding record in the resource-type-specific logical_resources table - PREPARE stmt FROM + IF v_new_resource = 1 + THEN + -- create the logical_resources record + PREPARE stmt FROM + 'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) ' + || ' VALUES (?, ?, ?, ?, ?, ?, ?, ?)'; + EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0', p_is_deleted, p_last_updated, p_parameter_hash_b64; + + -- create the xx_logical_resources record + PREPARE stmt FROM 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (mt_id, logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' || ' VALUES (?, ?, ?, ?, ?, ?, ?)'; - EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - SET v_new_resource = 1; - END IF; - END IF; + EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - -- Remember everying is locked at the logical resource level, so we are thread-safe here - IF v_new_resource = 0 THEN + -- Since the resource did not previously exist, make sure o_current_parameter_hash is NULL + SET o_current_parameter_hash = NULL; + + ELSE -- as this is an existing resource, we need to know the current resource id. -- This is only available at the resource-specific logical_resources level PREPARE stmt FROM diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql index 0130d2054e0..5cc07feede1 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql @@ -8,6 +8,9 @@ -- Procedure to add a resource version and its associated parameters. These -- parameters only ever point to the latest version of a resource, never to -- previous versions, which are kept to support history queries. +-- From V0027, we now use a logical_resource_ident table for locking. Records +-- can be created in this table either by this procedure, or as part of +-- reference parameter processing. -- implNote - Conventions: -- p_... prefix used to represent input parameters -- v_... prefix used to represent declared variables @@ -58,10 +61,11 @@ v_new_resource INT := 0; v_duplicate INT := 0; v_current_version INT := 0; + v_ghost_resource INT := 0; v_change_type CHAR(1) := NULL; -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. - lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id, parameter_hash, is_deleted FROM {{SCHEMA_NAME}}.logical_resources WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; BEGIN -- default value unless we hit If-None-Match @@ -75,44 +79,77 @@ BEGIN -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; - -- Get a lock at the system-wide logical resource level + -- Get a lock on the logical resource identity record OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted; + FETCH lock_cur INTO v_logical_resource_id; CLOSE lock_cur; - -- Create the resource if we don't have it already + -- Create the resource ident record if we don't have it already IF v_logical_resource_id IS NULL THEN SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; -- remember that we have a concurrent system...so there is a possibility - -- that another thread snuck in before us and created the logical resource. This - -- is easy to handle, just turn around and read it - INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) - VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; - - -- row exists, so we just need to obtain a lock on it. Because logical resource records are - -- never deleted, we don't need to worry about it disappearing again before we grab the row lock - OPEN lock_cur (t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted; - CLOSE lock_cur; - - -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL - o_current_parameter_hash := NULL; - + -- that another thread snuck in before us and created the ident record. To + -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn + -- around and read again to check that the logical_resource_id in the table + -- matches the value we tried to insert. + INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id) + VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; + + -- Do a read so that we can verify that *we* did the insert + OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO t_logical_resource_id; + CLOSE lock_cur; + IF v_logical_resource_id = t_logical_resource_id THEN - -- we created the logical resource and therefore we already own the lock. So now we can - -- safely create the corresponding record in the resource-type-specific logical_resources table - EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' - || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - v_new_resource := 1; + -- we did the insert, so we know this is a new record + v_new_resource := 1; ELSE - v_logical_resource_id := t_logical_resource_id; + -- another thread created the resource. + -- New for V0027. Records in logical_resource_ident may be created because they + -- are the target of a reference. We therefore need to handle the case where + -- no logical_resources record exists. + SELECT logical_resource_id, parameter_hash, is_deleted + INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = t_logical_resource_id; + + IF (v_logical_resource_id IS NULL) + THEN + -- other thread only created the ident record, so we still need to treat + -- this as a new resource + v_logical_resource_id := t_logical_resource_id; + v_new_resource := 1; + END IF; + END IF; + ELSE + -- we have an ident record, but we still need to check if we have a logical_resources + -- record + SELECT logical_resource_id, parameter_hash, is_deleted + INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = v_logical_resource_id; + IF (t_logical_resource_id IS NULL) + THEN + v_new_resource := 1; END IF; END IF; - -- Remember everying is locked at the logical resource level, so we are thread-safe here - IF v_new_resource = 0 THEN + IF v_new_resource = 1 + THEN + -- we already own the lock on the ident record, so we can safely create + -- the corresponding records in the logical_resources and resource-type-specific + -- xx_logical_resources tables + INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; + + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + + -- Since the resource did not previously exist, make sure o_current_parameter_hash is null + o_current_parameter_hash := NULL; + ELSE -- as this is an existing resource, we need to know the current resource id. -- This is only available at the resource-specific logical_resources level EXECUTE @@ -153,19 +190,19 @@ BEGIN IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash THEN - -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) - -- TODO patch parameter sets instead of all delete/all insert. + -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) + -- TODO patch parameter sets instead of all delete/all insert. EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' USING p_resource_type, v_logical_resource_id; - END IF; -- end if check parameter hash + END IF; -- end if check parameter hash END IF; -- end if existing resource + -- create the new resource version entry in xx_resources EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; - IF v_new_resource = 0 THEN -- As this is an existing logical resource, we need to update the xx_logical_resource values to match -- the values of the current resource. For new resources, these are added by the insert so we don't @@ -199,4 +236,4 @@ BEGIN -- only the logical_resource_id is the target of any FK, so there's no need to return -- the resource_id (which is now private to the _resources tables). o_logical_resource_id := v_logical_resource_id; -END $$; \ No newline at end of file +END $$; From 1644408eb698256e9559b4e5ae6fc3d775904f3e Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 8 Jun 2022 16:58:48 +0100 Subject: [PATCH 24/40] issue #3437 db-type arg now required when running schema tool Signed-off-by: Robin Arnold --- build/docker/deploySchemaAndTenant.sh | 10 +- build/docker/updateSchema.sh | 8 +- .../docs/SchemaToolUsageGuide.md | 125 ++++++++++-------- .../java/com/ibm/fhir/schema/app/Main.java | 13 +- 4 files changed, 89 insertions(+), 67 deletions(-) diff --git a/build/docker/deploySchemaAndTenant.sh b/build/docker/deploySchemaAndTenant.sh index d57a19a6c67..e95248ac74b 100755 --- a/build/docker/deploySchemaAndTenant.sh +++ b/build/docker/deploySchemaAndTenant.sh @@ -21,7 +21,7 @@ while [ "$not_ready" == "true" ] do EXIT_CODE="-1" java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --create-schemas | tee -a ${TMP_FILE} + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --create-schemas | tee -a ${TMP_FILE} EXIT_CODE="${PIPESTATUS[0]}" LOG_OUT=`cat ${TMP_FILE}` if [[ "$EXIT_CODE" == "0" ]] @@ -54,18 +54,18 @@ then fi java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --update-schema --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --update-schema --pool-size 2 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --allocate-tenant default --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --allocate-tenant default --pool-size 2 # The regex in the following command will output the capture group between "key=" and "]" # With GNU grep, the following would work as well: grep -oP 'key=\K\S+(?=])' tenantKey=$(java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --add-tenant-key default 2>&1 \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --add-tenant-key default 2>&1 \ | grep "key=" | sed -e 's/.*key\=\(.*\)\].*/\1/') # Creating a backup file is the easiest way to make in-place sed portable across OSX and Linux diff --git a/build/docker/updateSchema.sh b/build/docker/updateSchema.sh index e8dc21f4849..df2c401286d 100755 --- a/build/docker/updateSchema.sh +++ b/build/docker/updateSchema.sh @@ -13,17 +13,17 @@ cd ${DIR} # For #1366 the migration hits deadlock issues if run in parallel, so # to avoid this, serialize the steps using --pool-size 1 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --update-schema \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --update-schema \ --pool-size 1 # Rerun grants to cover any new tables added by the above migration step java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 # And make sure that the new tables have partitions for existing tenants java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --refresh-tenants + --db-type db2 --prop-file db2.properties --refresh-tenants java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER \ --pool-size 20 diff --git a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md index 53b3df9c1f7..76af693b8a7 100644 --- a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md +++ b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md @@ -118,17 +118,18 @@ The following sections include common values for `OPTIONS`. For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --create-schemas ``` For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name fhirdata ---create-schemas +--prop-file postgresql.properties \ +--schema-name fhirdata \ +--create-schemas \ --db-type postgresql ``` @@ -141,9 +142,10 @@ for the IBM FHIR Server to operate. The FHIRADMIN user should only be used for schema updates, not for IBM FHIR Server access. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---update-schema +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--update-schema \ --grant-to FHIRSERVER ``` @@ -161,10 +163,10 @@ for the IBM FHIR Server to operate. The FHIRADMIN user should only be used for schema updates, not for IBM FHIR Server access. ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---update-schema ---grant-to FHIRSERVER +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--update-schema \ +--grant-to FHIRSERVER \ --db-type postgresql ``` If the --grant-to is provided, the grants will be processed after the schema @@ -178,16 +180,18 @@ When updating the postgres schema, the autovacuum settings are configured. ### Grant privileges to another data access user ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --grant-to FHIRSERVER ``` ### Add a new tenant (e.g. default) (Db2 only) ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --allocate-tenant default ``` @@ -209,6 +213,7 @@ After a schema update you must run the refresh-tenants command to ensure that an ``` java -jar schema/fhir-persistence-schema-*-cli.jar \ + --db-type db2 \ --prop-file db2.properties --refresh-tenants ``` @@ -243,9 +248,10 @@ Edit `wlp/usr/servers/fhir-server/config/default/fhir-server-config.json` and ad ### Test a tenant (Db2 only) ``` ---prop-file db2.properties ---schema-name FHIRDATA ---test-tenant default +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--test-tenant default \ --tenant-key "" ``` @@ -255,8 +261,9 @@ Use `--tenant-key-file tenant.key` to read the tenant-key to a file. You do not To add a tenant key for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --add-tenant-key default ``` @@ -277,9 +284,9 @@ Use `--tenant-key-file tenant.key.file` to direct the action to read the tenant- To remove all tenant keys for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---db-type db2 +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --revoke-all-tenant-keys default ``` @@ -295,10 +302,10 @@ To remove all tenant keys for an existing tenant, replace FHIRDATA with your cli To remove a tenant key for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---db-type db2 ---revoke-tenant-key default +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--revoke-tenant-key default \ --tenant-key rZ59TLyEpjU+FAKEtgVk8J44J0= ``` @@ -317,8 +324,9 @@ Use `--tenant-key-file tenant.key.file` to direct the action to read the tenant- For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --update-proc ``` @@ -335,9 +343,10 @@ For PostgreSQL: For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA ---drop-schema-fhir +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ --confirm-drop ``` @@ -355,12 +364,13 @@ For PostgreSQL: For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA ---drop-schema-fhir ---drop-schema-batch ---drop-schema-oauth ---drop-admin +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ +--drop-schema-batch \ +--drop-schema-oauth \ +--drop-admin \ --confirm-drop ``` @@ -387,10 +397,11 @@ For those using multiple schemas for each customer, for instance, customer 2 nee ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---create-schemas ---create-schema-batch FHIR_JBATCH_2ND ---create-schema-oauth FHIR_OAUTH_2ND +--db-type db2 \ +--prop-file db2.properties \ +--create-schemas \ +--create-schema-batch FHIR_JBATCH_2ND \ +--create-schema-oauth FHIR_OAUTH_2ND \ --create-schema-fhir FHIRDATA_2ND ``` @@ -398,32 +409,36 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---schema-name FHIRDATA ---update-schema-batch FHIR_JBATCH_2ND ---update-schema-oauth FHIR_OAUTH_2ND +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--update-schema-batch FHIR_JBATCH_2ND \ +--update-schema-oauth FHIR_OAUTH_2ND \ --update-schema-fhir FHIRDATA_2ND ``` ### Grant privileges to data access user ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target BATCH FHIR_JBATCH_2ND ``` ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target OAUTH FHIR_OAUTH_2ND ``` ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target DATA FHIRDATA_2ND ``` @@ -560,6 +575,7 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ +--db-type db2 \ --prop-file fhiradmin.properties \ --db-type db2 \ --schema-name FHIRDATA \ @@ -597,7 +613,8 @@ and grants permission to the username| |--tenant-key|tenantKey|the tenant-key in the queries| |--tenant-key-file|tenant-key-file-location|sets the tenant key file location| |--list-tenants||fetches list of tenants and current status| -|--db-type|dbType|Either derby, postgresql, db2| +|--db-type|dbType|Either derby, postgresql, db2. Required.| +|--schema-type|schemaType|Override the default schema type created for the configured database type. PostgresSQL->PLAIN, Derby->PLAIN, Db2->MULTITENANT, Citus->DISTRIBUTED | |--delete-tenant-meta|tenantName|deletes tenant metadata given the tenantName| |--drop-detached|tenantName|(phase 2) drops the detached tenant partition tables given the tenantName| |--freeze-tenant||Changes the tenant state to frozen, and subsequently (Db2 only)| diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index d0f3f88fe22..37bc9854dc4 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -260,9 +260,9 @@ public class Main { // How many seconds to wait to obtain the update lease private int waitForUpdateLeaseSeconds = 10; - // The database type being populated (default: Db2) - private DbType dbType = DbType.DB2; - private IDatabaseTranslator translator = new Db2Translator(); + // The database type being populated. Now a required parameter. + private DbType dbType; + private IDatabaseTranslator translator; // Optional subset of resource types (for faster schema builds when testing) private Set resourceTypeSubset; @@ -1137,7 +1137,7 @@ protected void grantPrivileges() { * @return */ protected boolean isMultitenant() { - return MULTITENANT_FEATURE_ENABLED.contains(this.dbType); + return dataSchemaType == SchemaType.MULTITENANT; } /** @@ -2228,6 +2228,7 @@ protected void parseArgs(String[] args) { dataSchemaType = SchemaType.DISTRIBUTED; // by default break; case DB2: + translator = new Db2Translator(); dataSchemaType = SchemaType.MULTITENANT; break; default: @@ -2267,6 +2268,10 @@ protected void parseArgs(String[] args) { this.maxConnectionPoolSize = threadPoolSize + FhirSchemaConstants.CONNECTION_POOL_HEADROOM; logger.warning("Connection pool size below minimum headroom. Setting it to " + this.maxConnectionPoolSize); } + + if (this.dbType == null) { + throw new IllegalArgumentException(DB_TYPE + " is required"); + } } /** From 7ca7633ed81166f2a0d2a360be65be379813ee88 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 9 Jun 2022 10:26:14 +0100 Subject: [PATCH 25/40] issue #3437 fixed schema migration for Db2 Signed-off-by: Robin Arnold --- .../fhir/database/utils/db2/Db2Adapter.java | 7 ++- .../ibm/fhir/database/utils/model/Table.java | 10 +++- .../java/com/ibm/fhir/schema/app/Main.java | 10 +++- .../control/FhirResourceTableGroup.java | 2 +- .../MigrateV0027LogicalResourceIdentMT.java | 49 +++++++++++++++++++ 5 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java index 7491b9774f7..a6af211b4a5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.DistributionContext; @@ -209,7 +210,7 @@ public void addNewTenantPartitions(Collection
tables, String schemaName, * @param newTenantId * @param tablespaceName */ - public void addNewTenantPartitions(Collection
tables, Map partitionInfoMap, int newTenantId, String tablespaceName) { + public void addNewTenantPartitions(Collection
allTables, Map partitionInfoMap, int newTenantId, String tablespaceName) { // Thread pool for parallelizing requests int poolSize = connectionProvider.getPoolSize(); if (poolSize == -1) { @@ -219,6 +220,10 @@ public void addNewTenantPartitions(Collection
tables, Map tables = allTables.stream().filter(t->t.isCreate()).collect(Collectors.toList()); + for (Table t: tables) { String qualifiedName = t.getQualifiedName(); PartitionInfo pi = partitionInfoMap.get(t.getObjectName()); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index 67d3943ac44..9c4428b486d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -140,6 +140,14 @@ public String getTenantColumnName() { return this.tenantColumnName; } + /** + * Getter for the create flag + * @return + */ + public boolean isCreate() { + return this.create; + } + /** * Getter for the table's distributionType * @return @@ -194,7 +202,7 @@ public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContex step.migrateFrom(priorVersion).stream().forEachOrdered(target::runStatement); } // Re-apply tenant access control if required - if (this.accessControlVar != null) { + if (this.accessControlVar != null && this.create) { // The accessControlVar represents a DB2 session variable. Programs must set this value // for the current tenant when executing any SQL (both reads and writes) on // tables with this access control enabled diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 37bc9854dc4..7d2c292ad2a 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -164,6 +164,7 @@ import com.ibm.fhir.schema.control.MigrateV0014LogicalResourceIsDeletedLastUpdated; import com.ibm.fhir.schema.control.MigrateV0021AbstractTypeRemoval; import com.ibm.fhir.schema.control.MigrateV0027LogicalResourceIdent; +import com.ibm.fhir.schema.control.MigrateV0027LogicalResourceIdentMT; import com.ibm.fhir.schema.control.OAuthSchemaGenerator; import com.ibm.fhir.schema.control.PopulateParameterNames; import com.ibm.fhir.schema.control.PopulateResourceTypes; @@ -2598,8 +2599,13 @@ private void dataMigrationForV0014(IDatabaseAdapter adapter, String schemaName, private void dataMigrationForV0027(IDatabaseAdapter adapter, String schemaName) { GetLogicalResourceNeedsV0027Migration needsMigrating = new GetLogicalResourceNeedsV0027Migration(schemaName); if (adapter.runStatement(needsMigrating)) { - MigrateV0027LogicalResourceIdent cmd = new MigrateV0027LogicalResourceIdent(schemaName); - adapter.runStatement(cmd); + if (this.dataSchemaType == SchemaType.MULTITENANT) { + MigrateV0027LogicalResourceIdentMT cmd = new MigrateV0027LogicalResourceIdentMT(schemaName); + adapter.runStatement(cmd); + } else { + MigrateV0027LogicalResourceIdent cmd = new MigrateV0027LogicalResourceIdent(schemaName); + adapter.runStatement(cmd); + } } } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index 9ab4e67907f..d09b31cbbf6 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -835,7 +835,7 @@ public void addRefValuesView(List group, String prefix) { // Make sure we include MT_ID in both the select list and join condition. It's needed // in the join condition to give the optimizer the best chance at finding a good nested // loop strategy - select.append("SELECT ref.").append(MT_ID); + select.append("SELECT ref.").append(MT_ID).append(", "); select.append(" ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, "); select.append(" ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id, "); select.append(" lri.logical_id AS ref_value "); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java new file mode 100644 index 00000000000..d73f265e01c --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java @@ -0,0 +1,49 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.control; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Populate LOGICAL_RESOURCE_IDENT with records from LOGICAL_RESOURCES. + * Variant for use with the Db2 multitenant schema. + */ +public class MigrateV0027LogicalResourceIdentMT implements IDatabaseStatement { + + // The FHIR data schema + private final String schemaName; + + /** + * Public constructor + * @param schemaName + */ + public MigrateV0027LogicalResourceIdentMT(String schemaName) { + this.schemaName = schemaName; + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES"); + final String logicalResourceIdent = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT"); + final String DML = "" + + "INSERT INTO " + logicalResourceIdent +"(mt_id, resource_type_id, logical_id, logical_resource_id) " + + " SELECT mt_id, resource_type_id, logical_id, logical_resource_id " + + " FROM " + logicalResources; + + try (PreparedStatement ps = c.prepareStatement(DML)) { + ps.executeUpdate(); + } catch (SQLException x) { + throw translator.translate(x); + } + } +} \ No newline at end of file From 89e14ad3133fed3a8d24af3910c739ea9d598a7a Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 9 Jun 2022 11:52:46 +0100 Subject: [PATCH 26/40] issue #3437 fixed missing grant for Db2 Signed-off-by: Robin Arnold --- .../java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java | 5 +---- .../ibm/fhir/database/utils/common/PlainSchemaAdapter.java | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java index f6633952b37..536404a32a3 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java @@ -479,12 +479,9 @@ public void grantSequencePrivileges(String schemaName, String objectName, Collec public boolean checkCompatibility(String adminSchema); /** - * * @return a false, if not used, or true if used with the persistence layer. */ - public default boolean useSessionVariable() { - return false; - } + public boolean useSessionVariable(); /** * creates or replaces the SQL function diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java index b715be2e8c0..189a50cbd38 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java @@ -64,6 +64,11 @@ public void detachPartition(String schemaName, String tableName, String partitio databaseAdapter.detachPartition(schemaName, tableName, partitionName, newTableName); } + @Override + public boolean useSessionVariable() { + return databaseAdapter.useSessionVariable(); + } + @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { From fde6f803efbfd5716cfb27bc826cd00aa25124a4 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 9 Jun 2022 12:36:12 +0100 Subject: [PATCH 27/40] issue #3437 new inserts need mt_id for Db2 multitenant Signed-off-by: Robin Arnold --- .../jdbc/db2/Db2ResourceReferenceDAO.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index 104553fed4a..322e2961363 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -11,19 +11,25 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Collection; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; @@ -471,4 +477,72 @@ protected int readOrAddParameterNameId(String parameterName) throws FHIRPersiste final ParameterNameDAO pnd = new ParameterNameDAOImpl(getConnection(), getSchemaName()); return pnd.readOrAddParameterNameId(parameterName); } + + @Override + protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { + + // simplified implementation which handles inserts individually + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (mt_id, resource_type_id, logical_id, logical_resource_id) VALUES ("); + insert.append(adminSchemaName).append(".SV_TENANT_ID, ?,?,"); + insert.append(nextVal); // next sequence value + insert.append(")"); + + logger.fine(() -> "ident insert: " + insert.toString()); + final String dml = insert.toString(); + try (PreparedStatement ps = getConnection().prepareStatement(dml)) { + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + try { + ps.executeUpdate(); + } catch (SQLException x) { + if (getTranslator().isDuplicate(x)) { + // do nothing + } else { + throw x; + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + dml, x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + @Override + protected void insertRefValues(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch. This is the multitenant variant, so we need to inject the mt_id value + final String tableName = resourceType + "_REF_VALUES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(mt_id, " + + "parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, composite_id) " + + "VALUES (" + adminSchemaName + ".SV_TENANT_ID, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + for (ResourceReferenceValueRec xr: xrefs) { + psh.setInt(xr.getParameterNameId()) + .setLong(xr.getLogicalResourceId()) + .setLong(xr.getRefLogicalResourceId()) + .setInt(xr.getRefVersionId()) + .setInt(xr.getCompositeId()) + .addBatch(); + + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } } \ No newline at end of file From 9d9684377850f3704f73e132d01684adc9b3a721 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 13 Jun 2022 12:37:27 +0100 Subject: [PATCH 28/40] issue #3697 documentation for fhir-remote-index Signed-off-by: Robin Arnold --- docs/src/pages/guides/FHIRServerUsersGuide.md | 36 +++- fhir-remote-index/README.md | 170 ++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 fhir-remote-index/README.md diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 4052648628c..dda5dd4569c 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -27,6 +27,7 @@ permalink: /FHIRServerUsersGuide/ * [4.10 Bulk data operations](#410-bulk-data-operations) * [4.11 Audit logging service](#411-audit-logging-service) * [4.12 FHIR REST API](#412-fhir-rest-api) + * [4.13 Remote Index Service](#413-remote-index-service) - [5 Appendix](#5-appendix) * [5.1 Configuration properties reference](#51-configuration-properties-reference) * [5.2 Keystores, truststores, and the FHIR server](#52-keystores-truststores-and-the-fhir-server) @@ -2070,6 +2071,10 @@ For example, consider a FHIR Server that is listening at https://fhir:9443/fhir- The originalRequestUriHeader is expected to contain the full path of the original request. Values with no scheme (e.g. `https://`) will be handled like relative URLs, but full URL values (including scheme, hostname, optional port, and path) are recommended. Query string values can be included in the header value but will be ignored by the server; the server will use the query string of the actual request to process the request. +### 4.13 Remote Index Service + +To use the experimental remote index service feature, see the instructions documented in the [fhir-remote-index](https://github.com/IBM/FHIR/tree/main/operation/fhir-remote-index/README.md) project. + # 5 Appendix ## 5.1 Configuration properties reference @@ -2260,7 +2265,16 @@ This section contains reference information about each of the configuration prop |`fhirServer/operations/membermatch/strategy`|string|The key identifying the Member Match strategy| |`fhirServer/operations/membermatch/extendedProps`|object|The extended options for the extended member match implementation| |`fhirServer/operations/everything/includeTypes`|list|The list of related resource types to include alongside the patient compartment resource types. Instances of these resource types will only be returned when they are referenced from one or more resource instances from the target patient compartment. Example values are like `Location`, `Medication`, `Organization`, and `Practitioner`| - +|`fhirServer/remoteIndexService/type`|string| The type of service used to send remote index messages. Only `kafka` is currently supported| +|`fhirServer/remoteIndexService/kafka/mode`|string| Current operation mode of the service. Specify `ACTIVE` to use the service| +|`fhirServer/remoteIndexService/kafka/topicName`|string| The Kafka topic name. Typically `FHIR_REMOTE_INDEX` | +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|string| Bootstrap servers for the Kafka service | +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|string| Kafka service authentication | +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|string| Kafka service authentication| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|string| Kafka service security | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|string| Kafka service SSL configuration | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|string| Kafka service SSL configuration | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|string| Kafka service SSL configuration | ### 5.1.2 Default property values | Property Name | Default value | @@ -2406,6 +2420,16 @@ This section contains reference information about each of the configuration prop |`fhirServer/operations/membermatch/strategy`|default| |`fhirServer/operations/membermatch/extendedProps`|empty| |`fhirServer/operations/everything/includeTypes`|null| +|`fhirServer/remoteIndexService/type`|null| +|`fhirServer/remoteIndexService/kafka/mode`|null| +|`fhirServer/remoteIndexService/kafka/topicName`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|null| ### 5.1.3 Property attributes Depending on the context of their use, config properties can be: @@ -2588,6 +2612,16 @@ Cases where that behavior is not supported are marked below with an `N` in the ` |`fhirServer/operations/membermatch/strategy`|Y|Y|Y| |`fhirServer/operations/membermatch/extendedProps`|Y|Y|Y| |`fhirServer/operations/everything/includeTypes`|Y|Y|N| +|`fhirServer/remoteIndexService/type`|N|N|N| +|`fhirServer/remoteIndexService/kafka/mode`|N|N|N| +|`fhirServer/remoteIndexService/kafka/topicName`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|N|N|N| ## 5.2 Keystores, truststores, and the IBM FHIR server diff --git a/fhir-remote-index/README.md b/fhir-remote-index/README.md new file mode 100644 index 00000000000..f449023bb14 --- /dev/null +++ b/fhir-remote-index/README.md @@ -0,0 +1,170 @@ +# Project: fhir-remote-index + +Project fhir-remote-index is a stand-alone application to support asynchronous storage of FHIR search parameters published by the IBM FHIR Server. When configured to do so, the IBM FHIR Server extracts search parameters from the incoming resource and instead of storing the parameters as part of the create/update transaction, it packages the parameters into a message which is then published to Kafka. The fhir-remote-index application consumes the messages from Kafka and stores the value in the resource parameter tables using efficient batch-based inserts. + +This pattern supports higher ingestion rates because: + +1. Any locking related to inserting normalized common parameter values is decoupled from the locking on logical resources (`logical_resource_ident` table) used to ensure correct versioning of the resource. Create and Update interactions should never see any contention unless the same logical resource is updated in parallel requests; +2. If the same logical resource is updated in parallel, the contention will be reduced compared to synchronous parameter value storage because less work is performed while the lock is held, allowing the transaction to be completed sooner; +3. The remote index consumer can build larger batches, with transactions not tied to the semantics of the original FHIR interaction; +4. The remote index consumer uses batches for all the search parameter value inserts, making it more efficient than the current JDBC persistence layer implementation; +5. The remote index consumer handles normalized value cache-miss lookups using bulk queries. This reduces the total number of database round-trips required to find the required foreign key values. + +In addition, this implementation eliminates any possibility of deadlocks occurring during the Insert/Update interaction. Deadlocks may still occur when processing the asynchronous remote index messages. As these will only occur in a backend process they will not be visible to IBM FHIR Server clients. Deadlocks are handle automatically using a rollback and retry mechanism. Care is taken to reduce the likelihood of deadlocks from occurring in the first place by sorting all record lists before they are processed. + +## Processing Is Asynchronous + +Old search parameters are deleted whenever a resource is updated. When remote indexing is enabled, this means that a resource will not be searchable until the remote index service has received and processed the message. Carefully examine your interaction scenarios with the IBM FHIR Server to determine if this behavior is suitable. In particular, conditional updates are unlikely to work as expected because the search may not return the expected value, depending on timing. + +## Status + +At this time, fhir-remote-index should be considered experimental. + +## Build + +To build fhir-remote-index, clone the git repository and build as follows: + +``` +git clone git@github.com:IBM/FHIR.git +cd FHIR +mvn clean install -f fhir-examples +mvn clean install -f fhir-parent +mvn clean install -f fhir-remote-index +``` + +Note that at this time, fhir-remote-index is not built by default so its project must be built explicitly as shown above. + +## IBM FHIR Server Configuration + +To enable remote indexing of search parameters, add the following `remoteIndexService` entry to the default FHIR server configuration file `config/default/fhir-server-config.json`. This entry identifies the Kafka service and topic to use for sending remote index messages. When this entry is present, the JDBC persistence layer will skip storing the parameters and instead will package the parameters into a message and publish it to Kafka. + +``` +{ + "__comment": "FHIR Server configuration", + "fhirServer": { + ... + "remoteIndexService": { + "type": "kafka", + "kafka": { + "mode": "ACTIVE", + "topicName": "FHIR_REMOTE_INDEX", + "connectionProperties": { + "bootstrap.servers": "broker-0:9093, broker-1:9093, broker-2:9093, broker-3:9093, broker-4:9093, broker-5:9093", + "sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"token\" password=\"change-password\";", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_SSL", + "ssl.protocol": "TLSv1.2", + "ssl.enabled.protocols": "TLSv1.2", + "ssl.endpoint.identification.algorithm": "HTTPS" + } + } + }, + ... +``` + +## Running the fhir-remote index consumer + +The fhir-remote-index application accepts two properties files defining access to Kafka and the target database: + +1. `--kafka-properties` the properties describing connection information for the upstream Kafka topic containing the resource search parameter messages sent by the upstream IBM FHIR Server; +2. `--database-properties` the properties describing the location of the target database to which we will insert the search parameter records generated by the upstream IBM FHIR Server. + +``` +java -Djava.util.logging.config.file=logging.properties \ + -jar /path/to/git/FHIR/fhir-remote-index/target/fhir-remote-index-*-cli.jar \ + --db-type postgresql \ + --database-properties database.properties \ + --kafka-properties kafka.properties \ + --topic-name FHIR_REMOTE_INDEX \ + --consumer-count 3 +``` + +Logging uses standard `java.util.logging` (JUL) and can be configured as follows: + +``` +handlers=java.util.logging.ConsoleHandler,java.util.logging.FileHandler +.level=INFO + +# Console output +java.util.logging.ConsoleHandler.level = INFO +java.util.logging.ConsoleHandler.formatter=com.ibm.fhir.database.utils.common.LogFormatter + +# What level do we want to see in the log file +java.util.logging.FileHandler.level=INFO + +# Log retention: 50MB * 20 files ~= 1GB +java.util.logging.FileHandler.formatter=com.ibm.fhir.database.utils.common.LogFormatter +java.util.logging.FileHandler.limit=50000000 +java.util.logging.FileHandler.count=20 +java.util.logging.FileHandler.pattern=remoteindexservice-%u-%g.log +``` + +To configure the Kafka service, use a properties file as described by the [Kafka documentation](https://kafka.apache.org/documentation/#configuration): + +``` +bootstrap.servers=broker-0:9093, broker-1:9093, broker-2:9093, broker-3:9093, broker-4:9093, broker-5:9093 +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="token" password="change-password"; +sasl.mechanism=PLAIN +security.protocol=SASL_SSL +ssl.protocol=TLSv1.2 +ssl.enabled.protocols=TLSv1.2 +ssl.endpoint.identification.algorithm=HTTPS +``` + +The database properties file: + +``` +db.host=your-db-host +db.port=5432 +db.database=your-db-name +user=fhirserver +password=change-password +currentSchema=fhirdata +ssl=true +sslmode=require +sslcert= +sslrootcert=postgres.crt +sslkey= +``` + +| Property | DB Type | Description | +| ------ | --- | ----------- | +| db.host | All | The host name of the database containing the IBM FHIR Server schema | +| db.port | All | The database port number | +| db.database | All | The database name | +| user | All | The database credential user | +| password | All | The database credential password | +| currentSchema | All | The database schema name | +| ssl | PostgreSQL | For PostgreSQL, use SSL (TLS) for the database connection | +| sslmode | PostgreSQL | The PostgreSQL SSL connection mode | +| sslcert | PostgreSQL | The PostgreSQL SSL certificate | +| sslrootcert | PostgreSQL | The PostgreSQL SSL root certificate | +| sslkey | PostgreSQL | The PostgreSQL SSL key | +| sslConnection | Db2 | true or false | +| sslTrustStoreLocation | Db2 | Location of the p12 trust store file containing the database server certificate | +| sslTrustStorePassword | Db2 | Password for the p12 trust store | + +Note: Citus configuration is the same as PostgreSQL. + +## Command Line Options + +| Option | Description | +| ------ | ----------- | +| --consumer-count {n} | The number of Kafka consumer threads to start in this instance. Multiple instances of this service can be started. The total number of consumer threads across all instances should equal the number to the number of Kafka partitions on the topic. This should maximize throughput. | +| --kafka-properties {properties-file} | A Java properties file containing connection details for the upstream Kafka service. | +| --db-type {type} | The type of database. One of `postgresql`, `derby`, `db2` or `citus`. | +| --database-properties {properties-file} | A Java properties file containing connection details for the downstream IBM FHIR Server database. | +| --topic-name {topic} | The name of the Kafka topic to consume. Default `FHIR_REMOTE_INDEX`. | +| --consumer-group {grp} | Override the default Kafka consumer group (`group.id` value) for this application. Default `remote-index-service-cg`. | +| --schema-type {type} | Set the schema type. One of `PLAIN` or `DISTRIBUTED`. Default is `PLAIN`. The schema type `DISTRIBUTED` is for use with Citus databases. | +| --max-ready-time-ms {milliseconds} | The maximum number of milliseconds to wait for the database to contain the correct data for a particular set of consumed messages. Should be slightly longer than the configured Liberty transaction timeout value. | + +# Asynchronous Message Handling and Transaction Boundaries + +To guarantee delivery, the search parameter messages are posted to Kafka by the IBM FHIR Server before the transaction commits. The transaction will only be committed once all messages sent to Kafka have been acknowledged. This is important, because if the message were to be sent after the transaction, we could lose messages if a failure occured immediately after the transaction but before they were received by Kafka. + +Because messages are sent to Kafka before the transaction is committed, it is possible that a fhir-remote-index consumer may receive a search parameter message before the corresponding resource version record is visible in the database. The consumer therefore runs a query at the start of a batch to determine if the current resource version record matches the message content. The following logic is then applied: + +1. If the resource version doesn't yet exist in the database, the consumer will pause and wait for the transaction to be committed. The consumer will only wait up to the maximum transaction timeout window, at which point it will assume the transaction has failed and the message will be discarded. +2. If the resource version matches, but the lastUpdated time does not match, it assumes the message came from an IBM FHIR Server which failed before the transaction was committed, but the request was processed successfully by another server. The message will be discarded because there will be another message waiting in the queue from the second attempt. +3. If the resource version in the database already exceeds the version in the message, the message will be discarded because the information is already out of date. There will be another message waiting in the queue containing the search parameter values from the most recent resource. \ No newline at end of file From 451e804af448adebd1f0a068af7ad0591cf27ed1 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 13 Jun 2022 15:58:22 +0100 Subject: [PATCH 29/40] issue #3698 document schema changes incl citus support Signed-off-by: Robin Arnold --- fhir-persistence-schema/docs/SchemaDesign.md | 72 ++++++++++++++++- .../docs/SchemaToolUsageGuide.md | 74 +++++++++++------- .../docs/physical_schema_V0027.png | Bin 0 -> 212701 bytes 3 files changed, 117 insertions(+), 29 deletions(-) create mode 100644 fhir-persistence-schema/docs/physical_schema_V0027.png diff --git a/fhir-persistence-schema/docs/SchemaDesign.md b/fhir-persistence-schema/docs/SchemaDesign.md index 814fd11afae..e464cc8cd27 100644 --- a/fhir-persistence-schema/docs/SchemaDesign.md +++ b/fhir-persistence-schema/docs/SchemaDesign.md @@ -39,6 +39,7 @@ The following table highlights the main differences among the database implement | PostgreSQL | Uses a function for the resource persistence logic | | PostgreSQL | TEXT type used instead of CLOB for large data values | | Derby | Resource persistence is implemented at the DAO layer as a sequence of individual statements instead of one procedure call. At a functional level, the process is identical. Simplifies debugging and supports easier unit-test construction. | +| Citus | Experimental. Distributed tables for large scale deployments. Most tables are distributed by logical_resource_id, except for `logical_resource_ident` which is distributed by `logical_id`. Search interactions using chaining or include/revinclude not currently supported. | ---------------------------------------------------------------- # Schema Design - Physical Data Model @@ -60,7 +61,7 @@ These sequences should not require updating and should never be altered as this By convention, tables are named using the plural form of the data they represent. The following diagram shows the main relationships among tables in the schema. Note that only the search parameter tables for the Patient resource are shown. Each resource type gets its own set of search parameter tables. -![Physical Schema](physical_schema_V0024.png) +![Physical Schema](physical_schema_V0027.png) ## TABLESPACES @@ -86,6 +87,8 @@ These table definitions are more completely described in [DB2MultiTenancy.md](DB Each logical resource instance such as a Patient, Device or Observation is stored as a row in the `LOGICAL_RESOURCES` table. A corresponding row is also stored in a resource-specific logical resources tables for example: `PATIENT_LOGICAL_RESOURCES`, `DEVICE_LOGICAL_RESOURCES` or `OBSERVATION_LOGICAL_RESOURCES` etc. +Each logical resource also has record stored in `LOGICAL_RESOURCE_IDENT`. This table is used to hold all identifiers which could be the target of a relation. When a reference is external, the logical_id value will be the url target of the resource. In all other cases, the logical_id value will be the resource id value. As of release 5.0.0, the `LOGICAL_RESOURCE_IDENT` table is used for the `SELECT ... FOR UPDATE` locking used inside the `add_any_resource` stored procedures instead of logical_resources. + Each time a logical resource is updated a new version is created. Each new version is stored in the resource-type-specific table `xx_RESOURCES` where xx is the resource type name for example: `PATIENT_RESOURCES`, `DEVICE_RESOURCES` or `OBSERVATION_RESOURCES` etc. Unless payload offloading has been configured, the resource payload is rendered in JSON, compressed, and stored in the `DATA` column of this table. If payload offloading is configured, the `DATA` column will be null instead. Each version is allocated an integer `VERSION_ID` number starting from 1 which increments by 1 for each new version. Row locking (SELECT FOR UPDATE) guarantees there will be no gaps in the version numbers unless a version-specific `$erase` custom operation has been invoked. The `$erase` operation is the only time rows from the `xx_RESOURCES` table are ever deleted. HTTP `DELETE` interactions are implemented as _soft_ deletes and create a new version of the resource with the `xx_RESOURCES.IS_DELETED` column value equal to 'Y'. This column is repeated (denormalized) in the `xx_LOGICAL_RESOURCES` table as a performance optimization. Most queries need only non-deleted resources and the query is much faster if the join to the wide `xx_RESOURCES` table can be avoided. See the [Finding and Reading a Resource](#finding-and-reading-a-resource) section for practical examples on how data is accessed in the schema. @@ -106,6 +109,7 @@ The following table describes the purpose of each table or group of tables in th | parameter_names | Normalized data | Search parameter names. | | common_token_values | Normalized data | Normalized token and code system values. | | common_canonical_values | Normalized data | Normalized canonical values. | +| logical_resource_ident | Whole system | One row for each logical resource or a reference to a logical resource. When the reference is external, the associated resource type will be `Resource`. | | logical_resources | Whole system | One row for each logical resource, regardless of type. | | resource_change_log | Whole system | One row for each CREATE, UPDATE or DELETE interaction. | | xx_logical_resources | Resource | Resource-type specific entry for each logical resource. | @@ -115,6 +119,7 @@ The following table describes the purpose of each table or group of tables in th | xx_quantity_values | Resource parameters | Quantity search parameter values. | | xx_date_values | Resource parameters | Date search parameter values. | | xx_latlng_values | Resource parameters | Latitude/longitude search parameter values. | +| xx_ref_values | Resource parameters | Reference search parameter values. The reference values are normalized, with the target values stored as records in logical_resource_ident. | | xx_resource_token_refs | Resource parameters | Token search parameter values. The token values are normalized to save space from repeating long strings. This table acts as a mapping table between the normalized values stored in COMMON_TOKEN_VALUES and the logical resource. | | xx_tags | Resource parameters | Tag search parameters. Tag is a search parameter (rather than a type of search parameter) so no reference to PARAMETER_NAMES is needed. Tags are token values which are normalized in the COMMON_TOKEN_VALUES table. This table acts as a mapping between the normalized value and the logical resource. | | xx_profiles | Resource parameters | Profile search parameters. Profile is a search parameter (rather than a type of search parameter) so no reference to PARAMETER_NAMES is needed. | @@ -134,6 +139,8 @@ Search parameters which are part of a composite value include a value in their ` ## Finding and Reading a Resource +Note: see the section describing Citus for a more efficient way to read a resource when using the `DISTRIBUTED` variant of the schema. + The name of the FHIR resource type is normalized and stored in the `RESOURCE_TYPES` table. The `RESOURCE_TYPE_ID` is then used as a foreign key to reference the resource type throughout the schema. ``` @@ -535,6 +542,7 @@ For example, for the `STRING_VALUES` table: // Parameters are tied to the logical resource Table tbl = Table.builder(schemaName, tableName) .setVersion(2) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addBigIntColumn( ROW_ID, false) @@ -648,9 +656,71 @@ This was necessary because version 4.0.1 of the fhir-persistence-schema cli does The `fhir-install` module contains scripts for building Docker containers of the IBM FHIR Server and IBM Db2 and, optionally, bringing them up via `docker-compose`. When releasing new versions of the IBM FHIR Server, the `SCHEMA_VERSION` variable should be updated within `fhir-install/docker/copy-dependencies-db2-migration.sh` in order to test migrations from the previously released version of the `fhir-persistencne-schema` module. +## Distributed Schema Support for Citus + +With Citus, we can distribute data across multiple nodes for increased scalability. The cost is some search functionality is restricted and some queries need to be executed in parallel across multiple nodes which can increase latency. + +The tables are distributed as follows: + +| Tables | Distribution Type | Distribution Column | +| --------------- | ----------------- | ------------------- | +| whole_schema_version | NONE | N/A | +| resource_change_log | NONE | N/A | +| logical_resource_ident | DISTRIBUTED | logical_id | +| resource_types | REFERENCE | N/A | +| parameter_names | REFERENCE | N/A | +| code_systems | REFERENCE | N/A | +| common_token_values | REFERENCE | N/A | +| common_canonical_values | REFERENCE | N/A | +| *all other data tables* | DISTRIBUTED | logical_resource_id | + +For Citus, all joins between distributed tables must include the distribution column which in this case will always be `logical_resource_id`. The logical_resource_ident table is now used to manage the identity of all logical resources, including those that may not yet exist but are the target of a reference. There may therefore be entries in logical_resource_ident which do not currently have a corresponding record in logical_resources. For local references, the resource_type_id will refer to the actual resource type of the target resource. Where the reference is an external reference we may not know the type, so the resource_type_id refers to the resource_types record where resource_type is `Resource`. + +No support is provided to migrate from the PLAIN to DISTRIBUTED flavors of the schema. Therefore, to use Citus, the schema must be newly created. After initial creation, migrations will continue to work as usual. + +Because the logical_resources table is distributed by logical_resource_id, if a component only has the {resource_type, logical_id} tuple, it is more efficient to read the logical_resource_id from the logical_resource_ident table first (this value can also be cached by the application if desired): + +``` +fhirdb=> \d fhirdata.logical_resource_ident + Table "fhirdata.logical_resource_ident" + Column | Type | Collation | Nullable | Default +---------------------+-------------------------+-----------+----------+--------- + resource_type_id | integer | | not null | + logical_id | character varying(1024) | | not null | + logical_resource_id | bigint | | not null | +Indexes: + "logical_resource_ident_pk" PRIMARY KEY, btree (logical_id, resource_type_id) + "idx_logical_resource_ident_lrid" btree (logical_resource_id) +Foreign-key constraints: + "fk_logical_resource_ident_rtid" FOREIGN KEY (resource_type_id) REFERENCES fhirdata.resource_types(resource_type_id) +``` + +The query to obtain the logical_resource_id is: + +``` + SELECT logical_resource_id + FROM logical_resource_ident + WHERE resource_type_id = ? + AND logical_id = ?; +``` + +Because the table is distributed by `logical_id` and the value is given in the query, Citus can route the query to a single target node. + +The `resource_type_id` value comes from the `resource_types` table and is fixed when the schema is first installed. This value is not guaranteed to be same across databases and schemas, so it must always be read from the `resource_types` table. For Citus, the `resource_types` table is distributed as a REFERENCE table which means there is a complete copy of the records on every node. It is therefore possible to join to a reference table without needing a common distribution key. For example: + +``` + SELECT lri.logical_resource_id + FROM logical_resource_ident lri + JOIN resource_types rt ON (lri.resource_type_id = rt.resource_type_id) + WHERE rt.resource_type = 'Patient' + AND lri.logical_id = ?; +``` + + ## References - [Git Issue: Document the schema migration process on the project wiki #270](https://github.com/IBM/FHIR/issues/270) - [Db2 11.5: Extent sizes in table spaces](https://www.ibm.com/support/knowledgecenter/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/c0004964.html) - [Db2 11.5: Altering table spaces](https://www.ibm.com/support/producthub/db2/docs/content/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/t0005096.html) +- [Citus Data Modeling](https://docs.citusdata.com/en/v11.0-beta/sharding/data_modeling.html) FHIR® is the registered trademark of HL7 and is used with the permission of HL7. \ No newline at end of file diff --git a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md index 76af693b8a7..29826f56c93 100644 --- a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md +++ b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md @@ -21,11 +21,12 @@ For details on the schema design, refer to the [Schema Design](https://github.co ## Database Support -| Database | Version | Support | -|------------|-----------|-----------------------------------| -| DB2 | 11.5+ | Supports multi-tenancy. | -| PostgreSQL | 12+ | Single tenant per database. | -| Derby | 10.14.2.0 | Development only. Single tenant per database. | +| Database | Version | Support | +|------------|----------------|-----------------------------------| +| DB2 | 11.5+ | Supports multi-tenancy. | +| PostgreSQL | 12+ | Single tenant per database. | +| Derby | 10.14.2.0 | Development only. Single tenant per database. | +| Citus | PostgreSQL 12+ | Experimental. | ---------------------------------------------------------------- ## Getting started @@ -112,6 +113,8 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar [OPTIONS] Note: Replace `${VERSION}` with the version of the jar you're using or use the wildcard `*` to match any version. +Note: Prior to IBM FHIR Server Release 5.0.0, the default value for `--db-type` was `db2`. As of IBM FHIR Server Release 5.0.0, there is no longer a default value and `--db-type` must be specified every time. + The following sections include common values for `OPTIONS`. ### Create new schema @@ -127,10 +130,10 @@ For Db2: For PostgreSQL: ``` +--db-type postgresql \ --prop-file postgresql.properties \ --schema-name fhirdata \ ---create-schemas \ ---db-type postgresql +--create-schemas ``` ### Deploy new schema or update an existing schema @@ -163,12 +166,13 @@ for the IBM FHIR Server to operate. The FHIRADMIN user should only be used for schema updates, not for IBM FHIR Server access. ``` +--db-type postgresql \ --prop-file postgresql.properties \ --schema-name FHIRDATA \ --update-schema \ ---grant-to FHIRSERVER \ ---db-type postgresql +--grant-to FHIRSERVER ``` + If the --grant-to is provided, the grants will be processed after the schema objects have been created for a particular schema. No grant changes will be applied if the schema is already at the latest version according to the @@ -333,10 +337,10 @@ For Db2: For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name fhirdata +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name fhirdata \ --update-proc ---db-type postgresql ``` ### Drop the FHIR schema specified by `schema-name` (e.g. FHIRDATA) @@ -353,11 +357,11 @@ For Db2: For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---drop-schema-fhir +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ --confirm-drop ---db-type postgresql ``` ### Drop all tables created by `--create-schemas` (including the FHIR-ADMIN schema) @@ -377,13 +381,13 @@ For Db2: For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---drop-schema-fhir ---drop-schema-batch ---drop-schema-oauth +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ +--drop-schema-batch \ +--drop-schema-oauth \ --drop-admin ---db-type postgresql ``` Alternatively, you can drop specific schemas with `--drop-schema-batch schema-name-to-drop` and @@ -547,6 +551,23 @@ Note: the jar file is stored locally in `fhir-persistence-schema/target` or in t If there is data in the DOMAINRESOURCE and RESOURCE table groups, which is unexpected, the administrator may run the tool with `--force-unused-table-removal` to force the removal of the unused tables. +---------------------------------------------------------------- +# Distributed Database Support for Citus + +IBM FHIR Server Release 5.0.0 includes experimental support for Citus. Configuration is mostly identical to PostgreSQL, except that the `-db-type` argument should be given as `citus`, for example: + +``` +java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ +--prop-file citus.properties \ +--schema-name fhirdata \ +--update-schema \ +--db-type citus +``` + +When `--db-type citus` is specified, the schema tool builds a slightly modified DISTRIBUTED version of the schema which introduces different behavior for some indexes and foreign key constraints. For details on the DISTRIBUTED schema design, refer to the [Schema Design](https://github.com/IBM/FHIR/tree/main/fhir-persistence-schema/docs/SchemaDesign.md) document. + +Note that the datasource must also be identified as type `citus` in the fhir-server-config.json file. See the [IBM FHIR Server Users Guide](https://ibm.github.io/FHIR/guides/FHIRServerUsersGuide) for more details. + ---------------------------------------------------------------- # Database Size Report (Db2, PostgreSQL) @@ -554,8 +575,8 @@ Run this command to show a summary of the space used by IBM FHIR Server resource ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file fhiradmin.properties \ --db-type postgresql \ +--prop-file fhiradmin.properties \ --schema-name FHIRDATA \ --show-db-size ``` @@ -564,8 +585,8 @@ To include per-table and per-index in the output, add the `--show-db-size-detail ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file fhiradmin.properties \ --db-type postgresql \ +--prop-file fhiradmin.properties \ --schema-name FHIRDATA \ --show-db-size \ --show-db-size-detail @@ -577,7 +598,6 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ --db-type db2 \ --prop-file fhiradmin.properties \ ---db-type db2 \ --schema-name FHIRDATA \ --tenant-name MY_TENANT_NAME \ --show-db-size @@ -592,8 +612,6 @@ The detail rows are tab-separated, making it easy to load the data into a spread ---------------------------------------------------------------- # List of IBM FHIR Server Persistence Schema Tool Flags -|Flag|Variable|Description| -|----------------|----------------|----------------| |Flag|Variable|Description| |----------------|----------------|----------------| |--help||This menu| @@ -613,7 +631,7 @@ and grants permission to the username| |--tenant-key|tenantKey|the tenant-key in the queries| |--tenant-key-file|tenant-key-file-location|sets the tenant key file location| |--list-tenants||fetches list of tenants and current status| -|--db-type|dbType|Either derby, postgresql, db2. Required.| +|--db-type|dbType|Either derby, postgresql, db2, citus. Required.| |--schema-type|schemaType|Override the default schema type created for the configured database type. PostgresSQL->PLAIN, Derby->PLAIN, Db2->MULTITENANT, Citus->DISTRIBUTED | |--delete-tenant-meta|tenantName|deletes tenant metadata given the tenantName| |--drop-detached|tenantName|(phase 2) drops the detached tenant partition tables given the tenantName| diff --git a/fhir-persistence-schema/docs/physical_schema_V0027.png b/fhir-persistence-schema/docs/physical_schema_V0027.png new file mode 100644 index 0000000000000000000000000000000000000000..48c12d2e6d1f54b5c55d4c5c32195a1b87e89fe3 GIT binary patch literal 212701 zcmc$_Wl)t}8wR@RF6o913ep|YjewMtG?LQYjdY2$bV*8gcSs{hcXxN4XY2R9-#cgK z{5Ualo^eJ{_F}Dj-FaQt6RIdLg@#Oo3<81B-b#xrgFx_&AP~$JB0TUfP2>tWz+bQq zqHk3YfyWckC$w{!O-5=*xJF&#_<@YO%Md40KF9# zQE^Q@Sak7JQN0s78BP24`QnT43r$~PUqyfE<6&r8i7@~2#t9>l)jIl;&O}x9-=ZIi zDxtZAso`@&!2YxnuPNhv<1jT}P(XeC^5*jCDTEN|%R>2RHW7*U3j^1`_rH+Ld#h^x$E*L?JjUlG)eRWtfPH-O{4TB1+A(A6DGeoIvmL(2Im>uGeDH} zYPrIfc5NnFZ>>9B182*as@g#If{Y!zm9q;3!mC2^nD+5SIrY$ICC0*8LOeuh_6_>c zG)UC-4kbU+ZeG`H>YPx^N60;oS@-q^bCM{{(c8XPDH}sMcS%Z2gvr_K46iix3C+&{ zYv=Z0S(b|e>$&N9I7m;L=g1@%eA_K}12--5jQ%^!yhbtgCEL_G~WlLdA*s8~gFxu7t70rLePv&;umV^qY@urad6~mpMEdGoC z`mHL=pOje2Y%uNaNXmr^TyN(1?zr8UEU-SrV?e@p@l|G+i-fwZB4cJwA$vDG)_K$e z*3UQ5#A8LFuF}6AgUCOez(PcMBdV z7TzqpyMj$M`Hi}fwg5sbuwH>QA>NynLC~u;PcwBZ6K66t7Fd9tARku0^s*C{i# zYkUdzUy%FEMduc&EV?HVe>3-yi;|BDZMiHRrRHuldMcHVJI3vt4h-TrW<#_?c}2yz z3oA;MtyFiLQ!w33#i5y2!dgw!_R%;pCslBUiklAxuL=w5`r>qH*2I3@qWaA|EAz52 z$d^(GU$Z4Kjqsn#i=l_4+XHae$J1JA3?cYTw|p8qfnC@q?nhQv_xmh0ag+M+ow3;49d#xIQHPq*aaB^AeY6PinmTJ1HIgVbYQb6T37yRKBVuJp zR+BHV@eeyUy94Uoc2T|O&x8(igvbq-9xBb$61xfTyp$>)jy{!4+Qt2X+{Zq-k1|uM zqKwu*0jc6;yDJK=4A;yF)Uze1oG6QT72u3Gq!?ago0`}La6Fy}&1|4t(C`rhq}G93j$ zKnfvVTSVscx%FSSLO1GAX$CN3Z0*+i%lyMk#xh2fGL( zs+glyvOOIDwLoz#^=E{_KifTm)a6Pox#ewJ{a^MQCf~b0a;=`!We6W{2E2I}max#| zxkqu4zEqwR3H-s7C>9TYS+gtfg2v~L=+j#oeby9PN^fq#j7QW6I5!X|Qa+lEepjNdmn#1Tmq3eJe-DN_}78O}64!E6{U ziI1ca^R#-1g*wEAl#TCYofjCmwElCt4IMkzanRLlDObcJcIoFJu!`L4m*eZa#bTyeub34->Sa_7kqe z!vU4SgvvKNTS_slcZcr??K78UsMx&P>dtir3xCJC`oU6&9jKN{HsnX!!e5siyM=1j+^Py;QXgr;YN%X zYHeJIfo`)bcv~VlQ(~8Wg39Ylp;g23vJ2N}Yi-ZaE;f2n!zaBvtmg2Y;CT?}3@CQ3 zGcToO--I@opx(d*XRW`-#+ofvPy&UrRwqT?oL0F>hb>3?j=`G=tw>nvyn!k6^h7j| zlve{PR7HuPbc{PW+AZ&m!Eg#B7eR|rjjhi<2csQ7z`a*H2w3{1#rEX#oD$hS2EW6y zlb20drCG647OmgenRbGXSh0CJ*!#S8@${oya|q628_Wr5zPB9tiwhZr^#8wUq zot)|ZxfknzK&EWEQaYT*1Khe5V@G#)a3@h6Jc!`T0rR;_Q~UmrR`SWACblZ)XC^z| z+HACG&eZrK&ul41E6KABffJePRy}+9|2oWE{>Un93h?zS%^%VN_|eS=TE4W<{`x6a z8@ubr8wX(v@+~2wCPL?9hF15{lB1^2M=KFUWQD9m4>JS{ml%GkdHz4mdy=9}vc`T9 zKi4@j!&P2Cr0|Dw-%EUoXo@5;`qBTp7`i(mZhK|!!uBvCpM?5%s2L=L8-`3G_* zQBQhCGDty+pNonLR5LPA(@pacl&U)| zIruML|>x=a}mTVi8of>n#%2$vc+aBXj%YLM|1HRrq$U`h%Qx3c~%nC26T10qJZgl=> zgHXI0krfPXhi%bJCKn;qskXg#rHjuA$2;Onh}fPxrRy}WQM6vy$I?O=>=yMS4%HlxF%JGjfTgyLVf%;bc@(pC+g1P>Ifu3A9 zJ09oP_zIlEVpaN^VcK-@Km7*bkt4R@Pf25&p zc)kG?hcjM7QP;c2JLK32H+0u*g8u>Yd&l@?*xmUmGoc?h% zZMeUKu~8jNUo6&&L7*<<&(A|GgkDM3*-hREE4M@MMk5}F;-EQ(!C#Cf4vQ$hd-&pv4m^(M0U zywJR#o<$_bdEuQLSV~q-UTT;K@56TEd8@`q$o2>vVXq;-Y2OQ2sBGFcUp>NU7+M$N zr$*6kz2^6;f9I{z&yc`43K&>a(&ymgL;L!=ZS1p1RjbJSE6SVc@U)ks3Zp}PRW{z? zee>My^j-5Z=|14sFzgA3Jtzvb-35Y5GP!c-*-FJLS?fYg8?6&n$gBBv32sq*%SNXB z(~&){5j6Q;9$~6<@yKcxvl-Dceb>r_F<%WzY~}t&QYW7^FoUb*_^ahOI1^FFmL;N- zrUZX{E>zKOal;?S;vD1`YAZKmCQg%t(<`f7HdTWC?U{lYFC*OMeED*e25g_(EXr|nr+K?kZT6ViRsa{Ix@=t4OW|Y1bUtx$z=O&; zm_){%ePrsxx@GHF3LM15Ki|EcMa0r{26| zR(d1H_IK+Y0LBs1`RzAKyJ0j}d&T)KaDDG$xER&2=v#TATGS$u0yJMwyuwm4w!bD` zj$&F-4X!ZaH(uurdRA}3MwrM3&d-+LlUN($uP`fA`<5ljcc6xbXk&-t zMW{~!XQFy$s=xQ3NqiB=WQsos$!Q@AXpm@#rylEr8NId%+OIputh*KsoU2bU?KWM` zko+Oc9I>;gvIXk{dP7SBrao?Q=+2Ob+5dc{u9Hw43K!F8+Y)^!Q9#Rr?1$ch2%3; zFf#_DF45SF4-N31Pmjs=Iv(rtWFN27rNj%!;hn5C-U5S!s?L#jY}zRuO?r+>aE0SE zP;Gv0ni~Hnt6tFj+05*hqevy18DhO7!EmivxXLoEhD+**S18|6Kf%O@akGaMxZ7># zO)Rv8DC(^7BAa%67nG`q z>ZDOE6h>gJdCp67NN5?Jqpdw6KYz~F8BUer=FBy(5S9TL3Q z0?%%tCmJc%lpz+alBCO|-~)>y(TecDo>+7Q$=voJ5fEUGfVM{$j079OC>I)M`pbp& zw}dk;Y`BYB(;_Lo9q2<}L;87B8&I|HYh%#E--)ZHn9;R|zEAw=6xmW$6qgj=^YIUq zsOqwi6CTnp@`eLLB6urn#^Hc&M7vdI&rP;f5I$a}J&w|fB#9I&X1(EFS4vz(fXAbZ z(YqAl&q_{xkVu7jv@%mVHJ|;yjv>%`eZ!e5(;5hsMwD%kltJ~L3T&UW2AxQCy=`a2 z4=(af)hSuqCWd@k)%1^n6jfS;FD%r_rAC(8%ks+-c}yz8G%9Xk9W^;$p_^~!0zOQK zfub$GQJSh#u(nO|uaC}bbXa976_;DShdx@AH@NRG(r!$T#A^ALsg{{|{wEltbh*C3 zF@fzlO!13vlBFOHJB14g9&|Op#l)JV)*fo)4*2Z}L5gOTD?an`oqe*w_~{|e*8bw+ z$o9V(0Ps}G!8t4J9uqPLD>?NvfyBk-Po|;9PtL~Hs*dGhK+b)767D8~y@E?5t(=bX zBT*s!-rC>4glK z+nb1sUm5s?hiIn>WYEYA71>{1{)w>CAjsv@5dJD7g!baM)KhRRdI=OPY<4iXGInK# zxP-wL$c47fSp&1v@2BEZQzO27!7TYuC3#w#!exa_PEPLX>YAxurW;nLH?_Vjd&IN7 zxhaAPYh-H5M++Amsg|3y=Kj-N%X-(b&Cfm_DY*$5MUYXqO8xd zeoXJ3;MGbSHuh<%pOM92ZcU)F%%jo5@WQ9sTSA!{KL*%y6e-`@qRVnmICHDUo*VF- zTwO`3so_05Jdh+&#;m`#J5LUgMvs?)3-3{0-G>wLr45sBNneQ0$RJN((%C!USRm_5 z20FgW6D18W7_LwSx-+1~_*svp8^I(TF588y?VkdYTD|6HU-x&?ybTSb ztr5>sg_U1cp%PRL5UuhBjfRcJ0rMcWCREOc?%&w(2fP;Ib<(7oK}d?Kv!@Ej#njl6P$`B%lfk$m0 z&yj)xwLL%a)mqI5^@QWE=OyZP7QWACvG~p4dA;bev(Vs>+&5qSiE7rn0yxXZ4u3d} zM#tAnuIp$K#N6meNJwE9b^~8u?ku+nwLYB5=H}+EF#Fu1;4x}`PG&d#8OEHCVXPx$ zIzlp(%#HxUBq7m0dVuQR6O;JFLBxMK`F^I-808zc4SME|>Ct*$^!flzM#;qu=wvmR ze66^wwG-GdCJ|8>F1=c&&-0VCoZM&m{Mf0WNgjm^K~h3OLLTP>0+7m2-A_PFtFWey zONeh6(LtGMGLIcO6LFYsV=*l$ zEDUONJ`6jYubYpO6$qufjLSI_6#U0)9fLL-I-b#`@GDOh@6PN~=1 zt;6#R2*jkP>#j9!r+c6uw_KBf^lwg9lZ1Q(U&$r;uB_-+n@=IPJ)DJuZ*kXSbHyd% z=7MEmZJ!c|^wP)H+i@8==rT>kuqCTGq;+M*r@|$3{Of{!y@wsk!`fX!;Y$SdyBkuj z7!Ay#i&%;+W-E=8SPY;WU5?S2nVBQVg~$No*FyhX>j`JIS>op~8v~`MrzZ(`a9J-l z20q?j!+^4La;2V)9nzw1Rc|VTj$>QJK-mYFw>-C+k z_r;{76ibsYyKR%orMWB8%-jaUQ_~m6!`vtJbo@>+!H| zIooErwLmE+NNkN+yB-q-k3o2`(J5=(jKyk>1JU1?+~+|l=86#y<#(B0G~i%VU@$b0 zvya3Y?hDRi1TEG%3idC5rphtVq*>INAkXZ2h-3X0Ew1>*PhyZx-?R^#oeGQ z2(8w_-rm04YMv{COknP6E`d>t#r?ueyWSRxx%mVd9-Snl$>n%1O|OF2VQc2B=tI>O zOy9BBS4ZIRFT-NQkAw$|k;{w=Fp* zFykzT2`ersxpuwmgK)Avq^+Q!FrU%V+^o-*OdUP=sVyluIQVdsl9Ccc;Ze9(d4~v6 zFV*hIeH#n8?rO`~7by5lp86Pd6KjKqV#Zs{br@HW@Eu7P2q)&Y27w^C0I&fsb?dlU zcb;tGNhb<(rtx4x?elwOWlOmC9y)E_r0@Y*-S-;W-N1!{w*mYkq#upbd=e1|VJRac z>dwwikhIXI>*bzm^`~FGSGk`u1ii}L&P{eM-9cv=-)m^33iFld@81sFt@o}1wW-{T z6dykrC|23>X>V$$AqFNBaBp8;CjXVMAQk)S-N`WbG7q(o;9Hp^xdCW(_fljnpXt?k zhc`yJMPsifBep+Am|>d-NnV=rgR>`gIHBD+#ti`bR{zNFh_s~tP0k9wO2XRZ-PDp9 zn?ljarcL-3KpU3F^rN8rYdEe|TeG2>8ab4H{})Pt4{vXD+P9RYPuN_lrJF#=J4g7r z9DFvhf9bjwN&|5U0#x{e*(wAW1auV+?@#M_zjjwvSN%x&oL43b-`75$pPiljq%Tvc zSGcBQ`pJ8LOa-{S0Le9^#}s2_WqsM&di`4Jhnj`Q<5E5ePzS^N$UIOHFv!I3uMWqX zTx>25=kdwN$i6CMJ0H$X9g+BD=PU6!?WwHQtmP|hUUcXx2t9JVl1~k(G93le(IHz* zm-YY|Nha0d_*X4xEcY!oH4P2lUu(&P5@LY9=+Lf=aiV=aF35TT@xi$%)-Emev)&}- zdvC0m`}5x@u3*jRD>RN7kEPF5+zisK#=_x>oQ+cEEBlnkRkERvjj1*2>c76fvja-r zi!uX32K@hO0rK+lC>a<~-=v`fr>u891@BFl+3@^HoxNOrA<=ZMiHSwmk=yX_@<-1 zdl{xJb9gTyKBe%bjwnFO;xtkI%_N2jCF`(%{5|T7!$<^l&c&!uDSxrW&$!nb4fZe5 z$@me03iK|8i}KBzxmw3_AWT?1t}JuEfA2KofOKquZEZq;8-AC5_-Rc}+Y}8X_X3?J zF2To(_XLrWsX%+^adl9r?DfSxI&LR`L_zSD+WYZR9oTzmcV(bV_whmlyWsr+yLO`^ z5^xxe8gtdq^|F>LB9oES(C%(=z+eWu6L~w+Wf=#X~Fqz*SAh4|DJn+lqroWWK#4}41VOcIe7+o*qf zgC62#RIb zS+!-zIdi)#m1*dlEvh`8vk~$45jUK(s8!w0Vx^h~VWFWNG#Q@$kk$Z5yRQ2`>dNib zC2#KTph4f|)51go;Illgj}S31`c}JwD-8S4d(M6wTtV#$0*wkDF_)#Vh6Vw|!m7n; zl&@dEy@FA?x>`+hSrP%744?zGy}z=$IGDwTf`UrovCA%b0II4)I36VD041v2=h<7U zd9lf5kW30F@{k|Y^SZ=M_rBYG(RUCjWVa>`DUCo1Az(8?oU66c%6o#!E~QUmPxhkbHPEG$sw zG(9n^u^DRmE>@wm2e9_dblcNqJ9A=1MTP5nBzO6zU$1~q=nsG6gnSEe4WMVp0@~u8 z@f?I#3h9u!=wQAMLo5ij1Lz`v81#E7o1hO=QJ`;@u(f5uW6}%oJqN{AVGDHNcI5fgj57zI!dM)D4&;0+>69KYaLr8zzz)$NG#Pj+j9*dCvSO zU#2XJqgf)n_73gE#3U^T3gQXQcvdMA5cHd2b!+ORjV{&(E`nD#kJF^m560|4qYw4U zbKHRQ2`|`J@4P_-)Es4T1o4XqMV{N}huym{TC9$}XTfBNcY^kb$57)czSyCOEH#Ph zh1(iGe!pO!8P%@S*ZWGCd<$;Agl^>eovG?NepZ;LW1}8rJXG1XEQJ7P>EVLdEKoB+ z^B4AI8G%$3>_{R`QNX)^!Wjtk^YiJF4Cz=eFXPsy^?V{HCsi(Xe<1*_oFwS=2JjQZ zlpoZW)J2o;6YTBd+=)~}X*j3MIbXf|_PM`bTI&J0rUV|-p(NH?8s%edqkc@!yWRW( zy*5F$QteQnvzl8wf2*85@mgu)1ypKNUT-4J#m!|AJQ40#RTfg07ST_g4SViS_)&&E zlj$=zM|!f&rPmjd$E@u1Kl&w(IuyUW-?H;2f82cLZ=RHh_8zVV`rYgNcN$*p zzUSQJx)*gWx08Dp4d3H{(m((7uKxLX_g6k2pFrZup5e~kEu?GY_;Yf$JX6P*N66mW z+l7ZxxXZftyMi&Krb~5j=N-cP)8e>tWDEU&#WUcd>rAc5`+-_C#pUYZ#%qSg7F^3BAArAV~z#83NyH~Se}{oBx}Snwx5xhmotUXwd`zvA*b@*Fb?&KYW%=M_1EQk@JLIP-X7?8%$s{@^9C_o3}*JCNJ=t@;hegaKYd&Fdc%FUSS-)rTO zr8;VA`S`dK-|D)#E~aSVIB`2bC>1IwJ=^*~Ua(m35+X2CejuAX!2qtON-_VF)~un% zZvqCN2I3Mm`Vu_$5;b-XO$?B5cxk|x#}LDZWhvk8Yu}dR9GHRRaX5aS;^L%nXp&aT z-c?x>0!HDaHe>-R&H{Xmvw$N0s!FnN`9u4>ImIDU<`NInD%edE{fcFxd1YLVgby5j(b6>|53DRsS`Y>9BA8bcNWOk(12pv9t1PXBm|)qtFb ztj+D9dUIF=KkcA0R^ZE(WzzjwD^oqOd0^j~y*}!AHU#Cg>2yd)&>s-qM0r2+axs4k z8+Vw70m(l$69Rf7fMBc{bGuvFMnkc+JrAxG73<6I%~^jh;ql6gr)C=F&{Uf*`dBDE zK*UGOcMfY9q9ZH!>alR$4X~knU|pv+m`*GpUqpg=^^CylGEw0$^}ITRVR5<>%S=1b zm$-Ed0(r8!;Ea&Z2D%T=<#c79j_nCRfoQzLdK$X;AUhLBd)7~#X~D8Pr!hQeYYWoM zk$4!X>6d8>+TQX+2WZjUmuc;%i^Pg;5>f@6Zw6SaxrKNkVi{g0viWePuJ4)mA?h*p zJ}9lX#~gAR_1JGliN5m6M0UjC-h)6i*WQu2hC%}6j#-n}F(g;5(W)^rdHy|TD-n!o zL_2fR0kuw3JhtNz>#y-jgg%Pr=;OLenqwWjzaV^x=&uvV7!U4s|31C$mQZ^1qn1^%3MI&#YeoiJ1%-zad#AJ3Bj0P9f$YL_4rW)&WY`#Lu^D znxj6h5FTWvBK%7q<$`Es=v~brzX%AnUcs+eqqFpKhy4m|z9ca!Kv#5)IVHxO5^6Pu zRl3a{4=tN--egPM$&*AYap=q9({)@^!i0S82^AC2IDYm#zw z&#!9t^LodY9zeSgQ5Ff>yrWp!cL@(p!3iJKk*7wOw|(BC=S(m!9aQX5>3%L*GpNvxZrDGZP>p(-jpB_;q;(9@jXNTPGf)e$uwIA! zHJ}WCp&I}YutoRv_9nPBGkAWTbiySV(zsFOhdr{?{GQ@6+|N9=W?uU!q)RjeU20UQ z-)cI~ScS{urQkQ1_JPsF$T$~JIV}q5;lSQZ7X|ZuO2i~13@GpC;(0hU$JsCu*;azt zMt}4j>-}NeSMRWnEJ|EQcls^ZW#AF^c&Rn2YknFc{2)L%Q~Z(=IwtwDOi4z%_~zT}$G{HikE6UfYgXZ4TfjcKT524q z$|A**$$@MQRtY`&(0HCsICJ;Yt}h{+lJb{XaJE|jMZWc5=N~<;5}~rq=32_*TL3OY zFwIxaBjvXv^d?h-b$`v?m6;F?N9abUJ)+WO1f>E!mYo21`N$Z?_`d#9Zt6iv14kmx z`MJpzUg}(4VaA~jC@Z{o3>_-S5Aa#j6gN-5S`XOtYDdI@O&ud5KX%VzKQ+g@Q z>!IQuVXs&yi^n=e#5k!*8Z@GT+_v%tqvch#lB}F|HA*5j#@bdM`;FD$bGgf-XN24N zVD`&n>fZ95@UTr61;e~3KmkU(`$V$oNv_J}u`HDqY)FndZa#T3fkLpL$?U}zhOC-h zb5X(X(d8doh^HaZE=HgE{xgwxIvvip@Y2`&0$*iue&mtyB{F4xS`^-?nOdg=AYUK` zqaO2>E7qrt{?ffgmA$J^1QRV4Rlf=*)|_cFOUrOCA7$HiHNEjRQiehlrdF;UIpr4> z*VS5))Ao{;=A{OBRR1UC>folGymEJzM1~d%--0sE)fTo^d-I*)8ICBeIpypc8SzJS z$;H6i_6STdBDCDKvTmQb>lrZX#1G^BeXXQ639g8)TQ zwY!42-~>WU^o8Hl9h=~jPxQLv*{4a1C0%>e%XC57{^3f}s-)b+;--vA9+^v#sQW$v z`bQ$7>H`M1wA5oNsq$+tN9uSPUr9Le$v8cNO3fb_IOt^y6(5h?*h@{dvXo#B_dktQ zWD${lLkQ2#rkzuD#p#(yWZ`#}Xm1}uTRj|j`4vJ|7o5Q&xPq;Z;qtfIIZ+`TU%}6_ ziLf0qgWf7DxsZ}iJ(o?;TrbT}A9>N(P!?=#ND1`2SqO9{HR|0SGy20E9S!k?~S7dcAc6rjfddxQBj8Sj+UVna&uVlUy zS{p^`QX4KHvrXn^THi~qOmtD5%TaiOyZ?y4u4ev1E(n_Xp&rVPTrMap)2yxQ|E6-R zKgRO`KUY`U7Z60RH*v3wTn?H|)+Ux%W)h-r@1Z*QiHWn>PImB6e)2oVfvFu~1fx=idZIOU=mP zvxvpBmL3=ga^CnrL6r3-A9y4y-}iE)_}|#jP}&r0(J_iIck>tomyO8rBTw>L8qFV& zScuLF74NT3`#37L*SZAT0!mrnQ7r;;2UguRG~Tp9+~BYCT|zJkjszoeMurGxSfUQ21t%2a)6T2is_!wq)^HHy`5NPt!P*+WjwaZbds1vEc@ zqCoWR`B-Q>4`DGOMj-JCfb0XD|CKMim;)d)lm^4Tcc`T{mr9C$Bl7sl!IS>A4#=+xd@^ z38bBA?O>_BPg$+`f@R8u2b%t4OCRG~{-X!_L*fFu6C4@*RDhkZs)u7V|A3iMB)Nvh z>6|)6F{Cb@?`|@A<#d<~aL29#bqqJVC0N`?npy8rcJK-%`hqlfCNaDdaD2Y2R%@s} z+;%7bee-6s>rX|;;#DyoEts(I9l2yQ!!_@Tu`o{^RHS553J($iznQ87x5sk(h3>3| z2_*x0zGAxsE73+PQw{2H&tkl$0*~{;*vFaI14;C60W5f9zi!o7$p1Aq7VE_ee@=_v zac;W0x@52-!p_dM0c;R#+GVZZ7^cMud?&8SS`-O~79dW&jCW$}57<$qpZyIr9N3;! zO{skg7UnkzZTsdtUNzz`{f{mR{J%wMK5Rh8-9|Y4!xytR(k1tN^pngH`AeXtrpXuj z`}?!lZ%6}F&rjfe9AGphw!diAxb8){cyVeJ2ghv>``|7S$j+_GlhLu=5>h~&GvJ>p zNKl(_La$alY;$0~Zh+;f3dQy4kAz2?V9v@-fu`>*dzN1018%^YgBWJlV7~ zc!BP*sN*%s*txN7!k=D0K!Mh@GEeCY>$|^^5nQ&g3pc~9e&a7AzFu}xi}!s=kD7+E_L zzkTl-FLBh++Lclfkod~EYfaY~IJ-!pSgE5=ED*oZwR5_w^&@1mm@@xyaw&t;1c~F2 zqHtvVMu2kjov*W@5)vW@fEft6$7Qn=ad)}D4j|*05?Hbc3?0Gb-XVN0N10;5=nyUu z!qS0Qh$3A9TCD=WngDRU+H4%U%4E2+-46x|$Ey9l1A%;HJz7Z8#N?Ix)j{ayU}BEi z4^5CSO}Z;I06zA1hhjs4xNVli59ex>G&B%fbeZX;Kr}f^IUlM_AiwL4 z^MTo5R*7Jo@oHya5}!-yVPfpbiS0Zyak1JD(P%OO;0FEZ)k`U;sk@yQ9YzI$fB(=j z+!{(I5cK4ECObIM(9f(sWLL1+d8LkJrlxppC7qH2@WJ-H4I0 z?5nD(V)cG-J(#P3%uZd9v9|8r&;ILc@&29^#JBQ3D@ha$cLku37x#$5zADBdFAd}0 z?`D5a98YZMx@lwfyo__SaOx+b0#{lGYbWkVf z40qRa$$_tok#!zQAYqcdm(>|q3J};ZAb_rhOx{Z^9)?#3vjDdT4_fKM5b9Rf^9lrb zWl9bX9D219;peBvn4~1cH2L6v5pw1L0tsBmM}VvZfMxOvcLB8$4Zr$&9*7ZY%%?gZ zZqHfu+d(7(?kW`|3aLDP0RKbD#Dor9To-_LL(q2!U~OKbS&3ns%xVa4GL+=k+|2Ll z;X%d8dFhvj;&Hhbh(^q%KUe(;s3vin&%g;~TOFqwDo@WCz1U$AB z0Fu43zi*-L-2+@<06^7&K>h&%!}CoA*5NeDz8omICYX!m;Pwl`!RCN^V#e*~f&3NHckazq=aQtGspx%njIEG)8KW znA7|-fS6LTvAr-bFaU*_a4>3=g8&FOP%@_jU<4=-p}ikYr9kad#p>hV-x2?Z*+@T_!@Qg1g@&yX&>+Ec)3g?3II>L05}=qo~~|g1`ju<0Qcgn`6B^% z>jpsh(Dgh|0}uCsYjHf7!E)N0tZ3n~T@k+68Hag#dV-Af-#9+`+qB;HQr>=Gz=MGI z;rPsw;^MHt&RGB+N@DI+UcZKn{O@<}c_uS6`1xzFh=3AdhFTbN3YWot78_qNlx_fw z4|3b4Wy_I&6D9MTXO_HA$t&S}X<$GiR1muXMg%icxsXK7(cTddFzEuqsXTFF0Br!E zX)cMY9RP=aeOmKB{!zGiYED<-?b{B(*+3w}zNj9=GVlR17WEx3#U%JKZIGlPI+hl|; zr0J(DYTo+b^8|#4kVu4n_<$Ea>0Ojp<)TIiX$+K{GNrR*b6(Fz$ma(H2O~Hi%=iJa z1i_1M0B;IGwSlDC4Isd$zqM=(?k;vA(ufOpuR@`r<$HHG@0GT{OxIg*KE~DA$w_+% z7FD_R;v0w{Mkc=z4 z570Z$R%mBdA#1ZGyk^+f^gvM*wp-0FN#V2)lTw!8ZV1e%G~}g=GQ5`)ztRVUSE(YQ zkCmxz0+2JrG`%<4?F|03)XXcN%A>Sa0~CSTCrcIr#T-dARMIy%KyvNs2tbHQObiCF zT(SILlz?IYpIMi|?h%j=VBzAzfdD?E3(#uFR;%j*{FxVpq^9P#13_AYg?c+c)F1#+ z8c>P_@v|7x2?@1zPyfR-t*)+)nb*ARrDbMTsJPOZwJ|L<-1iHgprI2@ehIaCFiMRp zyC>PUpiR9_JJ#EU8VR^6d4okkI{~n)p-glgomx$06rXDlc|uXlVc7Af%L!m+1)s zGBFqs;I-kiRi;2{!+jNt1ZYK2&d$y(t75;uOEKZI<{&yfYC7%&NN_+_00VM7S|H5u zx*}xKY5ZJW%~7IRQ?$SI}imV1z!>pfi|U5=hO!&49}7H)SFSSUa6Zl2{GP-7n1n zZtmP~KNywJAK>C~fJ7u|W=30Uy=Wd@ChmRQe2%^~nt=i89~qGxO&5UZVIaJ+cEbxu z1ElwSrw&o%0L(og#ps^5X=7~iQjVIe^s7uDi>sm5pB==YtLn~begYZeqi-B4yqZy3 zu|E*hwx6%@L26m^o^2RSh|rD;$$6n#E;Aw>SH+)2b;r=6PJ}hkMW`*V5>H$4kSSng zdn@{<)=VIQeKegLA;$qkKtP1ipWdCUc8%9qs5hVeq?i90A#=Xe*u|3`=f}K_UT~qI zaHSLCJIRMA--;9;@!|DXc5kBtg_oSzQX{(W?Zd?FO&)lPMmMdhhUn?}12n$Z@cvf| zAbXR8L)G#*cZ2tfZjthceS)%GE{Y#4H0tV4n1F4gKSX5t)JgLi4wO zK?ahu|8TXt-x0_ksz8A|_udhZtzgWydp#ePG>KEXhx69D)(5Z6G5ud!w`5)>5 z$j9=91eRK{S4PZcgLjWN*8CDFw2i_K9o9c@Dg3N_R-s$F(rwQ!6i7>e;YKOUgs@5J zKb!urNrTpX`YaS&Tvu6xcxr(U1?DljX9}S$jYc?{8#u4x=CtHu1QQKIyO5dQn=K1! zU-vpqzHqlUDSq1*!z4~uT^Ap4S2EOU&{Gjo&>WWd8qhk29i$l-b<3%aIR3#h-4y}n zE;;(fX%V!runebojiN4J( za80$o9aK@oCK7Hq)nY#K=vt*=a%b?crvtDB{{nwn4eImQA&KA0p~_YT)TTxQ`&E_8S#k*%iCVTo1OSJoF|9uORSoiwmpl`7Eo zl$8ofR-x z>pj`YAS`;T>{0@6MrhVVWJDe?aM=BwtYRtw=)+z_t%I(C`|FFrTVBiBc(7fV;lMy% z3Ht`aqXz4yRUKG%r*M+)VesPB7F=^c=YeRTi4(4bLq94zp_lWJTy1s&343g0fqNlx zRejjIeYMlgKAHEb$iN#ZC#0wD#nw>mxQk&2 zLJ87=WY&qp{)W)9Zr(NW=UGyz%6g^AuPt^*svN<^VDdV)blTRkn{C34+p(JoyMyI6 zxg`c}-iEk$Mg3ZnGOD{(jwC=6i3uCLG~=SiFJ)&$nbp|r5*3!OT$rg=eA}{XbhTB` zeR);AO1CZtx%CZ1KpnY}9ruAtG5Ldzy-CtQik0f!X)^w@ulg7KYTY-UylW^N9*uC{ zFrp-X$ICH%YQglU$ow_Qw8K(}%@5P^#r}q_R$)GDXfPQBs*#U`3Wv|S{bFWzL6n=( z8mhN+54T;L0jKC#mANm}n6)T-xPrA3I_SY{aKBtjb#1}~TDc`Pw7Yt)a#>zgigzRK zYcn-bek+Q|2`fsQh5soFiB44n2W3xFy@#oGvF2c}arkV`1I9_Byd7E%6ALp}8Vzy6 zLo`y@_jBB*V{%GA!~xy`ITPihQ4cr9WZrFl<5%aewnvWLS{K^SToPdvi)ue|^7E)WKa!W@ST%dujSi+kOzey{d#0WIPArijW zi;Q34cVF!FkOhwS-im9(?#lj3=5>B0DirFh^RNpKRcrF_8lXJpI^&clw%)9=He02-S_~5BPf2J6^p}9&wwW}YMiQW<9 zSN&&~M9Mu&sh(>h)^>n?R? zn%jwX+fBRI`aWx#Q=5yeHea16Qo04-UbNeo4 zovO7>54^EWg`B!2pIpmlMjHnnulnQ5T)L#qiQa?zO25DyNu1CZ`V9}h3CV>M~sa770~yE3qq-1Y)}C&%b2sL7ro4Q z3vg3Rt-)NZ30}q5xT@q<8CCIz4z;*x%iFE3o=(?^M(Lj3Pd;1Yj|oQ{e079Pct>2Q!P{F$MH#hkqbPz3 z2qIEaf+Era(xo5*(%mf}-Cas6Ez&U}CEeYONHg@%4MW$!(C}^WeShm)>wM>|^Upah z{KuGQ@B6v;UHiJO`HeSZ9sjqa+z5@y%ZgrbXHBxZg?U`SNGWGamY^ zc0zC9OF3kSC)T@s4vp3F)ET^Jo4fjZjM%)0iNf-xa`YUkqF=Lq2=?VHm|PzllbaWH z!_Ky7-rG*g#G0syuO*vW4RJjw;=2pRlT*x=Np;JX4Q=HUU4ueeCPtNtBgRmRAC08- zO(-IQ97$iV#kyZP4B}fFgsWcF=7v92M~=Em$1SU;Gq0Ewqqc4)%F1`KYBNMU_l8tg zGPCeWRRw1a$y}&RuI4Q%J&v-mY#3h%WDFw;?nsZuK6qW{8_?XN7`bdgg6W~QLyxKu z9F|R4vD4nWRH;tMwthTpt(a8Xd8ohgLQz<^jmmW9gCWFZQzP|nWzFQ+CDd8FhGmo5 zdAjs+k&NJATm~^e3qoFYfW$DVoqZju z(DwJ?(DT6RB_64_-!w?d_PWEg_xm!3ZL}bvVWi0p)4Un763ot_j8(^Bv9p!u?^B5E zvU~SdXM627J9z_#Bh>7XJ#5cWX74BVr!AAJVhS9%7P|z?p7c)Gdta(^^#{fc+UOGc z>ts?a76Hf16^6>+Q;wM&vOvtDUPHc^74b0{kZcyqTzh0jZeA+A|DEacku`t5O`P=s zrZJtIhZ?-H+SaVFDLWR5BzRtbCCig@#T*#%%<7Tb_?w~JLSM$1T1Z@GE(@}`C!eA! z&y{4c&tXuy$HDb1=OSO$hHeWneXbXs{fV=j^S$Ay-bSpT`is=e6pl%T%~}5t;iX*^)bv5bp!ZB~nari=e*fs}?E(L{ayat~#H{m& zsjSg10y|7w^6T;EN+T*`u`!vCr1)Du=w}s6ne+HXQza5!dC4bOm*zEeO{d;2OdQZH z_G9p2{z8(Ex)FZL<-^cXr2a)CvI^(@(lSlpHywjjV!!C^&7Y_=PaUUnsm0a*yrcAS z{E4oCNhU?aJU!7Sxwg;e_bFt=*tLlV<{wT6U!##5;O>_pHH5t=HnNM1`l8EDy8GHC zBqa6ElDEjm?fk1_?LKaIG4Tl(8k3Wf;oXjOc^%hL8zlxChsrY3n^_gT<|$u$Xve(x zO^{%kCvnju0zI-s$S+Xm2@!VfBX`A`0B*-q{F) z-KL;N(GUFu|Hvv{t@w{(E$@c=E@Sz4d$IXl31-Zl$n&F93Pw!zkUL+LWAB!FNIpSK zjm7ZwZaf5L*semobkK`ys0rP6am+TG%^DNkDzB@ja4D)wDvp1JMkH7jOnt@ek4ZuH zP?6F9M+yBD&)NG{^PrUaKwh=*Go}%x>`mQrvLa$k0e8wgVOhb&6Ru|ygPT*&L5dkT zOKlvs(1L*G7pqwg1~VNiBOWL=5^XgPNJ(Ct{I+r&yJKV8(G-LqEl=hVoI-@CWa4T! zHFYNFZq>6(!pu`ebui?AL@qxWJ*A;pRwA@6KO0nidStKYacNvOep{$krjUL;7sJ^OtLG8#Q|fcO)$Js(NRT>pL$iOoSfOOpEOx|hjvs;h@?ZSPZu|K zlc(s`M`T&;EqB;PFLsDqe?hBPhBkt&RD;krP{@JcqfXvLzb`G>YBC>JIf(7LHz$Lw z$)QmcK?%+qkHa^4LcfGT^sW*jHPb>9e8>#g<~PsH=m_3RzdY?8Biv!3Ym|Dnt?9(w zWOW(_N!5|BuL_L2TPY*~s-m=kG;N_ZEP+1XrY^5CC>i;zNAg9^+424gE9u_oz4h{Z zXq1(By#2`I{cyrNN*mevbWD7qjKWRd$4pAdwOt~=63aZ_`Ba$s>Ou1cs(2w~d8+B; z)+RNEimBf83GK#4#%Ou9kA|<9L|xRP{ABd}&{iCeP$hFy`crM|D$<^di zBf@TlH^F98uVYQ@bGL@&GNLSbFq;yT!EU9U{XT^pq&Q*H-Zcc`Gr27P^agzqy z+5C8q`$yHUFmcXrhKKDIyY7AUm6K~LA(L6iHmf0QRK}^!ZrMJRF#p_EYyh6UqZQ-y zG$f}=TBz)8w1?`bNUrbb2Z2^RY8A<|=oGh5VblWxd=8bp7j+ZZx*lI}lONlv^r3ty z9utPA*TY{XV+ZKmRFB5j^~r}lD5YlXZ4SKmIfNRjw>f;2;|VL5E%>uo)R+LdxwSX#-t8gAvDMEqlDToJ3W*?|16=wGP#AkA)9RIZ=;kJ(Ces7}o z`fwp(G0HrA{rp7j5>13bID1HJ^sB(75c`YYDk@w_44!=>5yH3jMT-ZSffN z0o;4N;d>3ad2=5c)#4M>v@}Q^Mv6W%3TEuP*DskO;#>t8)4J*!W_Y{Pvs=?)g{S5I zh>M4>G_brG#Q9Y(Yo~m`R_|>qnx%kC%zPdEoTcQNe2uDStKD}oJ&M@>nc;L~|2LzkIGOp5xT*@;+ zqa`duS-g6)ADcHxo*KMkkOysr2cteV@?slm!Rm-mu9=Qkb|ewh-N zz24&Uv4f!^a6{GjCu!tq_tHJ=`6vy3-RKXrjPPAJ+ZU{reAGzS)DPFj`3jQlnBmI1 z8&XmlieDAZ&J77f`5n=`xULUPtJ-4g(z>NO`g!B=mB099?m7FFkItwsQ}i>Y zI}Vq>UlHV(`Kq+h(kQCtc3o4C@pGOWM3+erE?E;;YbcBTd4ORy`(26Co4E z`anD>a?Porjm@pthSTn5liKpEc~0ru{Lpe8bf}|jeuZd`6{#+nE_BO7PkDrvQ~hV$ z?4Z3K&4OH@!Q5f&6~bE&R`s7+_~sld4(0rk${=qQr5MJ-yYI|>yV^h1Cl-m63efnB z8ZT8fSb=mEJ%Vv|HAx$cAIS2RqzT6=-eHs?(Hw(Gspu$*no+Uf z=ivJ4#AMRq+>}HgvLW$exja5hX~e&GJM_9QG)NftTl6JFn}f|HKyy?Y+OHw-hNT> zL|3~pIx{oElq+kUXC?ZUP7P{~L16a7D>aesYg5G#%$ZrB&cy*fDbyg4K5(jZiD z>RZ9+%Zl4RxGuHqPRN`py6-BHyo{$zBGdyF2|YOGd2)r z#iov_a0XqPrhe8pf}9yFk?e0x_*$u-Er4Q8H3fiN*^%(UOadQqIlil7VM-i{--f+D!OSiPs^O7J-9jEZ1(bBpx`Ag-w5`j`Z@y z8PfjKr7NoHOvc_24i~O_qCr}wH^HXk-v9LZ?9$(WBW0U<3_Ud441+rr;mZVxMu&gJ z_FlFTYF08(>&1#6WGX`A&^jh+8h7GMW@I~I{QEkYl#5s+SKWkrVjAd$i*Cgq=`6?e zc6AaS-@Ql_z4-lYJuO(7_%Pi{JNl92gaLOM98v6D6diV!>}W&u;cDrXc|gq7L`xyg z4NW2^+z~F5@!jHlgC)ET-a@$29ED&PZh2l_azfj|&a}mA>`Y;S|AT6xq+d;nI<1qf zuYWMkiOI74tPZPq_K`WE0HTjW@XB(;2)FEW!D{``Rh6Y@t|Gf+GXGIP*uW0GVbU^%i-Sxz_MgT(!zfi(YfcVe*$I7;}@Sh>D+`1&L*a#%;z5_5AhX8QMucLdfdWD z3^cOX9{p+GHNsI@S7jzE#XyB7Gm9oYetDzcYxR_Ptg{m-9y6z3i-UjbyJSxU->YCN zwjvRX7SSl?jTkYL71Br)RTSj+n+(LK(sEzWJx)-YDgL1HuY$^cxq6*lqJEZ)=i69E zN2xrt-4ejT?616aV1>&zKibD?@l!Q=bGkHw3gT(H+pnZ!mOg_qL=R#w|5J_Q%=7)sfvB#wR@y>ij-WYUsiT z%B}D)r*}!B*qr2qwhZ>ltJIs0wX{!aTm`J(PhVl(FQ@~U5w%5ip~WUN9;B@8bTp*@ zRgO-{3+MCFNU@=rrH)ITKW*HmVDXP;OO*zYbm0^WXLy%wsmwWJ6x8MGLXp^oUL5kLMPo?f-aTs;&E9js$?gG(?=yHHPog>! zQp#2V*lB(OM?wzkR9o2936-<>Jw{WR=Scl0 za3QuY9r3lB)3t#k4npY~;F5tNEz4om;wrx%OJmRf9c=UU*fqk0tP37y)q{n*ptTiq_i=cXki(N|hXRIy~r#eF~Pb1j8s zxt%B@^(uWnkIIbiky9~#bzI+vOKcRJ6enkh_eU)7G#W40sf5yzCu;o}GH;FLvXLxw zHD^D6MPmP)B|IX!;Aau~x+li~Ba{^3IOPvtkT)*mVV|TZl*1kqfcjxEPG)F0%Ht&gVfw}8973f}=?{(mc zQY7b1Z66g(z;W0AcGAMB?rbeIiAyqeF0V6(p>4gJ>^uu!4Df63g)z^#_hMJ0ftZ;< zy7q_5(tZ6c^l7C5P0-QyQ|cPVuxOaUc-2mHbd2ODS7t8ei_~LP9i}Qf*DICMHIJ?0R|BI|=klSF3|TlV&^jaX+JuAV!e<>Iy4{D?;_}9Ag>0W7A7< zf?QRMvWU0}-aR9IwdLV-icl{c>i%veyJFQv_s~h*jfFK%044Tj0$Abx4s+Wrf(*; z3ngX|aUwjRQSu>yH)^qaVnzy?Yl`6y-(Y52WXykJHfzdrR#`aTa5o6RYm)+d`ClDp zHN|A&57k_*9|lRRj|I)U2*^k`{LN3ixW7iGrf!ptW9jTsQnVHRaFaSs`?7yyxNvqU zAMvMZ^A80p%mh)}Mm|=O3Tg6;pb~cbvT=1{)>7*6!a;pk?BvvCqU}SnIHS>S%(%rz zln}iYt|-((MFPU8sQ%k8-j)mx=PL`cMz^SH%wy5EGRTb#son&`cysm_m*taEv05C0 zX(wqGqL+&D(Hr5kE(wy9!^u|l_d&IRawZ37)UtC9ny}$J{Qy7bBPbYwmEvFF_{DeR zYNS1U=}hK7M%^>t$b@hgyrKEN>9=fV#pU{fIqPG(U?dO;mp@PC0^$Cn#_+izy3dAL=r2Gs;3KC zs_#IVQd7W!i(WS#w!B-ty0I*#Bz=WvQ^Xt|9FFU!fd-nVk-2iLrzI|xk^yWZe=<06lyB% zb;+`CUkbF)9bAZdquRGP@Q_;U=5Zj2wOy*jX_(5hln_ zT$qgH)CQ&ng{Axd2{$h~o!n3C9E5CodQFyD=pGnaYELrylDUg(-mAenKGiTOTZ4OC zmytOM$^$oLkr0RMrJmS_LOkY;mRhf3M`EU#dE;J6RTLN-SXs5rMbB=XzlCNK&PrTQC-e2r znbEkCQfg&1*}w_IpZ!H3)kF#Vw(|llGA_Dp$+TU+Y!c+%A1&O1vVJC3q+BbIo`l`k z33rFSTJ`&f?0ARw2P=K)5~~vKo%#v`hRS~a+{Jj6S#H0r`)F^3%xB@dhu&)nzl0+y zN=a*!!oEXU!rr!CNy=3FdRBJUBSJ06s9;beB&oN`MzFO+^z3(B|F>=`cRx_Y`8scV zr9!gm|NcO1Kjr7}1OCsk8S9fusQCkw%2L#n*j2zD_E;*Sh(4dviGlk zfI->7i7(^HT$I#qHFoR(nf=gWxh$#q>3?297k#^F@FaS-^HPbe6eZn$!=LvP)jwrO zD4qn#r%vS-v**`aa!ry-k^HN(T>{~8kY8dScK;lU9c&~w2hdBlY=4u;bOD9E+fwag zuclySMWo`8tLHflQG=UIah+n$^l?N^d3blG+w?c9Y}*8pY%#ZsZ{LscOp$h zm<1bgRuP4_y!e&%Z5WF{gqDFw?fkUMV?kR=;pywzq5EEGfT1Rg@{aVNixCU+fK=OS zlsN}w3f9)^9@7d!Knw!c3DoSv`foN~xjzpunA7TCvXEbPuDj)G^d+eX(jYBG{4H3b zqSW02(?(XxcnT`&H%*AoYc@Y2D1Bjv%^D#n{K-pR?52@bc5j*uV*G>KkVxuiZCPv? z7dPTj?^)I-w-cEuIUX^EM0SdNR0&=eC|@{EAmmu@agy>^s%QrF0=!e|fL^v;yndu)=Q5 z;^C=k%zwM4(*JY;F{3-^g|i{|z92P89W4xtKLy1-xLYqRdH5YPlUh%Ps~TIi!VJbN zwO!AYm@ZDeypshNlcN&BDC|3GR65fECd^UcCh-S&+rmdDqLJ@H)=?e)8t$oyC~&O}8Fu5-E^#12%PXBGa_A;NO=`xdy zj6W3|n#;DL4pqbEVEn^=sK`mHcf~ z|B>cR?0b&0HxN*Q{)i_Yo~B$(7!Ui~>2st6vI zuaRXHw(2upLkT2Xvxb>Zk`A*Zt3K#LZ8(6z%e77LbLju2E(v*+C2#hEb+Yo`rn?*7 ztWwUd8s*d7>yE5mTAwsn@#*_E4!LqQ<&a!33}>%XOa%l;{K z9p`}||VF^3;DXy$M&bcLT|8)%>pu6c;~Flq(yb7nj*nD!1m zlVqA|Z#uOf?+8K1==~#Ramn|LF7BkB?l0@Y*S!!Xs;hQvep}cw;ztbyew6+_N0k`s zdm+J1C+Q5>^2RsEOaen5>|jCMfZF-e-K&T_rOXM2uWBXKp+a0<;wih4<OMp_Y1_pl=cwKhW5Q<*EQK|qQOKq;7DhYiW)y{YxKHV%xHyWClnA>33 zSLjQjO22tiqw+?NI$iGuziB{78+;gsM-hwH%ni~mS@NG}90wRE5yk~8`?TQZ9gG9H zJ5bsu|1-v(&gbpA+;39ve3`Ee^3Qz-&429qnRMSa@#Cv@964U z02*d^&T3r+01~`8T}PZfBnj}L@!U4L@~0(UXU^_N+r4Elp>F1?Da8f5@$qpWP85}p zAPI;E=wqT_K{@%r4Q4Z~c{{p#?~)8ZW4oTN;72hO(l8PSR+L38_FQ;{+-l6B8bjeg zo9`)~P<&Qs5UpqqFK}<-?uNpDB0`bR(zK#AWBZsrx4hQ;iVR{}S+4Brrg_N2pO7VA zOVE7=5KKS`IsSQ=4n*3ty$(kJrkD4V*EMe_7ufMQ298}MCqqjIYs;DzudbFW_IM6# zqoN|l3LD9>+f%8xs%$P;r&!Hw$BOgB`RMV55;B2=ylGVPoIiB;!gcr6P$AqzF8=_i zYK%JU+rvPV=7ZW_@SY@gmBVKBGrEr1IKwI?ebDtsODR4g%TNDRG@4|r~+4@Lc zdyJZH=NI(5XxGwrg6;?U!@m_?TTT1``hWH_Zis>33eQxFbgfS7#}BcmZJ`eYPL`sl zfRqdnDjX;?mGeGXCYCbtSj&t9LQYvg3Bh%RQ07yohMGH~o{iTyWb2*o*w+AgL!*n+!#2ttAZDhNB~5>N3iv~zN%aAcUmxBt60B>TFgFX-$F&(9}$*bAY3- zS%~>1oHFzzgpt>R>@|Orml#vi$FT zF)w;GhqZK^yP=)i@XZ1S*9T6opdvim1HhowYgtLI%Q0#^uDefx@C?72juWAE71`&i z8TSw%yT}d{pnz7@>R=}Jv-hDd`FwzePW=UHQrl(P324>@0_3PeCvDs-hZSilSdvat zKL9>gFNX79Go2dT4;QkMtV>@v22r{P106PE3^kw-H4pSL0roi=Xeri8d!b#Epn#qZ zJ;4jJj!hf@&cAY8?Kga@B=z-*NR} zo?4L!5HJvnXh2O^0I>cKfIvcH) zmPgNhh(3}!ZCy0Fm{-`1IqHA3GCqOrJf<`RW9=ogsc!*m?*mGI!J(l`m6O)jAo{v# z+-9)TE}$kRqo{Ztx&$td7taAdiwUR`eg-^=M^sdDKyhsisDjzq**UHce+9%Lb@!Rg z(||x6QV}4Ec&)PKbFrbP(csPrgn_&h(DS^2dKMwEo+lg7kTe=AHV}(wIvA9yKVAp{ ze*QQJ+TwLI9b`aqz6-_kUb`Yx(^gxo3VV*K!vz?QjY6z6bK+qWgu3K2W zfECNCr#nN>>leco8#lSN5-le*55!M0-Hs9})xLEr9+AR!ss=_YzxE5UZT<8hm zp{Dgj`By;iAo9ouVF!&TMHZ-c$O6XOw?kT9-h>BaSgNR(=K`Q|czNmqC^NFzLkDh* zaMs6^l)48yK-m-xqkH zTa)5|x>*9BJ9v0aO1}ewF~su}`W^smpjOZL3Gmm^^gqF#PrUfc=GCG)$Z?K@J6@Yd z@MZa~jTR8r+={7acvEuPAWz!*OFxxrla^d+;m}>Gz7oFCws9DHoGd)S0z16mC00Ev zc3e{gx>eFZQPy?t%iV93cn==X3J4^FNLLKj`0J{)tgPnh^6X;ps<`+SOYJI6}3^3dZ#*BdK7zWnuaGGj-dm>g`vDG!zD%H2{O9jqBEh)mXi;!#U1r?q06hjR8Ay>y z!;669gBQne%E!uDG=?PP{p;_4M?^&ATLUJ@m)_45n}xn~|_^;;Nq3=Cr` zDlI}+mfJN5${DXC9U@j;X>V^KATF2=h+X9T&N!dXw(BB*I9IZu2PzT*fq}#=WogL- z7%N9oXM|oCB7n*R2o!O3cmX>pfGYz?3`MqQTUF6fQMZ}_SFz_=gg`5x69W3h3aATA zfvTgohGt1WAU$Z5e!LIlB>9ZSTLE$838(oV5cu^#S{N@kBLF619y$at{cdq8uRkDK zjMcfYf>6qe@)Qdnn76g<<%I7xqvJ)Urpjt+KC!9a3xvT5ufv%b2xlK>(=4+ts}r1i z7f7bk$2B5qj`khUT#XWQuWwe&X#$8Y)4i&5#XY_#j-WE~a8(pkF3x8+CMp=IypO3} zX1%xp^KTiDLx`j463cBO-DeFm;UnhLU=3h>PhpMNV5I`uKOA zgYyfHo_w|}1@LI?LZS1$2%ik_ny0+aw$K2jCK2r92MCFYfGNgqHuxN9(_Pm*7o%4| z@2gMv%2Nm=be4vLL5@@ij~&f5zX&h{SE9rzuW5sp)3q_(tWD&faVhw&pJCl};SAK= z)L-9qSn0Vn7tX&uH{}4!1p1TO&f~^_aZ@mM=tZ%pBqyjcypkpw#B+6y1bGeDWywd;E}4pTdk`fvCGH3!C%NfkY{xm}=}GL5PnE`We<~lrWw&C&+7|~-ym_uh+)g|% zHjUqj`5!w9f>m{WJRHTDNsjHR4hq7oW2CWHG_J17`Nl$D#k?jSU*Skczx9MLK%2G( zImp7Z$u$#S`9v%aocR$gnVhcsOG4T$>RGSka~<^6r)QH0M|6wx5_3;azMNkcYyzQ1 z_%B%e?A6hnA0B3%M^>);RhoO~Ti_-P!s5mJPgUR>N@|O3TIH_|%o2wq7WQv-N~czR zrBdBAlz^47I=sA&En2TcLiXJqsChP*w=AAs*|$?PSLT*AH?aB*Bp>%}?8z6!#wEeR zTH*raaG<*5>MScX%~!Rn6ahe_ z$m;!Lry=qsGesRwC=gF64K{;yAFhM-pFZGuL;%mHhsFuSrMhT`o`D+mGrkh>)h- zTwk|;SS^{f?@#EJ^0n_xa}l5@dzZ%>LMR%I+)5^LC}0gt-t8^ z!223}2Pp3Ur-xAofTylM%c$MagOPXsyd)f@N(rzC96W=bU|DjZ*zOD}F6B3P0QHbj ztI$kK)ADGu_yfcXtc-?c<*}n=f@g^>nc3Y4WYAly95SWGOm7yl{%g-e^kgLStSPpsEuD>JC}x;d|HW`5t1Fb-5lu~x-b@pMnOzDkUMZpF@Hu9@wK z6n63~6z{*R1<*XR#iT~Hm1AgqN_g14JJWmNzEu+E(iRfOon3_5aJv3Dd+hhk6=(N6 z56YtPh!A_@_NH$p2%~bLYx}@t^I=r%ZaeMin|w0X@*vg~eX zLmb4f7uKX*A(;;?PL*WRtc%;!p0)*%C}-Oj-szIsvoqTFQ`1RAR z5Q!1|Cx&zkt?!RAUZKeC*t{MKh)Yx8RGNuC*4Y~6&t8hx#KVj(Z87S-RtPIO=&UjD+ z*V^773&bBr-r#d`(;28759_@Yi|gbIcx5LeHU^u#HL@Jq8~KW)+pp&JIJz7?WtQwy zYV_V{OnbPgS=SafP`Lv&07IFd=!FEb>h;sTiH_XXc;)#vi|0!Z*Tos<9^qR&`7?GR zy7Orex9-?``o8K`kCBz8P=jXRP?HnSywLfJEemyUW4-HI4*PlIYH45@V*WvcHKKsz zPJJ`$Bwp{M-FW9+TnEU#obP_3vt+mHkIQm*yVQ2Lh%Ty=A?WxeLWj(!X%vNI7Rjgh zo(9UCArl(5?*PO>v7tE2g;vl|o)`%=s?_H%p*Krz>CFJ>k8rN%zmZN%L@!abZRaAV zdRaTP)k@YL3}r{yd05BaCIvbpqs87aeHwcCWAuTjMs#dD-iF+(GdOvyHEhwhay~hl zB(NaVEiU%ooNABip93v$f5~6icQ;tP1twcg-xC&T8G7bgaj6|;i{nnR%7TMQp`RTd zL089~|Hz!&S1(?>qnDMczc(s?Srzfq%5z+mRAKC9{JsDRsO$6lhZRR_PNa|`sGezT zeWjTVZ$8`?tB|sn++R70rttqQEJE=_W$7%R-F^(`ls0HckUkW`1(`z>xOvvT4#~2^ z4Rj8ET2fe{S4MbqPZWF{^gja`hTkYEzY{I(O}@<*Yrh}Q?|LOpe!W+qNvr+;jE#uw zf^B-O9J;!mjBZaMN9Fv*-eFX`5Yh?!(YHO5P)rPB^HE}91SRaHT)P<=RqGrIe>j3m zZidkLW?FPC`=2?YdjfBn7AJbBhMS~bB_ z?}0@FdVXCiDXbS4SfUQhNc)l>@aE4(!&WY2`y_D| z&nxGL!(Y0=M11b(d~Us&66Os2i(3+zF);|^PFy^T!D-j}{?f)U?-IJYH^?RAR~$#?*DX%~Fr+2k*9;SZkOvSKg@k8|*j@G&vCu{Lz^ z@6SDZ!%~eh*bFb-5>@{G|1%;2lSB3IC+Ca7Q^W>&d;?6i?=-$`X!aJCNc<5c`3Hj& zM8DRhY&4gk>lG`M6xk|GW3=$CFKMsOXjslt1~$*Fm6)y1h?BSee!M!@lo?fyWrR8< zSKaC|vT_H3z<=O65Kx<5O#xz#s{kcr@*g-d=f7_t8sTePHIbBJ3 z7<8Nx*Ys{(5TV8Yr>F@Wt+$7g>zi_Bqa}FHw%r34gYeib&Sy0aFWxl;;2G)0-)e4` z(}k`qmq}X2i4;Xuf%*Hb=0rE8v2sfywaaBI^R`jb-sDbcuXz^hbJ$fezlb`o1r4~s zsz!ai58TCgXOuHP6-nC5ckp-n9%qWE-kGWK^)=F8%TN<|)jr=D`(2ZrvTdmK&$D7O z+m+?r(-g0oi{%}ddTp^;U8AdandR>S}f~*29n5SGoy@ida>Xe(j6K zr@jR@cj(V2+i;pFt@175<)@zU z*1=gb-4G7E-k+!r2__%+Dj1$Uel(f=IK_r5s1b$NgBY;HG3f!0>%~}9R^%|3hq`!Q zYPjP6>!(HcH(qcv|IRTb(A%avbZB|yU1}UGj`HZN!XZVs@rpBuc5;7pntd~WM5hr` zrAc^pZGX3SVaq=teuofztD~YyhA}xp-I@3DrP#Y%P7Iw#uQ&jcq5Al_dl)&vX5o*? zpolP<4GB!js*t}0e8d+c<)EC?BTe9~o}%R16neZ_sBB(yq8q;Sc~a|gxe6^bx;v4x z$O0h|9*5qJRfs4SJ<7seAKst+9x!{celS^a-nr-;l@KLs3C+r;e2rsB_QKwE2lvtU zxs7kAzqCH|3J9m(BRHjn~^qqvY?|Knno3bVMKuV2n*^E+-$0&dhedcHU^w0 zTiTa~Kb$Q+=Di$zT4GgFG|fUM&VkyWK9e)7f}F-IAJ3zc^A;zqJihZ>Nv}SnK@0wH zw*HySdZS%SwhQ%+?fnN4O5i=UF?P@Ya28xbkRC?rvtim3;LNwM)d3bD!b?D#s-A{ z#R8mdLZWr+1M&o7>%3$wW)}}j>9<}{hnL<>0>y32$o_-7JhmB$tUFI>jI#ju0LR*U z1sjcvDWRnnW5{m`e6iL2$Iw^D8;1?P(_tU1hERg#c{>@#4g1sboH*!E9*#c~No(`% z%q-4SsAA8!5Ehxu1B`{^@MAu zc`oTvHTHcU-)#2T|2-B2wLnM*&{3#Jc^02lCv&)s{iwr&mh(b*oN1ZEv?1=MIt-@Q z*d76?Z$7*_e_Z!$*bx`I@ul9CY@Pa7eddko$e6|kqt1{-*nH{!`wA2GfSi$YV;Wmvs4Pzd z?1h416a)JIfJF+%7R5ORzlN~>M(>iC6;WMzyY1$13!uLQ<7ST70X~GeN`bDC6hy-i z!PuAtg##NkZ_2waRvAc)^pMzIhvhQ^oi6s@lhqicb=<@rn+KT$)BghxaVR41=8W+1 z=MWCMnDnW5{U2;dYHe!SW=r^4#qOnNO-@o;7;NmLJil zMo;$S4p#Nl-3qGks9Ju=3n%ia3E{*IcX17@8Z?M-5(CmS>82aJWXO-mwbNQi3GSl z;wnZ+?R{oRQacXjTAY`kn*R@oO7($8>>=ISAk}G>p8$+@+je1$dgdQMQFE6#PxbWP z=CLt#NLB+ti_`^&*z`4TP|-FTXa5rV0k9q)8ieMY`A@8;7A^IV1Qi?H2CK|hKfQ;R zezAF7wKR>D(Gr4JqdI-pzNG?>*lJi1+0dRd{q&0wsYjoCBYflk2;@vz|4H>%ogo@N zphyM!1oO||8LB@FfePQOf%awoYOM1M9#IGPWs#@$b!~41-Ywy$C}rb zO?6##aBv~COM~G5fRYL9Z8mYyCmC&b>1)nQIBnH=z2nL9AbO(AtM|5v&oS#?0b#%X zU`+rn`VUg(+YkII+s!J$P2Hx@{@s1%HC0N%psOf=)A(SS{>~gOr9UmFSNdt4Yzj5# z0*cMOCd77zHoFWP-2e{Opov2QG$gO(R4#TN!t*@<0UF@lH{S9!Gu}?W zah%X`lHNj_QDL8+FNEQduSB30F~^>ZOAfDauEXdx43%&&=-ZN`hej!99E1xIxpAN< zJkbY=*YYq{584<55}~aTmE)TA4R5{cuAfHbt1nB)-}+bRpTU_A?Cmy<9%Bj6eP+XW zY60a$W)+)_GZmh9PY!=8fF<%qUmIfT7;nh%WjK+AUHIq8vyCl1rZ#H zPOD+l_+9HY(c`7|xS-CUpfOr6U8uV+?c>G-7d-hodm0~AnXvO$u%*L8FGB%|vQ%gH zvzc*^XM_2K7?dT1w}g%#-f}L`X?24O{K0?=cm2w-*_}S7tDc8IZbkB)DJ42>LWAp4 z=a1+8dY)XlsCo%Ur9y33%+t$}@IFW%z|~f7cYgJM|BmCTj95e9V$1OC(+S^u4wItv z7$GpwRdz9cP$Qdim*UB!*<^>Td16xAq)yNHs$m-oH@YV8Cn_7nQCva(Ex5Q(oo{{* z&J+tBWn47(4*l_BcGOY*8<5RwaT_sBy>TJJ4vZ(f8qdqu{u<*ozMV%)a@lTQShFef z-m5|zvvgkPGHU}-`Mqa|EYpR%0Aphk+8bS2qG$+(U$1qxOqV?7>;cb%#0ulFcprAt z)*%2;qpN&Nz|wfeEAmV>0H1jJ^0fnr=D~A~fD7%3 z4;$D#HMu}K;r7g!2H+%x^$}}#-i^@qZP$_i#=Df&4zfV`YP_@8`(o=B;@4J_(7`F? z=XnOv?bvzFE9@?!xa|&t!{Oz23fY>Dk~nR<24&1iy(>eH0viL3|9N7VZ=rp#RsRoJ zgpTfJNlcm0%CE^(8ook&G@YRHFCz}Zrbnll*T=9OZ!6u~FzHuU8bGkNS)*n2UHZS} zzARWhnR(U@9`A^)TF3!=#O&}e7KVu=J4ZA#_@ zASUxv)HZ7XsP24(9Pw;)xor%Zl+i1@4F|m>%Au00mw;U0G>QNRjeT>}C)TsHlYD|A z4X5sq>iV}>gy6w^&IM)u0jNAn%Gu)CDAEHq;zot_5Y1g@`Td{sDD|6F@O~V7q6S3E zH~4=^$9yGhV|uxNd*f&c2|Xn^2VZmi?srSEY}A1{zHD$q@vp9`Tm~PNe|_-q9jw}} zw6}(nhI(~6vH<6|Tz3TdV=c^?*^Wmw*YRcg_1%dhvd&@R9YfGi{Ndh_07Z|a z+=_Ie35hJ02&b56FSQYxt2$s_LVS#lcJe=tO<{)2{$DFkVK-#Eo)J@6416Fz0yx`9 zRMYLkE7P6E^z__OuFt}z?(XTd<#!}LKNN{@q))s5lH$&t7VO`_y44KsZ!efS2o(w+ z6Jl6C#;5oFC4p}iNBQ)#zr3i35le34*p@&abVJo;g!XXZ@MRKFvc z&SU2;ea0Kad-n1|y0*&@_hbpqxgaCWVZ5kc;zk&)w*`^!i#k8O(;Q^`U1mc(Ekk?< zDNV-O^d>5g(VlLLrZGVFVxTf@j9eh$stTe?1y}jvJqnt<_ReaxR+CMYMk;cu@{zA<` zME%n}MRC%f%QBoEyK=E>4ZYgwgbDZZ0K>T`HKN3(hJ$bymF2?`9+k-dKuB*R^LIOfeq)mh~jdGF2SJwo@_ zhrW>9%_HIx=#5tLuiRhjp1$_Qk;RxD61;jeZ))u5@zWk8qQ5+47ONS=j}}d% zR7|SoLP>o-4w4c*wy$OMvO67a9_f8ZP6ZY+4U;7d7BbIRv ztmKfq`&mpUb7T1Eumq4ix}QvFKP{|U9Pyb>^TR;n7jmEHW_$1lg5Ryz2XYI^Q=TpF zs<`*z1pThu3pOWXbtW}_bmyxZM4f@6NFi##rtoo{Kl^>(&vkB?n*vD@N;yxO@4U_{ zc;i{loqH0q9}f5Y}TuIMNDc7l}Ozl^b=Pb!+nFSH_4 zq5{GGpKp)L&p**Zk$hG1IS<|-mTmnBr#=~z_2l;;DaSC+yRpqPrI+&r4}Ce#xS-%~ zkt>85$%5Kx$hz;YXdhVhNDef?PrNi9E2rTGucqOeQ+d{VaG+gY-1AjB`ukntIbN9B zke;u?ma=PZ16JCtN90li^JUW*V_v%0VX53`&DWcJ2GO+04xo zHLy2YkwFt-THrez+L{~ixJssjXas5RGtNE<_}Ra!$URc2Y!r4X(~qFCF+x=BJYy6j z{KC1e-;6~5+}*#07J|_Knt92*#(m@?q;w$WkrroGE z6n;sHt^T4ZkH;J_+UIvPViGCugQ8@^thw5dwDTVTj zG09Vmj4YYNo}AKuHoDm}eVjP1u%i;1hi~nc(jMU2?dbl~vRuyb<;ud8zd0ACrRH2Y z1~UF2apA*kG~T#^?s-!dCv*kW1KhWIhT;>^-^rEeacb%GGD%VJN?J_e1H3?B6A4_}?i}m`hmX&9HZ`)747bc{3AJ2~Cd*5AX`TTn zHbM73>2>gY4T!(fi+?PB&b`7jLx>Z&=2V zg4DL4VA-dJNx(#&7Q(XKcOQ?Ty%SR?lEF|Splbk%8!PZ$_Q8QVD)DUBJ>|*|Z$CbD zojL1U_WjdoGP*q@q6n9q@b$}ds)r|oEjCV9bndm_T)_Re^uK#gH?1lQg&l~4fyXdDo9gB_|Nf-cq&OawItwB; zsg-?yU-Qn?m`D`%T~I~G#no?z?#_v!JDKN=9i~sFMne*Jf0yh9(JX%d$XHcAFDll7 zMOs=f=U{rw-_PZR&J1ooNs@ny?jISO+D;tB0AVP!e?}4|4gUKqXM90zWE<;k?ggB zPB)+(jcBphF^GZ06Zsdj8|ViWTK+Dj9w*(7Gf`SGnrUSN8H_^0HDP?;A^ zguCfBjBX^LlW=Fv(Uahi7)F5_(yQZhs)Q8Z07^fpd|f{jDbiFl6=vaPd^~|X<3A=G zeBY1EU8;L{^@uUmM`z6ky)oT~eVcBZWJGp{7joPwEfdYDtqD)PvXys5BL>sU-Fd+2 zTH5dkB+&$W#pCUAOD}-qG4YR+AzY{Mrxo+2dBtoL7?n-}_pM+QHM!e~UHAYOEwT$; zUINy0oAyXl_aNr+*3~m!HFT2rTNhl z*o5zDmN)GxX5;bR>3;NXXY-$Kj(gE!L|SSrEg0D=$<$nQ_sC8s?LDa8qiI$M!#&AI z926REv%7)8EZo7De^SYp2_XcXFY*cyU!)1AeVr9CQQgUSvrF=Ojg`l`z)^WpV$TWh z6Q_CG|8kXR+TzElh@!5+)ej3$E>kU~K$p++MCyVK-4S9?oWA$E2{rt|kq5r3dXB!c!;#JY%%Q!eLRT7IY7^ zx_});J&w&R^qTIzi0AWwRM!xecY;li~i zawd&1@sZ^x1SXhERa|7{IpIsQj@9&dfPW37Cs9Wdwj~?csr71>jQJVMsd9Z1A(nu5BMg`qe4w<;q>Tl=p1@dsb z)IZd-ErCc0m}@&?7v-D=_D}Tk>_&%g@YJho9c7|i>j&w+ytscUjH|!r4BF#jiin8h zsWG|i&(C#P^}GEXp$A}J1kew@%xhtQALYxft6Je0kWvSLu@?oAp`k@k0&xsQtb>9+ z>y4i)Q}v3k?Do;QLjO)@YEK8N|E#KuJE~_@NeD%^io>-xzqZU++2ne}_uPyz5_VSy zxo3~P4`gL!;US>mF*?9NeUMLHun)T+jjRi(Er%K0(3CLpnsrU*c_uuxup-hdc!!%nooS)+xXyO6t-+v zH7pk^%Jl}?eHh#^&3ze-(|R1!@Ae}1;?Un8@>si~=jX{=pW3VK^OI7GrR;YRddN7j zKf;9CU-l2z=TR>2SCEGGRQ^2iG&SEa*$e#$(WNW;lJk4ZDnnZ{BAANputYGoX77vK zCPmJnO$;@<&OJ%5H*H5tG;FKLEZ%ui)oa=YQ&j*cT->Cjq|NbSviQeZ0Ux;{gg=l* zNjDsJk(~p$vyze$1W0ZIIo)ia3J>uIkTW9C4Hy9?p1TBO3oHQyFN^~;F%Aa}p;thw zept6;n36j{XC}ui)VX5Zop+0r*)Iq`#7D;@d4ibkeT0UO9;`ROAt*@xYHwBy!1#b* zLI{x8L<5^e62P%x8UaAM8nkU+1}MIEply2CLloFxS`R8b(7mD<31Cfot80d z3GmEM8vwSLnBCAHI#C8Rzlk|5?|+t0G8SF{(%_k8W%p%sMvQ&Nb!FsW0o??LfB+dl z^R*6M-MOS+6#TXbZFv8<80m z4Q{WzkpF}ScCWo$Vig#$B7>_{-YD{<`nPIchLi=Zy6niwOH`}7;@~NNXI%jrtht(? zy*Nyf6yV)ps;!#~&CM6*pg%UyNJl2;b3%l*%Lem!2sFt*#x#Rj@A>{+%Fd3p0P0oq z8G!nX0i^DH-|sHs*Dn`X+jRi)_HQ+07qI}X7-3E90bKtL0NCS!?d^u8QleWMFgvRY z0Gq}De!dCxd;ucyjqOYo90VrpTLc7Wfz(F#$@au>p$6Npy*V=NVnWElVK>zZ5Uqzn z%u5WKqhb1{K-VEeI=g&}bXuFr@Pc2PA4TqQV>C=zTUr`9no1-ZsMRiQr2AD}UZ=F0 z!V}s*gD6#m%;iUr&}v8ywE3*%6&(F)j1KKN4sFzke`?VL3@tD{%(tWR+-Rp7MlPkF zYn+~<=wR!5Z`Sl8KGsf$1z7~x zbeNWb)p!wFoNO@A-NnsSOs}fD1D)PYj#hg&0W`9qv5^D77QsW5DehzXQ;Pv|`DAXa zrluy#kxytqo;e(}2%p>~-=6>oe}FMZi;ay%#bDOHJOqLvT>$(T1aP3@4DXjZi420M zQDI1V2sP+z4p85m2H9ysUd8~^t;7@ud2Aq|vyJfB0JgI*0$croSxA)(IM|`U(jqa-a^P#9mV=xv~@o2#sFRyA7F@3d1aXs31uK z>w{#WvHC~c*GZ9V+>BWZbU5_UyDw_qbjVe63J86LxTR)(xcfEseNA|@x`RUdg(plBK706~wim0>Zx5p;JdZ}5 z>`}OyI4OcmqIIG9^*H6iF|aiVcIISh+E-tH-{ns0HDx*u)IVGSC{d|uyEz!@6nLja z@Y~O?j|Z7K-{d4a!P>1WDcuJeFu~w=!z{*lx+GGsDgD!z!QyaWD*>fiS`LokYap_| zl9Qkh52*lhl@p}~-GD^_WG4;)ayA1X?ZNS^+#(e|ca_?zZ}!oe82;#7CUtleT5?HN zMs~(P2coIQ>2VZ@i=951Y%P8SPl|u}2Q$%ju$4D9Q1aEnVL~L}OoPRdIbN5#tCDBH z6TVo*<9s=S$YnL!0{AD6YkdUHUd8GW;CNA1T8)W=ar3|0(=~o11I!{A1FXY5phvU= z8{)7%&T@R{dvggB=3d{opQ}R>yZ=qjNL!Uoslq}Bplg@5w<7>TeQB{ZklyOyK6&3U zIPL9&gZHpVIUx{0_aOrv@?rW2V7eTS*A;;J2aS+Wqhvjxv#bQ(6NW%Asf5Xah)wru z?|bD&Pgj_60S)Ti@=GUyYP5BU+RAoVncd>Cj2T1^aUUWy+{+(S}q4Pr=;d_OU{GC!&V?IesFuWzXIl5 zsc?+f`@9S~!A?z`(Xo?wg@9$PRTmAaNQ3}?{6?3RjiX7sxJquJs*i2d=wdK;X}`GT z$IVALh!wJjiDXPyjW->c$ld;kgsra6Ah)%Z%W9hKQ}b`4c6KFvDROdhPxlwN0WN#A z0Ez(|TKCz4A87KDoE!m+iXqTWUhf9Zf)Q*rIR@4|6;(d3boZZvE-U&oYZaN?=`%BZ=fD%D_oI2OiG>m>S(?UkVVSfeK!H zNp_BNS+G1qJ4{Xju6M%tA1nYEDqenmevR8MDM0?)I5(e`SzYnXerM?bOVB!~W8{1N zv+4UHSP~+g7h>5@KTOS!o!KxKN^_jqAov5hoE5NBpZWn3i6`t-yKY@Ks;vSV(7|SQ z3@y+%fVnj_HGFQ{>D3RHvqlDdv~q}H$AA~}ti_BSK7{qFhL#5 zffT-V;e8-{0SGxX4;|fNgi*NPPteMItQ=cl$nFF#T1W@dctnpK@|CEe6w2K_m@)LAh}!Ixw@Eu2{M9NdW_?0@)xWAO`d$?P@!{ z%*&!$*y+KQD}_ZCIQQ?vHW18`0BT%ifq_h2vQB8+%$@jmL;J{`;hGK(KPsfOEwU(` zSI6aU8Y)RXAF3^MK&1KIk8<{_E!c`M=LS=D0{lLb;NW1$<9#m;lAPU{_zxd~KqwgR zUQ{78)5njZBLv7~&tAOfs&$r;YGje}@_c;eKDIeJ%#|dvy1P9HO+{^$>>2Rl*rOoH>xD{w?q;|CC+O49NqM zSnfv8>(Y5nwc2` zgYa<+b7?~WAgLGvW%WcZYxP19!U0)*JsRS$^o!T&`m7irS3(WwI(3@S1fAxD)>e^AAyTwyqi;JAg;$(B$U^+lnV@!f)v(CIutEe#BRXEL` zEv&Ji5#2+88gTNoi}llX`EjZE>^E*vdZPPO#cgyr=slmVd88%d7*PWhL=>l|^=V_( z+$+(4dj-XA5oMiYax zVA7;@wwMm+jS+fjat9sOAL3&}u)N+4Q$L2>2z;Q7f4HV1PdQ1>ccJZlQ+Q(Z=*;Hp z00TTu(XtNZ57J(~Uc`AXF31}5*Y^C25J{R3N(hXqL&mQ#Sho$sH$khRo zDyLZCuTx2B8{$RA_hj3O>o3fN_kCx-7KZ{^j`Vbcq;3J%Ekv>V>>zy#iDyCdhaCyP zKnetQceyjd5%7aCNjZD$PU5=b&f@5B<)q+C1U#gR1U|-?c^x~PL{qYCKROV7$iLl)?D9yk=_`dTB)TdwF4J{&@2-$WOfqgvz$?YG^_Bw#p|kn&mLDa` zDzXf|B2EM>WKfv|!peT15;fYzS}_*(*Dhiv@m;!=x+G;!zr( zGcqtoFljs|XJ-!J5Oo_p34qAXz1=Agl9j^DOO5XhKbTv}>8}dK>e(P~>gnwj1JN${ z!!9@qmSY73q&#-ITkjI$ts49WtANf}mQni~@CoYke3}wq`ll??r2r?4vfDzFOuNaRWj1PGBFmo;q9tZRKLNT(@LJR0+yj2Mb zt`5(@hH>oBZyOr_Tc8aRBjO8^ZEe<;Jk$BY0Vn+)jSbt{cI>#mx*#Mm)arcAFM+*E zLp5RHD^{7^0=En}dSGxKFj9I_xBYxWSZycbK+s@rSYoRIM zr~M0+EZSa2kdim#1rHdCr&2%i*Ks#6D+;pM;fB|)R~=~L!%s4#v(X}W9`M$PPgs-+&2W-p zft+1q>q~Hn5|uL4eD_H(GTlLQ1nJ}Hq=?21F4SLy>`w;B23-z|m3lgJnssA(EjzmL zw8d24GP*s=`N;kX@3T)6RU=~)z1yQz+7A7IxBDjGq)19ioimq8N@ANnfrCZPU`Jhu zu`rERDXe_l9D0oDrIs-%!cQBVMmHY-CrvpirglGbbi)TT3}|elLWGaYn<7Z^C&6XXd-*7BfJ0s$ZxH zIAz(h-Irc{ID8S6nW{%DLT=a4&|l;TYEE#6!%Tv>nf~(HPNht#=@EKgD0)n3^$o+Y6q>g5gyiQ-F5z&hlK0T%;POcuG9!?o*tPz?CF1_ zuM-NVy0K*3W~a-0igGO>&au~cw9p(SjUen-jv{?~#T*pG#V*oxdJ|?jw*6ePavKQ* z!L@-48V!LJtBB9coH(t1Ak4`$qvwCCGtitoI8@`bnxiP6dy{l^b)Dnh@N13cnHhSE zY1<)%wS&_HKYqbioaqcc&&X7?J1Zl?Qk+^=opz=_dD`^(??p@V2e;p`fy9vvkEh(> znSwFjQ`zh13xk4Z=BxV4^W6i2Yma5py2=9U4W_Wrl)-6#X09Lp6c!kcz!p8p_m9Sk z+a%8qO2{&XBeHM|+n=n8>yd4KGi>jTVSjgDx7h)g(aksJ^80kQLVE7d=Rwt>m*{1i zQ_+bHi?1FwddpSZbzc`l{P*WcgO6pZvh{!P3O;ebU6o~vFp+zp!fAQ&kd4rnh3+ad`{na}=Hb0^G`Cu0#yicBPwg}8J8^@H zc$yT9uWtQMj?>#~&U}N^7}(B8pPQlAkBXjJ#Arc{2Qhya=AB-_!f0--B)aSy4Uq$$r!l9F2cw}nGn3Ozp(-zFKHehX?w zV8jcz*#E##-xBK6OR6hBkxcc2OrF<4IS_$}%dO~R3jNstmRjk=q@Y*6z6<%5R5!Gs zw1w1l_S|8fR-1t7AOT!jQ{3&r20pn?#s z>&)6*)xc=Xo1?<|LDNE95X%*tu~LGxTfw`~qS(0H*PyI8dbfvPkDm}9tL4A_3eE#5 z0q?O~<~(AOn*Y3#%dt8W+!j(lQ&~{q8l!&S5VT3O;O*uMg!0?$4Tqby=RwjP8FnrL zqY-xZk{Xtu##xOPo(o5unD8%7u$fDw`k)BI(bTvsD@s=FYAy5ey|P<87-V)3;3AoZ z=R!Yvf-}ut)n*dc1#YwjR``8&ZV5k)G>ZhR%{K;r(|Hu??^BAymp*Ak@CS=f(D(bO zw`@^tI0)23?#Sm!oAk$bca_u7ZcNH9TIjvY*YH6h8#^w*enD~|iwFO^(lqA=S}e1p zE#CZGP@)nYt@Q_Qzb|G}qEQ1uH$+^#F3&NF=MgK%gU&xQNsd3h5-OuzX{>zoRmD}g zGx$GZKwgVO%Z0G%RbB>nC-mW?#_R^kG2TkL*hckKaR1uj%>93XK!(Gy)T9Tf z<9cc0nAuIcXc5&)RkxgXJfeCo{$<&fpxb zX9^!Z`GS6MGw`7P*K^6$X93irw6T*FJn<^588UbTpHB`X>K%~mHb}}Y-;x^aYtV81 zFg!c#am4gw0q0Ay>Sx$M`)8xk9s7(Qgp2DrPhS+$e6bY-$4$C#iVoG57$K)R{UTAn zQ0#KdIdoTew!tMsUq>_ZHcsCcShFX7_Cs+g9K^We5Ryk5a)NGmBcUY@Q9ayMO?Uqk z{K5SRfCzjo^%DwgfT#E@NQn{Lu-xpbAJ5VFk$O@w*xpwl(1z4@F)9V4%t5IoWUAx6@k3r+RH}pOI@*%(xT8(gQaw4JT>Caq~ zC+YqiN|M7e7Ff9qR^@%=z)U;Sk-*FFe}W`#|2N!X*}c9yt!v&jebH?i34G|Y$wQ%W zzjJakCN@%RW%Cv^0q}!)qlCr&+2b@I|q22tFla==N z;P&gP+wk^GrrdE5Nzis>9<+f!WoRkq(F*i=Ks?<=MnOhW1|sW}`pou=Wd4eRQpqz0 zrt|!B0o@X-MGPwA{kmXH9((jrEfUl`$~VGz3#>THsJLjx_4nGtlVSjp1Oeh1l^&H> zGp~8OB?|0&CX1TM_K2T^lYM&Oq4Rh^ zeQQ2sc1M3j;&qb>RGp~U98wB<5zCafnlij-)o>#*w2@Q|2SK-`NxOF9Hcn`WRJHXo z{{HDK1DXec=*e8;9R{Yc@cX(jzM#0Eu-10#e0p4!c=e=Oqz*4v-A5^_ksiSy(O|tC zKseklZ=&cyXWe=JBVB+W)X;b!24H6FX4Z9wP7M|r8i1+Pnn zTb?V6!pjLVq-)OyNHa0H)7nf^Nv)PG{xY=eYhjuICvo)b9&7(tMjhsR%9HjfP8PRY zXsFNzhU2erWW6l4Q0t6i>aS2%9*)GDcA_uKwMk$%zN7$E@1)u3#QU1-OwR0QV(K zpMNa6AF-4dQXlcI?F!IAEY)>he>OrVZ!T!va}J(#tAn7hSq)|96KRh^{dLAe3kN^v zyd`xx|Jkd-ZaIeX)|%`5GC0QQ!?QB;5ifVoL(E9+Tl1Wl!cnb6?w^e+-K_e{Z}YM` zu#&s97QgH&K?WJ&2E zVe5|?8aUB#Q);ZV1dDgbc+nKZ&$h;?XmLlUhA}p$M7{d#+MZt}bT}4)Vc2k1M@o-d z8+3zil znQjv72J^RGA5KhFpc6DDX`jw(R#7rK;cK84)*$}g)UjsQpMGCcvu1yRf%EX;J-gkP zV&BN_2y2AmJtBQP^FD$h(+P|<^JDrb_84v~BY+)%Ak}p857I&S&rQ?=elH~5d*}!6 z7Q@u%Hz%KkrCMMdOkhy}wAxTz)5x9)PV7cGR}p|~0lD<7sNY|yMxa^!6Anj<-XOcX zONi)g_IS7+I*vyf+*2Oo*%X6L*1#US2e@JFXYcd(7(4T1yg7a<`&C3K!Ob?m2`w^= zxv&2Z+$8#GUFgn3nc~(f=6jhQ?N$e8%AtDLI39UANf*xs`-ti>X_NV^2sJ!b#xP>D z^+U-a4!@i-e{?a>o~@zF5j*`=uAlmog2bwZ;W#)-BjQYn{|mLE{IBQO|6G+Qb!PnT zf|h49oBjJgOkyn0Ag=Jv zgCqlf)0CY^wIbhF~tSWTiE5|e@i!QbMQ2*_hPR(YnB$JO@n z6@2;G%s;pydYCs;mQSolgB(oURW1KYi^3+8StASWa`aJZwEHl3EH!ZQJ$?$~!(b1J-H6&&QT6}1eYNn>l@V@tRhSbg}#E_Fz&f6e^vdfi zg&OP|z=J6a>{O4S?Ow*RmrEkb#cmRj?zXiTjyVZzAeMh*Yz81aK@e1j`6t1rs5q-C za8CKhU0>s$N?<77^XE-l>2f`5ET9xbVUy~-oeQH-=h*rEv9N*%t&;)+Ui9@P7Vh1w zf7>-V%ufiVds58b(wt$FIU2U-!oY9x7iuAIlFnqFf9?ivDYfXVZHic?C zL@v}d5ad_M4)GT&ZQmQCqP>`81-QnbQM*tOC=X>wX%v<$y@|=qQr_MR-R1t~-IU&p zACu4|h?p;LR4Z&?csCFyYmN-7K$GQf{8cPoc-?*mVQw)BW&M1&Cb#O4mRI&sZ5MPi%B3&KDM@(IoN|aD zi>p-IZ(ZEU-ZUPOc*7U2tPW6>`QADcpUua-?OWxovZenrQfl;zY@uyN1aX2br6VkL zA=~%s>DxYh!qUv*6D?;7TOkx$+LmJp#ORHt6_2A@Es347&jr=hTsFT!oJaUz3dk3K z<;>J1t8NZ)jM#1PRrry(f zr#|H#OPGaMBIPSy9d2ad{816+{#<@8sI)V`I74(HLoYWm)=L8=+Ht0;80~arE4qIQ zJt&jbje@v^E40+57f#WHt3EvmkJPbPZ_en7P z{p$+C0hma_bp7SQ0zg7uU$X!tWPx`SC4_yWUILq{(3iz8B!-4CTP&t1QVnsu=;|ym zz`y0=iNo2Y*=V#)a9K}0?{&Wu{;HW+=8@P)<^s-iUhtU7Y@-K3W|Xm<;O8T(hlf4` zsbsv@$LO;!6y8*cMg=s;2XQ~1>e9CwwN_d73e*qTiQGQ7z0i3pd*@p?(CCGcRIPF! zj=JuLXwls!A1qDI#No&zo5McInvr180EKzm{8CMiiQ#c)2N@z2yKK?y?zm(=Y^r`! z_~p+q^vDpn1=?_j#lwi1YEhdz;ZsL@y}Yn0f27-BMG{Epg1&IN%z{ z?<24+Kt>}i-<$*3?~~BZWa(t0@1pMVD1zK;KIa-3?T5RCv7UQ*CnjqCl1wazjcN=b zY9%diNh0nmHNMEV6({5F{(v;&xb_Qfw$7Un#qScnUSqI(vU&oa<;r4-l+Z`D_HqwE z=C>`k`5vkXj}vT&^H=3XRFG-87uZ)wbSIs33Pb!80`8&&0$qn&& z^jBk$5j|ly%`ph{dzN=`2wVbUh%^N9MCE;KLR@sd+^OhiC3W}6jG@aOSw)sZO?`D_ zZ-hbw$S-ZIk~6B!fyZhU)rOFDleBaoW&ZQ-j34;a z8J@O~_);UyY(n08Ka4L=N8TpHz3vbqW75sefvxs^<*~(HnlONbefEHS@O!s2AqTvm zSLDl>(-I{K=szexUJxco9)2&%?CI|VP9RXZwEbPa9vlQAcR;3bleZ$A3MsV;V!a`Z zU@Jyx#bX!M`hy7JOt?d)!>tXJ;+A$^8$ckqVfZwzq%d|x5VvKtgiid@ru_mugcHf* zd`aXEY+R|8nt6cwCga-T5li=Lh)P{(iYfX9OhH~C+~8nf+xGt2(WQbfRlWLvR9iF` z%mRYyXRrc>e#d&}UE2zqx9K7IW3B(0k5T>2$N00RPXelxq6~yn!%_rzl#1QxFq6uO zW!V<89QDuddrh!()Oo084D^YDUHL2N{M1KG<6?rn>PqQT?gOJWo$UMgq_D~mvlo1=|axBI0^q6?U)^5JT= zYBkx5lO?)e@^}y+!tpk^M8dXYryS9q!4ugX72WBeGww{OME^W(r-(t<;N(m4L7y%Y z|BZ&j-w0T=nVE4)e+8qZz+I7ZE)<>Hr^HSA@e2t64>{WzQ~$o)T>{>Xjhx2qk)+FD zQwL|!Lh8=JmC@Q#BnVOj06_aC*h&LUSc&rf)QJ{k9v6Jvh85$f6%ps=r)4TJm3u`fyq%+c`n@;0rco4NS4%(r5C$A;XHeZY11=77x5R27P zi2Y{-d4wJ%gqrs;{10IfrLFR1rRdN+CdbMoX2?NjH0JrGCtp55W4aEt0451PFN4i9{{;~dH=)r)X%6jj@BRH#4;o6o>^1Ow7ed>A1i=^Ntz<) zqxGxo`QP|`ugk-_8wLqgj-x?^2O<9taaM~M8OgN%@iUxP_J3fs0OJYE!ejOLya^&s9G=Bm~+I|?4eD%5bb>qQqk#C+(yCR?)^aP&l4?)Hd8aqH1^3x`D$&ejhhSja9 z1^N#bfQWiYmPX#KC?ZtwJvAzO64RZX^0lv-HC0u`ZBkWUE{$Mf&7Cf(u9?n)IzxK_jc6?rPAZC-MN{(v^-LF_XSOf%2TQyP(g zAQwEcj{XMI31-`mOE9V{CgbzEP96(^ntr3P_t2ve45xg5g>C?Zjx{q5fnxsAZ13y- zlcMNG7sF|2f2N$l9ji$!36`hLaQ;9$|7LTmPi(Mi_Et&C zZX!u@OJ73ZETIO$=jOtwzsv!cKaSB&v$O1P_5-Vlz<~afxK3+?q=(%G7rdUe5M;Wc zns|7hk5?JyqCZ9Urg1;xD38&(DM7E({=Re52$sEns8$KzN%3|UV|IJw64RY zJ=%hPiYJsfP`#=*P~0#?X_GZy{fh3t*8i5+?xMW{NYch3IK4?Uxp*)eCj+ zWtkg3v_gTF_`DEqf1$CsUn;3^rycrI2}U`YeZ%tleJd(?oubfdivy&cwIYE8}I6|Qr995AyjGTnh$+gK&jnaGK`sT=y$U<$ToyO4AjdXcc8{;;~jfQH8aeasa1uX=RWyT6C6eG1n> zufDLoN|Fn(4yofHsaXv}#pOH=ab_W!DV_AJ3SyL1eka%^X?&r8mgnMiw1)d1bEcKa zs(rvv+JJ=Ht@?2^@;KOw_5Hn7^6w2=5g1=wEpY|hE4m6X=U>DEc(-G1MFoSFoQuaq zUWTj9&En5A5sp`^7t%x38`C5KA8T#gG75`IQ7U_U4rj(T@ghaU?-5Xg@IX~bYFp+5 z4u41gh6W@;0b-?6&vxI0Zi~QSX}eTfc{`Dp2QvMI(EiSZ!5IrFr+27JwQf0mYL)VQ z3<5t?(1d>Y-TVILcmMveU40=cq}n9nD?e)pB=_uAZP~r;lq-+w2i#41a>Q)?&&jGC z1@b!|`e`9csxTh&{#~10f zFH;!sFibKWnT4veGiYu-qmZ4A?B}EBp`p7r<#jI__5?ijWq|St3N&7_ayJY_A#^2ol8~01_g>1)Q*>{c8+TMdy88BS z)|0jb1;j~^`Rs5DPf71&^y|Z#h6APo{GSzAJ5cY-93vUBJ7NOaj7Qj`(_KX;13UUS zRMgYh_~V`M5Z#yB18*FH8g?*2%8WQxWbIqTE65ZNXn5Fn4e-^f?=xAHvZ>kdGOT$w6_jP4qQ$pa~(}|GqB&AtShqs*sC1HAs`*Y?f@YP;N;3BpU*2=YH0*8WXjf`B&iy941BdBZ&pD~ZAA>0 z7foLm&Lh*U;7l#<$8WlAXYq=YJ>y8SpG~-#w$bNASOSBZ{BX8pz=GLbT`Ij*o2`Dr z_TMg2e)8u$L25G*SQ-fv!I^srQMu(0(8rqx12;D_y+58JVcEW(_{C75Bt_U?Ct+}T zQPIHgZ0myf>XZ~DA_RqjR)ez{{{|9iLD{}K9AZOJfb#Pd1l&zUHi?AIwe2<*bU ze1w0nb56M};*I?=paFesn!4&~i2-l0GfFie0DA{pKUVzTp}z~Z$bc8&70+WaeRFSj{%DRglLsF*xl*Q5h#zkhD=rxoN-lJ}kuSHei=ubMoi1OJuK5Wml z*-=o*wlAW}vYLAJhNQOXzmnRoX}V(pulajH0qQ^abH3|OGO!SyH8YE5;szG?C0}Jp zeByCS>uryfLprxoWFQ??%CIkD+d_8!SyCW={{gVWAb2f!_%D2@8o_hGS%U4y0&cPU z)UnT)D4ga70U9NP`!;dDn$&nJ&w)=bV$f~2vjAZVAkH3gP~r>++V~KCIq{MICsq#y zwRZ4$hF|q<@yn-=>G&r6$cxhy_Sf-=AwTkid6U&cYu?O<+OMOCu+3vKWQ~Lg70Uke z<~M&La=z5B=4xYDu_jJuhAGYY0PX1HcpxquZSY8R;~`aUq@v`W%uYrxu>$aa;P)2od9v500GnBCy8n@S(5u_p4uTX4_SpVwi$WYu zAG0FsN}AeNHRb-j8O~!e{KLX@qaOz~ep;`8^D<3I_EBNBHdYTPAJHC3g4oe|0M|m=P=T!2|jaG9^f6kOaDBbF24i6ceWLNalT^sumFW<-s@AA2oT*25sFuDN9xo%gdb-mnX|&w?p#IbX4`iyk}@FjG5AGh+twblaCd!`78sJ|eAnIf&LH7*#!37U z-U8(w%0sJ5L$6Fpy}QAApKc!LjUTz++$S%#Q>)i~X~^!4ywo(bA!(JPd}a1}@9(Uu!HeJy;ct1Ic4XpPs* zDv|aG1xGh=cgGe=dKj|0SlViV)<$uKD*{e;Vz<+GY;|;-0VDgtDWJvfsW>W4(0y?= zs4#)c&l|PntgqkIU9{fF6Pk5)xiW4WW;d1VJ5sCVIToua6!KJ81u*Im1*1So!^CL|tnfYT!N`;%peSWp&@rP> zghCSn-fhj#k3pgH`UGGWz?dbWYuOeq`7h$2I%b7Xv2in-(Q5sZPl>^5<{AXs$29m+ zZ{@^70_N)!b6nD8mGw|A)oiM(^(#Mfyef8-_xV`unq$*gFyJ;tv7(hwta15L_e_)` zf{z!!4js%#mZnsCJN-(nUqd+N>w?R#E)(wa!pwa{(^qZe4O99x3qbAw@EHQCbC=!$ zOgSKBya1^4Mz6EeX+dl%VfpVLxwOg*#ga~*JbkK+A3QobivBSPHf--y#oG^avHpEe zJ{-AdnKZ^hsny7(Um_{yAh24%KOytsCw{Ai@_ZezDy_yW4t2kSX)Ii_Df#65;NchL zh2EB4^_sO>>hE|oX|RuD@*Gn(!(uUQcz4Jo3q?I>ps1*so*x^muHM++Jr&mhsMJdA z7ue*yqMn}opt*jg#s;8@_S;s^Nds7PBmceMYcDKZ>1y4S);WM$>u;#drdI z)x4KYLLr^5WUe-+D=z8$KimSlEHahdGN>XhJ}4W&wi*Eytvg*s-b3h(R-b#x$?o^v zWACM+f~|RdO}XdK4^QzC?~{=90zH=oA*|n{6Bn+{>hBpF)10<={xU(2Y>=OSM60;X z$ve4q`E(pE=@gw^rs`2eIfZeO4}Vo%-#8P~M)jTZhFe%wUz+A?=0_&UJ^(aOkII+lCC)yEA;|6f9i%ylxYe^WQ!H0_xfc*!<@E9#8 zN+@_{9Rl-jfUJkd-OX7oQ2D9ZZ@SW~b)x5)cc%kLYS&FyuNxwrYC8&mhF$|~XqVmT zIOS|ecF+oG^4DE`%Gd!rJ1y5DXRV@yim|*nt!Hn6{(?pRL z9=qKuB0|FTksAU25ZIgAv3Twi6Zg1I=vfpbCnR(OYe6jJ$$5Kw`+1vDF^x0Wqhz4j z@9uUMyaI48-T)foS3pexG+!&_4ZuT$Q9WTYRe&$Zs#_xpQ2i<1q|@Y$-ADXBmz=e= zwS?&i$M+Rs1{dC&bMuK4tL~BHa17zX;n`O*3APmu+v5GU=idhER4N8r+H5zBYID;^&i0zVEKAl3n!u(BDhFLyFX4~WP%b#CdQEUs z>-?d*dhqyE3p$_HTs<@M15ok-Gw^Wg1P3zXlr%w+)^4RYkO{VQFq&gGx zIXwWp9@EwUq$17oHe=>iz-9R2b!HcF+I(}Gth1rj;IjmN9MB$NR#sO2JOSjDfNC2f zpmNa(3sd?R6+O6Ic^iTKb{)TcCL0;r{Y3-3@iBMEf({aW>Z4 zh^&VhYOC&2?a;UAciJ32w>Ig;+vA5!m#_JsbJd)Z_wi>>W}ccAeRM1ms;TgQ21R3= z=40)-2Y;P%rR(7Zjc3$w6*^xG1lBYih~tgqt1L`Ws%#uRuD@Kq$_2DR;5_?;ss8!K!*Vgv>T<^zEzKz%VC`Scug zD+mHQKm3)&CTDwdNkU%Spcer(J3CBbPp4t8*X!5uu?f(66TIBahDoPEp~dP&Xur1{ zaFo5ZyBKrLjcL0_Y&zJ-9W`-?NLS_S8)y~zSGdgFk+}*Xo`g$KD=R_@JblyMsxaN* ztQWNsUq>pcAB`SPb?YQO-S*W2SPF>3RX$2*kD&nI}DWjoHRP&m2u-Fuq1 zCsvFb!qq?Xr(9k>iL>`5l{1c*RPm|X-tKN(f<=MK&~wig=#F2%;?jo2bi_F6aCmei6rBR1|0Jr7e+uF$_!|~y%<~qtu)L#!wFz3 z5yz}?v~YKu02(^j)$ekH1`X1{n`qU$70-yXvZq;(1Kupmy9(?#n4DY!BCY#oW(Z&k z62f#6Z?7+4-UDdvl>z!C2xX+4oSt}icz|{sI206EEr8(!9Lhc5-pxRq1!VbNgP;kx)Q7M5IHSp_EcVO1eR$JBAJ^@urpTl8&LfMH&XA zq(!=8s3D&{-uL(SeEu(ZGt4<>&f06Qz4qGIwJy~>ZXmkW?k=DWwDth$8>LJ)m?n~d z(p}tdIs4fNwFa|v7;yhQkgEy$-d2C-kR0(uh;RSR@1`+JpC2-f)gtDk2U_XWrq;8f zuWBcqYw=CTY5msCgFay%{28Y_BpA`Xr@wYhOiK3aW*cY5r9|oV>yPrzRSOg&JLDuj zG5S<4wn&6&)zpx#Wy$F~7h+)^z{BR_FuydDOhJrxo?N$C)mqb)qDIe zbf5yIw;`V*a^6@x)RdKoZ+oeurzk&^CBG~f7Enf!snSzaqM=7EHvu0$T%NWR`^Do- zoP9Cfct&h!2kD)C)bt zrufu9$$O=9LJaiMrg_9`mFqBux2?HlW48MGahOI9j;8vXr>YwIA11cz2uXML-?gFa z!=>H2Nmr%0^I>ezcp*KMF#9vtmCabs@CNHzFS%Aa%Pi!q_*mGuFrl&`TZTjO``!MX zSNr)n@=@+Up{wMAg&Cq&b6j5=@Nl3!uf(~vTEkI0o?Oa{o z<6S)e(yAG6bKl{rSLV~HN5&-dre~;SarL)^`5Rz*%fY8N^M0SgkExQoMEgSFw=4a5 zx68`}waF*eA|tac%VbhF2!Ww(amA24rNd(e`Q8BzG_{+nUG8g`Z*bSlqx$PSw|WQq z`EgD*wgk^FXH^%))Ob7BxE#y0W-I07RANr)v-atVA!||2KClEKC{Hk$_FK~7Ywe5q zK)p3rvCE>>uqma(Cx4`2;t}2klIQly2e%v_icbp}unk*o)XU4dGhZT4f!p&FF&wX zf|vD+`xQ`}(V@2Ni|c^^zM*O4@q}t?GZgh7{z^KgShP8p@V(hI z6u&TfghXP5lM5%bPSu(2e<>72&MC0#l>gu}_ZjuCFV?*2m|9eNKO3ou++7b0Z}lPF zA8d%)^;2TtWB*KG2I~=ZVX-?KoqNEpjU@Bi@d`T+N^VB>Yvw^15kZbQYojHmVmEWQ z5Z?8uwUb+)A5q)4vg*di<-%Rv!4=T6`z6yy_j;o84t||8b|W_xW~q&bUXd|#hC8&(4!G;*F2YXPc*=)W6%ZC^v&Crcs^<;~9`O|CBCmg%Y945QDpTs9|Tg)`=q zb{kc8W_(lcUMe95i#jq|{0fA|Cbo zUt=>)dpQ2@9ys|PuxNkR?8~?-ZI-{)rXmIjd!-jCa*k-3*~tNq!<(S< zm}6U%VcTXW9BU?{`MH>fqj?a2f8Uk0l?2@FaJDuI{z$0Jjm#>wAC>7|_=Bezjd(A% zR@ocRngho2IUkWck}J?Qnyi1rQEPKG``)!?OFFsV|K-hi3AOr>VMoC zEQ^`VmMr?N!Y*0%+RhO<%#bl*OVTNht=MDB=_u`Ke@HxB6*c*Ph*rpC;qC~h57EBS zCR0ORYGNC+aJ5B;r%JBfy89PDl(JNpsMovJYP~HuKLwqA0y~V_vi~|Fp?VM;OFuFp z%LUt{NpIn~T$H)?iJ!U0kpjHtdwk|NAOLPSlW;fn%Ot;u8^#r9+^~#n?Osg+Z4`|9LWN8k$`~& zp1#3BdUif+<<}Qy;c17}-L)mERMFFoj}g( J{FIL*E~(Wm{Y=T=i+orSp(5|O|4 zUItvszLlRp*t5el^r?kiXj%+jU4MF>D{!DHC)Sp|EwGTu!pvtwHIN(2{*G&vkevEn zdxg~F(QW8>A+4qMVwSr8aA~(wZRGWVIZ1#lTR6~)v@U0zDt-JT_&zz+)jFO>14c9Qh?R%e zS=db=PI3MrHZH!=X*Kj)zIm1H0J^VUo07A4v9MN);>&)l`X;VcIsDKmXu_G4~3Nl$kK-)E2@ixqPL}%sBI7xOfqz`(*B_P1(Do z1CU}ldgfmlId2}nmLD3erwp#8-WtnRu+3{d7Sm2broK^Jo-a%xcQBC;kh=5sq`Ifv zkhA5hLe)+lQYZDZ2#8-LHu3{+&{3rr&8OG ze@|^VL@P3O7e0MtqT8WA?Qc*=qd@MfDQLHODWPW+D#>2-z+Iat-(vp3+VRZ3qDK-R?5lUrG!!}W#Yst2fTbmgKyR; zZ$w@8@1(oCa<^R8NWNUU4nA|b(!!-qe*0&wb zzWY%Jm(65SgOLzZq<;1+35?gK9jn?#H$>BX&PJt2ZE|jSWb~j<)l_D=5lnRzpL&ts z{e%1m6B&igH5XJOQKn_`teH;rk4_!^Zgiaf-Usa`nqzvbp4{A!}0GjsiD}InBKJezq|?85I}iX^*Z* z{jMKjNguX}-$4Z5e}}~=IFFh>J~*HB!-=GNbs9k}3^gOQa`?D;!j&NL_Bavap3H*$ zaD*n0;{q9tj}tdq{+PIA-!S7*YeT_{;bVTio8RK&Mt+~>CgbSqpfY{K44#7_SM|oU z(}J!m?_4<*UMfwR5hS&T4pk0 zSIGB*1MlSxF_vu|p4~vA{q2*d7mfxk6&#NU+CEe4L=_c1)%U*3D3Xz!6HnN<#l?eF z_er?!=&*m3F(ai!)GF%sYq`2#jCeg`{&J2A^)I;*A6>Gox5bC-{h`CeR63KR3zaIU z*}pZ{qF`+2-yeTXZnKb~Xy-3i`dZ>!( z0#9Zm(zpwOz4DIo@P#xtLD;vV3uvd%X_a^|ez)QTH@L~HV!x+Z}mmRG$+&k=W zS$6RGP=;jOFE7&f*f_t(m!`mN3!c9h56xQ9o@{m_`efu6$<%TW-FLjvgr1lpTq2Q` zPkFnQv49?(N{o*j{OEZRQte~I^&8WIjvFfRb3Uug^kk*d4rjLV>i)9Gzd_MP#dYZkC9Ry?uaY1Rs7> z9p&eLddM|WHj1dWKBn5G5PLN%8)G@Uph{1T7s-CAkG3tUL zYFkhJH+^(wbCR*C-rxyeI}?rGt%XZJz@=O3PzqZ+a~$1vBOZba-TuN#$ZU(1nXCUH zbLJ)jE*^$AQ1ka@P!Ra~@4YAG@A32hng(x}zWqV?sC)YCHjMsPh7LigwBPeEo#i&^ zzTx%39o=yFMGL*2dls$oBliM-BYx^MDgi6E-T&}zp&5fwBnngES)I*Z%R#qVg(cTL#q!PDol8!G21uxyd{GwRqRLV@o?SW$Kjwl7c2Rb9{BX8I%k{Znyu=Sm* zrH#s?Hx?JHa(3wXj~6YIX+6b)E3#x_4##oXFHLcpPk-h#lv;#k9=mRBU>Ea%r` z_1+qe&O{dz4$m4sJNjqn2@bJV+}u7cSWZ>Epo9X0G@4UpeW`exSQUZeTxjK2FM$wigAto2Fw{Y?z^UoQS`Z4y1B2oKj;5s}ubq<_z+gKWj!WHmQQ zv>1ym@rc)*x)G&({Uk0L3BbctX&-br@kdWn-@A-ycF-DyrfoLvI`@Q1)RyI7Vma(B ztH?Mr&j@@kX{FeK2P^IIrvyVU!!jbDTU*~?zL9x=|NCb?0r7m=i7zDxN$2`glKM&Scd5P;??2nRe?=ygnExkKnBexE4kn zVu$&XaElNO+%)K<3l|Su?Ua7K8Q{DR0+o7Llb?{+(R*=g*IcKAx16DO(o%v(+MxGT zvOJ^0nTlF>!Uq=81C)A)6wgkHz0*spDmw#D@HQU~@pE({?c7+KBF&y#&uh={VXs}B zei;uv#(I%IX3R}iGSdWU{dXVGi4_Ql!jkl7@%lJj>=%x@y&4bqv@tzTFDpC@UAsZ0 zwFQ7G)FK2mXV8;Qt;Ll2~YS zefHmFC?wfj|8iIqfV>rq20SL*BPX)s-ziUb=en|XB_0y#hJ1ms-&Hs5A;_rkTIO-$ zQ7s&+e=s$VBqnl%qBFDinvZ?06RP)N2*&W}nzWWopp+e}pXcvJSBsOF^c`FnV(aab z%Xx`VJZ`XlNKaIOR+4x_ryz2gXA9Fs@0F7`9%k_I_^@EVaeap>g~8ziT%)$sJwe_) zl1P=we|yh68KN+gxTZWQ7HfLhsD$)br3!7ky)=677up8j@q;7ywt3q3#Ny+hzw5w<=*OeUI&y=Jm3zTlXZeB%T{zG!=;d|zvPKJkx7mPuM zhcQUu#*bF&iMIyi-)e86$`OeRO4HaWe>$)0t6s4KnPOCeb(-F527(Jj5Xbbz?CU2| z$DASRe^Xpw*{mU^Ku%ryh^9NNx;~y|c&B$s39vi`GUPWGS*t5}G7mGAi{w9@MUZ8i=-uK8q>3TwUErShsQZ}w~croc5BS%y81HW8^?mN<8%s5#` z^tloGuR#>unbLPSXZKPXF+ZY$Vw+z{zHu0*3$?ezjeiH)&W$5Emakz?sZz6&h1vsG z`||*<{7CZ^wt>-0WR=hS6V42gj?e4j^iQ9}uShjb*DN_x)NFlP;QFXf+4lSQ>4Uk8 zE-J|-YdyE_=4&zk4r5YiYD3Rh$l1|J_^Srr9b%K-)plM?q1v;@Wz;uh zE&JZdwuZxNO-Q)oR9HsSF8NH#_rU$@d+q581H#qIuRe*MmdV_nw&E92z6!{@c#a;v zs~YI1w0;1_!B-@*QaieReT)mwqkz_(g97I&9`tR6&EvIreyuP z0+Z<|)O6WospoL%AJUwzmPpIE>7%P_KR?Sg$XB*dMw)pYYB(RA<>zEu31kk{NU@jM z8lJICElLC*B+j6@Y`BRu32?5SFSdOdsm0o4$uAr&PqJK9`W~-u@vAbd@Ot<}euMW& zdu)3ibFs%zSHo%Rx8fF~tHS`5+x3C4t=EHG->OKo>IPTre>6{Dvf!%j&IcBQBTMI= zQ^dq9!I=FqqN{Gtu5qvTilTN)cE0Nr&4t{Bi}vk~lCK@Ho}aL6{tMr%{%#-3D_7>gZJl6e_4mv-7f09R&x}!vH1NYoZ-I@Y zT@tPIj1Z>IvdW8U+;qN9X5EgMNf9y4%W?ePY7H-Pxf5*_w(|ZeEBKe^VCgtd9-;HY zPiuUe@hSln+r>F}nKJG&z07yM-se;2zcH#GF>1}GtV9gn4kyqaLyFA4uaVL|CS8f~ z6i?b0_eO7NUDlo%MH*6^&uCdRN7Owd(H_I8aummENv8rjpHR1AX$pBpJyxPMTb+G0a)~uk zF7pO$W#j7VN?t=ISh@=>Iwek}EVCG8rTz;@L8zL#j5<9Vd{|pY@#-*#8TW@d-ugdRMom4Tdp3oYv{PxMIXRLenR$*x?xyj6sB*f)oEd zFhvX+6O)zPE<9wUNxz}qrMws`am}2)c-zOk64Lhbmm5z=r(_&v1-jm%_}6-eGw=JC z=h=_!HhZQvk3GYpPJC}RuJ4i^{bGM&Jw9cln*WBfyR{`f$M1S@n!yjTJw;n?*{8PW zeaO2U4;8p^D?d+%C#iGgu4#=Y4XFo2&IPimsY5qtl$fr6ME95-4ouM=EPt&dbJ)M_ z7n9Izqh*J$Pp4P5;@=67)K7+S;o==I@JPc~bhqNP;bfoVt}%QC_%ryM-o5E#exZ^n z<&8a67xt7-St+aI_s=5sSKV0d)TQHKqg z14q=RtJ?mn3+3{f!@f2)g%&wV+b{ZVkXnqpRC-CPzyDe=a%c=fr`mu1>)Ld^P7&N` zsaP^#B_!x{1B&bKi`#*|Cz4a_8y-12)$MpJNmK0ht#Vt7)Jy5ZSg9rR^^7zjO$kvX zc1YKx?I3dbR_7jFKEZyj;<5l5G3m+@E!o>YaXzn`D+w^a-yQV2l1LS9j{3yWM5vv^ znE+3tdr})&=(B~4=^B4t_6QdbpGjo7a>0JEhqrH}6BHj6=87v@L)v1sd(XVyArTJG zFiDc99=taSVcwESxfRQH6Dq0v@#Qr1&JsO((3=t9GTFE|xFZ9wu>15S_4`ajb{}Xy zXvJ~_kRc#NuG%$@^LL>ocOjbnR>A|TExos6tFEL!Tu}_Xuk`l#XE4kY1pVxK5CPSk+zb8yZG4LZ$!~%O8Cn z&b04f>h|581p|5L9s>_pcP?5IKo z7?D!B@6y1Bd_$+%E4|e%DpU+Khor6QZkCANEq2vgi?CNjc)fde^8OAe&pkUc)XY)q zHC@?C`m_gmgxcW){993bkA_Q}Ilrc`roM1>WB-^gS)=)O*HOmA>Un%XtQSdKskm+0 z_uQJ90f-)A5SNFNt!l1bEUOfPil59XR1R^IjlSF*+Zc%TrMIn%G5u#=V<@X#&&oQ1 zaO)|UN}!?ceZFD@(y6|Iz7Iol<|8C&qkLBV2|+Rwze9(wdk6YfX3x@}aIEqMbpN&@;>vjwTk111Tslw8=nyn!p`NiuXL{-3G{ z`PUz7TSA=n&e2I*xKlSEhb|3EH2LdVRVvPyb7xN|#XaH%))gPBSBVGGxW{}nGT}De zr=iN)aH-z)VY{E+NF-&9j&?1puolXBy}MvyS?W30+p$Zj-F9GVm&!fQfvD=4fFr&p z?5+HqUIw$rn3?(aJQfW7_oqYHqs7Lid05~U@SY=FysKj+2q=QYq?yxk&P9Qo({h%% zYY$f4(AX5XgO4&fFx!{xixIc(W%D+fZL{^wxl_K6(*oYP{h>&E_;i6uPHXLMLsnTg^%5dYhU$VI{1)JT|55)fN3C6%a;e9xij!ZFf&A zrrJTO+~));6SnhSp#D zUnK&s>(iy9Pt)reLEMTsDJ>80QJk{Pg`znGNoT^nl~sq!=DzA{=eu5zlclOlLwYW{ zf=&Y%u1m1T&doM9oS=qwgcTpF2YTw)g%+fQXJFx$dayy3f_#vXirhYb#rQ!-s{p)F{v=?(*Aq^+7|}NRz41eDbGCM?;9j zbluf?U3Vqr6N!`xS0trxzSy3r3q_TxtiZhvFpX+wP>O z8u|3N_nm9#RlfeyscLMd5javp^Mm?-2}v2EFr$QE$0z!m$#c?Ke<81%FPD<+uP#*v ztt_flL;6{?BP%lkQrEF2n14@ zDqINzA+qlzp0#KDZaPDyCx}%A{c)UqC%sGEy(A35%VT;Uv0ij~x3*s6*cn`5-gq>y zds{#-(<0$}e!)%V65R&`8E3z?JG3oHuB4l}Hf``x;pi7q?-&wntrIlXW=mYgp7_5a zF7_01ys3G%Y48*0LOLoE(+3^!xxtD;@kV(|bn&KYvCVhOP}`EVG=AagD1s%4-6w@a zOI*eEMMN^AU?tnxwXE-25G!d$u-Ih?CM7&Oey@}-?tOePOBNcS7pT~i?FjQaJ{c@$ zCXhg1bi8;cUhJm#XJ2)i8Q0}4wyDu2^ZV}+_W9})lNuY5j{GG*H(faG{h9wxR%z2U z)tCPcPQi?5i>*=VYf!xxh%|Y8q}8&_gQ|P3y9>-e{7yE)`SpOVrKg)T`^hTCy~D#m z)s|CFr~^yY$C@TiRCH8BEp}$gJe|Ya@duG7^lYet1VDX48UObnLXrks zBTxZJ`i@?$HKm-`xDRw44Q0512#8iiJyU|i|#H&qK|OG3)_ z%*atamF{W3va+JD9C-iQl37Cd+Y4aRf|X=`Ygu=9A<(8B=%LJ-ETdqH;Im4jjyrW{ z`YLC1RJWpLW@g}c7qp<~j4EW73;v|HR@|*KL9q5u2ejdOOH1<3Ny%(lKO39k;h4lG zi2yA)0m>|56-HtjXnTYX`Q;N)7n!f{JHBnsQcR-Pe6Fck?Ag@vcT5T6;{NVmq#50Q zx1;|Y8$$x{rx-*LU?}V6T`7Td3)E|o1WM3ZmC^Gv^ln+`#ZqzUAu&cHAI@@8%C?hg zzJ=r+D5ykh&^terRHFge{>oahH0E|gjWwRqqFMs}VV2MZQ(|2pD^BD8EUNn$1>NuL z1pU72H}g_ZEtOLQ?FIA!qV5BLAR445IfR6si+WxFNMPi8zUIaRPk;dj16tyF-*Y7; z1#Fdn?qQx2k!P=OrfkJG5!l_$JRtK zweP1XLq9P9->RuMlu-eiA5aV5NYPhbXGT#ia1$#lR?rQb0bt1isty6ZDLhvvY^c=% zGP8w0yXx8`*I}5b`3iPLcm5jvGwWW<*}^gamVW@<{ZVK;pvd6}kgbUZ84`ZtKnv@4 zl*G+%0L}pd8Udi0|HAQwGhFNypdtWFcGr&|+9z!Q94Q+?@dKd3z5{(1G5}Zxzagi* zd_>KBZ7>N183euJQLua0y}7%!2<{(0Z<|u2I4ko?f@UI z-RQ!yJq2SD7ES{CA){@6eyYDv(0hDpNEa}+8~~PawVppgaUoa=D$nrr(`bMBMV{_; z(0FC^0KmD^&6S7i&aV#8a-1VT{1x-ZVJuBYF!WW;)EW(*d0AJ}|Hg_U%KvO&*BwsY ze6dyB)zkBcifRzZi8MGY-2rHH3?mdMnvDA+z_Sq%6L;;_E$)q5rGv2s9skM78rCQz zB_-=`Z!S>~P|ubHHS_*h79dgM4_pHUR~yNaLIUZYIbRqW3Z@4nJ(vLExDgsLtE7-2 zo(9^lk7mBS3-*5Q=Sv}gWCC6Z{o`^^)L5&J*BI{>z{>$2d!*a9Uc7wK(!t{d+X3(s1yxQB-l!bU_i04`|kk9^ojYuPOk;jtU?HA^S~qXtV6S- z6BBy@8h>*%2Mee)@WWb8DIRm`0SsY4u6%;iggQ94=>|a0lC7TZZXIv$onOxlymt9d z_vYmRt`K}P6Bws#KJ@`L#HIXPh>I%<2p4gC9$Q>pUn@*0rHl4;1Y)71u(Uuv;5Tra zJfP8~=Qa;L-knVVIM9&vEH(t2#-y{$3XPuQ#jz#mxz4fooR4pO)p!1lh64^gD?q_~ zH{yamxKK zW=;evVd!)D+)zkdYp3)ch=wzXS*Z5A4>fBbXCVt2yeGCp1y@qSb2+?ekT(;|)2;74 zv?p3^iqqz&!eO&z##&q`By*Zl=sqKDUj=Ky#XBp-n=m!y;wInM`dsZV%IKw}8iw*CruD}Y<}?D2;G z2WQqf1pD0bGb&K?VFR%!q^(Wj&6_vXcJmY;2M6CKTtd}lg7mP?Xn82%Ia0`vp)D^x z(p%e`=>PHJNe*oa8BCRYSB3kQ)(haw{yPuG?m1T*fgZhOPMg)sv~Ifp(Udtik(Mbh z(NL4wH^x#$W2D&*`AMzfLTfvOC3L#2rn|-lHldS~ZbrCfGyn5(X_Vd={wWo0PiiP~ zk{YvWdyPK&6grXJ>N=gRG@+r3SWFePFMf}Fro~?jvpO!cntXoqDu)BVnxAsYNxWz} z#e1Rz_SFdx+IS_l0thADI|07|HUG71Z2-At-00$XQ(8u5w8~rw4S2>Y(7PYt@;6){ zz|TiFsU*4)9ayyA0LCAWN+?<}4T92e0DxNYNE$IYptaS}(SZuzKrfC509_-gggXIp z9OwRh#5jkp$59`X$5vtfYd}u|1B-E=^kAn7?)0Y%2XHesXY067tRxT$@&Qct>Ud1) zgPB>fB%>F+-0N%)Xyv33G039W2SC^^Ej_1Y-D00c59m1c94vkJI39ha4e*OaMLhNk zq6rr+GBTxql3kqjoe&e*hoa)C|78EVZ5=3G7CyYK8ZcoYY2s0SSvVp+L}|!YhDnVX zXuZqPw$7>KHIdsc4|sbIc}Zs3Qfh+kZE%R6x@s{R|CjroBGpD#_w6%P=3OWz2DtTk z3F^U-bi}seWQVMeiP)>V^t$LtP_N}*k=1o14wgaMa)&~gVaYlOzMGu*#-gg$yx4IL z4--@JpP59zV$~O2sRPp>ruR&XN)Lv7eUzp;{<)YI_PvJ{cXe7oC@F*zae2zb1eVfQ z1_m@>odqgqt*PQZ7u?33_W(z)VJ+GEwY+=|5NZI-v+iRzI1nC#Bj6J7{0v`QGN~vq zfGEYaM%P^sq*W6kP*yaMHyv+H+y{hBAk@-5*WkQ4l69BX=a`5<^Z*U`HV{+@0!vK* zz`2c4J?7Bqk|Gf60(S=sG!_t^G9Sr&38VvqfZWNi7A!B&;=dE9y}eOVT8|Xj<4!ZF z0qTNVlcfWGx0i+>foj;TofUH3Ap_hTv)!4RypI;hqgB!C<1sMJLO|?MjgF0uMl(7i z2gvJq{U%qGJ{rjPx&cu!HJR4vOn9aV3sg>N%!gC^z^W||z}c1y&B8z6_}MNc>WJ9!qDImy~j zru9Cfo-uIzQnt&L*&sLY{zrVow!oGarbN7H7$j7Kx-@E(EazLE>kEyGp?Y5Qq*9U( zk!6FGH8aKILKJ5vY~L!#*@cOTlVG7diE;9<_R5;Z{zm-Ttyq=C3c51T<-Ht58igxt z|A<(;mx&)D1_`g!{{RcLf} z41p7eBxebImn+4YHhkeFpv2-E&ttGk4WHC18-6+C)z(tC{uglzNV01HGV_l08;|Xj zzZEVUd%s`wPt=wOL4~LF3S=rek1`Le%_GE^90fhZnHI`VYI^)rHw(59K5y`coldAYBe(|u@z6S0dD`5|^2`#?B(#$< zO!<|_J!g)$KeOIxnNceRnXg`_dfC~)FV4Cc6;tK&;c*YxK8S*yi^FzlofG);^r8^h zbb)|13zTRxG?LQ4aDBq&N#BoTLQgln?x!L~N_rJ67=ywED%VwygWKv`RCr{{>ykm7 z%wkV9g3H3K)ga$~&X|orzEw6rvm!ViQsXqpU9dwZ3dQxgL@MJl0ThF85i&Y?8oEk5 zr9quqwl6HPFK>3U)k4A(4{ofYBa;DDo2Y9j#$Q&C?XS$ut*dK8xO!=}J!=;8v#&!D z13^8s=%^vHc!#%2jj8|ig4J+(I(o?U-~%;}C5B$)y${{4d*ZXj-yW+@%f2?@)v6>C zkJHznDljQ-ZK>qFh8O_u7D(Lzv%3{A+nCfs+TV0L@4J^(yFtC}>&}9O)z<&V{gx3Y zbBU{=7?^JFYk)+SY&nUNsq&WvLJv&Avs%!rNusB5!P!#Pv3lB2dv{#<08(9L)zIJ6 zOFh9{)22b=FM*B}hht76{hm8ddR^c{ZS~db)BhbL8ux*N#GFt)|J54~JdSM(`!ucO zLxnICwyKi5oVybS!^f874jxt{u>>XUM=yRc7Nd4_pC85vOGh6*ZFgo;^4{~NXZ~^; zC&G>@v7-!gSdlq!%~mVg5|`ywr&7IgC9Iblz=j21`niBVid4q3DQSFXQ_@~Joo^1` zI&z6#^UgHC7A65#v$ElEd=NFHg=M=R-D22dG5%~l3JLyuTS+xW;CCLrUXtsYP)k^M z<($Fjjkf;q@KEw3E9Vp-yS7kgmsYxI7ztNCEP7wZ&F+`U^!LZAY2zj@0{mY+E3uA6 zA-;c1`1*y=&(^CVFRlM73+jKIUnVdD>SkpN#dZQm$mXaMg%2$CrJbF)W|uhJut`yS zLg_F6fcL8DHrz~N81}YG&C_(!Ji{hObNyD>yH@kCrh1%xOLk#E!jCD%S7>#|Lkk?* z%X{f9YWc4gdY~-RnO;Pp6H5cjRtquLj_WDj+x0QVk8AiZ!Bh`c!qiU}A{dySP_G@{ zx>yNZ+?eD|hY3WmIooArp|jttiEjypj+ZR!XlW)-P6kG%e#fBCdg+bajp36kFrHX2 z@K`>(6!PA%NrH+JxyJ38BQ64|6y#V;)Dud7E9Pr2>YD!rrFYhD5$dR|zEIHU^03?K zpKEP|6&<;HfnNwLU4)x~*ErX3XFz}TPc-YvJAr>VtN@hRLd?2~PJTUqq~?_gz+IL| z)^#{!`0QN4$Q-iZsapp>f14~ltC1qZAxpAi_ z!uGdaU!+(lPQ;q-$aiw0oaMdR>WE}FV$hu=H~X$=>0!(K!s9i%-G2)v1q*;3$=X6O zaA^+p?@tRPtwfwX8uT=I1O~_0bM^d-Jl?KIf&kQTWo=>K5fCW8<(fUV)pw8omm%__ zAMQ57awsEyW0Yccn;Y0PFA~XnEU`b*ytnLP0r@AN78%|==Z#i->za6@+15z{r7`Xr z;spuT(739G2L8#Dee)AdKPRbumg-Qy*e~DBoQ@BKKP`$zL~`6j`L`G@jrWCtN2)Nr z=L0;D(47)*z!fo@Pfo+zLg1$9ioLl_=Q@Hf=9k<_F%HZ_^uXSPH+j)SgzIjel^fYaocd5{zWbvka^Wtli?|SK3$Pq$9 z=^+t$*R1NK?%bD*w!`OsRo+YktzI8z_qJ>%n_M%^xyi`frb6_l8U3l>sGggBzdb}K zVIemH6^K_J^O&uOUKlKXYI&7+)U`sXUHzL$RX1(;T-JP|f7ko^ddz0_khSgFJ9?DC z?77LXkIk(YMV9f)!L;j^!ia!?)(wTdSI&Y5)0$C;^+nZxlnVMNVI|*OxDh|Lda$Fg z&Gvds5}&*Ap)K#HD|s&`?+6`ZZr6%^-&`3Gyfc(xoO;g`WttyDv6Vz;lppOY;lRr)Kn z(kQq~6Y`2TM$|3Pc6Rs;AFerXC)GHRi71JFulA5iLSC4GEqdw?F?UZd$sP@8l6~4n z;4K@fnq}~ALh3%w!CZckU#ZZ)L$H8P`QLQdxoQ?ew~qH$b>ucu(NG@~MAUAPT^>qL zd#86tg)Gkh)BR7y$IeW8+w+6@42~@lIP_q|^(o&!&K9CFDq1bbTOghAjBX?fGvdzX zL>-2gE6qB5F{VEDYscM3myZZnUWF*)9Jggr>k8c19zG3;sK=G?6H|Gf^;Do(>XPjd zfWza;sBDbau=wqZMQTiX0Gxv=7fA=x8b;k=DbGNR3G&N~#Ef`fU#sMN zx+kj57w;JP{goTNs3201w`}_`$4=wj^>7G-+9^oGG^Irb-$KT6e0J*5h^M zL*<4QXNxwypRFUA%31{n-%pp_-~H?xpxH%1^BiPkuy{anxgH*h$CQ@KELVv5nCl}` z7xL4bNArqG?|;Bp)3v~tj}x)(lm+RRO*~J$L%KYzHEC}XCY6C=kJO_7Wsm4ipj-SB z%3Eo?>pXjNNhF?#Wa!HeUy}ZRxg!=9nVZ^-d}ST+5%Gv(b*6s4uy#5>tBFXS2?E?> zrXHL0wJpKVym5E$T=n%0vt7JqD}N60JQqcJVDOa*cNDep_9Yam@KSwjJjqe;L1Jx$ zt!0)kN0|(!AsO-&&XN`C(iPY*3Uwul3UUM5+S2Dzy^G^iZ6V{P?Ft-7rKxF`#v-Dr zW840rx*A3vNbe4fTqi`m*yi-vXl3MZ^&J2|DJbf=Xq38UHZy$Oja*sZ?i-lmwY2N& z7gfT!e+r9sO#NbLk@;qxxqESL7?o;cFEOeMR~KxdM~AOdHdDu{o&FztB&7T7e`^RU zZw5i*TpPuJKAW=~u|0`}z5I`lXcA#)r6uLty&Z^f+L~xVGko)2+qBV5Vguhbwn$n9 zdM;RdHaO-CkPIwTig~x+6SkP~7>iGK(ta)cWH2MdBVmm`Vw!hk8%*T2{pGQ+wGYCw zj^~=9SO{8Dvi%d2g+$@-lo0)YvQ<4+r`1!1lHc>!ovvrjo;WuozoMXjspRSpP32tUM|Nh=LR+b4_Rx#%ogJuN^Mn z)Tman8bm>~-btk4BzmH2L8Wz%jZy-cN|B9}|fXNYWOpq;a0sI1t(NXkMbt|#Wj zkn^(oTBbBu9`^p1qiVD^jW(MZscMi%#o7j|MPg{r3~Gu@Z{;htyyn#F{r%gY7jUi) zCkKiFmu7WtS`7lVeyi+&R+aDbO4bu!!d|#NYWr+@cdXJzj7{AHs<|mBcYn?9_dssj z9YQLq_R(*srG2#4hhYX9z8;+4`uiyN?awi%#H;gnG>vZPrlsw7fLV$*4B%fAblb-h zroP4{4K!3)Qk;}(LNPFqj>iWwKi@fPUG(AG(bj)t2;ns1E}#;Q?QGyB=ldL0D-v}S zOO5k~>i)L3p9mEySEDADd6_?^t`$wU>|xd%`X>Sno{}M=*>f6v+R7cWHMfue|B7n4 z0KTf~eb8id=^w+umOrQ7gNz;|It^^H6HDg4Dl5a=lnAtjG)w+Jw2=HkLvM*i*tAqg zSALS%wO6;;�@bPM&TyuQHA87PT6SFsXrBiIUkNF5O%7n~avrNc*CjGprlP8_3Uh za=NFSQpy6K3o<(smfmAy_tNyMsp)+d;-m`=rSsY^@}~X2cu?F~w^hMzAV%kEC038@b=F_Gyls2$8kFrZl*wo)%&cqg1R@O|%v3Tnf?Om5ogB~3j@^AeC=PmiE`(OxLOpLx|A!CY}-_zhYVQVf;X z6_x5iC|*>HEZkOOp}puGe7Q+v{Utd$5-;N8_}WB;GyME&l#h_d`C!KW_NAfKQyvk) zj@$v>kDnb7iIjrPL5arlgf!Pwl@P)2iNWxt)hQ84LcV%$FY=&4Dro|JY@!#F`Lhnr zOU{HRSsM%6=q~#wXwgZ*;If0UVUymiIbuO@^DW&p@JBtgZGC(y3lP%G;4v?^0$zfP z)>tbXh1Oc|U>63e{xZQ~_#)8w z2r8IL%JAzD`nfBLTW7>R!-2=Yw*B7Yt+X_8Nr3S0qaWTk4|^TN_Xm5t)G>T-Un~ZE zL3P+;9yvFbNtkPC#EqoVH1Tj|`ySmj`;oq_zmG2a&xiuy3B8cd7WA{Rj@nk;@E^y! zta#TCv&HY>Y1_@_Jk0x{B^&GEO49XNSr2lF-o;HVccvYCD})w^ z?kE-CK}HnXRh97qZ;ZYS?)}*VNn(J>@ct8f5E;=YHk>i~Pn(2eUZh6c$z^FBpWg+p zOO1#8w(7*&TqdNaIC^>=VGQXyO@Ba!+Y?1yiTz$0`|GeGKisfhJm<$nUSgR{#*eig zj2sJ3-wQ%LQG|GxGIjr{K4u49@@0aT9Y@z^!c>WC8t{`?kM%Lg!t<0zjHHq4;vr;2 z0af`6L1cMV?i^?xbdo7wzxdggRb4bRrHEJhMG)#5hMLgQ={X=tFKSjrdeA7-d`aH3G zG*`yGC8c~549ij&dVy3)q5H7*r+=rcH*9oyt9kD=vh5WZ+Ao3sSiJLjLZNvp!B=)e zL!s$M5K+_;COfh4GRJB^(=_l*#j5{|`TEKeyU-yWw^EHC}kdm)A*)1n=wd zE8sByXZ+NT3>uQX{V%stZ@9Z<>7X}spO@%A`}kt;6>feew=4S9_}q;!9JEU|=^*@k zp&Uz5L!3O$CrzMx?}OR$+8jN1cYOO02K)_86nQJLq#38(AT_p>%{@DIGCUR9&_orC zWiAXJ=PwX*H1Om#q{-1bm!4tU)yY{18BIX_`qY|4>&)@h+VQ970uQIw&-}rsKjE+t z0scp(vLzwWabereo}Gl0{?-W3m*XMXJNGFSraV_L{gsuvu5y zNJ(UEWs`73hF}23Or~R(R^YHx;f3mii-F~bPb8G#H>|04hp`<29UJ-#3gXrsL7*NqM9UQlERAk z7lWf?LaePnq3J}hl#E$LH@naUg;S{&9X<=%5q%1y)(}e2c(L`_@$iYp-Km>{`kU#o5= zNWcUUDv@tp4*3gSdoVv)oW# z3BJA^N~ZzBvmdHuY4)FXu0YP*=sf2AiPc@Yp#=K$V@Ur!C7zBr?{-Go=#K9uI%hGM zGVv46Wmt;CpR^Wmeu9Yp`VDswX=;?NPGi7X(R;s7|EZ86O-D@s%v-VB-+Mo-X;N6Y z}P$SkAs0sR_%;)zA{a-{RwPV!(!RRrt~EZ zI;AfjZ#)XybpsbvhA*2QShfX?76yY6BYSKH56asI7UnYAdLaZlbSyNFs^*zzORi9!&k^^gbUx zc=vW(nUzkJsaB0e&8b#t+eC|LW5BCs9U;%3o*5cjv@|DkGdxD8bhF&coRChvG>s3XbhI-K&ph!m8Ulr3BE|pa3dMfEpha_TMmOC(-Bo=jo9;Q zupF8Zq=1Ub*xE8qb(eSFw18KQ&+!M+ksv2So|2tY7w?Xp4>=TRWL3349&(RcT^&iBUohbjhdW{Io&4PK&~0r|laj}I4Ju+U zt#Yyv`boOMqwN$g2_on;5LWGgeJTEotA`@gI%9`vyQEScoOB-w;)n&qQ>$tZ zu^P^^GDWM2o-`u)o=2{|r_ zKu)Us&ou7+6UvQBnxVH`#2BGcP`xsSI@rSAzwxuY*-1Q;-WLf{dTiG-P@x_B`gdATeNDT9^R@cLsJhY^kW340-h3z5lQw!H;XH%ZycUSCdlu(<0 zrxnNs=koTD@!xOvP9pE$4ywk!+pDjrr!2RAC_Qj?(5dOZQeypWCshUNCX8s=&8A5d^PT5{$VfrH=fc0)6o0Amk8j-C zK1GE9S@S%<jwN z$N73xb>mXY45#OWP5xxTvq4Q|`2s~73;mYrpY0o(<3Hdwt^TElSu5@P=wj z|9uJx4t`Iu~{`VChB`>Vt9~P!a^haW-JKtq8>1LTI z_&|*S*(d63Y_7kTx^%z&FA#dvNB^b}Z@Q6ud)`!K)}SAY@xxR2LU#{C{r>_c_rK4* z_+DVoOTlnBzXHuC|C^>&o&PO^9sh9EH&U#D1d|~$E%>`9ZXTF=7Z*WXc#yj$Omu); z3l2tz)U(pz^@2Vpz2U)(yb>&s{$(?{vDBbd={H8=+3f{ni^(y2HV%DXn?PS9xe~0l zzUgI)JNlD3CN_zjIofKLdAoKwl2?-^_53gxtdmmS0_@;-N+_H51Grgv=+CQcmTHsrT|e@%;6> zf9+8_LFnIS=GRs1Uqo{s^V&5YevtS%BHW{I{v6vurmd@QjGVad*;>oO^v`lBx2dS6 z?~s!{51(Jy>3%oQ8CT3MeC?YvzjXDh&uzfMi8muav4y71%uZ8yS;5*U+6pW1B#Jwu zGak+A*9k#YIoY4{TDmGbds6j4(@8Jt)7qZCF@w>ommvD-tv%grvP7P?AFg=igjiZr zAM3pkKenY~KvTjDFtklP1_q?z0(6H?|2d_EkTj#!ovt8rHb9a03sa+fq==8-yk-)k z@mcKUmN#SJX+GwwZ91mvgNrD0aJuz z;7N#8D04Vsa8yY@JE;;c!M7reB(v0ntk1&pbx~l;MC%G;UoIOjQVfx`pr#L}|M)og zE2o#lxzqIe*MGCVGmv62Q-)5~Zz>{-;<@E#YO(z!O4>&siQlx-z#~+YG_CvVd+5Qp z&g<#R|v{%TNy|Slz#xgz~68O@^44&PofC$Gy{5;=e;LP4$EFDLZ~rIc81Lhc$NmZtg9_Wkbe zt)vrKYU4)!Ou$#G8Tzy8DgG)oTbT@v;A{pz=Z(IMgamva;l)asG@f7c@O)Qn^N&bR z0m3Zm+kGom92nJ!sc`RYQZhw*-0k1N@=vBvb#%)_kf|l^SLzdqH2q7$F3f_@UByr^gsq1W%5$hJ!ke_-?&_xj|IUhb=W$JXCeWe~5}L!)Q+LJdA!dCz!Gu8F;WCZAma7n8r} zjpFYN`!Zq~9hJB>G)AREh&NYk&TMXhBVvkInQ)zmB8f~jMdz2)`}S`f-mlB~H(2Ym zo{oQuKm@5=DlRBf zpZJ^5e+_`T*&J2=u>2xi2;ceb953eT$AU)7rjkhXn)R#M$3@EF=3c3cQ-~xk5$N7k zgw*aXWQ&;fzvr~*Rj8j7pIUxdw1OJmO$8k-EIJSgZU|IZY2s(raZAONDS!lZ%yWF8 z*1#GO;tRdnBlSE}W5-N+kN9ZWkAHM)nh~?~#p{rdVG?+rp8pxEQ7dkmip4oHh)x}} zZ|r(iHsJrlzxn-AAO3f9xdK|zxT;59BUapv;7}0%jz9ImB36GR5YR}bq4eA*mp0x# zOdErq&s$R~r8+R%XI6cPBK>4XJ_(#x?8Gn z@^@usMxWxG8J;?=y<8nXvWPm_*>tE9D)HV(HT&aff2Xm5#py4_5>*~bK}Z1@rC3pUJ*lGQbes&Mvve?{J=|LKQiMa?)=U$ zOlvq5N#g2qtKsRvid6c-k!Ud;SNA|MhAJ*1snMRq*n_2YjNOwnI4IlTa9!VA8w zen32thv(XzuQRItf@<7s{u)(R`9kckT`byju+@N9g^zpvkW4crNoZ7A<|D2(RY0(y zXjDWNA;3{ni+pW8Z>}{`d;07o{3)-?%bC@N+minM_)oKMW*Hv2N7)X~l*ru?g+Lu;v3)1s=j$(c zd@#K6B|sfDKiY;|ADz}~Jp={oakaqLVA?m+=rs~J(;hqAi(09^EcXq_$*Q8wLH%oc z`xB!z?q8{O&5sONL$Fig!?d>fLabIZ+D4%(vt~qkv!)h@kfQ}Bdj*kSR;1xCSnh|X zWaz5K=U>SW5;g7qS8B6w*%!W(DGGU4o3S%F$T;ux!8|$`tx}*!HK)~!sX&Y(z1-Z0 zmFUGDh;t2G{7Dlp-JCzmeyT2Ddo9a7fE!I;{|LiG_7>SxPA{g)g=XB2+moI*wH&%zlaI`w@wuAz9uTp@9`a=G-zV*z zpB5t1JBq#?-*7}4d++!xfe%^GN$;8Lt_mU(R7yN^C9Zif+FqdFAaSclet;Opx?iX> z`S)JwcrW??v*~-jks}tT!ut^%H7-ecZA`VuLG8ARB?j)cTm)w=(_uHA0VG?a-h9Mr zS40c<)p$7{w8du&8RvDbylyTIxaQyXOD)uVoxXt?%==7rt2b`3bW-*mU#5O096dR5 z1AXOWoXr*pi|%fS_GKq80zdoiEmd+#5maK1dTNkH}%kf)}Bq6+603)+ZhgAw+|p9sSc zhaJ_C#IQ@Od&R&r!1<|i^(2Bz%irl6R&6CxW==Ir1pz@uXG`~fs~Ux)*uC^YYi)n^ z(Q0Ta>!(?{8xQ8re-or_lsoY86g!*Ooa6v(AGc^>NUZ)#Y5aISGPxD$H|*q-dnP7p zJ-my{WdYd*SL}9^y8!lKMyYlMXvwwN3QMI)*iYt!}hgvIvv zbDpRQ3xsK1-(%o>-`&PhQa-XL*y-d)<^LwU8A+=y& zAzyWY>v+{{#p`lb_3Qn8p1|SaP9qVk6N;e+^XrqirWTRMboo=uZ{$+oGdFKSkG*24 zv%9G)+^uM;Zk^=LE+htZ@4as$5ikyPcl{fYM=Ip%~0?z zs9O8WFIJ-jbW6{kan@B+r}$*PXj&2(xaqkvMqx9ymfjV6GA+Fvn#GH-A^mqJc%-nmf@IH6$#0&&?;SJh|Qkd%D&Jh#s ztpRUcoo?3D%XcGg>_SI2?^TbhiZKmOE{qQQ?_7?iC~}UpQ<37iZQ#yHJt!h04b%lq zUopD;X1KK4G)8wZ*xQUNJ(zx#ORh&i(bzm=QV@nQw#)L=sf)W3X zn5%!KXXe>7?D~? zwMIyG=u2eF%Iw+74)`5+7vXMG>al!%f6E7M?MDJTXTdKFrnHcT)H_TRL<($i_l>wo5& zCdYYKY${l{q=y`7pcyetIMB4{K;ShuAE+dAWn_C@KYG+?Rg-Kmn_;s#b)P2wC%>k~ z9@`t1OM;nF8@`>0g$Sp+JxgRx61286O9kiKq4GJh%8qx88&kSC$N~{=aeHCjv&ga6 zHnA4sM?IzanSOn0AoZz{^omf;~2EV#mjE> zhbl>~E>P%Cs?^xe*7%WPq*%{;mNEy7{4SAdPs)d%Xust3gw2^(fhJ->4dPsHOg zmsiGq@+Mqz^rmYunS1i0afhqs@uzX_g|(gDq6__*`Nk-2e|`S_=|2SJStD z-u9}0xM;$v!^SW)q*17sbK|sD#n|YRMuhuWnd`SEjZRk7l|k%EGIF!oXNIR>lC zrVF;@AFq^w;a+;Epd%%`bcA-x)KY}_n_NsBoOD+wZ9>7bt)O^*i|-TD*(_|c6~z%S z4j*4i)ako^Ua<6vo5f?Q|rJqo(Xewju+CFN;rg6Cxbl_3NwoZi{Ftm%V&s?PSNc@)C}+PMMce6CUHIYD@7esn?k?K2|y+%tQr; z)A}Qn!)dLX3prLrf#o~haZ(blsN8$PffV<`5vf?Z9ZFaeZRH;(rMJx*=hnbmJGYQV zCNvvRCeJPPUamFIu@XyeaUzh{<>1kG1*|T8+aCJxa@u-Nd)~2ug*<{Y_YoY?pGvmL zn^TIWs}n}#io$QbhzC5?1zInV7qlij7JsP4e6gRc7z`@esXWhB>~h4aB&_FbOR|za zTx*rtOhbX2jZEJxEtq1o!mLA6RV{j5 z;dX0az#lz$zx)`^z<{c!Zy-j;G^?+4(}D;$UsBa1e4?^4#`c`v-r-oj@>D*toB7p( zXhN9+%&0q^vgI+x#6|=L>2#d08;7^3;wlDQOJU+O*TeMsV`k$5E6b>%sSl~sm0`9Y<@II{yQ{ic2X3Q;zGmEE$B1Z_I;2$m}8b4o%lpGag00G zAbHzuW79k`v7i;D;DTNoeQM)b==9-<&!x2u!YohmmnfX{Z~^?%KVNz5_>Z!Sfdlk9 z+t;as<$@)-`ZFYxw72gDJ5uhCZ8zU#hfMk}BzycRe{u;Ow^}DgE7k+&4eI%1e<5k@pMBKh2SJiC~n3%cjZ4|NVJ znARrb`Z>-BJfJY&7i8RzlhxSMAZ{2@mpC2pIofFaxvAmNLlj+rF85j}le$uGAuRtX z34K_JX^AeqMvS$9pxLOuR#qg@J5QYsm45dr{mjemJI}{CKaOtFDlcuNcpSH9h&00p zesueMMu#YU2NZ^*^p0_1mf>@rV@C1vw$!YtTwAVKd^7ukYK&Dz^GEP6WOnS1#{}ON z`?tIl5Q)ZpG}p2@kXJy&txqMKFoC7x!5t(;qra*}Vz;7g?~R^bQrw2Z4&A$t&aXq$ zuCqm~gvNHBVmTYOjmyDFm~kRxy!&FL*G9q(Me)r^F%O=S_6&0o@qY3!1H>|A=g2-) zAY>1C;I`{|!HIZzpv`gLeCy;mH(bE-QYd|gw|lQ+U);y=yu)_{8}a26b1R4U+IG8^ z>`gZ?k&h5!?2JEYG6kBsN-vPB#~cLNn{v4FbTG*%a#x_n*f?$jsrR~F$2Jiwf!6Om z8>6W+C(N)3GW-^Y2v?B~C`i9PnaI#-S5Nvd)b_RGiIOCE@e6g&!q%Joy7RI1&Jx|F zN3D}kTIZEjcjjQe;ULML0towo%PUszyKauDSIRk(uDJwrubf|p9ls!R>49t)&g>U! zk-sfKS-ZX^JiN%Wh_5+k3Dvk{z#P~Kq-}I)M*a1NzZua}J(wV3Nee^k;-$XmT-~(O zB`!hZc-Ku$jxJ(WkNDu<=dE3%rLqCd2d{=ee$Zc56N@{=sC)e)?vIL&6so_~L=0y~ z?HZpBbq>$Oi%5iX>AybJkmN^7| zERMM4J`YDy_rVJ<)*z_1F{R#CM5EtS3o!op5h$P0?XogIZZc@Mr{6*p>KyS>yf^P| z;ADYXoDf@w{`L9HbbUn3S|6R#b%cpa5VJ&!QXwRL&&cV>wq7$=m4?8)y4>~!8^#7h z!emlRLQrTZX-Ai2nsMS!unZdfscYwT;m5!Cn0H)45@-G%8~gQAvox4u0pW9sO?Zlx znjwTIXVgdHDT+lo@4-hBQD0S>*9G{aMH(tI7Fm-phHAEpg7(YpWh6M!pv?)^3>g|_ zI);l`aOA`zbo<8YiRUi49!qA=jW;d)`&AnG=6AmajV3Qpow_Ep_@>KPo}mn98$d*O zMsnSm>*m#vH)QC1wu{`G3<(^KKgzf;@FqJxjo>Z}8AuW&6?7m=TyjI0I9}pVuXdUe zQZcf-v@CnBKjU#suQ^T!8Fkz39pa*77ksp!IVn*c%f&W2f}ny@)S~yQ7=Py6^I@eq zrudL%zm03AkVE=T@(a9gx9;by9@by;L+tmKUN%gi7!lFV6kUh@=Ut2MGIURG7^y%q z9gG>icjn@ z)u!+d;OMRKVvs3)zcQSCua}80Um(X9`fQ0{P4@KdG)_CoYpYXbEBRi8def%RLs0^`J87BF_ z(>a~xv%Cev`(?UeI2r3Ra#vWKT*U2Sim|tNg=SMc|G6XDaeeby>9rZ|Sh@X+^az2q zf+LBOoy(I;{NJ!`k>0AnPkM;bCmwK0CeG6LMKXu8xh$+c6L7Lb*Icnfb739_5#7OYv--XB3*WK!qpNfsJF0MZ zyASTgI^VH9DCSN{=7}lf=6ywc1j3D>x*h)bG)I#K1*5zBYUzK>NAk}W;>%st?ekrGQwgar28V}}mmi%)PPf^gwqG%I{q97BrwtPFLKgPT`%z_da9~N(*SW&_@|hz){Dp`%Vk+~liS<&J8VzGY zBZ`hYH*3US*q(*0jeJ_v~bZ zWKdTr=tMp4*F~ulR!8jj5e(0HnJgI?|Hjho;H?B?iNW~{N_N=RpvP(f6)|)AfW+v% z?Xz{c$nEIiBf#Bw`v^|70Ecz8_bT^xo_>APrn+&tlY0v$&m^mR&y#H}hzeH|6B#yX z#3Py*TK^(3=?v?gT|@2ehYGR_ z{3yjS96ez)neGb72FYL}o135Fs?=OeB<2ZxTh~Weo7dVPX`$e+Su#A^kOsI3Omg>qosFwltPuz;qocx;1QG3zLxZ*Sg*Rbt^cF*=I8bE&?|WdlJ_R$^x?B#atmL5pUh)s9^OO96z<>*rxeLv z=WP!Ao)C0Q!%b-qRd^6hTAZZ%#rqgpiH!hZsAK{T6uWiDpjEW_ez0fMOyS8lGxAB&S-K6vTD&3`xe-Dm z4x=%4Khlak&b)FIk;4s&RWoqS7a2Fs>9fQl-Il zDA3e$G8ZXfc>}}CjuS2sggLVb= zcQBVPI7It@wpi@>u_i;vb9^1y3h+{{D}`SQJv~tLaRJJeu)71#gn&Y&x{QEMQ*Kax zh+fRZp_F(-9`7bBbx+AqWq95lE@bOj>zH{I7x~j{Fd-zHKaFxF@uzA=>(-apFqqR1 z9a-thKxRlN!1Y`^T&C zdY>-40i|rsPu9#syU(q7jBl?#1hL5m)hs$ZhekYWc*E|aoeCp}Ai?4Jh!l#G{zgy$ z2^C!|LHHZI_?^)B6Dc#rG5mtYUvu~7$`z3!LxWtar(dD*?A|C}8s~9|!}R}j%~+vL zP|SU)9y96=%?P&gNdj=~D#NZgl(aveSgQ`aF6|9pRn*7*UA#B>Ch zeoDI#-$>kfio~-<=hEK}ie@QjrT6wpj{SaR5IjL+Zf&aVQOyOn zyp`T>Mt&a&8*V(JSY5oNtKsoR0|&OjjaUP(VT9b`Kcd3zX?Q?LwNK3G7zBs4Ot=Z@ zk3;3p16dw~v>>FZK5ndDC8e--?AUpE+dSx8J-o?)&Xrck5F_BX_OF3$L?g@RQILy3 zphBI7Oo{$*NBN&TwlB#W$YZJhCy#wriSTpWvEzvz8Wjg2&nBin5U=tO#^+bQ3)4iP zdVv9g99&21V6C`*D3f0zdP-m}me@e9@yM@5wu=wR0&riZQRmfJi9iNn%TyJ8Q~No7 zrJwTS2DQ@=9FY-CB)gCP-Ic1?kDu1MdOJx?u!sk>Y{*}=Y;$EWtho@*mj;C;-JtTw zOx;S5SjD6JCdcmVVx;EE!!>s$h#rw`Zml+TS1Q6;=Fs+96C8ok;T;rgeDr;Z8oQF% zR+{hOM0b5=OYQ#Q68UND{nyjq+M%BLINVK3&pv<`teO~4i&oPB^CCDTzGO4f-Ebb7pZ&5*Es$ysifiJE= zeD?+Lb8C-WPq!w|AvuPvPsQSj+u8r_MMT%Zn^8|&<7WFl*xD~TE6R`StNZ*8>Al>` zZGFtxl`y6X(bbG_AmCSk6eVEWKdGt67f2gyPK{FarD8_#4ZaFvSS~Ive%KXG$FUMz z<94kc1e9J1th~QTK76?^1VauO&>qhfLISE-=HhL1GcMSgnXS_i2R4w<7EV4w1-bA_6&*l)hR?B*)Wq~QqiDntm*}PPYxo1NP@y zHePt+;A6)1M#GS}*)UA){aP^@ycv=bQoO2Y-&iXQ>Oteeua8!0gmkBHDw9`bM?-6&c?i7*^+?`Vk zFW8ft7Mon@XWc>itI5%KtkAjqeR~eEXGGHZnOKhvlz7hu3@NAJ8m^zqA2w;yz!_0E zLG9jGEZ*HHTY`;g=RfsS4}fVi)jA=mu8=t4AQ||gj%p*(@Ku?yw3!uUN0NS%B0A&O zR-eT2HST=B!u!RRjG4)!uOYn64sGtuBuzqfNti-$`^LsF9(d2$jRVqgSq9c4Jnu`U z8+Gc}<3X#lA%5IY&fx+qvEx2drlORJuv{$3zCHuh3s#0L7VwGXu6%`94up~#y;xy9 zx&kRtVuH}7 z;NIUQozK#SWtP_P#CP2^ERg(ZhIqqlKJOQMXcrJ@rEH$Z=mZ?bpue0<)xGS2ct|g?i+@YPZUQ?xz zX01!t6F$!Hh&Cmrz6{U25o&cNSQaZ8DDDn~|4{NG~70emYB6K-~S8%4xs?qBZmEG zcd2>FBpetFW&-FHtwP2lK;#Mso|tdQo88aofd_a%d^}M!gBm7aS4qgpg?FbI0aI@r ze0-!r)ob6|k1--HZ+U!n_jP6}0)8jd^TkAdYi$Hj1OZ)>hXrhJp9 z8!%c<1Av<8DyzgDi_1&Ue+e8+qaX~Z2*4m#rSQX`ljA8Vsog36Vx1E@xCjYZ*(c(5sG z!)yKSrxuLhQ4A&v5R&j%cK|kx?a`_v=cjSh55lltup>acG5VAJe!AL*f?2mA7yx;e z7hIOOMDB(J4k($JFhC}t)v2o(yaf2Mt`#&3D-^iNhVx*8`I_mwcesDacuk3=bb=Ur)-gN4I6Oj9v`fW5HWt}@QE#aO_^ zz$`msR)&g_VcjGq7SP?*X4lt0MJ8vq=O5flc8tqqR1>HiJ03otYIty53%5I&xsg1`d$;Ww+hUmQON`u*BW6E~qn*L?mI zp9ZwP=qH`kC{ag4bsXf$pM(9s6@PyDpF;k*H;E%%AZLhJI55+_2-hiWIo6JeJu!&m z&_bI<{t&35J%$t0rHpT~aWB%#7w9{1Y?_1kip9n_*I7m~Sx&+%7`|Z18v(lHfg8bD z^u#8C+lm+%6w3qSVK~vOcQFHVNnUp_2;8zQv;xepH89}q07`^SLh=pFvNb@2V-wjo zyXKd^o%lT(xmx9=Wo^y;Q0xFOV8ZZYe_t?EYnMlBaK6AUdNqi`TNFSF?d|OkVUW}{ zKNNC#p>AN(u263BzPGlfhPQVZGMw5Sj6*X5U~HUbBXA&qekgmoDgJo*{<#QbeI2mg z;6V^beMZI@@L|@e3sB0DgNH+YLgWiTVGoJMbt6Wt#A=TFqp>l(ybH#Hh911fl)|8c zZTCihTx4=GlUsOX zMZmP1Dons2?!+pZz67sI`E;hSP8)q=ogrJ$i-Ot1D1WX_o{zE|W17*;2VFeuru4@g z!zU!sOdN%rPfs|B5AM!47>XPGft&e>R}gS<9-K=8(-hseuJBLO{hh}X3SrX6j~|KdfWW@ZkYSDsAMp$#q|#%)XQbR zMp1KBxe+G%cfNfTrbg3FL9O#7i+zxJ;^3nI`4o~IQY<9Ak<2-7p-p~YSo55M z0h?@d4M6uu1zoX$+4Vz0fQNhb>>0B#&D1;qbn<#$maNY&HhK(p&&fF8Fn=}VJ)y=gs82GsnNB+jg(o%up+0~%!D{kgWS=v& z^>4IBwjEtJ6HE_5p|u7Oj5V%D)F#6zd*?M0L-afJMHyh+KpJbOznl9jB>kAGhpe-& zqVEQv2P7cURCHpklr8gU1&Tc`R_vKSD%N4esl;d5QZUc?_Unpfb4HxSPY#cA#Ax`D zJJ*&u{XBE9b-u34XhQa@^U`W`Kf`|d^fN#;u7D8}2L*+pp<#&wL=LV6@aaZ@N4wz7 zVf(`aeR$ZGOHB(mf)qYCcAiI_gu!3Gih)fV$krpj-ad7x907~Lpi%a(Uuhuxz<;&^ zjv##eK3}Ef40|Xs7n+@&T|BQHb<^c4dNiZPSVPxDftnE@#^$P*Qg?-ueFH_n8d!0+ zCJPhuAkj-1D-Ln7J#1NW7Xf ze>&ZJ#?r8e@aTj$%1Dg0c;>j_c4}}Dx-EiMCC89!4=<~g16-8b43_-&pr9}f{WOse zPkXfoB`uwADg^Tr4#DWU3e3P6FKz5cz?2;mETKQVZYrmA+rd2^f+ZdS&U&%oD==oW2tD*{i zN2)?-ANntHbGJ$vf0vU=k4dqIYkO>PznhSXU1z9?p8d-He&hAa+C}!YK2Lom%pOav zzCsmqXISkqg$Ma8cV2%zU@#udH84 z8fo0W6HFbu?n?VvPMhPvk@?+=MJw9KmD=d&vxWX}OepKG#oidNfhjTk&;JC+#@_gj zTy+q7Omc>Z&)t1mNip}Qbm}PY&#+3uD3r-iKKNP+o0wX|f(MfC0wa2C^ytiKlnPCHX%VnEYqNQSc+4s_ z0g@TvVQXYvw2gS{kAyw~Fr}wkqllp5-kK;N97+|5xH^LhR65OqP18{uI*UG_-97{p z0gG-ZsN*vL#CN*F{D<`9R?kB*=(M8@m+j$G34pG^>=$fOI>48*g3?^@dbj)`!~o0k z8wfyyv0rcVrwGg{G?`QjH3LCmvj&*4{&G2>low6pwqga?Q9TwcINOT_m$ji}0r30+ z>^sSdagf-}of;suik%Y;&YVq+ieq--u*W)32z7cegmKK2aD8elYECRL@pd8ntpt|D zduu@yo8l!iywSRpW1kORWUA(z#5o70px;qKLYWkm+J`ALxF^S3ASyUaB(?2yw9u;& zkNWsKMsreIf(*~YF=mtQyU1tU8zh3RYLCZ@iSwqrNVL!hKtE#Aa5?ACjC$h|5leGA z7m=BaU96V9w1C?Ms9v-WN|%rKW33!V$;U!I&%Oi)`*)k_k+)nWzbY~HrqE*v3_gan zyCh0p^hO2uh=YMNxhw0t8jfc(@@y2OrdC}3%CgO5&iRwN@qiF=aszrYDP9-(JJrlB zJ$+~boWJ#eLeGG72=tuOG|$$34}ODTy(A@pS0h*&NL3Kdo7y~-^QC@n80LQ6pudGx zQyun8L=eS#;Pv184i+QCK_U&4`rPwoqp&nMxTo3e+FK_DcQ^eBqTtfe|N7S1|FzPU zbrk;X70Z27dyQ0B-MkEM{x#-; zhElP>;%k(v#(e*Dk^jl+pCbS50#;rZ!~u<~yJ(yFnGc_y^}HKdqL`FVy1wRT{HHf3 z_*XLYyqd4=##~OwFIts%!ZU-HryNw9|BQQ%byM>FpJasf-vy^0bu&F_Y^8Yp%w|}P zlwp-9`jPkjDmek}-{C_1_q*#HO_+6nD}*29TTr9cfq#+KL2>o}wUMV8{ksj}4QNT9 zmVufbY7(!Wfrksk$mje*<$t@j9MH1E$TSOM4o?a3lhZmh{@i(Y5i9+ArSz1kk<#P$ zywtuaERw9UdfV#HK!V%&Fw>mgd9=p~+@v1dSMK|%vaht}#p7x#4YX6;S4mDbFxYzd z@9x;p=lnYsz{wP%!TL*)^ zN|7=|P|sN=biACK?nv-x=Ujp0309TLl=Spjz8E>@NbGN4XAGmHeHK)Itzy68>+5<` z;t#K#C1@Yje;BR3Kcn~AR5RNf-*7tD!-c4>z4OS?{I-8Dz%tvYkCvLO<2kS>LU}%E zaKD5>cuhB8>T}x{@r+AU;uG=Yy+Eg&9ym8W2K}EGh{lnLIg;O`rr_0^ofsH}f20t8 z9F`IBwfL>se=&SuRt|c{-&nbR6fkWa{&Wsgb4M(wkgN2IorP$;;Ttb~;vI?xVG<|n zZvuVG?~LdJ75i3xNa`u%g*BdlNyi!NFh5n6e0onHoD!4Oxp@FxZHIs8&WrWLaW1@m z^@wG>9uAIHgL-(Q_ABk-!q6xc9Rn`Og&89E7J@(Jp|b5}6LGHVDtp^lxfMA_Ao~!C z%uwa2u18L{GkbtN_Xx-BQO6PubhslnqJ}_r2E9)ETNY_yJ4}CYYvqtJQ=3zfaM@5wVuABGc}rFXMgvspS5>K{ks4bYHWIJTBHceOsPUs$LcF4yohHyHP?{rAoJKdaK@?}p<7!k?mPGa}oN6?afi-m;I&+?K>d+QdY$#V9k zUc4sZ9Tnz~2|Do3LjrH9R{Nfc5iWaHj<(cCX{Y=M_V1So6Eyx$3(&c%cnSOD7I$4w z@{r_+!lb$}8abOq&}D|DVrb$%`ZL4(=bnc@d6ycAulY@`O6{go39IsA$NkYO#P+tz zHzJ5KU;P29rc+-!AeYGbZSxQL`*?+ta;dn<;`!mlD>5OQ+Or%ThQO1-OC8YC8cE~r z8o`{EPsG)2f2$)@<@J=VCqOwnk-|b>=Y`29QxCb?=6ON-E9DIINi4x z74Kw=wwl^GO-cE}h2=Ii-atMLR}xA=gGp^6q@S&(VD^a zBbv>cJrG2jein!R%$g|@YpGOE;p~of%G?@NHLTOY~Fy}7$K21ugcl( z2?glEw+|}*oWBYhV=|IIk+syqiP3yYsuPLffd8q7Y@MO%I{9VtaWnPDSvpZ{qi1Yb(7U_-Mee}!yM^leCA?y5 z^esyd1zT!gduCA%ByHjxL}sTo0W_OvM5az09gJ*k(mye0a~KQSzuy_D02NH6g=3Sh zFW1|ok9JD-I2kaJ%P1qPO144mkwr7iPZC6fHW|;m(TotPZ0cn9EI+@ReJQ7k-I zF)nVdW5chaReoY9A<&Jq*WhC5reqB75BnSfYWX$eZ%y>G1s0iXOO~qkjAjE`R`g++ z=e2+$fP~(E@%C=~{xZwa^5%qoNN-poc*w?Q>09uP@9Bx;^4XM`|H%?Bba*kU`%$K0 zuu*ktayJ^^Q(C?yUF>8_8y?R{lBEPDjAV-DvoFY%^wDYFz8$laret+^iHX2zi z!ddUPMz`6@`(eMdiL!3KsOnV4U9lm3<5AfU;@MY`s}&C4uIo|Of16*Lq>~7>dOjtc zeg%#fY2*K@KqraRY)p*$o1oJ!I|OiP=3!A=vl)zi-2t_NF9Ap=B{!d)cyB?Vul@vu z;aWwfFFmHb}KZ$Pj7P$ra#!rI_;WC1}Nry z+AI&9T3x@a8uFYeRlF5)(3ovY-^cW%+#&38MIO!h=10h`HcfssrgSxdu|l(EbpOdy zg`-U%e?iuH7hUjM_F`v@{`AbmPTa%yHD`5=y z>ai+1De6HNpk1?|j4jVt=tT(X&I<>$D!2Tw-?A&x5=Rb8u!DCZr#!fPCl0j z3#ZtM+l_A{U+T2@`y*|m()y#)zb|dh$Q5CzBK$vey=7EXZP+$!fJjJ4Nh1hIcbB4o zGzde(NVjyyD2PZ42ty;?-7$=GcXyY>NDVQ-yj$<*ectcS_h;6cwf5Tk+ULH`K9A#~ z`uR-&Q-i3Gp`e8=>*YJBu?)-Emlvt? zDUe2P=TGYBR}jV*RS14!$Ln$^~XXiAhvng=5pnY_VqmIH|r zA;xP$MV33MpHrtgpi3(-B{w9lR~V8mqd{`Cr~6q4QTQgsCzt^$K~IL!Mt4KB|FBJZ z)GMIBeDKa$p6IPhE5XG<;+Tt*PkV@gbdZ@`Na|RVnsAl2Ry8>TbRFb}i5X{WUk({8 z&{sw$uIWB9&2$u{>^WnCWTSk+z!|4K9)IOU_#I;ttStF_fVff#1lm?zP~MN4xmcXPOoBp3-uY9SGw;?sl(G?Mhp9k5`Wx@Ss3_!cn{EK>qlZV-=ubOuzc>H zFMe)m^Vl5o0kk|YQV`B^aKcVWq@Z-*Qd?OE5Y8u7yFCWkbj3ebZz7Pa^DH&)p71d_EuJ;`41lH*%~t$6z4QZi13G`Q6SKNhI}mf$S%_2_ z=`8z6PdwvUBLDVnMIMa5Y08)@-bkPH!F*Au3Mz{-u4 zJ%yGZn{Cq*15~aIi$e0($PE8K!XZ!Nj$P0_Zl1vo>3`6zh)|@!awKQ z-Hyrq8mp~iykbv ze?6PzC{yu*!kUMjo=b+yVLMiPEep-hpuaaxOb30w)cOTaiFQ_-*%K&6K5{7cP9e|G zRrb*4Vq;6W6P2l_Vjj=wM>|6hd#U2CZQGuE?}cj3GEA=4OV3))-;@>KGITsr-WdH{ zjd#Q!T=xVWO9Hmf+og%~e8#$EK@@IoF)9Px#C zx>JrcJ92n<`R@6@cKab)pJh4AY0z-+2(zWdp*%i{0HeH*4ijh zePOQ{w8V+vo(;u4bMsAgexw4hAk_>m;=R$$CrwLdQh*()+B*D*-N6_V+t_@nm8=yI zxtx|)0lAHjFGN0GZS(%fzcr&ioN8Zft9Z|#0TJb`>~rcE z;+LnYJF+U?I?=b{4}u!j;Q%V4^Y9MGD1X@Mcy6S}%XjTL@`X{31I*eOR*(NsOpBgv z(NE8%VO#{7us6*t#`i@b`(q9WcEXhKhF+7z)hl=8v-&rxtXA46cLA$*r0S8db*hrn()z{2A##2j&t)Xbd>O~qO zD;05nPW_$RpC|p0hnFFk;)%_EJe~v}kDjjR9lQrUdVe;a1uXrc1~}qBtz8U#XlJ=N znL$EBzu{S2_Y_le{TgDn!_R;q#lc*DKWZ0xG_$oum%aT%-+a8c&Pset=*(L0$EEPF}2%gg2q4t(;YJE8@%-XL$+?3->p*%CTmY~{vd#88y zV%HJ~mHm#+dWaQbnBTAUQYx$e>q%S5atnX1co9xE78gC&^VAHXs1JAL7p`YcY#X#j zubnE~@e{abhf=p5?(!G#!j5b@PcWNVFJHcVc~fOC?YU&Mvj1DK@b^=|c$5kG6&5MN zehkb&($}Z1LT0__)9i|&Yr(Cx&phlbo@NCuVO4A>K5e>&T`>}q*;`(AuDtXdaZ8T6 zOP*T5EpJv$gTKAdNg!Hqq$GVb{9$qvWWKX~|Ib|(=iBo%#Y2|lhs_?k zFC94@4HsxUy`?rn9)Tz?vuV)eWXE#N_`;IRu|lg3R%_XI?5 z|DrkCo?Y5`Aub#}2Aq|#{wt{ynIhL?MOJZ;@rE{<&YDCP1X4g@;o#t4Ygk^_~A1Pqg!%iA02X}Hw{0COZO;rAR zU<|Vdy1OTihJmk6R*0_vKJVbNf)Sn7XqO>3SwgJz*8H0sbx&Q^ zuXb?A>1BiJmk=z?S~(8kMg8c*l<4Zc48|8MVDn94VIoS(x77)ty~WxXBWk;n){D9q zFK6G{8@_Pghvwf#o)@U|QpPD{f)pLi8}*rhd3rv@LT2RnmQUNyxqUxei24I5UwLEx_L=jsh6I0^FfRNyrNuD(@KKQ` zyl%40psmi$1i&m>R)_BEPp_{$M2xM@v(2v9IV^d7k!)7wIjUX$5sJf-=|>VR^T#okB{thV)auk1&gW+hbLKLA*~Ju3uwY zKVa-gacH&WLQvOKj##& z7dP#+?nR07-A5|OgT~I8k2P|3_HjESZhnL1q1WAwd&ZlOTkHw7k))KA%8g+f9^Q{) zl47!xtm2D;y&qv=0e=o-4dRQO1swI_=%5a!n~CDOcel**p7yVT!+3AbcW7OzTniKa z{8Tz1R@h&f_9(5>S-#SV`jm#G@@<^vkd@V zja|@|=|g1!wPdVv>HIVdmrW;OiX;=p1Lctn2FEUGNH_ z3U<}Smh?jCS{L37>=*xmTZGmiw9;lTol;a@Q_E`DOIu;amGX_WI!q;(w@SVN43DBX zIbW5i6GOagv7}y9p_`KsXG&Vn>W%E79fz;-N7~6eML{!E?%Ecyn9t z(r3!}?y5q%mQBBW13giX2{;_2$X6CuEyF9{{_n$25-UFN1NVFMD^Qx)X67dYh`8Tt zbEWRSFWOuQJGR9>3M(NtsWm8#fX#YpHCFcIfpXqH-W3BfT0~I`{ z0zOP&jtY^|2pj#*`+5mc;0^@emEQm0#uT$UnC? z4$3H}Co5&ta$B}z_}QcpHt1^iRx3Pg-i5j1vxa=4k!bz4hI;X1xCN>avgz(|CzqkX zNkmW+Fb&9}J$|UwY-g_KLO_r$!QIhNvA1`4OPOtR%lq;}cjW9;h3yNg_W;6axCyS| z^u2v3(v+2s1~1H$V9&6p&e&>DFMe~m4g(w4dC~VExrmoq#Kxa!=g*$o^0f!!0)9H< z;d>#g2|nwYpQbSpDQPJ^ZE-eux2xp)4^RA}(qO+@b537nj*^lYCYoJ6+ZQzXoof#d z&LZxE_V)K>f81)ftMBDOd^D0s%3kkZ(e$Y+9{Q#L2XKNoPl~!uwj6z*jW=fY+|}0> z46kCdizc~*hE)>z!H)?2#V0Fixc5f-ZtI0sk>t>mnPK9ZLhxsUWKgc_sV!H=A$ugd z?Cv~749eFWr4Qy(Cm~^i|L&QuxTEDHtW0gKU&^1qB8@2B!o9lCC<>DW(cG1b`A?OT z$OP}whm=a)5=0q6_cG4M4_&@p_c`=NGP?aCbx}7YtVc_5E51`QS&_@bX>z zRBh4zib_wY+P9pUPnLC;3-!!IG&FLJI2Ixqv2D5Gb8p{%{rU5I!v&rXx#+MG&u_Tk zwnDS|hrQ)|(hcxkl!F05wUy^n&vR?+g2^Jy^qLyH*r>E@MUY+mRIZYqxnVyu*Yk1K zJQ-D0F7k2y0?l%UY+86wO!QJWjc{vQSgye{o?V|5qoMgN1u4&%ADc=5IQgE_2Q&q+ z|3Jk(YhIZpK!qW6*5VAQ{ZW+`$hZoOgg@(>ty3VaeXo~KDA=Kgmc0da_|t;6T`{qK zp{IQmfT_*Qa(T&%S*=$YY8vHhpR_w#_$V{;HOG}DO9&FEd<@S=iSBQgJ21*X=^xAN z<*061cR#V+U#|GUb2F%YK+w#sd1S1I>m$&++_Gl6*Q^MKL zG=)>1yEwPEaY!BC8S7;7-12%wJ-V?N$ya1&HjR)RibXdMn4w2M@O#!3wf%MS=+Y;b z?)$uXlFNM_Eia{j;^0BwPL5Gf*b9HrAAduMj@F45K7Eue8*Bd!lOI%Sg^ zoAqcjIOi}5&R`0@*lH_4o#O0NPxsAfV!7E>vh*(Zd48fwODc-bv8P4W(l(FN>H_rv z^Bw5NNIOY#j~{ujazPj}wxoN?p1}hfP=?AGY7twUO(W#50vfg^GH^X|Ucq z$^5?I;jJLyfVJ^p4%LjNR|9oZz;Q{I{V2FAP|tvCvz(yN$W7n*P#ZB};q(Iq%ZzA$ z#Go3)Vy(sfo>mF9N;O%nm-}M<;%p%iZ{>7|no2S#-#c8tMFh$I{{6Uu(!gqCz9qlQ zEK6BZ0oisyWpa8wTxGpAu7E7Q(?4ItpDi9>8k*xEB>_lY+GDKk#$#+2s^i4ovES#8Eb6H||D_kwnq_!0OcTGsfQ`); z=fyBJ?Kx)C_#0qU10u!w*7}u;J(ocZ4c;8rhppER1mMznB?Pk8VF2GpaRuL9w+cxQ z|0pT9a83Fx6c01Z5%H`x69&Vmmh*-)UplM!gHlXh=Z?_F>?a21%aZ9nVJkb=dyz2qqz} z*kMm0lSMn&XuGqK*9AxezZE}AZ>?)rlAwlp&x`Gi&k!@y)zdAuA8U6+4nUH4zsNQd z{eAz^>4t3@?f#|D^=P{z$=_=$iPWt`IwrBf>0>L5OAYQ4XYx_EG{=Wx`bf!Q?yGCbtY4J0M_RsJIL1d?YNhYPRlPcV z1Jg9?U+vgkjnXhIOBcJ2vUm2}{Tf20R|y;K|~rDy29UNmr*_#+Rm?KvMO}EhoCPwebJW0_^bm4?XX-UtdnyKkl5A zzPu?ME?`qeIfn$PpuUnO-P{U2TK0RA*=CuCZkX^GoDgLzyMcv{Woj^y$gY(-3iCl0 zX^eXi^G(c*%%u#>e3K?~?OBNrvqc|WWW)_`khyBcu|+f$V{c`5T$#x~CFgf-DI&o$ zS8OU8Acll@1!ldhLz7?5LP)i~?uEWTVuRuM;j+2ocJligpaSCha^|=$!y*O7>I;MS zRvf#t^q)B?J%pZ~z}Wf&vuwH3Y+8DtW`a{cz2AtQjF|LKvG>%%l4Aq-BALv{T>eDA zz1OexEem1wuwW{atZ~l!K({6mkzuH8TnFL^5OSs_Iwr!9|MJHjEh=!}%^B$W z_XPQTje9SKyldcCt-c=8u>>!O4cHT|mngrdCJttWz&AGGEGIh^5fjw*R&&HkC9Y1I z4o5eR!sAi-?;B-7WRa;ODrSzEi-KE;bMBNZb6!=zd8y3tT?#+RA3)fFzjU4tM3ic^PABTKmjFnZjGkchf2hddO01U=-!=f@SMf zG1Q$JqZB_T{oZ5Fc)ZEz&G?VzicLD50~d9Hlh5tAzM67e@p7DbgBRkBqc9?t{IlA~ z?3Xx-a`>cj0rE)*Pp3*>Ggbe2y7o5?&dN!RKd7z*W0dNh3*|M3{d~A>y}ljF81(fr zn{4X5g?jv1v_;v0}Y>djyH}1#arr-$LnHY)l``;c{50$bqLTA_b9~(4< z4$^XpqUYXBUn-j1?5<5bHmGV_Ylw{CFbz=~xo~{lZ$*&x6^lGflwz(EKSTC%T&3=K z>JOp8d9qo+kf`{%lz|^Aml5PPZ@WfiJAD^eX9N$5R9zSbzT`tx;OXsQ&uxq3N74ad zye{)9az$JR!}&)-6iKfmQbwbuh(qWN{cjo%MR2tbH-dPDp>Aw}^dv;w%HTKal_ag+ zxdsaPE_2WBkOan@M!KK!m7dMV#pR!%qDTT3V+XqiCDyY8_vV_Y9e&F;8~q+mC(1Z7 z^+(hiM?*w+FSes=O2MC(dfw~aT-;~tlH;?Pj9gr-!MO*w7wyj04A@b&eMCbATF;IV z9zWc&04&H*J9+c`x`g;Cw-mTBJk@aDeQL5tYz+OiDFSWz5Xdth(VxF3i!o`#NK!t7 zS8{3g=t5glu1-~#4H|+w@+WKz8KW*}kKTpI#oJe6sp%>N6;-x4sj=S!8bGg?q_$^M zGqpma-BNO#qzXG+v1uvt9%Wt3f&LfmX^ZnMMGK;jHb(jiPoZ8Pr&n%Z`!9ft-WQva zpoJ?UgXqcMQ4fW!WlpJh*?mWiSl!1ojVUL|M<*tQO8AG*X~%%no3lbG96@!M{oF{k%fl*O;iH|DMulxgiLbg}Y+!^vPSD$UMa0-GO0C+7#kW zwKT=n61=}uH+9Wzg2a(vVsdNACP4>UhYprV z>B_5O-Di#7Bvzh41-TqcwB|SA9C@{Zn&J$pBhJxhMEVkN2E=iRmnCL?eMZ(Pl~KWR z<|DfLo5!EH92Zru``i|&&ywc2A^6%F`+4(tdPsVN(JB5R-Tu4N*PNFHR~ScLr(CfF zX=YOk@bQmYzGcnjOsA`$FUURnub8)^-v8 z8;!U)!7=2`$D2VLh)i~yYt56gU}me+AX@9 zr%H29_$;l8?Xz7!;ErVZ+tPTJm?v8Ok|J12ZX!MSEeZ#-m@n{ByOR)dTRkI;0ka$& zVNVX{n?yZONL zU4O62|L{k$FE|h4*c#J1cjV1qK~xrDaiYzul{d$noljDta_yffU@;T&d(a*~l-@e< zjNaY>C$1bnbR-r>iuXTBZQDc0eVY4XdY@^FA>s1f!A!Y_wfNywZY*>K`B9pq2f$&k z0N`&aad9Xk*6`_3IQ(>!xRf{PK$<*?SJDv|@Hg#K;T!H5Rv0OyV#1NxZ-Sl$EGG|g zFT^HY**#@PtK@kuxELCgD)U_s z*t#MqNo~1*kj1g~d@dUCG3sV$B~H|hJo{4|EA*M4l7Gfqu3skU2hPG+;)@dWF2X$@ zZbSqgO&NX?v# zcRQ0&nyK>vVG)~nst(e!x!b|qj5Uy|hqy=RsLAYUrVr@!Ghqi>8>PeH+PQkaj0Mrg zyTW}v5wGVq@)P%Tg`H_i>1+;H!8|&Bp{(|)bz^NS*5JdyVWA8BGo87aq^G(whOi%u1>Ll)&eVt)f88CVZD0FQlIhD#dRO|IZZ}G zDA3oHogdTE(wbJj4%(M@&p9fcsj%wex@TJ464UaaN?w>B}Pz;vyuK;@!1J}zr) zf7=(ZO9nSmLR!--t9^`*HcxvlsD>*nJ=usMB8sB(CiOISpTUuZ(&Y$SfO z51FZ**>+=fI5rGY#8U(P%0aZ5v>o%A`z}p(mW$`Nk7qaFSqK2C5!(!*OE)K@$ zN_Hz;jITMw3^O4p7v-xfjK=4$zZknoNbfLV(nL{k$y$$<{yuTj>og4Ue+U{rl2kD) z^}1(j*XeRPJFI2@KD44L^KB<|ZMUhT>^|wc`P9vE8coK8l=S_#UET4IIjQ8#WiaGt zXjrK^(N=LOF+f58?!c96>jS<}cGgppr*G{`b$vz-g1SCB!m2<6Pr=;j{vxYY%oLuG#L!~Wo+bXHwpfGja7DK%>!-ittFetzw(UR6h7nC# zgTu4Y)Xc~A{s$-v?#3k1k6kQaS$6QvCZ^uMDHyd+*haySNmwbs&Mb$8W5kq4%Uy3o6BZm z1xGvoQA@hwsqC;)ubhrEpTDV01D_0re4~-#m%ejPMcJCS!ozI8y7GrSm0rgM9w7wm zd75Fw;vm*HD&CNy+8!Xe;Qx{IW|mC*(pkW`eYCoWYo(|-Vj()R`7`f^*zA_}<=mU< z!=Yz?vmz(=_2^klb)m(WqryUtMNuvUm2X5NHaW1iIr7H{%g#ZUb4H9Ug|_(n`|C} zM4i}zWcO8pK6vC1)F7WMI1!&t9B<4D1LhTx`K{y&itCRRMZ|6FQ2nKk$RoP}3^^ zWmc_6*bmdGzHWQ_?#n(dm5=XhJu`g~*r@<%3dn`9sHm>A|Jc1pQVhW_KBAtfhlzz| zb0~K5TXJ4R9xvtpaNR-I+#&0e8xp~`w1>S_PuB~-5KlfM&|rw5`;CzxDowJ4@T}V6u#G zctX;aC(llht>^3CMB4aTEQSf4`kdbY-S!jsp5>Q})8v=rvUpah!K^48j?y9y-x$RK z?c%Tj)5`Bq=+5}?z<}%S%+ppJB#P~_^``^3%hGaQ-npS$=lx&m)T3xr&iVU1lo{qR z<+3}cp!g*73%)Nv7V@g`nsl~x6AUmoypv8+9Y8I7z~|L03-JE!R%CVQIWC2MfF^sR zZ-UQD43_>pUq3l^KD9A|zH4)ET#>u#CVd==m${Vafp>LvUnctvZcqQLH+{iR%9+>y5C|4PHph`*QDis{&p8dn&+Dt<7KNm)F)TyRqS(w>?v z$L-+IAz}e@#n(1E4c1&2z)^~;iR$Mt$S5*O$^{+fy1!Y449eUr5fr~f{yyoELt8Nc z5yN90ZY060+fWn4N`ntYS*i9{chq~mraQa=;WUf$VUni0cdL2bm5*XDUN?vxBc50H znY$1!!DwO6ikc!4s5vhE_Y4ayAd3Z61KxcyjtfX{@o!+Ft;B|#XKF~tK8I2}k+j@f z|5E>Q?g8>xxDHgWXFy~d*|lITnDbbI9c5Bs(x>(o6g+3Av$Y0N!x%D6 zZQl0BTcYCZ`6GYfFWiW`K=grs`nQJ1GFdVdJUdbgQu=g)-qAHV0^0<4$wzTM&bQe# zA}!nW;k3rTR&xn>oZ3Fzr8jhQ;ogocAg>l4N8(ug621*BBkI^Y%ukwduM?E-Vp}cn zCKK{&`pk=>?Vz?#3qi>HjSM?_R=M7(%4Ewl`YtbZZlWOGJ5cjn%S_X?<4zW!4$1O@ z?TeHv%CxU?`{cMb?r#D1y=~OEwqQzWC*$IpSEuBsYU~{`K*r1?rCiBoh-U zD^us4WA!{qZz4+k=FqN3e$`Y2Xo;|@_-q9!w0S2oVnnoPVhpEC<1;4W( zfz~M1^*pzC!NhcBo1HJaz9~tAwv$b`#!n`fE2;_N&x5VWgC%5}k+tYp3i%j_f9o)*yCVK7c?{n}~9>UXLre9@iz zC1k}|u33yopirZ`Tm)0efV37;CcAZU$jY(emVle&-6@ah$|V_c4x70aOs9XG8-c1RS*ee>M*G)GRx~d^F~t-YZuMM5pLx|Se09KHXrVO~b$)QTfax1R5vsQw z_Q6({nvPSvU-Q;ryi0ZSx7P|DW%=ye-7e)J-?;~8#f}FEm|)c2rKlA9OY=X%9U?W` z_w1OA-0io8!W2>x1Hk zZVzx$9!NIVJt&vO&|5!g%SPD|gU`2leYSYuv|rS50rP5~8}`?+J=UR=tp#R{P*_PM z9~~&=Lmu(kizTWWr}smP)h;EXfNRJ;*Xhh0DNXUDs04dbDVlWQhGi`|id)|A${It2 zIlPUuI7*9r8qG4X-}KIuQ6-Ahc?=0#WPM z?nen&zrcy8af$2FXHIQcO%)0uul$RGQxMs!ri&+UtHVja52U z->h0M_A;aAVI^BCDp)6)4iiIXUGV!@{tO8VD=J7`c25@0GM=dchB@=bhC6uj0O$$5 zG6nUig5QZVN5(06HzYy#P+0D{}x|cT(t*P{ZY28;hFx88h zm6cnZhu2bXKpLD^N@QQVd5=TPj6)5tUXK!sM0DQoF#GDuLvclM3p>8kJ>GP58XPHl z4$^pXQSLI-MHQnoB+c#bn?0p*hWW;`xX7ALlV^EfMVcTRHsdz{{ui{$KX|`;0`ApV zstGs6+wgGyX0+UWpN2@y-QEscri{EDfS(liZZdq|>z<_QA^yxU!o-5W;476vA#j(g zs?e?03;%WTb%^h9RLH`22!_JU{D9?u43pVqv#I1z*XV;#Jm*MPV2?T-gq6Zb8gVvu zY555qBV@^7W?l3p6o6HT`a3+^xAN`5=K)Q~AqtXxNmkDQCEfA$Mlmjj>mA+rk&Z-k za;%BRMP1kBeHP|PGulY^(+6)reCRAzT)|MgiWq7Oiw@Xd+{%eG)MoK zKHjXl-W*P+MjUf8n$N|%?7h_f;n}7GadrRgs<5o0qMTdI*9}boTBE22$LL;?;)^(l za#ffiXEBOM0o*c1{s9<**BQe#X$5`1Vk@!fI+j}<_kBG*YV2&>*Co;vQ`$FU%a#$< zNIcy_zWH|d@2oY($m0UZrmCu;1BD0{w(+KfYE#eWAJAZu!{1brzB-j{%dHjgKfdyF z4^RQJ1GHG+kVTR%cHbWA159F6IKQtiZ0oG%5MWZN;leXQ;czqH3JXjJe9Q4P%Ouit zrQ<3M2Mrmu;GInVc8%P#ErLehdg?P2WP>v!<6vK3CdJq=GxGnu6a5P(2weGbW^9mt zS*}>s{Z>b$ZXWJ=cz*OBjS*ZCwM%y8P59gO_Vhpc|4wNhYxtIF2UGZ9sIxN(%|d?)AYx3}Zywfv|D z=Z_rW1cdp+2e9&SUO$6C+F+F?n_F} z83dq8e?T=T7~MeY3=EdVUnb2Pd3|{6@2fQEEAIG_N0MyGt;73t8=CKhoKCHl$n*a- zwQTyiNpQTSSN`v&D8qZ8cTl7!m70XIyP{qo4$EWcSSk9>?&}BdY9A<#BRzKQ&+_`S zm!bN8d;^sclrN*9nywf}X&KTQOIDKsc(O z)aoD>dEv=A5sT!usVu4qH5yq2T%}qzR3IGaLd#2c6uqSDWTER}D>~KIcmlPU3^QtZ z0IffkB;AeZWD$tI;`*L+0<}CFR{N*lW)Ezmx&CPlM&_BuxUa+2KA42ys%u3W9Xs4+ zbg)j+_zT_tE0zpLNen%{3QoS*KOxDuXyPNWjp*b@=tv*?mvaTd+Wf-yxFCl>K46b> z^5$?MMN1hjcj}NjUJy<6I0638R@`%{SFZTD7w-P|X@4+@n*N1VgkMZ^b7aF_5h|8> zpxKQ$Cc|wib-9#sRp=ixtwStOP+C87#}+Be3h*C03vJnx8zcd1aYv0g| z>!ltrWADe!#f{0;LK^$yUB>^N1!$0nEQgnVxiC=rP=RIGhd5;YD`6cr2Dg7koBnav z@D$gNACJ(ldNW_#9&Q+NeI5t^4#Kp_l)X_hw&!?4A`gO=%e;@lwfoq~l3_}TqZn|f zS{wS7c76eaOHG!21UST#eN-aSQ8#7F3YKOgQTk)kZjjM3U0l(>Z}o@vf2B7dqkv1A+a^uu(m;8uloeJi(#>X*$wYYb@Oen= z88^+}ml%8hr4=N5#ZW>K{;{hPBU>15)lwXI;_(mXiAZ2Q^U2`m#t>d9Zt`3DA0@Q< zA3OD$kc&#EXjDV%?mY-}a<|<1vd{u2+_mNJJ0OdSfKA9r(sS~u8>At{p}AXJv2CUd z2BuukSeN}9T8#555-|k8n8Id$rI-KAUYq9s=f0om-7fd(9*Y_d-9g`Qa6~@PxzEH| z^9}^+LqYNV49iA#iVCtW*j-FYfn=wpEbzn!Sa1T-*+)9AkpvpI!8|VvVerN8_nR^Q zX2-oEZl?Yv*T9MmL&V9@c25C8K zrC+f%d3%18(t3SWL!hIftzy%4F}Y0jdeES4D98{mf#<(;o&WzWn?2X)Wh3Q{Qns${ zdypYCJd5nlgVzT{kT%_yjj14wR&2Z0$C}+B^*e3%t2zQ`{Q!v%<2THHDZ|y`)oq0? z=vcPsRXx|=t(g8VwEDYaoaR^~W%}Z`0byVSLTmuJiS)4BWHJ-p+=dc8RuV~g z(TM}@;sSpW%qVX2GgchaEx~O7gj|-)2FN;0#mSXFT26`XpGp6~RFFelVYMth;`+y0 z>&`P6(o2Ib5U(SJ1$4E7xf2q~{h?VVTekZ)$_|jjA^BQIeF>U{DQ#}g5tPYlc4rl`7*Y6o$5S^qS(&5#cOws zNN}F4LZ0v!dF`)7)?QSGb3<<~KX$fEo>=sS;T`k+iwO<=F@B8J9?TS_N|CFkSX$lp z|2%N~brD9N$8=>);~bgknD6^~FY{HdWJ12jwP+(g;wregIee{MNdnVtK3G^^_0|VT zi)%_LM2oGod*&-#*7__@y6&)G;NiU)WbPn+LP^O;oagg@X5|SJr<;nNm1VD>f)ont z!JX84s~BwzZlDwr{#w=iZIw&*@5)h3CT?vbPE(!ITJb(M;@8Fn%Zzw*vM+*QF@8iD zcrY!q!U3p+-vAK_-Nl!xFuvtWVC#^U!(dSP1X`Uq(zjgch72R%ESYsyZnOW|hVrkl zjV!WVc4ton98xld5E=(ui>ZIp(7ecvnNJfU|c0;>Hy94}8w%8KqD-f$B{nz(jfX4~%j@03Fh#|E0_k zO#>ePXL^C1eqZ_C(b!h2Voz-;r^h8(@jac_3YDQ+!vimD&rHVu2seIHo35R!y_F0D z0%2d1^eEVFryJ>^a$RDSX(mchOwu=dizPh2w7~{kCUIgwI~4xcZ5sOjIFnKVZh!Y4 za9~Nw_q`1MLnP8ahQ@qqVL-NK$ypJ+0OhajZ=K*2WgB90Im9R^t@7Dj`!avG={%F6 zcO`8YrR^ajh6bRBsLh&@z6`&2br!sz%9rD)u|$CzjZ|O%^Ot3+Z-F#pO))PgxGXE| z-=_zjtHXAm87O9c-ZEMMfZbqf{-P`wvh*5Et`Q7AG+Xak0_$WEay-o2d*O(!A`DJh z+|b4KALtIl0f>X20;$iJ*qof=s{b`JWtI<=ciB3L+xqBIvWQ1;zTj}KDcB{+sA2N0 zu=my!H?)M>xPTVDd4Vq(hk4;78R-z>hiCfNIJJ{T>I=l{-ANRME1|K9>X5jUk4SK`oVd$ARfggk!O8kNcU(&iv2_xXEo#_u*u30LIR zJUS^}8}z?ldPuWhtjAfI#$v}yL~nSpM_f&R%|hKY*!|O%_{iToD>ds+n}+jt{@h+y zm#8Vf0cF}m@}1dgM$K%~ov|1JxERKOBBOuo2>&u{K$MXfsiT;(dX>34pDkt(?jUIX zMzWFptL%9t&75(4q>Ng`Nz44eM4~otpAU1e+2sn-@W6$1qJ?zmtY?b&sHV6wtmtIm ze_Z#=CnTPr1mQb!zEj=SJ!;5-gj$(wE-eK&B^1Yk3x`g_+

VAYb6g+7 zpWxrLKOwNo8Ge1?YPP_Im2@|?_`2IH*oCtl*&h_aNJffa62uLEY#q68Z7Z`#n{SGL zpOifqK-pGhRURAt{97^mGX7d2)Togx9`2D}X?iL;&Uf-H5-$saI;knvA&ax`2UZsH zqek7L8H+RUUw7%RBNQb3AVcoOO-KsD)OO}HDF97ba8KLd`e>&*s)hwPa%bCS#V!#j zwVhk(N|=#{%x(d&==~Axe|AUg)_~=;!H{9_G0wu+niCA3HqOOM)4^YA+JX^TC-e9knPcw>@?;7ZI^-DS^Ir;v_g$q z+dFo#n=YN)=8@TGcaAuvhJlaZ7Cyyx#SrH(C24qGpu_~IG!f@W?iX)n_5QtH8RODm zQV}KHX|l)r@IkB>v$wa>#B;qnpZeN2OSz+~T{VUOXcvuw_sT(5v?4gJS zVO1zHuKwmjtnUh^do}{N?vb^7{L}al%Fq@Nes)qK*Z)YbA^)Ha;~I!>sSomR9v=LE5eKI_q(ai*(d9^YW` zu`)&B#cO8Ey+(m<^XL9^1B}SnU+J#2-Qge4SA5i%^6pTkSv`EW37^B^cM}lV_i5DF z+~>?VnB4v9&{KX3RK(prR4iMv0LO9lKVKu|Dy+)QY2R~I@Z8fp$6tOOTJ<%lDHwM# zQ^6Vp&czI+Xo=n4CK=c?N91aJVVa)wjxy;QAm-+Vfbx@wE6-DOB44IcmzkEf9?9J@ zSXh20LBwZkZ*1gAGirafQOT)h^PlKt4cDPvZC6M|cYXEg-Je!luf< zaa%w7Tz5b3=$hOe;8*xi@%=!vUGFw2^V!dPmL!ur%uUe!!m|hs5k?snaQ4(X1qws!4USR@SG@ujmKoa}-$kAG72zfBSnc1FE-(U678t;Q%$F+#m zKhIq{=TO+SbS8D39GC~i6?0hvky>R)I7U1QUyub0z&F?#DRroTS&7y&_ zym_zFbv8}to^Ux!xp1iEs8X`^@X+oQ@RS;7nRCB52!-#pVut1Mj01kJ~=$)e)BgYt^@W%22!; z8rwI>4-K|x@LsrmWdhsS*FAQU#s!Nij4X_o^InAuTuElk2JtsjIE(f+Rv~}QU-1nU zlacZEX?vG_!^0tw^08!==B%#2b8!bFcfMSXevQa_oN!seaw0E-UaPUc%tM&nwPeoj zPKjD$qmA{f!ZzLLV~o*D2LQjXj9ARB=(Dv`;PcS-<7SC{zyd@v0ROxGik6qwqHq@>-4-` zIAar^n4%@x`IT|jZl!RLjBGlCv-@Z(Z3{x@9nfH5tDeVr^_QT3!!>W;)nn+#%%-cI z2fK6Xj?2`pA(CFiN7e@uu=D2`YfiGp)Cw}J!AP?0c;L+APK1h_?TV!Sy;SBpBfRR> zFi}KzGYWa|G9j^33>R?Pi+I50g^mCM`fVD}(Dp+zz(@09hg2ZbMzVCmjLh8?C$kop zul)Jb&AX?5PUPc?jK{{z)Etq6aHk12?-I@C4vw+szRg{(1ySOPPcMbBl^#<+TX76E zY2~CU-v7~ErrctjBg zSEp^Mtvl?`N2@#b1N^RSo$zW-=Y|<-A^8uD0DIziF4xQ=n$>)C$IEiGI^z@r|pW4m{iQ;+GV4bcsFQ?uHQ%Y+@R+jAm-l#1?X!?sLY)_Qpb( z?9SCe>ps}i!Yw4P|6h9@7%D1t>X$)=j(pIM^G@P)9%_vI?BKuza86_Zj1vE26KoJ@ zb(UeOw5BLqTIUWG$UH{?P0&~#JVUoEDHWRZ(GsT!wqrR<^v355-eS+reGFGBLN*9N zMZpj>Hb(45%sP2KSakMK*j(d1k%>B5WSAS1CpSVfO^1r)rg|^I5~vM^4tcC1s7Ws1 zt%H1dCiEjcu)LTY(a<`CJg{fU{ZFX8vBb6Lc+xUq)|c^*QgilEKR>2k?`N?}4OI-C zvh3b&xDqsjkZz!(Jwj@%JiSdT!L61h1!P6VW*kU}prH2nnGk-dSw%ZXjU(f1A%Urr z6PSc&T`vWP7lkxIXk^TzR*Y5&K-OJbgr^Vlym50`>i72cai-tG`Jvpp z3b9)4O_t;pj86_?yVjbKYS#yVS3>JCcjN91lS~vJ!h>MpC<%PQi>-!W-<}DaHpyFA z4fXWXn61yOLV7fwu#)m>_kE(Ffx+isor+Dpgd)ni%CO$)^3;OMXP#Gb&9 zG*j5`3*$V1bFEz8)KR2;R&^P5lxT{11A;}ooGS7?J}4Zc(NpvAn7qbi^pKRB(nMK9 zL5}n!A2E%L(ePR9w)(|0BmLV5^ZhD;Y#P0tf~uYirkuc-J=4vh0}g||gjFo2DdYi1 zLa-K@0u}kL>K!k3c=0F^$cMt$;G~@}uIw}wGBiA-LKUlf2Muzof)nUAZUy;#AxZt*15X9K3y99yMC^_z z`ziO1`5oR$P_7&EJEan6@P@^ATFubXn(k?w+1~SaJ4?i9pKt9gj@Usyfx0@O-gT~7 z-K>8MAdR0L(2gTa5kNCe8yqRYdGIpT8K=FT$<{nQf|7zX6G#_eppMS(#Izq3%da-| zgA&rjvryD6CwZ^?D^;cenuF0QOAXB?4{Tf(i_bPTpI&;S1B5Fe5YTi#0O&2R_x$u| zV{6;`gGiuMw-vmm2F=s86zJhEDkjD!EsZ!;to6zj1(59Is+14_d;|)5w~g-CYem&d z`~WWr&>h_0y6F}=)$GA6=_sYHKqU*7pLppBs@F>06j+M!|nNCaU#&>wFk$7 zXNMC1VTYTxvbKi38*LgJZ7nQ~V~NfAj3At##t+#*J;OK+v+BtWQH+&Ud&XN7XyO-yXuet+!@3XV zi>3Dc*bnodEM(H&*LW65NY;%5|OX~>{?SyY*Mj$GbtjJW8;<1+vavPrLzS3 zT|JCq(i|}IH?9jw5_hWeB|H7XyXW{Qa9W6rU60e0#WiB2NJfG+m+t1M2AN(gzq-04@?3>K_M1ZZ>bhk%!0Z<5; z45xIyP=~NCd!PbrHYaCiZ2+s=>i&KKYg2GveN9a~o-e!8;hf(K85ppS+5jdJdgB4C zBsN?AD0|)f=dE~l`;W#$)OU7PMe3X0I#LM?P)Gz^Ex@(_YuO;se(9?}Wo*EwGHdkk z8=+p@*m1J+)K`qnl@vSGYZj9rg;uoTS1^RPwn$>Qo@^f4NOZl3!WoS9q75&XcrTy2 z8SMl3Lb?yibV>H-EO}z40VEu$-m=R!^Y@G-NX}L{By1g+;D34sDFn5p!NASjlxMu9 z_=p%7&qvbaEG13R0#rU;er_(dFEi+;_~XTZ7a?B}dG-mO+02a(464eK1CdY%>5rvO z!-B-EXWt$xte~(kZPZVBIWi`Yk53XFvz`4!uH5*1PbGQL2J~oK4K#M?GQoKG(pX$Y>%w$|-q{nI#_>Ug#`H`>=v%fDqo zRP%>nBw)M3TC1Y2lR38ampFF(AdoN(r?Q zIG05NwvN$l(Ku^)*`_MxQFfl~nj6Mi#>0ssymt+UiWk&T`Ju{nS^=@IiKWQ#l-|wi zVoEOIjxSWA^y=_<^@qIqPBAD}O!O*mCJ@3m4@pdyfiZ~^?EGc<$j?T@Uw`JCQFYaG z6pU;Kxy{&-BWJD?mu)MeG$k)+UDu&a&(5$jn1mh@`~G-%44H8}_!zET8xb*u*A;F3 z)OIRFvU+H>gy1A1ujN||aDru;PZxuN!6OL0ZopoM@Bq1{cfYb&w2h36#Nu!u2dL)^ zOH%-nwNS#6=JLS2Gl01S772M9Ujs;#s@E&CxjG{XQ;bMG!1Et$P7)~kEaP%t$G`de zv!SA8p_&ukVf|D=nn&c$42LNW{&&|3FA?b~9Os|or;`hfJCBl#E-oDVF=4@Zz8Gah z@t02+zq$K_jD-n!JWNPFM(CWo40K)N-k;X{VqIPEYu0-psTcWtr@FR%)`o7|oBk}h z?`As2gjHc&*zW6&&E|1Jxs-6h%>?BqybtMvL4WyfoDz>Q`^o2z$ZStfPp7-}6DF$8 z9UM3$>}KpJ<5Pg5s40kmr>6rD)U=^#-JwFMp5CGENB3Rf>tf1BkK}jWe*zKOyBS&1 z?W;eYoYU3OW8f=b(x^{%iX2<=yQ%NmU;S+5@P0bNN;#~58>B+WUjT(t!qf3spOCAu ztaE$=j5{zCuB}1Mo2$tx;l}J}dFN7+bb(u|Z=&GSIz9D8JYXStls2I)L)=nayk*H# zjumkUXvHmfBZHB=n!GIa7jb5o5V?~xdNfy3-888 zQ@Iza4z_{YP_Cvg*~`-=kU{ClsQfmJK1kJw@yrZOHfV9%)-~;h1*-J|xpSgMs>I$u zaD^yS|5)QluBz^@!vRE`GQRC`M=JS*;cXaK z_xW(FtdAd70oe@w{v8t35{AqA$#zZ9;a~=& zURYdojNsd>m@H;wuQiQ~D*%TCS8nFT7_?AUY{R(5lXOT`@;>pY zIHh_e{3ti!_(w<;S}wtQ+RMHPgBsrRI3%I5FeV36E=&@4I+1aV*7(Mi1O`v4yJaY< z6h96f-=WcGzBml~AIUKRx67dnBj29Zx4WuDO1ws0tXASoQ65-?k?wKmqz;0$JvBb3 z%j)*#$NliwN1m&W4-%s*Y2h4)Npm0Gg5H~WzTHFaBc>;D){ zSS0IpzB#sTzMS_a1$Pk}!u<&bUq8Ejz8n9!`UVOJ4}4Ca)!_{Iq<2frTSipz7QE3R z{JZwJn_W8Ldxode6ecVjtWGJ)E6(Vj6Xog}2gIk(4`x_g@~ss@3I4#*@SEr@1Rm2H z^K*7w22B(wq|2^_pbJDgFe2b5dVH0WDdDf!`Rwr^#OwPw;cZ&;Y-Hl4bcyvscd|jA zZDjmL^N3d&rCQX@44*&5Z^%CT}dIt+*(l*x#nQx0F?{ianTpB?$RG(sE)VR6gSmaj}guN@~9A34#Ln3x6yS+ zx5jtaEm6lRCS&qNBnc&VN0?xr7d?@KoEd*4MTHr04y)9KUwZ4bFK_p!8hoy^*A-u8 zxjY^AtG`)|=y&VK_~o8VrN4)lfU~YJ&Z6Jt3}egr0T&8~<|l)fc9Hd6|&+;l$TUjN2e7sgI6aA>quwrlzK4a1dd>+PXV|(Zu1|(8R>x zY@;_Sj+M#3kJ25XGYO(>omIqk$JqS&@I~fgj143(!MkGVqV24I`}{g$^L&n(^t>e>OTFdP7U|W=(*y*l{?{tYG72ZC^{%spVvAhqo}H)ho`+V(9Gq z7LA@Lz<~CAaV=YfXA%Q$JL8S{XM!55|Ks)NXQXQVp6>ZF zFX2&nWy&mY)O(@y(V9temxhb0vFUmP!EOKe$8os zu|@g@djZ3(eEPF-tr}ZiRJlv=QiCaw*8Ns~E|R{~4zAzNH%Q2wyJSpnF1znZ)Byq1 zOU7B7Pu?!QJh`;no|#||blK(nZM;Ros1mll{g6Gsy|*wuoGy)UZX)1P!;!8L^6sJG z7_N!D0xv&rf5Br7hcw4fporqTdZyp=i8UCgS^C(wCSLaEcFzQsck&Xe?Xk6> zoj6OIZ$p{BI9?mJ!!7_1y0#Q6Ql?MFY6Z0ego1V#UQGavu-o+!(5gWba5ceS0!Y9i zeoLT%T(%j)Y(DiiM>?4wVDNMp#^-Q=1qf0r#a-!tv%$n3c>@+bDLL8paE=+^Dh2}q z9Kj#*UV<{fcw;%B`{8^53&rT<3!iXzRNBNcSXG-Oz;!v%DMUO~3%$S8*Vz>&9D{A0T$OKZm<3lLbL(->#w%0rdLE^NlZs2Wckz@X1UG zR&RODIB`M2lo;xthR>VIcSHE56b6<4D#ox}R%YX*D6_CbqE1}MTrjt#Pp>JxaBbWk zqQ+J_s$r*dY)Ojah<5k7tB}zJeiUb(BpQFMW@!(GHZ|QH0$n@`qTyiYbr|cBvg7lY zC)NDW71m1i#gmR|*QV=*16ER9+d*hF2&!BPdz(vxCw9jf^hL>6e|mWo9T1C)u(X0k z6)k{X)zGP|j>*HGkD7j$$`0YdN=#j!d65dKm14zJiB~8Gf89G;F4my~fwZuN1+{Xq zW>5$wo&HD~pN+G#h^*|p7fLdK1e*|GkbEHkZfIy2$XBtr)jw{!r_Gj54i*l>Ap&^i z07l0aAWpE|pQ5?By8|az=NOrvukYKW5el}~-F@O@XI}+~iaT+w8@>YI7yvnX+#gnF z?0;HMk;Y@-qS5i!W zGl%P|uKz@pjL@W<%+`j5ADIPFtUvX(4jPFd6c~dZFh_tx+655WI)e29Qxq|+?T+w* zI{*`E}hJQ9?R;y+ql{u zK*Gp~3NSo&0Di_hbaX#_PNy9|F3K1Fwt_>iZ$nVk1-xz}5t(F#ib^zUAnzQgEVU_Kmm=4NT^ z{%-L#<=PJ!w&?+Tv)xtrmg zE+tCqq8|nEd7JRdYRNt{sR;gY^`|4t(ZEt<0`$O9hu1(DOFD&V9&iHed}L z(7wI@TtM@D9ho&xatrKdL^!{;6<&pG$;f(lP`(Hupt=G(%2%`C!SZ71n7qZf1R`k0 zksh-kE9+VW2hM_HblOU5w94*nuu~3=$a@%?hfvUu1E5HTHO@FBMpM$ef%ym*I?z_@ z>J#yE&2ZI{!K;ht5leq zbxj=TKgD0C5E?oi5(Dw#NuV@=P%|cLagRSwv2A=@;mArcm*XPx#XL|azOA$HhK^;@ zQ`t0ejN$3BNmH3s)4Kb;93Wwl><>{s7JU8jrvY?qq)j25-V5ncW)$%bcpt5VW}8~p zXU#vn989M|54$2W_E*TtPqvu#GQ9bAiGmw77M6s687#|Z)B0Ub43-LMzZhF6aY6K> zM1*)(Sp}L{V8o<<~g z-M>&{YwO8R2T++uDii3!wfhCrF=}F>bT(Onddk?ws9ztJv6sE)j7FV(z~Hq;2;W5j zKZXZU5k4KuA#8r*8gysuZ~z^S`4&=K$XoG?+?5B$Wu*nDQ7WU7$t(#%wL+7}?O<|9 zIO9alSl`jQ(nIcqfgg4^7Q6NWeV_dB{K_*rAVGzq2MAjB{r7HcM6`2+jp}zHW)Xlz zd=EQf*8fbSPxtCTBF{o3*4JA}=PJy;>ql8lc%uGjPmri!Z4`lt>x2!P5DKa3Z>)li zSd*`vv+4sZwz6uF7ls_xo5tUsg}EK;<(t5h^6ARlrPZl+h>006A^r`ob?X~#3j?rN z5qS%5sn3lam8ur|20s=&tjjawB+(nW+_s!m)P%xlZcV76{D?VLxOIZ-$Ja>kDg4tn zyhq6l(cfc1P0!d2tbIOr6QvkkuQHGfFmS$SY(R?pVEkiu z+N80uuEv818F?19ww-e6)5nZ_gO*peV?})S-o>wDaPTprVwWxLA_Smd>u1bo4ZRck zidAPlI9&OfmXk#Q#g`tTux{=iA`)Z{^6bVu}69j$z7i; z@*4_Hcd<`4nz!3&@INKFRRGYwMM>MXC7VIkB!L7Wnsp+4^{gxQ4L#F+zm1P6+XUJL zA-sPI9Xk)JOwa;X;$1Tdz^fDkcyeY{`@$Fu zkfcs2z0ptLzj6xT%))$rfcmN3^bddZ|8fs~UBceM>~F=xT?3F^J&W)!nhPbfydpnz zQg8Pxl?&7Ag4vIsDBrWTAL1Q4#+-nwMEUke+nj-c`(!=$%YPGV=qdn+9dOOUXsEW8 z*2TeC@KXQTJd4#mPKScvssb2l0P8j_N*MSr>=tj;eF?q;h>Gj{*s% zUNmLqX%~G}C=?DF)-J(+xm_g9TIlPhD2J^zxv)`2@0VyzfxxcnzIvI`zymN7hu44W z?BOm5sL0RvcCEGAu%C)_z%%my)%f?ZhWh|p-pjo@XZS~2&%*Ipl(V?q5oxei#JZQB zHHco`KcBY70gy%jh=^Y3pDyEyvtg=6aMyMUKNRc#rI(KPcfTuCJ=3riGMfi-i(Od+6uo;hFa1BquW6`mu#bnQm@vMYI2ezDS|t1+|?4m z)dfhEV;A--BdQ^~-SA#yeM9DOepJ23Y#rOa=-d?>NwM3%0U-SwfKm^>gQ(mR#@$NK zW)>{`A^RU2V7fX}AOQYY;Bs%vdZlgcG275?l75j5>bv6UN(q%8MaBFdh~tJ)8}CwP zu869h*W!@SRQ$k|aK5G8$0@0@mb@!mPH{~1WiC!mlWpy!!y4RovrFKAxoL*k_rEKv z7DjWZ#C%MrYx8)oR2&d^(EsS+{NO3#`=NxXOx*o#&FhN%ZcPL17N9`$FADSXf7mI{ z{F(VqIV~V~wW*v!NNXsp(SQ%F`63O1V`~I|NAGBNeA$sWAV>=1|L-sVm{^s`oMk(4 z2L%m*3g^!Slj?`gNZ#EqGZ#q$^u2w#2Pr&7+gC0=I zz%`4u{I+Uipt!Js|G|k8;hy$)rYu#OmnG|44`L8}6`4w{6mdp5TgdCTL<#;qM>$r# zO}1!sEy8~H=1~&IZc-icj>F*dK8~r}GOFgukHFt;0NprniE97*hLgO%7J!FB#4(nw z8&WdZ!L6>Rx3#cTjx(|BPf_<`&R;p=iC3T>2IPlz_x!Yp3gJ0V2=%kueMK@1<9~n2tO-> zom{VVYXT4YYr=@5#oky3(r#0+IiN^vFd03t-3d=Pr^vG z_4;v;e7ZeT6sNUdTN=By2DqdePyiKgm|?2j>x$V`%bcw+yo#^jB)spddW%X@l)%C;gL<%MN3EC6 zA|k1N$8$va)=UBQ5XX%#k4d{ZGeE1IcQ&X>3f%VA35Nx*#(xDQL;d8e;t-S%0ji0M z^|jSGXnnGGURf*b_<@M5`l*ajGf6SNdX<#t;pvA}f8`HM<%}Ce+QbD2l6t1LLc$2$ zp^|PekXkJiTb&K0I%{E(xz0NuzeE|m-l_`G!2yB!k6&Nj8AyhY;-3TgKqlBTzO}`- z7Tg~A$gNGlOVJfZl7Bpo6g4FI(>^{#o3laT?G{_Fbl-*|H&XnXv}X z3+bj97-{ZJ=bZZ|cZ@VI0M1WOCXv*HMEYD$2+1AXtOB~7TJ)az3rfvorH|cdQ$Sid z%Ukbp`ArWUx!}1Qn2WWb`Gt3RliHz*%oKbwmd%SIKo;};tDVH8Ek=H$Lnbw)_!f~7 ztm+AaB`E>Cq-R3a#!WU(^6zCxK)E;2W%8>t1rF96@jekE)$K`foVuIVm?O{aMWX85 z7(4{Dsx`YzZk2HD5en`kKpwSgXg=%~S6XlHMA^mO=Is(5UgrGx-qjvv|KS&Adqvy< zyh&(~?Eb9L=A3r<;ajgIXQ1&RvpJ9}3kxKNW{!@}Nb%WQW6Tu-`U0+fW+T_s^Fi zrTMEQP{!fFnf%o8nru{#idChIi=nFtDbgsJ`^D2`6)6tH`=@lTXWw%5kdJVy$dc<0 z&pI5iC(?>}Td1-4wgf=ofJzPV zsU{Nrznhel``@Vhd-_DAQ=p$0U8R9x5Bk1kKAVK9aiY_Sx=T{>$tggYy;TSKi06H- zkacc+KZpkh`h-%IEINQPD>1*dmBG87{hE6>27#;=hC6ba@!AtQ&IPiyYUVfda1X6$x2{GCF76h_F0>N#kVx(&~SLTn$e z^nU!hGs=J_(>XzNy^N9YvAapI;O<9|;kaP;7r&LKM4;1S>mBkf4t|?biQlLZuo~d( zp5c{-gkizq$1AfO``ZeW41GZp0PRLWOt1`%I!g2u*NRjVa1r}VUZL^!vXk_&TYH84 zsbqV4{po@)wLK{cVmD59&b@c41F8#pLs)Y{&_uWQm}UDjQh1GagoH)xp%lvxe(NE^ zZS#$G5@H;b?BKM24gZEIp8GoZi=`?jr)SlQflKo=pc}wK34SVih%A^|yKg2~d9GUM zx}C3$+4Cjo>z&?t^k8O3SpCY=qhxNgHU}YjfSaWeReVsU&kPUWlJ`BpZCI>!BZY#9 znQQgncSj}X2MI0X?U+!xVEgx-SOHF1B2O_lPk@Xvh zxheBBU0E@6D2L}qS#DpurHI`sW19%KTjc3g>P z{;_uI1-Bo5of^{n+Hq@mqV!!&mV4`|@Xn_|8CE}yhN%@H)mZwMcTe^A#6bDTd-e~w z{Z1(*6Idy~s*o9*I0NS?W`VM=vSSMdz!v9og$&A}1xQa>49j^BzbalZ&AK0fqVj;l zB2i7Cls`~(pU~4s)8dDlXB?NeyYH{=MG$jHfE;N7RHRoj0moEN92E3q6G0A^TfT>Y zhVH4P9$1bIjlJX^Es5bZyT@9d7PGjY99=;~C^fr=diyKJN7q}%;;EF-gFA#j@pyUH zZ#wp%GlM2)G*9hQtkluhCqk>v9}(3CkD*PV;47iT89KPIt|H;D**n>f|62;d865N@ z0Ls-@iXrGN)IUdA=g z_~xoizh{^rJlEd+uEt9++DLl^%%m`q3YmjB^0(A$tyWA_jyU&4MZ!cGP2Y`Z3m{u2 zYyRhi@QDVrGQI2h4ih!_jc29}!zz(J+2i+*=}#`+g655PQemsWUr(0$yx(0v2?Lo0 z60ROxs_YJ;B)&L4&Ft53QIr7gYiejvAn8w_^6u!TcbN;PL8?{wGd7CBsZ(IC8FWK4 zavpZ=&3(U|opYyI=;5f+ud>GS0i~*kWD&qWdxG7z+WZV806j_QxbWAJ2%TfpHpz-= z{7HO~g70+OWS$eo_&%=D4?XTkJ4&d;j81 z90-u^{6xAbf^L~+osQO$)SvZKf)_JSjMZgF!&LFSd5IRDD=&fC8`m0+)%smeN+sim zq~&*LD?w`O9KW=(vr9CL>V*YKIVp@CV`vEvj=!m_f!iKx6TEn=%W(|0OhmX^iFaut z(R2%1(de-`d;1Gvih_-GT`T6+Z_R>kOQKWpjijoUc%#CZ1!#v>UX%neLnxsCBELtA zL;ae9&sD?*K+rCPz+wk>;*34vy|e;V!8F|Livr(qR{PRJ7fLVAWdCM5lMEGjUWf1t zfAPGL7?_sQencjObCIdLH_$rvPEp>t`WnZCK_&$if8!ZEI1d}e}n*q(rXWnyUx2zM+iD{v~b1(a{#&AJo+X?fez;arP){Ky|4}cRoO_+Ua+trtK+j$wvJ;q3KnTh(7qSnnlxOL>Ar7KWf7?S*Z(!cB z+GbTy54{b>>x1PkTmHp&L+X4g^t>`;p?j*Ri;4GZru-PzA5kOa{>6d$nE&Dvc=G@) z(7TQ8P(X)+f$E2UtJGQ&m&<_bR*nRLz1}mDDca82=S6|y{hYyVXb_VWlrP`1MlDkA zU;b<7Q1nJ#-S6^Iv&xLVb76X9C9-et1D_gS!)?L4e*s_N@Az3ELBsxF2wHR65FmZ| zk_N94d3{K)cz9zS5SF&iNfg?Nsk}^po#20gmz|$h1k;g{m4exsh~wF{&)c%)k+vz} z*lb(BY}q<(HBN`MsKD3T<^Fq=slN4QAG!m=9JH99gsAY}vWf8@cq#+)0pH~x=PsRxR_#aAA3{|kOD?G`;EsfR(8z5*#8G~2 z-}^Vl%Z2?H9DILo=kJfkhY3=_q5gpe9Y;(0^4#9p zwKL}_pb-vML3fy|h^n$o_zOG{hw$gB@`dT1IDd#O^EK!tCA$uNKa`*Uv`I7gKkyv9 zH~)4UY^^Vf`&>bOT`Ww8pWpw6B5j!dTLYmPep!8ZJ>UqE@QBfIx?f{;H|W(42ZsKW z7mSyR$T*mM0*pES8S>c101S(gyQ=QLvnz_D)V%*1G#WyevD&FEmGy^M&d*hZ7h*>N z)4E6?@EVyBe*r?iNGMWGj);ISvv$tDh14WeIpUi*FdQTlrwmReOS$r{gme*MesDFs^RbG{wIJ0*3C zd$`EMeGa`3-$3TVux;xsMbl}zm0RECpR1&Or*LsWu;gZRGFl)oO1Aq6{zvuVSo4T~ z`(^`}2(=b?9Xt^P73Mv+&E7@ya$-UgshB<}P>mCEjdG_GBiv2FLgWpbdFj0qN|Wa_ z5?;<{N#jO6!W$&>%7kU)9PqRc2UUEbQG}76k&7K|@MkqpDgBN?O?(3S`p*@78 z^nkZ{FvU;qq2SypTKXd)%Lx&r)vi=sYHkYa<|| zXB=Uxq4Y1yZYQ|#ci9X1j~Y`7T`Lj-b6u9{%&WS8yh;Ne@;^yb5~Zht_bFs-4-Axy zG)Un8tM)c}{z*-t@WBCet%)CSHbjBTw_m;4q;T;SNsBDf|EnRCKRBy)CbkKZGMf+O z|AT&hM@V1-Nd`Ig#-%&-2!?ZHf`bs9r|RZ~MIug;P85G7iqsNhK!-yEI1sos5^5w` zjpQp982RujX>#m)-!e8g?arAQuD>6|Ft-=;7y0u|7aBnoRsi4Ab>jAbYO< z?KQPPg!(_~G|}Si8bjJmKLd)o30IYaajb^KFTySb;<9-F)|oaTK_!Dj&zgT>aKb;U z(wlMfu{fHs-YO@*Y|{mY_QN)RmSUQ@0wLgw!po?%f5k$Mq)6&~)ux8?|2C4vZ}qkl zy=-Vs>Fm{xOAFBFedv@O-1Ptuo!7#LjC{R+z&!T}o%hRXM#aRJQc=l-+%SH**%rru zs{|(&SQzC6@_k^mDA!SectOeqmFgP*3_-`sUc~rm!wPYjUD##+?WC76(j^S&Kd`h7bq3 zRxLL!yf<>W|8UiFilFN~jJdz&l*OwU^=R8xfD`9O@fU%FmODjoxXg> zSYou-JFTy1-m%5oSGgRQx$0i)IL~^e_7x94j^cG2MiBs-5r*mEXR_%$>AN`V9KH6h z=CFt-9c$jsLUFkqARqTH>{!>G1P8p%FhDT)cK6*^C{)h&PBA|qGGA*yMoPaqK7WOP zQr60Qxj&UlkPO_N!frjd;embtE@mdYNSb5&T-_BEm}<=}TC4?U$ix8GA4`-#U`%|E zp9vS~=?20?eBTq2E)BL!wNE1>AB1dLcccrsAXMDFk9`k%u0CRGT69H>C_fve;egb6 zl2L)Ya|MC^eDg_I#&6r>#*9oX`I;@;vW0Y?e!gyRx`zj9xgufxv|jI9{&?Mn85CgNtN^P%S){z0t||5Nfj$lSYmX>k;m-kVz8^D`s@sT z1`jvdpbc8W9mRP}Ewly*-Wlx~figB9!1s1AKysQ!$C@eM;a6==6NY2u+-i zSH@~x32d^yGk8SJx}O1-WMj6RN(ihbAMQGCGVFh|Y9?O@(9pYkDzSRdu}U4Vf6B@@ z13dfxugfZcj5VZbZUVR2p zAhi(bJLm8TY}na4l&W8+C4@UM<2gVhGE(7>#c-Qc&S-%WmX#+fhWj%kBv~*f zVPc_m@v8~jOe%fJwmy8Z<9m~ws3to!5Z&qzY@Zrm^A(mPnz27BE z#y<4Mfefxvi1hM9c=DcxT;0mU~-pHHQBOl~;f zCGo!P^G{CtOKfCS0*n z2jdWNGnh-$mO2hVCkG3R zx1e~Ivp3%xN>KnD^~BCOwveuidd3A#LVG{^zUwU1WM22S_XqWH zoW^pC!{}Y6SSt2)8aA1+I2hm=b`zv|JUoWi6g1lLaWcbPJ@lhdXZAo7J+42Jo0Mig>@oR2_(^$3lMt zY3;1Q0vdV(2W-|A$G4)Kl7w5O#X4u)_DZ>!mJfH*sLnQ$d)j07VPF? zJJTgT5{3}T~u* z7#Wr~f)TOlE|l4{0IUTFApIkRWU4@!L8%c?M`FfW4xk+;i$nIxU-}6UQmVv`XjP>E zfTnoitM&}Ff3_?D7y4|SQcID=fUj0+5^eT+8TW|c6aT@ek2m<{d zfxVGN+H1$+;^WNN|4e=zV1nh?|NqY^{BkdbeW!b%&3AhKytf4TtXbnb0>JwI5x8?q z`C%nG`^H|lAFm?IHkdzkh-LBKwz}6Hwhna7xaFCMY`ZCR`*4nCeYPiCSPP= zGKxSiP*Vi~ifh7sD60QVDK_LM+$tYWjT4#H@vXV@7_!UOClEVL~fO#d#DP2ai+?i)}<{Q>X`S0BH&n$C})Iuqob9;U$DxLrdm zi08SeiHg+R_FvnrTdX!J9+>#)11GF>$6Eyt*kir}IiA&RHMEbwuwRV{b84jk*-`u7 z3={zQ0>;E`hvO=>-uIvZ`~~>Ow9e9#y@$4Ow&nlP45Qqsk6g%=a$J$dj9*xLEoVfNj6e8V8P%AZjRmq^8wVkc9=Bd z`KIPjaj??OA(>5|CZ1f{@OkkO0j4&+Xd}%3d6zIOYkqjG#g9|g9Iou?*gVk95rM&O z!dM4Fg3&PVt zW^V-*9f2ffrw6wp)+jCL_$@5ILp*2k(qQ{H{A&OZLE{==@-SFNHs&~KTwpU#f>7x2 zQYN3OmhybCU!?^S6Xi?ge)1mLpDwiruI0ud`AW@p6c=7Y1LT+$AXSWLW)lFdT6qr+ zJt+RUw3&>>dj_c~MTpg{$vX#ATcaE^q$)``$GBCMr55JSy9Ql=XDR%W5h(DS^EJ|M zCTGH_#)0~k!CMvAw1iK>!P)`nMg5~XRFKa_l83|pqU$Z=s@lTsVJs921d#?20qO2i z8tLwmZb@lW1eBC60Ridm25D(&knZm8_t~Cv?!6!0{|A4^gPXnAUTe-L#u(3-v6a`P zUXu#jnS4$Ot<-Z|^8VGSs5hTh^%f@fl2&i&v}RwcJnT}GC6X}VwSEvnberBZ;K^lU z-7TbZ&KlI&H~b7M(GkC3yr6???Q3wd@KQP_0&ky~q(^K9R3##)ogzU4n}elDhomjN z8VCzSGTsx0^;Aw4<&TddU92~xIG=rV+vzxyZftElH`6h_pDI|rPawy%7aX0{vS;hw zzfotsxwa&e=AG8Iw!gN|>u9yW_u0jRBt*XO5#f9Qh;Gp?j~62{Y}pg79!rsN3$Fi2 z`2Bo<2&QB;GZ1wXl3>t>&rJ*^P3aWS+*zSC{Ne4!9!ntu2SL5HFU;SngFTtbYB#JcT)GCZB?IaQ zTo|TA0_Of@MN0*UxBD4`=Y=ITy;O5(zb8ruj%zX)ACq(SW;oH4p_)^?Q_jqpWMZ9b zTpIh{I48N#U%1EM(I|Fq4ff+*IL~aieoY4O^@h6Gjh4^yR6eA0(?Sqmqdb+q6p!!9 z>dFf((2*_ap0(MHODQ%EihZD!tO1xFRU_`PVR*vs!~=XB*Zg+3P>iXLj$gw_wsRlq z@~#n(yf1UPo-0*YJLxiX7}9h11o!c|5!N?UvNw4_c$Js$vfZ5uQP}oe87#G#d9iWd z>_@FWetWUyeKxW?s^eI}B|c6C?bC^C!6Cc*>BX~$?ib(2HHnj24-+Z^>^-#V&&4M6 z*KS-@K6_SWyD3bWq@}61UBZT3(sZuOiRap-;iY!%tb4L$Vqg*N=F7zQmZh?iL~OG5 z;>~?a??3<2cw?lfsK8qoh!=CSYNjawf1Nu#@R$jQ{Vm5mo@BS*i3L~+ue~OERtuHM zrdp9(ud{GpOS%60?oSVgYlO5vrRUz~9bF@`xM9dcGPc$rGPZqGF`v|P*X#P@tI=Ct zSxlEq%ufgowXQ|1H&)a$xaQ>=&X4%~?YllpAoLmzX(my8g+tQx$1z2r$2lm9uWuo0 zry>tt-!}29kHAgv=RBEE|MaHKY(j9D?v{AMjb};i`T8R%GDtbAF5eBJxGpO~0w#?*syPI z9kb50eyXKmi``zalTLv|cU+k$?kD4a@rHf+2fw_Pa;>s5#-PbRC}uDTzyCOV zR+(}iqk~#8{pPWj#oWJ5yCUkp0PO5v05&W1dQFns?}CCzONvITL++Q!PucDpZv<7c zXk?DxE+o#C`nk_Lcw}GbG^D}$^qE2e!gRO0|~?r zS?vdCj}#+HA_e~ipZ)`mUi)5D&QnVZD&Ty4`gV#AfkQmI_rIVI$b^0h>OW|{7FpuH zq*BEr(AVGp2oLX`m6esFi_2WeuohF623S^1Z)`kZV`H=1nw&^-lMoes38QHRZ<_Cl zt(7cdP z?34ht`5&kJE1XJi?mwU&K>>U%BnaG7UEN|^)8&sRO2){Zp+Az+nW`|Va35I-qV;C8= zt665BZlfAoFJ^K0p(1A;q*-@fWGmp9^<)+*xHRhuz<>zCpn;IbS*OJxXM2DDCK9!3 z;Z44)bEet3xkvc;{&}^hcM((UqeVtDv$G$Ii3 zKuAKfkwQLo8eYfle$I|0T8oMYw|0~kD4Zwx2Svm{uFr2GD`NCimpuygDdX2zqpzsq zAN&M?Gm^{Gt`OA*Su=qb&li9o3Osh0av?jh-%AMao@>fcewOrpc;isw;(XRfdR-1{McuYk`( zF=kFT#TsNbt4To#3I9#}=H_N%T3RV(pA6dY@wtIS zCK>YyybM$nY7o1V2N2vsq@}B#$+%V_&5a-;j|cbZ*M0Am`-vtcWmIf^5wB1%%`h-g zicyPkt}-zwo9Z$%0{#Bq^(k{Oy{dy9ZiwIXzKS zPiKC0!j#WZ*51jtJ>Df(qk~8 z-7r=V8yJaM&s<;FO4Y$(ZpOW5fO)lGU3h5b(<0T$56d(2UdfmgK$G$Igs?6RmafkR58fL=t z%1>zg@K<>bmy?Eu+e6)!0tU1mE}EG3srT7gO=NpplAQS@f3G6fRysLivu(2Tuf+uYo|dHc34-05d|2^_ZG`T6unkbi_< zzrH)1t4Tsi>IGgg-hWEWr~-RosGn<9T2k`XQ*v^jMjuor_jAXNLJu&&VBJrj57W5e zS^pjq1q^J>OuU;S1yh+Ctsd|xzzj%)&}w30BD6~NWwu4egy?k^jBS-yRA7F|8(ve) zP)A(9!T;R-{*3S!!Moj5+JBx)2}wUwh;R&?{dH4IsLp|O=Z{$)gWIXy?Ck9CwF5|B zpLyL*zsARJ=Cew~b4NpXgJ|`!*H|A0!4)#(yeuawuOXZ~&Mz()ug^4DmEZ5mIdll1G{S6h=cG7y-NMl2Yk&Ht>mjdpA)e%hTLA;mWseffoa=NrbB z12HkNMvZeg)B%v}O-2P)j(6rD*T#~^JAQ;8dZQ9a!EpS{%#8U|ErExJ2hyK1tDX$` zG+5O0i;Hq4riw_l=ey!CJZHq>4jvC;l9J8F%x{=+X%^}(PtBmpL5gBFL3gf+2WKI; z1Fy;~Cq_ywVi$OpW_*YiwXaUG%!abB-9;yY&!AQ>^#?-u&Al=b!X*=i-e)6+UaddIVVbT;$9 z9;uaCN zKdNJPxEcP2?zA1A7sXr1gIZ#>yqO06l;^N z2o*eWl&kIh(V8jliEnchRa4hH8lCtuiwn;xRy$%kiFFhrtO)fD3?>d#bQ#-~a&vRD zRg1i#FxhLT%KHu_S5MDU{cR%smA;hBrhG4wmhaV}4Csh&LG!$q6hhL>?%p2gDne8W zilFFVa6NvjR%*Vop8EN9Dm9tfYc)drDrH3>S~{_Nf0uk-Hp16xY;0`Pn$Y9?@1Lqx ziU%SAubjyAo+CoDx@eL(ynSOzN(x66O87^JD=;o<_Hy09+j*O_VKexiO=N20L* z!|VAr+_eD2y@%f@5<~oAdo%=W2HQo6%kw>Hn5t_>EW?ok5+qcVF2R?)IMXBY)YnZPFx1t{g9TBiJcMr#`)Cp@L#<%W1 zwQu&tyideCZhB>5kqi;3J&eM9y!?@8{Y%)h(F}UXP+Z--OZc$n>hcUKl5a4I*IQ&H z&3n2`a{a~)MI1fSVc9lzEF+jqtv;G`L4pTnJ^k|e=A( z{KR(^GwFGIV>YFL16jq%5pCWz^E=x4Wp`K=b>wfD2cnjjk0?z(x=XmqMNjxp>RD75 zd!c9GW9bbU(OC;kuVU=%C;CCcCMLlw@ z_NT31jgCf+26LQSx7G`>t6~zcK9n^hR@1=KFSNv;UWh>2Y6p*-(R|$-@E8rCy}FOb zif|Tzk~1fQ@7*Ro68MW5f@c|&u6~dU)FMy{99&^_W~-M4g4YeK!R8QB@#V2HI%{j| z@~SHJStzMJQF)Kw3;8|SS&X$}blx`x6PwOrll~}Hix>mx^sulnbUICAGdvO!uueE- zhTemep8ji86gK#>NhI*PZEtVuw1tqGkK})VkANo<4ZbGO4nbJEky(tItu2x==SpW) zbMOT52Z_aWpnUEao_vMt38T+Lg6ld^jr*RG35taus47~SS4$CC31qBc@head2dM2= ztW;kq5X7!!{$kpu%p^i3+Z}43x|3tOS7cH}#^P8y51`7vp#v5fuKT?rh}Hroxu z+pZ2|JX>0|W``?Hkk5WWpkX(ViHky~^_<^@Wn>5Y|IKu8jmy`OL4L16Z}A3_6HNK8yS z{D=MRG}2b}-b2VP3%?Tu9336UZO<*;CWaVjH#s95zJ){CIr3GM@cf>LjYHw<5KaC& zF2Oe=|9&P~cUm`X72F*BB(YGkZydI%zsxc+G9v2E&dx$3A`DN;>c4MJGC}u0x7?f5 zA-O(QCOwEP;I&=6583=Zr=Ix+=O0^OoH9IPW7UI?oolqh zE4|9G)QQ?NWwi+J-yob zi;R1po$jv)I6FIkS?yi11CNHxKddPh>l0OMOh(<$p;QLj4fo3vqkZQZgTsOOwosbZ z0AUg^DM)!8PHDF`^gzbQKSxufq~*jCBshRL=s>7{PEH;g7S;sY$|6_!j#)c@A>^q6 znJ7CO{p$8p!iarkvMu-8p!+qwqmqw6Y4&s2libCJqY4LZH{}b}N*Ty%!2ogWwUH+v z%$1-aLb!8jyB%PmXL(Ajh;MsxzKYuwmg+xb3`{Q%;72X~I z+qn$Qs?xzZY3`ky@MB0v-7ICVbRWm$pbkz_c!QAo{^A5116MI>GdL~ z!KXMylmGY7F474Z(^=2*sE2USjRe_y#O+_!kZGbh$ewt3L5H>|}t%=u#K z7yP+hc{Er08-fDP@^;RqYrypEj;!^!uH1&@gqXx@HvVL$iq>;8O75QqD~|}{55FsY zG&_vQGLPoMCX!~uR9nCokK?^+2USsBWAb(g@Fk;J-_{xC3 z5SJYJl-suL=PW_w$$Dg#zW!2wUJ6kjpPC=-(aFfbLt??^xlD#++kSdCnWsE4r*Ao! zf)uOzR1W7byVDl_@YJq1LY#p`C%E#f&kO$B?w=|d8xkIFKdk2^j-=$kD4bNy6x+p0 z!}-9O3*ev?}b-hX}oNcfa<1pJi9UT(NBD3?jA8EuN5j5a`A4mZ8}zOK{P&h zWMjxO6?0zJTw+A+z_$Dqt6BubV}1oY4sUMs9<4;7;u&Qe| zs;DKM$n~C6UUwm@%3awz*)&G>!6>?BJn=Ks{pd`#OC&438Z9Y=$#Z2(@;uYAVpq_I z(hhY`_pCr-HpJ@fvjyZ0T&G_0aV%}=F)UF;G)pQhcVvmJ&?a1{vqF9AAP?VI;bqA3 z-?}EFI2@ zZP91#H)-&|j5NV|($t_#NNuwB32ou?Uu;6C!ilW30ljVUY6j6S9*^0Np1v(%yV;E({mt zIl=Y^wOkQx-59FtT%lc_U9QWISG|;G3e&xqS_DT!BCI`av6vKdiZrVnG^!ngtE;&I z31}^wa?dio@Pddb0tSIj;6j7LVe=!?o+}fT|HQttyE_%yZg6vc$j8TbyHw`Rfk{cO zqjT8$j-6rYqwj(X*U#-peW9T9_*+s?GA=F@InK?Xhg;kC<}az-^^ww&(^zbq)J5F) zFf*?&seNQsyOQ}wc2D>Sd!CVyC#=@0#T##lbRft7O8V4od-w*KAT8HuGfIkkxGcIT zRu5~%nS>|JYJT6#4oO(#Q-R|fRG%hG#zR}Vyw$cYrrrz_G`=o^^{?eg>4x@pp9Y7l z>4{^BW(!B0**$s45B4oq#AA`;$JZ<*4XqV<>#i=jkk*Ftj39qG_+g--{RA}AWJu!T zNK@qC)yNU2T4+LoG{D=>oyhQWz3a%i_*|0&4gLhE`;~R)G%WzI6#LJ^o8)A3r=Ic2 z;VGanAF=m633dLHCJPvja5&gi;IKRJ`Vh1higvU&DlaumJ18!r&3a`uj6P1X+m$pd zzQH$pdTM>_QZ;q(&9s)&`yFbELB1PnjB+2c%b5Gq;P07Q_*&czGX>JL9=PvI_~P`N z12G;3(OKH`zN`|~0*jtDtF*$C6~QOk8#Uc@9?MSYDRjCeR^62Lb|1yjO)-aS-OjK% zY@S2DHJ_*)+^tMV083YotEHw06c+Y9POTc}K?-XK!l$o7zGCt}HaOd?fX#@=u5x08 zE%CMbIy*1#7*N z@kb<_oH6k{E^^>=3E&4*JTHUiBI&djA$2}yXODvNaXZN4{6=zbx*E||-IY7^)FSZv z0IJ^uDB}lsEZeR0H8UaurkCt?b~yfS%8mPEKJ*)($E4h(G4t+fR$d<@*^i!{PGFck zTA=4M%iucugdhEv~(rxT&Yo07axq8CF$a*dQ#PHGZ){lsYwCBr9 zjv>h!0eU47MgIYep#UqdN0`E@6zZeEr6k+04Iwd`4R!WKKtD&Z>50bA{!uGWB)Jy> zf~}{9aX}(YG3<%~)Fg9cuYC_&{BOeyFAE5{U6?U!qgt^>+2K*y?zQG*SJ4xT<7wJg zRe@@m`_Neb)^=0X6oXZxZVS!%7hDph-!`)zCmghcHs^f@=ExUyISOuXI6g?{=r_!2 zV(;x<`#f%Ub1G4m*?E%*X_b2qrB)z$n*g`DGC}$p{aRg}x>oHae({cU|W3(y@aNFcT0S@sCQr6pO{TsmMFmKMB>~S>y>m)(ke#B z%IPX{=_a3Y603qz{3b28YaQJczKZ7|vDUXLbNv-mlk9+s?gfd3v-}9}=jH8r05$J! zluZ=^B?Ca@9aL1K?Avm+1;fK~N4E@63wf*&y;lj%wu-3DEA zv$Ij`*3S_3U(i9k*~_nw-Ff$+K*;P#3SH}OfL29J3RjT1e^mQ=IXFu;D1PA$sZp(g57aF zKyQ{}3aWSl>SMQuT6@)qvI`strd;+El zggUcLUyvleu?NFoHUzr!(psNOEIF^vbq)Wt=|LQN7o_C9(TXQ(ahe{#C z+T=&?BlC5G-&)+%oh2-h)n9rkjPT7U6NEhq=nO2g(8^3!IpyBE*7+h%BNzj2@ zd!WxX>0C&F#3GzsPIB0H)m*r_Brn+G9;S9pi+U7Se2T4()N(Ym+{{X=hv0V4Uj0Ze zbkN!@4E;UG)O#yN<)3;3DPasoZNX3x;3P44q{K$6pnO_-xvh

qo|VVBfhwMogcW*2k!ldF za9B(gf``HSy`j*cBV9i2Arb-yLtwK?i^Qy(h!9}sU%KuebQuZ!MlxyB!h=Pm?wj@|jQa|1@sE!C>fK?r*pTFn&5y7WN zWU7_P8Zq>nmbmkB7tRfEL3)36;T|}dvpSPQwH#ZMcTHt$?xZ8S_-@9l#-AVE=$@$! zk|rr?DvrrSXtEf${c2mfD!nwgn6Ii3uWcJt6ws=Tb~CbJ|K{jPo2UgjPk702>@1uI zU1N#PfPO!RX5i#(DVdu7k_XRPyi*l-@UYA*+wj5lb+xMv1Wg9`78DtU2%5}zApMcY zokc^32{!v2*0YH2?ng>W z_zPjN@k<(;Yj+TA5)!wAxt$Zh`-f?5R&J|^IrlaiAWuLk#aI=Oc< zyCve0vt0n|q|h(!*;yZKNTmiqY|(badTUGcdc zzXI_Z1%SED?z~v-*(MhPnJV!pGVGeIIhwrjl#0p^P}%LDH&9gq)$!(b1^@OXpU^1# zjqgU>F=MmLDTYo}m0zqmpl~IjbGrd9wPn z0L4UKAFm=LSfDlksp2f|-mX}ARDIM5->gGN*q3=K)o-tB9*#>qTd|C0Q7G~yM_L&b z9>mbh5ikv_3`iLSrDA+ER#&sx=uO3gHyyRM_czera9H-3A+?{fQ$F)|XR0GzY`XP_ z4duu1h5gDz<|F|h7iRdqTc~Y0HN%f}Nr%JY^(c3@8o#g|Q)p@LwuJY2=D8{+?PZ<0 z?DmG;;VMa;pT+2RyqM#mjvk7QARgSnH8blpapwrJ7{)H6q*YZtKF!x{Jia)Z0yrfN zG6BG+-r%B*-kiS}Z4{pfLIKzFomSx069E54;02U{vgl>q&u@T3W@}@kCuknPHkK6N zDRDRh36Xygxd-m_To>DvJ`j-ztd^j0>`E)-kb)u)R^AH@!W}K9VQ~y?!}@!~_zQpw z`xElS0JHiVPR{^|MF(7wsHo`A!NG7+@d)@IWLQ)v_wVaNV7iY@|C6!E5uiD6?23jB zbZxw#ZU8c^Du^LK*)ou+=m)>_yUa=(^84F&%B80QuyE3(6AXa~yornqi>nb3o0a}F zRG@IqP18D9iG1XYpYm+ zQI9^OD&NWz+sk)xG`6?1;$JxY=vd)E8pvK`5GE3-%c1V zrJ{lj9Dq;=^KDeZV%M>89eQxHZ-)GZ&1{U8S9i%tqgFf(jUH5IjjM26@`?4qYS+c1 zdq;HkgJhng} zSZ~}C&wGf&*tFXWu=W##yJ*MR@c#j}0eJoY!?MLMz|CLC-A5AW?I7OM&tUfd0XTIk z)47wqC6mF-@RSG8BM1tPpZE2^CIIpwp0u|FmOH1zx+th7FvH7I%XvYlhCxUu1yD6q zs|Z}}{d*+H==I>#iP>5n36|+udK&TUx|pVaF)`Ukqm}+c8w**E6^z*en5o1g4Bf|5X&wXjfzADd8rOD@E;4g~=G6Xdm(K6L~lpC~Fj`fr%7M)&$ zQv*_dnVHgc7*UB7TN_8BLBXW6)T__QjPk?x-K#e$IcC#a8?V$_9(2?_s1Um1Di{(G zu;hhC+1n0fX2OW;(O&)BvekUSl|Dt}Zr1lsLngA&mE0{34}cX%O;#HZ zHv)O0%$Dy%wJB0Im@OEGie=uYIk?KU#+`7c2jWx&T8;Aey~gv6sC-RRQwbnHxR!6b zBz1m%4s7{fT>yUdcEdej|6?HK=`rIXnm%Co$-q(;DNafRdjAvPpXqS}zVW)nBzRn^ z9=Ja-0)pfGZ0d>(BrEId=7>{C5GoI0;d=i$5Im{3yZ|&Gm?AJ!ra^t26QFx`xN)-E zK_@05F%RJb8r9S$i(=Nmp;MiEFp&-i`Z&%7u!%}ak^-2FN#(ogl_Bhkii+y$x?W|T z;pf*@M~u*(o*st zcHM@Z-hIPAG6$rp|$vWf2kpRsBh-F&&ZOIh57O6{jODp~R**S@a0q7(EGi48kt z%ad?VlpvCsy&ZQRs>l0JemLvgG5$RksL6Jlf8=^roi=t@CA2XGj0rqG-ndXXaO1Xe zuZ;Wwr?O0`JH?e2*UMT^OINlb9a{xZy8~%}53v00$Ip3yihd0V29WsF4h7^&dI~x1pd+m+r1ySS*+#$A9 z9p2+N90U?Tdu-kp{%>Sn}sO|N?$0o28^St)7MvJai+ z15qn?B#vUHr!i^Vx!a}+y{M_Q`CUARxHp$S;{U<-LHrpS9;&_)L2Ir#98Oj;PBX#78K$xR6p+@hx6p*sw%}zLx98)Z}YeuB_eF73-~LS#MMb z2&65w>a#zlx1lm-tLqthIK6kmyBpCX+QT}XV=J&W+mOE*@FhkepL;5EOM1J$h2;5V zxR|3S5-;{xIBl1y>Z&`yC%X1oyNjPok>+DNjc>aP?#3_-WGq^gJQ(`KeT1XT^gw?R zySKuO=1hS=`|xqeRc>85YiYtRW|~|d!~9}stX1LjL}(O)+))RVU1_|#C%emBFv{}} zIWo^CG<+r6Sd4iu@5%M=1Kh)~IAPv6Y{e@fm{kdSndp2+EIhsL-#;uBwTKRpV9ac; zWf8^kyyi;P+O67-Cl)8+yv1qEiKTL(fe0rD)f)JglRq zi^93l39|=aei%FM%2bk7x${<>2={gbFGr@*$E;`6nvV1P1}(KmJnDU7PS zako^^yGKD-*$z#P3CR0o@^#OldV>;7`*`Z=a%p}D#pU<>{4|R@_ojNO4nFheb%PFi z7w3rRxze}x_B67)lAm2nU=UIgPid}*jnD>weoVUR?dwp?2}LxBOG>7|;Y}~=`|<%3 zVSue%h;IcQSC(>KDzJGVMYdl4O$_ANwPPq{@7%vX51Z!r?5samlWWp(?g8WgbBa2v zDPB+$e=iR}_PmA~6sv95*3pp+I3EI@EW9}bB}O}h;3!zPP=`R?sT5qeceG4tew5?8 z)(G_iLitg5afk;d{yLx#0gYE2;7)}QuhDztazOz;<#rTjU@``ve3o9ef~sRF1=GLs z#msevtTm+>dwb@ry!|)LR5hFI zPkJ+HcVtQYs-INtw{dm5Y*lnuA89OcX?fTRs*8VI)rwym^Itk;ty3#(csN|cb51E_iO@1PI|jEtGlT8uR=L}K zzcoD_Ogi4^Ojqd-?bBLfY4FhN;@TI}9C_Gmx)k=TQ>4pLY1Y>A7bZjut-tUC!Qa%> z^me6>v}3EeVN}?wgO_6r9?Gkw?sp!1@o@8iEO&V8*G_1r$80n!DDmpm^y=M{6UVyk z>xO)Mxe2@h!~QsqLHJ@rrw3A8n*r)4qt6&@qnUVw2^JgzPgs4}EL#Bp|S zH^(`kEQsQEVuT2-4X1h{<@z9$M1dbHI8C3dI##6sAl(EJpkDk|)S}v@vz6RzMYR&4)(>0)CLAkXul!FK9=tEKL?VoN(6CyS? zyP82JCA;M)*+IdZL{~3EBSnD1b?l0|#=)`7cXnSxvzJcOw6FY2LbT5LB`b=^7yArOrcQ0hDJq+g6a=WXQAtr_yHR$ z=6?i)nWUo%tPwu>p})YAxpuM(Fbz+S(iTK@C$$dDiD8liyvIQ}C z!p|ozC;6P$EbkjHNE`pIIa1Fs7sqn2(DG9zteF~>EVvgsDiKJ$mD@UAOj~ZINoD7n z7znPqX{g!m!*k?v_t!sn&F1-kG?1@i+euZ&pzA2iNYU-N8A+|F!l;~UWX(Yn2U@AB zmPfNn-CDlCJwsGKTniV*Xq5yM}xcHJ*YfZ6=>>Eod@%4 zKl0sq+iSc%#Ic3T#Z{Z1XB!z{&KJ>qQmmn&$!ur3E|j-8dF&YG&c`2JuXwo5V;!s= z_?%&`O$oAVx?pE~{4(D~n9Ih)rS&;rxwHuQ+{X*I1~yV9q8k8>!W=fB$3{#mfEYDP ztxMi3#x48Ke=B~y;gLA@WUg-{RZ!VcntBx56hYTK(3Mh}$9_r=xL4lQ5e>S|&*h=< zM~kmGxnjjO6J;PX|GYS`-$yx0j0v(}+JD~mw=C7w2BrjxoVDE6yGPXuNwDJ)T|ek# z(Vsj?RoYWXR3f6fThWj8m&i6K5FAJ2%i^6p^XVoMV*3wyk)=Y~lC$w3eG!*7HA$?jR!G z8ARU-<@{$xsedu8hTH*ByK9-xI}p)FlY|cG@^+Ad)Tk|jh7 z;q&$o-Mcgbi~qAr@c#;${_8LBVJ4<WtB*~g* zd7542^~zOJRHkeFZZXqyD@yY$>Tk5GW4hQ}y8rMv?0H|aGgBQ~DX6Jt<{qu`PKtsj zFDUj`hYHsUSBRmAz`I9YqibNwkCDA9zMt_$DvfHlGOvtW@uGTDb~Tl-DUSU^7^G0_f+Y6}*yv&g@ori02?H~4q znIM06;yiIYdb>bff8a63G{X39n}CS{R9Drpi@fnX$(6H-Y$FsOQ#(DB3h9;(C)M^{ zbEs$+U76o?l}xJrIo*=6_sELO>3L`9Q`X9`V|nb1UPvB6C*5f;%AF|jf9o5846t&_ zuL$&bfYK19T#3i17-G-LEtt>aK64dDV<>+j_AHE@aDhrpZufoQ#O)q)*0F_afj) zBBfHRtflUx&OdlS!_J2rb%nK<@48P| z1KJKkqLXH&{^K(q8Zkz{wWhc*r(_8+sO21%0}p#s57ifWEuHKoVP(;b1}sVozDrRF z6TZCh5GvzW7_C%zMD&ynoqC3z2eg>$g`lYO4v*rFRiISHqKK_^=uT3d*5t+|HHEZ% z%4v&iu+1h<2qUWsdmb^6AtH8lwHbH7g#X(^)9_(;y#0%IGK44)vBT2d9=fY9)KnLR zdYPs#i`^7yWs1(`W0w&ZBI@uZgbhiFOsefFVW_ZZ45OsxsNUUVRhK_)^Md zl&%&zVDM%@OLjrbcUgs?kG6MviX&9y@+N4NFvM|-?k!g3-BA5OMrwgfy0<*AQioO+ z`ftd4d)J8Ex1HZ9-wj3BD58wiQb_8VD7U@Xb@R7piak4xEpcnyxTAQKo%-C8@>m#B>p$!pI3l7UFmMbYx&}) z*xe&NtRlY$Vzdu%HOxgBO)j+-^TU{a|9)IN<-N<=%vKdyRfhG9 zghXrdbBG1p{pSgWs46a0Rfn7!3%zNZ_Ccy)xOlSSDIIat3Pka`sHRT%nd3z-jt$}?1VJ)DY~NB`Hx*i-_4`1Ee^Pq z=bL%4r{VysM3+sr4UrDm3cubj`h!d zQye*!SLbi-=DP7csX*ZN1zOR zuWw#|BM>5<+|KBgA}u#s+mw7@8%BLAecrl)Qye2xHL`HwdiX%H)D>o{ygjv$0oi|d zk8(h?nB7H5!6`v;A)6t6xya_Z)`>sq4g^CR*~0bjh0(-bOX>NYiAk&(1rpzxA}Iz& z@^35a+SwcCQ;5B!Oc+(HHDtjIOr*4r*32gJ!t4^3w?Gkz9wg2FZ$HUMs*18w&{(xI zg?NnBL?Z`uQKcZX`yc%LEc|HM3tfdg(zlKsS39%i34b3(N0HuD{{70QnQFzKlIeXG zo@H*xIwnjby&ff0C(HVRb8xsSSLmU#5`N`Y{+j5@Qk*$6Tt%)>IlM0gX;97icd&~; zg=T|ni2)&IoE}dB8714_h!ScsY3p4*mQZ{bB*%*J=$d&Rao!2)S{ZsM#FP>K;~&T7 zMuUc5$X57PZ$Z=z<9m%RzKH(K$fZ!cD_Yx&62^<0Nskp`iBpV~LOI*W7sN=jz-y%q6R` z1C}X8bCH>l+Itt0G2o)Bimt)Bo?;+yy|H;+Zq@)doIE1zoBU;n;X~={ z3}y`3nl&;?l7F5v^`v9@FZcDaBeJ)YC(apbwgZwWeX%WlFR0jB!at0ID*P>|!U-RO z4*OZhy`&XY*9jE=E*o&J$xiR8kjkdO0zz7Pyw&$BZY{80M4)x4w9CH~gBQu*YpUIHGM zLTkTckcy2J2;RdJq_vL=tjs80lGIbJV9140IBoWiP;2deRk@=}72Qnwnv#a5Yg7^q zG4G9-)&Ws6^ebT3|E~=q_740IgC~LPL95<*>o-AThc-3ynWO$$c=( ziUhQ%XI>s1EHD_Gk+R0K#Kgc5fH`|O5V|x>J74D-2m}$U?Ug68HDD!*e-a`iMP9x{ z0{PL4U1vZ@pr@-@n1U%P#2JVRIcniQf4(G>hywaW;qi@W7%e2Dq|6j`!v$ol{(E}R zP-hHF2jeDC7oN9KpKt)f&<%8Lx>^Y(f`5S_Y3qrG%FWQBot9P?{3p^SO(99Dnyc%B zDW-2EOH}o}%qM*A*&ohO&M{Xo z1$wv_LWzPZ!Y}n4hK{Ig7AVWaR4gqjv+?(WOaNoDeniT{J zfN^~P{xnl5=QY6Zqm}kqvkb)!bCx-7uUlj-FNK`spP*|R7Da#R5a#u*ZHVC+X1DJZ zmBGyUyA%lHvQV%de5*8`@8`kEJ|*>>&vJ4IP=#HK1ezZMU?yEbW_S*B>S7_fD_;uH-N^5kpyc{K{Dz5L=ry%5g1}h?N=~~(~K7uLM|4>y?X~R6J@(`wF@tY z^^tl%x>`|S7T{=59e8iwQ#Ok~%wReJu3YkRa=$t|pZ^25M5aGL0wQY;xPt{?0g7CA z%>Q@9N={xL!HPqZO|$D%S-df9TEe~iiv(0_qq}NqYPi7bBdi`GhM&-Ch=O_;F&qoZ z3w&PJDBuF9d=o!~@IG0}DVr?xcLBZQ7CSF!*g@#onJC}`qth0_J*N#0APWag7zcH{ z^A}QElS<-aXr@Lkw-mOY=0>T^`VNSevm03=wL*zp(P}0adi2AsLN&DFQmQ z+xQa>s~R!b{qr=y_J2hLFzvm&<#81Ypp+WABB>L*Y|P<@1M9F@4_U&|Q>0;i%2DsJ zvdXad*42c(g7sGBkle+-)I{

PLVITz-pp=e37J8xNLR&r-jK2tL1EJ!shXz)&k zcSoKvigmP?X@OmIsarXMF#-;18D=oFw6tJED*^_$=T=wm{sa~p0TG#u-@$M z3iFBiM$+}`U*QOMh}A-Xslnjy&6_uWCkfpH#x!NQyyW!W0qDU1xSZbF>Vcz3>mjoI zYL{E8ujymO=d}Bb*e768f|xmW;$dVALCn@UAFL)0nnnYl3VR&*>sR52s05tFqDgBW zS7$^J&|1dM79;lzd*ZwPo*TbI3^*S=Z^UhmbP(PA{v#+QJRzY6OWPHu=I_CYp=1ED zLN3hsZ8{@i+GQ$dp%j@M$#1tksq@Zemsi7pzF_{dsw63A8a3IF`Es30=?AqPvGY2H zxb40FkF&Q9in5LWg~0%o6a}P31f)SqT2WCzK)Sm-mQIyYQo2F9ySqCU>26qZ>0Xv| z?eqMeGv~~lnRn*>!xmFfcs7)2j^J0*hw0n;c5FX?{<^3_Vf@*ybrs3>a0|4F7Ll?v6T+Dqc~ zFj72-q#DxIQF!OvxMb;?3t6MH)TBn)FjCClE})c}SDknfIvj7wb+#gj6^#4|JE)b?aGVtzO-X2=@Hi6$8T zX86*S6Q#mPQyvc&g!YjH%oG}gve4$31wgZ#e?Qv0)PH~@hW=H z?FqYL8B`_hTfy`TAAH=j=CJ-eOIwBW^R?UsT_+*>7Uk|9VO{1$f_~oSL?e^XuZnu= z^`&rNL)Wac*9FEYjas|=z_S2ISybf5({4btuTj8=g&ST)#>VylUmNiJ(gJK6OqAhL z{ZF&}PI!PcU_IJbw@#c#8U7YsR|U<+UV+j7>B|@Fz1iw)Abmo`h9pU6n_yd-YNxJr z$%<21Q1jllB?Ib!RB9`RMpjMBn!S`s3%W4Mw-z@rmcP;Up2ginVRZgzN&W$KWRA8} zKb8>Ua`i{p=IS&e*~U9v9D|>G;xEhC6TIs1#%#qZ5{+smxJN@xsjU8hgnBbebG5{* zA6SD?2>OBxi~@zY<|*c*p$;U@9{mSb7uK1GthfR35`w^(+JTNr66|rRVABVc0;6oATkw0+D-{CYhkV?{`LD9~x{ zwST@&UwhY-=FFI+KN4Fo6d>m|b_n@T3m_LdqN+*S`7pLVrqc3Po4}Koq+Ci*JSB!| zr?Xdvn0D*;4!Mki$4OkwPNh_Dv4$#Uc*_IFGjh_e_Ol#x4N|fMM`(SZ&7rVBD&&%O zzTSl(e{zg}MVkEa%sDd?BW3L@0C6!@%UADBmSZ=tm)uXLJVz$C0(g#KDPO;HLd6by zdlxHpt?LRn{jz{)VysX-251F=GFzp}LiLxPO1&x2{;`8P8s#nq!6E?N!Cp^#MO|36 z`3$^VaxyaW%k;?|m-Q%RG?Z~1xExUmFR&JX%npj0KtY2C^>7rwMdKpfmU zs@V<~vHLk^8yq{z+oOE2?(_+|#2BPJ!DTw)1BjESE-c*Kgs(1wo{*EfcORmt6e7oS zwx3ZxjXTQaIN-eFG;o+PDm?-0SVXE4B|v=(==zG!mtbJz&J4A!V>!X+#DN7lM=mp9 zT}4_N*B_6{2oNJ&7E}1&c6WCP#`pL4Wz^K-*B5;yeTd79ss6KHn*aGHw&P9A<^HX_ zUDi0~)4?t|n~=O~e+IsM3zlVPUea<)@uC%@-+cBwN+j{)_U4)suT)>i6`s9uy=B*C zbD|S;6iF}=X}&${d%pvC{XS6bd21CL-y&}Ux`tj@_QEy9d|t*aONdT8wcd0PZ{e({T*{)bHj71@2@JDsbAOiSqvt_6hqm`J)n!UVGSunN*run*wdITl z^1;J<;RE6uLth>=R~OI2rUzS33gaaHq8+QAbh;+Q!jYhH z8%8<$(m(M_8L=}ssMB*5I``7B!<2%kvUK4w%|GLcp@_lIy<0P z03<35$n;;0lT*5^JqG#&)Q*l;1FR^{z#08a*!>L<6S^M{@*;rz$H&)~gy$BlkFICC zy+A?@1*twB?W@{eH{3vbK!M_lfx8%FAS8iZpTnRB9}T2eiGivSn3{CK;J%zh2^Y{% zMJFoIsm{S1rH)JBv3zv059b3%vN3oKpeiH~W&v)EM8J_54W$DI?IrNP*n&&^1w-74 zHRhFmP`=q~#y_uLvmdQA;1q0+*pJ)f4yPAS`(v#vhQLA?oSx3A845F|=M6{|O0_yd z_is3hwOu2c4VyaknYtK5vnRw1Ls|BfEL|y1^*MI9|9l)W}LCd$* zKdYM-25Kc$X}2Zk1i2auJm*n?A-`6QM@Y`DUc`?T4#3z>S7)H548*UWos0Yn_YoT6Vl}j*yE}b{A7!xz zN_2V_mSDhe9~OJv1fq-^CFYZ?K*bcJ?YjnpeFAeGBuO!#8ZqD}Lxp<0h5Jna$=}80 zB`QA;_rMexKAO$diK&+B z7A=p}+-DHl1=w)JX|dPfEl+;Fd2B>PJLoM?`UMUS4wPXHlnoI{Nm6g$ej7n36zSW- zmICr&9NgSdpQt<{37M3Jfjdt4V(D(3{g$YiSzaoyY+wBa3{czsT^lk~C#MPxMkfi! zrmE^RS$>^{>Q3i<{H{Yo(0Wz~q5-E|JYB#2xof8%KEa=f;w)2>_^!akB)y#_DXYOw zd2(rr=I>My{N~uY)V|uIk|au6H)c<0uDO=0#Kt{dV=hxV6&?Dp@z{ZDMcU2_aS7~S zTAjXY(rAiFp<7rVb$My%cepuY)G#cfaT$AJLClK6BchfM~iw}6p!dkPsDpwp^u9ukOJpu3in)!t@lSiGoSEmbHv z7`;ddAeU6m`8>)UmB~st8)=sd4z{n#nor-0qfBvQ>ogq;k7I;|7U2iDo*{~}9g(%L z=rKA{k%S~Fuj~AN{iA}#sDUxQ**EsDquvHm;hmVa{hZa$ea4((8&qaBDQ;BE-?Mfo zjcU$s)SbngiPL4b$G zC>KC;T7a76kq5$;SW9QUFUlT1NAeA%VA07W52V~8P+2tXOvMdt;Gf*?Ghmf40M3k$ z!jx20PykY4AuIqK(g$1WJC#Hd8lenultOjG-N2LymLsvS?z%cZbaZrgp!&E>@O$67 zO&E3geP>5yj<05A+oeDumjFmLSPwWarGwbx`;t9<`J&8VG4mwkpG(tBhwHSSd#p0z zQ?8he+UR$kXC0ds<-4?tGE?I4DQrBSqnUg&nHdz$cB-zVfX!v=w1Z#Ag>9fJO6L|) zh%l}U&Z=ZSF(v)5jt^x{TiK(1{1JXVR#Q_WO63Nxg@Jv%EXG*jHSaYd)jiSsKu~j3 z6GM_;Iwm!&i0`kx@;--yhVMK6Iz8Xt93Htz%@QJAiMOY#_mv+qR#}QOIavv6K}d#8 z>ay@pOlpeKMkR)aZum@NHT=XRV8z2-J;%J;e6k5n%XM6#8S)xZo-_77GO&GvuJHK* zO~~&sCoX*zXqO5?^2|jpQtF@m4s%b69~!V2u{k0bUd8d4mVY9{R8S~`5bA!xlrk{e z+R;76=qS;?Y*S)1NGc~McVvwQQzTO?R2$b{jrMUbagZtst5IE>PgKN7U^RN968*b% zi0OKXt4KFh`}olZznCV8)(GC8`))hN{2d=`mJApnyf5X-?Hu_ zD@U^P@n8;aD{f@!T9QmPc_R6c`e#%K80;Ra)=;YPv6?$V0q+IlIy}j~+Z?MVTmEhr z?$7?_4Q@(A;_l_@g9G0BMD1i+^$6$Fd@1rbaS|3uHd#SC(%p32k;M>Mg`(1YNW+3( zWA-ya+lL}wHf2ILDm8|RYZlQJ;lj`;_%$zXcE}*S18yUCxG-nU9U5faHb(DOgAjVf zWUrZR_p|HZj5@Ifzokq_^<`p38SiwT<>}o`$|s*Y->%+adsypE!^0B|gl5E7Dht;e zWxGZxB9WmG{A6Q;5(9;cwXsB2rb2uB__@E4o=-5pR?2b@?FtP3Fy64SfDrt?ZmPY; zYBaeu`ShdBo3G&t|F|2vezawuR`&@4bMvmTUm>WR88d=UQEgr7vE_9szyi@nA7W#z zg?M4V+*_l4F*|0oFg1PRTuQ8{pLe+a4a!$3L?Si1c_r#(I_TCOcKM@J? zq|7hwcV)zFAz(ulE)6)S@6&XM?37`lTeV-l7MtAC@Lz`bbDB>vhbz&inv(9Nm_jXA zt!Q%W@acI|40+}pzYY4iBNysj9QBGa-Az-$y#uBMF+GGW>h%XKHSB6<>BMO4>I2wU zOIj-!)h*;p-CIHo1y|R0XRNFjnR@L%xaWO}_yR1`N&UX}g~w*zmZgc0)|AWNy{gLZ z#sepi_NNeB*i2BlH|vS$@0+!Lla`I`yNIFwcxVs`2;PG~4CU47sfS}r6Pf%?^5vhc z0)H*V6pHVPIR0Il)->cvh~tCMfo!}qP+OoP4#~*+fe92;j52~eR^n?OB`mul0CQXP z5Ndq=4$S+woVkp_0oyX?ZbLe@#+DF$zOhH=tEH4t&!)M9NOj>JJ42(qzhKt<1MeG2 zAqVA;gSN@~TPBIaqD}8FrC$ovHcDxRc%44TzZLee{AC;`=!A*pMLHL7GOf>dV`(+M z(HockIYlTM{QvlF)+D7#%x!sI>UZ9to8A>o_vVe~{dDtEN0aHBvvs)D&1Go$jZ6N+ znax!CyWie~3_KI}k9{E4)cnN-w}|{k@8wo`_sEO9N97X~Y3Ake2w}r#l8X(JrXf@V z(?h9ik{0oTk!#jsht;=REyj}PzxDe)5ZZxG;Fz5k;4*`U#gwUr-g$GElOOw_q4}5S zyKme>!uHu#Q}pX?H=09VZ-f|OyImWTOE-R)tYv4Mta%Bmak({xBWLxRt`?h4Otd6j zJU`yxK69YZ9He%bcs61jy53$o4{lx>$sK!TfY)-m|Gk*73Qu{j&ogLSE$ntn;rn)~ zQ5AXba{9Vy!Cb<}kog;-M~$`nF|T?IUOy%vklXZPjxK9H`8z1=^@O6Tjyh)}KS=KhcG6%` z4T?0mMm*RM;!fCGM7I9Eo$H=F_v~EoeaHo;inw;RxObeA*dJnD&2b^}{_<*pZMxb< z2D!S|?T|73`wei-CwYyzB}7tWJ3BAa@FuFKdj?Nk?H9TaF9}Io2oL>c^J7f9_3Pbn z$zOT75O{%!Mm6S|Qq3s6QWy|1b#r5dX61IoqqiSyXtWsT{uOpt&N1U)c*XzGUa^wD z+MLL(=Z3IgOvZSv?hmInrMf?Pm7QmtKZqC_l_H%8w_%*m{eE@(1s?wQA!YSv6M2Etr#r z6~5Ry(wHP*J5-4bk+IKOF9BApTZV{j1eJgEKJ0Hw*G7tdHOx&eQWA(-^+0N z_U1$VUy4JlS1LPc46FjVRo&6sc5y{l9A>u_E2@YAgv_GKMuQ7+I!1>rhfqSZM`;-5 zqDos`68$_}`yxaE)pq(@EIW=h;T<*n>J%lZ={1dT>PVNT#V2nii!eS*6&(l3n!R!Hs*Gx!co8cZvQ*vnGPWsbgJv zMQEX!FRSVe zs=L+GjPI{vRF(hzvE8E2LrT3-B%$z=-R9#bAL?_^252w(autA((uxi%8nDAjDcfBew-!r1Gjr?k>Mx5O@Y;GCe!KQ6*428B z{amVA=MzCv(q?Hj>3Z3{+(p_)-y(hFGg2g0?L{ADKauNc_}#jt$uMT28K_Sd@upFL z_WW?ZHq=1l22$ab~zam*9874254S;P9-$(;6b3Kc=P+5L$b-pP@TzdU1mYBBmdMMJ4Z3Gsq8 zo9~E(n|^m>DpzA}o@7f8kZ|}{teSos+6d+Oi_AUa+8q7nEs1-_##!}6_XO;$@~n{C z{>`N^Ga<7In>tp~Xr-d2+s4b`)pc;@p&K_^@O}`r?Pcm5q}k4_UrRIIYj@hiSUk2o zaE4+#t?KFXnk$dpD-Ar)XO}E7Auz4@%SqhjN1xTC7uD#-2P+g4LFUZfS)5s~{c`OErcZobKU4*%-D zrPMKop}qr6s`L-kK82>~&j6{?J>=v#Dty;Qao8S}r}1RiafYE9UrcmVGkJ1-fg|Q8xshkwF87g>bK)Yl@Yk3T^5659 zMVGkE|0)+p)@!!`0q62N<(*A4+T_*tL#fU4?K?cXd>^l4AGlT*NTnz_JgA5*olYW` zuKm#7Q-aXv#vvWg>sVMNmLRv+6loT`mdP4ns9HLfTbH=6DP9CI%}HJbzpXhw^}TnI zur+^VhL*uELgR%gaV%IbH@E+tekw^>%bb+)D_qGA87-p@~62ap@5mYl}CA|(rw$MJ+h*=8K$d{tYY zzw;R;Lp95wyVhBD-i(Pdf-xf2_DK-`Az79xei&}z`(`;?tNyZ7fuzm&5SCHDX_0el zo7V1P#BH3?7o^bu(x{D0VZ(jpG9NPTFhs>KQ%N`erizE*c5atX`Buy*u~$=qi=7#N zXWc%tr1%w`7w0lgTdmDz9Ur-%%=lE^a0Gb4PL>VfNzG?M^GhQe5bc<1dNN`QymsD6ZD=&pjQe2+b7M>?HW5FnFkS zrQzY+nWhj2g>Tc^&$Q2q)))>{MweHi4Sp8 zf`M7WwgtLag=vcWFb~8Rei@IFJ6_FYwfI_USy8#r)2#QS;T{{cgrA3ja-8y@epXaI9*s|& z`R2f!m^X*RzM%Q08j2aP&TPt zU2}K}IQ#K7&9@!jYV}je`#o8*W-W`~m`TaUxHk#iH`Df2CeKUu7O=O^gv>SEa!De2 ze)1ygqhMKu>&LKFNg)xZ|2VaO*iq37!9z`AL-b}qZMR3i>1*(OK(_yRgPH?NMo_-TjIkA#25G zn|qBvk;QQ_8y`lo9ulAK-1_evvjr&W;NHKkbF`}DA6rR4UmRX77TUapA_u61sL=e! zzYEO{@!oA(URe&jym`u-wVg8E@U1IZAGRceY4SSf;cWAanD@fp0WwbBWvu#lWL4!+Yq%PBywVRxxUXiaG3#T`BamQl79Us|02_0Nw%~88 zxD@CeD3#9@MWfj_=^CaCj*5P99yKX4+tS(3D;y(=S~A%a=a<(;F9mY&*PuD*ysW+o zq^IUScFL0`8%2B_M~mT`<}Kl$MIYde*ohSQVl47se~gtPB@zjKejC2X%u9F}yvVPJ zAOU4Njj<*3d{ir_<#&a{-urJpcjbOa_ogM%Yjrx}os_A;#?yiae36L~%{&ghWi!*! z&e65SIDfQG1AMC-lt=5Lrfn}YzT4;?BS-Cj{$jT$cdCik4;g&-`H#r^x(}Qgl3dnr z_`f=>#aK*x`CTYg{>pnSv63r!If?)LaEvy7tQY51o9LZ4DbY!{DyC=l0l5^?;Mey( zquqtQdziF!4%Zy2F5)zkY1Whk9RdT_j$tdN$hGRZ`CN1-r=X(?ghH(VrgAgp!q<^+ zOhU80c^=bvN3Tj#+sbQ%ZwAsrg{53XfccRD?*7Zg@ z5j={d3d_4qoF9u|g=Kgo-`dS!D`8j)R$t@BGVv$yeL>E zJt*xJ$0H#O2}^y-&CGkmUk`5Dw;DmkA{&9_f6TaOzgeaIrNJaer(SW@N&mv)5C z_p;q)A0CBG=8hI6cgL^~6mByjg{ox<4d80kJ1^y77W9V(jNgew#BhGzY&)53 zjC-nxoX3;{#QkS_l>$>$y!?0EQs!1?K zl7v>ht}ltNMxa}m4Z1pB%wv1xj=@tO!CfB#KDLfY2b~*#aci4D-zp%2kk0&#R>xgm z2K``x10dc|8V1vsDNvm&sGEdfcQyu(@l5!<#gyA4@ae?UUmd~MJYEmkX^J4WlT+h- zcVdg?rt;(q@7$7)26IpquI*+rw{O5$onTAb=PeQTn%!IWjLl63v-V26y({g8rU*wa z|DRtuY50|O3}ON#sFlTm*c&K{mjgu=Bl_&@ilPnWlVu(&ijg5JLvZ!^<%{K?8U@DH zv<|v>b_d?mP~~xHALBkraK=onYy*ZGZQPa^L;l3B&2B|Ew&Jzw%pEBV1dzx7UK1Do5!)!g> z32I9OVh)OZb^iQE{{6cp$obQg8#nE|ZpChqj+C?2Sx&v_2Y1@<1SUo7us=N#9l9%d zCSNVDsU|oTAfz14GN9jI2H~Sk;-Q(fOvl>F-8>BEdHHu^xk{l(+jqF< z>r62G>(U|jt;)ySYD|K^1sLem==Pp3a(X1C(elOD*#`JT#P}V&5$4M{Tncwl*TMB@M^FHe3Du`)eQ& zaN3ku^kdn$(2M@HDghldyN^#s(qjFoOtqHYd;KAah4et0vIYBIy0=gr@dt4n_}n+P z=>7HNU;eoylx;sI_g^xF z`06i#Rg;scFBWHWWLTrgTb(6VIB_TQ)(yOFWN0z3A)A*g8i+4+eh$_>6^D&A#p>@z z$_uc)+!(}-M}Fli$EDEW_EBjo+Cpq-T~CxXK0~R6`upDRBg30N$+uD64!YVxd@bwWmKZN=5urd$E#bcL4C}Q z(whT5wb0v;*qP-o&nBuq9DC0RWg8)K@nk$3;ydJHeio5*;XS#6DVvenCI!}a6%8G_ zhMoi)4*9hQU2ZCGH1V{QEPAihTnkov!7*`@8$N^#eji+ZiDR-Go z!yrx=GoM<%{u$RC&mh8d9IOwIsmQ))umm2-jTW_e0zZQ?hZ$KI2%c z18`1vNcW_Hb|34>g?X+bcU+0#%$u@p_CbGM87}+fW6du8}di4(MFm+Z<_>}A> zZGx;BhNZ|=Om`u|Vv^GhLA7z7$MWeXynm(WPOah7KHpuQYE<>hKcVW8qRCb zLyY_mr{cl5)Q;`1s>7WPRljtwCWTwmNJ%wZj_*D8d?O2DT}Srp~sq}U$NP)2>!LSMP2wa zFmL;4s%*P*YiDaDjp$}}^88fckqX$4%ath%jxLgg6TAVIw2XcXW82u=`m&B!n})F2 zv{BdO>^`^D-$q)p&d!n#SB1n$GJJ~V7ZNuO!mUiocuAqNec}z5R-VhS`|*OrLs`9z ziKzLWH@+5Z<3>$cQFHQ$*ZjnS;LCN+Tcv%dfxI5iqE*9(y_>kXziD@H^7==fB!~aD z|6XIeNOUy?Ij=A{inY>8mYs%_Ytv+2rZi$Jine5Ym-laG^VPv}KY!+4hq0i7{bc`W zZZ)OgJIvMXvTtoBMNi*Fl*o95LIr_(HzSp@&xu4TC)s2$fw#uiTvlDNC1;yWM#+J2 zB4P%qTVr87cS$U~zde}8={Z^MOId!HLUyqwBNFAzJPzsKfhOV6jIZP!jumBNdF;+y zlBtj1E84IaOI9*F=E=(2*?#TMxHp#~uRFq7=w4}$cdOYHN}k}|<-eperngT4a!!XXK9E_yTGUm++f;G z0>hAcUMoog`}K#o?6#i*AK9Bd8wl{huA=DQ5Xom9sqyngvM|b(hd0b~t69w3utG`H zH}#R8M=Vq|m&IWti00`#j+jAP(PV<_Q_JJ%wXI`V9?OHY2d#EcA0I`!)ORZpqd7vi(6{g&7WOGZDaw>6`A-a~H z9{yINMzoTEAb^F zA)TDn;d_5CvM$2o*!}LIQp7>Yz~1zz?nnWT``9LiJ$z{T!W3p$MQ>QXzpzbv_QJS7ke(Nr+Q|P2a($zR~P+;+wwqA12X@|H| zOwP+a6QX&2(ao7RDlTjk)E$NN{aNLXUOeg{YXD3e!{!{#HpR!G8MBW4+ybzT1=zSw z?(e*;Hdnca?SD=br6$kU^Qf{nIt<>v%_QO?Y4myqzYpG+1}zTiZr28H8nAIDo_29= z(Irl@+^+@YSk};V7Zv8v(!Q~t3A9U)hdqQb^&RWR%RZ+6Zh6^Vb2`e&y<82wLk#1f z`!93!{*cZzyx)R9QheWtP-4Cs9L!|5Ny_J#tuRrPM&XwUK$?5mGAzWq*Ee+;5L~mW_R2Zd1 zwiPx1@LSh+hGIG79c(Vf+!?Oc6zZCv59c3es(bb+(&-=G96Z(bsHi_!cGbtVEQ~*+ z9qvv$vquzuDpRRvOCR$9CkkY9<&)$Ca`rxFWBk2gRz@_)!vM)f3Vs*JA{#J<>GGH= zU?1(-=`LWKD8#UHS;W21Bce7>@$m)%VaQ_l}QVD@ru=wy2Y`L?49O zoQo9{hXM=!%4S{2=+?=Md2lt#yP6Os(UMvfnPf($e{P9GXu0)|bmS|DgZqPLW(lF<saLtKaIJ4GY+EVgxjb5KX=w#El~97e(*>r~W5^zVQ>56r%U zS5EnPU^@}Lh(|s1jzO4Ffteg-agd@zE+%)EV%1jsH=4jXvACVFR+ZW370yFoR9rDqO3sfD$`|8g<>Jg48S1Q&(QCZs>;?QE|5?W)iiK!h=lYf zCl`ojZg5t8Ls;<$bX-R~Gs$VgVC*tKB?dO6-JL}H=-AC>|3XgLX5H64lS+G1g40V% z_e&H{?Yrpw!>E}}x|m7X%>Q{%X&M4R_1FzFJ{-!C4r*%5EzYI^D`G_%k>kx)2&2kg zE!Cg0f`G*UuG$!`IIeRd1sfOw{YzF__jzZqqjd;utdUslkK6g2UH8$3^ZGvG&W)5Z zvY74~5J4SM@3|FP4eQH@;&U<0zh?d59{iP+FuxD0e-%)xLB3VVCf&!zz zwoh$Xu~4sf@sWA_#sqoa%HjG=mZu9T`zCI)0Y$v^U%~3@uT5aIMs<$Uw7=ff-5XXCJcBPG1L@n&Z~Fy$0mz2Ey`{S#a#T z1)=6G=$Vo8x!4!PzS6?YAs!m5@$|)Bx^9J~ijbGPrhh2W?ovc&eeLgLDL~Vs)G#+c zhY>t93+iItq^acM;`)E1(?3jNua*OZ@3G=XQ{194uQkb`Y~zSHOXG?Do4Dq!2M}KJ zEC4<3Uag_WKq_V^1bHo4AQMz{-5Y;Y8~Yl#3Rv#WPC`Yxy#|opx1f*j?B_eN&9quZ`Lw}ZhJY|(B+*HA>>7ljx>rn?k_hmCn)x;{B@UJ{#W6XAt3qN zn#%ASad311AtXoi`x{oK?W9hj(4AE!;RT?DF^vb$KRM{rC35~L5X1R8<;UBp*WnH( zn$!ip9SB^G053c9*olHPdae+NqHS06*^Bb}fto`j*pFHJl>*6)88DsnZT$b!O6-}S zNe_{KyxJ|zfKnz8R+M24Z1#~BvVH>hJ6s5^-W?LaUi+?!#cJSj*sfH&KcTOt~N9!APAiyM_gKM zz0tMQmtLF!%{MZb!s>PY76I2%b+A3EC3oEYG#%5^wb!{)9GG@1i5>W z+>*}*bKZJHasVOP)U3#=&9+~nH)wps>`m1oO7OAR@S217{GmXNYBVKygc`bs$U&2*s=s;|sU;wSjyy?9f5&sE9!vc0 z3GsbvW!vE`rG3fa*>L2^VkoGO+Gy2-ZI>f#J#u~M#C&B8 zU#oC3b3x|))!lSWrFrvf=CE?WiWA%jeyEXfYuLHjRbadeQD+FrnUbFC`{x88=smSZ zek8%>$}2V!q-X$5a=G?joOW0fG>ual$~0VO^M|nl;4>_eOH4}Zd-J8)g4fENAue6^ zol}7H#J6O+WZa-<|z5O%+a(&!)ALWx}=&+kQpW_8^EH(i=k4}0$OYSF#XU0d}HrF;Fobo$71 zL%4yUe$-^iff6bVsP9GP`a?z3URV1zsxc>MMr7a*EjeMAoCur#86MNs=V|}^icj^; zAJWXXC7%&_v6d;ve5D3^blMkEAiTSonOIL~KFRd_@QP|^u{AG=9%pNI#h7kb�Ho z+vF?F*eSZQ^N~?nO6?MvWfY&Ig`D?V@u$+0bA5#40k*Y!y)W#xFWO^sy0;LI{Lj_j2({XldEqPH%NYgUDh-A)B0_rLky|?72G@@9RJ4IOzOy#Qw zwRc2j8y0Ia9|td)g#dvg^>-xp7Qvu;sJ&kM^?Lw{mmA)a03l|YnNPU$w%qsCCo-C8 z;z>4W{}MEsf11MqJ>%K_tQW?K&4)?-;S#p9ej8Nt&b^$<+UKYth^j|e$APntlX`XX&-t~~(<0C7= ztWz$aS}qV5WTEbPI>zZ#+2b{5=5#QG*DTNQ@|)G*3-nELb#Jzq{Z*hO1%GiY@;I3Q zdYJotFs{mgm+UOL&CH42*yDxmY7;HXs96sm7>^>_`-jkQOh`K`P(UXIe ziT){v1I;P}WLv}Agf%3Vzo|Vlm_q74vQ!Yosr)8=iU@@F-~8^{nvnB_-?w;Hh8M$j z_qEd<{?qiJnRCgxynm-D_Wui7=Hs}&(C5dM+0bhyOc-31HuawIoL3>{oyaHv5}Fs)rfXPJXnfwb$jYVr0bAN`(bYuRh6<{}q5&{$c&1$L_LpqH2qTmjyk$C?%_}Iq2 zdY4jnw5Q4b%FZRsvI61Z6_CzwUzIM?aQx{^&0M~Oe<1oUpNXNfVO6Ib>XrjnH{)_4wh%YVE7!V;oR7C>ryfF4cP94u zdv_Hv;{=yq|vNT z?l}1pUaGj@j-fh{FyiMHL{Bz8Xv@5deD|-)yki0fc9dA^E4;B!KTGq7 zx{sQ7zx~SWvZ>IUH_G)3SUc>vI3v3!ItZ^~QQe$ul|904n&s=bf;*YA7{3!Xm&lfPNwk8B%w)@mY&=LR9m!61Jv95I}FZ||6oXldZ29}Iu}lycXv|+LLm`U zpOm&67k`rpa4GO(P*~ZWf0|WY?7Mk&?k)}Zq8k0gZX>-rvH!Gd__Y5uZtOwzyrv*l zlPB2m)|uL8*kL|DRek|gVT1AYb0qA4da`F-KOa5NregD{sdN!o3ZG>D=i900XxvcR zKJcCIFI|fZS3+d?t@YjHLT0ajf!j*y=iyZ4NlI}v9?h!~;bw{xdL8oLA$dNfs09yo z6+b3(-N`sE%;w!89cw$>c#J`F%*Z-x@W9rLM{KtbW@X8mDJuUmkAx<3pex{VSW!Ry zr#VAibyXZCL;+uet*rE;XzUB{q!^U2Fpy>A2ESmQTM+c=k?`CV^sE}Y0dPgi+wutf zz%HReZzrLqv5pu1iX15 ztHZ(qH%V>T^TeD|)ZRfd7e?b#!#!4LVPqepbCx*;R7<$vR!fh4l0tA0TVpLhI%RGg zdWq3vm;?Vk?8l1xfA`_+FE;N;jQJ0ymNRf9a9@}ARBb8b>SS4$ud>e=LD(K)p*q^L)5tcJAUGc&xQ zW)j&S(mmEC6Vhky7(Z%WerD~b;i3;%s(*$}O%&>RV`@}gg5z&-dFlrNU@RK>UL02D zA#|TwqGMF@8C<-oOizZ+c!V5baT22=~YeF<}3Rmlb<>fEk3hEOhieAG|#jss(Agq`vv3UgIC;iqHBPGLY zqpYx|64m2ka)lN`iTzm#$e-T^JfsdXcGAj{F!@uQP3`kq21wRm7R>WWy@yzi19Y3u zC_Y2!-uc{%jSL~?O=)K^wRiU<+7H_9*qlwca^+tkdb(Nsm!9X_*(nR^1r0snX_`*_ zsG!K%sLwq1;Mo0sF0=eApPKQlHWqoa8WsXh-dS>x9f1dMXf5-}QvPAFkldar z`!*JhSV$JZR|ba+_V6(6SdJ_ctTR%`R6Vo|sWg3XR4e`TpXM8krDa>Odzz-I{7h#?Wj89gb+|Y*h}uR~c5bFx zPf+J6Pdf<@o|YBpQWJr`89oZ`_`* zITdL-uIlzYf#s83iGckJrpv!a^>Ht2kVa$G{KMH;40U ztbiQlX&Dw`CHVILVF~)OAcT0B|AO>CEr1%0(VCYy1XbuPG(LWETOZ{Fj^X>(ugc?j zZSu^LObH60r{(nf0v85{R;(GuqttXi7*^}#MP6_<5B@xl{`*Jl6;OVvRII=H%iVsl zTkEX$K~XmTy{$qB8=MsFjYHC8)6Bp?)OAl^slL-Dv-6E~oo20{QoT2j!5I!+wXaQ) za{o^GT;4;Q?zgzfwg)?&OwF{@Q!Z`N-55xFlO_L7jXJWQ zpsoXBNa~Xl$?#^pJoV?l3{qbe=<@zpZ9L+dUt2hQgq|Q#5=nMalV{eKkbV6nEK@^zI|qqZq-rz~GnjX4_o4NPcY z`ghL+p8sPa@|Ig^?QZSng@NLIY>XvuzFEFV_g_*G{schn<0r&`l8rYN?%*$JpOhuF zJi`FC?N%P{oms4CHTMo@2SK8~R z6B0PTo87FM?wS+lW5DDVUI@VTAFQFzri2412hGtB5KBt#$2;g7mVyI78XU`2rtew* zKNv#7dCI>Sg7F5HM=nqXp+V>7!$XPp^e(HOV4EnjHES;NIBbnn(4nC~1Tda$`1-|s z|AZ@IM*HReFUsCBD9+`116?4vy9S3OXz)O=;4TS^OK^9G#oZwU2rdBvixb>kgS!WJ zcfT*^oL|-b@V_7KR&5nU?ZUh>J>5@tKQoWziR7BbT{VDyfsz&Z&e)H3T*7T3?*l8o zlANMVwZiUNs63=-o;>rqgm%e-egevIe_aeK#Cph_NRmy@?m%X=vBM-^Ubz{Xo)PtE zPNH$qYxfT%GMa=oI3=}5%Lv&yvVJM>3rRv&`LK+*zS0DiU&fe@)aV#%%udEq`wvFw zzeg{hPg^`6faoc9TMp~~+1N>56ds@>lusL8+(faCE~0or0A9bWY=%F(ea_1J3RYg9 zPxz4b4M7;l9G0I}U0q)DDgs%Juib7vl@)FKnC2VmwJ1__ded z532iaekn7$xkvI+W2tR@j&d}g3q-g-e@qE*3NJ~1zWF~{*fw=XITsFT^uTBYejSkS zJ{#o6UTx3BbMyMc@+Ap|A_8O)7VCMt}OHhDLgwk#iMFtb&Z5tMhXY za`$rB;{Ti5`-M%)D_LB*iUEdO1WrD3mwe-@MSYHY)^2tndfg?BZ?ecBUoA3dR2aS2 z`8qjG;g_V%8l<|r{a*2s_JgqC)ye?-Z)^Ui7~__VU@Y>y7PX|J>p}~CA|?O7;#aF@ z#%-?FiksEY{=TpM1?N0Ks}F$*{@oy9__tWsrhl-k!~~}teUzqJqH-fX@c0a`gx$(D zL3!pWgIQGkU1C9irBJM>$d}F>J>B9*|7;~6$qUIrFSc(W-g3*e% zvXZKbvz`b;;u>S@5pk*}wJjyk&L6!Yq>Im2{w))##y}uG=gd1u?{nV8EJBq#p2m_i zJYm0P@ywKvht=-O@RkQIN6V6eB|)$mU?33TY`Af#v+q45^v2Vn1^F%BGZg=W9I^7U zG?}wu?5mmhUsLp?GY7CGb9&Dw^h*wYp0nrwH95#uuT?^2Q)PCp4AcgxX%>1WgFeo= z@+WxmalJ6eqfWlIsMinzMY$(jLXGIZj0IG{L}veXZd zFJ)A^>G^vLJ8RZmo~ro(EK#if@V7J(hJT)JCpeao^)JA;0~+EVEOdzu0xn zSF9~;g5~I!Hns!}=y`f(N>zsIaZlQ~Oz8Mb82_zbvuq zs#F~XF%+$%e@Q`sz}E4`Nk>graM;TKN*SgKYN&5$fB}$$&ex|0i$BJSuhe0YNJ@44 zm;IkCJD!W%{~x6g9?*{~Io4oj%f8%W@Ac-n1;F_~4llky1(*|#)KCBuDE40Gh|0YZ zYH>hh9h#MT2y_RCq?AwzaX}x!jJL=Q5F5SlbTFb-TI&JrY|hx5=??452R1?n%)VVG znNb#iPm$9Sm|X;AN$^&IHfXCQ>&RZMIyfHj1v3#Akhl0!4U#cbD;G*|nAD3+(UC}w?rR}H_Gr!N$J%O4TuJ2?d8U4u8aHf#fNQ!F+mh!tF z#3KL${b>p9+we#A`V|=mD6S8OllhKf?^_?(M_Y*exl0?JehSvwbS@4P5W2U$XY(rY zs0+Y-cwWVxgLuma_jbvps4VEniB7l;RI)G)7#@??+3aS|?*q21wE+$S$$+;`2Yx*+ zknRJ91j(lbcuXiYW{YF|KcdI{9vXO3)89DUj6h|(Bs6y>n0{E`>sWKM1U3Zbq5^E* zn$g}2psgO?>*QGE6Y|A$=qNr{+bp-};d+!X|`T8nLHW{zwdeTzzgEwbLdT)tSBu1{MoKm)I81nibfL&Mi>#s7s zJPqyW@B23N!}-{3ovABW++}B8Nz)}N(uJ4>yARpA_#Dj!Op{QR8+k!JAb+F+MyqYC z0#35HF;a=m)12C5P-l!jPI`CofD_fjoaE}L%0*0E?CyvxxqhXHTg&i2ddUbewu^a+ z#0&fU0>yAdnsE@99Y|Rs01C%0Ij&gmj_||K*{T#$HhpjD@7?k!X2}Ef*8l}1WoN2O zq|+j3g(%^loc!aY&05hx#io-+a&BdB;=Ha7RG-MrAX&7wr~uN0siqIK;w^0Dz-n{F z77kM}(mbo2lv$rWhbOOQzilE1{rX>x1~Oa#ut|h0xQdLaxW_w=C4pmJwgf=g=Y?_C zY#3m!!wQPu&=Coxo=rdT{1$~C#?4;Bm;H(4krWY&a92{F^$X66c7$vFazFtsKzo@= zlrb}ro6S=NScb(#kj9smf!1y`52yNPsKsXul+ivY{me54kt)?zH?;KIAqBnIzr{LW z$B5Q>u1_tM5AHe{uM`~Fv@uNs5x1)p5k~~ zlU+DpCdDSA=-C8P8nr#@SCS1p6mT6=%sgV~B)&{`*-XxQ{(OfN{uaD9{P3i%EUUxae&ESNbQBa)Kfct{^*85h^;J7saazD=T^fYJGo6YG#K9UDsi|6 zmTQ%a#q{oc7)oE^(qp}9woGi8$J~A7cW$TNE>`;jN;x6Sfr=3cLnJ8@-aS z!TJe>h6(Q*v?w!(XRth;4Y0+!AJIv)6cOX^;RU5tjfJ>WCgk@?rwx~+cPu1T# za#yw(NAX@T{8hSvenH=S*EB2Vn_5+BlI=UwME%>X4~mKbX>C~%F9So{94;S-+z$&a zRUxrUb|xT%W1*(DmOzjNx{Be#X7t>)6}LoFNiB{sGH@(}aO`0J;sF~}o=6l`u^Ij2=})YR3)7l~6MwlwqZ0W@1!Ms~TmZG;6A!_j-&Dt>I;$ zeRdfbMK`fOLN7k!b<-^1;b6yo7=zOP>|puX@#d7__a1&q0uH7k1AS3tv^B~gf|UvI z3}7zg7_}j2L*(LC99YMQ6G2A|co}_ALj><84;Nhf41)Hx~L2VbS%LfE0>#Uu!p z9yPp7OZlTdLFT4Jow?&a-QLW|#bcmUp*szl9Q^>X* zHue<%lr^KUVJ#}B#WF0uWrx-WKFMS5pl?0AU$+2~B7O?E49Y?Sa80I1Zv4A$+&o}k zhO?0_L$NILZU4CI3wWwzMyO6erV&@imX`BtC%S7(+}OtUK6Ea?V_PXn;-YV6q-#v> z>CTuys&9a&EIa~+5IfFZMH=7DtW@M4kr7;wjsjdn#2?3td}*kZI-Y27-S*>@R*J9Y zt0~)1=Cpd+(I&^EjXBA?&6xJ%G*;`@u5nd3-#Z1bRcy_ z1bdWb^1VeOjHvR_|K=Qtxr2HK68^Sy!Xgn*QCUQ1n8KaGaK-rB4=$y4NT=V;ADLIwIHP_N;%Mv z!Oo6j1M3(hs3WL`Cf7YNN_S|jgCW%%VEIb%p4Nz`GHu~EpQac0lyjl|1@HU%TK*L8 z;goc-*W5)|Fxogszbu?h-nCy5kM=PjKKuTXiSqXd`I;M-5C89(t&(lK(?wQl=wTIu zD9Chp1jpQPIayaCoy}xpK9WxeZB#HswhGg9{uL)mL1kWFsKg)XA{@mkim_Xw>7>yn z_<151Z=xxa;+>b>L|t1{3f9TN6CVSEH~Vir9r3Y16R@al_s4m)F{Un{jUjw{F|jF) zp<)Xhfsd#uhhj4;`f`%m0+x6r)wYpUou(BAJq~H}9^-=kwECm12tQYvl}|7+gdZB* zirtKO9fmhRpojhY^< zIP9F$ZeiUP?K{^JKPLhe%5OAcUEZUon%H(!;P)!kYUun4&GftOSP?y%)9&TMZOsUs zY%ZL=xa;`_oQVX!aB_OyJ{MqGVSXh+re%-D7K7OVes)a&C^m5yti6XiE-UP0H#R~> z(h|?TcQku`Pe^7&zPZ-F%GWCJ`=si8L$gcRwo2Ig{5k=jiS z38{=n%+-qtwC|6cclcWvArDL+98b`=n_37YRmn^3noVpJ%>W#+Ll7;IK_Q_+{I5&c zaLK@c%20qjJupmkj}NnhYGZmqBeSP&LSdm6n1%s#V=h}(`erKS(@l~bix<1J5U=wj zspYsp^4iM|=9T5>S{l6&gD+T@UpX7(cES})H3Ano>OjGM|dy< zk=;|l4Wnbb8*c4b0|P3)EH?Ae@htXsaPGf^ zlP!HlsyB6zEj^B@o3g*zPiLMzJ2JUJxLdnDAUsi>O*I}NVAK$;m0mj8<%%G)th>K1 z{IPJ@3i6FrkU!bH)ce|I2>1T!XV{O4b%96^YtH$YBj4`=zp&45zH(YEAe|$6f6aXM zO$_y>MR+q!CTai4zW$Qb)0>EM0bD!!r~bS%bii;FxX@TkEWcm~41Sqa%sF1&A@Y%f zwy+nk#e=iJG7|f>zspyq%3v_`EA!ndk=3&qDGqlR9LNRI^g1Mw+}Ggub2INr1rd{v z88HiOzUXIW_2)Cbi{xp-)qoEBy|OPm-ddP{j-RPmy7WLGD=#0$U_qfjW6y29hUGV< z6mjkhxS*4nv{tN3C7&wdIfPyxUY4qwyOKQ`XGQ2w6sD&bx2|XGW3&40-vUH8UBZ&1 zQ|<<)wAS(>s-FpH`!-8s?rAXWeY$u{XYLwQn}W^6qOBr27Ddu{?GS(!15Ar)n2bN5 z>6|dGK?py)Sp)`zuE`}lkb$QyRlf3bR>%xx8m5}&a>p}y#lF2n2~HIuy!kU z>CRO_V2Qg^r$8qQe^g{*VE28x3v-h6391i@_Nh?^KKDBEG=g6EU4u$9J|01!#39M* z2L{Xrmdxwrk?XbSm1E&SKjye50pe4i9vT~Yvhef6DJ4lCzVQ7my^x;Qr0a`ujyv9x zt_QbPF+a%(BvLS`$1=gw9?dGO_@Az z=R}CddNG51m-G%gmRgqsRN*?}9-{I=t9M7M}7L$J)J`Lz~i9wiIiM9KPxeTa*oCRgG1uDOcrfAE2G zO5iONc?NE?Q^eM1bQlz~9(K0nSO;=^w!*=la_Rv)zl&9gm^Q;g?gYg-21g=K<4OBz z_Y)g_qlJNL+6#>*2w6gnb>o>o*DLBh`IU?YuRRz*><_o=uw1jPW}viW<8;>Y9ShmE zKMxCPkPb=1vsp2^3tT1jy^n9-{$UZb_h!TJ_Q6`RISzhJaU^WksswTDOdYO_5?%Zb>K7{|!oT9@kp3@@1DMpTbQ^bx}c$Vo}Oo*!gt|9bP zbl&2RWF~KftNvMNbe*}`$J%Jo9(%82!hgaH;f~kaKl@Yc&sHv*GS);ROeVp!)vK1K zzvr4fywK`HL(_gHK1k-~nM5#eCv#tl1b}8m zGf+>r$I9h9Wgn$bNF=zs>S@gTWQP4q`htVwl60@pFPJBu=R{4%iY3q4Gg0>`=&PQ_ zt2G{&-VQK#tkQ*?_Wt(yy4n_0j-WY4y*FNC?J2krU{mWXTD#D&+BZ%Ee9WQAKRYKT6BES|)!u9urr7()^$!InQ2J-= zxC1YTcrhKmFR#jTK|SP72+AJ5a@@RQ-0v@N$JP1Y&V%y7iXJ1omF< z`7{(d?mJkF)wl3mjDbkv1cDO~uHK$;oW<6$VartbKdu~g&bpR~E2(=;+G=|$*CWsTX+e;0;11#%4d0Rc$!0vYbW(QA`oL9hIiy4MbY?ecsf<}!7I{LodIHSo z7djV<^LN7V2m%~PoVUn`;q0!Vg}@)v z+ihaSG|Mk-Afz$XF>DQKArM2ev8E7u63LtScsEuc*5ptO)eOTzsYF9*?7Gt*6(3%o z4tnwGMtq94Fa>W>jOH${A?_Ye%MDzqwYEjQr;)}0Bq^El78B!2>Lb=d$nZ3tRSlhg z47JjGJ71b}s&K5(vu3`ses}#QOWBAfw>N)%ptw^aIH zU^$|_9Z8XRKW?)UU3Gw?eUJWPBK}ny3GZJ~5P3X7_~7}M zsfFmC_l)BlCe(W6=23pdaN{8E$5}7lBx*?r>M7yLL{(-icBXRvW((_e_KZMtY&lEz zhD`Ow-SK;eE2%}WgO>yZ!8@g?^LZd^dmr8-gQ1Q!G1lhwfm5xnB) zC1G$e*IALf!+y#x~2obVsRpW4Yy6 zY4|p6`BNGPa9n9m5nKb~+c30eutIpB#UPVB`es`WBT-%_-~Dm#jhg;td=UdbA@qUd z;PFjyOs~J+GaQIa8-j&MNbw4Epj`*|k?;v!>gqH}?moMNEeMvOv*S&LOK>{6$r`bt zSkixCo_uu|3MEX+kQC4(1!ZMSZ8->ushk@VYQ~zzuWmE0kLh1orxLwnNq7?Qveenh zu+r5#$C6p{=CYy;({kFa(`l(r=KhjbLYk|VS}XUctpZP}c6tOtsC+An#HfaXF1&ck z*?h{pDv!{VK!zGL)Wk@wR(pMl|7D$LM2c$S%%s(%zK>Kq*i*Wf#oV+T)18K?AKCv> zQ$zJ5Q4_mrL^v5BpK0MMdy*yA&ZjdlDQk?gT+9m0GP+g3T$=N%oK4!c1fcI)ttABq zsUuL$XRn20(ezlpoBgF$yQ98(hV3uO;n2D&l@TcYxj$V+36^oew`o%3?k%*?jGvO# z|7F0ySP*qSwR9UlWrkXmd8m>yRl1x0Wdp{+t{`7z;b75Z;nMSCnVAeMF0OAw1@bIk z8<yhORr(T$d{7cues!EeiTG@bS7F6_)HL-iYG^OOAP(k2aAz~X zfSMhls18VLSPqX18N@o#{aRiz_*Fr54<3m)GG$leRT%^3k^U?ccn_4yGS`=t+I}gJ z3jdoK5MD-j+WVjcq`K8%ohf$a!pNsmEVCd{+vGM1Alf#)IJiT z8R!aUo(SMFyfc^;*L0Hr&`^+e_Syfq0NB{>Y1pRDFrx=F9E$CJxgzAIa$=hs_=9iX zB0MQnWq;uyc7BX>&&qi_OyP9QU=t=KR-LYF3J%Nsl(#fIdESX;;rK3V+$U44=L=vB zqpCh@7wMf~lD1u>KHAg$sE zC}jwT)o^(vT!eO1Mg%c%lMnV@-N524s8k`JI=hompD8fV;xRHzlmFKLQjG4UrGbU7 z@0S_T%9Z+>k8OjRnt=|h1qog@+~hKX;RSy@-G1k9BSg)DDwms#-ii0(W{W7oCoTIH zRgDiN>_?$w8Vxyi0fsEostc}WEZJkH=j0f?*l$SXCLj*@7cvryfTIfku%F#^7~pFe zqL$x^37$o$aF*G&PwBv7&2%I<4zElyc)mTo&tWd*Bs5Oj~r+b|7X`k%+@PPSS4*Qa{EwQLr1Ox zdk`#hv;;^hN{qT^b{F!M6ovD6{yNwkRWgW<5h*s2$TF$3=dyyQn-p`V%*62jHI(V4Jj0yOFwNR_ga>7zA>~zOPEh6xj~|1OI>OyZypu7=r8;aI*lh5^=pZ zjVOnr&6fh9zvG1EVPhDPUBk2N?3}oTdpJl#H$Sn!!XI{I1Ts-mqkO_cEgI70t1M@C_=9Wdtgmi-2I z4+fwzwi$`k08*Ljz14>IAs_B-pAZN6mNG-^!ID0lXy4J3;fPsTxKPJpo_*)OAby?a zN*54FCkeYA{k?Y!^Ydf>&Vo!w4=##AZegf^sc4c3J3>ty_J-|&8yes;8BD#wxJu4a zhThSXBV8Ip#_Obc<_}69{BdgpTG(&;g`K<6^s>F z;V6JlXzx>aGFxt=b`q!$kyl>jID#{Jc>nqmt^~ivTxEnJKBP+YdOAvgJ0wM6pL{yl z^Y6Q)5QQi>y2xYwPoIOs(cj>aNh#ZHMEYmwv)@VfA)QPKWh8HV13^^|Aw$$yrQGN< zIR3Yn`Q=Y36eGA{qJUf!qff(LOBPSO+IO(*;@bpK)P|(sTVu2Gp7MyaE{=a|tJaZ= z9q(JEx)X8XXGzpTsZ#-m#7s0dM-fMr@GLc0KFGhTahqbjhp5SR-A+ZGE-L@a8Pl>; z;Jp?Ys8yrWE>6$*%FvqHa+V*_<+J4_f=Zec&LOX3c9y;6|}+hUtNY9pvZB<-+au* zB8?%WEK?G<%Zglm4L3?RNvNsAv4i7yUeucH6O#WSG0iVe%V4ZYpK{s9>}{|UIy@+? zzb@2sfze2z5Gu7WO!V_ZFkW6tryG6+OQ&-UQ&!HTJHwc;rsp6 z*3+JtMi7YCU3`|!6z8tWdW}dA?8iHJz1Eq4XeaoQ;_)MA_ab5lWsZ65&!KeN5{Xok zPoauK9pQ(_p;o#SsHV}vz!C;-W#yI61}C5M81!#_z{8sxrE3Bdv?dYPbEpHiZw6Pw z+@8)$UPi3|@Bn~G4LzDg18PJ;n|1cWq>Dl|Me7;djn>#_J|0-!04iU6Bp&~LZqL?A zB*g!bM94a-cbwTC$@Vkmy-*YQoZMn5AbUwO>vD1=_GWjU46b znd?~DdR^}CB)6DIY7YUKoN_sxZhMb9eB(E~uByZ_E|p-W6YJ2Is!S170~|lv=?+&k z7C0Z8RzcSg^izs-%;o5Yx7h`VkYhWg@L~q--`tFoq-OK*Q~H(KQCx94|8;DO15@SF zV83Q*@mafKtYmkTJ$1UsT>@XhRaN!0f^T1E>9&5A*t^}9nl;&v2t$DTP1_wQ>yMTvN8`iIjPnUw%)gK4Y&6dkCwn}DWEHkvC;hr1uRWoH5o*!DyX_s4U zv9<(1;Ei})5#~2FlQgchiko3*E^AaGzELnW*l*;?Hn)3i#rr+{+GjbW$g?^yvrODE z+H-wae_)}g;^wwuXQ33iMbi^6UiTjTG*iw`oN`m@#t8+KHAAzq79#)VeUl_nMr$=I z>?0)TD7#Of4(dtNqT_=Wvos0^icMDw&8+QSk1~f#Z%|tvo@$d$^tH)m=i`d6xc}~N zjf|rDTDg2ZH~do^`#ewK%?Qj(0o|*q8QTBjM7lWLUh4x1IBF|x-W>xtTNv?vubAaN zQ#{TtDnW-f_}}nfpTPKOVrFT0&tE`jP-k=!>4tWH_22ky1u+?y{JB3D>dad`5ZVIc!_zhCpSC*7IGVHp=qtv2kpCuog|b%!~w;+ zag&yLwc_5dPsBM_U~N~Pk!hXng4L}=Ar=JrE%%5JR>mn=zNCQlUmPH9l!gZ{)}acg z`{q9pLBO>VFLSLt!rGfC`Yv>D6tB}WP~T`I@uXHWaLbL87Y>duq4`ae|AS77Y*`=9ugXK|Zr#?HA0$H5rpap(0al^OH<54kEQM2`~N5*bv5Fu50LM>y8IOZvUl^UwjS2Mt7 z(N_6u2y-W{KK5o2Q^Dy=@T)>O;~CE)4xRl10urHp!pOY|6g>fVF6tD7)mEXy?gp>^ z;0x=^inbMtcGl;54C@qlb3LVw9fsze=pnni!NMD_ZRvv#-UQFL9v{i|%CQ@_6eOBK zMO$Rz420r70@wB?whC`^J&Ed~W zB?YYt*aPX$6~|Susj!@-jQUc!k$kH7x8hs;1sd+N{#uewV!jkUhh>L*Q}#BDYB?zb z4^aBH*ss!G`F{>$;#$MC#SXCitg$P;;Y$ZAW$=f{ZgU4Uz;VOD@?le(TFs5@jYIRd zuYz);gifbnxl_?=7adS5D=SZWDSRm3zc2r@f4(&&3f$>FpfgjJ`H1F=NCCkDRGqpx zM)(8R412!c!VA*Sz}YsPBD{!i!~(tFg1~NSs zT%e(5_33$9^?*+!6ulSaRLzPiI-kJM?TwEU#-;OabLMcQ-hwvMTbA?FunA(KsYd)W zl!%It+~~4;L=h+Smkd@#YTK&l6+B1U$ynwKV;nmx*@fBbDtTFb!NNXza-m^efj^`k zdZ#H$@*%;8k|PFUBkzZJU64>EtYqd+E}T4AL`dg+u1YC(LN1CQ*w@`VzQ?_`pV2B( zZjHuYN38jj)%s-d!B#&`r4L6*sO=I94E7T(WZ>?di%r0Mbqfc#(6_UAafZlwR95~O zj8C8;l8GUYUg$}L<Z|YJ23Pkh@Y3cwyO0yak1nA+eRnO>lws_s207|gm z>#qQ%RY^c5P1k)B|I4%pQ10PM-P_RBA(+$czuY+yMl-h+SMUp`aMkYWC3V2^8NMy= zo;$US$6vrH-~Ec>a2|2;G;(jZp@LvTHi`hFO2LU*8lb2~A)#M^E*a1AccfW*v-G<5 zE=$+f5)7xfJV~m^Jns(|whKhZj>XM;yUX7~R2;RVh29r`^f!JFo;+-8#ieCL~Ftpcf83f9A%Q!;`Ju8wE3_Qh=ALy-?8M=P#@Wn;=TP8YU%dFqtNPk7EIJo5#w+_D#(Tzvb*RmCCL+ zN+evU+P~xTCr`h|V8H9IJ{5;vRzaU+JlWTWL{uCras~&z0MRKoz`2}GYuV~=^u@7U zJ5I>+Q1S7Rur{5*(#xm3qM)G2&CUG^I3Ks2><}zsV)MVn00pB@E6nOx6&uL4vJW3e zc^1Q?-k`jo@=V))@dbK8qOpF8p}h!euC2YB>h>Ey>yzCSK($DfECmtK;8u1f$2&{l z{q33F%-=Z78N;KI`}YNoju3vs4=(@)SN92CHlx%T}6GEAED1p02Xr zrH+xTP$;k2?P)!m+g*I?`?X}_59t{Z(?fguW9u}d4YWJsDTY97fSkp|x$Oj8hRXMg z8J*wVJSC4IG0d?aJXPpayg4HC5V&LJ3xr=!mi8Kmq~JYtOdDYlcG0D>@i9zTXDSbP zZbZ+Hg5^5S=D7YFz52TRsPd*1W;vHI(&yT-zW2?VZuNS%|{f)?%H%-!)^i(xGIS zDvF?^0*|1hYjwQbTnDI7*)Levt^yK3zmA@kJ@%Q=NH~!)A!o>d{GMRb!`a};>1q4< zh>&fY6|i&#HKVxP@ORQg6H^pp)(_=ZQi2$?gjC$jh&oy~25vwqSn%`FMmSgp|NJuU z^fYqy#;xbXp)6{LIJooNmi4^;kz!f1bcu8VVGY1tgI-VdT6b%}Tb1{z%976z$Ql{F z9wXkAvYc0lCuTjJg`SXAaat=OX7US)%#ihpi+|Nt5eq82z11w=ImK3Ynj=aY0i-l^ zz|qVOn6lX#udq#ABoTu}tjt7*#|M}m2b;mJ$OYr4v(agBx6jyn+&FTb#DSeEAX2%E1Oms1|o zKD$0TBs%NM=@CeuCJf)PTq!O`eo9JexIb!)i;oZR_lJFvt#$)c2^p12>Gma zg*j;93}*W%6uS*iR_zYQ3&HOl5*PJWEn5Vuw`bQ1l2>S$IVfUQX5n6I zA$WT5aVdOaD`)=HUit+k9G|k-FEHEMn6r#Enf#@2moqOH6pqlpD}~AWoPSs;)XYE_)=#e{a9i^&;W;ycm^-|k* zWeth~V!U=oi}f;k6apTc-jA1RN%t#&?)KB&KI_X?_yGv++B6<=R{>VgQG<+(jD(3v zxK|6PbN&$Y;@cWZ4FS|1sVFJiHhCNF$+3WWxCGz%s@fW>?Os}oKv`x!=kK2a0`Rrp=#c%MS~4<%d#!zJzy129M1#-RcJu|M zhW;&3P$fZ>AzDG*ufCnn;CICe210-Oso5{|ax92-c-F9>W0TY_P3_ zmA>2UC5|r(Gi0o=p4BCTsVms<_QqI`yrzfXs~7-0P%bx5yTxebl>GwUq11FS_6;NY zKll~R5AKZ1A+!N?DBxAl@O5;P3Q!exwR{Hj1z|vdvM?-b%MEn~*fqBAgkK-CKKWh>_$993^L!=*4AIYUK%~G+5|qS zy#t~S)l1ar2nVoKIZZYw)B3-t6DEX-t?UjII6Okoe{q0s`h->LBA>9?A0SGV@>C9xK>QYgmD&Rr8CFny0nytafOQvVXb2hzHK+MmO7XIwz|-tS ze^O&+gWTB3W$+UN>S;acuFGNE*7F~9Qw6;|f@UDMn@M$#R|`Wa)WD)meLliykl`wz zF%&uGBbw0(%7HC=PT4&B@xp1m!_Us3cn{zK2=j}x0A3a-a}&6fh62}%0DBzY+uM6n z_x@_ZHZCb?8>t>pO(XAR{eu5kTN%3#{5Av%1$0mXv}XPRP@%)7zdc5p?6*9MVqW>^ z0hjCmmGy5%&t+xZPXgV4M*9y=|M?e9$N~R6$hbW7ZA_`4li4(S(J5JZci|`$K-Hq2jHY6qVA59Q#>Rcb6f&^FDx#wT&>T~<= zi|;*Z|GqRdmIwa$My#N|P&^B`PqEFNj*Jkl(xf^ALt0#~A)$Ry->kcdmV`PcCMczSIu!)buv%Zt!$B7 z*9C;aXuHwy!_?&8e;)JCmEt~})DAujc>2zL@G`TPwpj=r>ft%+Sp8HXz;peLT*Yg@ zcH*f~DM9S11#d?(D_x3o_4L&ivx|2no8|8n2gj{kbtW^sq)5eY+b9>})nQ7b37Hv) zptqqTPWwioea%cIu&djar!U3KzCmK;ggl-d*sQr;X93t)wGF-gqHn{+GRK!a6)PAV z6qTt08TFVK;Hix^B``b%3K36+yCDqu?Qf0#)>LnpwrZV7vY403(#HZN`$|dKs^w%Q zwCYY>T#n=Y7~8 zt+-dy^i|&Q^HXN9`-CV%7iB9Zmwp;TLZ64I@tlCIf{o4l^P;^-Lkr7e{nEqjKETBm z^Q6hki>z!bN?c5ZBSA22W_e9Xv5%%vn>XLbN)gfN4vt%6<&tDwHkLC0UphgyBLmty zb|i^|!r8D26d=pQiqzaBf2n8OU`Gmcw4re|u>>M;Q&tFggZs zRa~mIjz$*zuM|rFswXICKy%Yqp^wqJG*^*-8C08}Io#z*gh%ioDEv2-{o31@Ej z--q{&brxRD2awF{qSk36&q7UwzYO_vfeB_il$p@r#1ybN=+$W&$O1}9zD_pi0W z__Mu??1?znKL^$|EwSvrNS{!4sz9O;_j;Ar*Tk$;I(c|y^W$XmRnyoF`bOS1_}=>k z)f0v1v>A+f5L$2Yzvd*i7XZ};>!2}PT)8dgkA7%M5)-SAnFH(GLF+8ed9^)i-%Yx% z9769rt4#hmE|6Rw^z(*0!%>t7qlSbL|^M3sTyw-;E zg&<`5-cIfKG9$wW{FtJFyfK7%C1B5L&7ErA9{FZ^JM8q&+9AFVkRirgT}zc9-dyRv znm4n-SHnieF-nI?-lvLtgAoAMc92L@ zKCj0^+M`e`M$a^j|4>%6^BEtX*-$gb{#VZj*PFCozYIeoZ_tdoOhKSM?jlBrj;;s! zve^TIiuk1&*9*;T%EqJi{iGJYu^2ev7h`AX!Jro08(?X-?d+TO_{jkdJYCIU7D@$_ z+0Pn-4(zrs^0OrT&RHM0 z*Lj>8OGsf;5!PynTiaRtFgu?F(10t=yN>Nlj)^6Ousf;W-_*Aw9pVBBQ(VJG{Qak9 zj5j>#PVSF>@^GhOd$`7eu10g%5I|zZ!J0-=9N^d;6;zcmjx>K`!ZFAIaBym4mVRUK z2iFa{k&;K7H-Y}SiNEB4oQt}GK* zP+BR4DG4itA zkGF_U2aq^-$6qm0Xd;7<%#L2P z_~=WXqgkREABcdeU=@z(l@1CG2QEkYUOa#a7=Vb+L{JjY#;6t_z@S!keTCJ&Ac|^> zA^jnv;?Kix5wVRDmA5IM!WiDsZHt4K;<5FAnCZFP!7=>T%q1zBHWhH zAPHD4DiQ*wgynVUd{{}VN3e-Xs=M)MLj1RY^;R;;V^2>t5~pg48S=OpF#~{i7|l?$ z&KN_(!G>!nL=>Szar{DS1^soUA?#Hp z6J6SJwqNq!FJIk6ml8^e?urUCuIZJXPYJZOeINin;0QVDI(~CvG#|giKSrKdFY@lQ z9l$>8?FG2|2{l6_eXJ7%pwbj){1^7$uTDX6d)fljA6$T!d# zxlgLEEtler{d#R+eb@#$Rlvg*c~*PpNR`9w-T_o|WyKnp^N2>d1m3e5zXxNvyr`KI+405<-e zz~sngJ?})>$0KX!2agv=E;dyNwLF44abB@b*ZD`9JeB==2>Q{rfv=^+q+uU{+$+EV zaF84@E36U4iyAv%6IwVP;zznFh~T#YCmNJ%hRJLjfX$0?G)o$FA} zA{i=YCad?N0uNaRl)l&-!0*3EnMd6>En&U(=;b?-eoXgf?dJ zwtOyDxn%=gw3PAO$sGs$yW|%s)2O6+OX@9t)7R?^x-+EnAYovO87_vbmv1F= zeW+O@I%C}s1%u@lIOSbehM^O67&aO%sNZVn^`~%)7_7d;f_>vj71!=lTgy%^$E`7p zDXt-%_W63Qiwk=fmoL6~SBB!EfkZVykak*Y4jziV&8eTl4#>nD52Z4>V1MVP+^iDdy#A{WqPrqNz#zblfDQ`A$1MYb#!o_fOA^^)> z-l%?trAt>h;)!{{YZ(Y&Fz9e(rVjP8zrxaE%LQSymKQ&7okNEfeX@-yh~vG}UKO-q z#(Ni(pP_+VIWf@vL=VJw{4(7(FU;)Yi{!Rm|pCf8I2w7+1qZ}MyZ z!KGzLut&$|GaFr$QEdBSlq9ezusDAJ~bkd#jkwZ&d>{ zSC&NX2fP-h+MN8s)~~F!A|dps*n-w}YYJZnRBpM-68Q{))Q?c;c;f+N`gI`g zx7}&%$Hm3^5j<*s1=g+nc4(-NKY7=TbdjFOS>)hTbhpJPrHCwc;3{^{&*>R6z!97G zdrQhhkIQCcTJC6m@~~sqel1e!NxmN0mskX^gL6xyLB4mZm#?SfqUIMyFf5lF>uE0P zu{>c2M|56+`dZjiu5FHpdI8+>jfr;)VB$vt8ml1C3(NA6d@qFA*2YiKSFs+Y)-hxx zo3aQ5vb#H;m~FP>p?~~<4D5%{iH{0Dl^Z39&N{FFP)T|0FF?D%ZAyA4u!#AEN1=O* zhnwc!iCa@|ibTGV$N7$2XE@nAy}mZDQqdHSVKi4Zo*ith@vAb{i*F%M@TI_O@ehjU zLuzvHC;|v6i{i=`wyvz<_yE%hA6KYIc7I#hkD=%r&;hTM^$Gex{Vv56oNXjgI_d*- zZLCqcr}NnmnR_zX?Ck70=Y{K~>rCWu=K(IVAuxYTnYuJ`9>E|YP%yN{ZJ@iV-Kc54 zy&25#9&*Pj2{3KLW1_x|Ep-Tk89Z5*3<_7j;fLq!uL_Zpdv(+aG_NnNUS4K2x_wLh zwL8<=IJyXLC&b{Zcp4^rch}WKy??PhvJQot(rV7`__Ar)x_o5-C3-VrEjb>`?DkwS z?Bwxu{X-zh_*EP74hqmb=kqzg;W+KbraFYNtsX~m=)X8=s892v@1N#7UFZFUuiW0| zQ0~1S+J;>$UkTki+63T51PFj5>wT#Lo1Mnkk?P&=W{viBze8cO93zPJ_JDoU(r2U7 zfkV=wNn2-zqO-}}jKldAE5VP9Riq>xK|GB(e|CW8aW1t8^_{k;||DC#oQ1FKWP5?_tM9Ew?aaR z!;$uI#nxICDNVR}3yUWGSrCuUqkO<9n+SE|a9uXg9z}#oWr9JWmZ%Lxdm(cyf`(j( zEn4sg41$8WMV@AWcUp8K z2hxF4aQ={Um#nO4o5Ck6q-M?z!m5!^RJfzT<^`5? z_kK@W24>jb?-sHrq3$Y=5vAau{lC6!ivCgQq`GLW;D^`GK11#tuN%mB{8II~4oKpj zH5VnD{7Yf}-z)^j^trHo7W;d)>mY;o%z`!r!EBN*=FOAS()w)Pp$Wn_1)tU=g2fv zZ0&dyrRK-or%|2l)16b#qQl08UylA$w(R-Wo zh~_{bbvelOuDx@jQM8KHSiUuYxW|cE)|wnzGNQ~HW23$1tq>N)#tdL2TXqhAc3a`@ zfX~rmbWX=)w)RS+T*NT)d8kiCGsn%1Tz_^F9TlcYh7$9zsFeN)q^Ups?tm2DK0eu*QBnMa z56K{{B+~YGUi|FB^8aSDb+YE436ErsWI6{2iIyEaZ2Z z_N@8J<0%s=(*KRqzS9%KdKWCF{X7xFyFNnBaAtS>*3&3pFe0ts8eU{2CMjvC09?13 z_vxEO%@_JAmA<_NsuKl9@fzq*o?2r`83qvSb({XIroJs+9{KCLx+1hrps)s%P{_Wg z>O}(e&aS9m3Q(QDFi9_8BqT_yv^^8}SMEM2Wsgg#)&#%HOfxw>xhx!S<8EemwT~@^w*|}vqV0k+eDmI5c}DqWr2_5CEp7CtLN)#o5zW(`T69l@1TIDELHy}2<4yg zy2k0!Qsgppg`*M*a7HLo6jo#QuWAt~aSaAsgY>d$fK>6@TSxFCAu(fAs64!b+ozfS z6k|nygZ}dMS%dvjK>e=luBNV?FIr%D>sT711N(x1Gtj=Q$Dcz(`L3+N1e_geV}S`U ziJ~+;+GZcPphT;dA9L1)p2PM|t};S5wy+DA$I*!=zt|uon4d(kUfr&bVEg7Q6J{9VO+Nsn^^`hRMIH zpvx)hpgIH}H%9bKk_E(fK~%1)#%K5{lr?N{!ee7PmgHp>x9btLn34--k&a* zckOqy+@uO$XSMr7zka_zhY2yct=Y)@?RPPiysYwA-rkp)%^N)M|L+qGZIPTG4jO?^9++}Qx#a3aw zF_N!V#D?Qfd~vITm30t`_^3BUEWFxCplz(L*&8o1tWm(2b*%VPS2 zbe+|XOcz}}5|i~}q)?(E>G++6#h>W4RP5DNobm!%(x)m=8qU5WRzP%g?0ZHYh-=K< zNRXrL53trSl$DP{TvRcY;~%l_OS7}yWK&PPX1zm1L=!!E#MS{H zBn)NVEQ8# z-uu*ux`;8L3 zl?^sK4rYSVm~jB1v(SupyM0y8`I|%VgNAZ%k}|Sg1m1&au50KNNr!)1MqgGC9jaP~ z>F;J{)Pb)V`u+XG#z-#C(?dT!3CJfW`aiE+ha0E?xu~doQ+adVw*8S`hS@2OJwNH}R4AUM6-j)PBijO! zGC$_Gh9~$f6z1vlRS$19YWGzqHYh?krh?p|i*#KfO#iCK_6COSfIAmKm`Dh{O8SYI z4x(hXG(Jr##Cf=YBTx47&aZtfDhlP}c1%?E#>6rSI=sH`EM-Q-|Fuv5J>Iw3wOl$&1K;~~X|>`$5Ggr`L`7w< zFt)x}3Z~pPiX@rE6WBlR4ASd}C~){p@VW>)aIA`cD*+@KF8XGTHZqgWDLkC$9gTeaQXlF` zfp_nncluSVOI$UYuiP)j#m{q(EPa_jY72jRV)mJn^yKTvMp<9#knSD!s=Vulh!>iS zX+H+=EM9VzdZXqN+jR6oQ6|^fAYX zuM!TQr!wph-n_nMI31(ARR%vP7 zBK;isxQu<;^Sjn6xP!$h1g)4*gQz~!Oj%>iWbA&ahW-ghCMc+cg!8DXIkKMGjxFc; z$arrHx%zyAfuyb!0_-RUnQ@J|#OXENW&|Ta7cNlv3>=i8rv$oxwW-M5iBk7M;AXt1 z_MKx`%Gl^TsBxvU7(B1Q;EYzP8Pp3db281{Rh^eZ}g|6BqO zwWAp$l<{&m(?W+m3r@PwC!(hXRz{zD0e}rbd+h1qAN@`G5;-QD0~AyVg8KK&A7Q^j z=n#ccf@AhssBT4(#r+yZw~_!60{-i~7d`*Zcq2cg=47)k`L+W-ph%k1Uj-J6=L7Ga zkLjojjbmd_>q~>kGR#JaosAZ#a#n>r$XOi(#J3rf6~11>mwOZy#<59_(VLSt-X#u;IbNbR{;{l)U#ujr`KEtsUvJM$8Vj<~VYem_ZZ4F_HY z!?%OTuosY3{hq~_@sYE#wd`-jYC5WxJH{7q+UZIP|2Oew&Wa>@a;VVyxXZ1q+{a0W zAN?)4YC{sVXcVgNGj9-xiQzXEJ{+R}g#O@As242J|L}cX>!yO)OlCyHTyAhxa)G75Q-Vj;H5ozZ=pa8 zsLouQ=9-9UYu;_Rwoys^^VioX2YTnR9VCtgglRvEw&0;U-_Q7{2Qz<%cmoPSaQoy6 z3|<5mqbafhGZfNbdtJxX22IBrIJmyfJ?}f%oq;KujDG-K*qDG|sV!vv{Fj0+`ezCZ z#9UbMs9XV`RF<3geb4^5yA1dPVveX6R z56_tF=2LqLmh4DFzS;Y01AJEJ4fAUp=3^)0hIWU=`)0i*RLS~7>vI484Q+1&4y+(q zk~x@yH+<%X@#PO)^s7U3>B3j|#>R!-z=qOT>tD66)(G54RE}2Viw)^f4|I3#O8igV zPoE=|SmS^)5Rw%ZqCK8ucZuDnKkS4%mOI9+4NX~Y9K@>?EGWM>K+O^9?52Sh_PA|8gd5zfu*#M62Jxm(Bv*Uadm0K$*->&+4 zI+?mJ<_tR`K8w`~^vdWIH1rM*k;t=cv(3{=xu8GsqwDjCM;(`(V5Y2S9n{Z>L_P5! zD<>Q1Nnp0OI%${^N z|A$HOZOD|J9C9^1=Iu?4HT>SjsAue%Ndba$7Fs9h#uJ zpf0lhs(>tbgKpkGJn{2s-Qf2AF=v7m&B1(mw+N#`Q!XTH?*8vJOxHYBih*d9M zzdptx#4sX8Y+&iI$A;jk6Y~|_{XRkmQeeTAx)E+T z;-e#~mYGNc3lgGzqpgD(&tRy0kJ=0L9E$Zq1p7Mg9*Nh2g=C z*T5d_2$pDARk2CeC*$#y2D2y4&ABVE8xneb9cobsNni}hnC{3n9WoPf6$10G8Xk%D z7;gR^tK1n5o2yhwZw2UjIo$3&@00aK(yixyQ$#a5dNFc{7frewa~!SV{`+DRLU?&8 zD|~oCnQB95F*eG7IojxJV5qHr5!EsFH2SK+X8Az)xP%=1yka-wh8bS8aqWZgOVom2VN#!7_sqNuQQFa=^Tg$C)00V-$tY2RR09Q`tG zM2+h(W;ZlWH%zn>v~|O7+{cN@qNuD)HlGubwOCT+C6&l900GEvvt*f=FPlh9*@Jp!Ays;Yf3!#P;Ft&>;J zRUIl;Ma{Snb0AMw$ORqQ{!3i4t#+DW5nKK^CGlrOOAK~?a~Mad9`E!uKPEre4=ibr zh!g(BuqGbqzoYY z^8PJgf>Y{X8=Nim-0g$9S)qsphSD;M(vteoC%dA`Epo%*@;Gf$C$(+$!QruOUwf(- zR2bRl7RRfhj$9FKWkb44LQTcz>frMqz)@8W=nnud54n6UYoqmtD|?iiV8Syz2mBsZ1szRFS>)0evg zHg}Q&YRm=Bf#ZDaKd<3vXtimN>QPN514MQbCi7$V|6eaa6IZJ~OC~E78_V1SmO0tI>fq z{WX&Ni*rUfwsy5@BI+~TK_DKqa@CH*bN90p5_k!Lnf-Rh&6Rnk3>TQR8qY^({mfhc z8nBK<%SR1WrGX-iMwI_VEa$A(!sag#=S__B)`OvAeQ6o3aNl4zK#N+>^Rvw6D395m zu;XKmr5EFJyb+ZUM?_Oug7cL>`<hmPeL)uXM2ACT%92NkoFZ- zv8vVe_@VW)4IO=|!&JVJhh0RNH61Cr=3Na=@YOpxBN(n$&OF@dz=giPUiVrzfUj#P z(0?WK5L35k`u? zOO^3{g~xMla7MGZuO-@$JnhD5VSeD&IF33`k!PQ@2w|P^UGi+2!TrXq*WD#E zK>;t6)UvFe#n0N64Uawpx|S7!y_!q2cDc{9bbU!9Ikd_0-_fWua4M>V0~)6uPC4bSFwkygQ_-X<`b@- z`+qX7lPO-sF~T>Z>`q(x;XyJM{0!eT$(c6)Q?w$cW^~@hVV;RMgB5=SUkx99`n2|S zq|#uH6sb^KI>Ci&gEMGen%owex`o^Pr5}4mV4JZaVD(|QqD66>7v+`j8~&by#n=(G zyOUvhmC@O7^`AM`qs)f#6mA@-Os$?euRl4L_i#Hqdrm3-4{GGV4GfW9uYVE}lE`5x zb}jMRXkTLu%XYtJ#ob@^3CEsIM%KsLllcb;xoA&YtF9+_aD?mKGCB?+=*?3+4UzxN zN|FLeb0_%Ux0ImGU9$A646JA3%hZ1qVo1(<)mJ(iwfC(r?g1c#;&>pbk#6^444wJz z&i9{IyrsI`&rV;43(?r5%ydBm7U~hKyDgbFegPjY zqYgvi{H{(50z;!FEUiV?$ zHR-`i-#|lKSYY3AV8d}Eb##3DWLNy8kzfxUvSeUa@;NyOqFWCBIg17!>VI)b_Oj4T$YRk%AdNQ|B?k{{zhkHd#DoY0ce`p?Z;U8+a{WN`C8piECi2tP+a&!;!Eb&VT~^j z$oU{L?(L)~ld^m~Zjp<=NlEDSmZJqmBWFwj_G~@-EAq+s&6t0L!gsNgE_zv%FAr*I zb-Z8wKaxJs8hW%&5Z(YrY)#G^0Z!+p(|^pKfAsA2tibaa!*J;iy)v`;UlKa9t3ChYV-5rC~&0mi|=Dx4mK!-^=f0Pe962*l`U~*4x%$f<`_{CMF-8+C#yUR z0`CW)rW-JwUV%i9b}CrKFD>HHh2oKDI%uUQWlix75vjLKUk>TF;ga!hof`QRYnuV^ zROVcZk`w7ttZ4S4`$9`=m*dY|A;`$>I=0^1k&PLiR2j`$T2ja;zW-w_V3c}|l=tuP z%|?_e(R{A)(J~b`;t-anSr=l2iYhO%Ef9Ph+y!2qWo6U_8pM+uuG8+0*$nUf8L%V% z6d0xZk|fCQOl!(mW0Q3{aY(0Bq%Lu(4Byr(C}oVJ&kFkpM_tzOTt4)!)g!lLuGI$S z(0afKdRtu@3LCma%Ex4JGIRzUnX)Tp8xA%TQWN-V2bQKm=i1r7A`;Dl3w?^oWlXUj ztRKXHh1c%?M4)#!1e*qmvrMfLv$<%b%UuJ=DEDKv{jp6R3k_S-#GRvu)yWMmXIY}U z=}|ZIzwB;S^2{^v-R(nxAa~}-rnnjzKU00xx#YYv?aKS#uc9pcFR@%UC!|foAG1qQ z^4B48XWtuq9>|9KR?8S_;ky^z(fgeoy8gx=Q58(pnRsP{cT`!k(qkOuPyGJ+9dxEC zvGKV1O`uGKfUX1uPRyC0r>NZgct1qvS#XDO8= zCq0&t4bCtm^ytKz5n07{&(qE`NpI!rwKm6}txvmnvaxtmXS;gZlhS6rY8qLlSW3f= zW>t}bTn;m%Mq%spdsTY@^bRjBr$l+fLl%qeJWR1B!~uqe-My*~3{U5&#v*5fk(Gck z6V?m(b>HhA?rCk@{4!ig5V%Y}klTblDaa{uZu=%Ah&MYgcXMnevM5WO@1b&BNXd!j zhsFu*C+u_^t{Lr$5?HY%%lnAZ=k9%ZKYj&Z%SjejTKu1@V|07`f5Uh6d)_M^9N+Cm zvvdMQoU>ru$O>*NUc9{QMW02LKL)lZUWTNp^K>>BT+fX#+*)$KL74i8TJtCT-i?5tm9#b`i>I+B`gKly*h;lj1z`wf8IHOM^2{nb!bo^90@c zIZswWW3~Xy(j3E_m30JsA$AvlucvceD0*Jn!q>ss98spYY%G9Sh!vwPP}i2_1PK|D znzbD8dVEM=Qe8ZWAKJVK(+B*`B-=__NMdoA#N*XgQJ)$;o$T z+MkIi2+Y>J-l`Inn9xEv&D^YN^DU;zWb)DCVi=)Y#nDG+*KX%eQrWbKV=}?TrF#V zJPaNixmlbYh9MT{;&wh$MVf#~+AjEfpoXLKmo6H3msAnci^URis}`G4z)I$#|9$uA zL6uxq@_!MTraf+2taQGv2EOP|3eq?j13q82ktEc;l4wg& z+lcXj*wI$9^<1Bdb34g!;TrWo%O^=uSL8Ji*!)()s4J>2=uXOOYk;@Vodn^~&PF-~ zt8|i5y9Wr@{3*dpEGonPe*27c>BVE`1{f}gl93dLyJHkLFfEpB+hOA11mP>|F63&T zL%-SC^#gI-LJXd)7R8phN~O#LM123eH}NiFts~!`Ojbj>WFf2BJ=+QeguZB42d3c{ zSJi?11J|@Iq&a!15Xx@7>Mhh1w6Ur2{I`V97_Uk6h8%y^y=;Kw;2T;E7F}kqq)Ca~!hEdwS;%}K*4z-)b)||(oeOOPP#ho7M zKSDoTb1_ys`Dv)zY-{Qe3rKmG$hFvzzf;OV)4X6^N`R_^47u);i{A)=UG{1)dB( z*%{8I{P9E;e_BliJ0mbiSC0g&cBkGiY6U)CbF#d5>dPA~`dg+-0h}MB7qjH%GrOK} z#2Xw{S*?+I?4GhesGp!4iQ>M!IT1|nSxeDRA|IscrHv3)M4h>tUuFKsWD3YNePmOm ziA-#GZ~8`g#Mo_IeWe2)PwDAssR0}8!(ZH%qE8=Wwl(l}FS%;lvs0n2 z=WD!u+Z%(KWSH3t0s^4A?fmK>JBg+N(;RL98vVSDK|alutQRZ~Ku-=4`-f@FK!&Ab z*C|D!rXA=3#qt>IcFoxD#&HfjqfEd> zoH1)KoGCq$AOrg$ZectQGWf7rK0nbC^%Z=Zx!}nk5wh>5^H<=fbc6d%6f4F-32v@D zMkssJFXUyjhTfu`m;(oi*cDvUUUB1`(a6CMh{47vo>)fo3ORH}#0{8wVC=b>$Z%=S z=-=w(jq~UXw!=yM`d|7uza;rrD;d|BY5Z)Hq;5fyx82|U`)CrZQh_xkGE;^GXo=1r zma&t^#_i;T`+rziY8(thbz3^y$CrPP{cEqB*@)wgc2GX2F8+})k#R@f?tM5WiIL~D zYq8hk98=mOK6CI6Gb-a-~Y7p8h%{^3wM z=@6vp)6ox3Fjk(ywS4i9OxJA^Txt1S;@~Bd8WVX8?)&ae9kW&;t}tKvuNC$Ud6b$S zOII$vkc|<+|6#qbXVFfZpL5#-kr_D+ZzT8V(yZ5?%G6RI&SK|7cf3@X9GWmCtL5nC ze3x0?ovIM)i`I|MGv*|c+QEVqtA58t71})N%0+iM^Lw-|S-JFNq2$dQFlj_egEKmq zsFL$k?XjAExk`W6jn7f1zACYE)6-F#9ur6LdvKuaE3ysJ7?m$R)F_LrL{o(Ju~l(9 z3-Tg$5?sgv$HDNCW{<@7`5~cPbI$%sds!8V%td{E zeUABc!}!=(IXr~&8OXN~Fe~}47&r#)lXhAb&UA?m`iAUV2{bgY+iM9-U+j z&F=gjVZYc}58&l>Wqkl|y3qa~n`A6+dft7@NUO1~r81rr1eUoPS z1r6q#kpuVTPUk~+jV&2Qve0{G)uVGFWmXRon0cojBBB0nsU&Vm zBN7~aG(`_0roIY2Qfk0zhH4LNVqqe_ud{Z;Ie)HCfMU@mJDmP?2n6nBOk8G{hI>xD zCcbUP>NqIeQmKa74|{CtM8XX^cIp*X*bxl{2Hs3>q8P-oA~6-m7#y&AFwGxIZbYo3 z!257}y7}ZcNw!xhEO#=iK_&5gf3rv?$XD&dmYu9j+KqDEcy%u2=R9TSV_HYXqnUI~6ipNYnLPN_;E$)Kd)^ zH}nbk-^T<;E8mLA&V`pzZ@W#$ez22P;piW-!$B`%rHh2BYlpSv-;{1y$GVK9CQ@U7 zB^1LbMFC^9)Pru3kb~PAVh<0*ZNr0Azm7MqF#fmaBCYRz{t_FzAL|Rg#}!C=^C%iJ z?Lb;S%MtepVTKBf8KSW~MZhFo!K>!4u5C7)MMpx)6J%E{L&}^E%N~b$49o6es`Awe zTy24F@;OtqNNRE?aZ%cUUUER%^ZCB2<3q57* z0LrML1ov`#ZIb?lLk4RthKLKOV|Ft=YK_FurQ*sT-d+y;ellmZHvYik(XE`kSzpHS zVjYZXFtg*angV%u#*WaXw>CjtVST{%kcEXY(j?wW%1gUqDzP?W zD=jk)X}8bjj3(-s9aW552d8)7#ki?+1aW#|wvgyWl!dDsFHsGGO` zb%PsBu|g0llt68p<`1pwuVGiESIF3y|P(vD!ai_SNZYkz!BSjVuuEtWs zXI8OKsEf1swv70WvqTWbo^^eN)-DbSDu(3`|KSdoh2~9$!wvw0AMt`V=n1NtUBM^1 z&@eC$z#RCg?H`-zLT_K+t*tHbY~g@^4l)%%hlhvOoJ4ABYX5|iC}uQHXjK;Fb=33* zh79w*XNGbKJlJcS)Q}v)<6gm4UPWs>VEt?;fk(OJLDFxGt!l4oF6MzmDQhZnx|u(U zZ;pwFGUxVR?KW?_XVw3DWfU1)+4r3koT;iccw2lZQIt?Gu;9>ASqnt{*=MJ`jt&uY z;d=^z{in8UwYd1x)YR1dUb>Tgt-9(9Qm{%&DZ%WH>?K{tZ-f52x0_tGj?cNXh4422~Lg0A0mbLYncWnghFQx&S+F?o3A}w{#aMLiUAve ziHi%@nI{!<5AZ55*k?owHXRr9A-Y1h)T-jZ#T7n@*v3jz^JnggZ%CyQT6E0xSv+KS zyy^$=$n~~?xj~2R0|AcS+9n#zlLYZgJkR7fa4bO+8^^5=Gc?v32Oz$sIFS4F1 zCv@8z$pzS~p!Azu!!EZ6>KvB2-^0V?OUIdlPZkN@!$Xnzov;yc+si5_ytkVxrxX;- zaB6?(zsW59g#7w29&~?HRwXMZm*;i9<_})9f`x-CIYMVO>who&bRhxdxbn9Ppl|wn zT-kEMji6VvVQgUSs$9$!Oe;!nPA4ruGzC?p%FhY@Ogtp$7@)1(d+sEN z_f3{ilQQuRd3%zMcG>36z5=&(lWDc177OTZK=b6bJsG2BK@=@?Z8Wa*em7TaDPkG#~7FfPsOKZgE{ z34`kH3hA))#J0|hc28G#T9m>i7)kZfbEfsV{9pg2ed1W`!I{aIJ8+#lz5u&Ymj|jy zL8uYM;il>D)m|ch8z@C%H2jG*^tu}r~NJzM|w+ArSbxh>Tpm&0Q_NiPJ z#Sn8^i<+8J@i?uSA5Q%)2Q@@z2fd^|-X6Q%9H`<*ZNBZ%TC0zKM_{H>CO@0i7zq)l zHI%5QCwEor6*7ZXIRyhl%;2DGp+ahRwMmag#pX9ki8uNqpS@Hog=R-Q5l4ec6Zav!Cfj!ro4yVRb?~u|QjT?3pG3L39R39l6$5TNXXo{d$tn zOG5PQX5TDN=5Rr|I4J<59f-f_#?$Z!$w)5?g z=0}ATKyn2|4GptO6uS<$uJ_05JdU-{$ONsmDL-&?C)L+;f*Nf5rhdyGuC)n3?#`{A zA8(;X(s>fH10SPMiMcVz$;ZFEgCXE`rdV8Dgc`}<`w5V`RNKunZ*6b;0E}Y~fQcXc z_fM@MyzvnsX?}8wgwf;PF# z-98}{e)2NH5#R^EKJ`%X@$qE{`6Wrm5v88laS{>nIF3YN(=vjA332*p`DEtriHR1C z@MmXdXoPwgr6OP;&tH{hSZb@VI27-T+4ntgxkM{@R8=3io0&Pk@actNm%zv%jtMgJ zRBQF-p+!UAoO*6QJ3!L7=65ax2=bS*GKT3wMQTdQuMW$N@0>R}-@&62Nvfz|Ff%g? zK3(*X2zV+qJa^N^tgfuQ5jF97U*zbL=6bCMBm?w6nCR%}(}!Daes@a&l~9|Tn}W|b z)0~bgDf}K+$BqC$+s_1%mlWUIqulvpAArne!kW{5DV2VzWks)dP&**)Y;pI;l0?SoNC31^ zdPeoxBfj(JyS>X^v*(f zF4N?+Ul-t7YDBnVIRhZy1Ry|E`J$K>uXCk8j;n8+J)dP|JBNp3-b9Vrpt+cgnx4K? z2Lfyy<*WN89SKokQYGM|F-p zQ{&?~A_=LKnn9kHdFv5RmOuBY1aP!b@$uX(c;w{DnfxAugM%9zT_J@t-B5tN({NZe zX2rn6lXiEnFA))ScjpA!9l@zkxB>`z00&jq&`{LDfi?xt9PHx90mv8S>wN zC!eX%O9muhEL&t85I2)qf+X%?iWPd7|JMsZ>eH-K#UrwA%1Yqu;$m{T)IcKSJ9E=< zbK`Cp_+SCW?{%i&?=M)@{zwl9v_iIUCZKyil9CD^LmQVAoAX|9&y1G%OH)Ib29Zv# zR7vwnNt#j}P~-73fiz{$K$(&%JG)_D!v3dHH#zw)fHq$3CL7h74WQN5)#=ul!ZCsB zDY&?BwY9Z_LqbwTf`kFXA}J+RV>A7+?4;vnN?swIJ1z#FR>c{%EAi|XNDPtkDEaw2rw^|^%|u>5 zQn#0WxSr;Sozo1`F5U*bX&!of5H7@s9rs&kJj%B=^{L`UC;%PJx87CH89By^v z2pjnC3}P}Hw#urhVd1kFn(zKgw4}RbR9za{f<{)t7Xi#z;nqpKpuzizRv$Msa$$SV z3TG2=c{$_C>M9|ZEzWt{9hrdV2?apaF60L;21z(MIbmUAkDllPy$SW42zsCVpAW2hzn>jM78DdzsGL3a??XK z>N+hYf^O;HPN_XVBTI;(ddeFdY;D3si);B3F_W$JommwYL2n!#npIK8Y#K1Yo`Wc% z3W{hz)=YqDnQ;g9n8l#^7vRm_IMwQ=`;O$iG3uL~6n0-8C% zp!Ejqxa{WMC<9H+%r394*aZyUc+jx1v3mf3eQ|M7Tug_*4KSFRUF{6LnP)(y+MmIC zg-V(K9XZiF0`@5jc* z{GV^?2mI4Z-XVMpLIuL}^83GB-Sm+?Hb2UCN5V1R)oNiEda;J|`N_;f+R3d+gV9Px}TOiH}S{c-4 z&8xwP$@%y@r&;66(Ujm4Lq%Sj8fDVVKqKeZ#MDBSE}b(XeYkvW!yq^*4^*~W8#Jch ztQtR&Rf`W7nDWX*JOBOtuD|s>LOgVEqeP%zP#H!o+w59dYw%Vf1$9{ffL zP~E`-&PHEf-`Uf%5b&Ob)|7h6coKct%xLK71M~CAfYe+KQYWN5;!8@#kqM>IsTW>s zgrH;L;Pe6!mChf>%+C~{Mted)#R`l%0`DOZ4x1^Ow}SyN7yu*zUSEm3vsJ)VFj33I z7Z|j7YCRNLzU?~}5P)qJpA<9e@x(LuZ8x#2{pOR~DMeD{n+<3^qKDSJyHol^K9E2z zlRx?E{tbj{Hu?oq;NTi4Sm<}#_OJZvw2bX_FrvP?ka zWO&w2?w$Y;cUW4SwCbf~o8vg2uwiZX98F z5M3`c9LeXQJ5ug~1YbOQY-W#zsag^~O=eF+UW{+8UX8m)ZoTi}lNO}eh!$HMg^R$B z6o*%QPqsW1wGhOmn*b<#-nMk$8)b?FiLI=+49d#)S9!t&6F24@w~cS^?ONI1o$g?U za-Yk-)=Wn)>O75Yd&A9ao>_^gIPg4f;AsVbz!IJblPr@a2fSr-6f8XHi^^Vk<0r7fl2f&> zA(R!^$@KL+=C7wPvK2l$QNy|X9mKGAXmNG~K~JPEyj9`ov6cr} z?qq|VHq?)wKT`qj$aEg3J%GBq-26PWbfmVTD60Ki?~V+=)>}vH$udJE;$Wf4mxnvm zE@@+*8kr!yyk@9$h@QNMQW*l42hH}7H=IU&b?ZeWEl{j(DS_n=XeckjKT$!$H{A~H&|7cyG z-!1gHg9TP9_bw$UtwHZ$NM2t6UuwW)`?3vOm)pa@9m*7+bi$?euH7a580}7Cl?)+&t$wXW+7I^-Cy<6>ozVOw3wf zXw5k@hg5@Anm#^Z0Dz}GW} zN1+$^h9hq7#mo1_24`Hn4NlaV(y{v!{n-ok2QWw6O)>2WZDZl*meo$H=(zC=IR>vB}}q$k(fbRpCq z-a(xrh)8$$S))1XC`|q4m405gvh-xw;$T1&A9@Sx0RQRwaAl$X$WB}I6qonq%NIE? zb2dPqISm*gtn)SifV?F?y2vzz%;V^@|0x;iRJ$0Y3hQvh7tpo34g75Qby<4k9eirt z09sl7AP`5fLdVN~n)T@Z^+@d_KRs?v+Y(N_#vk}y;_*4ouMe2AxC~Rj7F)2UakMHs zEmrS|r2H6`Dr32U`qrny`|yVFa$F<)Utzx3v!it#TG9Bt$;5p?3aCz~Qs0*x^;>xTBxM79gf2!<~Wi}f1;pc|s7BByVf zFVf^Iawn6gg51xvPscm+geCZsWL01I%U@2i>yHA+8^qH=ZD{jjnN)|ajAm>4qRu@} zaSe}5%@dv*1XwK6=}3&DEofGd@oQc`YX-nbu^PPkL2Oq7po_^D7j0WSi1uZU4l=H& zw`^w}?3*o*i8#->W5X-_eq?2co;;Ddld+MlZ@`Cg+Q_@|segEc58EQfNjtAIRKDqA zip}6m2Hg^+H`MiO+pUE}-@l#UNvwM#%WJF>b^rA3+CCJb({lRKCR3D9GmM!bGbnTI zhzqD5db;zIqTn%Z0LWsn*W)cECb|7Q+S*VEm{yn>nunAx@=2F-62ahrNx)4 z&UVg$bioHz)4IY^AR=5Q_o&Vf+nT$VJ9V!6@w5QCc2m`kkW0MJlhaXCHybXdDjn(q z+6(B)6U|=U;zm|2Q&?RK6!%>t0r>_XQS5&5(f!?D-}`Sa)TuL}-nAT#7?b~BSzj3x zSFnV;1h?Q4G(ZS$!98dQ9^5rJ1a}Pt50F5R;O;E$vI($2a0%}2?kuqHkbCcYKi;W@ zsx68+Gd(liJ>6f==Z(f<1wo3EwDiFMA3^ObkcekfD(CAc-!-tV%gR5a5~~MAh*cp& zgiN7o>%4~o5z7$&w5B{^o3V4FGQSwXGm`o}UM_SLWFGrP#2DGG+XPrYJsDwyw{zg} z>naKEy19GtHN6UO3Ow`i_ZL~WmE%LZPnd>Loy)?31-i+tV}x^~)jeRnazjMVV^G2} z=u@MS?=5PA)1zCnAEW%?NfNaCrJaAhWd-wEt;J9qd%DnDpszzM3l1Klb-M2)(|#7U zmKKkczeFo=1&(e{KiUoC1++!cY;<-S?toz}jlXjgaKdJA7=qDQ%*SmpEAwWi=9Nrk z5ip+C#yk9t2RDs3egON2<4Y(3(*!28|CO~jIWDQtJNfXz@tu+|FK^5Nx7K7jM6^mK zzq$faG4qIJX`W%8wnXb&u0P8M=0$sHANZBB2d|X;il{}zzfVzRQ+@&cUWyE#g803% z7Rr0=k%XU_!7#Sfev0!z(J+19XqVbr#p(D;L&oaiq=Z@)_M@d&iFB!qyDk*kW^l`` zrUv>Yl@6aJ+cXfcJ8bw(u;!CTES6i2(My-H;NQaj##pO(r5P3Oq(4jbr2c)p+$JNd zhY!x|NEdI8tLq++_8|Y1V95K!6T43DyqZT3=KrAE zgV+r7x7niSDcJ3rV8gfe&z7{+97q+4!84jWF3#7l_qVVCJ)RGkT%zYD*jvt{!Fp?UH*{UZeyG7pS&llfX`0rN)1fkH!YskL2-jeBJ%B zz53Fi5v}(>_&Dr3kV@BJK$iO@+LS6$H#m3s&w~{HfP)I|mp$k#xh|`KdM|s!t0*85 zma~?UcI$7^`VSGqb&%P@xo=6vG(Ai(*G2#W^b|0NSHK{gE*UdhRI!&nyEjw#m(N@R zp2eF9^^;~z%s*s)&!i&7nqKg8z}NVRpc>TWZ8zax(jqlUR47-9uCvze|M-8Y-?MB1 z^EZv=cYz+4PFdJ|W7aN~V%#Ws$1GcIi+6ad7&cEehxByMXUCUIeUlQue>w`shA{X) zJEv&(H3WgzVVZQbj)P)}L9`mISSjANbK1jOM za7XTl%&>c~efVIfil?()#%pIWWHkUwVk8Q@`=&eJeX|3k^d-p}p|e8?3BuDq4S`MZ z*?8-b`sZ$=l=E@^^}lSSeAg8AZ2u7lpcC=`W!))L3rD}NKgBD7kECW#M+v5An;_r; z@T*r6=x`YCloHMTSS*A5PDu_a3wLZj2x~l)v966Rzwx5YcqbC0{gx8kO<^@sxjYuo zrz7$Zf*_~VCha5t4D?=m4b(*D8C;jo0=hw(>YAi8w#R%Poi8)utNSo3zQc3C6x^Ab z;I~Q98gPqo9E5AI(AK}X59r02Ge}&dU`s~N;oTa!LDqhZ)U)6U@se&t1P~D zI0b7aAF_ziwK+H@#5K%Zo|GVyVWX4ejS-f8XTbu6ye_BqV&EUveu+@9q zedF9-_nx}u%_r4LUDSAXsEgOR+J*A4fM9Ftokr+?WNHtk`k?!^*+K3pUv}>xn23l{e;Q;8xvefG~g<5tgrApLT-{_#07{O*&w$^FCkg_72At9WC z{o?J>A8P7<@Bs%*PW8d^z!(xl$pU$~Z^`^=>G5y$&aY9crCw+|5|?BiP0hTMM|t|Y zkBzGy)%!;TZhFt;hcL^7rONQqhKDRIx3v7QlF^A(EDkcf z5uaMJVma!>t#5kVXP6|68LUbt;M)~a^!Rn-)I-psp;a9TOdaQ4dl&NEt4T=r6YJA% z0Y09bxy2gDgrSheN}AjOT)gK0^cio4E;MKE(F*(m!1#rKA;&a=C@4p z1rp=7uVWR-8UH~mo?6O&{y~%ZJFruLbt=CGwf^+Vut4(|_TNM@kdgc_W;PA|NSCIv z=|b}-LK*XLe}896HLgrzoERh6giO18`W?F_GWL@-ggSeJwB2TbysehKt4M7@9KXw>=$87 zcgf^lMKv|ys^b;YG}OO@cY(d@ea;HhWqVmn>?c)g0ZxhbvmeAynD@i;*H*qq$9Zsf zMTUoEf#wAM5>zN<2smaauuajd%jVJbg^4HP8)9c9s8@ z`G;s{iMZ@Xp(xQbzYa9d9km-&86ozMnUxJ|W=nGbAgYA(JY2O(c=M?>_I6Q_sUN0g zqi_>FP7<8eMu`F-{A1%<1M9l3&emVowRo;(vCh}~6}BGb50&IyOic}I}hV~h2& zS!H5wgM4EJVXa~JaV{Op-UGWYt1Bdtw&4t38*EQY%5>m4CWd?+1s7;J zRCo3zi@uWYF)-jdt|TP*6cwv~Q(uTgQ{=WtIz%A2#61H90fe zLMY$T5z3axbgi0y@sUO-uiX2QQhSn8dRnLd89u1$5v5~61e%7QYir*J`OYc-?9j-MnE;b-T`i%(Q%@aRuq9`e*D+5al_?vDp=6ei% zCF3Mhf>vZ@ zpTorbgIdXuQ+u=tZ!VI%24R<^(-UuY|MEl#j$GzTd`m7FGN*RP2JmHE$?Ceb3`r+* zOMd6_;(DRCWoHuDcGo>zhvA0lEh&n%J{^PJV1^93U^rmOuVAM-b&t?m{<%Wz?&~9m zw`^#-0bR4VtjIA8zbT`I6Q~i??zoR8njeWzOn3(XhYLlVNVaEYc3gCkT_?tuQO})!0`${rzqjnr8Ma zp`lM_ljM_qqhZTit5(%QBZ@j`!)Eo<4thcbnb*I{VO|aEoZ|MDHs;Qqd|QUgw%Fq~czbkBfaPn4O+qa48NlwUXj7V+w5k%TP<4lXAs69t zrK=4P8{8N>^m6}5ZbUg7b}>jHBJ2MLio70GFz9=}`n>Jd2^C~3df{GOz5U?xBkNIQ zD9rOXry+|xt5UH4R|&-FAUd3D$B~*)9|EVK7GKN&=E`jETKEO*6y^D_zg!|tl7JuI z=W1$4liNBVG)~c%JM=;1xApH9Af$FPg~4&9Xfk@sJR;t zk^SHc{Kal(QWghj1JDUj7+1dFo#YM_vCL*Qui;v)sn7842L$^q&YWIb03f^0poU8) zkN7z+`c><(_tHP8uC{%B5(Zl()P@}rk zz`g13s4?xNNvr(PzfAm&*|UuWkumuM%?4aVUXDMQQJ;!3|sEho`d+EMZ%u z?1pkuc?j^go=1BIaSWNqZ>6JOVg~$lY!TsG^NlBPtg(biM>{B7%}X02T{G%eZh{b%JqEgjl5%g%VRS6(PTV@ElUNCn(2ppuO!sHx1R4f0X^ojLyd z_Mb`sypgFOtK#GK7M`nMqI2!zulJrX!Ar?bY~@}C#71PoWXTWWpT)qS{Dr}W=urZ9 zuAl5fgWDU&n`-R(4qbM-?#72?8XNbAo94PAOB}%x69J~Z%gH-4JP|;tZKIpufy=|6nv-X>no#NB;P6gY)imO1chBx@ZF5pZR-Mhj4d~ zvC8QKhk1lOr9m)Tf0dzI*i@AElyuQ7hlBZ_AM8$vOaZzxUuv`rUI?}zl2vHQ=%2%` zzjk0cR3|i8VDh70Lr-T!jh!z#vC5!jeo^ycZE#wZ1sEdguoB-Tg-4)59of7?^55yJ zx)?3#F(QK2e~La|KU%1bv(&+&byd}kjB|&#NdRGncyepg z^UE34A!$H(>iK5eB7^lo);lTq{qf6~!V8`OTgII%H*WFJ)yM(a=jkJK^y&(gvO>>9 zSgB{K1p;^pT0U8BrnZX_WDRC}vE7qS5sjQPsr&Ei%*5%B})7ea; z(Mi@F)5^z)L9#bsB(u1TS12?`+WCKu1h9;AShCKFwrgK45C&v;@lpmwXnls-hlkys z!5*!Oi8eHgi#iM4i*DD}57Wh*u~i&NS}tw+${ep~IAqx} zo1HaaK@y;PT1JYmGf10aD4@frs^0h7esrWEheW4`fg$9WrRg+(cG`c`0h^8 zb^h(K*0?#zNW6DkHO|jp$k#n@x!=z~Rr0`4BY#eP^8AQuSOk8CyX51~HD<-G-rB%X zsYL!B{=4F4dIMUn?}}TYB?Wx*I$q~$T4atFt(*4UxtP;0!8F~-j&r(p+JY~U_RKr7 zZr`k)d%>ExK1$myr37%Wvnw+)V(mZR`Lrh`k~8EmVkEC6-0o!5+b{d1?nm;UoI2x43s2^LSXLgX>gxF%zkk7_{EV=Dbp;(jhpP_m@zDR)Uo@Ys1cdYu=Z>&m|VD_d? zxlGJ)`(5rn&y6)kl%Z*}0mX?=XPRD<*jw9rx$V#{h+?&I7Ldao=PZ>e+M^odFdxEr z*CrcV)(_(0DwVW$u>h|A-#IC^)2P11Z;=Pfv82e{KeWQ?ETY45!#?zG!Z{?^ZkVtz%PBKIkl5&b>KJYrnZ^Iqlzn z0NuE*+ho3|jri2w_48pU#M+%K;4XlAM5HEYq=UNam+5!w(VS_!$Vg?ArShTs?Q>LT zkFoQ)?kumv(5#*DcD#ph*|W2a&-M>=Q8tzAd7p}@6O5?s^}-Vsu1@!AF`Ks^TT^(v z(R6j22h-ZW_a4K)y6YE>3gmi}?8DCE7u(JbJQsWe-x+1RNiQ#_5QA^ zO0;Lu&VhGhm40`pWIZBF3BJe2()ZB)`}Km`k5-#2+s8IO;()ET8Zj_EDmVW50QE}G z$ne^1uKdY!s^4-MVX`vHr%VJz|FBIh@&=HH}lpt?nHzKm52V$h(zY!Rd zZMyz4K;J*2@ihl@dL-k$dYBoz(-tN-0o831uqSS|>vE97y6EaGNmtNaladdWJhR3hK)+TM?>FjJ%^bm3?11IeB@8N@_TGD|M~OU ze_QGAqW2Z7!>&O&tgN-&G8_pw%n{txgE45B>u5-oTn#+i&Q*GJFZrHGF+a;>F&WV4dwlHwIP0TR8Dz< zbJCrkJiz~@u`OIn=9;2xFu)SH8xp))4lvD0J|KNL=vDR%tscbx%<3LAr z6s`4pqLcXDCEEenlpsphgBOO>vb>B}5DwoEZP)B{ddI=i%{m zJp%2b+PCfX;O9K^gOg6%^wu9Son~xN^+HHk#?L`Im-6Blt&Ln$!&E`qVxmDywH?1V z(>!r0x1xzAeCS#-kq!SxhDR(>`b&7LQoV3JjoOUU{GO{X1bw|q;hU%&KUim+y76)j z;N7lia1%o35NSpi?hwbJju953=j##8pdHJxBe<{7Wb9{Wva@sA-gNTRHh+k%4l~#1 z%j{uE;5S{}#-XcLp^(M7eZ~YEuQGdJbUUKT!$Ez8AUfX`chZNVlPIFRtThcC-F;=h z#Er~c-{D`O6^=@f3jN*H||K$a?vyxy8+#BC6`HjTM6@4&9N zOc_e;0YL|m@|_q`EO?^8V%x)aNd1Wx=QLql7M(64Z)z}?;3qbqJCpN{sal{%Xm)%M zH1znK-2M4OZkqS_VK{n0A+X_9m}q6LOj#?|gNL0k$?u#19f$=P$V4?RFj32`JwjjC z+nPWWlY0xR+jRmaN$o3t^Ke6dFz-!F*XthE*hp@ade7Uj>_q`FOb@SlxqXV7v^LtI z0bCaY*e-kr((Q2gY$P4(__sPAo-gAM4UYn@l!(F{oqu&jxx8gPLB9sA_)Yc;69kgP z6ap(=t*$P;FCa2hK^V<3ZjHORZgPAkov7hMb)p(a)RbWeP=$M=fKE0-jA<+ag{F>3 zV=UREyAgZxU_1Ebb&4%N+)$pKOLx#NV|Ms?zwJo1?=duJ=Ka^`WYM`Ep)QL5UqHe*sIBLLK8+QsA$hB?g26 zl0%8D7-4JDeB*O*MxBt?T%W^ypEDUMR~r^FCcMDHKJPj_@JkR(d9xl+D;OC&5w57s zZZVr;OqrR*HG1HsXg~^DIfE*K-rj|hH+j>c()GSd{aT=z#`q-^CdDq$jl;!d8ushe zD~&+qaOG23D|9sM>)yiykrZw8L~T4|6r0f+O}c~=dY*EGH{8c5mzW?iF)Hho;~012 zf^73-ow|Ke;P#H_DbF04{Z{$oiQ?e*TC4e|mv3RiA7tt9IK6%1&IIEx*ra<=o`Jvy zEeBKC5Wd`K^h zbS~*7jldvkZ-ZM;@Qx}c7B3U?8RBg zUGj`uEOUk*Wf9gR8IP2o*&Z3vI#G2L2r+$J$dH4d6B}g(ih~XRAb^liU^=A9&pAt9 zVq%P?EGFx*Nfu3L_Zqa9FZT)Aj2XRRo3nF%{RX@s+x61l_z}B3=1A&|M$j(2P zOss!osDH2vcI+P~DxT3(|8<^`Z;n*|5nBvMwJr_*4))|XpH^XT7G39yiW=Z2o=((4 zHyjyVRxtTisjF{6lT;lPX5mUlx&^&BzFdu(+c7E+mEQDE3TMk@lpNQ69NpRicW&Fg zHXBb-;qv=Mi-cnLZKn1GiuhlY_B-CjnPlgA37U=qh}0&;SC{|TWL3P;{5V$JNtAB- zV?aerSNzxU<;p6wb6@6p_@R$1m}Un9>x!?rgSCkbeyi;VVe|c+GTz<>5Q-TT96+>( z^*TpWt#dBW)o=)KTFr=N6J#Zh> z@F35+Zs(xs&ICTgN9yLombRaG`;~M8Wu4k6nwKRI`jy|8l7y@M3kiuQ61{FFsr_Ma5U=VD|kXt-nT4|ZW4N`Zcb4!W=;o{s~o2uW0`vpy!TFgpf=N%!X( z;yRbsE6w5H+L`7D!hks+d~z-7-`S>~=BH5%gY$eC{tF8l1w=}lrc<@%9o>=U$(#<# z#tZW$=saZna`>Z=3CX)#n_?9lbn}j26oF@aw6zW%8s_m5dKTPJC_$s zWY`eQ{-Ea419K46*tEEKnOBvKqz@5{4tneO>WWv-vg*PQ@CfJXFnhlv?)EjsM~|ZU zLqGIv&cAlpcf$u84u$qiQ<52T?f0&eLJ?b%-+0VzyqG zxBkcs?ShS%_kV?PeR9F+8Y6$vY`NV(-;$CUDFqub?f;5E3b&{3I^BOE_>8#;p;Bo;A3A z*UXv=FbYp7w(FoImuj{r<27`$_1SM=m>{%}~{^c%%w*1Gx&W8HzAf+B?iwZg6&cc~9nLzI>$q`nuY4nrWe*d6`9_ zMipxDW=r0GilF>{_P!p30J%>O+s=H%YOFn1qOx?~%>;q^T1u!ZUxM^9{HU4NweO4% z7shWBN5=XK?qas<=gta9kP${-Y|Y9ymXRXKu~ zaR=rp`s&GrOK(p-Q^7RK`gT`|fXj}^ocmX3aS{IMD%|1sM)d%vm$k4#n-jr+_(E9I z7pJZnwpiVGdsac6&p1KB_XQ&QyrML$v>@R?k(du%V%;g|k!a6Pq4j06IxQy*fLR!Q z6!W(*mMf;lz?A=1=S7lh7BXt}78BDjIwBTZQm&YzLry3t<;zPiui|R&2j`7#qC96e zDGG)6y<=S@PAcI&XwNTYOij_XSZCvkXWm{MP#Z64*MrtZ$Ou1y##%fW{*kb1J;$uLXe^k(JUl#!LOhn(Zn?{Q?eO4}k2u4oLXS`V)YARHbYHisT$pxL@Nqw|fnXgFoF! z8?`46uh`=n(+#SZ`C2iNSHOZi5b9cAj;89p+4iAbm*>mOI`Ot~PujgaqGD#%7eshO zSRxhQGZ*1MBDk>@Imv?NaXHHye>7+gWm-?jp4{Vf9adHJx(Ya$^~O)=Wrv3GqE^4M zd6sJ~sj8M6Dp=wMVLZn%x!WmQTNbH%c-w!W_bKCI=};FdMnuqe7v^>I_~2Ijjy%h1ssO`z4)+lzS2 zP7zi;;sLA;!IaZMA?tb}5urj#{M2x>)mGsumt7j2(xg4^Vy2|~YrKU&Y@J_Emsbj` z_-Ocx(5N3Bk&XhZeon$_+95wPm^oM`Z4>R%03Nlwrkc_LoplhNiknl{w<+5jZ#02> zZFI3E!;Mj$h=?D|Wh-WV|9r6xgJ6x%p1`dRDqTGe&M9I?5wl{NXQ;bd3Kf5-`=ejK zLr$=>aG0GX82H|>`66q#{;UC6cjjfv7fj3bFveD#?Rp2kPspw&d<>T23U8^MbSqbC zv=%M=XQc)Xf1+9lCFJ!y>{KQz#-?^8g~r6UvC%usKWE=`80a$iGbKPT>@d*FBnUZV zYbDRiejC~-+l~Po6s*V|wuE@FS)~=T4>@#zq=l*`2`IvBQ+s*p{jW{?2Y98B+Ls@u zsFY-wc$1VR&0?gjW%N{z3C9-=hPqJAsp-9m&o@`?rlX`CC-47xocv4y^4WEz13Anb zp}xLMHsqEFVeJhRmCCNN^Ooxk!l)Qm%C&c^3e?297|s1g0UIR*zYS?RLzwa^^#Djs0OSJP#s~#Yz+t>rj$D-w)-go%{S$tKP zi|`F*WhrNq^Y8Jfq&Q)IQgxnBp{}Y|5`A}XLX!}|XYx!62+1F{Og}vdtBD0WCwGa#extX@6#h_J>yo_Y<5h?A4XaT-@NtO-T;X1q;=`s=25_hwp)a?>Ku3J=6B0Waj^9Z{_}v+8i)1$O^38_& zw%5xxqF)ltCj9+F+0rCT;-S-fu81BVeLS`a$pF*&9#?9}cA&aMGd~_9YHK?&?W!EE zBJ$T`Fn>38qXVWWx1~~2LfcxEN{_ESHV+c6D`2}j7F5AMApVq^evxjKCVV|A#=(U76 zP#>G|ME_|R!vSQs*}XK0iyEN$Xn~QTjQXK`qoLY~mDP@bY0b&9Y9fSPJNzwpNY3C! zbs5TEG$Lm}j&SM;ja(F5MRCU;eO=yUwmnx;>{8f|WGmzu*ezX^`p-AmLzP%d<#^g6 z8`*hQV3VtHO43CmZ|hfR!BU#Klm4QrwJgFjS^m7DbtZuaDkT(^NODI3DYsrQW>TVm zb2_AUTRHMXSBIC_A}of5O#iF~^M|qiqOZH4i@*EfP}gJnd{&vpyCCi9K#zlw^p|J? zuJxpbdNLZn`<_f#ge?_{;lB=WejpsJupm`7C|QtNnIrY!0$$16F0SpoDGppJXSFRC z*dg)#X5aR?&?DPIB;AwrX-k{`%Ppn;m3r+9_k5d8&q*e&!W2js6lehou^UI7M)IenH)_R zR&Rcd(7)}k^yK;G*-?qenuX*l_2ze@=(l)m9ks?O>we{eg1>BJfxGrKpNm5W_y zCnLVN(iXlu$BFE2aQ$SxvmNTa-tue0@bE(HNh}ezLjRdi4U5=}*05%lD;h<*)ujl@ z8*i-xAqiX{L0SNFzql*a(x$4Zg|y64Betb3XEpYaG-||rzdv`!I!+8KLaqD4IJFSC z{nRBqP5S?~Sa61^aI$+1F@FHp%As9Lmj}{x@<6Mu6&t`>?oaFGKqU^6)*q3Jb{vFe z(0>liRM(Y@!1}de>PaUyzFhJEMBacQ^8J_9kp9y$ljPZNjr(u$VcaBd@F)u-&ubk`Q^@c zRjcDU33&1IOSGN&4s?`dl8`U52*Z2-oU~#s^Z%?9_GE3OM@j%g8vUw+&Ja{ literal 0 HcmV?d00001 From 7a9221a89945649576828b4fe70bbb565d74ae9f Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Tue, 14 Jun 2022 16:51:07 +0100 Subject: [PATCH 30/40] issue #3437 addressed review comments. Support whole system token values. Remove ref_version_id Signed-off-by: Robin Arnold --- .../jdbc/dao/api/LogicalResourceIdentKey.java | 3 +- .../dao/api/LogicalResourceIdentValue.java | 3 + .../jdbc/dao/impl/ResourceReferenceDAO.java | 111 ++++---- .../jdbc/dao/impl/ResourceTokenValueRec.java | 30 +-- .../jdbc/db2/Db2ResourceReferenceDAO.java | 26 +- .../jdbc/domain/SearchQueryRenderer.java | 11 - .../java/com/ibm/fhir/schema/app/Main.java | 1 - .../schema/build/ShardedSchemaAdapter.java | 7 - .../fhir/schema/control/AddForeignKey.java | 2 - .../control/FhirResourceTableGroup.java | 27 +- .../schema/control/FhirSchemaGenerator.java | 99 ++------ .../schema/control/FhirSchemaVersion.java | 1 + .../main/resources/citus/add_any_resource.sql | 120 --------- .../resources/citus/add_logical_resource.sql | 190 -------------- .../postgres/add_any_resource_distributed.sql | 239 ------------------ .../postgres/delete_resource_parameters.sql | 34 +-- ...delete_resource_parameters_distributed.sql | 60 ----- .../schema/derby/DerbySchemaVersionsTest.java | 2 +- .../fhir/persistence/index/TagParameter.java | 2 +- fhir-remote-index/README.md | 2 + .../PlainBatchParameterProcessor.java | 4 + .../database/PlainPostgresParameterBatch.java | 7 +- .../PlainPostgresSystemParameterBatch.java | 37 +++ .../operation/reindex/ReindexOperation.java | 11 +- 24 files changed, 176 insertions(+), 853 deletions(-) delete mode 100644 fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql delete mode 100644 fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql delete mode 100644 fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql delete mode 100644 fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java index bfe96976036..494a824f004 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ /** * A DTO representing a mapping of a logical_resource identity to its database * logical_resource_id value. + * @implNote use record in Java 17 */ public class LogicalResourceIdentKey { diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java index b7843676c07..2749fc70291 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java @@ -9,6 +9,9 @@ /** * Represents a record in logical_resource_ident + * @implNote no need to override hashCode or equals because logicalResourceId + * does not contribute to the identity of the record - it is just an + * attribute. */ public class LogicalResourceIdentValue extends LogicalResourceIdentKey { private Long logicalResourceId; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 5fbb483a51c..d2519379e64 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -264,8 +264,8 @@ protected void insertResourceTokenRefs(String resourceType, Collection xrefs) { - // Now all the values should have ids assigned so we can go ahead and insert them - // as a batch - final String tableName = "RESOURCE_TOKEN_REFS"; - DataDefinitionUtil.assertValidName(tableName); - final String insert = "INSERT INTO " + tableName + "(" - + "parameter_name_id, logical_resource_id, common_token_value_id, ref_version_id) " - + "VALUES (?, ?, ?, ?)"; - try (PreparedStatement ps = connection.prepareStatement(insert)) { - int count = 0; - for (ResourceTokenValueRec xr: xrefs) { - if (xr.isSystemLevel()) { - ps.setInt(1, xr.getParameterNameId()); - ps.setLong(2, xr.getLogicalResourceId()); - - // common token value can be null - if (xr.getCommonTokenValueId() != null) { - ps.setLong(3, xr.getCommonTokenValueId()); - } else { - ps.setNull(3, Types.BIGINT); - } - - // version can be null - if (xr.getRefVersionId() != null) { - ps.setInt(4, xr.getRefVersionId()); - } else { - ps.setNull(4, Types.INTEGER); - } - - ps.addBatch(); - if (++count == BATCH_SIZE) { - ps.executeBatch(); - count = 0; - } - } - } - - if (count > 0) { - ps.executeBatch(); - } - } catch (SQLException x) { - logger.log(Level.SEVERE, insert, x); - throw translator.translate(x); - } - } - - /** * Add all the systems we currently don't have in the database. If all target * databases handled MERGE properly this would be easy, but they don't so @@ -554,6 +495,50 @@ public void upsertCanonicalValues(List profileValues) { } } + /** + * Insert any whole-system parameters to the token_refs table + * @param resourceType + * @param xrefs + */ + protected void insertSystemResourceTokenRefs(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "RESOURCE_TOKEN_REFS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "parameter_name_id, logical_resource_id, common_token_value_id) " + + "VALUES (?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + if (xr.isSystemLevel()) { + ps.setInt(1, xr.getParameterNameId()); + ps.setLong(2, xr.getLogicalResourceId()); + + // common token value can be null + if (xr.getCommonTokenValueId() != null) { + ps.setLong(3, xr.getCommonTokenValueId()); + } else { + ps.setNull(3, Types.BIGINT); + } + + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + protected void insertResourceProfiles(String resourceType, Collection profiles) { // Now all the values should have ids assigned so we can go ahead and insert them // as a batch diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java index 5e08dd18efc..5f63e4d9616 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java @@ -21,7 +21,6 @@ public class ResourceTokenValueRec extends ResourceRefRec { // The external ref value and its normalized database id (when we have it) private final String tokenValue; private Long commonTokenValueId; - private final Integer refVersionId; // Issue 1683 - optional composite id used to correlate parameters private final Integer compositeId; @@ -30,39 +29,21 @@ public class ResourceTokenValueRec extends ResourceRefRec { private final boolean systemLevel; /** - * Public constructor - * @param parameterName - * @param resourceType - * @param resourceTypeId - * @param logicalResourceId - * @param codeSystem - * @param externalRefValue - * @param compositeId - * @param systemLevel - */ - public ResourceTokenValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, - String codeSystem, String externalRefValue, Integer compositeId, boolean systemLevel) { - this(parameterName, resourceType, resourceTypeId, logicalResourceId, codeSystem, externalRefValue, null, compositeId, systemLevel); - } - - /** - * Public constructor. Used to create a versioned resource reference + * Public constructor. * @param parameterName * @param resourceType * @param resourceTypeId * @param logicalResourceId * @param externalSystemName * @param externalRefValue - * @param refVersionId * @param compositeId * @param systemLevel */ public ResourceTokenValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, - String externalSystemName, String externalRefValue, Integer refVersionId, Integer compositeId, boolean systemLevel) { + String externalSystemName, String externalRefValue, Integer compositeId, boolean systemLevel) { super(parameterName, resourceType, resourceTypeId, logicalResourceId); this.codeSystemValue = externalSystemName; this.tokenValue = externalRefValue; - this.refVersionId = refVersionId; this.compositeId = compositeId; this.systemLevel = systemLevel; } @@ -111,13 +92,6 @@ public void setCommonTokenValueId(long commonTokenValueId) { this.commonTokenValueId = commonTokenValueId; } - /** - * @return the refVersionId - */ - public Integer getRefVersionId() { - return refVersionId; - } - /** * @return the compositeId */ diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index 322e2961363..837026d6f65 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -176,8 +176,8 @@ protected void insertResourceTokenRefs(String resourceType, Collection indexColumns, DistributionType distributionType) { - // for non-unique indexes, we don't need to include the distribution column -// List actualColumns = new ArrayList<>(indexColumns); -// if (distributionType == DistributionType.DISTRIBUTED) { -// // inject the distribution column into the index definition -// actualColumns.add(new OrderedColumnDef(this.distributionColumnName, null, null)); -// } - // Create the index using the modified set of index columns databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java index cea31177fc8..07a2b038d40 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java @@ -36,8 +36,6 @@ public AddForeignKey(ISchemaAdapter adapter, String tenantColumnName) { @Override public void visited(Table fromChildTable, ForeignKeyConstraint fk) { // Enable (add) the FK constraint - // TODO handle distributed tables...need the src and target distribution types - // so we know whether or not to inject the distribution column into the FK logger.info(String.format("Adding foreign key: %s.%s[%s]", fromChildTable.getSchemaName(), fromChildTable.getObjectName(), fk.getConstraintName())); fk.apply(fromChildTable.getSchemaName(), fromChildTable.getObjectName(), this.tenantColumnName, adapter, fromChildTable.getDistributionType()); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index d09b31cbbf6..4feef8f1432 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -519,13 +519,12 @@ public Table addResourceTokenRefs(List group, String prefix) { // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: ref_version_id removed because refs are now stored in xx_ref_values .setTenantColumnName(MT_ID) .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) - .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version .addIntColumn(COMPOSITE_ID, true) // V0009 .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0008 change .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0008 change @@ -563,6 +562,7 @@ public Table addResourceTokenRefs(List group, String prefix) { statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_LRPT", tableName, mtId, lrpt)); } + boolean needReorg = false; if (priorVersion < FhirSchemaVersion.V0009.vid()) { addCompositeMigrationStepsV0009(statements, tableName); } @@ -574,6 +574,17 @@ public Table addResourceTokenRefs(List group, String prefix) { if (priorVersion < FhirSchemaVersion.V0020.vid()) { statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); } + + if (priorVersion < FhirSchemaVersion.V0028.vid()) { + statements.add(new DropColumn(schemaName, tableName, REF_VERSION_ID)); + needReorg = true; + } + + if (needReorg) { + // Required for Db2, ignored otherwise + statements.add(new ReorgTable(schemaName, tableName)); + } + return statements; }) .build(model); @@ -598,7 +609,7 @@ public Table addRefValues(List group, String prefix) { // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: tweak vacuum and fillfactor .setTenantColumnName(MT_ID) .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) @@ -616,6 +627,10 @@ public Table addRefValues(List group, String prefix) { .addWiths(withs) .addMigration(priorVersion -> { List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0028.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } return statements; }) .build(model); @@ -792,20 +807,20 @@ public void addTokenValuesView(List group, String prefix) { // in the join condition to give the optimizer the best chance at finding a good nested // loop strategy select.append("SELECT ref.").append(MT_ID); - select.append(", ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.ref_version_id, ref.common_token_value_id, ref." + COMPOSITE_ID); + select.append(", ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.common_token_value_id, ref." + COMPOSITE_ID); select.append(" FROM ").append(commonTokenValues.getName()).append(" AS ctv, "); select.append(resourceTokenRefs.getName()).append(" AS ref "); select.append(" WHERE ctv.common_token_value_id = ref.common_token_value_id "); select.append(" AND ctv.").append(MT_ID).append(" = ").append("ref.").append(MT_ID); } else { - select.append("SELECT ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.ref_version_id, ref.common_token_value_id, ref." + COMPOSITE_ID); + select.append("SELECT ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.common_token_value_id, ref." + COMPOSITE_ID); select.append(" FROM ").append(commonTokenValues.getName()).append(" AS ctv, "); select.append(resourceTokenRefs.getName()).append(" AS ref "); select.append(" WHERE ctv.common_token_value_id = ref.common_token_value_id "); } View view = View.builder(schemaName, viewName) - .setVersion(FhirSchemaVersion.V0009.vid()) + .setVersion(FhirSchemaVersion.V0028.vid()) .setSelectClause(select.toString()) .addPrivileges(resourceTablePrivileges) .addDependency(commonTokenValues) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index aec32411090..5b7dcc67a02 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -96,6 +96,7 @@ import com.ibm.fhir.database.utils.common.DropColumn; import com.ibm.fhir.database.utils.common.DropIndex; import com.ibm.fhir.database.utils.common.DropTable; +import com.ibm.fhir.database.utils.common.ReorgTable; import com.ibm.fhir.database.utils.model.AlterSequenceStartWith; import com.ibm.fhir.database.utils.model.BaseObject; import com.ibm.fhir.database.utils.model.CharColumn; @@ -560,74 +561,6 @@ private String getSchemaTypeSuffix() { } } - /** - * @implNote following the current pattern, which is why all this stuff is replicated - * @param model - */ - public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { - // Add stored procedures/functions for postgresql and Citus - // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. - final String ROOT_DIR = "postgres/"; - final String CITUS_ROOT_DIR = "citus/"; - FunctionDef fd = model.addFunction(this.schemaName, - ADD_CODE_SYSTEM, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), - procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_PARAMETER_NAME, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ADD_RESOURCE_TYPE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // Add the delete resource parameters function and distribute using logical_resource_id (param $2) - FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, - DELETE_RESOURCE_PARAMETERS, - FhirSchemaVersion.V0020.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), - procedurePrivileges, 2); - deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - // Use the Citus-specific function which is distributed using logical_resource_id (param $1) - fd = model.addFunction(this.schemaName, ADD_LOGICAL_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), - procedurePrivileges, 1); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - final FunctionDef addLogicalResource = fd; - - // Use the Citus-specific variant of add_any_resource and distribute using logical_resource_id (param $1) - fd = model.addFunction(this.schemaName, ADD_ANY_RESOURCE, - FhirSchemaVersion.V0001.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() - + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete, addLogicalResource), - procedurePrivileges, 1); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - - fd = model.addFunction(this.schemaName, - ERASE_RESOURCE, - FhirSchemaVersion.V0013.vid(), - () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null), - Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); - fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); - } - /** * Are we building the Db2-specific multitenant schema variant * @return @@ -760,10 +693,13 @@ public void addLogicalResources(PhysicalDataModel pdm) { * "aPatientId" has not yet been created. The LOGICAL_RESOURCES record is * not created until the actual resource is created. * - * Note that there's no index on LOGICAL_RESOURCE_ID. This is intentional. - * An index is not required because LOGICAL_RESOURCE_ID is never used as an - * access path for this table. - * + * The index IDX_LOGICAL_RESOURCE_IDENT_LRID is specified as non-unique + * because in Citus, this table is distributed by logical_id, which isn't + * part of the index. This is intentional. We have to rely on the logic + * in the add_any_resource procedures to guarantee that logical_resource_id + * values are assigned correctly. The logical_resource_id column is the + * primary key for the logical_resources table, so we are guaranteed + * uniqueness there. * @param pdm */ private void addLogicalResourceIdent(PhysicalDataModel pdm) { @@ -1373,9 +1309,9 @@ protected void addCodeSystems(PhysicalDataModel model) { * the token_value represents its logical_id. This approach simplifies query writing when * following references. * - * If sharding is supported, this table is distributed by token_value which unfortunately - * means that it cannot be the target of any foreign key constraint (which needs to use - * the primary key COMMON_TOKEN_VALUE_ID). + * When using a distributed database (Citus), this table is distributed as a REFERENCE + * table, meaning that all records will exist on all nodes. + * * @param pdm * @return the table definition */ @@ -1425,13 +1361,12 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { // logical_resources (0|1) ---- (*) resource_token_refs Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0028.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) // support for null token value entries .addBigIntColumn( LOGICAL_RESOURCE_ID, false) - .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0009 change .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0009 change .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID) @@ -1444,6 +1379,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { .addMigration(priorVersion -> { // Replace the indexes initially defined in the V0006 version with better ones List statements = new ArrayList<>(); + boolean needReorg = false; if (priorVersion == FhirSchemaVersion.V0006.vid()) { // Migrate the index definitions as part of the V0008 version of the schema // This table was originally introduced as part of the V0006 schema, which @@ -1474,6 +1410,15 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { if (priorVersion < FhirSchemaVersion.V0020.vid()) { statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); } + if (priorVersion < FhirSchemaVersion.V0028.vid()) { + statements.add(new DropColumn(schemaName, tableName, REF_VERSION_ID)); + needReorg = true; + } + + if (needReorg) { + // Required for Db2, ignored otherwise + statements.add(new ReorgTable(schemaName, tableName)); + } return statements; }) .build(pdm); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java index c01fe912a2d..6d3738471f0 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java @@ -47,6 +47,7 @@ public enum FhirSchemaVersion { ,V0025(25, "issue-3158 stored proc updates to prevent deleting currently deleted resources", false) ,V0026(26, "issue-nnnn Add new resource types for FHIR R4B", false) ,V0027(27, "issue-3437 extensions to support distribution/sharding", true) + ,V0028(28, "issue-3437 remove ref_version_id from xx_resource_token_refs", false) // parameter storage updated by V0027 ; // The version number recorded in the VERSION_HISTORY diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql deleted file mode 100644 index 6e3d70aac49..00000000000 --- a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql +++ /dev/null @@ -1,120 +0,0 @@ -------------------------------------------------------------------------------- --- (C) Copyright IBM Corp. 2020, 2022 --- --- SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- - --- ---------------------------------------------------------------------------- --- Procedure to add a resource version and its associated parameters. These --- parameters only ever point to the latest version of a resource, never to --- previous versions, which are kept to support history queries. --- implNote - Conventions: --- p_... prefix used to represent input parameters --- v_... prefix used to represent declared variables --- t_... prefix used to represent temp variables --- o_... prefix used to represent output parameters --- Parameters: --- p_logical_id: the logical id given to the resource by the FHIR server --- p_payload: the BLOB (of JSON) which is the resource content --- p_last_updated the last_updated time given by the FHIR server --- p_is_deleted: the soft delete flag --- p_version_id: the intended new version id of the resource (matching the JSON payload) --- p_parameter_hash_b64 the Base64 encoded hash of parameter values --- p_if_none_match the encoded If-None-Match value --- o_logical_resource_id: output field returning the newly assigned logical_resource_id value --- o_current_parameter_hash: Base64 current parameter hash if existing resource --- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit --- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) --- Exceptions: --- SQLSTATE 99001: on version conflict (concurrency) --- SQLSTATE 99002: missing expected row (data integrity) --- SQLSTATE 99004: delete a currently deleted resource (data integrity) --- ---------------------------------------------------------------------------- - ( IN p_resource_type VARCHAR( 36), - IN p_logical_id VARCHAR(255), - IN p_payload BYTEA, - IN p_last_updated TIMESTAMP, - IN p_is_deleted CHAR( 1), - IN p_source_key VARCHAR( 64), - IN p_version INT, - IN p_parameter_hash_b64 VARCHAR( 44), - IN p_if_none_match INT, - IN p_resource_payload_key VARCHAR( 36), - OUT o_logical_resource_id BIGINT, - OUT o_current_parameter_hash VARCHAR( 44), - OUT o_interaction_status INT, - OUT o_if_none_match_version INT) - LANGUAGE plpgsql - AS $$ - - DECLARE - v_schema_name VARCHAR(128); - v_logical_resource_id BIGINT := NULL; - t_logical_resource_id BIGINT := NULL; - v_current_resource_id BIGINT := NULL; - v_resource_id BIGINT := NULL; - v_resource_type_id INT := NULL; - v_currently_deleted CHAR(1) := NULL; - v_new_resource INT := 0; - v_duplicate INT := 0; - v_current_version INT := 0; - v_change_type CHAR(1) := NULL; - - -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. - lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_shards WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; - - BEGIN - -- default value unless we hit If-None-Match - o_interaction_status := 0; - - -- LOADED ON: {{DATE}} - v_schema_name := '{{SCHEMA_NAME}}'; - SELECT resource_type_id INTO v_resource_type_id - FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type; - - -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) - SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; - - -- Get a lock using the sharded logical_id in logical_resource_shards - OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO v_logical_resource_id; - CLOSE lock_cur; - - -- Create the resource if we don't have it already - IF v_logical_resource_id IS NULL - THEN - SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; - -- logical_resource_shards provides a sharded lookup mechanism for obtaining - -- the logical_resource_id when you know resource_type_id and logical_id - INSERT INTO {{SCHEMA_NAME}}.logical_resource_shards (resource_type_id, logical_id, logical_resource_id) - VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; - - -- The above insert could fail silently in a concurrent insert scenario, so we now just - -- need to try and obtain the lock (even though we may already have it if the above insert - -- succeeded. Whatever the case, we know there's a row there. - OPEN lock_cur (t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO t_logical_resource_id; - CLOSE lock_cur; - - -- check to see if it was us who actually created the record - IF v_logical_resource_id = t_logical_resource_id - THEN - v_new_resource := 1; - ELSE - -- resource was created by another thread, so use that id instead - v_logical_resource_id := t_logical_resource_id; - END IF; - END IF; - - -- add_logical_resource has 13 IN parameters followed by 3 OUT parameters - EXECUTE 'SELECT * FROM {{SCHEMA_NAME}}.add_logical_resource($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)' - INTO o_current_parameter_hash, - o_interaction_status, - o_if_none_match_version - USING v_logical_resource_id, v_new_resource, - p_resource_type, v_resource_type_id, p_logical_id, - p_payload, p_last_updated, p_is_deleted, - p_source_key, p_version, p_parameter_hash_b64, - p_if_none_match, p_resource_payload_key; - -END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql deleted file mode 100644 index f28a2a70800..00000000000 --- a/fhir-persistence-schema/src/main/resources/citus/add_logical_resource.sql +++ /dev/null @@ -1,190 +0,0 @@ -------------------------------------------------------------------------------- --- (C) Copyright IBM Corp. 2022 --- --- SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- - --- ---------------------------------------------------------------------------- --- Procedure to add a resource version and its associated parameters. These --- parameters only ever point to the latest version of a resource, never to --- previous versions, which are kept to support history queries. --- implNote - Conventions: --- p_... prefix used to represent input parameters --- v_... prefix used to represent declared variables --- t_... prefix used to represent temp variables --- o_... prefix used to represent output parameters --- Parameters: --- p_logical_resource_id: the known primary key for the logical resource --- p_new_resource: 1 if this is a newly created resource --- p_resource_type: the resource type name --- p_resource_type_id: the resource type id --- p_logical_id: the logical id given to the resource by the FHIR server --- p_payload: the BLOB (of JSON) which is the resource content if any --- p_last_updated the last_updated time given by the FHIR server --- p_is_deleted: the soft delete flag --- p_version_id: the intended new version id of the resource (matching the JSON payload) --- p_parameter_hash_b64 the Base64 encoded hash of parameter values --- p_if_none_match the encoded If-None-Match value --- o_current_parameter_hash: Base64 current parameter hash if existing resource --- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit --- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) --- Exceptions: --- SQLSTATE 99001: on version conflict (concurrency) --- SQLSTATE 99002: missing expected row (data integrity) --- SQLSTATE 99004: delete a currently deleted resource (data integrity) --- Citus Distribed Function: --- For Citus, we split the ingestion logic into two stored procedures (functions) --- so that we can distribute these functions using the same distribution (sharding) --- key as the tables they interact with --- add_any_resource - distributed by logical_id, which matches the sharding --- of logical_resource_shards --- add_logical_resource - distributed by logical_resource_id, which matches the --- sharding of all the other tables used by statements in --- the procedure. --- ---------------------------------------------------------------------------- - ( IN p_logical_resource_id BIGINT, - IN p_new_resource INT, - IN p_resource_type VARCHAR( 36), - IN p_resource_type_id INT, - IN p_logical_id VARCHAR(255), - IN p_payload BYTEA, - IN p_last_updated TIMESTAMP, - IN p_is_deleted CHAR( 1), - IN p_source_key VARCHAR( 64), - IN p_version INT, - IN p_parameter_hash_b64 VARCHAR( 44), - IN p_if_none_match INT, - IN p_resource_payload_key VARCHAR( 36), - OUT o_current_parameter_hash VARCHAR( 44), - OUT o_interaction_status INT, - OUT o_if_none_match_version INT) - LANGUAGE plpgsql - AS $$ - - DECLARE - v_schema_name VARCHAR(128); - v_logical_resource_id BIGINT := NULL; - t_logical_resource_id BIGINT := NULL; - v_current_resource_id BIGINT := NULL; - v_resource_id BIGINT := NULL; - v_currently_deleted CHAR(1) := NULL; - v_duplicate INT := 0; - v_current_version INT := 0; - v_change_type CHAR(1) := NULL; - - -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. - lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_shards WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; - - BEGIN - -- default value unless we hit If-None-Match - o_interaction_status := 0; - - -- LOADED ON: {{DATE}} - v_schema_name := '{{SCHEMA_NAME}}'; - - -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) - SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; - - -- Create the resource if we don't have it already - IF p_new_resource = 1 - THEN - -- create the corresponding entry in the - -- global logical_resources table (which is distributed by logical_resource_id). - -- Because we created the logical_resource_shards record, we can be certain the - -- logical_resources record doesn't yet exist - INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) - VALUES (v_logical_resource_id, p_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64); - - -- similarly, create the corresponding record in the resource-type-specific logical_resources table - EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' - || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - ELSE - -- find the current parameter hash and deletion values from the logical_resources table - SELECT parameter_hash, is_deleted - INTO o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources - WHERE logical_resource_id = v_logical_resource_id; - - -- as this is an existing resource, we need to know the current resource id. - -- This is only available at the resource-specific logical_resources level - EXECUTE - 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' - || ' WHERE logical_resource_id = $1 ' - INTO v_current_resource_id, v_current_version USING v_logical_resource_id; - - IF v_current_resource_id IS NULL OR v_current_version IS NULL - THEN - -- our concurrency protection means that this shouldn't happen - RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; - END IF; - - -- If-None-Match does not apply if the resource is currently deleted - IF v_currently_deleted = 'N' AND p_if_none_match = 0 - THEN - -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the - -- connection with a fatal error, so instead we use an out parameter to - -- indicate the match - o_interaction_status := 1; - o_if_none_match_version := v_current_version; - RETURN; - END IF; - - -- Concurrency check: - -- the version parameter we've been given (which is also embedded in the JSON payload) must be - -- one greater than the current version, otherwise we've hit a concurrent update race condition - IF p_version != v_current_version + 1 - THEN - RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; - END IF; - - -- Prevent creating a new deletion marker if the resource is currently deleted - IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' - THEN - RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; - END IF; - - IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash - THEN - -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) - -- TODO patch parameter sets instead of all delete/all insert. - EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' - USING p_resource_type, v_logical_resource_id; - END IF; -- end if check parameter hash - END IF; -- end if new resource - - -- create the new resource version record - EXECUTE - 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' - || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' - USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; - - - IF p_new_resource = 0 THEN - -- As this is an existing logical resource, we need to update the xx_logical_resource values to match - -- the values of the current resource. For new resources, these are added by the insert so we don't - -- need to update them here. - EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' - USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; - - -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level - EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' - USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; - END IF; - - -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event - -- related to resources changes (issue-1955) - IF p_is_deleted = 'Y' - THEN - v_change_type := 'D'; - ELSE - IF p_new_resource = 0 - THEN - v_change_type := 'U'; - ELSE - v_change_type := 'C'; - END IF; - END IF; - - INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) - VALUES (v_resource_id, p_last_updated, p_resource_type_id, v_logical_resource_id, p_version, v_change_type); -END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql deleted file mode 100644 index d82815033c2..00000000000 --- a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource_distributed.sql +++ /dev/null @@ -1,239 +0,0 @@ -------------------------------------------------------------------------------- --- (C) Copyright IBM Corp. 2022 --- --- SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- - --- ---------------------------------------------------------------------------- --- Procedure to add a resource version and its associated parameters. These --- parameters only ever point to the latest version of a resource, never to --- previous versions, which are kept to support history queries. --- From V0027, we now use a logical_resource_ident table for locking. Records --- can be created in this table either by this procedure, or as part of --- reference parameter processing. --- implNote - Conventions: --- p_... prefix used to represent input parameters --- v_... prefix used to represent declared variables --- t_... prefix used to represent temp variables --- o_... prefix used to represent output parameters --- Parameters: --- p_logical_id: the logical id given to the resource by the FHIR server --- p_payload: the BLOB (of JSON) which is the resource content --- p_last_updated the last_updated time given by the FHIR server --- p_is_deleted: the soft delete flag --- p_version_id: the intended new version id of the resource (matching the JSON payload) --- p_parameter_hash_b64 the Base64 encoded hash of parameter values --- p_if_none_match the encoded If-None-Match value --- o_logical_resource_id: output field returning the newly assigned logical_resource_id value --- o_current_parameter_hash: Base64 current parameter hash if existing resource --- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit --- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) --- Exceptions: --- SQLSTATE 99001: on version conflict (concurrency) --- SQLSTATE 99002: missing expected row (data integrity) --- SQLSTATE 99004: delete a currently deleted resource (data integrity) --- ---------------------------------------------------------------------------- - ( IN p_resource_type VARCHAR( 36), - IN p_logical_id VARCHAR(255), - IN p_payload BYTEA, - IN p_last_updated TIMESTAMP, - IN p_is_deleted CHAR( 1), - IN p_source_key VARCHAR( 64), - IN p_version INT, - IN p_parameter_hash_b64 VARCHAR( 44), - IN p_if_none_match INT, - IN p_resource_payload_key VARCHAR( 36), - OUT o_logical_resource_id BIGINT, - OUT o_current_parameter_hash VARCHAR( 44), - OUT o_interaction_status INT, - OUT o_if_none_match_version INT) - LANGUAGE plpgsql - AS $$ - - DECLARE - v_schema_name VARCHAR(128); - v_logical_resource_id BIGINT := NULL; - t_logical_resource_id BIGINT := NULL; - v_current_resource_id BIGINT := NULL; - v_resource_id BIGINT := NULL; - v_resource_type_id INT := NULL; - v_currently_deleted CHAR(1) := NULL; - v_new_resource INT := 0; - v_duplicate INT := 0; - v_current_version INT := 0; - v_ghost_resource INT := 0; - v_change_type CHAR(1) := NULL; - - -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. - lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; - -BEGIN - -- default value unless we hit If-None-Match - o_interaction_status := 0; - - -- LOADED ON: {{DATE}} - v_schema_name := '{{SCHEMA_NAME}}'; - SELECT resource_type_id INTO v_resource_type_id - FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type; - - -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) - SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; - - -- Get a lock on the logical resource identity record - OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO v_logical_resource_id; - CLOSE lock_cur; - - -- Create the resource ident record if we don't have it already - IF v_logical_resource_id IS NULL - THEN - SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; - -- remember that we have a concurrent system...so there is a possibility - -- that another thread snuck in before us and created the ident record. To - -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn - -- around and read again to check that the logical_resource_id in the table - -- matches the value we tried to insert. - INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id) - VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; - - -- Do a read so that we can verify that *we* did the insert - OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id); - FETCH lock_cur INTO t_logical_resource_id; - CLOSE lock_cur; - - IF v_logical_resource_id = t_logical_resource_id - THEN - -- we did the insert, so we know this is a new record - v_new_resource := 1; - ELSE - -- another thread created the resource. - -- New for V0027. Records in logical_resource_ident may be created because they - -- are the target of a reference. We therefore need to handle the case where - -- no logical_resources record exists. - SELECT logical_resource_id, parameter_hash, is_deleted - INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources - WHERE logical_resource_id = t_logical_resource_id; - - IF (v_logical_resource_id IS NULL) - THEN - -- other thread only created the ident record, so we still need to treat - -- this as a new resource - v_logical_resource_id := t_logical_resource_id; - v_new_resource := 1; - END IF; - END IF; - ELSE - -- we have an ident record, but we still need to check if we have a logical_resources - -- record - SELECT logical_resource_id, parameter_hash, is_deleted - INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted - FROM {{SCHEMA_NAME}}.logical_resources - WHERE logical_resource_id = v_logical_resource_id; - IF (t_logical_resource_id IS NULL) - THEN - v_new_resource := 1; - END IF; - END IF; - - IF v_new_resource = 1 - THEN - -- we already own the lock on the ident record, so we can safely create - -- the corresponding records in the logical_resources and resource-type-specific - -- xx_logical_resources tables - INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) - VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; - - EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' - || ' VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; - - -- Since the resource did not previously exist, make sure o_current_parameter_hash is null - o_current_parameter_hash := NULL; - ELSE - -- as this is an existing resource, we need to know the current resource id. - -- This is only available at the resource-specific logical_resources level - EXECUTE - 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' - || ' WHERE logical_resource_id = $1 ' - INTO v_current_resource_id, v_current_version USING v_logical_resource_id; - - IF v_current_resource_id IS NULL OR v_current_version IS NULL - THEN - -- our concurrency protection means that this shouldn't happen - RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; - END IF; - - -- If-None-Match does not apply if the resource is currently deleted - IF v_currently_deleted = 'N' AND p_if_none_match = 0 - THEN - -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the - -- connection with a fatal error, so instead we use an out parameter to - -- indicate the match - o_interaction_status := 1; - o_if_none_match_version := v_current_version; - RETURN; - END IF; - - -- Concurrency check: - -- the version parameter we've been given (which is also embedded in the JSON payload) must be - -- one greater than the current version, otherwise we've hit a concurrent update race condition - IF p_version != v_current_version + 1 - THEN - RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; - END IF; - - -- Prevent creating a new deletion marker if the resource is currently deleted - IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' - THEN - RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; - END IF; - - IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash - THEN - -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) - -- TODO patch parameter sets instead of all delete/all insert. - EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' - USING p_resource_type, v_logical_resource_id; - END IF; -- end if check parameter hash - END IF; -- end if existing resource - - -- create the new resource version entry in xx_resources - EXECUTE - 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' - || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' - USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; - - IF v_new_resource = 0 THEN - -- As this is an existing logical resource, we need to update the xx_logical_resource values to match - -- the values of the current resource. For new resources, these are added by the insert so we don't - -- need to update them here. - EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' - USING v_resource_id, p_is_deleted, p_last_updated, p_version, v_logical_resource_id; - - -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level - EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' - USING p_is_deleted, p_last_updated, p_parameter_hash_b64, v_logical_resource_id; - END IF; - - -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event - -- related to resources changes (issue-1955) - IF p_is_deleted = 'Y' - THEN - v_change_type := 'D'; - ELSE - IF v_new_resource = 0 AND v_currently_deleted = 'N' - THEN - v_change_type := 'U'; - ELSE - v_change_type := 'C'; - END IF; - END IF; - - INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) - VALUES (v_resource_id, p_last_updated, v_resource_type_id, v_logical_resource_id, p_version, v_change_type); - - -- Hand back the id of the logical resource we created earlier. In the new R4 schema - -- only the logical_resource_id is the target of any FK, so there's no need to return - -- the resource_id (which is now private to the _resources tables). - o_logical_resource_id := v_logical_resource_id; -END $$; diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql index 01ea9279092..3172f41e4f9 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql @@ -17,10 +17,10 @@ AS $$ DECLARE - v_schema_name VARCHAR(128); + v_schema_name VARCHAR(128); BEGIN - v_schema_name := '{{SCHEMA_NAME}}'; + v_schema_name := '{{SCHEMA_NAME}}'; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE logical_resource_id = $1' USING p_logical_resource_id; @@ -40,21 +40,21 @@ BEGIN USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE logical_resource_id = $1' USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE logical_resource_id = $1' - USING p_logical_resource_id; - -- because we're a function, pass back a result - o_logical_resource_id := p_logical_resource_id; + -- because we're a function, pass back a result + o_logical_resource_id := p_logical_resource_id; END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql deleted file mode 100644 index 3724e0bf1c1..00000000000 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_distributed.sql +++ /dev/null @@ -1,60 +0,0 @@ -------------------------------------------------------------------------------- --- (C) Copyright IBM Corp. 2021 --- --- SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- - --- ---------------------------------------------------------------------------- --- Procedure to delete all search parameters values for a given resource. --- p_resource_type: the resource type name --- p_logical_resource_id: the database id of the resource for which the parameters are to be deleted --- ---------------------------------------------------------------------------- - ( IN p_resource_type VARCHAR( 36), - IN p_logical_resource_id BIGINT, - OUT o_logical_resource_id BIGINT) - RETURNS BIGINT - LANGUAGE plpgsql - AS $$ - - DECLARE - v_schema_name VARCHAR(128); - -BEGIN - v_schema_name := '{{SCHEMA_NAME}}'; - - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE logical_resource_id = $1' - USING p_logical_resource_id; - - -- because we're a function, pass back a result - o_logical_resource_id := p_logical_resource_id; -END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java index 172222218c1..614d3bf60dc 100644 --- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java +++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java @@ -52,7 +52,7 @@ public void test() throws Exception { // Make sure we can correctly determine the latest schema version value svm.updateSchemaVersion(); - assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0027.vid()); + assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0028.vid()); assertFalse(svm.isSchemaOld()); } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java index 539ea2c3b9d..12ba808966a 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java @@ -8,7 +8,7 @@ /** - * A token search parameter value + * A tag search parameter value */ public class TagParameter extends SearchParameterValue { private String valueSystem; diff --git a/fhir-remote-index/README.md b/fhir-remote-index/README.md index f449023bb14..ffbcf835e71 100644 --- a/fhir-remote-index/README.md +++ b/fhir-remote-index/README.md @@ -12,6 +12,8 @@ This pattern supports higher ingestion rates because: In addition, this implementation eliminates any possibility of deadlocks occurring during the Insert/Update interaction. Deadlocks may still occur when processing the asynchronous remote index messages. As these will only occur in a backend process they will not be visible to IBM FHIR Server clients. Deadlocks are handle automatically using a rollback and retry mechanism. Care is taken to reduce the likelihood of deadlocks from occurring in the first place by sorting all record lists before they are processed. +It is worth noting that using multiple Kafka topic partitions can increase throughput by allowing more resource parameter messages to be processed in parallel. If sufficient threads are allocated across all the fhir-remote-index consumer instances, each thread will read data from a single partition. There is no point allocating more total threads than the number of configured partitions. The partition key is a function of the {resource-type, logical-id} tuple which guarantees that changes related to a particular logical resource will be pushed to the same partition. This guarantees that these changes will be processed in order. + ## Processing Is Asynchronous Old search parameters are deleted whenever a resource is updated. When remote indexing is enabled, this means that a resource will not be searchable until the remote index service has received and processed the message. Carefully examine your interaction scenarios with the IBM FHIR Server to determine if this behavior is suitable. In particular, conditional updates are unlikely to work as expected because the search may not return the expected value, depending on timing. diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java index 33724b67d5e..77763ad8df5 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java @@ -238,6 +238,10 @@ public void process(String requestShard, String resourceType, String logicalId, try { PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType); dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId()); + if (p.isSystemParam()) { + // Currently we store _tag:text as a token value, and because it's also whole-system, we need to add it here + systemDao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId()); + } } catch (SQLException x) { throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'"); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java index 8b00326b21d..eb9a4a6209f 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java @@ -402,14 +402,13 @@ public void addLocation(long logicalResourceId, int parameterNameId, Double lat, public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId, Integer refVersionId, Integer compositeId) throws SQLException { if (resourceTokenRefs == null) { final String tablePrefix = resourceType.toLowerCase(); - final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, ref_version_id, logical_resource_id, composite_id) VALUES (?,?,?,?,?)"; + final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, logical_resource_id, composite_id) VALUES (?,?,?,?)"; resourceTokenRefs = connection.prepareStatement(tokenString); } resourceTokenRefs.setInt(1, parameterNameId); resourceTokenRefs.setLong(2, commonTokenValueId); - setComposite(resourceTokenRefs, 3, refVersionId); - resourceTokenRefs.setLong(4, logicalResourceId); - setComposite(resourceTokenRefs, 5, compositeId); + resourceTokenRefs.setLong(3, logicalResourceId); + setComposite(resourceTokenRefs, 4, compositeId); resourceTokenRefs.addBatch(); resourceTokenRefCount++; } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java index 65e7f4b94eb..d350684f7a5 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java @@ -29,6 +29,9 @@ public class PlainPostgresSystemParameterBatch { private PreparedStatement systemDates; private int systemDateCount; + private PreparedStatement systemResourceTokenRefs; + private int systemResourceTokenRefCount; + private PreparedStatement systemProfiles; private int systemProfileCount; @@ -58,6 +61,10 @@ public void pushBatch() throws SQLException { systemDates.executeBatch(); systemDateCount = 0; } + if (systemResourceTokenRefCount > 0) { + systemResourceTokenRefs.executeBatch(); + systemResourceTokenRefCount = 0; + } if (systemTagCount > 0) { systemTags.executeBatch(); systemTagCount = 0; @@ -98,6 +105,16 @@ public void close() { systemDateCount = 0; } } + if (systemResourceTokenRefs != null) { + try { + systemResourceTokenRefs.close(); + } catch (SQLException x) { + // NOP + } finally { + systemResourceTokenRefs = null; + systemResourceTokenRefCount = 0; + } + } if (systemTags != null) { try { systemTags.close(); @@ -176,6 +193,26 @@ public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateS systemDateCount++; } + /** + * Add a token parameter value to the batch statement + * + * @param logicalResourceId + * @param parameterNameId + * @param commonTokenValueId + * @throws SQLException + */ + public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId) throws SQLException { + if (systemResourceTokenRefs == null) { + final String tokenString = "INSERT INTO resource_token_refs (parameter_name_id, common_token_value_id, logical_resource_id) VALUES (?,?,?)"; + systemResourceTokenRefs = connection.prepareStatement(tokenString); + } + systemResourceTokenRefs.setInt(1, parameterNameId); + systemResourceTokenRefs.setLong(2, commonTokenValueId); + systemResourceTokenRefs.setLong(3, logicalResourceId); + systemResourceTokenRefs.addBatch(); + systemResourceTokenRefCount++; + } + /** * Add a tag parameter value to the whole-system batch statement * diff --git a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java index dcba1a443b6..d32f644a784 100644 --- a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java +++ b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java @@ -102,7 +102,8 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class MAX_RESOURCE_COUNT) { @@ -139,7 +140,7 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class Date: Wed, 15 Jun 2022 00:51:28 +0100 Subject: [PATCH 31/40] issue #3437 fixing drop column for ref_version_id Signed-off-by: Robin Arnold --- .../utils/api/IDatabaseTranslator.java | 7 +++ .../fhir/database/utils/common/DropView.java | 49 +++++++++++++++++++ .../database/utils/db2/Db2Translator.java | 5 ++ .../database/utils/derby/DerbyTranslator.java | 5 ++ .../utils/postgres/PostgresTranslator.java | 5 ++ .../control/FhirResourceTableGroup.java | 7 ++- .../schema/control/FhirSchemaGenerator.java | 2 +- 7 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java index 75e4a717767..19c726175c4 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java @@ -216,6 +216,13 @@ public interface IDatabaseTranslator { */ String dropForeignKeyConstraint(String qualifiedTableName, String constraintName); + /** + * Generate the DDL for dropping the named view + * @param qualifiedViewName + * @return + */ + String dropView(String qualifiedViewName); + /** * Does this database use the schema prefix when defining indexes * @return diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java new file mode 100644 index 00000000000..6a8d90d673f --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java @@ -0,0 +1,49 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; + +/** + * Drop columns from the schema.table + */ +public class DropView implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(DropView.class.getName()); + private final String schemaName; + private final String viewName; + + /** + * Public constructor + * @param schemaName + * @param viewName + */ + public DropView(String schemaName, String viewName) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(viewName); + this.schemaName = schemaName; + this.viewName = viewName; + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + final String qname = DataDefinitionUtil.getQualifiedName(schemaName, viewName); + final String ddl = translator.dropView(qname); + + try (Statement s = c.createStatement()) { + s.executeUpdate(ddl.toString()); + } catch (SQLException x) { + // just log because this means that the view doesn't yet exist + logger.warning("Drop view statement failed: '" + ddl + "': " + x.getMessage()); + } + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java index e5d3c0a2f33..4ee83a6a5b1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java @@ -265,6 +265,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP FOREIGN KEY " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java index f94a6d8d488..71e1ac774d8 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java @@ -193,6 +193,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP FOREIGN KEY " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { final String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java index 5bdc3f5500c..accf4062287 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java @@ -242,6 +242,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP CONSTRAINT IF EXISTS " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW IF EXISTS " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index 4feef8f1432..55565fc9212 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -85,6 +85,7 @@ import com.ibm.fhir.database.utils.common.DropIndex; import com.ibm.fhir.database.utils.common.DropPrimaryKey; import com.ibm.fhir.database.utils.common.DropTable; +import com.ibm.fhir.database.utils.common.DropView; import com.ibm.fhir.database.utils.common.ReorgTable; import com.ibm.fhir.database.utils.model.ColumnBase; import com.ibm.fhir.database.utils.model.ColumnDefBuilder; @@ -576,6 +577,10 @@ public Table addResourceTokenRefs(List group, String prefix) { } if (priorVersion < FhirSchemaVersion.V0028.vid()) { + // For PostgreSQL we can't drop the column because a view depends on it. Just drop the view + // because the definition has been updated and it will be added again later + final String viewName = prefix + "_" + TOKEN_VALUES_V; + statements.add(new DropView(schemaName, viewName)); statements.add(new DropColumn(schemaName, tableName, REF_VERSION_ID)); needReorg = true; } @@ -607,7 +612,7 @@ public Table addRefValues(List group, String prefix) { final String tableName = prefix + "_REF_VALUES"; - // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values + // logical_resources (1) ---- (*) patient_ref_values (*) ---- (0|1) logical_resource_ident Table tbl = Table.builder(schemaName, tableName) .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: tweak vacuum and fillfactor .setTenantColumnName(MT_ID) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 5b7dcc67a02..d50359a2b94 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -553,7 +553,7 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { private String getSchemaTypeSuffix() { switch (this.schemaType) { case DISTRIBUTED: - return "_distributed.sql"; + return ".sql"; case SHARDED: return "_sharded.sql"; default: From dd08e8799ffc8fb21b352c377b5388623055fef5 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 15 Jun 2022 10:11:28 +0100 Subject: [PATCH 32/40] issue #3437 tab to space and fixed comments Signed-off-by: Robin Arnold --- .../schema/control/FhirSchemaGenerator.java | 2 +- .../postgres/delete_resource_parameters.sql | 38 ++++++------ .../delete_resource_parameters_sharded.sql | 58 +++++++++---------- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index d50359a2b94..5ffd17e61c4 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -1361,7 +1361,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) { // logical_resources (0|1) ---- (*) resource_token_refs Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0028.vid()) // V0027: add support for distribution/sharding + .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: drop column ref_version_id .setTenantColumnName(MT_ID) .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql index 3172f41e4f9..292735c23ac 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql @@ -22,24 +22,24 @@ BEGIN v_schema_name := '{{SCHEMA_NAME}}'; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE logical_resource_id = $1' - USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' - USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE logical_resource_id = $1' + USING p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE logical_resource_id = $1' + USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE logical_resource_id = $1' USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE logical_resource_id = $1' @@ -54,7 +54,7 @@ BEGIN USING p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE logical_resource_id = $1' USING p_logical_resource_id; - + -- because we're a function, pass back a result o_logical_resource_id := p_logical_resource_id; END $$; \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql index e87822b7f58..e2278a70c0f 100644 --- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql +++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql @@ -27,37 +27,37 @@ BEGIN v_schema_name := '{{SCHEMA_NAME}}'; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values WHERE shard_key = $1 AND logical_resource_id = $2' USING p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE shard_key = $1 AND logical_resource_id = $2' - USING p_shard_key, p_logical_resource_id; - EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE shard_key = $1 AND logical_resource_id = $2' + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags WHERE shard_key = $1 AND logical_resource_id = $2' + USING p_shard_key, p_logical_resource_id; + EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security WHERE shard_key = $1 AND logical_resource_id = $2' USING p_shard_key, p_logical_resource_id; -- because we're a function, pass back a result From f37233b22d5d9cd2d51a481520a72e0bd2520174 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 15 Jun 2022 15:26:15 +0100 Subject: [PATCH 33/40] issue #3437 fixed fhir-bucket and corrected DropView comment Signed-off-by: Robin Arnold --- fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java | 2 ++ .../main/java/com/ibm/fhir/database/utils/common/DropView.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index 802a3a59de8..1860ef2124e 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -700,6 +700,7 @@ public void setupDb2Repository() { IConnectionProvider cp = new JdbcConnectionProvider(translator, propertyAdapter); this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.adapter = new Db2Adapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } @@ -723,6 +724,7 @@ public void setupPostgresRepository() { IConnectionProvider cp = new JdbcConnectionProvider(translator, propertyAdapter); this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.adapter = new PostgresAdapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java index 6a8d90d673f..8a279872eef 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java @@ -15,7 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; /** - * Drop columns from the schema.table + * Drop columns from the view identified by schema and view name */ public class DropView implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(DropView.class.getName()); From 5f7e25df6033e7e0764b9a41537c5d7c63102996 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 15 Jun 2022 15:27:25 +0100 Subject: [PATCH 34/40] issue #3437 corrected DropView comment Signed-off-by: Robin Arnold --- .../main/java/com/ibm/fhir/database/utils/common/DropView.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java index 8a279872eef..787fc3b8bd7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java @@ -15,7 +15,7 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; /** - * Drop columns from the view identified by schema and view name + * Drop the view identified by schema and view name */ public class DropView implements IDatabaseStatement { private static final Logger logger = Logger.getLogger(DropView.class.getName()); From 150e3c3a0a8e465af2b6adc562199ffa3a6b5d32 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 15 Jun 2022 16:16:13 +0100 Subject: [PATCH 35/40] issue #3437 javadoc review comments Signed-off-by: Robin Arnold --- .../ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java | 8 ++++++++ .../com/ibm/fhir/database/utils/api/ISchemaAdapter.java | 2 +- .../ibm/fhir/database/utils/api/SchemaApplyContext.java | 5 +++++ .../fhir/database/utils/citus/ConfigureConnectionDAO.java | 6 ++++-- .../database/utils/common/PreparedStatementHelper.java | 6 +++++- .../ibm/fhir/database/utils/model/IDatabaseObject.java | 2 ++ .../java/com/ibm/fhir/database/utils/model/IndexDef.java | 6 +++++- .../com/ibm/fhir/database/utils/model/SmallIntColumn.java | 3 +-- .../fhir/persistence/jdbc/connection/FHIRDbFlavor.java | 2 +- .../persistence/jdbc/connection/FHIRDbFlavorImpl.java | 2 +- .../fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java | 8 +++++--- .../java/com/ibm/fhir/persistence/FHIRPersistence.java | 2 +- .../context/impl/FHIRPersistenceContextImpl.java | 4 ++-- 13 files changed, 41 insertions(+), 15 deletions(-) diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java index 6860034c2a8..bab22adbc4d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java @@ -35,6 +35,14 @@ public default String doubleClause() { return "DOUBLE"; } + /** + * Generate a clause for smallint data type + * @return + */ + public default String smallintClause() { + return "SMALLINT"; + } + /** * Generate a clause for VARCHAR * @param size diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java index 536404a32a3..7af8243cca8 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java @@ -228,7 +228,7 @@ public void createIndex(String schemaName, String tableName, String indexName, S public void activateRowAccessControl(String schemaName, String tableName); /** - * Deactivate row access control on a table ALTER TABLE DEACTIVATE ROW + * Deactivate row access control on a table ALTER TABLE tbl_name DEACTIVATE ROW * ACCESS CONTROL * * @param schemaName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java index e7b0cf284e8..1c7cc32b2d9 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java @@ -20,6 +20,11 @@ public class SchemaApplyContext { protected SchemaApplyContext(boolean includeForeignKeys) { this.includeForeignKeys = includeForeignKeys; } + + /** + * Get the includeForeignKeys flag + * @return + */ public boolean isIncludeForeignKeys() { return this.includeForeignKeys; } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java index 039c07682b0..c75b26004ff 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java @@ -16,8 +16,10 @@ /** * DAO to configure the Citus database connection when performing schema build * activities. This must be performed before any of the following UDFs are called: - * - create_distributed_table - * - create_reference_table + *
    + *
  • create_distributed_table + *
  • create_reference_table + *
* to avoid the following error: * 
  * org.postgresql.util.PSQLException: ERROR: cannot modify table "common_token_values" because there was a parallel operation on a distributed table in the transaction
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java
index e2fd1de3f8f..7be7e257cf1 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java
@@ -17,8 +17,12 @@
  */
 public class PreparedStatementHelper {
     // The PreparedStatement we delegate everything to
-    private final Calendar UTC = CalendarHelper.getCalendarForUTC();
     private final PreparedStatement ps;
+
+    // The calendar to make sure all times are treated as UTC
+    private final Calendar UTC = CalendarHelper.getCalendarForUTC();
+
+    // The current parameter index in the statement
     private int index = 1;
 
     /**
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java
index 74d0a8a7b9c..1ea54e4170e 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java
@@ -47,6 +47,7 @@ public interface IDatabaseObject {
     /**
      * Apply the DDL, but within its own transaction
      * @param target the target database we apply to
+     * @param context the context used to modify how the schema objects are applied
      * @param cp of thread-specific transactions
      * @param vhs the service interface for adding this object to the version history table
      */
@@ -107,6 +108,7 @@ public interface IDatabaseObject {
      * executed concurrently (but in the right order)
      * @param tc
      * @param target
+     * @param context
      * @param tp
      * @param vhs
      */
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
index 9867b98fcef..d6e900123a5 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java
@@ -67,9 +67,13 @@ public boolean isUnique() {
 
     /**
      * Apply this object to the given database target
+     * 
+     * @param schemaName
      * @param tableName
+     * @param tenantColumnName
      * @param target
-     * @param distributionRules
+     * @param distributionType
+     * @param distributionColumn
      */
     public void apply(String schemaName, String tableName, String tenantColumnName, ISchemaAdapter target,
             DistributionType distributionType, String distributionColumn) {
diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java
index 753718b1764..2aa9075ba10 100644
--- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java
+++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java
@@ -23,7 +23,6 @@ public SmallIntColumn(String name, boolean nullable, String defaultValue) {
 
     @Override
     public String getTypeInfo(IDatabaseTypeAdapter adapter) {
-        // TODO ask the adapter for the type name to use for a smallint type column.
-        return "SMALLINT";
+        return adapter.smallintClause();
     }
 }
\ No newline at end of file
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java
index 662a310cd7a..30eb68cd212 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2020
+ * (C) Copyright IBM Corp. 2020, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java
index 777da38703d..c1c14c3df2f 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2020
+ * (C) Copyright IBM Corp. 2020, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java
index 64fe5b71e2f..746cbb3f925 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java
@@ -117,9 +117,11 @@ public interface JDBCIdentityCache {
      * Get a list of logical_resource_id values matching the given logicalId without
      * knowing the resource type. This means we could get back multiple ids, one per
      * resource type, such as:
-     *   Claim/foo
-     *   Observation/foo
-     *   Patient/foo
+     * 
    
+     * 
  • Claim/foo
+     * 
  • Observation/foo
+     * 
  • Patient/foo
+     * 

      * @param tokenValue
      * @return
      */
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
index 9ee3113c1cd..d7f32e8b807 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
@@ -268,7 +268,7 @@ List changes(FHIRPersistenceContext context, int resour
     /**
      * Erases part or a whole of a resource in the data layer.
      *
-     * @param context
+     * @param context the FHIRPersistenceContext associated with this request
      * @param eraseDto the details of the user input
      * @return a record indicating the success or partial success of the erase
      * @throws FHIRPersistenceException
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
index 5371e0382ac..b43d4241b01 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
@@ -107,8 +107,8 @@ public Builder withIfNoneMatch(Integer ifNoneMatch) {
         }
 
         /**
-         * Build with the shardKey value
-         * @param shardKey
+         * Build with the requestShard value
+         * @param requestShard
          * @return
          */
         public Builder withRequestShard(String requestShard) {

From 5aa932597d589faa8b9a47fb2d5d72e2719409d5 Mon Sep 17 00:00:00 2001
From: Robin Arnold 
Date: Fri, 17 Jun 2022 10:28:31 +0100
Subject: [PATCH 36/40] issue #3437 skip parameter storage for remote index
 with derby and db2

Signed-off-by: Robin Arnold 
---
 .../ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java | 6 +++++-
 .../ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java   | 4 +++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java
index 5b6d80e8f41..5833512dc9d 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java
@@ -38,6 +38,7 @@
 import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException;
+import com.ibm.fhir.persistence.index.FHIRRemoteIndexService;
 import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache;
 import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor;
 import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants;
@@ -572,8 +573,11 @@ public Resource insert(Resource resource, List paramete
                 // TODO FHIR_ADMIN schema name needs to come from the configuration/context
                 // We can skip the parameter insert if we've been given parameterHashB64 and
                 // it matches the current value just returned by the stored procedure call
+                FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance();
                 long paramInsertStartTime = latestTime;
-                if (parameters != null && (parameterHashB64 == null || !parameterHashB64.equals(currentHash))) {
+                if (remoteIndexService == null 
+                        && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty()
+                        || !parameterHashB64.equals(currentHash))) {
                     JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(cache, this, parameterDao, getResourceReferenceDAO());
                     try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", resource.getResourceType(), true,
                         resource.getId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) {
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java
index e4ded5ddd36..9e13a2f1714 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java
+++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java
@@ -31,6 +31,7 @@
 import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException;
+import com.ibm.fhir.persistence.index.FHIRRemoteIndexService;
 import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache;
 import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor;
 import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants;
@@ -545,7 +546,8 @@ public long storeResource(String tablePrefix, List para
         // To keep things simple for the Derby use-case, we just use a visitor to
         // handle inserts of parameters directly in the resource parameter tables.
         // Note we don't get any parameters for the resource soft-delete operation
-        if (parameters != null && requireParameterUpdate) {
+        FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance();
+        if (remoteIndexService == null && parameters != null && requireParameterUpdate) {
             // Derby doesn't support partitioned multi-tenancy, so we disable it on the DAO:
             if (logger.isLoggable(Level.FINEST)) {
                 logger.finest("Storing parameters for: " + v_resource_type + "/" + p_logical_id);

From c446a15e0d22b4109347028df86ce80710e54789 Mon Sep 17 00:00:00 2001
From: Robin Arnold 
Date: Mon, 20 Jun 2022 13:54:23 +0100
Subject: [PATCH 37/40] issue #3437 add instanceIdentifier to remote index
 messages and other review fixes

Signed-off-by: Robin Arnold 
---
 docs/src/pages/guides/FHIRServerUsersGuide.md |   3 +
 .../ibm/fhir/config/FHIRConfiguration.java    |   1 +
 .../com/ibm/fhir/core/util/LogSupport.java    |  52 +++++++++++++
 .../fhir/core/util/test/LogSupportTest.java   |  46 +++++++++++
 .../docs/physical_schema_V0027.png            | Bin 212701 -> 157711 bytes
 fhir-persistence/pom.xml                      |   4 +
 .../helper/RemoteIndexSupport.java            |  73 ++++++++++++++++++
 .../persistence/index/RemoteIndexMessage.java |  15 ++++
 .../index/SearchParametersTransport.java      |  14 +++-
 .../helper}/MessageSerializationTest.java     |  29 ++++---
 fhir-remote-index/README.md                   |   7 +-
 .../com/ibm/fhir/remote/index/app/Main.java   |  61 +++++++++++++--
 .../index/database/BaseMessageHandler.java    |  41 +++++-----
 .../DistributedPostgresMessageHandler.java    |   6 +-
 .../database/PlainDerbyMessageHandler.java    |   5 +-
 .../index/database/PlainMessageHandler.java   |   8 +-
 .../database/PlainPostgresMessageHandler.java |   7 +-
 .../ShardedPostgresMessageHandler.java        |   5 +-
 .../fhir/remote/index/RemoteIndexTest.java    |  19 ++---
 .../kafka/FHIRRemoteIndexKafkaService.java    |  17 +---
 .../index/kafka/KafkaPropertyAdapter.java     |  14 +++-
 .../listener/FHIRServletContextListener.java  |   4 +-
 22 files changed, 343 insertions(+), 88 deletions(-)
 create mode 100644 fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java
 create mode 100644 fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java
 create mode 100644 fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java
 rename {fhir-remote-index/src/test/java/com/ibm/fhir/remote/index => fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper}/MessageSerializationTest.java (90%)

diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md
index dda5dd4569c..8c60804ebc0 100644
--- a/docs/src/pages/guides/FHIRServerUsersGuide.md
+++ b/docs/src/pages/guides/FHIRServerUsersGuide.md
@@ -2266,6 +2266,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/operations/membermatch/extendedProps`|object|The extended options for the extended member match implementation|
 |`fhirServer/operations/everything/includeTypes`|list|The list of related resource types to include alongside the patient compartment resource types. Instances of these resource types will only be returned when they are referenced from one or more resource instances from the target patient compartment. Example values are like `Location`, `Medication`, `Organization`, and `Practitioner`|
 |`fhirServer/remoteIndexService/type`|string| The type of service used to send remote index messages. Only `kafka` is currently supported|
+|`fhirServer/remoteIndexService/instanceIdentifier`|string| A UUID or other identifier unique to this cluster of IBM FHIR Servers |
 |`fhirServer/remoteIndexService/kafka/mode`|string| Current operation mode of the service. Specify `ACTIVE` to use the service|
 |`fhirServer/remoteIndexService/kafka/topicName`|string| The Kafka topic name. Typically `FHIR_REMOTE_INDEX` |
 |`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|string| Bootstrap servers for the Kafka service |
@@ -2421,6 +2422,7 @@ This section contains reference information about each of the configuration prop
 |`fhirServer/operations/membermatch/extendedProps`|empty|
 |`fhirServer/operations/everything/includeTypes`|null|
 |`fhirServer/remoteIndexService/type`|null|
+|`fhirServer/remoteIndexService/instanceIdentifier`|null|
 |`fhirServer/remoteIndexService/kafka/mode`|null|
 |`fhirServer/remoteIndexService/kafka/topicName`|null|
 |`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|null|
@@ -2613,6 +2615,7 @@ Cases where that behavior is not supported are marked below with an `N` in the `
 |`fhirServer/operations/membermatch/extendedProps`|Y|Y|Y|
 |`fhirServer/operations/everything/includeTypes`|Y|Y|N|
 |`fhirServer/remoteIndexService/type`|N|N|N|
+|`fhirServer/remoteIndexService/instanceIdentifier`|N|N|N|
 |`fhirServer/remoteIndexService/kafka/mode`|N|N|N|
 |`fhirServer/remoteIndexService/kafka/topicName`|N|N|N|
 |`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|N|N|N|
diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java
index d70fdcb2e39..48cbf4e6010 100644
--- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java
+++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java
@@ -115,6 +115,7 @@ public class FHIRConfiguration {
 
     // Configuration properties for the Kafka-based async index service
     public static final String PROPERTY_REMOTE_INDEX_SERVICE_TYPE = "fhirServer/remoteIndexService/type";
+    public static final String PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER = "fhirServer/remoteIndexService/instanceIdentifier";
     public static final String PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME = "fhirServer/remoteIndexService/kafka/topicName";
     public static final String PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS = "fhirServer/remoteIndexService/kafka/connectionProperties";
     public static final String PROPERTY_KAFKA_INDEX_SERVICE_MODE = "fhirServer/remoteIndexService/kafka/mode";
diff --git a/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java b/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java
new file mode 100644
index 00000000000..1b17b6a2cbb
--- /dev/null
+++ b/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java
@@ -0,0 +1,52 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.core.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Static support functions related to logging
+ */
+public class LogSupport {
+    private static final String MASK = "*****";
+    private static final Pattern PASSWORD_EQ_PATTERN = Pattern.compile("[^\"]password[= ]*\"([^\"]*)\"", Pattern.CASE_INSENSITIVE);
+    private static final Pattern PASSWORD_PATTERN = Pattern.compile("\"password\"[: ]*\"([^\"]*)\"", Pattern.CASE_INSENSITIVE);
+
+    /**
+     * Hide any text in quotes following the token "password" to avoid writing secrets to log files
+     * @param input
+     * @return
+     */
+    public static String hidePassword(String input) {
+        String result = hidePassword(input, PASSWORD_EQ_PATTERN);
+        result = hidePassword(result, PASSWORD_PATTERN);
+        return result;
+      }
+    
+    /**
+     * Replace any text matching the given pattern with the MASK value
+     * @param input
+     * @param pattern
+     * @return
+     */
+    private static String hidePassword(String input, Pattern pattern) {
+        final Matcher m = pattern.matcher(input);
+        final StringBuffer result = new StringBuffer();
+        while (m.find()) {
+            final String match = m.group();
+            final int start = m.start();
+            m.appendReplacement(result, 
+                match.substring(0, 
+                    m.start(1) - start)
+                    + MASK
+                    + match.substring(m.end(1) - start, m.end() - start));
+        }
+        m.appendTail(result);
+        return result.toString();
+    }
+}
diff --git a/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java b/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java
new file mode 100644
index 00000000000..dd925c11832
--- /dev/null
+++ b/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java
@@ -0,0 +1,46 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.core.util.test;
+
+import static org.testng.Assert.assertEquals;
+
+import org.testng.annotations.Test;
+
+import com.ibm.fhir.core.util.LogSupport;
+
+/**
+ * Unit tests for {@link LogSupport} methods
+ */
+public class LogSupportTest {
+
+    @Test
+    public void testPassReplaceEquals() {
+        assertEquals(LogSupport.hidePassword("something password=\"change-password\" something else"), "something password=\"*****\" something else");
+    }
+
+    @Test
+    public void testPassReplaceEqualsWithSpace() {
+        assertEquals(LogSupport.hidePassword("something password = \"change-password\" something else"), "something password = \"*****\" something else");
+    }
+
+    @Test
+    public void testPassReplaceJson() {
+        assertEquals(LogSupport.hidePassword("something \"password\": \"change-password\" something else"), "something \"password\": \"*****\" something else");
+    }
+
+    @Test
+    public void testPassReplaceJsonCompact() {
+        assertEquals(LogSupport.hidePassword("something \"password\":\"change-password\" something else"), "something \"password\":\"*****\" something else");
+    }
+
+    @Test
+    public void testPassReplaceMixed() {
+        final String src = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"token\" password=\"change-password-\";";
+        final String tgt = src.replace("change-password-", "*****");
+        assertEquals(LogSupport.hidePassword(src), tgt);
+    }
+}
diff --git a/fhir-persistence-schema/docs/physical_schema_V0027.png b/fhir-persistence-schema/docs/physical_schema_V0027.png
index 48c12d2e6d1f54b5c55d4c5c32195a1b87e89fe3..9b6770cbc66f9414c69f9b5d3f6c63545e68adbb 100644
GIT binary patch
literal 157711
zcmXV%V|*M;w8mqn+1P6Aq_OR!v2A-B+cp|ojcsk%B#mu0w%NPyz4y!Rr`eg`&YU^t
zJkNikl)p=(A`u`#KtQ0%%1Ed}K)`&3fPhj%fP;X5{N)*u4FP4fBr74R?wNC*3vWCx
zg&UZEabEu<7g+y9ZIV#`R3@ThafND;mfuL1s$;3A8Pw>p>T{*In*D8%I+U_CpeIEs
z-D0(Yc$DF(MQZK@$TwPE?%{E@bvI|*(r+Aj)2Jm{+F9Eyf5z8U5eFhUWVIi4j4Lu-
zg);Xs|ASJo;tC<*_X*)Io1EQad6k@B=slW$t^pgKTd!AKU>qW{XA6iFBpiBwxr-`M
zrB5@SW-6Ms9(#OB$@MKYKE@nR7H3n^?MA{cc)Lizgp@zpNU
zih=6SJvDP+bP1%;73MGBn*<+m4G<6XqUS=g15v7?_jJMgP~adzSi5K}KD(v;a5Q#8
zXhg)-)mG0NRQ#pz=kxx-)75r){|B={C*$8^(+fe2y$H2Rb#r+m^09wBd_PX_NB2h)
zLRbts)tj~I4Eq+Vw48kI{{C6!v|FsDsaUs@=BLE#!oCEx=!;vk_Mvxoy^(ywnnLjK*a)Mc$?IRI
zL08_3z;J%*XY#x|=Condt%Y>KO&Kv`FwHHZ)o;;mD5-<*xO*Nia6)Y;xdvAI`~4))
zpG*E%Z({udSG3q)mUkAlAYv1wr!XpT^eG@;t4s?d5axobt#lC%`C}yUZe|!sXWZ}=
zA6H{e0Q*rzmlE#0Ol|V@v1`7ogs^TW532EyUEt|tQ(h+Po8XXC05l7!QL?-J2N5mC
zR&dF-Knvz=_+oefPJEzwCA-**rC+F>R?fSaAz_XD8=~s)yi0UZCZNF6sf>`G_vz0y
z(dRXS10eD_ViYcD;e>0pV&m~J`P7htAcQ26W?v;PxK|7Wb(L5{;mg*ey{%;iuwq
zwZD*|@6OQX9gYj?)mEL%WG_$P$!`}WlTT+nv2K&&!7G+xvpy;1c{BK@@5hjRIa%mv
z53nM!A!F?N2-}|QAuVw7R9$)#F)c3M`KY2vq?TuH_jM*VN1;Plj&}@G$8Qpi28t%u
zDC@Ml_!1tlXl<)Ja3X#ho2;hQ-PRpTz8_(VPa~
z;ClOdmt}!uz7h8Z4TthGXdN6_i5;#vbu-qwE@;)NAxEq|GD%-Ec%v!R4~HYIcrT)~
zgA|*SK}5}>WX){KLwDHsIqjmpA2@{V`wJoNf9MahoP2)#aog`8K(c(qLGeFm_qYKZ
z&}wF$JLPj@oj&B)wkm|}#j$qhT2qD9>|D2XkcPP44wQP8P3_u^X}SscW+B6S4g6Be
zC-;jy@dZ!g!if+ni$tj1dG?w>Y>WMqQE|D#Yi0%FS`l4w}cYB_-v}zBf5_
zHSJZNFa)o>`XS|tI&&Yc`l}<^xjqL{a+#bH>p2q>cSAQ9EeZQ%oUq~IyX=_zsM(9a
z3Rxm}aY#1lGBpxRbj-b{g%6r=ff*pQ?$pnMy;gcyf+Ox~P|QesBlUXDMw<4Gt$f&eMH05(w3_
zP{S|TO;CIqHgK`k{VC}42*1~X<^fIuCdD(Zal!Jcwmk)s5ulu*>j>S4m}DjbkaY)K
zBAp$??}%jR1dTGooiA{NL#;8Sg(gv0`o*jQPuHi2p3aA~>Ne~shJvpdXXur?6A|3z
z6+(a4wZ;Idi3bBFK6DP_73H#laaMz_QW&
zx6qziWQZw`ZuAkC!`i}6GfXZddw}azz^1|K3%iI?Z4nOw$-E7NnJS(~QqoYe*5tF?ki&(>VLeA|~57;PA916vsShivA)
z0bImo6r{Q$MT?c)I)j7d#d?Yyp7tb8&V>pvPT~T;muolnVwS5(+?BoShs>-@h;iku
zeDkLJC{*Nx;TzA?X0^uZ9XGG+9;+J6ICJn!xugZ@*)Y5a*6t43_PNvnJ|BYWk~)O{
z-Q}tM&QbOg+#{XU8+eLL?WBU1QF)Nocw
z`FrCmCm+ng?n*%WYbrZ=IOVjk*}hWJ8!o1h38WZywk4-P3p
z+3;>Nptt0cO4^&!Da3c;mO;`uuGpqHrPZ1Zx5Nm0McL?qt)3QOO*K?%XG?Q?c>2Hj
zJw6?MiWpMV4~W{#kNJ~%s4~`5zm2yiXqabsv3R0P2rWIq-hYs;e^I{H#qoO>iYm)s
z+^+F>9WbXJ7?Kmz*xd^y(q)r?>PJ>l<~slIA_$&g2gi+G4GjA@8Lt-D&Ug%$}=>}nV4{Hi*%3e+13Ob
z$yHLmA}q=_Q*NfQS!3zes)QaWBJt3LZ+=Hz1_X;IV-92dPNzV9N;8dg)oC?#{FYd%
z+CH)Y|7k5w05@1r+O})#;P(m=(im%%dE^jViyl}E`}mss473M~?fGk1Oq9o&4Thtx
zfvc3Hx1QG$gSy2BjYch$pdzN2!M3N_dZz>b3a(y0^Dlo;e5B8Viio4d6x2EFiJOBM
z%i^cAbfk@EXywX`_L2uTmagTya-2mo^^W6a&@{w`Z5BY|Gjrq4>S}ZZn-W=E4z}0a
zYsK0nl2P7A$BOOKvP$ZcDyHZ74cSJ1)3V9ZR+lH5hi67?wXx|*UkUw6o-dyYBTag1i0RWeZF6O@IHycyh-kvqq=ET1
zxIzDHagYAo>xTEv_s9yo)W9B3r9+T@+zt+K5(+J-{=K_9i0l0C64~>e73yzo<$iT1
zA)5ue?lP@$$e7aa)E~IU+IM^A0if+8W6E;r=KNkX92f7w3#+R_<<l?P8TcrZc
zxvjXtbCf|*#N&9W3@yE<8y$C7)@mx9z!P?B@gR8waJ_BdHj&PR`tkl=biI-^L3kc#
zo)bJz(OxpJKkn9(2=TM0nYY{sCVBS~X}6<5f3do=A$Fr=xo}A_u>(q?+&VEVporrB
z1e_UH$N=hG;0QwCLXfDolbOJ|d%NoMD0^E7hw{_gjVSA~3lZ5cJ>4Ks=I&RKsr*h(
zGMe~EJLyQ(R2Bu9&@fvfyfSX)w9_W$bD??J(R`s(>}(}f?GKZ(ZVvBh^;5{|4EuP#
zpk9>V3-NjHHJ)u60`=Q6iR2DF7j4a1mMVr`h=@R$%1#1TuSMc{0*wOzAzslWq~J}x
zFOfm3TM@-mp%~!y)r+Bt57snEh`dP=WvEMKhd|Y5iW6Wv-JrXR{m=SPxTZl*qdthL
zHQYo?*5&z>y+WO~N#!+s|Ea*HgXpn4ZTP+64G#D#bNk@Mp`29YW9FQCTHb*R6WZMd
z3r=m(Tzh{%Gz*&tk!
z_C4Z|S!W6Gl;6&lg{Q>cwyl;n-rh!H{;aG%qH9c4NU(XY0v?!^dr9HxFC0jV5iGTG
zn{Hr>K};@`Za}g~TQTc+Sf{xT$Tm;-)jQ}-`dW$=395IABtTxXREB>dxMaAq>11g4
zYkBzF2*=Z`qE1-<`KK5E&)g1%fBe?ZrDExX7b3wuJua^3k7Z$TPsq>Bpv`@)t3J?k
z-!!PGSHt9^q3indj{E0g&sT+QPjr&LSC~6*jlVzu6)>lP6z8E5syIDAxGle^K>WeC
z{&dFSxyFiW4j?2kBBt6GY9us9xMwZniQ<2g2A9R+gqnqiYX51%-n_K{mzU>@G+IVP
z6<07DY+2v&=55WU@M*N%P}XU&zi^E{FOS;A`f@1d9LO^!}5(e
zLzrUq)9rwLPAl~&BmGoGJf&)h8uJe
z#vA~@l>>otfD-l*M)@Ws7+RYI_^9FT{Dbe>dZOVnlqy`6OR2MZQc1AX+Uc
z&VF9iCYJF@Ahl+QW#OrtotxHaRUT}CGuKXw|6SH348S)($VW~)x{p`#=
zA`t~vhufEXFvAD=Z_POEs{CC11KyKmQnL=U{*jn>xX_s2VBs>#IYnu;zN5r
zK_=$!pKO|ydB#PW)S)WCrBBL;s1bUFnlUtFnC7+y_6cT|=h6%lVHjOS`=T)s>;4R*;yHl(9dC+ZK&<***%%b*>(0l>oxvVg#6;4yl)P~oC%
z>NDLC$vkbv_l6S?uD1Rj0;x80?XcZ_l&Wm&*T0`ykaN^*_UeeR+i%K=%MfvEk=?xy
z7+V|Ois{au^B(Xv0&O0fdv}nLC9v*R9a^!1BXlxtbCdk6^z`Fsipxg5zL|OZ7TuRW
z12??zQBh-FC-SEER5vlA#azFDAsX}s6rz;{RKlH_$9@O~IuL%#934&GlOKqhU1a7<4^2ay6|M$bvArLqo*CjbVyV70F1BjccYfMl%br5d<`@0$V7BTC
z9M^+Ij%EfhpZj*QzmvRniJF)-o8`Qg9(#fuE9j50${yfe<&?8|t~}iYY_O~cO&r$<
z3wzM~V5~Q6Whkxvlqk1)&`8tZT0M59KUiQRDQT4BL-5uz;l^6i$pn;;siRjyWOxUZ(kx
zpgUTV0hTbdz+Jx3?aw=FnWByBhUF=VHg7XrwSXt`n)+|?sePZ9hwa?m&zJ;dn`0@@
z#_o_>o}<^EM$3bS(G79dK9575JKi}WB~n=3I_aI(8z)6$y#$96nxeVWt^!nlJA`bX
znhb^Lo6Xf{vTmUYM7{74lOfqZ?zVf$46*zX{3fh!m;rgyXr1qzp?-Rk#{ghnwERsw
zjlm}P6t5Ebfjx@2bJU7~1if85Ir>f&VjcJ+o9?b`ncoizzc)c@X
zIBl_-g`Pzx8XC+-G05?a?AQt!er03LpHlXx=Ml;xIK!T}k0h6cbhO>C(dx?+Lg!+|
zpotYL3ko9~W<93=Y5p-KVm))D71A6x?<1Nb+BbZ4$u~5m2F@s&om}&(O+9X(jiF40&EOg;m%{%N2EuCX&k{_;y4QSS|
zGTV7mHY#>I?ik5XOPw`SEY9~3c|`dwGY#Fd08`_+Y;cSL(3?b{rbN%Eo$m!+=lDG7ZunD4&JYBG9bNW2mE3@T!-!
z|NDVpgS97HBi7m8lp=k`?zbKLkBo`%dcE_tuSs4cpjjx*&sf~tB3mmkGs_US+=~y~
zfp7lOV93CGzJck+NTJb@+E|g4OTBEBvr7m+w9nT;;parKN7;jDSx#^WgE}Pa2lty~
zeq1jP*(&+Mfkw#(kL`kk@syDyL2;05i+BJZu5i?@?~{HHJFYBt`@Y&3P*8;wnNgxz
z4e}w8@(ZM0uhouIlXSI~GT7E%@yy>nKd$9iDns1u%zS^w03;GTu7o=aEj`1^5O0X}
zVI%H2#&O5H_jh9wyugB|%XwV(R_Ot|3br)U>WJuS$Ynkk`CY4gZ&71SpS1D2^D@`+*jKy5km$?ze>du
z@eXWR%#ogb>S1Fwx`ohY53Y{|{Q??76GA&SuYg9mAIS_z3E?^m!
zC=3a^GkW6Qxr3zj85x%uYxIL!G0XnI`fIP}Yt|h*d<)|&>7NIBWj2LJ$3aga|9kUKbf
z0uB!=cn3?W3y2fFP;CoviMn_hg!=g3KfQ^92OC9~>Z{Fb_A7O0$jJ=8(JuS~5as*8
zaOm2)*HSff+8f-{Yr@2UG^((qZ4$&^i_JV`L)+FKeXG4Kex4~_#~3~=iAa}`Qi^^j
zAq_xKiWkPA9Lmr#+BrDi@7}EdIU{D*q>Jp4K8q>-Ju2&#g$&M%YJdmFgxE
zkvSDbN0q)%M=sC}gRn}UYVfyobS03^VFCt;VGL8eo~Mdns7p!3MveNx$6+FFPnQOR
zbjScy-_LK(u!LxjC{g8MX-V{Ib23*-yi~0-|IKcxI$?(5UpOjZR?%nJrpiL!e{v^$
z*Ag~^roh*S*~3%q4YFSU7}Vw#HcE~?()<5(O(uSU%>Z)-M{Qz0pmsEAR=Lf5*%0iP
z3)MRPj=7GjX;0$?vc3JvKP0``_5UmJnxx*b&2v@z*MHAXH$hj68lvbV2b@Qss+FT-^
zJ5i(PrXJ==Htm(o8t0K@yEeZL6$Lh_gwk!3!IOnd#11fKi7r9Lsd?Yd;
z#XGjK*}Y1?rr~;1Gc?clYo-GPL~5=tXqSC~`_grTpPS3ZkGE3h4fU$%XVsm^k*Ef(
zf(jf=iKaj!V;pC3@S8OcWAdJEw5t^@K^<3YU0jRfkSTZL(CAy{hOjXc8DlILL(Y={iLeQq`pAA5biuazYJh1$eRWr)u$1*<>2tEz-
z8%dfhE9mg9>`N3x+}DRlcpM16oCRMi2ee>3kaD9YGdt0Qy5Hd9vLnO__c9BVBB&DU
zjQNhg6<}pV_*huy{u>uLZxEBOW*7^zltBTHUVxV=hQOYsNX-dewUXhSJDU+hnI-(o
zY$M*ff~Vf^Q{ckoded2rpNo|tj?N8b!!nq{W(^PmD~8t^qK23|lJAN)j`ya4Xrv|7
zaY639Csl*1k%XabwL$d90lJ%LvXXs3eX<)a|HKREgI(ch#RdUJ>1?zp6Ph$eA^smm&B-X(5
z5DpLLM@`dKKi8~mq%d_3q*Lw0?>_2ou+kPwKkC7c?W>h?=KU)Uh_KB1ZNxBGkT~<}
zQL2}M)MHAxTJp$6iJhhjB#j4$SX`wK6L1jUH@rEcBAuW-4)63QW8>s@t``Lhiy3`#`OZ7dEW)$lEE5_
zM?KD98t*b;SMnYq4u>!_m?QO^cP<5yIcHRP)s5UjAW3MrZ-R$T60Oa&ybQqsC9;_<
z622CExcl@INCf9B#}XlRGnA0CfoD3L5??EJBR&L;ZQ_JQ(W?S=*=+}FA^AnIk5M_x
z7hy{4FqVKX3|o2Um;}FP9>9XInG1P30C&!VgCHZ(-@0=}r?PTZ4v?11>-LwQ*Y$u_
z@u~|w-G&oMsDhxeV$DfHZio02zdZz~XGXDDU%Bi(C$E-V+m&`hNE8dct!_K>aXt)u
z=%9cG-;Iid3rK>l2N#pf$Jd=-*EzqAkPREHEeG6}#^#P`)|R8^lU>$E-!O#uXq<0)
zq2>uZ!)(4Skcsw>By6Fuu{B)x(Zw0xRtN$NX=o1UQZc&Um3M)k4A249{L_7)zfI`%
zq8K%^>Zzu&?7cTbei&P`>LhG?hEi>gn>a=Mg_~U-%kTFmYI9)KCJyWe-{UKc)}z59
zVYM5Av&W|AmZQ;}s7$IHQQl+lcuQkQ#$3J=nH&qAU?Cm8Q$PAd(*9XW$CuuSLTAKM
zjNfUqq~V-MaFMJE+z;Yru`UEw8~SF6ZlS%-yCBDQhvnK$R~N&q_;xch5lX(HD@Qu0
zVd16B1qY@%a(kEHJiHANpHRippL6I8=ek>99S)TsqI@466|^yPyYPT(Y|SpDI_0!p
zCsyYjR`B~dK*qj7%0g`CppT`%Hz@W%0M0PUU6vR>VFKG(JH&ZujK~26feX%jzk*MU
z;C~MU+1{nKY5zXuP}D&UjDhI6s8qG^47f9viSBSPwZ^(}0`R}B{Hs&D9LF{vPTQ)yCq-1Q`cm
zi=x)VAx8^i^%Cf&%fgm^HiEZRtSrFkc}>I!@7M?lzKr$D$wH-CM44jtBj;uDARAEu
z#?}50aNQ{X`(~;ZTb^uhldM_==W3%P1zC$@*L}U!;UR-bZ^>|~du#0kXjk;18`GW<
z^iuZz8)x%9y6x<9~J^Sq8Iw|w|JdBzkb7a2L-;IbxO&Fc!U
ztJ%Dvwh5`Y&ofGX;A^J6cW}NnES^}bZBfpgn!c9^r&4D>rfLsB^g?@|RfTizK`7pt
z3D=&-ODsp9O{6P*2yad2;T;^=vGr7FOL3<|d%1&jItZ__F|=#NPnIW+#51d-@vZl8
zs#!TjO42o<8^huw)d-ZpC6khuQh_6x{%jqh#{K@q4uI$oDj!h&YkLnnc|Fn~x>M}s
zuQ&ejYOC=;@UnB6+NA(}vFV5S7mkj7)L%eHoJyNK90%HG3)t=^&l+X}`K8KeaDl!I
zF=~Qp>Rl&?%Je6OfN$iyo}*7&EOMZI+a~1=c}dk;s;%A$8{^CetVK|g@cgAqVWB{=
zGt?iOuktrBHI)`^;&(~|*@yI3@cuo;(=rn&3EE{y_!80FF$X&7u-aKYv6?xUE~~FO
z;{q%F1m~YFxXXAPGp;&>&+o68_QZ`z+#qrlc>607etTp35Nt`;xB**qMw+Q%wmx*1
zYA1UXm&3xtU(OA6)VrhP%J)_68E{d8AkMXqeUCE-PF}?N4hoi!$M^OK`9S>2@J?eMAFH-(m$>#~$-Wc9X3@|!#lx(c_aV(~Y4bxqP
zvxzM{`44)dijFs;*4NS+nB`}>GYEX(24w(17hH9M#bF%R``wq70OINxFPkBY3^kYw
zE)*OXrV-yjy*&ibVeyg+%1$u>Ch9^ib%!-Bi_MT?b>>At
ziFHK`ba)*VTsl4M3?CMQH_SI4_k0x`FlI2>
zJc|CJM$dU*1Fr%UFReW)44uDIa?!XCcIat?N~3xves(PLm#2a^tpSQK+5&7_bU7W?
zng_r}Qnu*@Dp62nsc)+L{3Aw>$^7rPl8BHpx0L0)qg#eZ{7>iFr;sD|@-L}O>ndDn
zV&VWv?vfLrZ}3&}FG)cZgZ-I-Iy%K27lbsqQuoCXvha83Z%VJX1HMXy9OW{qfA43q
zCltRGi}TEdPFZ;4n_-)UIjvh-5o+uBP?^)FLmKM)lAEK*U_vB0ef=%eTcu}j4{1k0
z_+=_?Hr{0%w@6eos?P3*28ArSlWv_)YxM#h21akjt|X3pQpH$jp>3t94IZivDU}XK
zA9|nR$RjgvSm*_z67Pf$JEu_Z^S^_c5xh$i~9AzTYN?b4f5q=wz@>AR+Hked78sYy6;
zBGQAl;uPA)9gM<9VYTz*K9Ml`J>68E)F*C20^)X!7E(o`L2iK*50(wkJoFR|$shxh
zFcvDJ4cs+A6-6=gD>x^?iT>H?1`Wyt?`Tl9HAGxvk9+|yjD;SbCs@5!dz;7;m&_tg+H(g`$uu9P5L?g0Facd!
zxl;a@KRLX2O}m2;N+7(v3bPi6b$3{4z4c3w_$YaOb|qx*DEWog^~-%U6^I*!Z?Xnw
zgu(vO;$R9Pk-W>qx+v1>sbAOzsomR?;(*Z*+ZQ@nEIIYIftJoXzGw-Z8pJRL$QCSh2$_T`OFc_ov#O!(^;
zJCsy(ub-KVJw-`_0BLvr4nwKnbSK1vb<8TlJvJ%W?PUW*@zI4)Rg;b=?#}#h$G}
4XB@l5!!t9S@j8Vkccc<
zVZK8$-~>XD+GH=3z6jvxa>R*&X}p
z9^;^siM{OpdYO<0+K(r_TJjv4`B9}=L-g&_qyaI5a6b)ih%98Fo;btXuHYi3C**Df
zUQ1Ve(3l4~0|TS1-OruHd{PCWyW4L-*hVh-@%#~XET^ko#W0rK_us8ba=^dn-!m24
z3d#O2JA4>kF^xYi$*gC5`1IHM;G)CZfMZX_tQ+WU0E`ZK4Q{y|wXXX|nR8{vZyvA3l!e!jf$xKCgzFTr`
zW)2)5tN2-xaoJs~%)qfO=d&=O5=lA~>Rmzi2%y|^t(a}{d^7t1KBE2@az+A=+M77_
z<9?GKwe=yO%9Z$y5&J
z931}J{_2Mlqtkftu=WhZ52OkhQ40C$e;
zVATdSUPXkg^c09yTUZb%avPr6T2P|_p?LM|*C_=`)xr(;Nd`}(3dv?XRd~Vvs$MId
zpdzBK?Jdz)zi7iG`US%VdZOv3$HWxL4kO&slgzAGLmjAWTbGb
z9d&Z@rLT$KCLZaUixy)S#G$aEB;;pK%T^T*3iY>LiI1(fV$PI)N7ROVbKR%HK?*LI
zL0XF{Ex8>k
ziCO{LmhN&}U%&}#C{|4>K!aWWrdmjxrtI*
zrhW9@XL6qee4f8yLXW^ZzIRbx0pygo(#r){_H4{NWJ>PS>-+te%7%XTH492@
z+D6GC)!;{zn#`CaMHBAdqAaQr?4Nt9ahdSFqW^yN&XhHyMH#i4Eg?>@F4u
zMAZzX>wNtF%44hWaQHzurb~wMrREd%S2I3bdD}loa!L9?&0^{oBcXOkd@-ctFqh>7
zr;ykUAv&u^5rlCyD7gh&WRIzf2z{!<9&41EL6&wsuUm>{Bt;NRDE!_68>0NzO)Bak
zF;nFSh?rVY$e@ymaXZ4e*J+%osJKdH5wgMuK79QID5!xjmpw`6yIcq%bav
zH$k`fe@asP$sAQr~JNB*`%Q-)JrEm*EYNTXkOe^)sCTXPhZh@>e|%P1~skV
zaacfNylkZ3FqRe-Gv+QFaJO
zZQ}5Bnlx*4yI5C3p0}KYrAoJXdV}i63XJ;vo2+MSVwJx`tC%gge);yb-DEJl3=cy{
zJRF53{0lK3_8@5f43|-RhFBc*_WC-4Zo~6ACRjqqA|Wx9wa0i!*<>~5%QKNz7a(cn%7HQ47FihU8i6b$0j4k+|h
zoYcWH0l_m3=}PFQIxT6fuxMWJ$K0y)5_T+fM)N|RK-KZH6q9RtldgQ^`yLa5X~5Pe
zICcUA!tQ%U@VJupSl%BIMPK?#ffplbG_#2`#rcD0CQN0WL@3k!C5inFkkXnZa+&aM
z)y=j}%kkwJQI3k<;E<3vHyvgnf!yxxqv-igkpw
zLt`_HydYWf2IqHlW%_Or#H|8Q4TunHM{-VmqgQozFn2$J&7e@(W%SrJ&W>>nPuY!>
zDGC#pfygdUe$b3Lmna1-V8c)SaBBvAkBYK$iRE2*!+RWx#lwqGuSj->Q3VKWRx&o;
z$L@*_7K=IsA~Tlx+vj!18Dw`$|wAdjJ`Z2TPZ%cQ9=|0p=s)isBS=uq{?%
zZO9=TkY?r}mW#@NA|MGYi7kH~%8VHU{}%1dEe?^#PB9Uo
zynSF9^PHh_uyeOG09|#1il-&??N8&qT>7+RiS4F-dSAiFClZU$kl3#AR_um;s$sZ~
zsjAE~ZHmpyCCv7wa$sAnT_L4O=?gR5YnFPF8NnjaU9oY$c%0^ed@@2To6)g~eSaU+b`hp1_687K)ztcf?H9
zR|FgT>$y8tI!a}6h(aIHE&|CRO#=uY#X>KVo=+c&({P%?)L4T
z=D+?tCUur&`i7~SnF4JJD
zu}Lt99lWytD8Ct_XGT4mtOCQ*gZLB98*DgmfY~b`Ma9&4TfbC(ciL31$Z*`wK4-x4
z?MD9i8LEm`x5J4zy8?-z|9N!vN;&KZ*i9(dX%TJbN!j^Y<;Czpdyh!&K&~;q!Gz>4Et?dh9phc9zNgeV{`dTsE#s?m{xU2
z5t8BQ|K9vpB1OBQ5Iz%X*e{fH1LkK}Gj;|`IzO(ssloLlmaqJ~gNWx3?|}Q8{>sK^
z3@W*tf27j(_SMT^LwDV{)6>Rdetu&@C@-bwDGFeZYvOkUUEN-ViL$v_~
zO?VwZN@mC)Wm5Eox?%8hm#_j6$!M&OSket?aI~URNGV_$fq%p_Ca@lxo{`z=)P7l1
z&fDJqNBQA=LD>icxJX)ML*%RZoo(skhN>aVHK6483uuPfd4m|1{L|7d60Wcf3>Lzb
ztrf5@QtQHIS!n%OuMO=%7j_Ow-wF?DN#O62qvp-84f#DV(cVc6?bm4|+l4;trV%JK
zRVCER@VD^BSd#r)(Q^%6nj+D*#^pxTV|CnWa3))n_3b|C>Kz3A@+53Z_*Fzwif#X2
z&I~dxDah>>iO$L?0mvh#EQ56|0Y)CEUntnVaUh*saXf~d~C
zj;i(}u0jb=Zxg@Sa!sOU|LaA<{Hw7a3j4MRxS8xfCr&7I+BE*TMb5pvnTU(Q;|{>b
zhMs`eieNyaXg_QDk9+(1Dz4N1a8G>jI~ik;P^2KATm8d@U7ax%v9UxJs^=6NcA2H5VUKs(*;Fq7
z8Wx?Zh=>Tp^Xn@J?DOq>ygk+LPQFz@5IkvMm9bfciJ0WCPEH0;{0QwUr1*x!Mnyg8@WnR
zQY!DIXK<>$sYzFI<2D#Be;g-2|7wQ_f)~m)L5;@NC#_iB@K(5_Dv!v-`11$A`?0Q8
zk3K$F9I-%;Iv!-Y6)QMn2Eo@~CJ(|s_)~y!PAyTkibA2dw_8Qq|9G|Z{L+(Qb_|nab9-|mDhxYQ
zYtVK0JB8*SfPldJ73@a+L!M_%75h7xnSZg>(^W>%XJk53l6WjM4qTuX)Y?KdvprkyWr<-u~=Gq&W`0db`)U28fn$$;!L+0Wwv2A2LciiACj06+50T_5!p_du|
zF)+^?w}%t!vUd|YOz)L}4cAH)fjoanU*`ul_^PosxKyVUL#)Gj9~lvoouGBCrd
zBTIBTJYku9nGMjJ%uQeV!QlJvW~+(lj@T)*mwlf1xyl@kC%&pb4h3L0jVDty&1xq}
z3}nzk)6c-V(7`HhG$qzf5jY@ld^sZe!|BA_bM&OSFV
z@m*tn5YZa@sF_;QF?u13PN~DJ!QB{WFGjY*Hcttzn2{0pR@dZytkEsxrR=Ol(OI#8
ziD=nlz4b}3Su@7PJOFYw0@s}i|B=zAOgvr}Ue*JZqh|@5VPMh{!>&KMV=hSUD7&VNaDi3iGP0D$q-g9_S>eSnAOM?`^zDK_K(ALz
ze`gSR>W9Bc;
z5-``>c=DGdJQM@Tw1RrKWAQLz$>IHL5fPl`E7Bt8)2%
zyi%Br#7@sf&J&!74Oa)hQOb#c^sLqIP<=jMZ7RLy6Bh2H
zQF#^{a6bnDouMybQm&Xn
zh1>1r9g#EI+(8_ZYMr|mkHGe@{%x3Nqv
zUH1Wxjbw=tKfU2CtLYX!N^bp{NO*NUEvcaZO?VTGE@SU%>_
zk}2bLzhEKLL8XFzhSxL^J_BOfq?e{bqnTa@tEB&d0)?kjB05A~;1ap3xe!_J|4$W&
zUGM$+GQoL)`X8Ec^{)+S4(=!@1I`
z%#0TCQP4p@TVxT$`IlC->|@UJV{zh~6Q`fLM<)jaTzE1^
z_H6ud;bCDMOD!*G$(-eTS~=}k5)q1)YV_)gyw1tV0MwNFC>=I4(7ka?txVe@P*2hAdI$v)N!rbxQjEynLje}wxS&lHXytK(8}
z@qr`}<5b@VLf}`~{)Hl9O1UcZH#%;0XJ+h|^_)2#qy+hl8Ft46wFV(#uud|ct1g5^
zIGK{^5jg1#e1(_7K57q(55MsQmyU`c5a|0Z!~wY7do5_%pUxMiq0?zR=zxMn(ie97
zt6+0#g?0BIZaN&Kpi2UWF%r3d-!7X%P1rE`?8k1POrJ}N4C`V#=rgxR>fe)sbULY1
z$m!lgg9SmxGaw4t-*vV>Zyo5+1J5*B^36G2T+r^%sDh}d03ca093FTY^ZKc5O+RjA{*=z
zW>k(x;RnKZw5Ojwk?k4$UU#sexTs&v5MZ0if^P!yDe}r@3=9g-D`A)2Fqd;8l#GgZ
z-*cBK{iK#}gP-j6=Yt&Bw92qpOLW+hx2!(Vf6f;dDSs~~cp*3Y1@Y{)sFTcG>*P$2
zz2Sq8x;pQzvoI_I8KjpcCuvyqC!5
zLY(Fkc*_kKQVKzWYPxtO3D(7W8KOP%CqkYab@^MIA-Yd5Ez&$mid{6^+_Vv8@@Ki6
zE!|HJ9${K24e^FYj;--W>qX0PPmVtX^dRf6&N>sL0g|AnOm|vtk*PN)Bv=vy9m1UPNDD$})xQf)K;y)W
zdap$8z8Ab^?Bg40bL#;cv$TsTtO;@}|7$cHVU%*x>3uy(sNbfBiJP9bKuL8$f*e?P
zaBvjqVPl7wptUz&8!9qt5q}#jq&j3T=yP<#DC>Thu
z4ej~)$d3&inh_{HCUi6o55ZGPmz6V?MEO}v3(_T0gmEMkYk!mfs-nP$X1sKEcjL)a
zMNY@aby2%9fa9t{!c`x!j4lCQRONbcmU$YYajC0JH=KDq#;2FEeEs9I?<24N5>iHz
zOK{UihlIc-=17w!R4DOSgg1Kqf=7dG+*~%^0U4@6O(R?zLlUao8m3;b-vAp-fG?l+
z3&dq9a9AtS2Gc6&=J^zHH`v1U3$CaGX--Eddv`qfADI5?ObEQ8=@$D?TV;`%E*%n{
z4c}+@d0MN#4Lik~^O!k$!B?A}8f>y+paC}T<<;u3tf#BQ
zXCdW%!qU;|(GCAJZT)Hm7%F}vK)rFnl$dAQHAMnrVHp>pL>tY*cp)3j+aCx{CTMr9
zPXxCn@SY5uBL_!cH|_-1w51Vljtek#zta6-*NApkPRukK%SMM{d3J;)ZB4hEPCP<|
zt75T;2nQzwAI`LEE7AT@VOu49Tugywnzdpd80yCvJVD6WS8AS@zdx9oAF&|(ROE0a
zvjgr+`fP9Os4$Z``P6dR6ST?5nx`I^Ulz{^|B)d>QnV^uU=)~zq1_Jx$GZC*_kvioH
zntrn5c!7vj$UfJ>yn#^f&&M2c^xl}SC4m@hf`F5<9^`RBQAgeBAeyMzKmC&cMbH|y
zH8Jb%9$+7`@BhwS1k>bYgNUFzk~j2+^<(Rqr*?i6`A&p*A+5_xr1j-u=^Z(;2-Rty&8)bu-!uQHUARD*8onQCaYVlf
z&j0kXYp(pz*)+zyI;od7CiGgv)2Otu)jrRWYe4Hc?H~H*iU$`|7rvFKUC{qnOv?D^
z)>ga24^af?uj;HA56x>(_=@8Ag4o`99hz=x?#BoJjDHZ~*Ypa1{=+p`
z>BA=&F=r7BE0fef(*63T$9P$Kcmq(#XD(u<99*p9`QntpbQ}?_*6u;Z1Q!ddlD>CM*MBy_|sA!->
z$!y-aXqJjf6(DUhXQ1(rRer9|7lvcqNh*^#M#R=MTSWN~&Tqb@11;cA|>x_<@)jtV+
z?&ariKe*eEtBehIGG(7t;rwt1eT!+4JX$J
z4%81}c+bOZ&de#6x`@9zhAkN`Y^i(labRE>J)|XKIj6rT^V$6^fbm``fbj6A9=8*Q
z2v^yjfJN8i#Rpe4Yal~@Y8Pr%LC3gYLrks+-ri$ZuF?Q?V!#61r#lQ}OZlDjhPhgw
z=wkH7k9)7+9jI3+!%k0UVrLt;A9P=RL=~a874;^l3`#m722AP*($|Jhc*hpd<4ic<
z9@Qyvzyjk9tIuT^lfDk>61P>(vn}3=U;5<
zKwpX6d_=?fa)*@E2|cK1dY2PSlwbu>tid*8f%rA)rh0)T7$5Qchs8a!a`6NUcgGtt
z46O?2AH0UI7EM6zA~3Cqc_gl2{Wn&vjRIIu!mGXmcAa_TSw5Fc-!O+aosM&%!oFB|Gb+3R*Yqm
znyg(FFi}pDN=yrOofMUknnXbM8(p?Tgo~?Fnn24fc@}D$53-*S=pX!sFx6WOA@1$*
zdZJowgPXX%w^GrY6AVsvxSqKKw+`Q8VnwDHbDmy%#m_G*obS;5u^9%IuxXr#F`j$y
zr~cH0v4rRqpAMePS%y)%h)5j0yYL8@#%&K}`cj2J_Pt%xfY#Y8#Te;GnIgVPTeQ8v;MT+5xn4
z^bT3^2|WYhkbT!u8_J|@)_mxG*s$?y#5-piYpt&I$db@f)O_yIe%3CvL7ki|yiXt8
zy?uR;FHcY6KyQ7uvegq*I*}*T1${8!T&`Lg+r1f^8n%=|t9tMq2qVSrI0iu>HafBA
zA6`FREZ3Rr+!7j^1OF#!ktwC8p102-ZoJ%37;)dUiOE`{t*(!J#kaNx#aF|Bk(l%=
zyg#dVE(QYUeB*_3GJ^|*TsGln_gE(V4%d8#;|oa$bnfBjFF9h02L1yWnHZoAElvg8
z{k=(g9a@|S0EjY4cUh%BL^oIj4QU&`BK
zrr$?LyB8};S-Fz5XQ95o^C7Oy>vuHPb^MY7_}juM(MdM4(rC;!i=gEnX{;d{&jc^
zBzNz7p0w1SI}TEbn4o>`%uorRkSBa&11y9`DHN1tX217WkHM3Lic3Nk!|v$xbR2u{
zAYt+JnYeP{=`e>SDb2+psd-4D87mjm^zvHPydm7-2{Qs7
z&vlsw*0`E9(I4#re?{DpV0ocXHPeg$dKOjS7mE;Ks=I*F>fU3=eN=T4ISZCbn=5P#
zYXgF62FyJiH5_fwSb1&)$V@E%Q~c^Ar$|>CIGHchLHW}pT8l?YUp;6HlUpsMcsVj8
zg`G$aN_bv0STz2g8}U=d3>lTzCC?L6HuxFp0Pc-^u-<=y{D?Ev80_Gk^dv~mgNHoa
zj$KUo5pYaQd{Z1M#qrP~Vja)-hSpP6@tqD(?vDFH&CPd-(=l67sWWsgJ#v2(BV~k7
zNN%^Lc*!2k=ZAD1u(T$=x+dY_NY}B#05=XZ*4jg6EQI{&xGz^Cl2<+IgBj4o?iffg
zaL`VQ1%CB2!jz!_a{YlKMksDGU(-EJ+J2D}{wXTrUJ;Jm#sO}>AcCiCc;L)1cRlJ8
z`vu4yl&G=8h^*|H(kxdx!8$ZUUx%v(xIZxR#j(OP`mBB$Uo)415;J!g8h%5Z>M4fk
zueA5$Zi7<{feXzD0}b>r;J!Ml0{fkTR7dOZ|1DDz1beUb4iIJS?FD9~b^QgCZE10&
zK8xP*6s9^%>(F)to02z1ujp$kCN%8bqx#MKX&_!f^68ta8P-6l)V92`%8BH5`(qB8{@N9wJYVF5Lo;Drb!RWU-v
zavwVL&^^bcNnKVBFF^KlFHDwP8io!1PXLqC(-XemAH)8MygfSM{e1HlCD-5xip{Y}
z%w@M`AWOSQ6BZu+^b=eETw7pyLkP9S_A^Z(`7V+XJCO(
zX%llKg>rwtxyF!WQE*wMCw@A&RNfM<HVhez6Ad;E`?g+p~5muYc
zzFBVq3E0kJQqzD#))(hddf2a>)W(zhYvs%>1m6gwg!#hjco}=MYYm&3VaZ5tph{c)
zbqQ)(T^?PZfAY)`$%VhT<2l*CzM!rN`A(L{QMSj;i$;R<@zw{(PW4p!cRNDpRz0zq
zP9OxO>XTJ{AE-0eFd#NSUb+ZXvtD);Lv)4n6r215QbRFA8C#hFim8i4^70wWC6J&G
z0FtGr8%ovvrrP~MR1xXWktvJIQWgj
zgz+s)(U!S>f5%ehWTx7$2HA6eP;RUki9a#x{QGn*se)re8j
z<@X^d@u_ZeM62OD}TV7mT+={*NKM9|ixn|kl-(R}?UQfPqaA;KyHwR^N40Twb
z*`&$WLB)1Fh|R`(?hlmZ=Ub3Q0WDCWGrkn`GJpvIC(0A@>x9UhHx|ZYZ}53I(>8N)
z!N7IpoJ~-#ING9n4$>vFhbJ;ttI8Q8dKJzdRaufEn#j+{9QM%FT6lsE=c6oM0
zj(FJ=I>OYA+f6A{%3D(fpV&D=A_hPX7gCVATy5q!RCAlo7nuW0XMW({9(7+S(LsK#
ze7N4gdmB)p$eKTffY+RRA$zULD*smzT5*_fY|S4Yzu$2jnj=|h|LiJWQUp$ea-iBM
zT-o9ktt|Lv3R1YH5RomhgQx0vhyXoLPrOn((mH>IQ1HT~4a@C!cqg!cmfTweJw;*V
z)3K+*-rxWJdNvd?@(lEncz}G=>v^IfN(9M!IkSbSagz*K;|xBdtcowgA7steJMe&R
zNB+sGz0J17etQu6#fky{;Q9<5<4Vtu--M3R_JIQqI-(MgY8~w&0G3H8)a(@fWUzMs
zVo^5Zh(M#R-_MVuexBb>xqaPbp08o4$^dZ!WzJ5Qq^;_SWk*R8Ta||hSAD%rq8obFn&49{veWl*v)#3)T%er5|Js>&
zw(B8Z`pgQ~IJ&xuYe1@qwju^(p`w-&Bx8S6qaG`Zso5DH0GWlw;Al#2v`2L9KxVnc3!B9TFl#Wa+jS?HRD-=GL
zvyRFgeHOpt>R8*o-w=09wW8L@7^xWeq0`V&A}}4XO>B}}9L|1FBu|YDWiCI@|GQD<
zCQ*632hPImGEQSJmZb-iyl^whj$_zO3;6Jlw~RQHblR9N6{eUB?ltD5^dta;@avz`
zvKwqdZB(!3;k>r%o$uFXZA~a7mP$<-+?zFt|4q8lH>=l
z*78-{gV37edf_qjG$`GoEnhn$Av!5lBl35I;B4mMrY%gG*^FCk8Oj=F_O$v4tw&fd
zhT@&~#2!qVPgjQ32k6#hNYX_Wm0xFh>m6+DH6NUYX7|^2*qz6^2KGl_;iEkqfm_GU
zx8c=Qz7oK#2JeVUNW^6|Z{ZAN7dH1NvK|irNQd9R?Pta$5dP?j|6u)eTAgIYUN8hFO^PD7`!NIzGflAc
z>KgRi6#)_bANP(#7%Y+(ym!y4NI4UGB4_0Hc=biTH&=+x4cYW%wt9)h+U_K6fk%$P%mWPP`70__
z>^I~>U%SbDjh?=+n(aeW>uqZ_cD2e(e89E>`|*YiT0Ck-Dk&$c(x^f0u9dG@=YY~R
zQ)=gTJUiA=9B$4=alDe}`|sFX)wG}n#d76y>a&HFb{aJY#N_p3FA6I_POT?vG0^dX
zGuzL>Z0Pl$l3TOWp6HFN;pw>RbX??1Gf>X~QA{KvSQY*c4$yI)1LReN?)zPYZ4
zYt6>MOy&lj8uz0W_^*TlX23u2q2H-YcAarn8NR$OlaSlAT=vUqcfWSj
zv(61{1%n-n(zpbQPPpL>ccQi4*9Z?%vk+Th7s>q$-)}%BGrU{Js8R$m6r5*b*RheN
z!Fe+2l-u#p`f^2wh16E#P1f6^RP07i)Wz&PTRH+&N_Ec{2&yA0J!S6X!UFqtzweQpg?SpZg@~Knm@125X|hg
zrvWDLt+(PNM-UJ#f*SMc1-)RKEO18gsw1s9U{4MUOrt|TpYHS0Seo?*xvQsxkpE=c
z25E7V%(Dxq589ZG{SLm)%Iz7J%rKOnliy2+g_;u85Dqre+CTDahQ#Nzv4||Z0Cc##
z7T-6|opW6RU``!dS$N8OauVxVuhU?qZjH2!3_xdg6$oPX&9Ev_ktU7RO!Vgx>Px;i
z3zeLa=+I;8(>p?ewV{5uN;<1}T1d=i=Ub2|;c!*WYF0muJ)Wmlw7s#cYyE>mlj7Sl
z7AMYSyW#?Fx7uXi6^y(gB>bM%g;HU{
zLjT2cdePO?`uxjoPZp;uxFf!1q-f#>SUSxycwNry5Iv7Mk$|7aMx#eQEO&!Xsidk)ylTu!E
zzUt|Yczye*TA(vDCl!J!*oTBret}YkSIx256PK?IDOW5ul$MyNYiK=|XT)_=P709T
zSPbYijaI0}<)3umk*cXzSB-ZDgm*dtaP+xsddH%i30GbesprgOzQ(kYZZkH1WZ?S(
zK5B56!fX>VC0wjo8)YMRYNct_)@bzdwkMi)cr`oy{5m6;kJ{s3n#wtn)A`>^4~yW0!H1w9x>Dn|U1Z
zFBgP$Dnbk8S=AJ(sT;k2N>sFF%JWVrmaJXB!K@>MvsjaU8;JOZvhGg_;(oSOSUlXZ
zT_f}icK())hd519%o);?P3#o!!ZG8`SjiA7AjQJdF#;)w`eZgUT8>I#San#1`eXu*+$XJ
z*p4M>`#t?n(DbY}LKRr-nFoxz)jvLBVSg@mdtCxwymrXfSM_*$1H5LBT}XuYmcRf7@fzHh
zN8`5Msx(<#ubmaYhN6a1!sHtjunz$=i0s^khs^d2k2L3}T1IA5!OQK6DUusqhVeCT
z+|Qirr@Kd3$o@!6fEZ&uGxCfaaqzV1BGtCZ|WmBBxdl3;7Z!`uLt=xK@K0VODD;
zr}R_ewV|$tWk2G0y8#X+s^MT_4Su(Ti^>FPH_Se48@(dI1qD~-jgzF^b2mZv!Y*@~
zrBt93#}jV{C-4PmThIL$p%JG}_;I6%b0|&fAZj?`
zMsUqd(FM_R5dA@)xKBbH@+{yEnt28ut+)bL>6~3|rep-bcyf67vhwz3YLJ8mz4AeKl#Ct$aXUC5JprOrg^
z+xRBXj%+sCt*?O09ZuzK2AK$6nBdY-*gWy+MnClc_HP-GOxchfeZ1Jf?nHK7BcRfY
zpp_d{7x4YnIGcdU(lW3gQ+j^-WH{G%F&f#9%kEsslxMA|UFjE=>3VhGd;i-#frO4H_~(k{Er|Fm
zB2s}`wKIPtU;kGJL7ZuMytIaOtJtZgM^Lu`x>$o5Biw(Q?y1}*!7%^O{cass-;l<|
z>lJI29t9sC5|__pIn)xg;P&Hcbsl}%%fjOZ$2z0|w~p6Z@O36Ruz}JMu&14#;j_O^
z4Z?;M-zti{Tomxi!M)Kb_82j~fFgE7pAF6dbwPIjUKUO%r=x?@`b=Bx%@(8k)eU*_
zWDXJU5*)McX{!M8=6uGIi#t@FZ7Zo*k
zCPzh3E1W*6-mNK}>Ho^Ibz-vKS&hM1TZgL4+OnowxMfb0NOe0?VY3!jZ@B0<-4B+`
zl!X59e#@*@qmGkEg&16gKN%#u4=EI5k5rLL7zTfMA@W8wkS|}1(u3i3as+()#rS4?c=jwTF`=hz0mH4>6}Jy%ljf?U4WWxFlGg4
zVV%y&X+1XrmzTBNKi&nN%R1CelV^ZQ)tm&QNX26Bv(S{Nj^frgeMi{Lj|H@!LK0%L
z@*{?}a}DuzT2NO?*dR?mUg2Y~ep0($q+k)kmtqlyVK3#0CH`+xQDb%#~H
zxO-l(eqxSG5A#9;72ta}NWH?Zx^>q!o#XVHTTaa^TH9$+Q8Kt=pdo`b`q-hoP=}oC
z`?`7Bx26fv-Ja-3?VyemVK@=3*4gY^_p+ECOWq_*l1}?$*KmTdAI)Wmy5Gd}vejtu
zflHp#b`P=tzg+|Og&a-(#4%)yU@m!x9g9bU)yaE8z6_$UadARBbIs05Q-a$oIT{wK
zr>ixTx~-Gu)?EsFI*wf8Bi}<6($f!KqZl85kj-X90}f?89(Ppx{n_!H!pHax)mGxA
zKZiTH|16cP2T6M6AZj+j@O&>c3g+7g=WmqQF>%_jSdv-sLL|Av@EAd00!91uuvZAt
ztyysLXVxz>
z4A4j}zgc0j;xSRM^vY7anU10*~9pKKMn<@Hh#&q*5JsRW8wZ=6XIfas#N=SgX(a_g9TPzM!;0S#5rX
z0BBIM;Ua^JNcUZGd{D6Nt3)5t&f8!EeD#a6<;%etGY{C8s<^$M94KS^n%(Vy83+f<
zvT`DYj9?rg-;d~~CO3rY_BSAzsM4TQ)tmDBGq69mO@6PP|x5amG&;3jq%lB)2K&8%VC@cGwbL_{^hqhxvUH
zH!)vYhK9S7C!hBV-Q@|@bsaLGdv=>MeJ`xK(?^LP1%bB
zwf$a8&qK-`S}1o0?ijxub-&+^yj<07n5y{JU09=G(ghLp3ovLM}8q3R7SUW+ruF({C%DCifN9Ny3hf64t|4
zLv7#xUMkA4W`E5|u5aM5_<7fPycIh)?iR9tQn0#8jGWe)AgK1;ELHw5wCZBAsZgaN
zV2OuNbe=rvo{DE-?ET`{>F*%Mf>T3z&AAIMXxR0Biw3ncmLA^}!6nb)&dJ0`(kVHV
zNXsqz|MFb3X0o3_t$<07)8ZHVe1O@WuX2*NIi|Og^RXe8U&A%el%hTycU;!Fwm+Gn
zyVAa8Y`EB^@%qNm(^&@N-7Q)5hWtr3T7aVrlZzuO@TQx4X4JljDdW)Dwx4wz^c~J%
z<++9u(7|rz>KyCXZ~aUj6hj98*I=X6IS6Bg^YdXZJaeH02_i>^&szFsQn|&XUkr5e
zpKKJ$u?R>a_6DBlhPcZc?YL=gO~)uH^Z0?}nGIT2QI2(>)^+Q#`MVX@?v9I*G764$
zWMFN#H2?8;P%k3}_aZ0qugn&rmC^M$Lri8y8w%l6X>w_FPa{2?PTl6={wCaPwPyvp
zKNb1Q!_4Ru0vS>gnoc)G0iZNFop&X)VE#0p%%*(by?y7^4}sNG0D0`?UaX&R4JCt894u>
zSM5MQ4iFXnN7EE?+8^_XU@g+wuNJDvr{~jMZ5zeZ|l`rbhTa?Kd&6moxp}n@;_CwPriWEDj?QPYoulMp54<
z)e&;nTP$3d?6qt^VGtv8rYnkC{LPIH)$7d-wcal9PSw&W2Mz#=qwv&uZ}M?-U6+e(
z^^kd&$35A9*=5g8^V$)J#IKJggRA;rf#W>Ue|t_%u74#;^N6o%c?H019e|C<&rxVm6{cC#p$nEPq`9yiEF
zMnjbPZ&L88X3jCYr-*yz+-U=;OZsIvM`5a9psQ)#&?5DQZr
z=+X1!2k@@GJGTJD&H!c8>kH_%sPquYza=*N?tQp?2jNPNRJ1rD62&;z8KVqUdwY1}
z?RaHiJ}oMZZAto+Pk3W5`@nc_%-FfO=IC9@O5JcJPcQo?xvKdN2ll9cC}#kq8;*3g
zIiAV}bFW<_69pl%D;Ak1y1v%?yHgC!6HA0KwV*~>2)ied5YQt!HlvsV1M*rx0V1_DFWUwkY?Xwnr`2AY&M=E2z!SpUxKzWEP`^fh04;_-TT`Uc4a1l$N?r
z!>=eT|9X1Yi@F*1BZ{xo+DMT^?Y*@}rhyGv(K*&WbC6b$)WJX2B%HfSyWZ@SC-ZNi
z_f0YmX*_D8}~mXMNqq4=CX9t@!x^5R7Y3AoGO{)DYv
z?qAwa?qNC4<6`-vzZR4{`Te5PZJM4R@^6zk%WGVrZ~)knBm4XH?$CEoz0YlL9}lbc
z9n)u-2d6WIL*B1BNB~lz*!_A38xD)X9Wm(Xr#9z-)V9O@$$}_>=1g|hF2dmBTDylc
zdF;fZcUUCgdJtqQHAQQ2*6|LqzDMrp5ktz4GOL+(mD=Ngf8(H`yh9c`K!rzoPltmu
z=?Qpkyq+!6SBT^trx`DV5D(hUBDTw!r@_ZW8ve9KZGSp96)ZBfxDCbnapG0L>1={s
zh(o|R`Lp#!=Qe}UkknXgI-YRxQlurJK#GyS9cn`lkB(3C9~X#MyC_q5;o;maSw1oE
zz_)vUU;^@zTnVj#&l8-veOIuOfyvg}q5X7;ERpodrNXSz`wJMEphSXtmET1RsCv}*
z0Gs0^ktmrq5_J_J_n{o!FYT!!Jg2c6A#jjidVmEq2q!_kpr_pXd50po$
z>rzQp1q2BW!WMdp>;qV(K#EyE-B$S15rx!@iQhbc{a!GS0|WL;f?azWELmR`2vvZ;
z)4py`2L~yac^FBfCZhq7DKJXj)L>}ni@vZh;sU@;jq=Qi_Nw}?C-ZVqczZEq`?@?V
zzD3+@_!~9y8lN{dQ%xrboPBkBnZ2h`5o83(_D6{tw_eL*tIalnrPA{0OJ;8&1Oyy(
z>!!Ab8wVUklw)2wEQUw#GmUJ@Y!=B8!MyZxCL6o?I9Xx8-~V
zJT@3;c>>TT1l%)XMn#xcS?dOo1>u4qp@noP2zG+Vv+u%1?(>lNcY;RUh+T&E_-+Ep
zWm+lX%f+djtelq=D5jn%4@q1#d`F)qCb+he=JAQX`N+i9RJY2qhr)@`i5%=`H*W-i
zF}=GIshG+kp(1bivuRWiy9qOj7a1v=Rkg8H8Qks!p}*H`!GVzl=uO%RbHVaG0$VDe
zo3oRUgptaVGS8I$5`ivQ9p6Uy7+@^^WmGOK?Xh&^h+#
zPg9arPaD#8Y~LSZ@ThKN!PStAn#?#Ddqj+bHeH=RY&yVV(U8J#AtCx;>S~|reDT5~
zt~sNNDr@g0enC#-75Bp{p&&TzBJ0RJfzUq7i6`BL>4k@pi~OxO{`YMhJE9
zo;7>`yOb#rXz=ePCGG6uSp!U`JNhxLY3OZoV%SzssRMZ+P4lif6-HK6yR~kFzlk;X
z?2cVwJ)(Q9<-u<`F>*l&-L~4WbxeAU9BlY+NsEGj4P60|W|l;mOkC&32>%D~*UUeD))mu}V9rwvAuwMY##tF&hgO7r%`k0oa
zymtP<5$zDV<<#mgjWD~S!@LYC2v=sn0ITPPr4Vbqyj3pDJIS_v>CAG|mMBRIZWX%%
z*Y#Kd*LB&6WYUG4LO;r|)WWMVbqKEd!!JN9f+VYb^?jp@5EQruu)qtxV7eHz7av0%
z2A|%As=jIkYNWTsSJ$&}FIEi*AK@Xg+0~UPfcaW`WHXVVfv3XsoUb+|g~{+Aahrcg
z9cc=LD5xUNI%0kL(84Bw?E1sq$gtML#WkYg$v=2bSm?c5F2>~11H_g&TxK@wjrNxH
zU_a4dJeD|(?;nO1*rb?9gYbV4ZnwmkxNOS}%t?8uzF@0VD41F0lqKjENd$|5O_#}s
z_gqh#-S}cL6o}Wt&0BSxY|+-^&If019wHWEuzkTTSu_ggfZq)>ur*XvXn-$-)I;mp
zMo9N}5GneC+lJvf;aMk%#>-N(7W84F(C~Mw`y%!kQg=`_zuC6A4AtiJcS<(4lAr9-
zx8dqV3}zsW@O=FF+oTNNCK)H0HA7oWyqbSwTO*|Ixk=ptR2KNgU$L-Zno9;fbE;D$
zBn?8};U*wHa^8{K!P`lx*h2RzINGqzj+%1xid8&=+*B~1N*{v4Ur?bQkemX=8Mp}$
zfsQItOKU#V?@9ch-k_U?n2~WFnj4KG*rYIG)de|$m+Yfxw2_;rH4V4R(96&T=1rwl
zYvK}8^T$u}`)ZPpPw#>HXiRB
zy)U;6f<)loMm@Z;u|$@>9W}skco+`sqO~jMBR9W}D)le_OeA*BmrnCI9U}aE$O0ZY
z2^2SaeLoek6AC(*i3?kGTteS<=n*jQ;gfNGZI$l+2h;9VIoQ5;ei02%yZ3S?mn>(L
zgNHKlSt%wyNc5F35g=tu@(s>!7&{rw3Imx+t__-<$!bP`FXbz$k;7Ia)UN}L)(*pX
zQ8lls0-lC|EtIOZt=m&gU%afZ_%(gqHwuiiiA*m)pnjmDe9j?eETsVYk<1<%LID6%
ziu;X@uI438SmrMj>l@01nRG?h7LpW#7n9oD{3(oTwO&${slA2()@f2(C%@lua*dY%
zns7KG!UEnY9dLP=-h8pHA^&mFS*dr}w@-`iXNT$pb+qA4)~2y$cErnjMM8$H1u|au
zF<~Oy;W=i~G6pDm@q)KmqI}}cMIgo3TV37qsY$$2a&V5-a+~D&o#^3C4(iV`KV9}a
z)UGt-XH&;tzb=0`3pdv?e|w1eu1IL1E-)h@zTpz<#*-G0VzdWQNTJ!!1V-k1GLQPX
za#T7l!=aW)wH_1GsqFzLSmjmH#kr@OMG^%Cz+l?W!meJ}!qfk%i5Jj(OFOA`8`dxM
zk;5q@+`Xn+Hon?cDOZ!Uq+w{RupVt~FZJkOW{a^r|3fK2q&4uHw;A8b?y^`T$@6lN
zYEowQ&hFB5Z4G|tmCeD*N?dUNtdXTz)5WiREnA?1mmx2e7j56SO!yV(JY!pLj$I_$8(6@hu6_1>_5nL
zOgaj?uJF)+3}n3ABIM#bAF>96+UeXLjcKHxew4nOIi)-`0`BNKEMA#&JM85qIASl*
zG14Hz(4!)EnIWRzt!$6XRTiIQ2aT%SOZpOEz>K}pvJadO-~thoT%q{%HqOIpKnSpM
zgkb+U^G#mAG;LlFv0gsCts;YTPC;cYov;xf+>Wsh{=3{kRb~Xk7J+;M*^@t9`7%QT
zGs&+Zd~66T*{U=9sOp5|D@`Cw!BEo0`^zV7#s+eCN_`~gm-D{eTfhwFTA(Yu7D7v+SH3T?A*?0gy%rj
z1of=?M!nEMQ?aeId+Y+8I+C5l%l{!FC_aE}`PTS(TCFJ8i^whWvLhl@q?yQg-7(!q
zW450tEbCrzsr|5XT-%h>rB=gW_mMcSt#-bVNOR{tUh<_n?Cjx~@a!Zb4?JBloc=d2
z*ck)kaFTNNUu@zXQDF8$_U@iHv>30|Bu~XxK{2*m!d2Q0hEgBry@8CIdz~nYr3?h=
zwa89t6QFKLYk^nW93;&yoKN+vF>Mvt1bx^<-{`A%J8$C<3(>QB7A=kIYpCyzX&#kL
zhTRb;<46x#lfBE=j!CUsi~5%(z28S}kb3V!ysAS9aFgIoAsTdL4>tO4N}5D1nyFJ?
zxq-abTqHEf*C_4}8Yo%QCU2}lG1tE5X(xMuFOe^_LP
zsS|!GKwI#^VkjxxZ*wZs=!DE18jT$m!{wH%?`Nt{zwKQ$^J>)uEw}Pl*>p<6Mj--O
zJEC%sYW&)$bJ(=A<4VJ-8M*5+b{$@vfuLV&xAuqJ6ylnFDZ9?I%#yuwS*dnY7klOPS)aa*4l#2^0TR+upU>BEfLMJlyrKwX_>w_9^ZM@%_FKnMwV{cTUj
zEnoFoG?y%jYiPmA$k|kqu3Nj000H{7j^WZt=w??JMmkk?xb{8mV%laHY?odj{J(WI
zEDn0)n*D==p5+Txm`mL$7HqLpd&|nyfG+$BkpFmC)>Z+C5}OH*;CP(KcT3(SMigHe
z1~G2}Y^Sl7WB>hO-hkg1=jWzV!gWI`4#%v3+gfGYKfI6Nr$aY@X(gG&K*9iOh}jJ2
z4nLUVQKc#oHHIp?gj7KVPP5Q}mWGX~;5lg8go!UNB8BhupVEcH+Y#2&>jkF!3#4#=
zy;nSM{YKpO;BiO3Xq0a}Ck>Y#Un?%0h6P1-o0EkAMs+1}t@rK|r*s~pPz%vm#Pb7Y
zV(|#Xm{gO$XiBE==$*(XzJW%462EF_tG4;=xHQifXm||<+ly}QL0GcbC8c}*r8b<9}%BGv7Nhy5B>BKpP@oe;(
zR~L3DB}zDGiXv&q8&-)Vk$>D3#=yG|0Z;7G^D#1D_})6TV(w!rKu}(oZogb@Yyzsc
zyt0B;)wiC0Yg=AlvBd@hw)4UZ-#l2p^Sub02J=-4l@ou!{u^J
zh_owd6Pj1*4UPCqAt0UrH(#{)wf4uoWB4l?r0yef$i;bD_p2#kzG903i9vz9GCw=K
zjg|+|#hN$dArVq&-oHyI10hU=JiQE@&?|6ux>YfyR@PZN$kzBY{IvK$WE#QoCx_iX
z*H^{VGJAH>(+X+!+JMW;|NGb?q#h%8xb=$UMdtQg&e1;X_G{Z?S$CiOUgu
zw=&#K;A~}r6ppAJR|eouHz*%1odSJfC2j{
z-YSG%V0_8@>Z(}r-dQ9{iA`(*tPA_?8cgqHJ$j}?mk0}mJk+2GMoIQV-u`^Sdh|eQ
zKd1|8j{W*iDUQ@KbHCKn)4%5~o7wi1g}HWwr66GHvD0J}^P9H~2_Y1RnX{}PNG&2j
z^=gKIji~C*OcusEQzi~0|P+qCgA-wBUy%$1*!V`)nA#D0WEeZrTYl2P4
zPHB};#QH)(6=7VW8ItQ;@5P0p&q{XL0a8m!Vk0Z((!P$*TuSG`hmSR)DT`t;C+?50
zpv{iPe0z+lQZ&T30HGMh%9RNEe+WkEr_IREqss+VB|C%Y+)5~-ofB)V^dSA_V7E=+
z(?hN=*eM<$+z$sag57Tr#OzMtu`;gQsZCsXGh>;y?3mW%;S_=CM$@+AoR!A
zd^?>MWpJ(i@z25S(j^>JDWW@rKxud>F}hs!W3ZQ&PG@!kmqOe1p0o76O3UHXv%7?j
zff!U!f-(yw87Rx)cGC90Ga+Bc`MM@`wLxC!!)c(j)
z4^Dwje%z)ZA{z2TB>N2U0tfU*T7*#OzGN753ivw@zrFr@jLGV-vDt#Mdh2+@Y(eKx
zRT7ti8oi^C(+JmmqNJkdeoOXN40Wba^CgJrC)>lnG9A`w>|UzC!EAk=%olTZrQe9Tlb%Y$e~ZpS`#Ul7YR+FxXI-;a8xiWZWTC97J3%ma7x@LQ7MpXa9$`x9p0e
zi@J7$ySsaE8cUGi?(VKZg1ZykU4y%OAhWoo%%Hhkp0Uf{uSIN|RYmFLsh
z=EhHTaQNR`U*sm=Xp-ZnoiP4oCY`w&>|Uh@Ep1MR!%GU{p)n5zga)ac_m
zx}3}uk8jESV1MCqJ6>c<-+ND*HwG#rl--H!m8a=KN$q#4eA|2HQ=>|2`u1+!|BUI+
z+vCkqe2Ww%P@l
zN?`b#-wjhhp5sJ}t`&iHgPG?t06km>-uIU;fPte?r&?pv+2L@mye^jH3roHgvGs*3
z1qH>ORR%F&8_jqH;_{0A3ljqa31|%{sl(G4NA>pWUDhZ>pI2w9w13_Jz~GVC$@+UQ
zp9vmZpQqbDY1^48{!Xy{HNdGNFNM2trH~OCI5s&6LTrA0eKllb{`hfdWq6pD@Qz2g
zN(jbeB%Y$%0XhHS0#M$5@(>^=C&$D=)!9V@lDYN(eA>7}pI#=j?ys=2;Q!`i3?DrV
zGRTC5|Eq_ta}aJN9ax<}6<;tIT3#ezT(88gFlL>v&LY3zCc6WJo3cfx@5+xirn>n6
zxd|BHK;6?w+qt{~Mk669
zT|dPFNQt*)RF%M=HEZXC_K;;0Bn|(v6v`=Jr4RbyI?kpJKeC4;L
zjgxQON#Z0){2gA90CgQn=QN};HYT+2=&SJV>R`yFebyqxRAjB^
z=1qek^hDT=({HEtzv(f-P9EBgPQaqTjaMSZivJZ|(UkfE=00Do(cyId8_jBzl5pp<
zp2Zwu9L&9n#E5LGxPOrIec;|%CH2?`KXldDQKbO#hgQ6rv^X*#;!JKN2pl$lWY+Ku
zSk#FAR;y*Bemr6*aa*{69iC}mQV00w%KiB((^K_Tg38p^yPB+-@UZTgewo{s
zz*U$z?a6bTy_K~NOnr4Xvy85G|13=B3ulmpHS(|+F9X199iYL@tG4R=ufbeaH0!8;
z^G37QptwKZ`FIncN;aYU6+Hk$-*(77azI=@=K);MCk~lC{%8=#U=1xV=oU?Q_oB?&
zzZJn(i2pE!iZ!f~OVF9ouyfOLw#U^F!L!7OtjBs$<5OGyh}Cw6g^KEy)K(t|Dddr^
znI$L3)A~;s=TcJ((_d^6JSbBM)pup%$dXjok	cCWsnJrv)|Dtiojz!fVsyVS|jx
z%u?Xh;2ok6q3s+E?B3Pw4u-t6bQ6j=@I^q}cjYTaIVeg>ruwFDW0X>8`Bpe18
z9Ko4q;OzYM?qEO1$^|hDs`w*a4@Q9eeR@^IAq^YSLqS@PiKSN0Na>oe1ABub^wf{0
zr-Bhy5X@;g{Z1&SgDH}Sk~8AYLCUyH!^jXb=S
ze1NP5@Wl3(^l`Ld0_ZTG_XD&N@<8SpB}Yd|FTaa_3LQ)spP!{3#n&1`++VNLa8C6hcJPL~3N8dLiG08@#zR0e47zylcF!Z9rb2
zQn`Uj(38{&S)HAw1A(SCQ1*DTW`=W*(%|f^&3R|cO{#68k9RaN3fxvQ&S&eCI^&G)1Nr;S%
zU|==Hh{+2U<%={n%2tyIon)6Go
zUdtUJ&kVgeiD~l)9PxYciGVj5DHyu|r^=}i`+rITfe4L=aHK7NBjwB%4t74$dKJf0
zgr0V(?}wiE+q>dGQiqMp(>5wV2JhrI@?6cR$_L%_O5Eu(t
zI|AedU;$#0X}0o4^L@e13?}XN!u!FaKla_?+5>R^(`2ju91zt5RaBL$Pq8!wF>L2!
zPaYx-2Kq8xs*JAcbKuv6=`vvWAGdt2Q2iIUy*+Wz#YC*v*>I1~Z+V(ApF9}p@}VRd
z&bjhak#RMF^A-2I~CjiH*iAi+~5K@1M>v>X9QmQzM
zZ3sk7j5^S+Dn#t_U-jgKqZx8kE#IAef8ogZ=|@f_Tt5fupE?xyI+@wNMv|qX?QQVb
zkaqRiYP2zg9xM|3U-zQ!M$eB1WhrLbf{pJ5NbzxUSnYS%@QB`I1YY#eyPt(9Z4j9v$}RhbA&cI%_;m~EmRYaomYM~$2?
zX4T35DmauUOCJBksrF?K${ZmdlRolB(@pj9Uy
zC`1<)7FLvqAv9+_i~zj>E6vMCW-OqX&?r;dr1Nvg4ZSCQXA@Zw%p7Tl+lPMH`Ya=M
z6L4gB_R^%S<4DMW-ud--{b}X<`QbzbUjJ3S8E3wU_O8vCjsxPzj1>|OaA_HF=X%g%
za4VkCZECkTDBw33K9Z(k7CGpB?D
z>G*iJvz?&!=ZAxTvMqitj2(VXNQZY@|5CgCe2``i5u^U^y&XQ&L6HE3UG558tE@&4
zT5WFc-+rQexk5XKw;AR9XIid_p2DNje6tfb>h6|jfG^T_^`x0EbCfUA3gAa5%#BDc
z^m;E*LY}!|?WxTFp7N5qy$BEjyE}{P>@J`1?>#%InGDSQw+S{J#pi+DlRmwRj^{hJ
zOlH!>fv8{)Nj13yBUZgMXm!fp2yMXAV}x+SX7vlO3l#ZRrX(@>T`P{79%v9c!-`hPK^H^c*MB6txa2S8tK!B%
z>8!jDN{c+>A@jBOY@IuM54@q!pYMog;cj+dsg@$TxV)f2gf
zIMTdtdN9=@SZVjCia~xmmzh{DTh946Ux@xCmHg}WNrvBxFGvaYj>;GLF(0&R!TP(i
zXzIr4kov<*r@|L|X_fobfR&uFr-NA9ra>@ohwG24lq_>4Q
z`rr2zexr4*L@(vQgY8#wXzV3+s{F)I>}7M?fvtFfLb3POiF
zGd_LOajL4{B~*0#`|U*bZ6lK@wk%aLz>o4{lazhZp3GO^I%|QoD;6UkmzZPV_XPiV
z5d1~|Pt+^cD~wUSVov*cBkC2{kl+AljBXShc)l=@$7%uOce0C{cf5TgpeoKNK`VGqQVe6ZfpN?O*-^_}Y%C+R-ZFm}}oBzIV3Rzq`
zqfP@3_@{};Bth3tfN~Z9z@dKpsM%TK)wXrr-`pi^)(C5jcbWEQypC8`k{;+4T{(WC
zflN>(-+@O$`VvAWJ3g2I`@E>fhP5%imU@&Cz3i2_902U4qV;@iZi0n_bT--473hHK
z6<1PWz|KuMpof_OqeZ+Js@WtY4@dzZ+81CzZvn!*_thL9BL1k?dNgi=B|kB7UqeJ%
z+(0;zc?_q}HK2^_w0J2SqB4NUTHG(tcjz>0$F}ccSI&m#KS3%z>UGb?4jO$bJz>aLC76xpQNv
zHM5qbOo`L7o8wTOS~t>lqNSE9fha;MR}RLMkeXIz@Ta@=q7NIUR~RkU#F;<1<#F14
z2A%;uTq{7R3^HKkArt+{C;llO4F^$fp@G_Ut>W^e7b{MN1~!b6QNRowq1G
z`Y%P%et%X?vJM1}QZZJhy}lF8lx%%CZ=(~2kSTOQ{LTw65zX=KyfuO9h}F@O
z_rvJ3O@T*#HLq<*guDNkUYY0Io+>)3bF$I>C2eCSpa~t|iJ&|_%S`<1bU=5)trfXN
ztI0sL&lLUbMOuFC!cY-%TD>GIG)n>sB(AqkSyly%pCt9KmCijvFDy0sfy$))+Z8)wik!hA$;aU1k2VY5BC_u(&%Z1)h|-U-GG|%
zvp?C+c~+`4cxFX@(bMj|g4$MZ!J%&KGx>w$zVo<%pp6qZAY1+&ftekC_#Yy{Cno#$RMarx(Xx7fQ}GRo|deq_%`>imin;B`S|N9QDber*>WHO@V(c_
z7HCp#^(EAF{tHAE&(?F
zdaq&RWsfg_$TolkqI@k6S(AkVwad5*aL^I?JClf_C@=o)SWuQiwTANiaGIGyP@P=C
z{tYT;y&X}!^4E8pkg>p2Z>vhv!p;3Q?ROeom>=k?-l$Ov?Ry!!^}Va(()^N&Y;5<6
zj6ysQv0_JdtFA3CJ8E$rW>lR$%O*;V&CN=sLIJMfPgZ66bnBE?HgYV^s_e@6@Z=^U
zbjT90pFD+A1(qKICbkM26^0N~hV5Q0LI>FjDON!YT&akAx_1R$wC5%
z|8Ujv)XsHTrS@Rh{G-u219vuIzKf(uHrFa0&R`cIw@sE7Dk%ryay}0mku_UW=ohX)
z@`H9ZudEt}%)iqqPELQ)n?Z%tq--(ir8cuyy8Sgz&tdI
z`ky6FgC2wMl%{GREPV{$C&%@PkcGrN4iZb;vy3udlbkywsEFRv7%+I1sR%m}(vir#
zipPcNR5)zB1QVU#Or0>cx>TQbKt^BDwMI1+@bxXAr}Z0s|JE_5FMN^bc}y_2VqhN8
z`@xIz7A0oK8))XO*4Nh?0CcqRUdLL1oizNPB7E{4JSJypxqkpM?1Fak$p;%1t$lGn
z-RaBo8jd3a0bR^?@Vhq#=pmXD5&m$0PaC%X-|9CI?GjkpsE36-@O{uGjadZ=wLO*^
z%unu4Ry3>4MicjT3Z5w6CzapiT?}FSaq66SMdcq)5t(b5=o}@Bq|Tpo{3M&#D^J$X
zefN~;GwE(>uLQm~IIHc<#2`xJ1OrvfgYJ=N#$Zo@p2=Y{7Ht2j?BBUoTQpT_C0jl`
zy)B-XC-Tij!fhFzf!?bf%9BWpTu~SiVyk8#XiRq>(58-&NWwth4JLT9%`n>yCiq8T
z(>oE&0-W)`al3^HA@6ViWI;}n+Dstv%;CcIf17sheGD{^Vb2#2;PpWrA%=O*qKQrlrx=TMqr~%Y
z^I)T)P*RcQfrqI%i!Q%P&~9CXnR$FHG0-9FuGn1gH|PxyCFq%y1P5n$!j@52eZhek
zg;G`KM=Qm_H+Lc)k`b8S_m<0+$nA7CT+zWZWT_wI?FwYqdV?}UE@yH62D8-1VNhYm8fZZsa6&{Ps|#~exMP3stV{HuUwSkSae
z`+4oIsOFFT|6SWlkDLKzc0zGFfPMjnP&SZK-0Wd1+x^Mn#0!o3k{cqTkNe2y3ch~%5dX5;}xQu)9v`x61=znm#AmxCE|
zHc7k-9%tvX6UGwUiONqp>2(K!bZXQ;Yz#s@YxrV=`+6ZZj;ezZn1i;=-*~_vzNl
zgwb>79jQ5WZUfw%nz-HGRR}t@Z};1ANsMY`4#}zAKor|AOGiP#o->_psP_tRG}}|&
z9W6TBQR72ic>zQWSX(aNQavh?5%0^bzH(m$=FWtlA+gjPVYT`2FdaOMgX?$ND7B$%
z4O)RWm=)Lb83X)D=S1crIrKExOuAI0xs*CI#so&R%}2FL3ZLa>j*&zx06kMjD%Gig
z(#U2#PgP(`2UX_wMwPJBuOg%nqpH#r?i?m=_(6fWPQp+HUWqWHdh8wA#bmr9o^9@+y;lAVDhqa>Ci8@
z%bJqv3j~TZ++~V(2Y*>eJGS=*_D!8i#MBf?gHO^jmsAXWISTZr7PZ4vUxwfnWtcz)
zyhjX_*Q(7CXo62u)m6Zm{b-)?e25q$Vw*>;^b^@UYbg1dT{&bTFDapg?QTXMxr;*Y
zFDTMJ?g4JA`bsuoNm@VPqNk$60B@Vo3Agvv=plXZ~{$=T8A-R-{;EjAy)
zuC~7*ew?(CPY&*XdJBR1c_)<9@grY2N$LLl73sOvlB{Z(`-29HAnZa7U!WSm{+)c?
zbl92duCuvN22MDREFMoX>v=|a60hLi3@~0ehNaZ$93TdW8c4^Kq@8Oh#JX;`1>!6#
z%e#99x)nf;*R|?ELj4RI$(T#@7ux<5$6MKdq`=b-6(aVYV&lAISl?I@JQ-73@r+TDAm+b*H0bjSnFX2f=NDQz@AIQgYj$oA*E
zA%8UF$x2ue^tJ10qAKGwhK_$>);^S`VEKizYBY=WXyZ~&`xu8VL5m+j*ov3T*l|sw
z4ksDQG$5&L%!)rTtP__~*@DN~J)pYh64}o%n3D%OSaM>8ouBqDa6_L_YmjxgQ
zZPoOqGJZT|UC1i>|B_+ZU2m3JtrDLE-|`iD5$2zLfe^J~{s(VRS6_uG$k^ql4Qg46
zg~<^%{C}s02Icio^3W|xvt37M=gO(jnK4ep1-KB-p5ogrrx^dn)y5$TPB(FKku}P*
z7+=R5tca5ty2Qs=k}d==BRJ{iC=Z<{f!FhzC0HFE<1&l?7F{L{>0|M@_{l&NO163g
zHbQKEwaiV)`VZCke^ubJq1m0D~
zJZRlrKWtF%71q>UD2IuD0}%B{X#=ud*>K>Rv(^=+NUO}37+BVA8Q6Z`dU|Yb
zdpWCP%fqFlyLir4G(nTzmv)nBEBsHbxb_<(FWnbsQRk0?v{ES_={x)$2p&<(fgC5v}A!XVm(OP#*Aa?n)6u&Y80mnYs{WCoxI}A+_SFJ$*{gSKE*gjM|m7=xf
zr7y(8Gy;KS3fzRCT`G@!;?plWbQ_%k-vpwV^IH|6zasR;dxP+n`V^vp$mQPU_fLM{
z>qdVvJ~8vmYdvxSAb!1bTG^#KV^P&u6vbSD
zKY^zOY!d_TGtkY}cls*0CHM)*=e;>wZM`~U)U2_Yeea8zEX;6zhA|kfe|=BesTX)J
z_oM2WEFS=l8`^ray2i0_PrVrs-SsE-b_k<_22XqoI?9U@9K)3Bk{XU`c9W27NIXr*1
zFUO=!&(=|HM|jd2!~iy4Y=cgIZ|`ZaTs$*Qhe4KtG#rJ>-W&fCle0o9@DfEu58!ad
z`=n2E)w=Fs{A0%Z)0pN3jTmAw>2Yg1#59IZhWg
zqm<-@#7LvOV3vT&w@>A6jLrG*7?HmIDbB$r#?jmde}A?boz}l5-rHLJxiesl=2S4@
z$-C1w)j`7$t@^;ZQLojR)8_*$BsSSMY{LGlk{RZfQ?n;)xmY&hK>2zO4^x@hTNFfA
z&X&2S=)64?9JCJ!1^G(hq?Q6PtC|+=U2>((~z3ZaR~CxJDCF-xy1
zns4>@;a!N;D*ke3^e)!>7~LQZ;%IDSti6Fqgs$`QGa@z4k(Y81@u^qPe+L`e>wvAH
zU}1?IO@GnK+xP^*=Xs8?>zu1DR@byUACi$U-f$FZwL>N7TmS1};;vqA2
z*4vX9@J!-A`O^7BDfh33o%M9dDjIo2Vb)#T^W*=mz<&Pq!a(m2&=e%V_8XT|4F;WW
zhzBb7L~~dDX_gH|NvYx(PjPv!#A^-E7n1eh(l!l2iC=O7Kx>00=2Q1wI_#+NQSnIG
z57CSwuwjcyn5))D(j$W|#5F!$>13qpQ}A#53!Q!p`0Z(A70n;Yh`KnCx;3V(#3AyP
z9F3HkaX8k{K?0PexPDn$A@>eiKX(v49onuLhdUQV#XUbTX#7D)>AS*@mNJsE=UZ>{
z)n=DhDfMIfkS)p(^BH$Y&PL%>mmwO@D@xO?pD^KQ_s^p21P(pMfc~a9EOuy2wRoyi
zDm>ChRUGY~rJ+3InTc>V|1M>fs_pC^igS>g2Mzk00o(waCCZEeDCgPp1ZrRD6PB4`
zsrN+}pWThGb=q%0H0aYy#pp8r1@F4XCY~(KX(sJ;H|frpt5nIQR#l06y-Ua7Quq?C
zlyb%32Jr_|ZmKI-71;L8_CWzhszieM!j-J?SFP~i#ldv$w1EOWh
zkg-FvaGd&IsN5;+t0CjtzZyuTaq&NEdt;~(YQfl@bRfVG;suIgp?;S+K+*0CB$pj!
z+DICJt6>5vURm={f3DTwDZhN5c?zHZW42m)}XCK=Gi
z(niNQg;g}ULzwKx)S1?nd3d`zE7BJyrIcPLrA_^saVVG367+H`p|&}!yY{2|(j$Oj8MuiuWDPf;}osE|Zz4V9=l
z11SSmS6xZ$P9D>0Y191f7lL-$G`S2CO`AT#l9Kt03Al)D^>3q@d3()U3H#+k27n$P
zuL$F}ZTXuMZGT&QI^ORl*~fHkP@ghC)>-6oLv?qR0M2Xn{l3R4M(x>DcVHSjQT@790aIhZP@jj6yxS&n|v
z6;hSr(>GE`=uw7Wa~pH;>Q#XW&M6&(50vSd&&j+Q<`tFpioN}cJ+LdwsEimsNWddm
zYK~mWg>Azdw5bHVsGl2#sxv_Z)B!xSZ^S4)CQ|zzN6UO7O`el17qdREwXL?307+50
zF0WN32%(izH#L1|VN=aK8WIuhugXeMlax=>N{rE~aqDPO3L=OoMT`jl9l^XLn6zD5U5vl2t0Vt{I`
z2MfROEe{U~W&)j}RB9wfrA}#MKB$l@${V2B{(4NSgE<}aOYy#7WKyL@#(@M{8kW!>
zuBjj|Lo<8|JQ{^7ki#YEi}tEiSjE6$
zF$UB)13`XK(&4VOmZ=^A*{%dzNL+6no(7Rd27Afe{?^wGM3lLIio{+Y%o+F4FP~pm
zl=o-aF$IV6fWaLDVn^b|@N>{qv4v|;0)kU(U8Km)xUdb7@&cdS=KAdMI+fF|RPGfn
zOuCk?V(&n&LA{8m2x~I&YTRGGu(4en{79o3lqxF7W0fgfCMKCM*qP7i@?s
zv{y-gB-3nkAoVq@kvJ1AM2sr;WN^Pnz3-?wtHi<-bc{U{=BCv=^2J$=T*NJ>8psKn
z_E(=lBk4RmW~d4XT>@)U-Q(6Hn8}!-&uU>Eb!iiV$dkEUVX3y`eYr@nLcjtSLeXZB
z?H87}!KuX4LH;|p6RRPlC@3%_Xk#$SU8zgOfKHDuJekd8!wa9Qlm7as0g)Y!CJYE6
zl79SIdPc5(_+bPy3)0P|4WL9bq0z?i1=2Vx_6Ih}n@Q3xZXv^p+bvUm)5ck?sKmlBkiMdG3>b1E0igT6jEy^C#2bX^CN~JW
z@-g(Yiet_}Eei$(680+NldD?Bd_V)J19encmJ&k+y&Nh#)#_%RU)}%4W?>X3q
z(l2Q}v$pVKQLYTAlMz-b!6YLj9#IQjL-|ZFy>aQjZ-8ofe>TdY`ysq9{OwlK;$KH0
zuE_Wuq^c}G?elV=9tTkFKk$eWa4o)smQo#sj
zsM;kYQRqyiL#VK9v;@&*@R<=^Wvjt8e4I5|rQPi$`#yhI4Pzu%@C{|vWUAx6gFlS+
zN7V57P}6%V3X}lkpk-*1MR|Z>_`Ao*gD|Gh&f78v-^GT0~#3
z@C?qAYf*=aYjK9}nom%_;IDi_ICJLBQwQtxAu?6*W^Z|YktGklLfpO$tp}3ztX&-e
zGJQ!jBS*>kW-{Um4t%KRU~eK0jfn$xAE|4iNJ?1n*rZVl2T(s3&;C@}K%N|)8Oh#i
z$+X$u9+{|H%mflLTwPryR!qHP1$>1!!SH#4MADtzL;I=
zLY5#NiGe+CeI^y~Q5TJ3FNwh>NHvYaKmjxl9_g69MpE&aq#lGAFx@uZsOC#km
z{lS2*R@d7{AOLx#jLDH;iG!MI?>z}Qw`(YVLzh!focNaL%oAuXz+c}4vf0WEGw$OJ
z>S(S2@1n^_COC@SZFIC)dAc5AtMR`;xu>Kyb<53r9U9W2S==8gk3G?D+7V8UZ|c}p
zJQxuDL>I4=;e8h@es{7-ztKWJap|4X>jQEqe`v#3s_t1(b27_~jo#;ApPezxG~pjW
zVDhL}M898iO;)B4W@L*5(ZJphK<4x=oO4hfa$%Yk_Td|qzLEN!&TfJ%y+=!jKGJ)1
zM!}@|eEPTiD_@%7-r-F;Ps?(u96sygo|EKqu>VH4=
zP-xv$9RS3oNKf7P2KP(*R@lGNCM6pRuLC=g2C%++R;!%4-;#PK(U6wj2)c
zs}&vI{&E@Fj(ANeLJyKskpds+1+^DVpTU&5+nv`M
zO@Dh64hj4r7aFKAa{e2cf5s{*T(O?j;*Jr2mWhLGl|fPEGC=XRy|g;N2fPf;Jg%mz
z+jeKTuf(6b^hj-{`_tkW`F~CzW16s9fgqU>_N4eRGO9vjMB4P`x7r+swe_^Oq=|LM
zm|$HA41A#S_lbyPgTIJQ{+)sf`O-1^+ThD2J+iy*rB1mp6CQmJ+3Ilj343oWqw|LS
zy4-MpobM-o6_$i$QGVgN-y>{!;$zxtIzSGu0!qt^r)sJ_u(zD*rFK~kc;
z+eoAbD|z8dZPUEdQ`5l1C%$S?{A0AbEXyRmvS1UYSfSUJC^`T_-*H0YrLvUlRpFH552js%*&;ALrw
z*la`
zd5^(@hJoQH;Bndse%C2&Sbk$T0{}eZ0CTt2!pzL9=e+@A`U?ECvz#lN{`wnmvUF>6
z*{}U>e-{D)ltjJ0X=R;Q0G0UrebL!y0+7X$r9g1inOeC@XEAua)0^u(?5x6=pVc1_
zL!QlC4!WkyJHBJZc!vJh=CK5tG~!MWs}-aE{L5x(Cf+oP>J$e^Sj?a*0GdisJZ;=t
zIk!FY=PS3!U|W_yIM3`YA|wMFvP8{DxJ>&Bv*&}Io>#jPWzZW)?NSMpX@j2QDU9wu
z02rv6-E8DMa2GF)=biK9lFNYp|B)wwmClPn0z;PVprZ#!V2R`~Mot
z1QT5f#%fgSRL91~-qOFvlE}6DJUC%s3%RrdvoIav_BrbNd@vfbZ5W+in}0jtH!
z^ZywzH3A281gxo;!H~BA>&*Dw%Nz3m(rZ!5+J?MiS7Om?snd*g_LHz`(R1dIj`AD+eU?Piz(
z7pC_B`Fh#A?-$Cn~tgb7v_|tSy_g<|ju*Eae@!6Gf#mc<=?xSX8QK%KKTqKUBpE)Gm&*Y
zvO+q2l|jw6ApM>q`VIb-d|6>dR^x3deidICD-CU1WgH-#5LF*q3i^^E;3!;89%=~F
z1zp~%Vi(CAN{-c0BYTt*KIA*NJ)tmlzEt5R!>wvM`=2d&Q|4KCFi7RDYVl2D1nwq)DsO}xzWLe)ne%ZSf1=8Nm|8IGz1%rWL7mQWVRs751zI~m=s8-pmE#>j6
zDe~Mgrv}~jP14G{fB63g_u#l+x6{MS#P(|!OQo1gg{CF(Kf(VEeCIavn%A+Zx2Sh%
zYyDr8mlv5gV1fhmvo6qq3cA;oqo4?V$77P6KRZbIJmUO&wYTkpH`@V-(3JWDOemu<
zApcfyf0>ZfRTK_9G!DE@D7IuU#4rZ@^OMA|wei?37i*oHHU0{9wlJx8oP@)=dNxzf
zuo=K;y$TI%O@Igb3eu`2VJ_NNB5g)5>mgkm7$7pk0wHn}n7|#Rty;j&l*niM6^MO_
zQ+{|D|4U2OdU+N4_v8Ou5{wnDH#giA=6D`3#BiK`V|d${}`BQ`63|G?0H4aBE{c@bve8
ztL2uDxpN%4!v!3Ru%fWeunD%~rP!C2!Vs#zI)lePZTTx622ry)J}6UTBVIz0fncKT
z2A&wDE%6ouUf0OB$F|Q3*~`ts>6I!P{@ggI>-kdmrJySw6+ke5DiBu!B!cSX=xVqQ
zM3|UP-9R(kdBEJ55KvVsV91{QrZb^3vLIS#|F7(#8A?G^h{0bY=kJIeHSwtbJ&+{w
zQF(s5_*0SwZuH^sjmZ3vx>QzaQz|$C0tbE|A%V}IOsCL?_)cbSHgsEe{4E>q;o4H+
z4oBzUIlHgSm$7L`HW7zhKB-*+VkT_k?Km682wM_wczuur9$Fp}8sQY3Q_2D5^z>9I
z?}bi-PNhgmMGtWhK`CnkbCo0z1_gJ^Ykk6C*UsHu(V0hj?d++hy_w_5?X>0i`R>ne
z?}9zH+wMI#Cc;bliCEG?L9ueNHG2+8?Cxqc>53V)3#W1&-6B1Pgo>Xy?M%im!XuD*
z;X>%2EZc3aYUKYS$077~cJU%(z_1slStd5{kj%lw^nF*^9b6J-;#FbK_#X8Z+gWFag0ohc7my#l->
zD8;*9`{WCSAil_{!PAX3H4kEuWL;K$h=fNn>fd})Ex9Y7=A!5MJ25}Rpsb}(u!BO#
zX^Ddn&f$5vvn~zI-yrAQ8wi6X%|maB&*^c#(G6skcmxjkp-XH~nR-&stUW(m(I(XM
zvRj4CHP_cm8#I`W;#>Oz%S5L%v;aG-)FH^A!-J_wF;E6Pnfpe&cWafQdxzRp4FFJR-6{ncy0{M{;fPG3hxwG9{%>gO!`C6Pa%sc>v4{_J+P_Ae*dez7he?FXOa
ztAy0=WFTAAbqUbdVMP_zNpi82jS)n6L_MBBbnO=
zDd{PJzN6Pq7=9G1GC!H4#-6ERR999Ik5Mbob}3q1(UsTpbejy!e?8&!%}qs(gy;CE
z&d|i`&DbHI`?Yw_@1Ag*U~s`ku&NfiO}vE)`WSzcSOpf(^I#=JpnAAmeQ
zNb~e#&uk_+uC=sq4=2$oeg>|M5oZ+;C1qdGs@$mOjt?AMm`4K*hyN%WrNZ&?G3IFN
zoB&VmZ_3-78(+D@B-1$o43-_j)kz^Sc-zwAm`{}%dabv|?&nvIX2VyuRYk0w`lKJC
z8O!k0go;@2HekZKd#4_kder%@T8Z66Dp7Ob8LXpl7;lqp`1$$!M^oK4Tx7=w`joIB
zgn#)yUM~q5nB~I5H_&VabXe-wXlXTCPJ5`IY#C@;1_{wlHUYbOkf+s400;V^RlIwJ
z7@b4t=N-i@%<>KnTr5l%g@6LS6KR9Qu}pj6Nh~Z>5l7+Ux?Ae8er!aF{17E6EWxc`
z)H+KI(o!;D#Bc=vY=j!!HD|+09r1ahjojWUAq?Y^fdSf9-+Rn${n~=G=iLOswDG8x
zV6(lj+Aa+i?Wq@D4ERPHU?zC{$h<{^zFe^BIy#HYFOf6W;yPcgyAXGZbryJz-!$pV
zi?zHbAgi`jcNab{>v)9BSk=q*kf{5g%dk<%!|-WRZ{XAHO$@g
zQgw=nAhNa}VTV(zP=BPs<=jP4m&3ts-#;#6k*CPZM^#?U(4HS*M(b}nJx!&#i!Nuy
z9sj`N99A#37C9nmZG%P!T`TA$Y0&&oKD{gb>UL?3dj~6LHvf>&)EB-G9(D}4R+3BvcNMP!QJ
zHHwb19fEJaHw!ftxQl@xH94o+ikFi^pPo+ZN$dj{a+K=4C`QzOLsi*S
zk(Vke+!Nm7807dv80PpvfB*oGH<&rt5EV|>f=DV-IyEX*rCK(?yFSaoYE%5Y0t{oz
z)fDo{?^~YDU4zk{30yPRI^?=WSNu;g7H;lCLC?Lj_&*R^RNMIvGrJqglw3BJZMe(T
zM<3dsPQFKWGKwaAFF3#T;x_HawX^Gls7`APiLqT6>7?0NHYpG*-jCOU;=&z$dz>@s
z>=MPz=Kfn`o$v4ew0F7Q=Cij;Ohq>y1^+obiJ+|@`v=~N?`lHlWTsmd-)Qk^UZ!0c`VB99~1V|o3*T6o(c{Prele7E7Vr=Ru!&+thb_5
z;a(k&t8*9B@pH-&O2}3Qp(k}AH+PEmy>^wjlYhEerXIzF&?TCci;~F3T%LndqM7Y<
zE_8}7d#_wOlMiRHw=$MR1!tp4{VKZSIghK0>yC7gv42}ZsUu<421A$p`aR)ARMM=G
zZNd-qZ*&0@gIVg2E2dAZ!~_#ldQsOE&#h>PXHdgn5IW}F-9PzC#*p)GwM%#lNj=$;
z7q<*YbSC3irkMFRhPI3DaG@t*a8G0-M_pa#gsC{%L(9neUebQADL9&o-%^p85X6b_
zf-3zD1MZs5ol{X`1gH&d>r2;J+`OH)U0doe2<@8&hkBZXhV|4G3X+i!PFE2^+ye(F
zc|7Q=G)Hb0NM7-kYd`Uja&{T}HrO5+x}@x$EwhhYQbGy`TP9JD_i8}S>g=htM+O>&
zFQNr}@>xvF*QQ5jSvJ|fbt&7bQunggt4^r5JTS47(e`C^7^*?GR^3vL@oKGk!>Nv=17yMZJEN7ox
z@hg_XRFphaEJ`|=P{ZzAEKEn&N4XD&WQ%#5tW=ic?_Ub|6USX&YlEeH6CNuK2yro3
zHva__4LXlqrvQxVF^B_3n<|`acnoy2-|bfB_Mh#Ar}DjJw*Mrr80Y49D6$y51Q%i7
zTfK;vsqzZCRHlH4XxXEq%5pW&NyqHz&Tu%K4EJa#oob=jlNtMLvESevA+0+U2*OYGoh$tWj9-0lUp(8y0Pl1GM%sED;p68W1-zth?utAfVLDs}yJ@Lsu@FObIe`v}2
zGp13D&T~uijA20Gk+Q)r`5+?+ML{>|-4h&rI${&x_7R1no
zlgD*={>=09ZKi08SE*=~NIzdY$Cj-%E8*p|oJn!fD*BgcMeP~WFQY@L*D|ISY2@*x
zHa*1^1{K=1ySxRiM<4g>B^UyJ
z=Cjj=p8hfb$!f(J*J4mJQf2*AS{q08;j=M1^*1#3nI*P?lVY7$Wy@B@N{dIPwGaDQ
z$;iV0+=EEMuDGkQXyM>g7t$iOgc1i2k7;ir{FYI-IY8B#?H40EI$8KDL*pWJ2b-P<
zD{#QA?kC1atug$T^?wOZ5gXPZW0>I2m7mbJ3$1wG!=7ZADHS7^ravy>eF+mG9T$$G
zN$+bx%WyK;H)gE-4qTF)JtSieu62QGy#8V|}b)G8WJT3aBBTYs{5AhH^
zFMzDA8j;G29w>Ue=objZ!(O8Kh>kg5q==4}yPx4YcWtp}
z5noQO9yD-J8V}^^>WHoVF=~!emV+NXy`EK83yJ5|&6PqGb2?`@gJ^-MCdgw#4tgp+
zYiI(yt+9`ZP}hfb7`G#@j+xfqR}qIcv?Hqh${NpX6lCw1&Cx#hz1NFK(~T5ZxQeQ^
z^O^O!Ub10stLp0()h1xcDcC>EoRJ(TYE4PF|32;Fx^*5V^C{&$&}bBjq^v2LkoI+-
zP+5nhT9&F7izp!=;uyP<*-`Xzeu3BQY~c}e<&O#%VQ75oLLjF6r4XA0p;DY27P_!r
zuf{`w7r2P=#O7{3ic$Qd99^1W;#5TwO^O0asYm{cLUw{cx&s^MfUGIYOs?m*{6MMg
zo=#t&fa)y%CS(s44~)4|(D8IrG~NciEd~>JvApl$^*Lq<$`4QH&x(@M)K9BDA4D=+
z;73@&QYbhVSF`?tM_yk$uvZi@Q~RS-NwX8GfQZ@r}LgM){p&RN_evEh_(Ie59kuRWxtN$nUhPUk)TuGa)-U)Y&*
z)*&Ps9+xmgziN_gNaq~Zc^;Q)=?;zSmI@1&YF3umm%Ene*{NL8iRI4k@!xyue)0%f
zLP;z@FIb;$RM2e2PxJLWmSd@~k@fc;os^H+ejsJS=6+znvkoc{{pG28sLH|I)ER_t
z{0pU}RG#TV8r71lVc_=^^{is-gpwp}ZueVZUQr!?`@c+9qB;(Vv4cXv{ewSCp^`zL
z1``;8huX=>=_g>ILvd0LU<&O{04%n!wXuOM+XE*c2Z8`NogmxR`izkM)|bgp92zFF
z2bYr-M$|6}e-t*jxVhE6O}=uFpAUvXTpt(-7gX)pdZ@(zO@_$cV*thHd>M;nlX|i>
zEFO4Cc~!LKU=Y^mD~t#j1*x1-lZf6Q6$$F|mEQG53?=w|uBeWX#(C9(V6|MfH|ur*
z4{!L_LWQidTJz~a_RYZrUjxJ89O+h~1QsVg29S*sSBH8M1X8$v7>chCxSl>Kr_tbo
z0{}Rdl$=?t4pW4-m|9Jh-V#w*n-KCso%h#DY*l$xzf>}eUeC1O0;d`N8ZuU{;epZkeQhuRc=<)YR`!jniYkzD7Gp|
z02Lv-o`*gkWOsGdjPUWZo}TBeUIm`ddq-Mm*j(zHs*w2m)X%}3J^QplShyEt4PvNE
zJh)2=vyR-5QZ0E+g6nUZK`VqP^h0F<$HC3r3C5b2)e{%4@^ee3zejgDzKtVA$n%#^
zpIIX|ch0=kyfS|ua|K30hLTT09mdoGS@p$C_oEo81mX1=45Po@(bM2Y(>7+6{((T9
zvns?#l~BCOC?ItS(kzWmlg+70@nxl=P2xQ_YO7fT@2d+c<(m0ABd+;$h4a>G5w$f1or{^?1edwLI2C%-i
zM6PkyzJPP^Iqz##-r9OiR~zWT@3DTUYh(cLlt1I=<#i(>mA!#1y?ligko_|%v+vs?
zU6=-G@$pCA+|jpxxIZ$nfvS5P!haC!EFfX_-0szZl;{%QR=?0aTgQp29;{idb=8Vh
zl({1t_cHC`E)3G;YC;|t_jU3NfRZA_mrU#Bfwv;naSgQK=g!sbNsAz~5Q>CM@ves-)uuhK!;kNbC6}o@WF~R(LG7xPr%m0%lGix2F9*Gt)Avc8x
z>;c3J?%-x`!X*1@dZU(WT~IA8YDwf$Fegyf8s!9DG>Q11r(52imc^z3da%gte_hC0
z4}|fgEtV_FJL3GP)i<3tlnQj1!giQwHdYa)&7j|63ixvzhD46q%Oq`Wk0?x}OlwKY
zlKGfPK7WlPkKkP5s7;GMkBgT-<%MR%>z)*psW
zz~>i|rme*s559AH9*Ea-e**EdW<#|rs~>d^-5wAS#ZoebT}jW;j*nVUk7k{r%TF$q
zN7P63v$gz7&o|VrD)dO(%__cd2GH#^pDKj5)5|Rb=XL`Ht`N@`5;;b07+I4#U%1$7
z79Hp{p}=ogcQ=Z2CmqC-`b)SQxe~qC_O+)Le{FnN6^xUC_B@=Jy1GnQCu82TGDUmk
zvw&Lmh|>LcZSi!hN8;R}pZ97T@wh{AhgzeP27k(E^yb{w=m6aRv&_RwXXTFE0WRxg
z1IKo4Ho^Ah4}K$~i<=r2*8^{JLRAQ>s=l-VJtltrJ&^A2mQW(1E@
z-V|g*ku5&SXx%KgOf_N;7eVJQM#}XumkIBDis7voviBs>%Rq
ztF&e*XEMwUMVq~dgsm=ibPha_edNSCBt9h5e7210dSv{jp&=$^UTsRjltn{X5>>t)
zUcrYTy8U%={l0qie6&rz8=IxogCnmg_rq%c(i(%m67<^gG;Os`XY+doTd}Mgx66ybVd_GhmplkmiA+
zbl`5}A%}kN?q{h7Iip3%8skepc{v<7-3eC3_J_b*-?iy^sB8A+lNv<^=^!UAE(i&8
z%6J+5F=|}KGg7Twd=fJpHRZt*jac&_#dO-2oF7<{xNb{cCjF%TFu
zx>r=js6kTn*Boi0KUV51!`<#)yM6bM50tp{TvO2+J7Qpnc+YL*Q!QuZ#f=|XRdo;s
zjz8K=VR+_n|1--~iDmmF^m*81YHXCEn`40C-(vlVdr%wT_Eq-o-MeqCK=`^Qz*MBQ
zw6v^T0_HED#KvCXRlT`uAYw!j@PjeCxfl}~d;7cAcD)N34K)~z5F0X@0Yt1Ub#!#B
z0i$iTCm*)pV?Dd(kv6FMak#N#~Jp&+qU5{4V+phpQyVT>|ZpH@^5|YjS#&$NxqlK$g
zgT4rkS4Ww1I+bEA2WRI9EN%Z8ObiSR%JsWxPEMuRx*jD*Y$}s@{~zJ$+1X-B*-RXD
z4M&RD*k;}6k27JeP3$|K6CeY@IO9yh?nGcJM2-wO6*$zY#
z&B&XfdhH$u4k6BTEPwLQ?dTFaJ=(PBkM1M7e&NCR7WwFJGW4TSW0z42!5$wsvZ2Z4
z?29SiwS3;eqy;Z`GwUM6)%l4s?hG?vdNaAJLnxvnP{2o|%(;7O7iUGV5$5B-KvcUa
zPkaW|CovFc9^j6z=-i?so&isg=dy3Lyuu*R#}6d@^Vl%L+DjJ$ve$qgkZg!88Pt~;
z6$Ry;=+MB=RErD`rZV}XNKyg!kBmfs--J{(Fjc*+uz&&0f(MDGB!Tx;tqVm{FlND3
z@2B(zy2Q{Z$odoRR+J8=iX0+ZObG!*MH+Gl;Bso?o0nV=2xvYgmA@wZ8QnJPDin#D
zx$Ip6GOOV^!}?#Tvv`tXE3*SDu=CzG8sIii4fZu4saYH1DgOwez&j98?9xNh=0|ZK
zSwSx6xe#4HuRfb_2W(SE@BA}I`u~*r-d^rw(bJl{5d3lOcaC6sj7V%Ge7u>Z+{sTt
zODiZfg73)?GwMN>p{ApmO~
zXlYPBFK+jQkmfqObkS!k(Xpi`4l?sOjhR?mKPqwb-g}BW0Or&y|u&T&qTtdA))UKw$8eoRaS;miDC!9Ae_svo{SYvZr`m2ie*rlc6-<_}R`DRmvzQ{L}$eEG)ufCHEGB*xOB^dY}RGWmP~
zQ|wzk!nGFna7*drb(yZ=;;5O_1-pl(<6`3$?;9&g=zF#ba;V!&pxa4rTr3a1nyzPa
z=ee`wI+epFlC%<(jSKH3nNX@Jlm4Y+67mN}+n=APq
zpVchp-`V6Jy6;&ODz=iHj>R7hX{j!q7H;)2;1IC6usFedtmCWG4j5gw_)X%vAZdH|
zXmDDh!Z4Bt4|YEzyq&&`P8?d0(k@!cVsL)xDEEk>$v{;u`WNpkGf1y?Sa396Kzcs`
zgkoESN28<;C8Pd`j=Q$HsyFDbSzR3ES4r9MSU*l-$?_5>sD5O3-M05TWV)ua3Z2fF
z$wTqrkH1?)N_7kQzG2mmSH6JZkl7#%z7Tfg784Yt#D_C;LH<~Rkf}Y{HT=E`_nB)4
z35XfRLikxe@Te#KH~mmaZ*=OzTJ$2`XqqQN6>De|V+xKX1}Rd8ka%al3cl{J{D`h=
z2D>Z}uuBcgL{Hn0%SM;MD2K^Tq$
zToVC8l9U%1K8y%IgO_Fh{n{T;pTK#Re?B9f^)iCAiVXOU=I=m@n3z!U-;tl3+lBkm
z<8_ZR5_SKEOH>-0(V`K5hjX@Uff4sy&dS#DzV%u#OJeU6lBLaE?%}3@3k;PfCWd$HH-5f5ALcO9B_l`1GT+m&v9Y$pk
zasSF@O%NVGoXod|jp_9K32!U^fI4n$pZ7C9E{!r|FUbF=DAHb2^5&FM(dX_U!Wnju
zNDV|1qNP58`Wv|j-h{@)`5n8Z;?-?cj+%gB5lsy?cI>yv7uy0P_laf
zYy*jaK_Zv0O!eze-2DIu4EO=KRTzuB>%r#CNK<~rt8eOvMU4_W^m1@@*>TA2C-9qw
zyb6%Qj&@@+C9Y0O9*;A@-XjR`IR(h;w;*M|r=@uP7SIGLF4n)b%^^hFt8oNV9?Lex
z`9&J8wcfL`ar0Zz35PLj9kl1PK4V854UEzxGMmp*_7v)ib04SOJhy3YRW0=6u3x|K
z7aZ_4Wb#>xx)vw>$E!l^?^lG{QMZX4L6{XA3Yo4{4G!*@n$E1QRB@#fu+<)}P)f#C
zXp%2?^$xx%ukKrZ6alo8?qktkyjFyoe_XG_)T}@&dBtb@N)K2z>B47p=;Ul0jD{Dq
zWc3yX2X9>Dj-d8ee3d6fPuSBbB7zy!WJVXlYx2Ow15KbG@}PlIdQNiW+yGkOMM$$I0=%`lSdM!
zk%+sj9kv*AXH>WI1?H{p?n&JGe!IC=GwbV=wvD$zqU>Vo{Dp(q0zs)LISw
z$j#4RAQkkn(GP_kmH#tHMc0#AXUZ#-bCj~k(Ia{~X^^2`ic=EDP$^$OLyiQ+B^Tez4}~PoEcZv}*p^N$({w7NKkT2}h!egQ~<2;)moH
zu7Ijryll$Lmt*!BZqSKf?6O
zb&5aP%y*j;1(1CS@~03D(6V+O8`W;9Xw-lEXLN>la@nac2i7K5z~U4I2p
z*6F-a%Br5&hr8|bX3WMx0Tzh$9W4c+NmV)r5E)uc$^m65>&=>P4+
zmFWi>$~^9c#czFfkq_>>4Q9ohMtDfj<&#=H75-2sR8bg
zLEYoumW{rODs4j{Di!ljg--K2vP-!Pq4^O`W1EMkcb~JPzr|{^J9YIQ%D5ip_thft;@i;JkMVnXaJSS=SIf!ncRuOfgR-kRUDkzMD&FcXT$%B`d*Lxt4_V!PuUiWj8+lBETD(Qr9J?kY!`?gBK
zRv7PxYsS9eU!PupmZK%(@aX8%cVcALGcX~u-xFs1WbuauyL(9rN*d{`wu%kTxODPx
zptVxbZ4kfavYZ>IVP~&tDl4=4`zmh%dOjZf?p>f9Ckgra(}}_e=%T)#?C!e!!eR?g
zuaa9|SR+jV96DCcl615W*bnx@riV0Yr|Kq3ka
zMm!);5$FTx0ppGVlA7h=i)
zD2FUXMWL-9+akk7E%zuw8?Rm*hDe(d0J1P1ThGWe|Gr-$N89e#n?)x9Qk30lC`!6z
z2u3T72HURs1S3}KmiS6@Lj%O;@s<^Ho|=kk-hK$_?CcDq=63-nU_sBCie#(^P*R7h
z%S%h%<81jP*zPii3q5dHgoM9jOMgOyquTM#Z`gqulkFwvtOhx8llR@L=3W2bq1(h^zaG#!QvS)p2!6e(?%x*=q3BdY*f6dznkg|q~oC@FI
z=b!gIboid0dqJc5+V(YQn*CDu+7LhjdTf68OH&*+kYbzX-C|GgtKN%*+tznPK)c?0
z`P0+GjSjtPhIVA-E3QEJQ1sJ3iJSX$9HZ@zWTrxdvMadraej~c0~N6CTMjyPJzYGk
zKV38{w26E^J*C5d|DguuE4ZmN!uMAb9eBq`)*M{GpV$2VzUN0V-Xr^XlJI-cckiAS
z*dFy$*9q26SKhEkv5^R1fkB9gmWN@U`82a~{Z`8YzY)zr7{*GY78~ONe+m=XRZ#AJ
zYjiHxe>^>@g%D`ZlBQbTWy2(TQoOFUuvlKzAJxj2NSfr~1DfuCrKwWktRJFk+B&@{
z)FS((cDTFlK!n3eyXhf!X(Uxrwn9(%t`Qz8_a
z9(?tc6nA7KLO0N8B-*{8OsPZh!_#hEj~NK*8ZW2rR8g*)Y5{Un@c^sXE&p
zI2AZ`=x!Sh-d1Evz7QVPDVM1p9{j5Xs|qji5uuGBj2I7@g9dj>cltHIl$8z28j)U}
zU-6wsm|UD1>h!d!du4frhRg=Ke(n!jRE4$5nKQ|*vpgiO!3Jw7CRLn`DcOxrj$!n&
zDQNr~;$fgA
ztr?OsIAxVN_lMfE=~TX*Ki~SZL^f+5XtMFDJO(!Oqj?FrdBY78{Qa!+7Re;1ITe&_
z$FDGi>{H@E0DqI}ukg@V)^k9tS6^_DJCJOQ?c+O4kbeb?8w&!k6Y0xRn^!YGa=57R
zb*wAgVW6euoZl8;F}G!OD}4RxvN`~vjoJ?Gx(88^eUX7>^BW`3K@dk#jN~O6t$x7H
zN_U#x>
zT;LC@hL>5>k=Eg|a5^r-r)=pMI=+K&YuTN=)Pl0V^34ORd
z|G;D;$JK{fJeUV4`ZlRaN=q+xbIFZ)mGn%;ge3N*T%Co<$S4)PPjVPoaA5=0D0^av!M8LFHj&z*)KgK*K2^m#&lAYr2dc8V79lD4?jR0p{L
z-+P}@DH!VerH*ogeEx5UCFos8h%l5Qn(>TXCYfEbtv^*WL$nnfL_GLKN^Y7vO`H6U
z>1TQmMSSPyTz?hEYqJdzCjOoyBrFtcqcdTnaA72tT}gNi{Kh`Y)H>QxEWX}aG5BD2
zJ`LpIcs;G2HVfRhNc!nXhWgvt;$gk9Ww!q^c{*`m-|=HEVW`?g%&{8rX$6SkR~szd
zS7W4}kbEiWyP(KLhH0#a;f58??u>{>;M3SO(;_+(=B-D8&$YeY@cK|occD<9N8gNx
zgLJ<$4b*E5Yx`&%r>$sgw<}27dZ^E*!H4P(=xJon7R1EAJdwUJq6T=0-H0BvFZ!}e
zqHsoVo<426xO$y&1Ox_D?ba>Nz1?W*?VhOmoHbpnYO6aAV_MsU2ju2Kl_uq@Iq9!T
zKvdu2eP1M=5$92i;l|97N%U@P~}1t+{^a867w*JRo2cAvT?qFsKvJ@$+2)
zbVxo7iV!eNbb6uI`?g-!T_+b+p3upE9x}lFLI4hNn05c>xba-}Od&7$iWNof(GxeB
z*%EnG$H(7Nd!8)1QKpuBdhMa)3&_a%v=70$;B-DsZ)Xg*Q_XRllgi
z%wCZ3K6cK{a|pjB^>>^>>>J=m^|)7}3DN&W1MsV@PdEKXk6c?q&K|Cp?3ZOfFa;{Q5XpmVi0MRynXH=39aN}UX2$W4Qv+x1-w4{WYGZ!M_IsQ&)*+IA-b#(gvPkbzgM-T^tC*U
zu!83!iU)AJ{2EoH&?^4ew)IJ+J4W8Qg}SL^ixMJvKGcA7dNL)W0-PoAqXy
z9uBRzh<26>_qHz)b8SBhecXP|Gq1W{-n!%Gre@akX<+tl`f?k=BVXk7K~vND{jb*OItz_^qk3vqkJZXn=Mk2yy?GC|~y
zZuqr11Gh5A?;rjHPMC-EC3&}R#!f`<5J)<^OSo}qwULig6J(QG30knFW=vt`1^7i`
zcCfH;z@KLxVPz4Ap`WvY*S+Sx(<&7(8k7)>d7l;k>}Xcz(DpJH+8Dv1)y^Y_NVEi6
zlES;zO#89xFjoYjQPV+Y=bN#l6R$nyZj9R
zibCF0sSvgn!)kfMgHw>eYeUR0o67fm9IBNpFJs#pxB&JZi~^H}XOt_Pj8ZRm?E7P@
z;Rg>9Y5JdeD^V@>xY8BkCHtzobAqp6ruvhu1hFE)S{sv-o;H`EU-AQ-Hw&R>tmBol
z?3LA3@rz@|X(d7g~KowMgb$gj19r^mZTYsErJEcUPaGKupENm2*^ARyAk
z_%UuY7(!8^+FZUr(*L!)aQqYUUrs2V{^2Fi`nC%t#KXitYd&+{QsZX_`61`$6;X35
z73+oA<%EjWXqQyS(=GACuxb2-L?lR@Ep7K?JzI;9H=-xqRQ#em=9#3T`de)H99S(q
z$2UUYvSO+XBxn-i>mNpKAW{3R$Oc##xzkP2SQlIxS}d!^bF(zLj`=y8DFf12;i;?IDU9dF1Qr+k*Y>%B
zC%9tbi7O-fhyCgpp?Y(en)R$nzL~~M2bhNve1$tm)%;|NQ;Qu4f+o?>QVhq*PA03c
zIuTXV{gcfN+(V#V5d{yy16ZMHDBYRD{16HNyx45bs0+gQ@hB&lrQA)9TduaXe(H9<
ze|~;?$RoVG?@Qyh+kg!UP2P9f8`r}ZL#!71KTUljDKrT-3>j>%&`nCb#$R-=C}<9u7+k5604r^#LQY$4>_p?Lj{YSoL`X
z5~BbUWO>Jog>gq&P!%qN6y!45VQW9%OtwVOy)rR~^-b@d>!+p*
zuJ{{KoeF4OwunlKnaRn)!B8*m)%m%*SREh%YwB43TfN>LTmfDn(?+R2dhPYkHxkeR
z-+8JV0f6_GjHRxs8Vcc{i^KAt9~`L*R27a8B|^5aHWd2CJ>NMhJqr9!PLNN1%^Iv7xEl
z(zhV_(*ijX2d3&_c#{0+;4Lbfcym-K4AZ}1Zb!nd4ytq{eQ))E5Q6K^U;5i8HREvT
zo9`}Tg?=DWei)}DU3rP>T7MuYQ`9LB`1AWKJ)g2&Z(%%=epaqDBz)C&h}LKN4)v{q
z7b6i6{5(x?L{{~N#XxK4^JGBx9>^CxOOj=Hl@wvBI>_zw=FhpNVKI*Ph&%AMdva)V
zpIox3-OC$@27%SX);i
zM!~HBxyYU94KX291jKNB6L!XwcaySag|RanVIFtvU~t8CCs+wy4rFs$V5|Fac8FiG
zfYc(bvN>IF{b7aQZ$ssss9}$0|ISz6GMuxWj~1H)wK(V15BS%=uX5TvqZTQ4W1Mj;
zElFIDD4;Z)`}7`+vX3Uq>mzEn?8CdA_YvN`qJ(7*ruqL?!?TZ+;L(
z!*B2ZHhX_SPQ{DzKs1;WxWd7xW3lAaUr~NaFPsnVl7>#sQ~%%h50r|K;DS;d`H--n
znCTRMO{V#-y#5f|MlYIrVyUYlFFp^N>sC!U@j^WF4<7|3v?UH4tctqhc?#60gT754
zDJ}l|-}{H-I--nCEL`bdLC*g`(g6jmtbMb8K#xqTO7$kY%`1Qlv%I6^A@Oya_$I(O
z!S*B0jZRWh4r*{9f-o6>
z01mwcUi4EVaYHQV?zg5$ZI?6eyv1K1gl0yS61)eBb|DI~<1}<*pO=7leqwW05iNmJ
z3&l%uKjlH&2QFJAV>{~uZ6RJ&_o5Hjqb4;jj(YWZ7WL7I6Zcw$_)y_R*}Ss~TJ!Lh_Q1C1T*b^oJU+reDuT
z%3cxn!#h$@{bY2}Ykuu>8h&>x)?72^rC&tFr=j41Ce!`g1?5T;zWOWt%}i~(&SwDO8
zj^-1K?a3E%k;g)Cmz&}&77?n*5v(GxSTo1K2pASXEvJXoO?YR|+iuS7JiCh%#<#UW
zO2fnP{0<+iTMR6HVWpH9HH@<^b1=ayU`hvFEDETtuC{I`Fc7EwQG;cd<6ID!Mch}Y
znA6F(#*evUMYPk!uo~wS^e2ah?F|J$fnI$`Ku)XdaLqJ+-SMyGWOQmQZl~3YCkHrG
z)B`F2U&R2XN2dWB%?G)3{=1vKJyU!dnu0la)VDH}+qvO-&FrB&JVo1FEpcbp-H{cIG
zk?Y?M;MirSr?z8rbLs^^gjOAu5#TQ5m+MdS0HBp;fFA>)FeG5S7f>BN)9SQ6TImlB
z4IPXCu`{gzNc!N11Skjfyvn^SEDmt__?jW*-{}lPeu{^^{W-hysSg-YjOHPa)bT0V
zJ4oBiuUm~T2QEtmdbgm}fy@7?edIwn-5ufPDIdqTi&Bx0UXt@ZHA()88siS|BYPVm
z9*Sgd!-M;7=HedMADJGf;}h>p06_V0OX_?}NVmh6148y%0U`V&Fq>a$bSOY0WTQ0Q
z9*h@j_j*X^27*Tq0Z)hca-DW+g*WL&NKyd1qM$O4yluI6B*~YzZFwz1^jl)|6M!d6e<^
z0t;vlTh$k=XuO8UHLsi(Z<~V=0SQn~vcHQD;du68X?PP8$zO6YLP95WX+OTws(<=u
z5J-Wq4XI|sHJep&7$qZlDpsiLMchaLy+y{xSfX)9Uh4ccB4ux%aJTm4R-;Lwuj|tq
zifdvqf_!;el*vG=hcV_95;58g=RFOsG-d;UM=;>L_AcXXe7|QB2i3m2J}yQEEajWZ
ztBP8xNK_<`+O9bh8@g0OdH;~OZqI5{6O>u8;{drD+$Y>4zS4H~4#C;+2i^aX^8<22
zW)-n143pGc($-fqa2oRBPsg#Js_)D@U#mN}%n*g{#KHiIb)5}5vvM=nUdJm<$A@hf
zPI$9W(MHmVr(TX9=>kr>PYNzvePg}WN*E^1vn!->i9fx*8|0X4n3SAH98R7!Ieb<4
zT?iM~UogF-?yrBc*Jb@5Pmcl|PtigPW;3HgX?Dy{t#mediL7dldB!v4xj7z!@4|b0
z0e*HA;2uw5tgN6uo1Awi80cJH!t{vvbB^~$21;BxG4c^*%K|9QTuyv4F!h`JYJGTL
zbZ+`4zU*mC{s}0nwzPfSM>7~0DYU(Bzh*0F*MA1UZsV+@8n3aNQ8T3;K+8c{3@aBc
zwwY|`e@l6|^cM~QUP6pw^TZ*p5M?Y}tf0;1#3X3GwLkd;^;%qP>jGT}0?XN3s5+;{
zH=u1NXxfQ79Stn%xXKEww~V2E{q#1Jg2JD>$(ffO=#=N5QRewke07GNZH*t^A3D!M
zJI5V%Fu0#iL@R3@d{JP&uMKwAX5%ah1}qySuHqFbW2UciqmT0NXMVeAv(>vnIXz)}
z8&Q@rurQ`m!{2NfshPpB{o&dIgy9$a@`!=%A@7evWa8g5_zu>6ANK|YW?GC0?&BNE377;f)5{FqO==BV8Cx2`_It;<{c%Ded>H9KLHmRWT!
z%wV*6xp(){W?ui!jk618>%kqQPBj=0C12kq`436CSA1-+rWsVBFV@Qb*oBX|wv;Z0YA;Jjg6~
z&wsf?wCwP~)BOqz(1S>JUsu^>1C`)QG{q*0}hY;5%f)#;w^!M5;
z!2lg2q9{syAnxkCNokSejL7+h%G1^Ez4j6hCLWQqQI9_(Wsi9Gjr@94i7#fZl+yNf
zC_c3Gf?2z)sztE&WYr9f$g`irL|fvECrH!db8B10oAzwA8Cf)iM&53l{u#iXahc(q
zEAK}s1x8J4P`;c%4m>PCT(c!kVxI|+SkYsJeztN@{XY<8biZ0I!^7W%?D+lP%)k`O{T3in&xf5Qp)Wac_xJ$lNieHx
zLKL{-{PyIGi3Y(GxR@VhUh3Dz@o{RY9ehLU!9CRjQ(RskR*=3J5xtzJzw!3>wSoj5
zy@hfloZ8{@|JHB-!X-jvrUD>bFdT;}}_%0W*U5(%ZJ1e?@I4hIGany|$JrTO_EFCN}AZLmEgL
zbff2zC6iWn&c4|T`mbn(0stT=#0~!)+s1llGvw6vH%!Kob4zvWtT#c4nZ00J3udU@
z#Q6;{00*!<*2Igj@$X#Jisgat>#tr18lsE8;SQey-od>VQ$wIvoCoILcrW_^^2Z{@
zq~FZ2L0tC1hjKF#R&ZCMQCZ=zf9R{2VO?m&D8RzNU{J%AB&7S>mj`?UaA1q{DXPG3A*
z>>GES3vlgDJTVJxoZGLuSgGqvzLHk}#
z4XWHnsVeoKX`)$}D~+`sb*^ErTRBqnViK@$w_|GVvFF
z&OhLC!r;wUSJd#<5YKHErScX@IpOvxargafKh_DyI-+}~tcy%Sa1tZ>w76Rp0@3_u
zmc#k4f8R(MPfFo)C)39aKMHe4$OfawNI~mML#qxWDjq;7
zs}*&F{Y&!WnDqjB^wDLWnUmqRVxxdHNz`n#y=8xecnVV|4H-nl7$0#VA?*94LNqnH
zk;UO$RajbTs=kjA0lVpV@v8_1p!fG}Dzsba`cHpYzA93J6P%8h+&r>G1GS%@A5LP6
zega+@CpUnn$RkjRqz0nmdrtr{S^_I8D-}4{*oOr&zu33{<;qDimyNe6;I;v$TA?iv
zl@8RlSv#-e1X7m&dAb0i0f~v~sHmtnLE!%l_Vobf`k(S+Z?LeiNYRNn`R#yMuXEXC
zcAn(aR7w?WkA~p^{2WDfopmV!!9OrT1H@_V9xrWeV{TvbTGjlhwp_TGvnz+cHI}nVOtiUbI5}7suL+9e=Ztgt@y8+@NFC
zo&%a54&Vl}JJFMnYg+?`p!2QGv;Cxl2M7k0SdOs!wyMF9v
zr63D&=~!72Bxhq|OPQxauzh{Bn0Fxy{+m46&HBz#_%z*x01_6HuQd7ENNA)g
z@=U}LPd9HyGrUpSnM{!-C5z0`(TbK%vYz;nLF+k#d{z5fCLii4FIc>0`2ZhF%q?40
zk~^jN;I1eT0s=y8qTsILnQrL?xMyj62&WSVqtsP_OP)g3wSV`0F1kF6$E;{KnV=x@
zpxR{+)dgl(k|bltN@tMUJR28^1_>e}k+a9aJ?fr7CX)=|1M?-3BIesCXQb+FGFb5)
zS?#j80p5o_yL)?{w!q4s-?vRn#sB9Wopy-#kqg{9s=6Bp*d!g31uz!j{!c+B;
zr#qc^u1+s(>_qIE+AsR2QnVU5@Q@a_>^PLAL8Y`Y4lo^rP2?Pxcvc6Tbxq(#hu5^X#$XX
z?u4dy6IPcg7BI8Pi^lATCXJ+=@rK1)JE0_w&!-^P2jSkIs
zA{e{Sy~+4hup9W|<*T!>jYtpotW3vsG+-TQ)=2;V!wWfJgDV$_-B1S>2G6L$CvaJ=
z(-J5|Wm{*_>gRZjeb$YzqADIChynV0j_aSTUwi{{mG@L=3I=%r4zA}Bs4N&V+`dr6
z_t}BN;IoX=1A>qeyQngN^#NMQvW9{Z5weD6V><>n>c)mE_I~j7x_N3tn0EfTxj#G}
zp7vJpMYH5Lp2rGh3vg-obX-*eOh}aG0s9&)CD7ObUw^y39P{-^m<^e7ZtT@_?P#tl
zC${ns0Cb%CdGEeAJ3E2-5)rlrZdC`f!GE2jqIxNj!1Y>_Y+Z`!a)$?p-GykDMmxsO
zel9FkD&B?4bHBW7LV)%aN`Nl|A3+fJI_j2_+atio#bplw5nyX80acYd&7*Wv*GdUMm%OL0XUbIDU=-^EKt_m-%&ay4bu|e|?mTktbN0SA$AuLzkR}E`DIBm{7gr068gpHnw*wa9(UoV>U0?p%8z4AO>Y5NIi1_2i9
zaR(qEcIzh}tOMjss{sPKo4;ZWB^$UD*h4+o2cqT(%Gm0Z+sN<$6P*J+B%+XRwZIqk
z?Tey@jveObraIf;b56N!Q=&9PJGD@c#;M@c8Y0aB&RTI8$j
zU0Ad3AcFJ{P0Um22GR^Z^>YVk>yt{MD(lm#T_15ClJ5%>_4rfgiCdfT<98auJ
z$1YIg(k-IE=hM0@-Wb>>SXPxZ^W*%+H(r$KT@kIY`vpRXY|{Ll&0Sg`#qC@@@FJY6
zI5!ui?zW^elpJZBK8bP4#~ODKXK=FkF!SvPQo;@5ecNDuWO!R!FdzuHMCU0G*@64A
z&YnMuxhIlKJYA}buJpzq9)maqR#TPn!OZ|A!!0_|n?fRHIZ#@Wai}uk&;A>J4-(==
zdbHAAzJZXg&yZMfuBW34GP)F*X1Dk?2m2fzWemRuV)S~OihCtLx^ZqeRA&SC9X5pw
ze>8z&+5H$%1e+iO`ANVg>-A!mocaO-=ic2rTUZY4?5~-@)!^f*Ik?rsNijMEjDb|+m>QWzp127uDXix;
zP#6ZfpO`)dzSOQWXNo>ymsilJG|7T6skyg|h{Z1z&y6ln3ch@dgS%!V;V#dF3(F0}!E
zEb_f(0f_Uhl|+gNexS<=_u*^fXZamIZOC~1)Dbo$lo7#rr6%T+^PBR(%C+2$%@iH7
zN#>#VJFchdBrR8!X>JN%NPhWgrAnPpqE*KpT2bdM_@m8y)UyF^AtFoad4oSUt
z6H9kkV{@|Pt2%FOeMWk)J#E_E&@cgY+ea^dc0jn^+0EqDSpt#1$A7x(nk=|LS5H1
zgpjZvN1PdYd8dF?5Z<4dgY6@2MTLH^WRybKci1e|T#mW6&CRIr(23SgY-LHOGrg0c
z_Hc@=le!;M5iAn+PVZVFY|*Z)
z4inQHx^Fv}m;~~)r8+F7=uS!*iv~?KFJsSnEE%=DF=lKjqRNuOu<8c>FbFD_`Y14#
z7uhhWIy_96^8u!>3$a;^HN)AF=EQ3y!uAizNWqbCj>fr}W>Fcx6Hb`SFYsg$Duq(S
zt8LU74Kee9>*y#dMk@r&yx@fyR_=wWwHQWRqL~)X#P5CZA1T>5n|s($LDo`=NvkGE
zMvk7om={*emPh$2%WnbfcWc#5F%kDS0iP}tpV*Kps};=F`sd19oReSylj-|LT$^|o
z7;y11qsO6U3NsQAe0pj#3JpUK%~esl`@Ou2_2~>woIw(yt+L4hSRpN(H&!-t3*TwE
z$U6?z&+G$5l&*OuTKq&Kw!bNfIke%T-E@Q%;yV$S>7?N(w1!e7E8aUU0
zxwpl$+Q~5)E(=TRm_VX%)^pgEAY#$g!yku(IbqAW@36`$q<5#ZT(&x2TX?p}Iviqp
zsv$SIsu%I)%k{%-VjUpvNjm`(RmMKs0j@Zcrf+8AH;3LI8(ICfhKB9JSwpWr@U)1l
z11(Z&T(Q2M30*_*#yiT((hOcm<&^gX$~7H-qcTS7e=sXp|3hoDxeI!(r(od6mqyGA
z%>`ZMUMx|m(8rSKwu@dOFu!?EAmrTq!xibp6V;}?BG*Y3%X7Z&OWxXv6;>1y18fy}
z%ROqP$Q|-&d%su@s=;`*lIq@1KKMVe7y)1C$h$u4+8ZsWzPjTsS;mhRh6F;DPtfIQ
zx>b>`xDTb^;ekPPc_etWA_)5a9nW$pzpiyOe?EMDp+M`$TGi+o6({DON+FCV#90eF
zLN(;JGk61`SR-|RCR%OSM*&xokPh0Cj;QVcV~jP;IRZYnTb^1&EG5!5BoeKhU_0NZ
z04DUHXFjAiqK~g`GGpShJhw*=VOQIwA?qe5Didru3!XT`JQ^Y=9|^i2MjTNFS-HcW
zGg33-t;&Yofo}PiU-`DP$##Nw9*JNZ48seJ{nRh@@R>&yI#|7v!S-{eHLDrmO8TA-
z1-~B+@tuu(@B>-DzY72PX^diNSQgPb3XocY2&nK3-bn@moRhP?-@JRXO?X=_&WUKq
z5-X%iyi(&|7MC>mAj@OriLU8DY$J!i7d?4M-V
z6H8JFBrt`8<@^xDW`s~C<2iioxl1#>zqfiG`_I3ajdp&}mw!Ftsn~If%7G{;MkrCI
zNmeIyKij#H&-4l>Ud?#n5BqI_DR^Mrgich1Z_n6^ZP;n&MAuC>9E8h(GU%}V_Z_|Z
zI9Z`h6Lr
z7X88Nz#)n_l=nDoxh1vT&`#RncmYt^0>kw>fNY`ET?6>sW^`Iy_V@m?ZE)Y{{ftIP
z`_YF92AF;Rh-#n$gS)c9U4*mT@TJMvfHAS_jc*h%;-XId@t0xbPzTSZ9DO&kShwNT
zrM+6@@~+$Ek}oV-4B<*m`QzNd1jgL)_;%6!aZ5FfocSf;UlG!_XW}p}HmVy1oAhxR
zgO^c5{dpD;R=2eiFa!SAHipE`5Ix>ul+vn9dg8#no83Z2>>I+YHqQfYFgSf2uj=Q8
z^p{AD_EKN1(eCAU=c-o?p|0ol)r^ycN4_`;?`FLjmY%hzy}r9SUYrdx&S_Xp(SEvg
z`Jq{=(Y$kXL<;$@o$elRYH20289x8BE&(<+D{O`g5#Rf0fK>6gxw(n^(biV6{S`JY
zHZbd785sBeuiUT0|D_(rB68Wl=LAWD2h3;>nGlc)|pmE&sng!`tYUrJU$4Uyx{
zu3@j|1*DRFV3FhPP;6xIhMB~^x-cafreJy%|K5>RMMeVlyc>V11-P+Dv>4p^zpTO@
zyZm%LY?uMqCQc?M`D74EhX30Fm{l>?9Cl!oMpFKN?1hNlI}E2536ym_-&@Z`fT)t?
z@%R@!Rl2YN%UcC>CXM=<`8Ki@3k~fe?;1Ezo@BLi7Tke}c1o9*x3|U69Pt6H68_jJ
zyr?_9&5!ly^8e^N>9T;)p29`N?;4epE52;h^dxmx9_75}Y5Ef7uT0WPd->Laez08qDRRf1vtGz}fN9
zAr==4(G&Dwaus@A(ikkUDE{Mn9y;s(Z__|a#DA@RphLfP72(Hzl+wR{)11yx_Ay6G
zS1Ft}B#ZS}4@ECF0BB|B3C=;qptL|0_tii*;$z4P09=WJ%>1Uci3_c9io12pQ~{Je
z!Ag`Hota$)Z%e^D5O;OnR~)!z7yh>^3>>F$X;jr-+MklP(?@qrq7`!WzEjFJ^u~O4
zU0WcX>VGW_{I0_Y>VZ@dsJ)W#`*XblpPaFQ+AJ`a;W~(>01>e11IAncI3MvDk;5?$
z)$}u*K73>WGCXwBfA=jRf+EmEhS9+;6G?}YrNurg8CcYGdmS?IZk9NQQTdFsf%e+?
zRW37J+Cj5HJRXA(E|wiQPvsQ1$8(uncV8F@i-M{v_uW_*V>0->*+lqbzVlbZpbP%cYEOK)HZco2SD
zPL;8lM@$uIaMF^Bc;G=lMI@+#oAqLv;DWF_P86e{EkSLy_>s>&$M#zy8?DC*Vzzs*
zA*~?og>;-(d4#ODvJFmEP^+GSG6hbmSIxz3zPPO3S`W4f_2pQW_$I$YzJ?RTh-r3R
zx&%HZ04Ubh%G`lkm@{Ljq!X~iE@021uIJ;uhZUb(i;Qr_>gWmPonUXOrd!-6Yb7_{
z_@l$xPuMLZ3!v#68e+D|6Wpic>oE17MUMBgXF
z^$=hgJ!*c*wAjV}Srlw!D(82~`JDx>>o}Q%Yp&4B$)7VbG+lfj*Eo^L(2YSxmw!KQ
z%dzdVR8{py&@aZ7J7i!L-WPB(5kbT@~?V0WWF@E
zOQ&|6*p3=oGoQ2A6Xl|?eFWpjkhk_sM}>#qtadN*=H%%Y9h_~;T9ClTE`=}*4Ff%%<+>D!Ct`QWnY2+!moHtZ&&Bdv-Uv@S31Zz;?
z?Y&<)Dr5*=u~Y~pf~9Wi3M6Pph`M8d7U%gC1%!Z_=63E%dD46}j-ew3vl1@qdTDFE
z=u?};^`$9-q=HX;yl-X2ClUxa=O^x17>9bu;oA^#>8~z#H+Gtoc}^Y8;Z;=`ljWSk
zh1eKC;~%~~RV6!^BJ4aae9i}@&sUw#J9dN>5TmvApfkg8&e4){uR
zp+puuQZ+z=&~Aj~!2UWq+DFoa9DDPm*WU#+zv2czx)0x9oUPgMjXZoKKtJt>RD?M>
z+CW~nZ|_`xu@f8Bs~op564t(46d%a$5eM2s3AJ$cERU7`c3pi_W5#nKAAb0(AUF)$
zjXhTzblkrF$ecuTw1J8Q^tMQ+erTX4q5AZ_lG=D*$-8kp3x9h!pnF^4t-7X^HNov3
z23GfreSRQa^u%TB1<--oY(0&|9y}Gi1bMz22zd>Drq>rg1uqUV1f-g1?Ilrb?+lrE
zfsvV!AuvZf8t(58^Zn`tn7CWDE=)L*?w{TE5cv%U7b#>f(7(HOP9Kgt`f6SGJD4qq
z7mGH7NeOY$vr2&A5{+Nik9?*g;We*%AwE^WwwZ*U%pO
zs23s6x(?FegCuR@2te3dQU}C-DY;$af
z4f5Xh9s3X08icCSl{MZbL`nZ>M5Mya7vk%r<0f-<6EK+QX!&T~y-)Xbu>0(aIH&Un
zI0NqX%XKi}PAjgu61{KbRtVk?=LW!;X>fMZGNPW;P3r969Fi&J)b;D(jhM;rr93tG
z3mOf~aOh2n6j%#A*9Ij!i<|~RiQ^iTwIrrd{KCf_gAAa1!o!s2<2Pt6EG5
zubOK$Xi*!tiFLK@`7H$uEXS*f0H{k?z!&QE)2>#@f
zbMqDsuFK}vBI03O1J%4*hR!>}^vS{Tm`~Tqi()Bxr*E2a>ODLzjn`JRd=~E|Ci?r<
z4pz0+&LfAaXujqMTRI#=vcF@fxe;;k
z8MVwBJ2NTvrQUV6{E
zVWNXD%|gR;1V*u>^Xo`#!{^4mYGXy=yn~g4ejAfQ
zqbv61aP~`MheM|+q&j9n^=8Q*bqRLc3CyALB6nqP>WDCnhiTN>jn;Y{;zB!Kv#AS>
z6{m}pXn@3{V}K%Z;1o`Q*T{;C9iJ~>xA^cpRB!zhOtXDjKX}fN%q72glzYEGX&G6c
zYd>UPZMyIshF5x8&JhMij#7NSjN($j63(AK@DQ|y6ri>>royW3OOB7dz
z1d^1myHeQ`B#r;t!1Woa8Z#%~wy09^``{(IF(Kz=y%PlkLxzI+Y7`1WK6D!Tg1tG{eubR$9fFVh!j)7>67y47H_gq|EaM?
z$w5Uu$4}v-fnmJlMTAjwVjPVO$+OM`$XE)>0zrx;k$0~mohIm$wgOD};5E=cIp8jQ
z*Hl^Fmv=4s2K4;!=?HC5Wg%18h=X~5Z~oq}mqlS|&~L#)w1j!2-m+w0j-){Curlt9
z4h2SP3U@oXOykxvt}70|wlBu=ypByM&;wcW@&otx#&Z17T@MD;eC4;uVf>2N-juE>
ziM>X1{GIu7ey2shmp~TMLsl9Nf$v5bj4vkcj!;YlR;kKJdRKxy!E>LNNcru9grQi*
zL1)n#;4j>l1IX0V4}2;`%pL(nnmGDr
zgObj^ZmqElFH{?w@~$c+u;+aAXQcI61QswWEzBPz{eWQtgU&b(jLxpE?0OFC1A|+0
zB|(p?b2RO@ypgG?9(#Y>IPk6q;hKe(ItK^I5o2w-SF{Z}?ooAlvqOgoa=I&fOy@5e
zZJOZuodmi<-zs+sr{JQKZ(>3KvUC&mMPnpwXGq_?^(H
zu_i+-0p_q>l)t7+n3_RMCPUqwo;H+>v&p6=n$_RAG34^+A^fpLZId>wZE#}A_HeP}
zb&sOw_Xuj4VZ!ul_oSfFMCJMOxXQp#Wd4%rgHC7H7I-vzeU^2%XuYjPDDn~!E_=u-
zQNPig2RY6^+vvf%bsfex;H;=p{kwK@(0THUhIV)ng5wdoglh3N)M;_5!;yhri?3)U
zN3=~|E)=R5qEc5Da6egM*s2F?aH?CWKYu>VLTv_s?J%3q`?~xE%KdqHycnwm7(;wu
zsE!J{J+}g;traP3h9V|_MrvxX8u;gH%3T%!m&07Y3YZ*^Ls9X)_yKS#&Ga)6+UW0n
zb)YE@7%{Y+$oX^rwnoHbGkE+Q6(xUJZ#`QY*8&xLLEQ=|
zfO&!~J3D**l2BOL@@3-NJDR3zi{I&2@|KpCSHPOzQ13b(Lc_BDSzLN$prE5aZwSSi
zLtMVY?pBTbsICoa&8|G#-4XK?IL)VV>MCY%onNs1sbLiRatlJ!oz*wDN9}n0gIJzw
zs_#8xW;Ku;F%L1@bI>1wqsu6>GsCn
zmmus?6@bnnE0f9Z9M(1u9FeU#?a##O-AR5R<+QdiHKov|^S(j1uss9eAx2u>E}^+ODEq>
zlm0oOZJnNNUn3g{rq$^ZLUBOR(dgz^qx5pRc{=ww>0Tm?6PqYK$lwz$UYXr;i^cuT
zX-D$W)My9LJk-*d0oYhSg(yZ2X3CJ_fv)C~0Ms4?0A6hWp~Rpz*xX;u8~FgYi(o)r
zJ4ZJhJ*(JOC`g;$i{)>k3{aMp;b1<}+M{1Hk
zv)oB4!*@&Ry9{)FF;hMOhCWzNa96RtiKzD
zT*_KwBtPjCOBjhQV8gvD|B;P)0JDB{0WXD0kA}*+0U_%4#6j*xnDVngzo&)+_K>YW
zDb&PACyoA4Q2kDtyPp)ekVarDsxlp>48{N*zO_^E#;UOOLO=7
z08i5))5o=UEk5aL2kL}T-9=LgbPk)i6(`d=fPlCe1g%iuAr@0yAFUj~Y2u{09)O<=
zsuqqg8cRWdjEHM_?&t#q=cC#u0>ehl(glhv
zqWb%pf!*1iPvO%cLWe60Jk_i)uWJG`F$6IakQFyU`*9d5_VD{VA5T%jSDwk)i=rCe
z%QH{<(JHeM{KT3i2!hA~7R&y7>SIMO#b@P&}8j3^NMNb>KMc
zB~yq`j;d+GS0LYN=5y;(62xolxR3O?{u@Cr&Bj!9`&zsbe6!|d-*hwc(jCbOmj5|VEifCDp|nT-+vS($)D)Z{HK@Bs$u`xyJV
zPQd2^-{NxcLhg^AXMg<9KQJk>Vcb;z*8tG_kehje0vR$f-p#bl9A>m&5PfE>f(2pU
z3QlH4n+Je!C;Xyvm|z-fGLvUQKo)^9`pYc^2Opa*!^#i
zRv}Sx{A;tA+3appOt^K-&H-hdfJ|K+Wc;dWKlMKWW|;X`A^WLxm2I!k?q}6X8`p0k
zdsD^Vhw#Sx6=pJLK#>vtqyB*sN5EyJTO_k_dO_g4~~Ghba!g?SA<9mdExy3QKfudRTeIFC3U(G0x}n
zUM&-3*M3gy7Jb*_B`vmu`7jcRoDS=CFPf6~4$kmwWII2xG)?|gif-<3++}QZD(z_5lyB;$|Rdw2}j
zoSPm@2JMbk-db??o=T#V0+-8SIH{1BEi=2i1Obf=GTcH_yASgj%Q(siH?+UWhSny5
zv9;Ub_ByO<8ePx$yEnN>hQ^+#dxk%?OdH#?`4iQ+orzW}DSvJkdK>Wc8n!DRIjaj!
z-RnK*aZ)bV(&Sp<_kVxy31*N&opNsrb^LuH;byTdlW=bofwu;KpMfn8qXafU?ye@W
zpIhBo02DC(32Ns~uAjOXy05?AxCg-$0ZS+jEa4J~o%DGAhdlK9Sj+HIl%$3~IH?nP
zuPzd^VP6)BFkh@sOK!Lo(qJjYCsO0erDgXHxbhZYSn2Qa)X4Yo^`seUJ$prrl
zq5yJ|K)p!m$+kNL1&x5sKqz^d&J_?O{+t^eWN_<^01X3xPeF4ipK@CNt$@G+4YZ=B
zRnC5V7!!Q~(r~&%aS_o8e}v6d>D2D@L5-<)zBS56FGFIBa<E-Kfh2@y?2%f>3*H>q9?gI9#
z3k&(S6VuO^i6m5L7G8_&c1um7(6nc7Xo=GXgekQ8JuPG~k%yoI4gzy%r3eMD(e1{Y
z*0lp^wc7wn)5iO3O}qqJjZV7!21CN&=jZ2X1o-%RMS%?i{lzJ50pKpPUu>vAs~)pu
z>akY))zd?j4**Hr2SgYQp((8OhUh`5FgganwU>5;gWKahFq0aMq9C(_=0}YJ^vQ*P
zb2cSUXoV$9^p`g7_zMXyqke~98>i=jA%O88mk32|qs|ijiySAgrKiTm!}Dir0MtJhirSljr9WoT3v
zASY$S9+Os6i+5rtgy(pxB54M0uwPwCGycg@Igz-k;dqV)eC;NapY7a|IhvR1d
z3g9ufZ1TL=$u=8FYAss)ulf~mCO6;^uy{(;R9CA;s!^A(LWjlE1ue!sVIo`7i_m%T
zQnkLC|NZ6uTQ4BT7>lTbCTP7ciXe25kNuxE_FcdCAjYhW;g_!QkI2Ddg7?MzrASV;E&Bx*tQ(scW@~Ydt#~>;VRPy5
zAFip5UM(%hfYxiL^A^E-ent1~!_B8~jNYeC&J7P%>B;7{tU{l|W^)p9by?iM=_#*r
zU6iUM;bfW|aa~l#yg%z`@Lan}p`LTy7L2yuF;0dMyfG9BC=2Hi8#ca38iEFT@W$AO
zzV9K3%=UJBFgZIF;`x>_;AGSX?W0Mgl*#B?Wcoa@D!j@Jg2TEX!Kie
zwUdK2c|5qV*_kVV9MBh}t$J*LGI+8haIH+QzCPG^d3x&}y7;eDO+xltA&|*uln5$ozraGck6qu!d(K3TSQAU8lB+eNEjj>6|C)X&ag|8@o4-3KcvX|w
zlvFDcJXn43nk2eNP5+FPW318fi*!A&?|g=W5}$-EvKQNvBu$dn<&iK+o6?PQS$Fqp
zyiWcD4c&7lgCuzRDI&)QdNeOepib@b&&hTUj>?&z6@f{}Kg+1S*1mKIK=gCD$;H?C7yp0q
zT{91%eAoCp-2V?$yi(0r{M(#`2DOLDI?3}v+1I?ngXMW;(*x(moV(0ILQ5CGZdFqh
zU?)UXdQfplSo(6te#yK5ZfvB$)$OTMSXyiBYr$WaK9bL2UeAwr3ofptp+HJ$M~L39
zrW|def9TiAE`H8#W3Qh=XZ!CsnfcF&{H&5Pc4^q*!xY{sPenN&Ms>AivtRM@+6vXL
zo7tOU!-(f?P;B|3_tF+hq#Vpk4*w3MeX{tms8;`>x~gbaLL0{M)P@~!xn!^aqUR`+
zEGm0&eFGzJ(r>}nqlyFn2bm@?wcwZ{E)N};7%DQnN1*ydg4!bYr81o=L2bX3r)A@d
zZcd0a9n63X-xYNKZA7RPJ5!;1gI+?V_g6YUue627)BK2T8vUP3)K0?Oj+Z2LL4a4t
zE9Dj48_AxCesaX0l_gV7BqZm;tA1oyyaxT^hW^_iL}%qXDZx$7=fFZ+WE0MivT2z!
z&T}<<<&Z7KygTh;zBO{7>Rh-)o}a|Ghk~#uJ~DmVNZvyQ-e>}LJ$?-5?s$H7{@nzv
zO_F)-?EnGc6c=6M&5rWx8QL|CFfdHXx2*c1bs(+|FO7y5`YMq4VsVnIPu6GHnqvYe9JmWMouE7O6Qi3n%tW^h6S7tn`0ui)
z1yt!{xFEoorEb5du4WP5ngm-dR|$y)Jw%Qg!g7j>H9k6>c;3&oER}Jv3|vBYbKI5*
z#_GD&c~vacJmzN2{1I@@ACdwGD9DOc(2-*qyg_x?cxoJ9!A{xISfTm2bd=#<*jy;E
zPAmtCDx}IC73EON`2FqpQfZ|j_snr};ZB^2~tMPN8uzTE-wqmcp$mo<#(@ZG<7+16vxt_v;G
zrQR07`Fk=0e^&Inx~9wJl9J|iqnLo^!I!m2v^>6Xs{zQbW`dHSwWKYdP$co-#z5Us
z5YGL=kQ`8Myy~yuYkN6gT`vF4-D~|3qJ;(QFoDLEdrTm6ZafXT?f>9m?w@Z~{c}dn
zepb3c?VRSoS(@N>G&douu1~y>dPGM}J(s~r!n&^!Y5;e#gk0IA5`h%68>C*jIv%=%
zlDgD_DsI%*kf2rBwV3Nv458E2K>~moqg%PoKhIkPJM08)Uuu0j>i@1MUmY*H3qw}@
zp6)A{^qS7la70|8_4Qx_P~6bjt+wm?L5;(*>X3S*w6?FSk9$PEfnwVftHQzOOy{JJT`l{090oA&mW%shnv%R
z*#V=lF=%8JHVMh0Lvi?n=y@A&K+3B+c?%X%e7c*%RRAZ0#HwJwUWA0M9hjAV%C0;V+;I
z9S&HCC4pM}B#zSwax_^m+<=5do;Z{xctclhtx@*=cxb-X;t8O<+`WOwL%~BhipRkJ
zNXcqzS{au6xwzD!RV#1wB*@qTU-Y_g{lh20WM&=Ck5IVM;FN~6uX30)FrWHTH`igH
z#OH_wr?lVG`;mWb{}(mi75i(%na%K@Jk#R{81EDsme{+f>&||~zPX6A1&a3y_7Ddj
zpH{fzqtFIROVbTiiY!rnX}=5L?O#rIc4%K7sDZ43YocUqEO?aQk@_1zh6lTq%05!v
z+)Y-BJ&F}awjPNF8bf#|6tTGRt(6mU1+S2Y6tJtfrVu^7XFJa2sD`B5T*KdvMfmeV
z{w$M6N-yiEoyoUT0XVnc7A}qiH#R<3N!`^Woj&O;G5x9tGj;bBscwSHz+3-c*io9$
zaTTyDvV>aM<41Em;QM3WbnNZ?&cGj3p98GlR&gYRAXunpyEjShiPx-WIXLV+NxL7@9x
zn6{I_1;B2AMj`s51eLMRg`UlKDbgb4H+|=GoK-dse|3a>A{4?pBbBTU4-c}6HA*Hi
zYRut{t$-#oZ+uE=%Mxc^z#67GcC|vQ%aYSZJw-Ei2+!vLibcSvpGn15qud&9CNC5w
zLDGFuLeRY=W*4Dr^5w>5b;?iD#0a}dsW31E7^u)Lxqm3Zwwa<}1Y
zR$})SYk7sYQE`2QkNOK`w}PyKQgzN;{Q>exwSMh~B;?o#){Xu3^UNjE&rC_AhEPOO
z4b2LyQ!o`E?}yr>e>D+p6a!iOeS~Xk
zx2d_^)tnfk(FM@NsSd<^Oa0baJl`zk9M8Jj8mwz{^0k
z8O#-6kC>+$thGe7-wEscW2i8coE4<`?Eor;0sAXiq)zlGuShRs1N?`1m-S<-*D$9`
ztkf(#3OfE@D^^>rz}n7RBIV@E0XOp@$6z86l-}pD3<=})fDbl;T&mw9gsCu();cS+
z2X@R5p8LoW4|0v(3YXFFVx&k{uoYL=vzE%mD^|!JDFFDT@c~RY*_~O(rnq^!;AqCQ>p>i9wY!X`C7>LP;o8U6zf-2_H>id{R_nr1^187=I6EZY#mZ`Vkb5+N7XAB@@F;9&#q^z9vl+uu9A4S
zDJ-vISA2r~zQ~I!@xE&gpoq5C4g5XY?6PX>`P9mvsST2d0l_$~cX=1wccpY|!-)U*
z2p^t>;lGsDunV*pvF{^(E1?59rGbFR>EZwv285dfKZ@DFxCoI1fi)RJElw#lV|;Np
z;wSST9D4)Hg7JOszrs!ha~D;yPj~QAZg#i1|4xqmG_GB)u-Pk%AunQR=!xLounPO~
z_sZDf#XnC)@eQBI6SBtxe#qH4@0#W^iB)tdg7^J#NKidm_pmctSI=y)#o=|o?wQdI
z+M|c9fU#jfPE2}rB2wkTm(Ew#
zdN^`#-P8#Ka~aKB+Fa8Sje6%VltWmC_alw^aTu~;e!@4O?UT^pqAnbuqlqfgfh
zKvZnl&V_DQ-XIbM4H$uxaQ!54I1QgdPq&5b4T6JnjcJmDhKht!^UB}t+$xO6kjcT1vfSU;6xj>+q?mX*!#*R%)>YsgN^(Aego@!b+H
zEdvXTMB1p4+^mRNFYTyr?EC{Z)nkT*ai{&X2b`}9EQ|0qOgHmp3@heukirJT*xE&2
zhBE!5Z7pt-c+W?l2fhcFEAg}QXs@`X`Bvwl9;f&e$`!n~Ej;qBcF}m+F|!N7G_03`
z%8%eJaq04nKheF{wrRsFX#iAC`^M(!uh^*rfxPV$5yDd83ryc|RgBuMkU~^t7Q{?W
zcP@oYh241Nyo&HHthtm_4^Y%#=M~{VZ!qPjWb_lx-K;LW8;{7^NCNaY(XaP;ahd
zDLb^G)Gc`cBC+kEG{Gc6-;&86!8PGQv2!Y6ti$hWap)-&9kn{9%(l8n%fAPUQbn?WTH1
z&G*6oe?~z~WYSfMJ~1!O@2>zJXgFib63Vf-uYQ
zDwFtv?s(dl$s1tkhe5dX-?OCkjJ)8wFXd6Bqii^v@!>lO>SbtEgcw)qOFhJVyx$yjIzhqL#LX=D!d_oFTdTY
z;OD35&zj`w)4ZGD{O_O`8E6=9TDzc@^FLtF)(_ML?s1Y*QYZoDwC{KU(gkMVsCp0<
z1B2WkD*(0y0d4;{mERG@mIqpvQ6-B;^niqonp)O)V#1!i&=E6rS_nxyFPK+<`XcFb
z`10&#u91>;@m9rsa>z!POLQ%X+JoL39Y$x#waWOBezIGJTOa&_hi>-5R3<_ANZg}{_$OVo~
z^yeH|3?yz`wjxz^4B3YI$ArxR(HuW&|D<*=st9?ce5b|)Ld3!#H#qaXdg<)5M`3AK
zU6L6zDL4hbzq>o-;;bQ;LNtGKtr!ZlGq~WRaJJgUg-z+`(0LFu0LFcvN=fJbBqYee
zuGhcO6rAymdzvLL6}w5ejbH{AQixZyi(#oIXLq=Hqiw0ajFjCGI}Lr~#DTIs!hGEC
zKMwRrql-_GVQ7xHF!e_SiE};)TFZ;+_bY|Qsj$(9Y>4k^8pZx_x~MtzgvZx2T3H}PW!{|p!S~DFsPr~#iZjj5%1}n8zoWHGm#*^C9exuwWdNg
zVlUoYo!>IiqhIMshy+krz^_xh{#0X6#_N{d`{rE-NdX2?mRjaW8at2m;A6O&8+#ZC
zYfEru%eONXRM-<@!MQa2slabQ
zCy7O6_e<&wD;;Uz>+?+sn0#9+2YFeiq!k{^4>g}Rx_m%mHG^4ZB_g4XB1D3=K;6x4
zFsH+WhFTe(E1400X1Utqq7bbitO5Qgzt=nfbZ^A4L6^Tr7{YkcMBfJUjTd5sVFQGW
zahl;mRwBWR_XB8e?0k)Gn668Z4g6Nlp0B-|Ub|q()w2Lq=5kveU$DV1TG
zZQ4rMw8c&fVUR_8l505XsMT;>?j_XmO9!I(hIKEVl59l7l`$iH$WXA{gaZb&M7Ms?
z$IO$ijsDS9$$KXy6DbGFq-)J5yh{y4;J`kUv?&y3khmwGnWRg
zVvOVODVpRuf@`T`+!a2Z6FA#f$0ILgTHP!1AZCXtshfJ1hBf3vh>1rB->QVR19#C2`jzFlKJh>erl#h7fr}P!k+NqHhzBat=P79k}zfxXHv_eL$xeW>_9CtKZ{Evck*6FE6O
zu>ik!^9w%=y(dFC$ToxJ*t0QY0c<(?Lo?IM@3(f@JzkIztX)qlbPkR%k)
zgtd)D_vFZk0Xy%+pLLQdLXs2GBnt+A0YfEda>iU+LbX@Q{x!zzyj39Xa{iG_nm
z7;hEm2$h+?3Fse%>mcRGyY&*6QlZp8ZVTPdH~GC&ZG;-q;zTiq&WZ{7Y))jPQg=*k
zslS`=WKOXA`2C*;x4bsP;0e8Odw>K%(TC?=83TFjL`JS93~c&CTZA?=c63q@mfT-4
zZ8I5v7RjlWZ4QP!O7c-}>JmCJ|6~_3a+AG!UfP6Z*pQh-^Ij}Wf)GE-HIsVkl0c99UDbBss!K%O*>
z&+5xr*NDQ;Ym2M%K(W!6GUaRIt#E0FzjFk5i)DlrpKWFs_7E=J^es+Qb%QUPx@x|g
zZ|+{gmuU6&nJ}mW)vhZqSWoGS{&v_xMhNwg7
zZU)@=nkU@q5pF2Oyy4G*rUqf}SdJz?-e*efgH(Gv26+A)>JM{W6_U_-Z^T8QTCPrJ*zR=(7?Wzp6Tco5=j^^o
z5j{^AD@LZ(dWYSc00!RE=_4wiBc;;yyqR&*lU)78o3VWP_j39J^Y8(;bLi+2xK|)<
z6m~biWR{UT>mC)pfL}p;M80X51L0>33d9!O7p3twm$q3wcfJ>{cm~qJuA${k(9H{+
zo;Mf)Ro@#%{|GyAd2b0!(1K}Dab`X_v2U2fQ8l&_scHBr#C4~VEVs$y4D~{RkuQ;p
z-`YuH>pU<9lVWyxs>Vxg#2roUlImBwMBb{qi4o2O^6p?fh5SDy_~Z39&ql_^W5D56
z>WyKcdn{;ZxIUn#EJml>aI9bJ;^Gqh``d-dSd=A40vCEDCRfM99gPElXn
z!QE6gj-P>3>~?lQ&$>@k(N%k6EkTPue-r8%p#_}o`Soa*>qGCd!_|W|-c)Opo{|H=
zW*`{gNg@&8V~MlK5gl*4{EI|*r;a|iJZaMr51HIrVI+zI)-P+91p=J|qijH^7zmVS
z#%l1u!uK`(Gq$WaA8^s-Gq{Ds4NT0XBCy1BrXdv10w}7A}(%<<)&jENDL%F
zPEA>n)@~<`$rC0U*BwA1g6uQ)+Z?H28TNuZE`{ukW-fLN&v(I#t#<;!uc4H457IPG
z&EB~n7^1coGNI5h*66`E6NX*{bogD_+#N^cQkcEhIV;6$*$?wI_?JH#nW*xl{uG&#
ztgN$vjQBW3^pC@1q5&MSqv)Sdve`;7LA1mG>uw`F?DBH_6^ot9tg7_A=?yaDmqQ%V
z526``8G7h!Z7$q*EQJUy)$;Eh)NejzS&o3{en+PqGgwQ(aB^)O!_I&Ao8_xz$=Db)
zoEr@O#E4bZ3HlgYx#~U~u8oz*WW!`RRC49mz0koLdkDReliwexQoaO8Sq!xKn
zhlkm~)QK%e+Zx-SwSn&CZXGvMW7%q29_{Wuts>MIAo3Wtsk2-3?C!oj{s7b>(=cr%
z0~@r3)!n&A^GwJ0u~z#*vSE&szqVDvQj5iCA$!4}Xd{usr|Nry;M+p@#o{jT1!fg7
z#-j?BF&)kCv0bf?%#h}8=9FsnWt*CBs68u|yXQS-mwF!^)PDo4iK%T1j
z4({(NE>sO*wo<3j|F#cK(OX=O_ZPiC#aMySP}hcG(rECTo>r9j>cx&#p&Ga6=rDdV
zDqv9I&i2h?pKhBj1x|QPISlUWw5FRkG+CI7SD9pBCARieX;T^~1z-$g?-fZ4NU0;R
zS`Cz!>JgQ`pYkg%8tx6OFT+~DdY?XwkUr&CP8+NEN6;*&oP89@o81*72|8F!04=yI
z4~bc$qF8kJ;JkjM)@m>+>*=y?PD%H-RBxa@TBm&yNl<7=&oYDk<`9Gw>(HZs84|ro
zM{g|)fi%IO1}%ZsqLx^1Abh+5b{Z!4Wtnd@tl}tJ@z+q--LG~WMG8T9eTD=vW*a8TH9SjRC
zOA>Mk-R5#3La;%oe%501W<6vXOX4av7&dml(u4+<#@}<#tiJ1@e1>FEX!kn~uI)+F
zw$NH$UbeFc?mCC-+3Y%n6I2X3ziTN!%e~AV-_YqyzUJjjUx#lOe1VkiCyY1D2}|GR
z-I;8G7yrW<HaQ`6rvi>(i!a)yFy1uVDrxVl!g=OnE#36B8Do;sKYwr!@I>wbtefm>aKOW6qC;+{;?`
z8831KPu;{mtKNq&40*G6P{IwfbG7fi$E*XpF%aCalbDLS(DhQw8P8HN?CUaN+rSvu
zV4>{>UU0b5LkAVP<2;xxVqJ4vav`V+=xW1chn)|SY!x}$@F1zWRNvt-Co}^rY*Ia+
zc<7vbl{|I6N94A4GC&v0_S(Q1az+2d*r!8>HSA}LkIlPJdY=B3Jf#v;mb_f4^~-LK
z@VXyP6i>CZvlUi`{U93lC!u8@g-Slqdqu$Vf(H>j)Ma;)DRu+7@dcI{58s9V3H%2%(&l(!Gu1o5kbRH^gS>Bk7N$|x_38!2?RxM2giFBAq#^m+|GvgEJgXw=XNa&d$3hK-XQ7oCU+J$H%e^k2%m$G
zkeI?^YRG=VS`-9T^(F@u+J+_`{t-1yXO{hViX0(
z+tE6naDXfTiSOfH0D!Y6m$S>ke*Zk8`ZqD}i?4`=q=Pl+3vwk&ynVZb>lq>FdJ|4-
zIl^5+jpaTQeA`S^)_#OJZ|J6`{v9xEtmJsvr4+5^L+43x^AGwplrY9v}b@+F(ECy=MD8Z0HwIT00$%3>+`O3!nEp7*Dvj}=IhUy>
z$$z=xqR)26aPnJsTQkZzWM=$Q4fg5%5Ak@#-2Bz5WeljiDyjx{mIZILEx)jJHn0*5#3eSxIR*rfyw4E6S6c~-G;Wf
zho(+jJxu#QkbI8_iQi4RK+mcFa@TNrMxHt%?yDf&iKa`yYM`rVJNO=^q2R54f~dya
zSylB2By0OF2gx~?F
z-Y;w*CwKL?!a3J5(ss3$UvoAMuV<5J{6k-77W(q?qn-bEBuFwa7X@W&t%OO`UwirK
z1jX)^E*JjN2Vo9XEz34Z$v*VuP8vh*)RjC8EaQfPI_k{yd7SU?(9m6x_n`+pf@Jb%
z1}Ig6FM|RDLu)ubCz{c#hQH6aGt<^l%VJkc!CvINgR1yqmPZTG^ON2$Y+(!Bezr*S
zm|}MQ+u_V+cBG$9wZi4&83jqiX!92Z$gumlz6QKm)I0z{kIhE!$|v?+;{^aTD#saN
zYOJ@j@oPOz=hW5vDR$yo3CM2>H)iKG->Gza@Ma_{5#0eP->2B3@c0=5bU|0#$#19v
zaL*(CGP3fr!6~>s3zvraUR`rEkO2+{)krq0Dz367~8M1I43OI-r5u|!^w#89wb0OpeEwN8RXC1sB4wbT?*82zM
zd#q@h_|Hdx0r~MqT-|P}mxOQR^Q_+Jsz{OFAPs~_qMKO>8{B3X}v_?lG{x_L$Y<;91qpv?j`w>JUnoU*(Gj}
zJ}wO+TcOGccGBixkxq4@?eDFu{p96T$jWlTt_N5{b26P3oNJm}c5JK8IOP>H*;!+G
zsuU_1=$VBfX-Kf#kII>at8J|3V{XK@HEsiO^e?FtJ-jeu)(_0lEQamdW_RR*!j6A?
zVURIrXBA~+YAlqpy)(kmFS++?w^bAO#MgIzT(#BI%dNpT#59Elm<$3*DnfP`FTN@&
z_1UvoS%Vp4;i33!+X06$)zJGueOX!AhCo;poqB+|)1FA9_#y-_-unP1{P#6IS^S>d
zu0Xig)f(d-OdthVSp>xBuM33h|FpQBMW3FY8UkuzB39dj@xHJ?haTCrt&L4W3P2AD
zSz2#$G(7_t_?%|^JO5u^TnF;)|CbLJWG~<#rd+?B$e!Ph$^1GSh+cp9$K)Ct4ahWe
zfX2vJUoS7O?uyFFMd>j4KAHqjziToNyJYUa4bzI7OnV!i-_L7
zC#L#c5zx@mMxas*@*SI!OP>2=0|A9YsK!tHxP^4A%a8N>rIyZA$KrEl+c%Mqf<6G@
z-kI}YgGH}Ni2>kEasW(sD}ak(E2|Ail8_#347CFeoKJ6kpdfUzUu^&7^Jgr80W-|Y
zhimJzR`*LEQpjLyL#-U^oR#B~;Q^Q;2ZZ^r=?>5
z=8b+&PC-#uYHNuj?e#4{m$`jg^+%(aqHcEsUA?B&zPR?cF!=zrJ$a=p1_}2Cit^mA
zycEZC_~)fcxB)3qD({d#gfliF!HwG87&1a6Op}nERJqmYn
zLJ^-Qa(nx;c>rvWye`tuo4*L3MU;3I}^*`+%OsFJq(LiS8*ozfdUQo
zB-ee*-LDn^4gVds^Ki|mQ&kAD|Bz#-3~K&EeQ}x3GYG5TPU@154Nuf@Cva|*TDRsn
z6Qf0*k?&OVBt9dw`>x2amM~EfB!_-+GEJ}`+$d5&NhAI>|JZ)_;m?>-yN|Stf?Uv>
zemefE=o+8yXXn7v>BjDGd8b_CYAf)TH8p1_1QJ?TV{A2l6#TvRAvcARIk>Ixzu
z8d~`HE`_jph$6UzN%OW99NV<8Gi;7Y$7UuUZpaelC3YW4QORIt}4eb8p-D1a({1PtevF{tDO
zJ?_EgWT&bkjzs_^3eMvjvGA}#5O6-(6#@mV!6}pF
z_qh-E;FvUvvWw75Np2TAl+3#bW-hUl*c_XUa8ppMb?u#7#LX8Y->IEz@()l+6T31T
zJIh-5oawRldd0r$(16-JFS7y>Tt{Q0v&H9Zjy8zouHIqO+phApArWTPjr2u<$lvu=
zDORTmH8TEQ%m|&I$tJl!VESivH9V0ptJUsx53hfSi58Hz4EopR8q!dXs-7=4Dsi#A
zsWn#GiB2`=TT`2kL43KF5RzL$p$B(Eu6IZO1K)liQZ;vsww6C$J@-kV%)
za%%BsXir;VUOApVF@kH8j-#i&sU`mH3iDV&^1IrQNW^$D
z?jT1RLHz6w1>%npuE&p)xI-qz0(V1
z_HunWa|%)NpcLe72cmLbGyuk9;|wKNno(iIAtAt6?XlS=5h)}q_Mo%SgsQ;sFnHK9
zEDiv2Er3Gc2M3qr@9T^S2j3p`5v7SKC0^4Tl*Xr79hZ>PZZcRc7wNRP%7kqMFW?nc
zJfJ9alLKczJ#%w;8SOP&fYmlA0>^8OH)a?A7NOEydvc$fFaLD;imBy(&YTft-B}uc4Z=Z7u%atpwwv&1}G!xpO8t`PQlhpZoOWM@<(dw)iGQ
z?zw_iVVkNAP3$7{Wy+25$vb_FUguqj$ErZ=qle)Ml0{$4r{)G+Z=^@JwEzvG;eb?2
z(%ZrdsN2r`*-ELA`+RXyL1(R3z)eV^FvI{a|5$tD<^3)Rk4-zWMBy{FY-5<;hIs8l
zcH1qgjs^p?L9^b1(mCZHqQ%#9CJkFe$8`S8IhycudfX8d>{L^CC*GgUPRJ<#n)S#4
zJyRp
z4x3j_c*4x@6bOAf!(u39QQ5FA*1AU_YYT*e)2=xZ|
z?uYfX4x%aI1nMW0OCIUkkoSoHxTBX1J94fmsXq(X>jWBaz#<1
zUkx4iD9wzf`2kPQ6xxsASVS1*1By=1Zf;IpE{z(U5Ng#*23oo1uu6-LF2}s#B!bVg
z6-}!bz>88w*wCk?aX#FI)140vV7ipNooI&u~D+aGHHIb2%3Q6gBkMMT+f
zx|#z~w^ZR%*4_aciOCz+8q+P?{mB+&3wTL9HE1bGjSy!ecfCq2h>Lv*`W5#S#4P$n
zJj&3JyeXf=tDBj*{tJ{H7sYWTv&I#*jYzoJKb9XWgl*9p8HN6Jh}+ybkr$L&Ng#C_
z!}U$}E@HCeiYG&!U!Fm*+V-MlHKoca+21!q(nY-r)qh=qevp~a
zF|I-aLv-??5Bo?|#@7q|m;m>rMR`>fXp`s>5
zx?KrcuLWlVm;`kt-1=Hc1BM$HIz||i#2L8Isi5Ym@Z_eZ84G@;l{h$0xSN&%}$v!OY$a9MxqNc!0z{x%JWUNsgB3bI3z+<=;5bgRk4v
zS%?(b#&Zv`KQXFv1!7vs|Dg)gkAfq9wIy3xUxOTJp
zsCt@)5+uBYZ2Q*s4zFoJE1$nT9xCHm{?#QU=%lr8xYYKPF1^aMkT>Bofb~}+z7TtrqmV60NLPZW_hA7TFw#|?JLk%r9LvbZJO7QTp
zr=4o+Ucl&`#kj*^UEo2pz{F@Ry{iT?1=x@pK7VaMf@pR
zXffvi8uGH;MVY^(8x%YV=4o>+pQ{_uSdMeg>MuSd4cYg#9)BBPmq=i6vJBK}jtrgX
zI+d4WR`MhdApOwAitYrWOx!T+^0Zj3$HfepNm`mVT`Gbtc9}IsC90!p>{?plf0^NM
zIHN-!iU=YN`}0p(VEk#ZFK_5hAHDJ;m4%yILc~jS30SG5J5%8EN2@@}(6ek$5-Now
zpR)B&l@Nex=UdqweRDq_mAF);basfqeu<~zxNUmoWav<@^|nfTjQt)h<2Mm}!1>$9
z)7K=(s3?o3!qJif(i(4%zymf01`9?TIDk6-e{WGRF`wU__@A2z=j4B0_@Db>e?+CB
z^SS<93=wN)`bWiAEe|2`=O`r#h~-@8qy
zFF=%jmD6=vTtwM(fl`+yL}zw+9&}g<7r@DdJ%09rsgjVmpgGn_=5Rd;3-}sgTxB*(
z*zH|)Ad%6Lp=RniiB{fFknTQOH2q?Mudubr6TOgAYW+g-#~Se-C(XOo1sn@Z{a@knJ#8ws+HM0kI+;&r?
zj06+Lf%-2`_R?EMJSQGk&=Wm3qX3bU(VU{^_h)?t5YyJ4s)--W%f&lbBaWe}yDQmw@%azwKud?kvC8tYtG
zLDacPhrVSY4BF$FP&tt@+S*4%S-yorB9M*-BoO{P?@D2!!&p3ur_k%%ZJnhW-c2rF
za9cfH32e&e0+P^vs^tozzpVZQeJT+ZYP!XuFHwvmx_#R1P?{Zs6NKt9dt{$Z6tgjs
zfxEdKCs6{CBlZl95xiF&X!CJ^Nq5Ghyo`jtz-gGu6{I4A9knGY$z4NT>+pe8&s-Ig
ze0B>Q@Flysf9ml59IKLphijfA1>xr8@yF4=6ww|AxihkkVUrgPW
zko!(tuxDhhQSf!^3q?Q_A%fQ^NY?9%ju76T`imKAmn?2A&_(8g{FB_vem_n|
zHjpmDo6(^q(HO=@^o6VzquL4JPIHxbz|Pc+2iv*Kl1S{nC;R9~cW|*ENGK%KhCyVA
z?qw8{B13tVAM^C&Rd1VQ8;~UbRT4CXA-Aov-h^_3j+Mb
zp@GdSKOr-dFp}8tM4`ky3t$S+v`jW&Q&~!r6zuEpPJTxl6?PGaI?&CsR8D2`9=>jH
zp@hYdE!(SU4$Zzg+qjjV$2sl5KKt~x{0dC|`;kA9PzB5E(#(|_5j)_OaUefmL3kTc
zsI`)w%!H#yu^h6~BGyTLa<0xfY1-LeU9_96Abx04Z)O}6dz*{WPGBzJi^`%e+Rm?^
z8LmaoAw-DU_-dU3Ndmm}Q61Nho@Tq|35pu656rA_hSo^3;xtS*6bI~%UrXzXlhc$T
zZ=2+LKqC5{eprD5_FpsMxZ)x~*i>3lB$IIBR^jMD@7$(Ke5=|IL;p=d0PuCBQh#sl
z1-Rcg?{7E9mTIg2hHkc3nSZ%ojuqbW5%LCC-|MfAMxi8p?2(54LrHlvg{ygLgza7O
zO}K`RiH*DYhoCBV0SA{*%!CV
zgU7f<5ayNm
D}W0ewrk&{}(Y%f0sp`9Lot31#1fzY|fUm4qu(B>HsX*7eo4!uF1A
zM@01XSJ9k3Yi(5t)(8`%e`8^I8w=b0XQhoWb>Fq@*R45*jCO(*XS}-CiOY!(S;)St
zo_lUY&ww;jfF+T<@Xnf-I!Qo=r`2$Mf^xzF`qnu6#QiIAr>1d#>scP&zoWPi3K1P%
z1+w3Jm*?^}G4_SWWk=fsJS;DL=9`%L-L_@d*LHb-i3j=*r#YDviP$fBtK?frU}EEj
zk>B1Z4*RS8HnKiVxjVSUUO=@mGa*O1{(*v9$XGX2~?^~OiYQkwzeXS
zS~W%?Ad$B=J1`s~IBNy)d_`*WxLz8uh@8-DcXf1hym{*o^#SZCZU9W{Vn^U?e}BJh
zMHqiQ1eNG_nn?-$I`H`7#M06OU}QlQJDx4{_=rlx|5Rr+U1WS{lO^oU&q+@|Y<|Ac
zrTf;epJhVIR}6@6RN+MSxyQuB{Ol!e6luSd{q*VA!F2HiGYd;~DBu8R>j0SB#oT3t
z0Q3Gf>){$cvuI@=XN?rMpP$=uJzdCL2T>54$(Db+92wQN0cdA7Ai5F)Af3y6R#OG$
z0Jv_Do{(Vj6PSGA%iXa@eg=k-eSnITHP{X)wAkPad0Z{ocRb$|(lU<0j~q6#bu!JP@&@qJn{;_#8*@YDOZlCE6s6$
z(<)~cn`wUl@bg(H+d=C=3pA;zsb;L?y#N3#otV*jy4pe#78_elzU@K&TiP-%Ii4bb
z!2WvEJ%E_gVHk>}SkH1R8%=RbIZ9M6_QoJ2E}hCBGxqC-@+V^G8Bm}viudH+J>H=S
zd_PbhXn}!+HK2lEtg{C+a}oiu$-4gabrCPXQx6jW#<3)CCIGb<9CUa2@Q2`#|O#0=b6uG@HON?wI@f(GpG`GLTd+z*3AdB0*lH#Rn=3Q*m&
zBP#$wO<%a?W?6Z8E*cZ4#oL_}L*_Eo;&Iilm{JWxsx07wp{}k?A@^a)8!9R)?b6x_
zK|0g9q3-!&P{*4$vW)Im4%uu}wnzNO?Rv_a=7u2Rz$9kzI%05X$3~zQ+lp5)8uRyi
zkdhFt&dg{Y&WF~{Bo|4ZRD$pgOnZtiWY}Ebjnx4!-bY5CV{fxnoWQxWw4
z^tuO!A8!p!Q-kHQN8>UGYJGq{X6CYEo-Z@|)Y^l;i;MR#)xE)Qg`npZ?`^ADK#S=K
z+x=_bu_^oeR4hZ+*Wg7O)-*!sub0CWulIX7IoPg*D!<{eN!Gk(0?U;joHzT5s+u<>
zEQGgz#>~H%B@g&M5_gA86wO~=e%;*0$v=?o9A1M-cg2Umpx0_`@`c&n*`54&Y4?Nd
zX&>j?^9DQelVj}Y_@W4ogn|Zas?LI_QtU8(8lH`p;J{p(x#E~1s1EL&%ZG{U
zqt&|Qb0BaBIo?7xaakfbTvK
zaSt#x;%tHdv&m34+d((Bd*Z!^pSH^lVtR(a`o58^W!(jefr*S|e&2=KAt{}vjKu^WI0FQf4D$3ld!(FIse=r_7hY)@*
zEC4$Bo;^Xpkma*xuC(T{xvU()Y39m?im`XdY}gtf$=DZD_WCtfLSct3cRkK4=0?;B
zgq6J7o!_za3g^5wP8=&APIr&X4iR@1Dwv+5p2DIr69+`iP+AglD|NkG`!>+stB>`>
z8cZc%>>mwvwl(P8J^gLLN&l;1#(d@zPHC72Y28+4(j;5I>P?Z6t@?K~F%#qJA)n;F
z<8_e8oOilB*ybcs{Z<-kWj>FkPYm}Z<<>fcR7SgZMn>d%C1MTd_l-oxTd)VOG7Lf|
z_ws)Is@#(RE&l_G1E$f!LLLatyOL$tdW#}D*QwAB`L_eJBkF_+@(}F(5p_tqrq2^W
zkB8`bD~*|`*#r$INW~4~!7Uz-ol_0;M6?zbMPeRbA%jR(5oeEJM87@>G3s-^Oa6`w8`5uNe)h?Ne?PoW&KtuxP=0g-
z&S7uVKt|ykTv{nfo^~UfGY*vd-yAX9#8D`6c*_0{Dq`ONm&kv~lrT2l<-Ova#TH{&
ztOu&BqjD>dO9Gocwz6W?w5wAcM`tW}b2mJT@HP24!t)Y2TTh6(C1zU(WJkDOe%O78d8oYIHRlxoe?a@EVzX}Mko&6+gPyJDI!r9smBxdD)EwK}t
z0g)D%{1o}Dk9r-)<(`vBxBOsz-BT`1XQp4Bfo{zXr&g#7>EXG_#biMS%dU)()|Wa;O^Acz
znyIVECnU6*G)nu4nuL4@c4)M#zzee_hSSjYv+!dUzYBl2ri89OiUm6#Mw(#QCwfUa
ztc1mgAxa;!ZgO_3>FSOb{zCO~4U5FYL?&5X_De@QyQ&vJS|h|LdcQmE3-9n=cict`
zQ)6bz3FgJvYnItQqr=R7(=||>VngYY4Va~rvlO8Gl@@A(fI;{
zVS!g5Jw&#*vj9SIR*+kcWNBPbL2l)PkMqHVZ{V_MU1hQ4NF6srTWFGZ1cPu1fA&-4
zW#xB8mLTb~pB}I(9>|to=u0SR(s~orW@w}Glm5l;Ajt?)vVhTVijYS@l-H2KZM$Sr
z8#_ukT3LJy=%q=?J;v8ct*x!y4t;v{;;If_*ej=EDhRm*cop37U!V#IRp+bpU&$ZC
znnZ)p$UpV=_w#cD8iI3xO1HR-&=H_c^!h8u|vY%$rpvPY{
zo;!KjQn_BWajvJJ=8&A=)BNSZ|PD*2q_sk
zKyAoL3Lw8u!NkBgmMMB?C}gfed;q9a&mPTJzmif>_5EE20$8i3y9E;9XiN1Q2G3~A_Kh5oEi#g%P$3yaGe{}_0O;Z}QQmVlwE%j9EDMie@M1Xf3)NBQaIf?s
z>Vvfu3rxuz#A--{;*zoC$d{mfG%>#A0I}$AT^Yut3HOJq%Wz*goHj2EW0TmC3%Y=+
z?caRMI%o-|nY$h#mOdZO9W)j6$smVL3saMi;O6FHRT8<&yV}1{2vsg85>{c#$9#n1
z{1++axcq!nKYxVFGX4W#*j1~7;S2T!eINf=tet7jWA82Sv~zu3TrLY$hevn`Myubz
zCbryshjqhuuMzxRRorMjZuP3c4UGguaM+V2Vkc2tsK3!+p3*tk4e#Z_4n%DGy>
zM?tC4Q=j+oS-a-CCKzo(*iTcDrA(1+W!W7~So;P+Pul|q!_X)0esHujE~gxCUzb-~
z^s;np^X|$wci?+=jqjjv3AP?V8IMcICDMf0LO_he<86jgVT_QuVJv^K?iH}ZyhTkdR|P`
zoHJOXYf?=LvF|GIl8)jxa$Lqs*-GP{sPXx$k0CYZ_008i1KgP?6CfY8XZRMU9_`LrVMwxR
zIssJ+%U6EpePrB}oKQ4$1jKnMy5)Sz9zjdC0DaNpgD)IjB?>!V*Qz`xzkc`__M2XWS%@s_=Il=MbyCBW
zmL%00pK)g#+P1F(uGYSr=bj0nwrEp`6)hC0yZ^ZH0W!+7MX#c%*9b2
zo`X%&!$taqm}2$z>zo|5@cu5p%+_=F>$z)4nh`mDYj;B&GGgA2kRvMAzhS}x!^BH+
zh=E%JXA9z^N#Q>zll$H=c){OFtALJieJ)4fxyxiCKL(mk#|$+XW_il8ObH`FL&QI9
zbQAz<;QK8jFf|1xxuwO|L1TMUWKHf&@=d{{!L|2hSYGRWa;80eK@sS*GH+*f7$Fup
zJ^I}4hl@aw($~78<7CSiIN-;?pgq$H`QFuu+}W!&J5ilCDC?>R5S-Wjk(5H&bP`M|
z$cI&7o4kI4gwLnr9ClxApzmG;j$sN|nAPR=gt=`8T%4
z2C1Psm_jsj1KC^2ys(D5o8b$5)|MFGmcL=t1}<+L_@iuJXm<>aYWhkPeY`s~9eYbv
zp+s0Qx>|h#D)vk-<#+L6Px7})6gdnqOSFP)awA^1>l@s&&L{9HFQC2cLgUYpJGnI#
zp=by1QVbwl6(KBjFgr&OI>%6rtn&Jj&?W1x5Lknc0T;(_=`fn)wYffAP?iVA@Si^f
zD~6Spg0tT~U1d>>3=(}5TrpPA8dO*HOj~}H^?bW3(*F9(AGkMdLtJ2u24c#S^CHDI
z@1cM=m4^DJ33KiSv#)0ehe{3)C*h6$tHM;0C8+n8An!K@O!B`@R?7MDP?~Px%Ut6|
zlk_lQt*pFOmohG4WW+9$Uw1$9nJOLc_6tze_ue|AE2pZ!Q1enYq`)6o8Zl}
zZbbDuRKIaEH1LcBw|PHSceVbf{|~lchO3{a#Vtc7&+YpV)Xbry4w-`GC^`ZTHm|t_
zHEPn5`I*t(L5U(ic4T##;_2TX~R%~V03Ov2w*XEHVb2m=LMKOpPb`Am8bM(T!=
zbdo475G|9QtAUUiOJIgBJ?|9CKh_J$pm7VZ&oJ0fD;HkjuSZ%FYC&mGw-y@?)v{c)
zCVy%%B}-NE_W;VWYDzhz0;l(5zIkOCXy0H%hHlJ@XaW~H<4}*!vP5!8C9y&hGk7?Q
zVmzNW5IGN6Wp;7Vp&4z64o;GS@K_sN-l;MOe0zOBP;+!Gws2_f+Ufv3dcCX**GZQu
zA_>q?d9E?F_{WFC>??~kdiH3`sV
zbO0OUn0=Gk=o}jba=Z$Xdtdj5##qv@8&<9U_A^K-f+D`FDZsp
zY(|M!~MBGms?6CSAf{GN)QmUCq%k^k|?#>?*QlV;48yd97jg
z3!Pq1W`(oNGhX#}N4CsPsKi#&;^q1arNws7B8~PrB{f`n3&GS!2O^2Z$F*5xtflgi
zC?isB9A;)_fdN~ux$4%ZWoeLPOD=7=nk!KH1Qrt~tY=ej8oH=8+JX4D97KnUU})jb
zGP&+d=rb-e`hvSkKZx6Y?ei%c`!1Z5U^QT_eVEinr{wSp12r#dU$L!h>D*>i)!3eF
zKoWg;>Ro1e)JljxKYa16wOyZUOY+*NQw$NKt)IrcBwpT($M_lpt
zYF`4_blm?brI|gJ&)%r>GC%YQ1zXEN!;>qV-|OhGI`MAr%5E%`+caG0~!?hKwMITflrmd=8Y7gl%o;@&QWy2EFWihopwUcD=V$HFZ2a*-Cf2DX}_kIf}
zu@LgQ1NjMDh^|Ydez)B@Z<^kqOD0Ak)hAUVVS_ly{wK==wqWQ#paN$a{Ik$4-}nd_
z**-Y=FkJY2lpu{R5I?lUe(T#WgjY!ec2nSRpu@t@!-m5mi3104hZyDlr!-O~MwX8S
zh$Fk4U9WCR`_!MX94_Gk!?RV!*fiFmz31~#p=M{1Vg|yJig?kEbmpvbv{^n$-p+qaooirgrW100h|(BEfGp
z1TO9O#KgoC00i22I6&h26k~e?;M>=+gnUz0fP7_nC>qJj@*BRaqpeL9Z~Id#b8mki
zPXGX8&w{UjWNqAMZf=f5!1efB5+R4>WUg(0e}9^Y5*|LjIpEy{7OysFyJ0^G{a2=-
z)fVx7aJE!aOB&he3T#9r!D*Un&xPM
z37F|*eKTlR={*)dA3*C$04b+;i&Q>M~zD{UACLMc+KH1Jh1`)Q^YH>(0UA)bZirp&78X
z*$b}`?L{X4rc~hgj0fbg)a3MWH-L%wv-Mj7*whf_Py;j@!~+PlRUMGiz5w>y;WX_D
z8y#UzCB?W*PJnqjoC0NKm0I(Ns25h@nn+1Oz_mAlQUI=p-bDeVv`8p}(SI
zVyfs1IsnTf7Tc+Up=&@hbDnCM64T#|HZ?uHD3LJ=s3F0CjLm>1gEpX>;Lo?+(X`8A
zg{1K^$ZdatU9c4#2wn!;Z8G4RYS%bnfvZ>?`EQ_jo-
zrv5`15Ss-}!D}RfKanxzbOf3BW7BH4eyk;y6aJKICgvtPA%@94T#YvM&G1~0XN
zWsM;bA@2W0_BR;Nh$m81oS*^JTk*P%_m;{GJ6@n_<6Ov9O28eo-ol8F_
z%MG@HP4mS-DU#={JteU%)-155aVQR>fu!t#!nLVk;RF~h=)_Mv2xr0VbTPHAam{)C
z<%A~se(h|!->JF2S@|jM{#9@FNjEQTQm3l8d%(;DlmH3w%2a8=2tf9Owf|3ejDoGS?OaegCq#KZN|4F$yS1tb^J$NM+_NaL&}
zKn#HlMe5PvedcJZ9M3;>MaO-9nn%EkJ@!^8$15WEmrn_(qWLNRq!ohm?d;KcFFnHw
zcn(}e+GzsM$;k`SE_+h2HBY8w_Z5jom@7$lS`8fGH@BQ-;w^-{NT?s>!T$QWu3k=$w=$7
zB79Nw9*jD91I&JaNso>|Po0BPV9?$Tq_f1)*r*9p{_NWMlfj;h0(|b+}AP~sV%_k2rf_;ikm4~La
zOSS7iX`b6N_>uVP;dAAf%Wg_0&IvW(dd{B0
z$nIVqsUH}!$ayDk^-}S2=D;6#Y+x}V;{-0Ru46w9cFO$k|fKPbjF$
zhhj*Bc)Rf8Z0rK#NQnIP$Nr9&N_z-p9_~kVq_y$p{$r8
zByn45@dbsz)d)}i6DD#B=?zgdthdO=oKP+2NV9{RdZq_Gpv24uvS$_|A`3&C7&a{C9A}&MgjLs`of24cnb6?i(=dQx1dvUWu3dzsN
zhg@xfIo6qMfQKW^HPkxY^tJuP(Un|~1n|mJ27k9
zy5rpocpsM-gO#o(LhpA)X0fr}F&?5&fB6D;TkEAS@0miT9<5_w?H3ZL>If>&V+9_=
zlqe?Qkj@HnA%=W%VB7uARiX4$p03{v#lTBjS~JZ>^Zri1%ADz#8gz>Lyw(ezcGTql
zGr*}a#hX^jxlD*_TKj_UkJD47xf<-xWiekT$9wx~jn4-gR)eng8RgwFim%@p&BM=N
z-5IjfCZ@_DsEgwSTqQx!uyEZ~L4!z-V)d7NeN-GzWo=}!G*%S%BD!JlMQ?0TzTDg&
znIHrmWK$Y$Xi>h+t_$#JFblQO(uCXA=t9%pqqxZx^nn4%N<~JLL8HE4BL|qiG)S}P
zgWl$#^At9DZbn3B;)@j-BbWRmip#vhr|Ek><6T5|iel(+H++sO2C!-jVUEIhsSr{SQYGcnvejZc
zUX;{AHlFJ@k}=U-_0TN15uR~hlUj06ana(IzQVP;@#kSLeY9(Ir2kNS%S{gz{b;lX
z2{_G>6@h=+<+fOfVC;cb-ZVQ093O;n3h{1?zQPO8%E5ki%2V6!4)lKhCXdH`JZ2J3
zCd3@e8f@8{M%uzYMcn*OO60*|dq%|b^OA_|*%f@S&%Rd2l!{2x3r~O;V}5Q@@!E@)
z;<;RF1lY10Nh)Hvvk~n}N=Ih3E%oEk2=oIOJZC3A)p^OA9tQf)2wed)P*z=$OCn|C
zPPyqtq!HG`TuC0R0{QY-fuI>+O&1hEJS5K`anm5Pc>-7+LiGWtL*P0D+=u32l~RSg
zQo5n=%NOxC_Z2K0NV``TTbIIBf*vtQV!IcmMlB1bz#5?@qZV(2w?k>d>1bPaKrbOa
zH$QJw4QqdZ|91hT@V}2wO*Me3_=JyIxcMEpM-XwDbO4(Gx=DdH$*QvKGAE+wOLZ{S
z+EYc)H_8tY)GsG9>a{XVD-2~n0SOb9G{y-tOi|=-bl}@(WcCW%+xSWQF~+bqM7bV&
z+iE0K9NjF7kZuxd$Lz~{$EbxFSZ$Yl=ab?tbhVwm>GIVg=;fMVDd0Vx?B#aa93JRz
z24I#8z?@e>1Z?9>02q_9yYpE)A34BZE$FnVlBfgpmp~^XlMiq$XLPoT3JTmwz}#VP
z)*Il*Hi6o6_aI%^V;yqD%E6(WlA3BOGXt#T@0j+~G=QM-Y
zGrj*}!sKq}YOJ->ft4y=sh>ZaNkF~859vZ~E5!KtYuq0XmrCG~Fz1xGqOH@1fPc<<
zyy#5r^HBfUYoN!ll$P$E`j_QWi*L!rb^*%tqeO$r#6Vt``FyKcWVMN5sfmuTw-wlP
z`VBVRP|GhE-G*(qL*&lf|5O3k%uEuNF}0Bf7rM#B3{DU0!;W4RhJ#=HUA?*axMXw%
zGSBiQE5zrYY^m~bp3Y;qLtA^YXKYNvk>*9Tmy!3?8VwEJ;nl&?(&0VO
zSNoY#&SKR9zyH-W-nQUua8{b@n_@vHSC_Gy_!*^I^B7Ir0Yl~2?(3Ify|(Q=9>CNT
zjmn2sT)e|(#lzBabE~|({14Dl8n6+BpM>xO?ab+^(q3U_I*;0}3V*`sl+}**{Ww;CquiF}xxtTT2dXZ3MdfRPWH;0!g
z&4sC`mPp)cDl6qo&e;GKj>e7N@5ST;aBEtqO;B|h+t|||?+z>ax_h(&y7^+d&;3>M
z_dfw*3=H%%xP>7jSKs{0Jc8ZXTkeEPyD}Oc0_^U-i$6JvL6`ujMjJlcnd+|9B(q-$
ziZo9J8xTzexw%KiySnK6fW(3Ii4r}GrlzKYIBV$uWFMc0Lwpu3g?gtgHEg2?&-2hE
zz}frpGs2~UAdRTQ&1x8bM-axW#Jx96c7VwX;k&dea5&%)P8?v5#}qJI*8>*$p`l=6
zf|bE`5&`_c;uA*YbZ|-bAQ(py227d(`-O3<>;QEmuJ^0}#szYlO5yt8N}S+H2dK~+
z=2*xt&BZCyU+^M*`A)1FPVHoC30JoWC|a9{c(kx4GUpT~YaKe(G&sB28u_=X3Z4X@
z;%smJ&hGX$Zvq6N;FoW3nMsW7>g#`%d>QxW0G}myEG$xItSgZ^X4g|xVG1WNOgkK2
z1HwV0`&khKv6479D1=CYo1h09CstDw&V~DQ_1_7*FZx?*9eRMkGLl`dqdJI-Krs0l
z0tdi~7f~T0_O!Fh%ggr?b8lu3CQ4=@u>&&7=~QLSxg6#5i=L{vmAt!w7Z`HGAFMLnI1;48wZs
zbB=&`Ue6VA3jDfx&s2Gs|XH7UE+UYpxQ)M1Lwl(@0w=
z8gX-W+B*@c|3K&Hci2R^sd=_Khr@QQq~mtpspOHF!0?bk
z>-|YhjML9cJV@OMY`ZzJhz;B${XwqQtn&HnkcOR5Nw`0UQOQPci0Iauctl>+MgtE(
z@s@!(N?YqN!UxBy%mH8G!=)AKkKQM&UJJZYrNiZ3A~Nz3{<9V?{;zGGApXo-+QxqZ
zaCxY;6h^NQrSb?=zCqCHZzlx~-z}h=Vnh_2)lg0FJ=qY5%!u+4)VcZ7;)V65!(wzN
z2CEG6&V=Q
z^%wU0ze&YKj?(hqoAkNAw6U{ES!f0kS60|5ZG1O99vL}SC#SQ9vTo1Byb|PnKX(c8
zpv0l>$wImLO#Im2?VqR6GDBy-kc3hR61Gk~p(;0J0^X?Mq<1&hZTOZWZ?MZszAK9O#p$h!!mKw#15a5zEMz_sQtx%N
zBkun*V)?np<6()@-aZktpd=ge)KL$K-iU?vpIR&A9HtkCwQej@dS7Zj_;N)GlMkU<
z9hlhNn+$zQZFKi?z5KmTDd0A%OVa&rS~4dEySdq?A>=s3_}D9vX2@C|H$4_BpVFfJ
z$0xVp@1E|SHq{zKK;9a4>i+4g7%oc9xxpkH5Us!IpM5D}T+Ympsb2=GR{ZyxLx
zx9RF^D{>q4JUjjrrR+6UhMO%e&Y2nxQFPS+QdGPGco-Bkd>2{
z5PW3la4iuXhKN!c7OryiX3)ZA?BR31>hFM$sssplr;w;&Q3Dwlh=YySwyx%um*Nl-
zb?0MItO0!13RZBWxmj*z=ff2I4`)UCc?w?0$ok
zY*W$v44QrW$LKyf?2x_KF`_bhD{lx)qP#SZZ!U#j=
zd-jtLj-rdh$Wf&?Bmz-i0H~PK!7i;m^3SYUT$^x{cNlw_bl=|H4@c@3?V0^kUI9$g|0;Le|m{Q#B@b5E5tjq@qd0-K!{|L>@#U-zOLCVb;c@fYUb~bEh7GM
zo?jqCcXj;L6sHNksD3O+I`LEc-sq3ay$m}IN+eLX>W=Bnd9v`*k}gt;Fo~Fp)Vmn3
z3fDqfJ-!#*fjRMYr|wO8tONtk
zt;`P8RvlR|GyQ7H<~<11Z(!W3q;Ntqtr*&SYTXPP7+_=WMHN4x5`6srA9D{(*;v3)YYC(pvu=TzLplvFa9?zOo}`AuIv4!
zdESe}{7sr5=nHtjY)9$+!s)k|(8h>2JNG=B+O~E>olw`)BUC(9S-0p1w%iw25NFOli2{Gk1At+_JMLSG3QvnogIYs_Ss&?|MJqe-w
zSf`TPfru%c&mOfWply)}?8ufau$dp=kO)qrlIeoru86;JRP29+?Vsx#qqcnOgJt|?
z@m((XZ{_}qNkYR!f0xZLiD)~2YX%Epd88n!V>HRS9ik?DrSGvhrV1VH72W
zXExOlzi{UB>!=F@bD%-v@yMW$b?Nt~z?(MpQ9v(ccocUn_2Izp^<*=Z*{72ShzO%z
zl??C`SQ1j}VA&;arL{kx2YI7|g{2e({QmDo%F?*!z!;
zKkRzG==yMZKZ^MM93D~~DR4wh&qI0d6_4pk!;>jVt`2+^{NA)_K|hrto95AXELyj~
z{do5t`(d5fDg8Sfw|d<2v5_^T*zU=oGs*9`y@eKg?6gH8c_S>fph><{$9k7bKJ2Z~
zwz0kh_TRs{|FCqo_Kd;3dua)o)<-fN5cF&Ii;49@RD&idd?&p+{G#Tzbla#8(EDj(
z=>R8T4~WCY#+Gmm4b#Kst>}H-VJt*ex)IpCs*H~{k0ECW(4hZ*dQa;b|CPyXA{_-f
zOR^ol=|)+sa0U~|W4Vu)Hs>|m8m8si2jT=XoLjbx^=cSW=5A@5IR4?%jo-b22Yj~~
zfpGujiVYKZ&&Y=SdP-oVI|l&M15@K-@BTke;7n5^A*&leW{cHA*tCx@+nJ2(bRv}(Ou8vp8k3fyK*EJD4dcAFimLoM{`}i)z^2Ny`81H
z?lL&E9VIR8whfFJ`vh1PHt?xUy{Zqe1id$r#5X84tkRLmlslBgcE3s6J1l+xh@;yTKO^9W@AU?jKoXInFr!qcbWLMa^1z4!EcdK15meSg3sV01FAAMS-Ekgx^^UNw+jchQAmRHu-8bnqj{tU@njY!D
z$7QT++d9CBECa%z>1=a}haX^!JuUzA1`V^Y=HK(Nb{;n6KZv6yoCEiA#sG34-iIK3
z!oO$I&0kF^JgfiA*oF#v4qiVM3sLU%ho6jF8VDw{)LYd5`aZz}Fd!>(gsfAg>nVO-
z#S{3?dkj0fpv+b(U1SHqK@YDt)pLG;r92YgflPyJU@H9gT>;b>*l&%Ro3r-vD3nu;
zppBXc_>0I{g4GNXbM7K}i#>LFyHlJOBG`VX3|iOEo2Wkzwhp-CJIvtiB2C})D;9^m
z^x(5-8s;6bb-CgSyhF9Uynweb$SF^!%DjL3Uh5dGmVQJV#kwgYQX-aPHldE?Bi2-z
z!VKfa$3c&0f%p8j{Cgk0){Y$JiZRQNyR)-I^T~am%zb<|`<%n5{pP^DZzD24HuO$Q
z5oIm4XkoUb4_}@3PGiqKgz9@fe|rZMtXcTr!xR%4d!1oLirj;
z+`UX~O>KK$#@=-6Q*L==!3p-)1jEhR2+~|1w7xLDp+5P%AJE`85_gAz^9oj&7Fpw}
z3~gyJ*F&ky!YZ*jvjI{ULO2ato7QrB1CzZXBO`eLP3D?#@dk@RsK5vCr77{4_rF%r
z1;du2qa5u3rni?sPj=G4U<3TeaS=8y1_s;AW6vqLOge*rWtdTX+}wR55+cA&4IyL0
z9Ek2Qq#r4$qtdtNM3(w!40XxyHx?)`M_K|M^m<=>mcaOl8COO%^j)Pdi0RD)d`rgD
zN-zC32)k4d2>yKr2O(|em==XJ=Mo_q5yzSd+-wx`YF5zccjj;sPbfUC)FyKX$%wV4
zgGH@0cFxE6Nnc{G;Zh7S9klCTMQoaL|GE!hfpT~M_z(9MTUw-n-Rxb_)Z(=fu$gz=
zqW8@xu(Qvc)v)p7455){Ma(Ya4zOi%_ooePzi4RJiqO6T;D&StJ8x^u+BIL9@56wH
z_dq}#kyZCbcyqHid)43u{B$}XoJ|2?e+DA=mw#NHx5wPR!%UQ>fS!2gqV(m3)%0p_
zTw=4&!ve%0of7O>4rI&i^F7)hrJ~3h(o9SZ4bzgQxVgA`Hy2wTFvIQ@zr1KrN&t2-
zzBi8&{!3e+RS4TJweI1;U)>gHew&bQKA}4&#xVg5a@pkM&69~_TMBmmra
zoynx%GnB|NoQQW>ngVJgKu_8QNkv3kK$y%qdO%2pgjdD)Pp60p`gE}MRmFAeqpymV
zXm75m0feRy`6+6Md
z)_w?kyL*7~7Jek*fZy7B=hN!=$O!i<9-h&u{(jaHChrQ{m@`Ty0(O0VFBmWpeg9~1
zY?SVV2ZT5o8i|6P(TRz3c$_B*>;?jE!x@m-p62my+Mmu&PJYa1Nrs05`H)&go{skR
zAANwBdC|(j>OUvqQ%0ihO1s*?DI#$auOx+_d=70S=0nS(dcBTAmSIffSDs0J<#fnq
ztMZxv%mzJoKi#ZI
z*V#DnQvseZq}9(M*sC=_K;8rA0SEy>pSCPVgxX69U~Z^AJb+!cEJ@DX2myBs!1B*v
z?j0}~e994mU!eEgc;eyqWM~Lg5ED3HnZde*G^;wOebZ>WtjJd9
z(CY@eRTByD?diBfh*u(^w@>&ALraByEIgKSBM{OihC$Yy-Vqf(MlNVU>}IF6$FM>s=omo|)$p5ShZc#y&GXs2eq*OI^45t-kv1D8@N4-ZrBtO{7U|N$LWy=6$c;XoKR8gEFWd2LB9Kv#=x`M;s%5$
zO*X3^=IcRA^X1Pk`O2Qkb%?AI%B>A!Iy=ht1mNO0&O2BFDo{JqVBGmsj{aFBwuHZ@
z%m7HwalB4+2K!MyLx{<40#-!iUxL~pZ3#ji5H2*aw5AMic9gAX1e|KiKZq3vYSb(&
zgd}{{Sc|5{k$OI?8F7A4LmsoQ@~eP%fhxbX*lK+?9r?`S5~<(|Rh6ZqEEA?G
z@N6vS7p#bT%CX-%G7=m9G!qx3L_@ac8_jY>&WZ>O1)mWsK;@Oy+ZG4V#hTavo?;A#MAZklXicJNzq56Ub=^@^OdNpMjyP?dXHIW`j!)3ILHSza{qYyWW
zvH~tUD-In|pUr-ai8vn=GX5ixnZ)I{Gk8;ptrNl{Hto|Q7cpkfF8Q1j-Q}hijyC9w
z6IW7hpirBUNmfv{2!>x+MXv30rV%qbf`fjJI4M
zy+=`Y#($}Sv4_o_i0~F4f*qi!8f0y{F0(Y3jdyuqMe@RdCy@7y&DdC85WOEe^^Iu;
z5;^HiE2HA?$}?sFsgfW(8U3@>j4RHP{JQ~
z0X-{>-S++O)Sq@3%~5l(PO325*W33*9gP%un~UQ3kXF9Wbvyo%`>xUU1CwIjY
z;R>QT(%P&Gg9nSRv6O&x3&aDZV_z|m0U8Su!}^5WMts`oQ~m81a4o|N_Ujj*Y9%#c&S|M+1yl2Z#F4K2UV#iidMzuUo@wFAP9nz-
z|KUoQ(FdLcil)0|{7h_WP3_2Ja+E#P9#eT)on0PA@0x@TguqOyM~FbP2msx47!hhv?gC{nrG691wkjN#$Z|$^bEFp5$E$;P&Na
zjO96AT5U3|KM^2@?&yfx8=!HHY2O(fw=6DNw$GcufRDLXRfMs)3lQ^dY0C*)bU^e7
z9(>)CQG3>TT}=gOm9n6@3JsM##OdTM>?t#Zl<9k(x$#+~gj&rls+7Z_pE}r-XJoM}
zSsF8C^93(&0sX1b&66BGLJ@|VG4ja*CbvJ=!2cJZWkfAUD+`XD#CwDcvP%JLPE(L)Q^%Kb72J+VO
z@lXy~ClYYlG!M%(6=r1YHXbWb-6^0FU0q)%;If`5egz=LD}V>r-O$CQmRiR<z04w)@_~;v0oQ(aR>-N|d
z@}$oPwtBDQD^fy2i=k$U(9lp`uj|w4wY9agLtr9~5k(<96Qd8`{Pv})d$e`7DjSbG
z1lvHR?h8s&K^7DJt8o;J2pARpXl3%Fl4*GRzr#yX5Y7|X4N!|e;9d9v4!wJN}#684z#&N7JvF{8i3HWx3Jk{DCO$+NC1)$
zf3u-Wh9ic%0)kqiQ75B*>347S!FuffLx5*R@1XmXl<&t?R0>&&QHIc4t2BH;yKClr
z(fri!E}+7kxkk6D-wRD19`p4M*+>1!JR8b@6&nSIMCubolMBEmWXlYbVgiVFY!VOAOX=7~u$>
z^0a{Zf(9UpnAfRWc1*=+E-W>7*LLiGa`dG3e)0|L+~beAvwi1rY6!d4Six__%Kmf_
z9|n&Hn7g2&j#B8OO1jWIWAanFk3&5@oYenhx9!@xx?T{%Mz5Wdj}{aA+e-(XR7n~-
zd#|OcgXQI;h%)lUX5cZwG`EY3Kyf}#Mn^jbgN?j^92NUAX&ITGSEYAZ8NJk`6q<@}
zrlWNQ&LG;|iTIQ=R9XDPd}K!MtUOkRY53a0jJO3x6hS7Fw|ODQW}uz(b*>Yyl|nQ^bt%jBAGoq)Px
zvrlVcK99v4QA!H3hN+aQud{f?B!6Wqol<`x#VX|RrsMRSE!>$CiWdW107L5V+TA!C
zgi6a_%?!Wu%FL*Zr6}cwf=4jNoZrZk>R|m#Us-a|irv33d`|R)+&J9yMuHm$LQ(Di
zB5ebQEJt|uWxaP#7jgsnm9sc@PG$Q=P9u{>LPk^8bw;>o6ZaG>
zaxl6tVuFVz3D1WC4Gyo8(3hU3r;186aOnw1c|S0ec$b5wuQ?~BQVZJ7#oLh?
zS=-=Y=&~B#bH`yr953Tt>^|#ZBdx#emgJ2%&t_|4DsfvPsEs@}Gf9#ai
z-~EACu(O98ib8B`WrmJ&WX0)f<@)-^DQjn=V$j^2C-TlJMq8%yXNgE4@ws*4jIecP
z0)4mWwWzTp)>Q40(8rta?U{Fz%RR}-;@=wkf+o8JN3o#VtnUsjSs>tt1#368%9|2RZ3C(!exR|9dxb
zsOrDhO2R{&$MmV4cbmG))_0S0INT@jEZ3SMi3k;=S&^ijjq-VLr=FH$)dC2CV?oPJ
z-ieAc1=67Hbf@FIi(d@xc)fR*UZavkg7iQF##H+Td1A*-2!E156>a@Qb#M#TLe5^a
z&1!<}HsplB`OWYKi}C)-{o$_F3p)B-`I})}NO-h6_S_8GwaY_ps|e1Xb)4^v@mmW_
z5APOqFGLW;i<0o;4X@vO>by@U0U27(#>SssmcJb}>r1R%TD1W+i5P&g8EW8!`zFof
z4n+<6?;cB+RAB!|$kS
zALP=*n5U5mJmT2(krEPJi4SP5Ge?=lh}-yWf9b?(-R3xNHf*w069imV|GrS>
zhH1209lYHzl0dAz=$|WYf#?{jr~Z52imslAkV}z8r*r&J3pw^)D(0D#$)SQfG~*~|
zc~?NWF@{fU<8C@a%O%7jqZA5Rxj)B^(|GaIp-nZDJkj7h^f@K?DiFU4ul
zPS>6`UxzY>1KsXj0yHUuzgk>%Z1@lC8^d(fms|#Aqw5LswjVTD(3(IU3&rni2R(_q
zYAg1WTZS4es8Ih^;W+$35cmW8dOquUmZW)4_
zj8yAY@R*MMw2M_XzaD+oTTs_n5`mm`#vk|6pyIv^bEOsJ?6+2}z?8tw(a#LcAngjT
zT!v3PCOb{k)PvO(P)pa!zVPdn+c}_+PJ~YRS)!@`nfpXT_5~Om1y^D_?dUc#p@A)R
zKQ4)n`nl_n@v6=|CTiqVX1|P~8n%*NU3+Pq!FD5K#r=8`6R5u5J{KE~0vCV{2_5O<
zYZO2Re?PjY<}9!gNTAwWYXc3!-si61N|FL?#K)gPjrbp5;cp_dHw6$gY|QFqaUF~?
zmX3b}kwP0w#gO_lZ-o|S$FKV`zeG#2h5oF>dmenNAS6K$*c+>1L59XdUc78u@f*EZ
zHVw!OY81mBCsMDY4@a70cLrjZA8)I!1`3_1L|(Y1$&+tQjj7cb9xa}!w!ufCb|O9=UCmoO37quu$)tpC$;x>nfsMsh;2okZ
z?>z3HEb^{D$&$aLPQ6ern#6YPKP67DO%d~b^H+BG&tDgQJc}yMvNj5(hNc9nIx6ySoCZ{$1@g*-C~qH
z3#~kOy?PaDtq}fhdk{RsaAG)4gz*zpzve7ss=V{{Ui~cPaA|vEZgM;XVvwr8*Gykl
z&~p*f=I0?KPtaPxZwZ(95f&Pi3!64S9pcxeHDOVl7Y-@U>rE_BP2jE-T_OYLD1<2r
zxs6dsg3%nLcOkGX0Su>_c`lyR**8Gc%e
zyz2KuKf)O9%~mELS~zX`Vyk8^p|l`qn5uH}P3)$iO&sFH=x4xA`me-KQ$&qa?W5D8
z#$4}rLB?hfnJ#3MRg}K0C!m9dEJVkJR-Bip7vk(v
zX~MH2&2ay6=(hQ|Y{x0+AkLRcX3ducoMX1ucBDtS)WRq`oRx+ja5deQ#A!k9Uf|}E
z?t}O<=z6%uify_%&GA#)ca)O0!IApT+G@8H>{1aAYrN6R52`P+Hv16egsjjsJE4dff_xJCB>r&BiaY&2(`%hs4_T}X&^p8pAVl>rM<31~J{Eak6q>Ul+s(CSX=Bn8C
z7xFqpr>4R;#5Ym;Y+>f*A);pewEM;mtuuQ0@njp`)O=mk_?K?E$1i8jmpmCao}MCz
zTJH}~Oe&CnV$ES0+>8Z(BQ*43H^XN(cK2wH?$``5HAK4%#vrN2oa?
z2kvCL3+!^wY77-7QDoeies%RzT{}(Kb8F#8)ZVT2F2(5hg`S1S2{M~kJ)rQz+Xn{U
zdZOsN$0A6V%1he^UQI)_oq61;8>4=$a?ub9{zBc@Zwh4j^!ANZnG~njs1Ij(B;&`1
z88Mgl6iF&ieI5cCjk{0Jo%Fmaj_Cpzq^`MfOcuK~#vHDO%QVT8s`ud(_P5av;<%T_
zO85>H(-}A~6V=PD^#gA
zRbNok=d|!
z{J<^GboKCBPYd!7P%nqIhcOT>Y?at^j2T}=2wb)>aqcg5rB4qQb0Jz<6fe-7{LHCe
zz8Tx_@J-f;L70~auw3Xf;H3n4SawG#Y8#)fYQbuqJFo#(8SOLU<9-*FY;$~m_e0G2
zwC@N02_?drR^c<8)fetbRhy0ItGN`?R>*)OeCC>2S&&nl*@&b=oguMxu4pp9X7
zECF5Af`ab`7xB{GJH+#ZAZQeaJSY3eGd5@Nw@-8MB6ZJ)v%Fu*$PE$LbkB8S8~&sl
z-MK5Injde263`S6L~eQ|TL@`e=fb!aGz2b7)Z}xd_|dbHV$e(YI*pM$zt-
z2b~|kFTQyT^&_4uce{|eo~QKQr>@#ttgclcn?Bkw4WwqcLqrj0m9p2@u}*tU_Z
zBj&1QsmO8R{l5?N6j%yD{F)MpGzB
z?+7&wiY;$SX54(Gg$2tid2<&X?W+TZkZzv-cqZkr9-FbafYcIC(=J1qeEydE7Oben~nIC9}Wpj~wymsn!
zaxW-#EgMZF>hjmQ4*sYIZ){0jfstnIAXvJ6ojJ48IsJ>vsTlw1UH&P{qFd2%U9o@M
zYgpV($oX#|n6y%sHdm7l&pS+0tNjTCm>6k|sgTWN3PH@%obEw`2+5w_q7cdm{;UsB
z1!Fk_=<)XepO%$5Vuhf<5PCTK4SVCLx7uSp+SuqluP`ZanNAAJubzhqBqBH}IEm>-
zzVw;D+S=~ebmBn(>9`k1;+Z2Lnx*p#z0n2
zg2Y8xm7(Vg|H7dzpfZ69XXTFqhs>HJ>0dSJL$y(J903Rw`YxHM4dGWS%4s+_F`zLk
zsE9kW^~-NAWh+bM7D{*2=3~vf=yY23y!e_@Dt@L?G#nPZK(wvtUtVx&yP$@pjMoe5
zs$<3$kOCH5)Msu7KcYSsCr!%>ZFIjGKq|WRKPiVLRl^VpsriC6kB=XPz5UH>)X4#
z4q0O#KuXL2h!Ze_-z_XVcN~^l5GU&OJ4@2&XJ=>6&#vebz`xQccWZGTbHc^TJ4m+&
ztm?Z2>4h*w`1<-D2~ESWjd0?Yp2fWX1S~h?DJdy&0XgFDHuv_t(yv)%FK-D=
z(uW>kB*EIbEKYAtcN2wtB73UBE!Cv$V<3Z}hlC(9`G;i_3v^rot^CPe*Z#dkX8Y|y
zBsPy=s<8NT?1*D-IuuwnBNwh#@P%3x`7}9<;C32KGF@Jbovg>-el3jTSzs7RSgj_|
zMRg&U^0%YME#YD%Ihv+>Et>=u@$N2@6mhfUY!tHcIj>=f
zj?aGQ09GOk!zOplxJG4!Cm-~I^afYgorygv9DN|9d^XIKIQoR94U9vppr|-1>jKy7
zL^T;O%h-VVf233i>JPTEKgEEJnE3GU{$wk&Fggct1F1K8-(6#ce4-We&)^Cg>Ro!dwx$Ec71
zKXFc2dnDm%=~&sfamunYz0i$}*~Xfys!sIYlNunEN0Pk!E}L+5vN2?Oz(`EKy9c})V~i`vI94);X)!Tx
z#G;ldq$TjqYk&N>2?rvHF;;dGy;yst{#r$Vju_a4AzO=LB-rc)+!ylx1qNc=?Oh&X
zf?Xi02e1zKF809aGnbQe7VO1C2~px{VAY+rOY#p2*Bx%+<%@ZWT4=IV(3aL{Pt6!NrGm$mRbWZ8DRm@9^!*Du$+hAB1>7&(AuPt
zcooi9Su*G8PTU;l9$_vyH_0v;<>|JBzIOp3vr@neU5$yb$TCsK;y`;>54@G**)AJu
z!!2uGDeJa1p8
z21q~xLX{hTOytQxu!--Hz7*MJM`lughhDSs(a}g8CWKdd(S)8F%0;c*VUxxHMW*Nx
zSmw~9tEO7IQBK)ODoz
zyaP{KjKhg#njn_3`SH>RhLRVx*d-Ocsr;}3x=@bbr}{zAe?!$o85fS01@+K4ua#lI
z+9Ei`)LQcH!9nYE$;m8yylI<_~BHTFJlQDl+QV1d;)s
zPgOieqm)Ov_7fX7lc{mYr6kIk4pvbWGa$ZG;cJW~_L>GDjYIwP?TzkU++n(T;{`82zaWY!hfJoVMW$bDTxNPfdh=}GZvq*zS}$s
zA^ET@b`<0*yzBqds%7qS%R6Rr560YhxU5N1Fwzyl$fhk)M7>5QqQgW3`$
zsR?OUvk)-{M6m-@G`OJE&3D|O`|kuu1Pmsd6q`@TNoIbBV+(#azPPe5wkg0e_&O20
zxAW?PsQBwJ$u4W8yp8-U@{!S3Zv&t6wAzPjFVx-SX~>`Rmyzs6j+XF%6O68T-({I4
zzuH#v&USSAfI07hB@9E2i{=w%!W!GL9~*o0pJ@|>3ojr6_n^0FdpGczsqkS4`E4?W
z4HrZ`8C4R8_Vi;Xa>5i<3|LXUXYk3IEig3+3jl8Cl@NRw^l7=b_3@jw{uAsQb_}7W
zJ?bNI#Mf$ka+E7POTqMe%XMqo)a$mWt7~!@d-={8W|p?mgJx8OFDd3D0~h)9;jQ)5
z%WHW2eKT|qZ+?DT+S@^wK3@zJQH~5&{Khd{dv?}!cmzM5sQr@PW1H)e_d}pLo6ypE
ztxe58BD`L>Zh1m03GywZ5FW!(Ty
zFJBq%o7wO7_66^hScm{*?(jJcj+lsZ@Se3@`$$RZ<{t(OFdU1ki&`@5_Tzy_fb-T_
zoO8ju2kJk$vNRuTv$<;o`Z8V-6jZ|+W+tzm^5cS_R46KtxQV3gRPmv$l@~~Lc!xGU
z=9-U!AAMo;5za-yPPuNJkmbh2B!>fS<-)*Ii98YOPIuf-c89%nwY2#V%nYwI?+(9s
zY6a)-^9uAeF97;lxzjp);CuE(TiXoYcRpADeaEQw(z5oY6@SZ+EN@E0x)L;a!t{W>
z(HV$gvqbOYH)RvhMelj~0omJUqsxgRc5PY9km{0>2kT!u2f^_QNk4W3Zn?8S5&HKW
z#l(wpy=DKUJg5mUHt!_G#=!p=6b^I~{{kU6)3Y$8T0avX1uUdNyqjtqvpyqP@F)!?
zCrH2}ih6H_22eo
z*Ft$7%)Klj;sE=M%b{dp7LwT7#=5BBB^C2pFC6K@*2Wh7sG7fq}|31P`4DqNZ7EK%LWf3zg9AXHAe`AJP6F1Y???8FtFH5~|l
z$zUj0%;0>^e^Ab$*zuhKpU+(Z%cjXf7cZl+1WEsT9kLGF2bwA8m$0L9>-Eq(m#cZw
zLgac2u{H7=O|{LgkEr?$9+2ML9(eIyadVmCJqjBqA+JHg-EBg0?$|eAvH0*|<7>|(
z%?;Qt$lR!l706lldmO9HAHZ&7Q|fb9eB0g`Q#FNUoZg;`b-7FyS~9@GUlcx;i`+DY
zy2$6nLrtB|JgV}#;Ve7V$>LZ}*F!hY{Rm=v`xLOUWuKqlg|N7o9vD$da=h$~(3!S~
zXw|Is^FF{YP+yQ8_Gnv5cjj=Cxf_=f`4i+k$1i3-7aU2yaaD6ANkN_zAgvUPq9(#D
zi+IyLDv8ML@erSp0Mnw%3&9TNVX2B~kjS)e&rMsS@3TvI-=Y?PbyF^+?Ad6*`-d|0
zoRu?-_48PGMc`a?L3;9-E&kd*PhB5vrKB8Sj7tb
zSvImgbL-&0D*0iIc^^t_%&iR(V^vp*V9vVpCR+omNjA7IbO&am@$W&skF${t#-3d}
z-_1WJS`|*Tv_&N1ef|9&@e4rtDpOEhjbefp)qgE~T)Uvg;NdabRl2GZ5^|imFcP45
zHY;e7m6SB)epw)}Q+G2;RQag?WboauAswk6hlB_#UtTBPO#NLspSgA)T+P>U;{
z<}*iDe6GMyH+%cRvC)tLKR)&IIBq@8pOlbh91edX*hr*Q_hX^BFytdaWgrY9Sp(PP;LNXa~=6N+^Do$=+2y+UTdknO?ne7pHPQ71(<5xz@9$;$&6qCEy{y{LlzV
zKiZP+Cf3T@pb!RG9*fyPQYM6`%-`&^7B4Y|R|4}m^|i)!Lm<@6VxZ;+gDECE#~Ge<
zS8NVCX?gIJmjgyR()ZB1j8aCyBH?J
zA3d@vfTclr-v_Jh1c5W%T4wWf(B{mAL#Qx;GQ~GZ*seP!vm*C_7r|1zJJ4|KKtrF_
z-;C>0glfq%j@YfpcnHHg6gfuM^av~`_S%~G=7VNUr%^Qh=}tqx*=X^)^y|ZH{Wt91
z{4;?feJW?myj^RhN0X=OGPzt-lzWc}q;xqo*|R0d#`S{E=d;ny(BLr@Vbde;y*@XB
z%27nqrBYM-?FN{cFKXWCxT4%qbL*SUO
zIf%dhbC4D4M+%%7@LwDH+2$JV=yLlsYBcs2V#fDiO9j=ckZD4>GS;fUc5=r$AqN6!`nSavZKQmO)lk
z6W7I9YK8cr&SjtnrURdum8e*pK_h#1_?fv0sXIi1Xlw9>%fabmQ3xCioygY#t4LHL
zb5EKPo3vT+;+GZe^hpt*#*i}_YDZ@Y;7+k8IT;idg$4Kc`udbGG1VLX{?PmR{wr=`0lIySaVobWTE=A1}oe#jgMbfbYaaj(+o|
z+klngC@~ib3o$EV?g#biphpGHHx)pfE+rl2
ziE*I!Yq1n^bBMD;xA#XyuC1@v+*os)oR^5b5*2Qf#Nr;$
zE{A1mIaYY>^Y!ColL`GSj`gptcZ(LFUM1t=TCpgu&bD5|F1x^PHm>|~M!ch2Z$tIS
zjKH-PYl!gC*7ba37?hx1h`#GrS2dH4s1kRgJ%v#b(x0EadwG29(x_ek*86S$sd%(d
zHoYs1@{Q(5WN|XSU`z$L$yQdHe4IpF!uAu&V?sIq{hRyuMb%*o`F>M?zC*mmEGn(l
zx!UYIiXI^$u7rpW6;2rOH{EO1(iPM?lbdxEa@RW;_`AU9`&YQHUNqYyf$^qF{-ekM
z)f(GH3nh3v`R*edmdJ!z>@};^3E3D|6fwAgIZlc23?@5(3d&U?X?V*M3i?=enpB1>
z;MxQg@#U)zT`UGz!TPCj)U#K@T?uM#&b^45S&;XoL}%oOvJo%wEI)ZB&k)r~YDXw!
z_AAv%@W^1^JMg!SuxzlNt1954`lnaAt=)tCn8xg8X0foZo-7B<-*pp}r1r-zPoLxs
zTEhVqv+(8O>l`GS7bBb@kWnFGU3wADFMwWYEFiM!5#oOi;w2aHWbLgbybpm4ae;ff
zeuXiCyKjwA$5S#|8q>E&8162q$aFPT^SlZFiwh~Vlg;0cVy-|4BtL&F^;@Xd#gZfvk9oMnT)3
zFug@Y()yt0mcXr8NmV1U%HoWJ@%Rh31A-gIF!7S(9pp
zwlztyZ-rVY7BWwb(zN{VZ|zL&5&UVB&%|1Qt#asm(J9mBVgdlXISey^$AUpg##Cq}E%+sNJ+
zY@z-(6l@*Lqei+Edq~(`Z66&8TKNDV9SZOFfB!MxUxon+8M%V7XjlzWVVkrkq7D)@
zry|<6YnadED((LH=+}6z*-Xc|n7q!;e`d29YAy{|8wDkv#VIKX-
z0HP=v!c=lY#>!XCXG3=~nopNnuH?0Eivw&Jvb877NxIVS*XdS~7f=)p&4Fa9{`G;k
zA0#kPa-LTj{f<;P*sZHTX#+U70^=2Gof>HU;a}6WlS;5LvUvh5LggS_PQZZ6mtT{5
zbBOB96;-m|g~i|#o1wD%BKC5)Bw#zK4dzSnLAXY0RuIu+mUDw59mM~
zhV|J?K&kwdL{e$8-vJpkf&$F3D22xl>O&{yl%cX0#-!!(R0~GO#|kH#
z8V-s=%z=l*SHLuCk~9sq!fio5DiTvN2L=qYrdK3wVX*Iaibgk+WoxDD&l-%k?*oG#
zd!Hx8d!LbjfB#r=gvi>=H>mh*daL>za&TxQ%ns~!vY-mHBUTzcOtQ90)y
zA^|G*XZ#G3gbrGBp&D?55c=@IybKA4c)qzkH1|VX1)`3VbA?f;(XfAXf^+5-i@2I`T%mNzx
zw4B0wL7O9<_wgAB5_aU>KTAHsUNwZI=&3VTMj=3(h09O%qed5`=!pSs==jL!fBFnS
za&nafH!`rNu*a|GvU1JfRHL}3t`oxfP$~Db0G!JaZ|_km5m3nE@b?w}>`nlyaXC?Q
zsIZb+$-Gy?TWmfQsHtNM3%gqnNKa?k-QDfcsN)#0M_CU7!k(IrkW`JTq+HDUr~3#@1U)R8Jxw=1NyWKJP}6Ih4mhIbb-P^O??z}
z4CU^b5nLdLhca)A3k49fL&lEA{TpI%7u8w9Sdx~(9cqa~uXPacToJ$deBd>qlYPr4
zJ6`opzg&sGMBVv+zGURdswy}zAp*yg1
zdB+U0r?G)^|H#*#AGi^@?1O2vNMdTU>~&Ov4MYt_>=_{}0D+8TX{
zn;aa&=eM*#2olDRGL^{$wGue8-65f8@7(P4_-)dcu?hqVdr)|*0ulmnKF?P}cYL1z
zzWgaTC{;g@@P>VV%LWNz*}eD7Jf}M%Gr_@JO@CQk9vJJ;bxC>U`%=#luml0Vb(o~7
zdAY|>^N}4=8^IX3KW#~@l+yL-33>D}`02KKu%d*DnF}AZG9MBUrOfap+zfCp+;wUwWDf*;D7jck8k
zAHqy!!9>wQmzm=|oi0&mepgN?aP;@^`Q65Upw+@``U1tw#^FzYc<0=oW^mY=d}<#P
zUjZv@iW~UD>+W
zD;`d*Pa?wck`-8Jq2{7kzN7o24@?UnpFe;3M4#(N*qMO`597KX)JVXWrzOdNbMaue
z*mA%Vo%SA*eeKxb*ncfPAwi@VKI_vXDy6JFk{I+9YRTL@FvfFj2|DRKb$6Oc(
z;I_Hh@qincfWca|;_cmmnQ}OI#sFq}5g4>Rdi*C+ruKe#yjboFa6y7*Aofe*9Cj7C
zg6nTX%@~flyY*xYPq)F&pDjPO$2`7U;mirTP!R|yo3qK27!YH2RpuQI+>Vb_%M9xt
zrSTFHUTda*vwb)Eb_#n<`2O8Zd%3c263YzfBz9E3F(;A5^eiU7GG}`6n+IZ3m6#AF
z4BKPY>!LS?xGZAe2fKM@-%Y*2;lXd!#xt84v&sbGKAk8g%7Mcz
ze!9~{J$rF_>p&g!w$pWFzX2{hHdR<%trAFg<-KsL76844=(IQ%9RPGeELk8iyah;mo4f#0|LFKQ$H)H(q!*q5nRV|}
zAj;qZqUT>-UiPhWfbOqU&@G{{pFir2z2cH}6rZlgFE*Ca38_sS!$roc4ME
zG|2SV`|EqW(o{{=r0eKdOEHs6`v#<*Ke!*xmNss9UyZE-xfJ`gddnH+TK#N$zqgmW
zVo(l#{6ovuk%p$asiXDe{>-8^NJDEQnT01@tRvp^V5IA-Z
z{bFXGhJcYk*ynAgDJBhrJe;qFzd7U`d47ADh;SwnU^u0+R+;PnARcWCBTEr!>qQBU
zre^lGn#pgqbs130jrIEKW+ub|&Rf3#;e>#3Z|dTwMG-{=^wMN0qRPfOt{*2^82r(k
z@UtrNCJg+v@GkXtzE{=mXxN>Jb#c1aw~w{*bqF#-W7;yprB`!lQ3NK_0!O)2+?%*?
z(veVVn6bk@LBzYeQv1i@Gk54c)!(_DZA=iZ=)e)oB;WuD^zsQWslD5F
zY8JYYs)9k|Zp`+r*reOI@wh!DYmbl#d7QnG92yMxMC(q~I81t2OVVs19G&+nCz+-(
zoU#tGS3EvEtl3Q!^!FDQlGC}afb7Q<)n&`50X?g$pT3�?$L7_|2E!BdON{R;2LF
zt*zph4}!5`+Hsn50QNPLY2cCP`^ml2ILqYt)+&tuogt
ztPZQ4MuQ_Uri0Po#03;x2KlN=B6HrD{b@47e`tU+8a@dHx0IQvHW8i0nYD<0(gUfl
z01?97Mxjs}GzaRosT@>p@AOWCPgrD8YjtIy&z}zV9qp-VfLF*(#bMG6
zvB3GplMlb*&41sj2F9eJ?cS)Q$EY}WW2w5Lx8>>J^ZTmjTL!yLM!~*>mN0~{a0LIt
zXwoYb0@f&Tb!TU%TiRVLAQFktqIycdlMDrG8jHlFql{l*x1I-5MnXbDsf#a=0C&K6
z-V4Mfc+mqqA+KAF#&^Y$-QC^gCWd$VsvC>GrA7dFwiaa%G-beg1mHLK%nxHhXCr5Y
z16g$%3^H=4pBRNeXyqQDB}+fZyXG;C7%2`0Dk5JmePu2@H)kp-FUi-TRCN(Usv>gK7s?q2JH|u0jY066
z58C#v((GZuTGUr*vT?PSfD%>~YYvf8&$Y88~g;ksAMLJX<%GRH2K
z^1N%D@%phjW}^O(D$s$Z^ga*n;_v7(E#2p=3S~X^AiVt%2yYJ$c@Ok=oCv}Nj*M_X
zfZei7ILvjQ##CsqmU=KMob5U2n#3?TDsToZ{zeer2;n{L0(X+v@RMThmwaj?W`^CR
z_s^!nTGYTo1tuYCLx6E!(MLXd`Xixi0`L?TQD?a?b=ZpDe{)x3d1Lvwyc)0W&c(Dn
zLgchK$~|ZpvtwI%c8qYAicBUJVfB8c1VAbM9q-Sp#K}M&m=GR`ShPo
zn9BJp@G9|$6ubYv3GUArq~O`ERwExldD7uz6IkeF+H?jIRgFgpp
zDpfB=fvCK;T1ndYBJQR0z?YJ;GpC7%mSzMqf2roa4>G1&h1bNu6tCYJm$*>@h?
zL{z(V92A$~iXl@8#KPi4C5YbZ@5(s9l4fuA=MpPcG~9{p~E3}1=xE7XYM2~ot4
z4=J-ac>1W=UP4IKn-E41~!>7vI%7UK#)=>0ybrx050B
z*5aA_Y7gu+uX$-n=e#zoWFSFA^b
zEVK#iDDOFEW?FQ-M8j`;{b0qVsVLOKzo`A;=dT@<1AlK#FQ%FTa5o>?$2#ooxu=-JaJ#C#7!giz)@rK~mIa`Ci
zTDEI`QhC@5>m_}+|2%S*eSFx7q{b@^=f;0>j
zo=Wo#DPgiz~}Pbf#4(g4e%~d6<36XybM;BU1;QY
zIvhKPVh`D8Z;HiA@sR?+J_j2I6_MAAW^eD#G#!?1D
zjC%gvA`C%&v=4<|4ktIQw_rljLA}zBtPWH^uD`GsVGGOWK+10@1hUQc9?wvS1`|hB
z9e^He!{slR&;TYI^oi~MwO?TMqmkht_L0KgG!|g@n&x4H$`LJ!ve?mPd{7f4G1ZZ(Fv#*b(vDIp+mR~9hQ40L(u|7j1F6jApy0JxKk593NX
zwwORz3rNppT!8^E0k|gsIMS6>fJa0Yl3FpKI6=5!O{nPa08PIIyYtx4c8A
zHk_`4=EEoEyQoZnT7*03#v$;a5{MP%Y0H{yd@Cni_V({ry^P{?&EG
z1CW+Va9R%6uIPQmyS>1#d9o%2_kK>a5_XCp?dI=R*a|>sMa%!wS$Xt~&rC(p{@xg}
zk79ZLGuGO`gg7(>xE8NYX(;7D+1mV_D)`6u%Aqk7%=XjP9h<^t-qlh>3q8)f&d_{eRp+L1f7U|Rzj`00w-S~2
zslG->e@x)&9v?;YIw3pP9dT=&u1Y9~)MJnWnhOy%o6o)n+6U7dF$sP)V_eiu@4g@B
zm);v(pdwl*ar*4CONgVqFuL0JPWYuvM3YP-#3^^GoMkOH{Rf!I@7l5nmNQICd+oRs@ek#MKde`<_UZ6vB{ucHMCcVJJN2gJ$z=xy(DL71}!$Qi=MkK#3
z6PQ^&4UjyYEYvp8#p{18NMlB9E9Z1(2H*EL_VcG44YZStC3nu;FM8b4XlCLHNxXev
z&OG$`r^!?PqPO!jzSYLx`pQoe8~*Q;9mr!q{F!R8?E@mpRUiWuQO}Q~>ip0kC>IeR
zo1gNl=Hg!D(aiPlQ^n5nrW9|kO0Fm=`~kTdoF(G&$4>cMr^LpfI#csfY_v?m^4*uv
zFWKIpwv;My_D^dW*pb&(2xI;mus0s$n(ay5=4<8;hpmf~oO1#ww#XO@ImIgpz%cbW
zG1LZVZxU3X+Q--yvKNSc&U20xhC6pIw7*tZ_k@`7{Fwaki~sIU>R{8v7`_SU^0^^t
zdxpN5`mL#7>X-Oby{*0uHc$#7H!0Y%Jlo^xUmglLO8humm2s@7qOk@5*jJUCjesGG
zw)?ApOgheo<_uv+Gk*Nm3#EKH+$qxGF*g+>s}$lc)f}pkI5)Ub=?zQoYil(aamF|7
z!1l7~KrsfY5hK6q3*Fr5XMP$+E|b4DD3CWJe4a#s@MmsEnDe2y_e#C%IVleux(aL0
ztYogQqe9MP(q`IfG~>VP-&whHBJ*5j&eAW;P9vuK1jk?)tsnh?6(#KL2#vEJ1)A6>
z5I9$G9B5?kbb@ebRLN>O|8vE)=yk!pn)gh4TkLul6rqdPy^m>bxW%b-GAhdX^Dn}Gl*pHgu4Uf2Cpdrsah}ra0ey1ZC
z>5spRd&(AP{dS+WI)o+!J-7
z(X3Dk_jnUNWpas761Kkznv1D6aTbH$nYd%*!rzpwoD+`R#GK*d%{6Lp#ZE9KPHpN-
zcI
zn26H0n9p3!bCRA
zEH}A-|H%a>yA5Y85+x99mS&;-=AoP)hKUwto@BbAE{f}!kVJaA>
z2Z&#rMkK<+EDvXav@lp+UQf-hFHdLZ02*=3_Y7#w$+@1buRZ;T>WS*5-Drcm4LAr3
z0KoTCz4dW1aJIe#Q)HNungN$Sk@(3P1|O;7a!{DSeQPv#YEV4BsP$`
zxzGg!r?Vhl2;~aUC^$+7K{c!OyRwrg2v+Q3_tIGQ8I7;
z&G&>tV+LVi01Hmn3k?ZL&*@~oh%O*mW#1c2Aj_8Yll8*DNJ2z+-}t{ULK8Y8^b3*D
zgsC5|ZfJHtJ&Y&Zw*bCM{;NM@Pds#VBjJ#b-md~4mwjxsw8N(7n?2fqMW;F2h!x|-en*O)ClzGVYYX6%3)ttKoP0ue?NyiABa^O$Shcc3fh-x6@+*n
z^L&VK{+DouLO1m*c8@DXvkm~V%B
z7Z);2jG6ztYXlEE7STX`mrcL42J|<$xC;POb=4lgUJ?w;(RBb}t#=+grWO`yayiTK
zcnpbV!!}|6M_?|d%g?5x}0EGWwLr${R^#6@~pZPH3G+ESnW>G_k;iOvbY=|fc{Y2AhbX|
zA{rz(1VnXWUQ3PZ?g1iP;eV1&L;UCtbxoisRRrX*Xd&@RLPA2-E;|?3{d4n|9rc9G
zT*!yN8k!vX|AdF~UoMuYz(fBV??gpYTDS)2Ugz(hp29E;x2KC_gaFu+E<)lVz)E7$
zO@YNW(Q%fQMbz~I$#wzc6L3G^BTI>_@|G$n9moOh(pdTS@(K-b0;bZs0EiiIk!`5d
z)hpAK0`;%P7AtW42~RortA0uNNBzR)4>?25>u-YmpA=9%0t(6Ck0vI2Sv4@p`p3S4dMU;?eGE<8q$B9F@S@%YkFzvf#J!=&adn^NviWd
zd85H#LvlQ$?Y@W+sSMh5xvY9Pc=%RPo7#zp3I2_S&7-6Akj5
zRN-Ey`$ezrj2~w(k*x_|ALVb0Q2|J_-huj0H5?ctjdVuy5I;xu$*VuBRT5(DEqX_Y
zKA(z&=7#DD52MYSg(m#@_4ow@h>wp14lc0D@bOf4t3IR0`-tRi{`AK_fHUV!A)6u5
zP!P!3)MBK)n8+T3(l^X2!R!1_s0-oWKX5Fsplmh8(+K&~M@F(8%c|dvAR&+}sj*2BdyQYA1!%dDr
z!pMz%t1A=8N#vN9-NgN|$`s!76Vb^fDL--e_q0zHdwte#*DlRZf_d1odDxR~JpBsYz6LU_bl!-#fyqvPi~Y&cmK7|uGmsTx9TaQ6e|!zu`$i@S
zC8RbFvE;Sq0-e>I3hw&n@jRj$ih;kL4)$@F2b71Gqaq9l=p<~>3jY7>i18TQ{QoS9
z#8^~zKe(*^y+6Jj`JZZr;whC-T&0n8^US(>bH+bova5j#7*Bd3$wKH8m?wPP&{xk7913tn3kpDBzT{pNXSIeZE;c&D08hoi4ahOj
z5~8}B?Ekd`0VD|_e|T{C@&-RRGM1nEXH8t>ev0AzvBwnWeupqU8=#JcW>p|EH`D~=
z5$=$j7=O}vwPpyvLF#UiCo9?6h;33}{1?G=c5mh#0BG!)qkCuN$r(1Y@+Ux20YorO
zMlmMlr(D~hR
zb$3+;USF37^jdLMUZ>klvW=L;Y7pLl9}Jl>8#6~!U`hvN&kF*~*27rc^<90;H@c5D
z8z}TP_)u>QKbdmN>6%#zx{JrGY{?Kdyv*$-TwbhB9$w5-Zfm!^h=gb%{x(jMS;F&*
ztcey!S25_C8Q==9XwIZE+9L8{w^f`BpC}F>h|rS9h?Cp5j!H42J>GWsKJxD)#}%(ngala{OsJToJIaN(
zK3Rr3R77bte?AqO{yAUXMhCiYQIJ&j9RZenAEuvDUR)%GeB(9LJV|`%KU95;tBCfk
zJnSC){Wfh|=XBN6+b4K{AJXfVBP#jjf9{IYAD@`GE<4Ymq29pjoG5A@o+So{lKit<
zaL|>nq$jO)1TOt_pPx>dPbMs2@RFJiwP}s4Y~r=14RJXYP9ZlqI=V~MeVSdYu-m;{
z@odWH1CvpHsOEh_dtLj1DqAAR2VhBQOBCV>?f~PD;@lX_CkWGT*TnzWB%BTW&Yr=b
zND7#*)ICg?>2T
zdb7`6`Zik(Qcduk-ZXnS!qlH~X1Ch011?`8wPn)gSxZRY{En^BGXx8?zW+s8MR)E%
zE&RAyUhu1O@K6dEfpvhn=uveSE?wiGsgexTj-!^{MSOJ=HrfjnRCdSV58
z<)XSXwsI4wX=2?vadPC;w_vs8{G=z~Uz=Is94m^m^tT|cf-nR;cb(_&=Xi}cfm+s2aL2E!>nTKbZQdVpbvKqJ9(W!~+XzChsr7Fodp
zKlKn5?Y&;z4j%(cZOk%{dm+GK1h{ep@24siJM`X>4|D~!ncww14Gvs?$pJG|Y*Vtn
zciR(ozq;qa!O0XH+*1*(u!q&M90Cj}i0ZfF&fg*6YFc~}1#{#^M_~R>(-w#Z`M=n+
z*C5qL%q0uKCB;MgE5_>*7s6j5h|c@!zuLjwAQRD5u1;N7KC4H_sYZ}@DDKO&^s0BG
za?<@?a>e#G@Y+BV}?xu0(x8aVobqsb!@kmeg%Z&7=K&7mENnr@w3Hj9EwM%A|jlt@OWWBs0j&b
z2beTbgZ_R*-vio$Nb0Xx!Wd)d_r(S;FtdTbUyve6?>}Z^r-I(s|27-Xc#{h6nwd}i
z1E;j9AkfMC8J_F@WJ)Aci>9}hI&pKn(wdqG#HoiSp^98Rjn<8My;&p-S1jFq8g8XR
zq1=das{tM^wJr+w#4oFpBt*no)WI6`B@gboV8Cpw!lMVPD?sG@LlcHnlH^=XX{US;
z5Qlb10Akgbpwjq8?}y6>eV?1Ti9_1TgvQlKdr_Se(2d*J4mA
z?$x>e>8e}N#Nzb)nj3#7-tdF22Fiht6e)H?2Fx4};=XS|`U*J}82^kHTX44iBhF+x
zrY^SWA&Z6ImI=SRBm2kSLVYD!5B^)>wumVjW3b{sg5x1^VvykY>4WNoFZ<0>{}3%&
z%+Ja+GoS_Hc6i@tgPy6#r`S``nkZLKJ
z>ERF^njElMP5nbJLlaf%3-6`1ieHPd%9xC5WzB;vKw2e0GW6zM5niz%JqoZD(E#Ml
zVWz0jNc~c~2fKqLzDGkGEeDhbW2>%zt)3FG_y?l%OnU8$e>)7#wu2ag=&3Lzp70G=l#kdo{Q`(h+Ky0(0Yoa-OtPi>o$x!
z&6Bl2jqEHgTr8g*`}}gZk?Xip0Z{^Jy@IGqc)uoN3Lswg(_ZR8i2@<-!y
zJMz`ZgMX&nSA&Af0_dJZaF|Nog7MqTa}KIeZ;vp#gKVfr%U>Nuw8|6JQdp!%qT8E|n?Fw>(+K>l
z7l9xbiN;=z(_P?uxMxajGbr(88E)tos#Q7j$Lv4)={}Re4vNZn6nun2w6vnZuBzK4?>j!%JNY|ozL$T*C%qd+@fDcD|xLvIabzH3Wdrc-`B
z&F`FYk4p_t?y^>EP&Oizru8-HgmJ3XjdFw5d&__8-$QL$OF%e*Rpx+H$2vYbDY5K~Ho~54_
zDidlUsZgNA0TZ}J^x(H3_5MZx%1GcHncNQxKwYH*MDw5V3X9d5sb4B!_;auKE5J`D
zp_Bp!yT|BJu}z49sa?V5wC>C=Ju|6RUBOZ0y)ViXiJPmXm}tOVRAF#?q*T>Ye;@`n
z64Es*a2AYyYq>4c5A)BUH81S+eep~JoWFOJbtY#^b$d=EJ~jYwIvgNYASlb1gXT^p
zx!HMn+aUA#v5F@y!8dLI;`4twfr{)gBc=%f^;(gT*q~<`tLbpHrKP3W?d|OsVEXWd
z^4}b9$vFs4&ZcmXu{xdEuooTyAu1WzKvj!hUk;SsZpjNBq3T}N+%{e$mmTBUJa4Qv
zL2By%?%N>p6LHuse>?^xJ!>U?PY;BIgcAVOy6JF$$nQDU_6Qg+ypj41qb%)of48z0Z<1hJmf(
z#B>=Dr#Q3qE7tk~M&!wo66#dY)FJ_w!=`Iwz;4Eum(5#Tfb8Anu+@7&C7)&Cw>M6p
zU(3Y8GW8u$__npTw-Y8N7D&Vqo@l1~x!v!*r$#O14j}Ks$*0$0b|P?qaQljj{mBQd
z3s^%gU{%m{gu&xNLd)Uq;NI(o$ez+hqC;$W@VLtPyQvNdq8Pp;m?)sXA>^tbE)kc{
zSjdY6mb25z&02y4-uEVJZu_UX8zL3flc(T$-Cu69Krgi_DP$Fg<>~fpV}lF;W*ZI0
zlW70HJEnt3_$?kQP4@Z<(s|*>J1q#($64NFXEoKU1EM`WQ-K-$QwJf6Hus7Ug+hXF
zsN4Ep^mZbh&3qo{sAwSLFmJkDX3KPjrQ4FK1A?YzwmU8#pO@9m&CM%OO=2Txv`^I2
z&Ay0u95xten37KbApW%hH2A8cEy|?{ICusD<#0VomyBAOY6Mkh;wFfNZXcdKI4J4K
z%H!gbxB;WA{*jt`AN`?&xC;&Fk6x1+L86)b?O|z$E5e|OHT$CGhpvS#>egFKrLwTF
z$d7gw$)w6t?kNIzq5;u&*vtA_zvEQmiYp$D#3;%mFK};QJ1_YlTKSwoPHpv-(5JP
zB9vQkeeStN?K*^!aJ4%R>S!3^6&z*&&M1;BEzW1U14Ul;&yM$dcI*ILP{-`;Qp%QR
z^zmf+>GpTG~SocF?1`4)B=gJYN1D`D~IM{$ckQJv;kc7A%sBl6?={NZ)a0ei2f__o~jR`GV70;egSW)ce#r>Z(
zDJj=H50V?Pqk%IhGd)wvM9UtT%GH;|h4d&(`=CA}>0%C_S0m=bDS5yph$e+0lx7l{
zaHfD>fpMXk&Ct1%cIH6h4$$qX!21BXSw`Y0d?H#;qZlh`xI44)uV^T82r_c=U3|0;
zlHxFEaUgrTxUv|WI
zfTcH{%j0sGI63*&x7Sdq?PfJRhSHk5$d}`aEcC_xIYCTsr$JlnJ=2orIeh$>z7MK9=yB08ybu`u`qEH`)Ox2Gk$f?ZNf?Z
zsZy3$0MM76&}mc!|7bOG#kn2$M4A>)NJONc+c4MV_f~YgjDpW~u5jxGsP2C|v@8s-
zCq$kRWA;tE4+qt!5mSnvcOvpJ7H-RzAdL)TOXVF&J#)(V{y~BX@~Om8>gIx9mERe~
zRQKZ0#DlCuJ@!0k$#^qFSn2e6vn-mxKFw`<>Km&P;aPZWw9h=O)5#{75la3b8BQDy
zUgP^VM>5i+kMp!pykiS>vis`g&52q
z({8)jR_B9DPA}!JiHYcat6J&i0gX06&`7bhA+{3Y;yLElt|CP~*&fQ0H#FkFO+sv)
zSldR6)1F`b3UG4cADllfO}$+N;A;B7D`(6t1iWhd?8X;4u8vWy^+$bw@XO_Q5FSH+
z_Okgs{Iz3rjto9Fiw5UzI(gCi5v^)tOnfCH#{&N}0PZ&g`3kK~%4s_+CpML{y|S8H
z#Y8F|v$?P~qaDd-q{u~9gZp|0*y}F?n~e$K$aNic7&AQu
zBDxQ~gM;eEBL~G??i3I=bZrHuE}ZVo?`?m*moZS-y}8rX(WzZjg?lj`r3m(O7x{iPPpI$y=DdSlv~}WJTj=X*)6Ag*@Lvi$fK7E*vgijS;T>=pENl`k@4S
zcwG+DHSBv|_jdMB6=JybJYa(gEDKgE=HUk~a_^57Ly{-27cy=c3i@boCp*Pjofw{T
ze$}QOkw?+AXcX}1q_Dlhud9xB)A>$ACwpBBrRk-P_BZximg^hE-@VUAPAOC*8knCx
z4#xvZi0Vl@fI}A3DI0+vmcQ`xEAP_1tpaVW2KZ_si!FoM}e%2H}^G*RKxP&~rguYr*|V5AHHG{rBIe*5;(Bgy}bYq)Li=^El>M
zzKk=<9_pdclpZXi=lY|-eC(|`YchDnJK5ULL$w`=-_bZVS}rrzyjO=Ax+N+$Hy(;b
zNaJazVXgb>;NsdB$(Ks9Dr)FHkb!eG<0^~x$e(+U8sRp>Tp@yl_%LDieCsOl?KTeX
zsr**&S20un_I71h3Dz;)viaQQdco~sZc-W&qN^^9cL!Q7QiRw|A!{
z?D0qK;LE@tq;u!|`1Yol!DJb2q!?BF_=Gj!Rzvzt+@T{f{7?y7n}035b`Jc@-8RAO
zsTy{bsm>SYEyL@yIhd92c0xyP%Z4wzF8l0FOxxJ_LL*JuJ9jwQq0PPYNu@%Ld14n*&P2$mdKS~6MT2eEe~z^H!!vr6DvnkKJp3`!1O0P6}vs1_I=F3)6+Y8FH1N(
z!cH3ch<(1U+$vwu6)S3$Av_6e0`aa8h>L|90i|NMdc_5>f!9xwc5f#U2$|yr(pL^P
zS?Dkl8WOuEhxq;s2!WQS%TqQeZm*^rzSFE)`s<@?4^rYBZ)mvd+=oYBItn&8HNvPp
zfh`Ue;ojrsf=QW>zmm(=JJZf_IPO^aDGHO9F&6jjXIG|g#_VZJZ(|&B7ey
zp{GpTM=7{D+1|ijp26g3@~b5nqv9M`)|kJ1^(lY6Jo;jEA@!*}VK@ZVHdjkHcf8ll
z@(;#~W(mB>>csz3*;@w1u|{2^CxM{Bg1fszaCZyA-QC?izzo45xC96g+dru60{0s|G(IuOz*=F6Kjl}CcC8yM?!F0NI`!6
zVH&d9t{bS~B0UR1&y3zEN>?b=zjc?eyU5-IBss~86OJp-@ln4=SG>A3yx9^R&d!`^
zZ#}zo+SU73joYFw%z!)yTTi!B|=dJv!xKjq5d~slCf~QNw?ydyPW9Sb_#)rqf-I)
zB5ck-=+D7l+(k1{;px*4ti#M_Av+z%i%}J~&--pfCDVQr6FXP)9nDE6CTkwb6Lk5H
z4q4}A$Yt`vu2TG)k&yX|ubKmgf}NY5$Chv3d}J4@5+dXC&iM#|c-*6#aASqJo#mmn
zx$|Mz|D-+>r!pZpps(aReADx~k(7aI%jEq54<(w*8loF~O>R-LrzeMMqMuQw85
zItx^KQ4DH-?`@xSq?E`FehVS|7xM1Y6;2Nyc3R2;OH($~h~`+$b%oulwT?C+6N1D1
z=Y8q~S0+wuRw|KS9a(arhxSG#R3=DJ^<2yHx~IUlQ^GOHX;;4Y$UXzLyFV~Ood`>l
z;NNa8^xlu*s{aw8RM$4#PEs9Ttv!~+hrQtjOHO<)Y_OHOLTAEzpBxa`(kJH
zIPc8%QN{RV+Vp%qo~NWv=JGWrEg6t@0aii75Ek{e^^7S-Q^5O6JiPJeLeD?+WbkSP
zwF$QAZDfI*{rZ!iq!9}m&n2g8!93-+ws;N(_GLr5)eVT6nXhf|`rCW-~8cw_J*5tRlP%TQ@=!Z5?=K;*I#fk6dy*Saa3FESTLZJ&Y?_}s8WFzW!)Q*#Gx
z*bA1lS$^J|LYk62n#h9w!Rejgr^s(BZ>reS7QQiX>*#K~sr7pkMq@?PWcTqCgQXdn
zm+FTafRg_EVPoN;VA)R5jpxvcm#6CXr&{8_F*uCE(!w4k0#D$N2-?!Ve)n5*ZJxTO
zR(LZ|drVjd8BzffC;nP5#IbieK!4kf(hMI(;K)|ZPegYyO+Z3=3ib8*rjTy-YI3a@
za5Y6V7Ok~MWt)w|?N1~{D5$H4pT0k+oDOe>3xf?tOHIivot}%ibvsX(biZxd;|86n
zx6401pZGZn`oj&{pl>@_?UkM`jvcvrdWs8}_~f1+_uMUIG;`(3`QNZ_{Rm$pZ04y8
z5CNr(izVECQ+2UkR)8RR`?U%c26@tY+GE9kI`H_7Kor;V$4yaWiJS)7ScrqWoesU?I_<*i#krJj*i%AeSc
zuqkZ`=~|=@+Y!oaUwt!?HGYfY@${$Y^`um=EZW^45#)z35K@8WWVttotamy1P49YDIg4
zG*foytA%g9X|GX`vHBZ8F!AN7TlCF8B0DxSiAM{jW`9FeJP`_hL=|q;Je-5lN2TU{
z6>Jk|?N@2c4^fx7n2bs=L>g(#B9vWRx8LR{xz?5zlS`y-0*tsAWHvp0vrz#!94GL+
zt~TMvFO`vl&mO}BuwudIA`b9mePscV>RaW
zYh*0xvaq6k3oQ0<4
zpg1|_-`9g3s>By-wl0?8W)HJrK{BIJqz1Rzp93E_XUAJ6L|81Z6)>Co@p4hf*}CC!
z6?GZ!xy~w0k5-Bc@2FVth6ma2e_tzweUiky{={Lsw&J;w78o)>E8qbxwFw8wo9ivA
zakh`WU3c2QGOZxM8Tgeaxv$Uo%f8py^w_lF8nX!T9aW?{95y9PeVM}rz8SN>;We=(
z?oK0e7=xo?*dDwDj9AQyg~b_=S24&W0zNG85*gn)ij`@K%4^TIvA-8HpBXx-44(s$
zk)$IFypmpziS-*a_l510Gg)#@TSMn>Fb(@9F8P;%Pl@W#K}Kd%TKi-u_;KyO7#`=^
z7pX03u8b+2KMg5IH6gIew0GoB4drfe$7CjnLGy@cD0ET})Xy#p{~N##ao^Ur*Tuu}9VGRrb(9hP~}dFQIgNekF`mhOZ)Pv!>JeP<1@`tvhJcq<{8
zH&@iex#ugg6W02XwU@yg#zKtI5B=t+jg_R1Us#K<*md6I)*MCrR3qBhn3JM?7Rdt-
z652OLdCHETpSz^OY_i}HH}!Uu(MX@!W{V$`_39YyHiM$(@lTYhjnqR~tQNg2)@sr7
zv)oY4*#A3UT~NxWL{~9Hx4}Fr0j*)96{*J@)uu}96h)rA2+F9O{#aAJpzFN$IG+dJ
zp6$B;&hJJv_TYPJW;tXZd|*GaH#VM1aC|!MsPWeE{&#H3w39BH84WYl?0sRWpWlzK
z5|gsxSA-wlu7y0wwc~x%Ao$hhJmj3z={Qv!Ck|#
z+;RpunUU({{(-xIchwDn?F>#Ool^fkJ!YKdXB13#LS~KzE|!)Rb$jXv=tjfRo~6u-
zGneb(f*M~H$5L6~hU)*Xz8qSVWo;
zs90Ub)sGo1pHaE$-Pix@8d3-i9F}YF{1ek+((_@X>2J|D$b|azk&vLTWi7J>G1*`G
zLUXt%C-=%ox$Ako3}xx3&DO3I?c%(vgeNUM|8`f)_LVqhaUDx%kuH<+*#~?-CxBa%
zj!=WxfESGdUp!?n0%&{t;n@%fVz7P?a&9kKWy+hg7|LJCz%YQg3A`h3B$>!S*$uCc
z9NI3xZtTiYUT1$G!R&}mU<Z~&_5wqGF+T-NvwxiYYzPSD8
z;`C_9e>LrLBUGI+VqlT0zjh3Se?YX*WPgI&SWOzs7FS`q5?UeemyyeEbbiTMWm?0L
z@Ht4Q-fi0-eLsRfUk}3VzY~p1RD)Qgm1)H|ig0#l#?)DqPhjS>sJu<~T>
z6T6UX{HBCHGE&(H6B`M~kOC8%dfSgz-rq3tx@bLJ%2X_XC{L+{vg@!6g*ky&=TloXu4H!>!bzwwya)1>XdL5>qN
zJEjj{mXs?`@g*WQTO*?ej*R(mb-hz!lXqm1Hdq#X{$ACtrWACpoKRPkaEL1
z${+2L{V~w*H{pi8J&!4<4ke)s@2P_q%t78GF|GA@z_*OTN_0u5rHXD!bSWf*
z4W%iDVhGJhzE9RxMyK=OyrmIllYSV%Ws`>gC;2tFYEyG43n9rSLBDQx?SaG)fuCJ(4q>h%sFXuksm
zG9mKolOBYE5W_I#mSZFgV=;+Om7ChY-{D+I3=0~!7^v0(I06T86GBRMEEA@Sh1Qxpugc6$^vGT<$xvs$>lk-G8YH4+?Xv>+}{sxK0N_iG!r?&_Whe;!slq!LV?>H58c2VxU_2`m@
zt{@bLKr;DzJw08)H+@NeNZ_(=<=!n!!3*t`2|X6O+3qI3saYMoo@nVEpTK>3
zCix^}qpn)*?Dsfn!I_(P{dz_bH+MNW%!%A(biR3x%?889^WKHtQfxS)u!5TR1MHKm
z1rg<)FW0+kns0xI$6cKsw3bCe@N-_42=ycd=<}DWZfRzGIp5i*65X(^_H_YvWMW1(
z-eASw+(+I-&rg=jZBH;bLOa6KvTZnU`0FLnwdfhpXQ+S|?&8S1MR#c1yv~Jp<+!1l
zS|DO}5h?8<+J7ez!%#XerQkH07CydNZldnU;Cu$f3xD3MR>3U7t)rjEdy}0qRBF~_
z6E1IJ?mc^GjEaotyuLk!-DaU|;t~PpalJ9uOY=VS7f${-Gj)Ll{G3X3--?INWRSB;
ze{B3lSWjWUrtu*C3}P9dI@+PT_(xgvsibMu{NOj=zbbBQJxMj7=MO^3AR%X>9V32}
z0?)j}aTo489-{*jF>*ipJQ+MR`CE1?e^f^)gGFcMXRrQ@&c6gV;u;0-M{mU>6qxs6
zF1P#gwM>{Ui>7{6v8d(s0Mi21ud-0NxBF&PTKlFvWMxG9#&
zS0!78Bn5QxzlN(G{~Q*wz4zwUj7IAX%|xsX`4KEEcI4-a;ENK(^k#`{|9;9h#`fp1
zm@q}l#FC^zb(?6DZdPjngv*|Po8eCOb432A>OZ>!Igh_5653~YOp+KB|7*5Tq$P??
zQ@bj|iQ!QSA6)#_BnpO~7-qB%?j5Fzx&W;bfz5HVMbshjT%
z?cBu|t*Hz`f1+OMtC>E1zq(tDJ0y&q?crV?tX0OMvJPGV?q{}vq&c>9O83L>SG8Y(8vWk%Ro
zhv}2*u2Xq9q=laPNq)SdyJD=o%;iN2vnw(J1^v&B$9;oE&E+uz^WcR0Ty0GyIMejY
zi3TZXE@w;v3dfTL7ASC*LFAVY^mG#N0t+)l+QdOe49$XM(cHdBFR~ewkm3&dMuTh_
z{j6|`;Zf{DC7gFrM#GdB>5fWyel^A>A^%UHbj0cl?F>A(=6x2)4s{oe|E)VpdSq{p
z_hVHFXdtsFKK!csxOz0v6u@){DTRNCY>9T2rZqWOq?HxK3qb<#Ae%6+xyJRI9%*=I{jq<-0hHOm8yT7$KxRpEDx3tyLNOj-c@K>PUNd1A2r&lCADB6=6Q+&P}=!=41x^b8=raD?(?GM|r(sGJ~w
z`8VsstPTLE%85XI#u%eH8-Xh5BREwpyp~)(GI1Rkp}b<%Z-fN_Ym2ggr8!7Pzw-Oi
z0HR>GDxkKi02Dn@3JMCi{ri9URkpy^mY|T3mj_^WN#X<0A4&nTMyEE(9w0$8bNCR2
zWwng+TYVnwN-C%B%8>3Oopqs+GT#h}cbaGUjK7|vXpV8D_7cJoV?(N9(T_?3An#Fn!7U*EV4fxTl>{UfK>t`f4^=)x;wHdtjWGdWxw5iZ@_
zE*NxIk$6sdQK*YIu?yAjRtG6ed`S57#uAD|hh0Ke%3sw_d{$eXfXffH;s6?wj=83I
z8Lt>PI;M;;KUoqb&5dOz+;T+5UzO!>Bar%uE)<;od|S$ik*TP^N2i~dzB=unfJzMC
z+emR#8Z5NSLH0E!jcX}wyd-p}-l*+s6Cn@M!zU0DcPyER|4%R~et84%OEtXo}!utxGG7{|sk;E-Bp~0t2`oaft7Z94vqcT?nu>Nn^Yu|}RkV71?
z-%Ko*wsoLuMw;C&V&TY~?DS`A>rcT_uBfaAdu2z9+5;NZ>pN)V+q10$L9N%fP+b7W
zH6rykzTdO9?M^X*)7R;n4uAzx(~oDm~l|N00z+snV=_wY-K#KrRxXaC*+?%F7{jk?7(Sa
zwa-QlqliWkkYHJ0l{p-u-@-fGj1ha3zAvBONOC_i$n0P@QI~d4Rht&dt?XTIkmQA<454N{wf54
zC{rR8;D}3tkUKg%$y*^1OCGx^%NKy7A`}4z8!Z4UlDMT3B7rU7$JwGm_Xh)Z%s@%+
zW@;ykhEM-yQM&C6+-oj(*k(%rY#l{O@6^#OH5qanq~m*=z>aib=*B-FIstiamEg
zjwl>ID^OD1>AJr+@Oj!%|3V-I!*?sB`jB7#e3V}oTJ39Jaw
zOw7}aDe%O)K@N@#$l=}rb)giB#L$v1VQfl$R+!UemYKg&@EAjC@!3Iy1Xa;UroCJZ
zPlFSo^+IjA81v#g)ZYxb5>mpj?~3o&i+uIYx>y(uXuMZoe3gAR&@2#=P=RiV^{wW0
z?{|mf+T9%<{^oUpwG+X1W<0TOi2Tth>dn^E^V}I<;k;xPhyo~d-KUiINLoNyoet;T
zeBBKTyE_3d3&ic+t8q(%E(Q*Cz|9lQAm~MSdXpJu7RfU$0Kl7N*Qn7VZ+)AtVjcu>
zS|T;x0UG~W{C;jl-L%!IMhXfQ%|vQ~A$%drNEBH@Zkm9U**sl;r7el>Vv)Xj^p47n
ze@vzuHolp3u*bzk=y5ojrPmQquCYP87_JGh@Hg2g%1vMpe=tzK4*Pam+Y
zH`Jii{c&+pb6YIFI_I(zG9nuW1NewGB-`PhCzviInI7x+&W(w(wIR}OQ%mYZ7Z>#vEBuY$wTO4_!D=jf0hmR+5*|vc9
zKux?Upaa$A6IuxvbF+LqBb#gn(aC)wN4A#5bwkRObqv;**9;Lw21
zS&6;;mu4EP>`;@EKFycHdaJUE{oZ7dZ)bEFE^Xsl>sXz%}k6
zY$-mBfg0uDoq~R7GwS61)?8qZSod9^aq-6Q2*Vl3?Tw7*T`MYHlL?4(@W=0aQ?l2N
zpK{MQ;F|pBH?3+;AZ&Ki`(_n){xkV%vNf#1W}Jo{h8&&;sw@ZHZ}~8mr3ZfCPCg{8
zLh2q6*!zshyDa#Ft1q~rLf}I8ijfRG54sTLgAEFB>$&-!yXpO_u1k|y@+Gka=WQ&r
zKX=&h67u<1;)z>j6guKCFJZzZXJ6QuY*-?6RxPfM6(x-;4am=l2`L1Wyr1xyD>k9v
zd|L`O!+$n&aTm8}rw)48mtoEMEuS*PzjX%J4$)I^37e
zki)F6clcG|mYji=%jX&)@gmKQd6qG>TNJu;Ght0`SW4vyn
z&O1-)u7ue9zj{y!1h|jb^KPnh7qnDf*FI+drP~EUua>lx@7M8
zA+-v+9O2>hkj`g3=Oos3MKI0uar($5(4)}S#C;@NDBUV3ov_KK8ZZHrmN1251`<$_
zIlrOeaBue>*T$hdAA$5_{W`_^0s8w26nMX#kSdxF#FUYn>r>;Oz3`m*pH}y3-jh3y
zdJnvLjoOdItFHx&wr5&9x7nFRXQdu|oF9p4@C>KFfyRqN3Cw|}c~YXA);tVk6Fzo}
zt{$YYV3ji{(@S{!XN~^(5u#+2{azLp`)&j4Zwg2;2Ig{*^kQi9g&rieT)M)(x>RSF
z5KAH}FMnc-3?Ezu-%Gfv+wc?bI2pqbaT7aqh=PacbNR-xqhG-GnK-6jB-;G|oJP4v
zxUL{k3J)IJZLJ}}SK=0CN<)GxTBr03S4wLg!Is^U*5
z%+s&P@IfMG>XEjS?4-9ux4us+1@)h6sW-J{a9RML8Q4F(JzV&z>MVt_vdu&f)t1!H
zvi|~Wejx7q_nr;>?J{;203hw^;U;-Qr{y(xAK3cJgRW8LQ%m-&(?Zt9bTh@TLDKLg
zSt`Lw7O~={oBboX<%k2ugtPgF9MtiM0$FAf4x6tTido<@RmjnTn_g&k8A?B>981es
zd^2Tu<(9d=~Ik|%xzrpIa
z55cIr-IHYT=qQuTV+&d(ivPr$w8q#~#G7pxkovab2C!|NO|!f`(Z#k1UW{jy6r;4_
z@pzZQ!~^sThBL;^@5}K(=`ZbCH(41^3DXUAmV<9TkgnonE+|3-vppQxjGcu|Ut%y&
z3P!?FF=L&O)qLu3KH-H%s{*I9Kj(xA;OSz%#To?kml2s`MsMi-RCJTcs^uV=?A34
zM|XOEw!^4d>NzAY`%}>bR{r?vhIaEkd?B@Bb3%|8QeQQ@&lyXQ1IXRJXg8~tCFlc)
zf!v6UCt$9>IiZD;f=ZnKq}>%W7=y%!#4j%>}Z<=1t%
zx9*EKiT~K2xF*1QGrR(>eyRbWboS~BT5`TipgAv^=y+~x!#XPOrSHf8j8EG03CwOb
zT)PGkjYN9U>HA;&2-u&0p|%Gv|D51#3z2fns7H^NY!}G%TR<2XKpGWhOMLjJ@LQg8
zfu%Z)Bz*fYGU2%x4J%pijZbbHII6*i>bc~V%w*?EFIkW0lO<^}x<)p~5#DxX20Y>|n)o)ck>Z9w(hLLEuEgrhL%vaxtvA!#s`>u<+sSBhhx
zM{Ruzxv*eK>@v|}XX79>2!zMLChL*8eOlzw!k(^P;_Ixv7z}YcvCoz3Y_gMU`bhca
z$uUh980Ue#;?YZ}DGH+0>RjQ_fRq6{4akDY)?{^_
zrw0(-PN7aLel^%5NPJ@M9F-R3c}#uyn?sk+k7X0%0V<&S1%O33dhkBLL7+Kk1v?l@
zKHm{%(zZ$r
zku-dkNC+nkzd9!QY?|&GtmScOz7cqTMhr7~7Kp>y7)=#1W1bE`M}PW-!NYWCb{sK%
zd*e$LvZ{i{aw_*@{Ctb@
zKI{dwhN^M>;Jf=IszRP@6JocifLHs{_B#E9QB_PA><^u4Xx=&cm4X)b>qF1bQUgU-
zxP<>i_8D@Z*>Ye?Tx
z60e=jAJsrDPb&-^hvU1eL&N6XZvV*q+4fLTFsmOwqPZq~8e_kh&GupDhnlW{IXJgz
zGn3gns6~U`&~PnC*(Xq~WES7s?9?Ffa$to`
ziVA{ic5=H7GsNpIJ3spVX&=uJ$w$L;7WlmK@@L%S;>vS}3f@Be!2Wi>hMKMh6(}5W
z)b%(nCyQ?`J-vWN7r%=miM*`#$dHaqSgX+KJGZ^ID-$j!wf=nBP2Z#c&;
zD$DBYu3@jW9yM)A$|aXvPY2$~`%*+c>L#^9_hX6hif-OFk2cDy!@LPUN?^*du$GIe
zOOT?{dw`oHII2{H6k!+xMATH}^A?FiMVG}-nw|C2CX%{+cs1Kf(K7K5w7C0PrZGEL3V{@dSUNU?m+SS$I!=>m
zSDDN#qouDH9F4+HQo2aWeujiVH?0tIv^El
ztpJo-qkDkkiy#HXy0O`{o<3$f8K6_W-zmupCOX;Q&(izatQYebOY{Tal(iPmeBewJ
z((0?tXxfqv-m=glwr{VMu#xL!?i)#Fcr`!if7V;uRo1-fyA}5$`qw^+gX8okAauDi
zQ&(39tH8r<;2u66Ew!A~!y%&gf6V4RQg(4Uk$ZlAzUjW*jA1O4OUR{etUy9VZ9kus
zBM)-c-OYCC^1A`n*N3hJKOAiXhD9Piz#3}*w9Da_aJpiMdHsF5_$!`pq$J+gJzlwSBA@So0BYRSm@~Gt_e~Y%--A2{N^x#p_N|4
z{z7?T!HUF^pL_bSqUjVR3Bjlg1F&4UXoCdrK!Y
zqW8ZOqOj?o`R%5@lV{{8rLholc7C42z9i3oWeVNhT_!6O|<
ze{B%w!@zx|epC5(p^9t&F+d3P;V&+CbGYC!3+Nn`lt+k_HQlE-KYi^KD=0W&DhkG`L?l+BYFwJ06pVqCE{#+pyV~~yq9jLzgGeb27_~4uNL`%
za1JP(2tUx=Ud@}BOh&)1(r;Q6a9)OF0!tGz&Hx=_C%5+@EYHH+9DV#RJ_>w~EFl4b
zo!{w`C}@?A?(Xh)VV8F3b=M)c8=n=#o^QAMiFoY4S`<{s;DZ>hY!PnV>n(ND)%xhj
z@YTKbGQBlNwxGiQr^61EkZxlwyG1v|NMn?Z^mC6)r9DKADY14mdXLMK-SuhRezMKt
z;pM#)bFI;t_iV__vFaP^5W0?e9zgkCZg163|+=YiMW$u{0F{
zl0f>#C1Yb_JHRfiYNR={6pM(+pPH3*JO&x<5GHieaY2fW(tn77f}&`qg1NvMBy_oi
z-(XUy(%O8mWctSsAK$&i6zHaPh1_LO1-$jbsu0EoM1IqqL4zFuKlwp>I$P(pu1>TE
zXR~mBx?C=^p1!^<#tUzL-&0eIKK~CO--D`_E}K>DL!nZ?i`{_6HJf{mgg126KL!Wq
z|8AP--e2o}Q5Oi}GP?%e`=-Bs_ldIs7vp>-oRbnCA1aSR;SisPgQ?j}x*W}JRSGVt
zIc-~bf$~8XO~Z=y9Ks;^Mf$B?SZG`)nPn8I`yv-5jxGtjhP5mjG`Lnb^0Y$r4Ti?M
z)62;qNBvVmV`^<6at_Kz6|I%ebY!XfYhI^&y5R*$2F
zKxMVQxocNnlZ_v5KA2<&Rjz>Rcnte>wv({F%P3;q>T1jV@x(ahaID1D(_Nw>I2n_G
zX!t?2%Zacp_AV=Z)Hf;GYON!hGLJR261)3O#Zd5`{g}=e+P8!pRz-G-fiCqxc@ncY
zu-NMEh(ASEOi>VQuv(ICLLr!NFFCb3c$0iK!A?$jb<8?*!toD2hReVfV$0-9r{5_X
zF^k`G@lTC?qL*`Po7k*tj5X1A7m;InBdHbM&JWiUt9w=5LEx`rZy|nn-!7Q*&Tgis
zZomwkzuUL=#Ym^4(Z@!=0e)dZe-&LfZC(d$Mc==}#{uGbvi8u~G^bzRdexe1kA{YZ
zE*!^mXY5RDa&gpyqSXN3g)PU)W5Dk1j{L|Q&^}?*Hzg#fl>Z-DmxYsUNReXdJ6Igo
zl65Yr`BBUQ`&wR%;d98(24R~*_s=L$=pw0Y~_I1Nk3`%1>BWyRg9VJz-hZD9qiVC{RcAKAJQOTh3(MW{w}euP5&
zkdV-&s_@v&XB*vIFWz_gdAhpKy~$=`ew(79@$O4zE`p8^yP;4kA|($|<{^ElI~pY5
zp-}6EPxVI)Np;R*%;g{k?Wm2Fu!Q^-Lg{odKDlPYd4SGKtn=1$pNWkzTz3c;Qkjo4
zRe_noEqp{qRo_G(We6q_V|#|rYwA4IjImD+Z~Gh=W)o7zClc}E${DQqgwNy!LIpW~RHu=`zc$4VTOn7>IvBMij6gPHSsGXFOvg`
z7eRXh6!6zkto@OVA9%b?MtZ?GD8(u`Vgn*hpuYswTv`@9Dn)8S6|%E$$!yhRzeUfw
z2b=&P6=X!=gA(B_C>Q^ZKPxdDPA3%!!&5v4iQY5b9S?8)AdXPRRzM;pHUt0fwCt?X
zTs`uX_2!G~1q?;#CYOyV041)ccFc`&=4DeE&H
zpd2u5Q65`1pYp@X-6f*Y>0r*@t1EacRgE}}vwm>K;eOKNBAxI-4UxW&WCQ?G$pNKC
zC%ye)N2&C=1&=;RfQkwB3M+Jg{F`d@|7W_wMBo2E6{Av~1)yS_T^+r>EYdaXxWWWT
z*>zXCRPRoT&2&E#5uE3Z-yR*scO1IqAN~(Bq3u3m@U4@BMhVr=Wr$0ik^7R(o^ewL
z-+#d4*K~pZi$9C3V}
zCevxNZzZ|8+<$c)lT@x=afOYfhbl5$3~Y}_#-cz!ZJ=G=Yy93PUkyPM3u=Hv)kYgV
zkh-Oua4=ZQU0y^H5u$OjVy(-5CG|>DtpGx#8k+&_&MFaDVKDSOLni2*H?L2dz@#SIoan4=yFB@u8h($d7O*}2aY2Ls@Xp%
zmK8HuCpKU!CtMzVjoX$Y(+hz3@c1VvrquMi=y=jf{QrV&ul)W8Y_ki43kZNvP%|Uu
zj*0ZY9Ya6jF<)ZZkVN=jkzR27zBU?%z0^xazS9Q^h}TQh06;g)%W0=f1zY@;_#OG|
z9`$}Fn?zzZIp-=;O?wNRc^E^WB{EhPYhQ2os_ER=`r7O`R!tXH3>oXo=ieF8wo~O?
zvRz~`(W!y1t|wj}^4d^i=DI+aB?|;%mIBtSbKg6KgM^WNG{_1oT`#zHi^P6)y1Wk?
z@iY60(azyBjJXjpk44_ugkrqvtb4Vmb}0NT`t>U*mXQ^TH5)yh2A75Ib*;YB!
zTe;HM8@}*I%MP!%Gbirai!hFwm*o3!`(9W(V?ynDO*nx_J~jJv-JQ*?+cna1+uc9i
zV~ep?2Xq$_&p$&{3e`TqqaU|})TBb7`5G^wg~@RCnna%kPzMW~J~MpCg&@V5_o=bL
z5W9aaM!!)q>Dzcp=XC$&cqnowAS_~Zwm5o$NrnYVrhj+ZT~xE#;@9rBqmJ_IV((*V
zy=!{*a-DDWV6V!^qjNs=Ozi9R>$RiEId9<8vg_VY3&4PZK(=Epb_rnb=-u9zkhx-2
zw_IC?oLn>O@<04zOokt0m*>>CdByH#utwCR?Mp6+r9DoL9Z~(GxUdGgGL3c8QBOu{
zQW>!a0(ZC6gQ7dQ?#)3FN!&0Bq=NOK@vx1pV%Zi!KQY$%6Nh5!+Wo2ufda)(;fKx9
z(Vw`85+Yn=ZNpCHrn4n6ZxnP^yiayFWWAbiM21qvH^E=><^
zU+KO3#zOcvB&#)MFFqy)6)BMT<7fPZ!g;m
zLwKToZ4gufO@$r*(PfrG*+lZG)E^fUb9)#ZpQtJ(-0=dBJbz4G>Hn>IVxlWVjbPnm
za3NX2)om8ycYp8q?W3nm*Ae-8)XZm4ux<@4t$B5&?+wPW$lO=@_u>2F
zFJF6(No~$X9FDxKj*+cC=5>T3#6cm-)y_Yci6xaZk5uzbY*wRd^62+xZ7%F_kr)o*
zK1+7*gg?30e6c1{(9BUhJ@?!SLP)*n-5T_JwHOiW*b9y~dNJ;afNb%*y?@2Jq%~pE
zZ7g$p=XdV0c0J*Dxa9VxZ}2@Fs`1J3aV+A^G+4Arr2wZ*E@%*AP1136#MRw>Gl4D}cvat+$aA;u^m2%ae
zzU8SE)_222{IG8Z=e_QyXQ(?VM@fCeu_fGq51H7^X`XERDAR>Gf%lhX58RP~7{`Rj
zM{`i9LD%n;sE$ykY{^>h1-nXJPep=t=AL2&**Wz7%7M=Ht-W|clM~tQc+~tcwfwc(
z`?fzs`+>Z+bA@!ggUjUC-+VqV82TcipUIr`qJ1Hrq*`l??tmcKzjx7y3m|#>=Zc%k
zL0i0}+Us-yk-5%t1^yN>WuvOa7CTC&`?zO4SB$D&TgLO4C{9IGH=4F$t^cL79o}d$
zC>{m(X=8sAo;L5GEH2^OR68MOZ1%SV$rkP3P7cpLJzx(7Uze$OqH=u9%!oz-b8Jp9UR7LH>ld(ZXpm$Gt<|1@Kxi0y)V#5hStYaO0@m7&A=8>qXFw{;78&fHq;5NBm$Z**XMz1R0yp$oYz_!)ucRDN-QJx%jAq&gF+O7(1bKIpQ#fN0i1
zvu7lPZm0b?b{vuJVDTyF%62?W^}D+5KXcCHc@J5|hs@ae=RjZl<6zNlPB6A+P{nxw
z|4BRzEemA&JQCW*h<_5az_u1pbK%Vq>GkF0DIJsPyc!!Q_CUdkA3b&m)C<7v+`rhS
zzOb)3r;*2m0>mtXj|&u7|LT&?Irgx=d~J6F?29PbRf~c@$xlmyZ!hBmIhn?MjkYl~
z%W&r6xx($s(IQ6ty(3d5QP`JJsq*Hfg4rWGDslX3px-*m=C53Iam;fWP^##|EROGX
zl{0@CX+*)$?B*xz=y@}8yUrY?mAFt4N
z%m)6MK(LTH$W~1!5mI!y>d>yd$LssI3>OPRk$U?bq%g5NC4)PVuh>fqLZtaR}kdHd7wc=U+b<=;_4fzw%3QhFS_^t
zs3K?_y_e3}{UaVdP8Zi*@X0eC0?|?fB5TgRR)ks3Bo@%T)g~NZ{r7^<05yH!f*##D
z-;tKwwx{~;FIJsaR%&9Ga$Xbv8$G-c{$mj93G!9fXRs#ZS>HXkUM+v0ZD;G;Fa1Xn
zZpCFK;47}H0ArP++mdY_j*GvBq$nEoWBhh0Zn+X4xtuoYnFuGuZv4cgTE{NVxRmK{
zVL{tWz+bQq
zqHk3YfyWckC$w{!O-5=*xJF&#_<@YO%Md40KF9#
zQE^Q@Sak7JQN0s78BP24`QnT43r$~PUqyfE<6&r8i7@~2#t9>l)jIl;&O}x9-=ZIi
zDxtZAso`@&!2YxnuPNhv<1jT}P(XeC^5*jCDTEN|%R>2RHW7*U3j^1`_rH+Ld#h^x$E*L?JjUlG)eRWtfPH-O{4TB1+A(A6DGeoIvmL(2Im>uGeDH}
zYPrIfc5NnFZ>>9B182*as@g#If{Y!zm9q;3!mC2^nD+5SIrY$ICC0*8LOeuh_6_>c
zG)UC-4kbU+ZeG`H>YPx^N60;oS@-q^bCM{{(c8XPDH}sMcS%Z2gvr_K46iix3C+&{
zYv=Z0S(b|e>$&N9I7m;L=g1@%eA_K}12--5jQ%^!yhbtgCEL_G~WlLdA*s8~gFxu7t70rLePv&;umV^qY@urad6~mpMEdGoC
z`mHL=pOje2Y%uNaNXmr^TyN(1?zr8UEU-SrV?e@p@l|G+i-fwZB4cJwA$vDG)_K$e
z*3UQ5#A8LFuF}6AgUCOez(PcMBdV
z7TzqpyMj$M`Hi}fwg5sbuwH>QA>NynLC~u;PcwBZ6K66t7Fd9tARku0^s*C{i#
zYkUdzUy%FEMduc&EV?HVe>3-yi;|BDZMiHRrRHuldMcHVJI3vt4h-TrW<#_?c}2yz
z3oA;MtyFiLQ!w33#i5y2!dgw!_R%;pCslBUiklAxuL=w5`r>qH*2I3@qWaA|EAz52
z$d^(GU$Z4Kjqsn#i=l_4+XHae$J1JA3?cYTw|p8qfnC@q?nhQv_xmh0ag+M+ow3;49d#xIQHPq*aaB^AeY6PinmTJ1HIgVbYQb6T37yRKBVuJp
zR+BHV@eeyUy94Uoc2T|O&x8(igvbq-9xBb$61xfTyp$>)jy{!4+Qt2X+{Zq-k1|uM
zqKwu*0jc6;yDJK=4A;yF)Uze1oG6QT72u3Gq!?ago0`}La6Fy}&1|4t(C`rhq}G93j$
zKnfvVTSVscx%FSSLO1GAX$CN3Z0*+i%lyMk#xh2fGL(
zs+glyvOOIDwLoz#^=E{_KifTm)a6Pox#ewJ{a^MQCf~b0a;=`!We6W{2E2I}max#|
zxkqu4zEqwR3H-s7C>9TYS+gtfg2v~L=+j#oeby9PN^fq#j7QW6I5!X|Qa+lEepjNdmn#1Tmq3eJe-DN_}78O}64!E6{U
ziI1ca^R#-1g*wEAl#TCYofjCmwElCt4IMkzanRLlDObcJcIoFJu!`L4m*eZa#bTyeub34->Sa_7kqe
z!vU4SgvvKNTS_slcZcr??K78UsMx&P>dtir3xCJC`oU6&9jKN{HsnX!!e5siyM=1j+^Py;QXgr;YN%X
zYHeJIfo`)bcv~VlQ(~8Wg39Ylp;g23vJ2N}Yi-ZaE;f2n!zaBvtmg2Y;CT?}3@CQ3
zGcToO--I@opx(d*XRW`-#+ofvPy&UrRwqT?oL0F>hb>3?j=`G=tw>nvyn!k6^h7j|
zlve{PR7HuPbc{PW+AZ&m!Eg#B7eR|rjjhi<2csQ7z`a*H2w3{1#rEX#oD$hS2EW6y
zlb20drCG647OmgenRbGXSh0CJ*!#S8@${oya|q628_Wr5zPB9tiwhZr^#8wUq
zot)|ZxfknzK&EWEQaYT*1Khe5V@G#)a3@h6Jc!`T0rR;_Q~UmrR`SWACblZ)XC^z|
z+HACG&eZrK&ul41E6KABffJePRy}+9|2oWE{>Un93h?zS%^%VN_|eS=TE4W<{`x6a
z8@ubr8wX(v@+~2wCPL?9hF15{lB1^2M=KFUWQD9m4>JS{ml%GkdHz4mdy=9}vc`T9
zKi4@j!&P2Cr0|Dw-%EUoXo@5;`qBTp7`i(mZhK|!!uBvCpM?5%s2L=L8-`3G_*
zQBQhCGDty+pNonLR5LPA(@pacl&U)|
zIruML|>x=a}mTVi8of>n#%2$vc+aBXj%YLM|1HRrq$U`h%Qx3c~%nC26T10qJZgl=>
zgHXI0krfPXhi%bJCKn;qskXg#rHjuA$2;Onh}fPxrRy}WQM6vy$I?O=>=yMS4%HlxF%JGjfTgyLVf%;bc@(pC+g1P>Ifu3A9
zJ09oP_zIlEVpaN^VcK-@Km7*bkt4R@Pf25&p
zc)kG?hcjM7QP;c2JLK32H+0u*g8u>Yd&l@?*xmUmGoc?h%
zZMeUKu~8jNUo6&&L7*<<&(A|GgkDM3*-hREE4M@MMk5}F;-EQ(!C#Cf4vQ$hd-&pv4m^(M0U
zywJR#o<$_bdEuQLSV~q-UTT;K@56TEd8@`q$o2>vVXq;-Y2OQ2sBGFcUp>NU7+M$N
zr$*6kz2^6;f9I{z&yc`43K&>a(&ymgL;L!=ZS1p1RjbJSE6SVc@U)ks3Zp}PRW{z?
zee>My^j-5Z=|14sFzgA3Jtzvb-35Y5GP!c-*-FJLS?fYg8?6&n$gBBv32sq*%SNXB
z(~&){5j6Q;9$~6<@yKcxvl-Dceb>r_F<%WzY~}t&QYW7^FoUb*_^ahOI1^FFmL;N-
zrUZX{E>zKOal;?S;vD1`YAZKmCQg%t(<`f7HdTWC?U{lYFC*OMeED*e25g_(EXr|nr+K?kZT6ViRsa{Ix@=t4OW|Y1bUtx$z=O&;
zm_){%ePrsxx@GHF3LM15Ki|EcMa0r{26|
zR(d1H_IK+Y0LBs1`RzAKyJ0j}d&T)KaDDG$xER&2=v#TATGS$u0yJMwyuwm4w!bD`
zj$&F-4X!ZaH(uurdRA}3MwrM3&d-+LlUN($uP`fA`<5ljcc6xbXk&-t
zMW{~!XQFy$s=xQ3NqiB=WQsos$!Q@AXpm@#rylEr8NId%+OIputh*KsoU2bU?KWM`
zko+Oc9I>;gvIXk{dP7SBrao?Q=+2Ob+5dc{u9Hw43K!F8+Y)^!Q9#Rr?1$ch2%3;
zFf#_DF45SF4-N31Pmjs=Iv(rtWFN27rNj%!;hn5C-U5S!s?L#jY}zRuO?r+>aE0SE
zP;Gv0ni~Hnt6tFj+05*hqevy18DhO7!EmivxXLoEhD+**S18|6Kf%O@akGaMxZ7>#
zO)Rv8DC(^7BAa%67nG`q
z>ZDOE6h>gJdCp67NN5?Jqpdw6KYz~F8BUer=FBy(5S9TL3Q
z0?%%tCmJc%lpz+alBCO|-~)>y(TecDo>+7Q$=voJ5fEUGfVM{$j079OC>I)M`pbp&
zw}dk;Y`BYB(;_Lo9q2<}L;87B8&I|HYh%#E--)ZHn9;R|zEAw=6xmW$6qgj=^YIUq
zsOqwi6CTnp@`eLLB6urn#^Hc&M7vdI&rP;f5I$a}J&w|fB#9I&X1(EFS4vz(fXAbZ
z(YqAl&q_{xkVu7jv@%mVHJ|;yjv>%`eZ!e5(;5hsMwD%kltJ~L3T&UW2AxQCy=`a2
z4=(af)hSuqCWd@k)%1^n6jfS;FD%r_rAC(8%ks+-c}yz8G%9Xk9W^;$p_^~!0zOQK
zfub$GQJSh#u(nO|uaC}bbXa976_;DShdx@AH@NRG(r!$T#A^ALsg{{|{wEltbh*C3
zF@fzlO!13vlBFOHJB14g9&|Op#l)JV)*fo)4*2Z}L5gOTD?an`oqe*w_~{|e*8bw+
z$o9V(0Ps}G!8t4J9uqPLD>?NvfyBk-Po|;9PtL~Hs*dGhK+b)767D8~y@E?5t(=bX
zBT*s!-rC>4glK
z+nb1sUm5s?hiIn>WYEYA71>{1{)w>CAjsv@5dJD7g!baM)KhRRdI=OPY<4iXGInK#
zxP-wL$c47fSp&1v@2BEZQzO27!7TYuC3#w#!exa_PEPLX>YAxurW;nLH?_Vjd&IN7
zxhaAPYh-H5M++Amsg|3y=Kj-N%X-(b&Cfm_DY*$5MUYXqO8xd
zeoXJ3;MGbSHuh<%pOM92ZcU)F%%jo5@WQ9sTSA!{KL*%y6e-`@qRVnmICHDUo*VF-
zTwO`3so_05Jdh+&#;m`#J5LUgMvs?)3-3{0-G>wLr45sBNneQ0$RJN((%C!USRm_5
z20FgW6D18W7_LwSx-+1~_*svp8^I(TF588y?VkdYTD|6HU-x&?ybTSb
ztr5>sg_U1cp%PRL5UuhBjfRcJ0rMcWCREOc?%&w(2fP;Ib<(7oK}d?Kv!@Ej#njl6P$`B%lfk$m0
z&yj)xwLL%a)mqI5^@QWE=OyZP7QWACvG~p4dA;bev(Vs>+&5qSiE7rn0yxXZ4u3d}
zM#tAnuIp$K#N6meNJwE9b^~8u?ku+nwLYB5=H}+EF#Fu1;4x}`PG&d#8OEHCVXPx$
zIzlp(%#HxUBq7m0dVuQR6O;JFLBxMK`F^I-808zc4SME|>Ct*$^!flzM#;qu=wvmR
ze66^wwG-GdCJ|8>F1=c&&-0VCoZM&m{Mf0WNgjm^K~h3OLLTP>0+7m2-A_PFtFWey
zONeh6(LtGMGLIcO6LFYsV=*l$
zEDUONJ`6jYubYpO6$qufjLSI_6#U0)9fLL-I-b#`@GDOh@6PN~=1
zt;6#R2*jkP>#j9!r+c6uw_KBf^lwg9lZ1Q(U&$r;uB_-+n@=IPJ)DJuZ*kXSbHyd%
z=7MEmZJ!c|^wP)H+i@8==rT>kuqCTGq;+M*r@|$3{Of{!y@wsk!`fX!;Y$SdyBkuj
z7!Ay#i&%;+W-E=8SPY;WU5?S2nVBQVg~$No*FyhX>j`JIS>op~8v~`MrzZ(`a9J-l
z20q?j!+^4La;2V)9nzw1Rc|VTj$>QJK-mYFw>-C+k
z_r;{76ibsYyKR%orMWB8%-jaUQ_~m6!`vtJbo@>+!H|
zIooErwLmE+NNkN+yB-q-k3o2`(J5=(jKyk>1JU1?+~+|l=86#y<#(B0G~i%VU@$b0
zvya3Y?hDRi1TEG%3idC5rphtVq*>INAkXZ2h-3X0Ew1>*PhyZx-?R^#oeGQ
z2(8w_-rm04YMv{COknP6E`d>t#r?ueyWSRxx%mVd9-Snl$>n%1O|OF2VQc2B=tI>O
zOy9BBS4ZIRFT-NQkAw$|k;{w=Fp*
zFykzT2`ersxpuwmgK)Avq^+Q!FrU%V+^o-*OdUP=sVyluIQVdsl9Ccc;Ze9(d4~v6
zFV*hIeH#n8?rO`~7by5lp86Pd6KjKqV#Zs{br@HW@Eu7P2q)&Y27w^C0I&fsb?dlU
zcb;tGNhb<(rtx4x?elwOWlOmC9y)E_r0@Y*-S-;W-N1!{w*mYkq#upbd=e1|VJRac
z>dwwikhIXI>*bzm^`~FGSGk`u1ii}L&P{eM-9cv=-)m^33iFld@81sFt@o}1wW-{T
z6dykrC|23>X>V$$AqFNBaBp8;CjXVMAQk)S-N`WbG7q(o;9Hp^xdCW(_fljnpXt?k
zhc`yJMPsifBep+Am|>d-NnV=rgR>`gIHBD+#ti`bR{zNFh_s~tP0k9wO2XRZ-PDp9
zn?ljarcL-3KpU3F^rN8rYdEe|TeG2>8ab4H{})Pt4{vXD+P9RYPuN_lrJF#=J4g7r
z9DFvhf9bjwN&|5U0#x{e*(wAW1auV+?@#M_zjjwvSN%x&oL43b-`75$pPiljq%Tvc
zSGcBQ`pJ8LOa-{S0Le9^#}s2_WqsM&di`4Jhnj`Q<5E5ePzS^N$UIOHFv!I3uMWqX
zTx>25=kdwN$i6CMJ0H$X9g+BD=PU6!?WwHQtmP|hUUcXx2t9JVl1~k(G93le(IHz*
zm-YY|Nha0d_*X4xEcY!oH4P2lUu(&P5@LY9=+Lf=aiV=aF35TT@xi$%)-Emev)&}-
zdvC0m`}5x@u3*jRD>RN7kEPF5+zisK#=_x>oQ+cEEBlnkRkERvjj1*2>c76fvja-r
zi!uX32K@hO0rK+lC>a<~-=v`fr>u891@BFl+3@^HoxNOrA<=ZMiHSwmk=yX_@<-1
zdl{xJb9gTyKBe%bjwnFO;xtkI%_N2jCF`(%{5|T7!$<^l&c&!uDSxrW&$!nb4fZe5
z$@me03iK|8i}KBzxmw3_AWT?1t}JuEfA2KofOKquZEZq;8-AC5_-Rc}+Y}8X_X3?J
zF2To(_XLrWsX%+^adl9r?DfSxI&LR`L_zSD+WYZR9oTzmcV(bV_whmlyWsr+yLO`^
z5^xxe8gtdq^|F>LB9oES(C%(=z+eWu6L~w+Wf=#X~Fqz*SAh4|DJn+lqroWWK#4}41VOcIe7+o*qf
zgC62#RIb
zS+!-zIdi)#m1*dlEvh`8vk~$45jUK(s8!w0Vx^h~VWFWNG#Q@$kk$Z5yRQ2`>dNib
zC2#KTph4f|)51go;Illgj}S31`c}JwD-8S4d(M6wTtV#$0*wkDF_)#Vh6Vw|!m7n;
zl&@dEy@FA?x>`+hSrP%744?zGy}z=$IGDwTf`UrovCA%b0II4)I36VD041v2=h<7U
zd9lf5kW30F@{k|Y^SZ=M_rBYG(RUCjWVa>`DUCo1Az(8?oU66c%6o#!E~QUmPxhkbHPEG$sw
zG(9n^u^DRmE>@wm2e9_dblcNqJ9A=1MTP5nBzO6zU$1~q=nsG6gnSEe4WMVp0@~u8
z@f?I#3h9u!=wQAMLo5ij1Lz`v81#E7o1hO=QJ`;@u(f5uW6}%oJqN{AVGDHNcI5fgj57zI!dM)D4&;0+>69KYaLr8zzz)$NG#Pj+j9*dCvSO
zU#2XJqgf)n_73gE#3U^T3gQXQcvdMA5cHd2b!+ORjV{&(E`nD#kJF^m560|4qYw4U
zbKHRQ2`|`J@4P_-)Es4T1o4XqMV{N}huym{TC9$}XTfBNcY^kb$57)czSyCOEH#Ph
zh1(iGe!pO!8P%@S*ZWGCd<$;Agl^>eovG?NepZ;LW1}8rJXG1XEQJ7P>EVLdEKoB+
z^B4AI8G%$3>_{R`QNX)^!Wjtk^YiJF4Cz=eFXPsy^?V{HCsi(Xe<1*_oFwS=2JjQZ
zlpoZW)J2o;6YTBd+=)~}X*j3MIbXf|_PM`bTI&J0rUV|-p(NH?8s%edqkc@!yWRW(
zy*5F$QteQnvzl8wf2*85@mgu)1ypKNUT-4J#m!|AJQ40#RTfg07ST_g4SViS_)&&E
zlj$=zM|!f&rPmjd$E@u1Kl&w(IuyUW-?H;2f82cLZ=RHh_8zVV`rYgNcN$*p
zzUSQJx)*gWx08Dp4d3H{(m((7uKxLX_g6k2pFrZup5e~kEu?GY_;Yf$JX6P*N66mW
z+l7ZxxXZftyMi&Krb~5j=N-cP)8e>tWDEU&#WUcd>rAc5`+-_C#pUYZ#%qSg7F^3BAArAV~z#83NyH~Se}{oBx}Snwx5xhmotUXwd`zvA*b@*Fb?&KYW%=M_1EQk@JLIP-X7?8%$s{@^9C_o3}*JCNJ=t@;hegaKYd&Fdc%FUSS-)rTO
zr8;VA`S`dK-|D)#E~aSVIB`2bC>1IwJ=^*~Ua(m35+X2CejuAX!2qtON-_VF)~un%
zZvqCN2I3Mm`Vu_$5;b-XO$?B5cxk|x#}LDZWhvk8Yu}dR9GHRRaX5aS;^L%nXp&aT
z-c?x>0!HDaHe>-R&H{Xmvw$N0s!FnN`9u4>ImIDU<`NInD%edE{fcFxd1YLVgby5j(b6>|53DRsS`Y>9BA8bcNWOk(12pv9t1PXBm|)qtFb
ztj+D9dUIF=KkcA0R^ZE(WzzjwD^oqOd0^j~y*}!AHU#Cg>2yd)&>s-qM0r2+axs4k
z8+Vw70m(l$69Rf7fMBc{bGuvFMnkc+JrAxG73<6I%~^jh;ql6gr)C=F&{Uf*`dBDE
zK*UGOcMfY9q9ZH!>alR$4X~knU|pv+m`*GpUqpg=^^CylGEw0$^}ITRVR5<>%S=1b
zm$-Ed0(r8!;Ea&Z2D%T=<#c79j_nCRfoQzLdK$X;AUhLBd)7~#X~D8Pr!hQeYYWoM
zk$4!X>6d8>+TQX+2WZjUmuc;%i^Pg;5>f@6Zw6SaxrKNkVi{g0viWePuJ4)mA?h*p
zJ}9lX#~gAR_1JGliN5m6M0UjC-h)6i*WQu2hC%}6j#-n}F(g;5(W)^rdHy|TD-n!o
zL_2fR0kuw3JhtNz>#y-jgg%Pr=;OLenqwWjzaV^x=&uvV7!U4s|31C$mQZ^1qn1^%3MI&#YeoiJ1%-zad#AJ3Bj0P9f$YL_4rW)&WY`#Lu^D
znxj6h5FTWvBK%7q<$`Es=v~brzX%AnUcs+eqqFpKhy4m|z9ca!Kv#5)IVHxO5^6Pu
zRl3a{4=tN--egPM$&*AYap=q9({)@^!i0S82^AC2IDYm#zw
z&#!9t^LodY9zeSgQ5Ff>yrWp!cL@(p!3iJKk*7wOw|(BC=S(m!9aQX5>3%L*GpNvxZrDGZP>p(-jpB_;q;(9@jXNTPGf)e$uwIA!
zHJ}WCp&I}YutoRv_9nPBGkAWTbiySV(zsFOhdr{?{GQ@6+|N9=W?uU!q)RjeU20UQ
z-)cI~ScS{urQkQ1_JPsF$T$~JIV}q5;lSQZ7X|ZuO2i~13@GpC;(0hU$JsCu*;azt
zMt}4j>-}NeSMRWnEJ|EQcls^ZW#AF^c&Rn2YknFc{2)L%Q~Z(=IwtwDOi4z%_~zT}$G{HikE6UfYgXZ4TfjcKT524q
z$|A**$$@MQRtY`&(0HCsICJ;Yt}h{+lJb{XaJE|jMZWc5=N~<;5}~rq=32_*TL3OY
zFwIxaBjvXv^d?h-b$`v?m6;F?N9abUJ)+WO1f>E!mYo21`N$Z?_`d#9Zt6iv14kmx
z`MJpzUg}(4VaA~jC@Z{o3>_-S5Aa#j6gN-5S`XOtYDdI@O&ud5KX%VzKQ+g@Q
z>!IQuVXs&yi^n=e#5k!*8Z@GT+_v%tqvch#lB}F|HA*5j#@bdM`;FD$bGgf-XN24N
zVD`&n>fZ95@UTr61;e~3KmkU(`$V$oNv_J}u`HDqY)FndZa#T3fkLpL$?U}zhOC-h
zb5X(X(d8doh^HaZE=HgE{xgwxIvvip@Y2`&0$*iue&mtyB{F4xS`^-?nOdg=AYUK`
zqaO2>E7qrt{?ffgmA$J^1QRV4Rlf=*)|_cFOUrOCA7$HiHNEjRQiehlrdF;UIpr4>
z*VS5))Ao{;=A{OBRR1UC>folGymEJzM1~d%--0sE)fTo^d-I*)8ICBeIpypc8SzJS
z$;H6i_6STdBDCDKvTmQb>lrZX#1G^BeXXQ639g8)TQ
zwY!42-~>WU^o8Hl9h=~jPxQLv*{4a1C0%>e%XC57{^3f}s-)b+;--vA9+^v#sQW$v
z`bQ$7>H`M1wA5oNsq$+tN9uSPUr9Le$v8cNO3fb_IOt^y6(5h?*h@{dvXo#B_dktQ
zWD${lLkQ2#rkzuD#p#(yWZ`#}Xm1}uTRj|j`4vJ|7o5Q&xPq;Z;qtfIIZ+`TU%}6_
ziLf0qgWf7DxsZ}iJ(o?;TrbT}A9>N(P!?=#ND1`2SqO9{HR|0SGy20E9S!k?~S7dcAc6rjfddxQBj8Sj+UVna&uVlUy
zS{p^`QX4KHvrXn^THi~qOmtD5%TaiOyZ?y4u4ev1E(n_Xp&rVPTrMap)2yxQ|E6-R
zKgRO`KUY`U7Z60RH*v3wTn?H|)+Ux%W)h-r@1Z*QiHWn>PImB6e)2oVfvFu~1fx=idZIOU=mP
zvxvpBmL3=ga^CnrL6r3-A9y4y-}iE)_}|#jP}&r0(J_iIck>tomyO8rBTw>L8qFV&
zScuLF74NT3`#37L*SZAT0!mrnQ7r;;2UguRG~Tp9+~BYCT|zJkjszoeMurGxSfUQ21t%2a)6T2is_!wq)^HHy`5NPt!P*+WjwaZbds1vEc@
zqCoWR`B-Q>4`DGOMj-JCfb0XD|CKMim;)d)lm^4Tcc`T{mr9C$Bl7sl!IS>A4#=+xd@^
z38bBA?O>_BPg$+`f@R8u2b%t4OCRG~{-X!_L*fFu6C4@*RDhkZs)u7V|A3iMB)Nvh
z>6|)6F{Cb@?`|@A<#d<~aL29#bqqJVC0N`?npy8rcJK-%`hqlfCNaDdaD2Y2R%@s}
z+;%7bee-6s>rX|;;#DyoEts(I9l2yQ!!_@Tu`o{^RHS553J($iznQ87x5sk(h3>3|
z2_*x0zGAxsE73+PQw{2H&tkl$0*~{;*vFaI14;C60W5f9zi!o7$p1Aq7VE_ee@=_v
zac;W0x@52-!p_dM0c;R#+GVZZ7^cMud?&8SS`-O~79dW&jCW$}57<$qpZyIr9N3;!
zO{skg7UnkzZTsdtUNzz`{f{mR{J%wMK5Rh8-9|Y4!xytR(k1tN^pngH`AeXtrpXuj
z`}?!lZ%6}F&rjfe9AGphw!diAxb8){cyVeJ2ghv>``|7S$j+_GlhLu=5>h~&GvJ>p
zNKl(_La$alY;$0~Zh+;f3dQy4kAz2?V9v@-fu`>*dzN1018%^YgBWJlV7~
zc!BP*sN*%s*txN7!k=D0K!Mh@GEeCY>$|^^5nQ&g3pc~9e&a7AzFu}xi}!s=kD7+E_L
zzkTl-FLBh++Lclfkod~EYfaY~IJ-!pSgE5=ED*oZwR5_w^&@1mm@@xyaw&t;1c~F2
zqHtvVMu2kjov*W@5)vW@fEft6$7Qn=ad)}D4j|*05?Hbc3?0Gb-XVN0N10;5=nyUu
z!qS0Qh$3A9TCD=WngDRU+H4%U%4E2+-46x|$Ey9l1A%;HJz7Z8#N?Ix)j{ayU}BEi
z4^5CSO}Z;I06zA1hhjs4xNVli59ex>G&B%fbeZX;Kr}f^IUlM_AiwL4
z^MTo5R*7Jo@oHya5}!-yVPfpbiS0Zyak1JD(P%OO;0FEZ)k`U;sk@yQ9YzI$fB(=j
z+!{(I5cK4ECObIM(9f(sWLL1+d8LkJrlxppC7qH2@WJ-H4I0
z?5nD(V)cG-J(#P3%uZd9v9|8r&;ILc@&29^#JBQ3D@ha$cLku37x#$5zADBdFAd}0
z?`D5a98YZMx@lwfyo__SaOx+b0#{lGYbWkVf
z40qRa$$_tok#!zQAYqcdm(>|q3J};ZAb_rhOx{Z^9)?#3vjDdT4_fKM5b9Rf^9lrb
zWl9bX9D219;peBvn4~1cH2L6v5pw1L0tsBmM}VvZfMxOvcLB8$4Zr$&9*7ZY%%?gZ
zZqHfu+d(7(?kW`|3aLDP0RKbD#Dor9To-_LL(q2!U~OKbS&3ns%xVa4GL+=k+|2Ll
z;X%d8dFhvj;&Hhbh(^q%KUe(;s3vin&%g;~TOFqwDo@WCz1U$AB
z0Fu43zi*-L-2+@<06^7&K>h&%!}CoA*5NeDz8omICYX!m;Pwl`!RCN^V#e*~f&3NHckazq=aQtGspx%njIEG)8KW
znA7|-fS6LTvAr-bFaU*_a4>3=g8&FOP%@_jU<4=-p}ikYr9kad#p>hV-x2?Z*+@T_!@Qg1g@&yX&>+Ec)3g?3II>L05}=qo~~|g1`ju<0Qcgn`6B^%
z>jpsh(Dgh|0}uCsYjHf7!E)N0tZ3n~T@k+68Hag#dV-Af-#9+`+qB;HQr>=Gz=MGI
z;rPsw;^MHt&RGB+N@DI+UcZKn{O@<}c_uS6`1xzFh=3AdhFTbN3YWot78_qNlx_fw
z4|3b4Wy_I&6D9MTXO_HA$t&S}X<$GiR1muXMg%icxsXK7(cTddFzEuqsXTFF0Br!E
zX)cMY9RP=aeOmKB{!zGiYED<-?b{B(*+3w}zNj9=GVlR17WEx3#U%JKZIGlPI+hl|;
zr0J(DYTo+b^8|#4kVu4n_<$Ea>0Ojp<)TIiX$+K{GNrR*b6(Fz$ma(H2O~Hi%=iJa
z1i_1M0B;IGwSlDC4Isd$zqM=(?k;vA(ufOpuR@`r<$HHG@0GT{OxIg*KE~DA$w_+%
z7FD_R;v0w{Mkc=z4
z570Z$R%mBdA#1ZGyk^+f^gvM*wp-0FN#V2)lTw!8ZV1e%G~}g=GQ5`)ztRVUSE(YQ
zkCmxz0+2JrG`%<4?F|03)XXcN%A>Sa0~CSTCrcIr#T-dARMIy%KyvNs2tbHQObiCF
zT(SILlz?IYpIMi|?h%j=VBzAzfdD?E3(#uFR;%j*{FxVpq^9P#13_AYg?c+c)F1#+
z8c>P_@v|7x2?@1zPyfR-t*)+)nb*ARrDbMTsJPOZwJ|L<-1iHgprI2@ehIaCFiMRp
zyC>PUpiR9_JJ#EU8VR^6d4okkI{~n)p-glgomx$06rXDlc|uXlVc7Af%L!m+1)s
zGBFqs;I-kiRi;2{!+jNt1ZYK2&d$y(t75;uOEKZI<{&yfYC7%&NN_+_00VM7S|H5u
zx*}xKY5ZJW%~7IRQ?$SI}imV1z!>pfi|U5=hO!&49}7H)SFSSUa6Zl2{GP-7n1n
zZtmP~KNywJAK>C~fJ7u|W=30Uy=Wd@ChmRQe2%^~nt=i89~qGxO&5UZVIaJ+cEbxu
z1ElwSrw&o%0L(og#ps^5X=7~iQjVIe^s7uDi>sm5pB==YtLn~begYZeqi-B4yqZy3
zu|E*hwx6%@L26m^o^2RSh|rD;$$6n#E;Aw>SH+)2b;r=6PJ}hkMW`*V5>H$4kSSng
zdn@{<)=VIQeKegLA;$qkKtP1ipWdCUc8%9qs5hVeq?i90A#=Xe*u|3`=f}K_UT~qI
zaHSLCJIRMA--;9;@!|DXc5kBtg_oSzQX{(W?Zd?FO&)lPMmMdhhUn?}12n$Z@cvf|
zAbXR8L)G#*cZ2tfZjthceS)%GE{Y#4H0tV4n1F4gKSX5t)JgLi4wO
zK?ahu|8TXt-x0_ksz8A|_udhZtzgWydp#ePG>KEXhx69D)(5Z6G5ud!w`5)>5
z$j9=91eRK{S4PZcgLjWN*8CDFw2i_K9o9c@Dg3N_R-s$F(rwQ!6i7>e;YKOUgs@5J
zKb!urNrTpX`YaS&Tvu6xcxr(U1?DljX9}S$jYc?{8#u4x=CtHu1QQKIyO5dQn=K1!
zU-vpqzHqlUDSq1*!z4~uT^Ap4S2EOU&{Gjo&>WWd8qhk29i$l-b<3%aIR3#h-4y}n
zE;;(fX%V!runebojiN4J(
za80$o9aK@oCK7Hq)nY#K=vt*=a%b?crvtDB{{nwn4eImQA&KA0p~_YT)TTxQ`&E_8S#k*%iCVTo1OSJoF|9uORSoiwmpl`7Eo
zl$8ofR-x
z>pj`YAS`;T>{0@6MrhVVWJDe?aM=BwtYRtw=)+z_t%I(C`|FFrTVBiBc(7fV;lMy%
z3Ht`aqXz4yRUKG%r*M+)VesPB7F=^c=YeRTi4(4bLq94zp_lWJTy1s&343g0fqNlx
zRejjIeYMlgKAHEb$iN#ZC#0wD#nw>mxQk&2
zLJ87=WY&qp{)W)9Zr(NW=UGyz%6g^AuPt^*svN<^VDdV)blTRkn{C34+p(JoyMyI6
zxg`c}-iEk$Mg3ZnGOD{(jwC=6i3uCLG~=SiFJ)&$nbp|r5*3!OT$rg=eA}{XbhTB`
zeR);AO1CZtx%CZ1KpnY}9ruAtG5Ldzy-CtQik0f!X)^w@ulg7KYTY-UylW^N9*uC{
zFrp-X$ICH%YQglU$ow_Qw8K(}%@5P^#r}q_R$)GDXfPQBs*#U`3Wv|S{bFWzL6n=(
z8mhN+54T;L0jKC#mANm}n6)T-xPrA3I_SY{aKBtjb#1}~TDc`Pw7Yt)a#>zgigzRK
zYcn-bek+Q|2`fsQh5soFiB44n2W3xFy@#oGvF2c}arkV`1I9_Byd7E%6ALp}8Vzy6
zLo`y@_jBB*V{%GA!~xy`ITPihQ4cr9WZrFl<5%aewnvWLS{K^SToPdvi)ue|^7E)WKa!W@ST%dujSi+kOzey{d#0WIPArijW
zi;Q34cVF!FkOhwS-im9(?#lj3=5>B0DirFh^RNpKRcrF_8lXJpI^&clw%)9=He02-S_~5BPf2J6^p}9&wwW}YMiQW<9
zSN&&~M9Mu&sh(>h)^>n?R?
zn%jwX+fBRI`aWx#Q=5yeHea16Qo04-UbNeo4
zovO7>54^EWg`B!2pIpmlMjHnnulnQ5T)L#qiQa?zO25DyNu1CZ`V9}h3CV>M~sa770~yE3qq-1Y)}C&%b2sL7ro4Q
z3vg3Rt-)NZ30}q5xT@q<8CCIz4z;*x%iFE3o=(?^M(Lj3Pd;1Yj|oQ{e079Pct>2Q!P{F$MH#hkqbPz3
z2qIEaf+Era(xo5*(%mf}-Cas6Ez&U}CEeYONHg@%4MW$!(C}^WeShm)>wM>|^Upah
z{KuGQ@B6v;UHiJO`HeSZ9sjqa+z5@y%ZgrbXHBxZg?U`SNGWGamY^
zc0zC9OF3kSC)T@s4vp3F)ET^Jo4fjZjM%)0iNf-xa`YUkqF=Lq2=?VHm|PzllbaWH
z!_Ky7-rG*g#G0syuO*vW4RJjw;=2pRlT*x=Np;JX4Q=HUU4ueeCPtNtBgRmRAC08-
zO(-IQ97$iV#kyZP4B}fFgsWcF=7v92M~=Em$1SU;Gq0Ewqqc4)%F1`KYBNMU_l8tg
zGPCeWRRw1a$y}&RuI4Q%J&v-mY#3h%WDFw;?nsZuK6qW{8_?XN7`bdgg6W~QLyxKu
z9F|R4vD4nWRH;tMwthTpt(a8Xd8ohgLQz<^jmmW9gCWFZQzP|nWzFQ+CDd8FhGmo5
zdAjs+k&NJATm~^e3qoFYfW$DVoqZju
z(DwJ?(DT6RB_64_-!w?d_PWEg_xm!3ZL}bvVWi0p)4Un763ot_j8(^Bv9p!u?^B5E
zvU~SdXM627J9z_#Bh>7XJ#5cWX74BVr!AAJVhS9%7P|z?p7c)Gdta(^^#{fc+UOGc
z>ts?a76Hf16^6>+Q;wM&vOvtDUPHc^74b0{kZcyqTzh0jZeA+A|DEacku`t5O`P=s
zrZJtIhZ?-H+SaVFDLWR5BzRtbCCig@#T*#%%<7Tb_?w~JLSM$1T1Z@GE(@}`C!eA!
z&y{4c&tXuy$HDb1=OSO$hHeWneXbXs{fV=j^S$Ay-bSpT`is=e6pl%T%~}5t;iX*^)bv5bp!ZB~nari=e*fs}?E(L{ayat~#H{m&
zsjSg10y|7w^6T;EN+T*`u`!vCr1)Du=w}s6ne+HXQza5!dC4bOm*zEeO{d;2OdQZH
z_G9p2{z8(Ex)FZL<-^cXr2a)CvI^(@(lSlpHywjjV!!C^&7Y_=PaUUnsm0a*yrcAS
z{E4oCNhU?aJU!7Sxwg;e_bFt=*tLlV<{wT6U!##5;O>_pHH5t=HnNM1`l8EDy8GHC
zBqa6ElDEjm?fk1_?LKaIG4Tl(8k3Wf;oXjOc^%hL8zlxChsrY3n^_gT<|$u$Xve(x
zO^{%kCvnju0zI-s$S+Xm2@!VfBX`A`0B*-q{F)
z-KL;N(GUFu|Hvv{t@w{(E$@c=E@Sz4d$IXl31-Zl$n&F93Pw!zkUL+LWAB!FNIpSK
zjm7ZwZaf5L*semobkK`ys0rP6am+TG%^DNkDzB@ja4D)wDvp1JMkH7jOnt@ek4ZuH
zP?6F9M+yBD&)NG{^PrUaKwh=*Go}%x>`mQrvLa$k0e8wgVOhb&6Ru|ygPT*&L5dkT
zOKlvs(1L*G7pqwg1~VNiBOWL=5^XgPNJ(Ct{I+r&yJKV8(G-LqEl=hVoI-@CWa4T!
zHFYNFZq>6(!pu`ebui?AL@qxWJ*A;pRwA@6KO0nidStKYacNvOep{$krjUL;7sJ^OtLG8#Q|fcO)$Js(NRT>pL$iOoSfOOpEOx|hjvs;h@?ZSPZu|K
zlc(s`M`T&;EqB;PFLsDqe?hBPhBkt&RD;krP{@JcqfXvLzb`G>YBC>JIf(7LHz$Lw
z$)QmcK?%+qkHa^4LcfGT^sW*jHPb>9e8>#g<~PsH=m_3RzdY?8Biv!3Ym|Dnt?9(w
zWOW(_N!5|BuL_L2TPY*~s-m=kG;N_ZEP+1XrY^5CC>i;zNAg9^+424gE9u_oz4h{Z
zXq1(By#2`I{cyrNN*mevbWD7qjKWRd$4pAdwOt~=63aZ_`Ba$s>Ou1cs(2w~d8+B;
z)+RNEimBf83GK#4#%Ou9kA|<9L|xRP{ABd}&{iCeP$hFy`crM|D$<^di
zBf@TlH^F98uVYQ@bGL@&GNLSbFq;yT!EU9U{XT^pq&Q*H-Zcc`Gr27P^agzqy
z+5C8q`$yHUFmcXrhKKDIyY7AUm6K~LA(L6iHmf0QRK}^!ZrMJRF#p_EYyh6UqZQ-y
zG$f}=TBz)8w1?`bNUrbb2Z2^RY8A<|=oGh5VblWxd=8bp7j+ZZx*lI}lONlv^r3ty
z9utPA*TY{XV+ZKmRFB5j^~r}lD5YlXZ4SKmIfNRjw>f;2;|VL5E%>uo)R+LdxwSX#-t8gAvDMEqlDToJ3W*?|16=wGP#AkA)9RIZ=;kJ(Ces7}o
z`fwp(G0HrA{rp7j5>13bID1HJ^sB(75c`YYDk@w_44!=>5yH3jMT-ZSffN
z0o;4N;d>3ad2=5c)#4M>v@}Q^Mv6W%3TEuP*DskO;#>t8)4J*!W_Y{Pvs=?)g{S5I
zh>M4>G_brG#Q9Y(Yo~m`R_|>qnx%kC%zPdEoTcQNe2uDStKD}oJ&M@>nc;L~|2LzkIGOp5xT*@;+
zqa`duS-g6)ADcHxo*KMkkOysr2cteV@?slm!Rm-mu9=Qkb|ewh-N
zz24&Uv4f!^a6{GjCu!tq_tHJ=`6vy3-RKXrjPPAJ+ZU{reAGzS)DPFj`3jQlnBmI1
z8&XmlieDAZ&J77f`5n=`xULUPtJ-4g(z>NO`g!B=mB099?m7FFkItwsQ}i>Y
zI}Vq>UlHV(`Kq+h(kQCtc3o4C@pGOWM3+erE?E;;YbcBTd4ORy`(26Co4E
z`anD>a?Porjm@pthSTn5liKpEc~0ru{Lpe8bf}|jeuZd`6{#+nE_BO7PkDrvQ~hV$
z?4Z3K&4OH@!Q5f&6~bE&R`s7+_~sld4(0rk${=qQr5MJ-yYI|>yV^h1Cl-m63efnB
z8ZT8fSb=mEJ%Vv|HAx$cAIS2RqzT6=-eHs?(Hw(Gspu$*no+Uf
z=ivJ4#AMRq+>}HgvLW$exja5hX~e&GJM_9QG)NftTl6JFn}f|HKyy?Y+OHw-hNT>
zL|3~pIx{oElq+kUXC?ZUP7P{~L16a7D>aesYg5G#%$ZrB&cy*fDbyg4K5(jZiD
z>RZ9+%Zl4RxGuHqPRN`py6-BHyo{$zBGdyF2|YOGd2)r
z#iov_a0XqPrhe8pf}9yFk?e0x_*$u-Er4Q8H3fiN*^%(UOadQqIlil7VM-i{--f+D!OSiPs^O7J-9jEZ1(bBpx`Ag-w5`j`Z@y
z8PfjKr7NoHOvc_24i~O_qCr}wH^HXk-v9LZ?9$(WBW0U<3_Ud441+rr;mZVxMu&gJ
z_FlFTYF08(>&1#6WGX`A&^jh+8h7GMW@I~I{QEkYl#5s+SKWkrVjAd$i*Cgq=`6?e
zc6AaS-@Ql_z4-lYJuO(7_%Pi{JNl92gaLOM98v6D6diV!>}W&u;cDrXc|gq7L`xyg
z4NW2^+z~F5@!jHlgC)ET-a@$29ED&PZh2l_azfj|&a}mA>`Y;S|AT6xq+d;nI<1qf
zuYWMkiOI74tPZPq_K`WE0HTjW@XB(;2)FEW!D{``Rh6Y@t|Gf+GXGIP*uW0GVbU^%i-Sxz_MgT(!zfi(YfcVe*$I7;}@Sh>D+`1&L*a#%;z5_5AhX8QMucLdfdWD
z3^cOX9{p+GHNsI@S7jzE#XyB7Gm9oYetDzcYxR_Ptg{m-9y6z3i-UjbyJSxU->YCN
zwjvRX7SSl?jTkYL71Br)RTSj+n+(LK(sEzWJx)-YDgL1HuY$^cxq6*lqJEZ)=i69E
zN2xrt-4ejT?616aV1>&zKibD?@l!Q=bGkHw3gT(H+pnZ!mOg_qL=R#w|5J_Q%=7)sfvB#wR@y>ij-WYUsiT
z%B}D)r*}!B*qr2qwhZ>ltJIs0wX{!aTm`J(PhVl(FQ@~U5w%5ip~WUN9;B@8bTp*@
zRgO-{3+MCFNU@=rrH)ITKW*HmVDXP;OO*zYbm0^WXLy%wsmwWJ6x8MGLXp^oUL5kLMPo?f-aTs;&E9js$?gG(?=yHHPog>!
zQp#2V*lB(OM?wzkR9o2936-<>Jw{WR=Scl0
za3QuY9r3lB)3t#k4npY~;F5tNEz4om;wrx%OJmRf9c=UU*fqk0tP37y)q{n*ptTiq_i=cXki(N|hXRIy~r#eF~Pb1j8s
zxt%B@^(uWnkIIbiky9~#bzI+vOKcRJ6enkh_eU)7G#W40sf5yzCu;o}GH;FLvXLxw
zHD^D6MPmP)B|IX!;Aau~x+li~Ba{^3IOPvtkT)*mVV|TZl*1kqfcjxEPG)F0%Ht&gVfw}8973f}=?{(mc
zQY7b1Z66g(z;W0AcGAMB?rbeIiAyqeF0V6(p>4gJ>^uu!4Df63g)z^#_hMJ0ftZ;<
zy7q_5(tZ6c^l7C5P0-QyQ|cPVuxOaUc-2mHbd2ODS7t8ei_~LP9i}Qf*DICMHIJ?0R|BI|=klSF3|TlV&^jaX+JuAV!e<>Iy4{D?;_}9Ag>0W7A7<
zf?QRMvWU0}-aR9IwdLV-icl{c>i%veyJFQv_s~h*jfFK%044Tj0$Abx4s+Wrf(*;
z3ngX|aUwjRQSu>yH)^qaVnzy?Yl`6y-(Y52WXykJHfzdrR#`aTa5o6RYm)+d`ClDp
zHN|A&57k_*9|lRRj|I)U2*^k`{LN3ixW7iGrf!ptW9jTsQnVHRaFaSs`?7yyxNvqU
zAMvMZ^A80p%mh)}Mm|=O3Tg6;pb~cbvT=1{)>7*6!a;pk?BvvCqU}SnIHS>S%(%rz
zln}iYt|-((MFPU8sQ%k8-j)mx=PL`cMz^SH%wy5EGRTb#son&`cysm_m*taEv05C0
zX(wqGqL+&D(Hr5kE(wy9!^u|l_d&IRawZ37)UtC9ny}$J{Qy7bBPbYwmEvFF_{DeR
zYNS1U=}hK7M%^>t$b@hgyrKEN>9=fV#pU{fIqPG(U?dO;mp@PC0^$Cn#_+izy3dAL=r2Gs;3KC
zs_#IVQd7W!i(WS#w!B-ty0I*#Bz=WvQ^Xt|9FFU!fd-nVk-2iLrzI|xk^yWZe=<06lyB%
zb;+`CUkbF)9bAZdquRGP@Q_;U=5Zj2wOy*jX_(5hln_
zT$qgH)CQ&ng{Axd2{$h~o!n3C9E5CodQFyD=pGnaYELrylDUg(-mAenKGiTOTZ4OC
zmytOM$^$oLkr0RMrJmS_LOkY;mRhf3M`EU#dE;J6RTLN-SXs5rMbB=XzlCNK&PrTQC-e2r
znbEkCQfg&1*}w_IpZ!H3)kF#Vw(|llGA_Dp$+TU+Y!c+%A1&O1vVJC3q+BbIo`l`k
z33rFSTJ`&f?0ARw2P=K)5~~vKo%#v`hRS~a+{Jj6S#H0r`)F^3%xB@dhu&)nzl0+y
zN=a*!!oEXU!rr!CNy=3FdRBJUBSJ06s9;beB&oN`MzFO+^z3(B|F>=`cRx_Y`8scV
zr9!gm|NcO1Kjr7}1OCsk8S9fusQCkw%2L#n*j2zD_E;*Sh(4dviGlk
zfI->7i7(^HT$I#qHFoR(nf=gWxh$#q>3?297k#^F@FaS-^HPbe6eZn$!=LvP)jwrO
zD4qn#r%vS-v**`aa!ry-k^HN(T>{~8kY8dScK;lU9c&~w2hdBlY=4u;bOD9E+fwag
zuclySMWo`8tLHflQG=UIah+n$^l?N^d3blG+w?c9Y}*8pY%#ZsZ{LscOp$h
zm<1bgRuP4_y!e&%Z5WF{gqDFw?fkUMV?kR=;pywzq5EEGfT1Rg@{aVNixCU+fK=OS
zlsN}w3f9)^9@7d!Knw!c3DoSv`foN~xjzpunA7TCvXEbPuDj)G^d+eX(jYBG{4H3b
zqSW02(?(XxcnT`&H%*AoYc@Y2D1Bjv%^D#n{K-pR?52@bc5j*uV*G>KkVxuiZCPv?
z7dPTj?^)I-w-cEuIUX^EM0SdNR0&=eC|@{EAmmu@agy>^s%QrF0=!e|fL^v;yndu)=Q5
z;^C=k%zwM4(*JY;F{3-^g|i{|z92P89W4xtKLy1-xLYqRdH5YPlUh%Ps~TIi!VJbN
zwO!AYm@ZDeypshNlcN&BDC|3GR65fECd^UcCh-S&+rmdDqLJ@H)=?e)8t$oyC~&O}8Fu5-E^#12%PXBGa_A;NO=`xdy
zj6W3|n#;DL4pqbEVEn^=sK`mHcf~
z|B>cR?0b&0HxN*Q{)i_Yo~B$(7!Ui~>2st6vI
zuaRXHw(2upLkT2Xvxb>Zk`A*Zt3K#LZ8(6z%e77LbLju2E(v*+C2#hEb+Yo`rn?*7
ztWwUd8s*d7>yE5mTAwsn@#*_E4!LqQ<&a!33}>%XOa%l;{K
z9p`}||VF^3;DXy$M&bcLT|8)%>pu6c;~Flq(yb7nj*nD!1m
zlVqA|Z#uOf?+8K1==~#Ramn|LF7BkB?l0@Y*S!!Xs;hQvep}cw;ztbyew6+_N0k`s
zdm+J1C+Q5>^2RsEOaen5>|jCMfZF-e-K&T_rOXM2uWBXKp+a0<;wih4<OMp_Y1_pl=cwKhW5Q<*EQK|qQOKq;7DhYiW)y{YxKHV%xHyWClnA>33
zSLjQjO22tiqw+?NI$iGuziB{78+;gsM-hwH%ni~mS@NG}90wRE5yk~8`?TQZ9gG9H
zJ5bsu|1-v(&gbpA+;39ve3`Ee^3Qz-&429qnRMSa@#Cv@964U
z02*d^&T3r+01~`8T}PZfBnj}L@!U4L@~0(UXU^_N+r4Elp>F1?Da8f5@$qpWP85}p
zAPI;E=wqT_K{@%r4Q4Z~c{{p#?~)8ZW4oTN;72hO(l8PSR+L38_FQ;{+-l6B8bjeg
zo9`)~P<&Qs5UpqqFK}<-?uNpDB0`bR(zK#AWBZsrx4hQ;iVR{}S+4Brrg_N2pO7VA
zOVE7=5KKS`IsSQ=4n*3ty$(kJrkD4V*EMe_7ufMQ298}MCqqjIYs;DzudbFW_IM6#
zqoN|l3LD9>+f%8xs%$P;r&!Hw$BOgB`RMV55;B2=ylGVPoIiB;!gcr6P$AqzF8=_i
zYK%JU+rvPV=7ZW_@SY@gmBVKBGrEr1IKwI?ebDtsODR4g%TNDRG@4|r~+4@Lc
zdyJZH=NI(5XxGwrg6;?U!@m_?TTT1``hWH_Zis>33eQxFbgfS7#}BcmZJ`eYPL`sl
zfRqdnDjX;?mGeGXCYCbtSj&t9LQYvg3Bh%RQ07yohMGH~o{iTyWb2*o*w+AgL!*n+!#2ttAZDhNB~5>N3iv~zN%aAcUmxBt60B>TFgFX-$F&(9}$*bAY3-
zS%~>1oHFzzgpt>R>@|Orml#vi$FT
zF)w;GhqZK^yP=)i@XZ1S*9T6opdvim1HhowYgtLI%Q0#^uDefx@C?72juWAE71`&i
z8TSw%yT}d{pnz7@>R=}Jv-hDd`FwzePW=UHQrl(P324>@0_3PeCvDs-hZSilSdvat
zKL9>gFNX79Go2dT4;QkMtV>@v22r{P106PE3^kw-H4pSL0roi=Xeri8d!b#Epn#qZ
zJ;4jJj!hf@&cAY8?Kga@B=z-*NR}
zo?4L!5HJvnXh2O^0I>cKfIvcH)
zmPgNhh(3}!ZCy0Fm{-`1IqHA3GCqOrJf<`RW9=ogsc!*m?*mGI!J(l`m6O)jAo{v#
z+-9)TE}$kRqo{Ztx&$td7taAdiwUR`eg-^=M^sdDKyhsisDjzq**UHce+9%Lb@!Rg
z(||x6QV}4Ec&)PKbFrbP(csPrgn_&h(DS^2dKMwEo+lg7kTe=AHV}(wIvA9yKVAp{
ze*QQJ+TwLI9b`aqz6-_kUb`Yx(^gxo3VV*K!vz?QjY6z6bK+qWgu3K2W
zfECNCr#nN>>leco8#lSN5-le*55!M0-Hs9})xLEr9+AR!ss=_YzxE5UZT<8hm
zp{Dgj`By;iAo9ouVF!&TMHZ-c$O6XOw?kT9-h>BaSgNR(=K`Q|czNmqC^NFzLkDh*
zaMs6^l)48yK-m-xqkH
zTa)5|x>*9BJ9v0aO1}ewF~su}`W^smpjOZL3Gmm^^gqF#PrUfc=GCG)$Z?K@J6@Yd
z@MZa~jTR8r+={7acvEuPAWz!*OFxxrla^d+;m}>Gz7oFCws9DHoGd)S0z16mC00Ev
zc3e{gx>eFZQPy?t%iV93cn==X3J4^FNLLKj`0J{)tgPnh^6X;ps<`+SOYJI6}3^3dZ#*BdK7zWnuaGGj-dm>g`vDG!zD%H2{O9jqBEh)mXi;!#U1r?q06hjR8Ay>y
z!;669gBQne%E!uDG=?PP{p;_4M?^&ATLUJ@m)_45n}xn~|_^;;Nq3=Cr`
zDlI}+mfJN5${DXC9U@j;X>V^KATF2=h+X9T&N!dXw(BB*I9IZu2PzT*fq}#=WogL-
z7%N9oXM|oCB7n*R2o!O3cmX>pfGYz?3`MqQTUF6fQMZ}_SFz_=gg`5x69W3h3aATA
zfvTgohGt1WAU$Z5e!LIlB>9ZSTLE$838(oV5cu^#S{N@kBLF619y$at{cdq8uRkDK
zjMcfYf>6qe@)Qdnn76g<<%I7xqvJ)Urpjt+KC!9a3xvT5ufv%b2xlK>(=4+ts}r1i
z7f7bk$2B5qj`khUT#XWQuWwe&X#$8Y)4i&5#XY_#j-WE~a8(pkF3x8+CMp=IypO3}
zX1%xp^KTiDLx`j463cBO-DeFm;UnhLU=3h>PhpMNV5I`uKOA
zgYyfHo_w|}1@LI?LZS1$2%ik_ny0+aw$K2jCK2r92MCFYfGNgqHuxN9(_Pm*7o%4|
z@2gMv%2Nm=be4vLL5@@ij~&f5zX&h{SE9rzuW5sp)3q_(tWD&faVhw&pJCl};SAK=
z)L-9qSn0Vn7tX&uH{}4!1p1TO&f~^_aZ@mM=tZ%pBqyjcypkpw#B+6y1bGeDWywd;E}4pTdk`fvCGH3!C%NfkY{xm}=}GL5PnE`We<~lrWw&C&+7|~-ym_uh+)g|%
zHjUqj`5!w9f>m{WJRHTDNsjHR4hq7oW2CWHG_J17`Nl$D#k?jSU*Skczx9MLK%2G(
zImp7Z$u$#S`9v%aocR$gnVhcsOG4T$>RGSka~<^6r)QH0M|6wx5_3;azMNkcYyzQ1
z_%B%e?A6hnA0B3%M^>);RhoO~Ti_-P!s5mJPgUR>N@|O3TIH_|%o2wq7WQv-N~czR
zrBdBAlz^47I=sA&En2TcLiXJqsChP*w=AAs*|$?PSLT*AH?aB*Bp>%}?8z6!#wEeR
zTH*raaG<*5>MScX%~!Rn6ahe_
z$m;!Lry=qsGesRwC=gF64K{;yAFhM-pFZGuL;%mHhsFuSrMhT`o`D+mGrkh>)h-
zTwk|;SS^{f?@#EJ^0n_xa}l5@dzZ%>LMR%I+)5^LC}0gt-t8^
z!223}2Pp3Ur-xAofTylM%c$MagOPXsyd)f@N(rzC96W=bU|DjZ*zOD}F6B3P0QHbj
ztI$kK)ADGu_yfcXtc-?c<*}n=f@g^>nc3Y4WYAly95SWGOm7yl{%g-e^kgLStSPpsEuD>JC}x;d|HW`5t1Fb-5lu~x-b@pMnOzDkUMZpF@Hu9@wK
z6n63~6z{*R1<*XR#iT~Hm1AgqN_g14JJWmNzEu+E(iRfOon3_5aJv3Dd+hhk6=(N6
z56YtPh!A_@_NH$p2%~bLYx}@t^I=r%ZaeMin|w0X@*vg~eX
zLmb4f7uKX*A(;;?PL*WRtc%;!p0)*%C}-Oj-szIsvoqTFQ`1RAR
z5Q!1|Cx&zkt?!RAUZKeC*t{MKh)Yx8RGNuC*4Y~6&t8hx#KVj(Z87S-RtPIO=&UjD+
z*V^773&bBr-r#d`(;28759_@Yi|gbIcx5LeHU^u#HL@Jq8~KW)+pp&JIJz7?WtQwy
zYV_V{OnbPgS=SafP`Lv&07IFd=!FEb>h;sTiH_XXc;)#vi|0!Z*Tos<9^qR&`7?GR
zy7Orex9-?``o8K`kCBz8P=jXRP?HnSywLfJEemyUW4-HI4*PlIYH45@V*WvcHKKsz
zPJJ`$Bwp{M-FW9+TnEU#obP_3vt+mHkIQm*yVQ2Lh%Ty=A?WxeLWj(!X%vNI7Rjgh
zo(9UCArl(5?*PO>v7tE2g;vl|o)`%=s?_H%p*Krz>CFJ>k8rN%zmZN%L@!abZRaAV
zdRaTP)k@YL3}r{yd05BaCIvbpqs87aeHwcCWAuTjMs#dD-iF+(GdOvyHEhwhay~hl
zB(NaVEiU%ooNABip93v$f5~6icQ;tP1twcg-xC&T8G7bgaj6|;i{nnR%7TMQp`RTd
zL089~|Hz!&S1(?>qnDMczc(s?Srzfq%5z+mRAKC9{JsDRsO$6lhZRR_PNa|`sGezT
zeWjTVZ$8`?tB|sn++R70rttqQEJE=_W$7%R-F^(`ls0HckUkW`1(`z>xOvvT4#~2^
z4Rj8ET2fe{S4MbqPZWF{^gja`hTkYEzY{I(O}@<*Yrh}Q?|LOpe!W+qNvr+;jE#uw
zf^B-O9J;!mjBZaMN9Fv*-eFX`5Yh?!(YHO5P)rPB^HE}91SRaHT)P<=RqGrIe>j3m
zZidkLW?FPC`=2?YdjfBn7AJbBhMS~bB_
z?}0@FdVXCiDXbS4SfUQhNc)l>@aE4(!&WY2`y_D|
z&nxGL!(Y0=M11b(d~Us&66Os2i(3+zF);|^PFy^T!D-j}{?f)U?-IJYH^?RAR~$#?*DX%~Fr+2k*9;SZkOvSKg@k8|*j@G&vCu{Lz^
z@6SDZ!%~eh*bFb-5>@{G|1%;2lSB3IC+Ca7Q^W>&d;?6i?=-$`X!aJCNc<5c`3Hj&
zM8DRhY&4gk>lG`M6xk|GW3=$CFKMsOXjslt1~$*Fm6)y1h?BSee!M!@lo?fyWrR8<
zSKaC|vT_H3z<=O65Kx<5O#xz#s{kcr@*g-d=f7_t8sTePHIbBJ3
z7<8Nx*Ys{(5TV8Yr>F@Wt+$7g>zi_Bqa}FHw%r34gYeib&Sy0aFWxl;;2G)0-)e4`
z(}k`qmq}X2i4;Xuf%*Hb=0rE8v2sfywaaBI^R`jb-sDbcuXz^hbJ$fezlb`o1r4~s
zsz!ai58TCgXOuHP6-nC5ckp-n9%qWE-kGWK^)=F8%TN<|)jr=D`(2ZrvTdmK&$D7O
z+m+?r(-g0oi{%}ddTp^;U8AdandR>S}f~*29n5SGoy@ida>Xe(j6K
zr@jR@cj(V2+i;pFt@175<)@zU
z*1=gb-4G7E-k+!r2__%+Dj1$Uel(f=IK_r5s1b$NgBY;HG3f!0>%~}9R^%|3hq`!Q
zYPjP6>!(HcH(qcv|IRTb(A%avbZB|yU1}UGj`HZN!XZVs@rpBuc5;7pntd~WM5hr`
zrAc^pZGX3SVaq=teuofztD~YyhA}xp-I@3DrP#Y%P7Iw#uQ&jcq5Al_dl)&vX5o*?
zpolP<4GB!js*t}0e8d+c<)EC?BTe9~o}%R16neZ_sBB(yq8q;Sc~a|gxe6^bx;v4x
z$O0h|9*5qJRfs4SJ<7seAKst+9x!{celS^a-nr-;l@KLs3C+r;e2rsB_QKwE2lvtU
zxs7kAzqCH|3J9m(BRHjn~^qqvY?|Knno3bVMKuV2n*^E+-$0&dhedcHU^w0
zTiTa~Kb$Q+=Di$zT4GgFG|fUM&VkyWK9e)7f}F-IAJ3zc^A;zqJihZ>Nv}SnK@0wH
zw*HySdZS%SwhQ%+?fnN4O5i=UF?P@Ya28xbkRC?rvtim3;LNwM)d3bD!b?D#s-A{
z#R8mdLZWr+1M&o7>%3$wW)}}j>9<}{hnL<>0>y32$o_-7JhmB$tUFI>jI#ju0LR*U
z1sjcvDWRnnW5{m`e6iL2$Iw^D8;1?P(_tU1hERg#c{>@#4g1sboH*!E9*#c~No(`%
z%q-4SsAA8!5Ehxu1B`{^@MAu
zc`oTvHTHcU-)#2T|2-B2wLnM*&{3#Jc^02lCv&)s{iwr&mh(b*oN1ZEv?1=MIt-@Q
z*d76?Z$7*_e_Z!$*bx`I@ul9CY@Pa7eddko$e6|kqt1{-*nH{!`wA2GfSi$YV;Wmvs4Pzd
z?1h416a)JIfJF+%7R5ORzlN~>M(>iC6;WMzyY1$13!uLQ<7ST70X~GeN`bDC6hy-i
z!PuAtg##NkZ_2waRvAc)^pMzIhvhQ^oi6s@lhqicb=<@rn+KT$)BghxaVR41=8W+1
z=MWCMnDnW5{U2;dYHe!SW=r^4#qOnNO-@o;7;NmLJil
zMo;$S4p#Nl-3qGks9Ju=3n%ia3E{*IcX17@8Z?M-5(CmS>82aJWXO-mwbNQi3GSl
z;wnZ+?R{oRQacXjTAY`kn*R@oO7($8>>=ISAk}G>p8$+@+je1$dgdQMQFE6#PxbWP
z=CLt#NLB+ti_`^&*z`4TP|-FTXa5rV0k9q)8ieMY`A@8;7A^IV1Qi?H2CK|hKfQ;R
zezAF7wKR>D(Gr4JqdI-pzNG?>*lJi1+0dRd{q&0wsYjoCBYflk2;@vz|4H>%ogo@N
zphyM!1oO||8LB@FfePQOf%awoYOM1M9#IGPWs#@$b!~41-Ywy$C}rb
zO?6##aBv~COM~G5fRYL9Z8mYyCmC&b>1)nQIBnH=z2nL9AbO(AtM|5v&oS#?0b#%X
zU`+rn`VUg(+YkII+s!J$P2Hx@{@s1%HC0N%psOf=)A(SS{>~gOr9UmFSNdt4Yzj5#
z0*cMOCd77zHoFWP-2e{Opov2QG$gO(R4#TN!t*@<0UF@lH{S9!Gu}?W
zah%X`lHNj_QDL8+FNEQduSB30F~^>ZOAfDauEXdx43%&&=-ZN`hej!99E1xIxpAN<
zJkbY=*YYq{584<55}~aTmE)TA4R5{cuAfHbt1nB)-}+bRpTU_A?Cmy<9%Bj6eP+XW
zY60a$W)+)_GZmh9PY!=8fF<%qUmIfT7;nh%WjK+AUHIq8vyCl1rZ#H
zPOD+l_+9HY(c`7|xS-CUpfOr6U8uV+?c>G-7d-hodm0~AnXvO$u%*L8FGB%|vQ%gH
zvzc*^XM_2K7?dT1w}g%#-f}L`X?24O{K0?=cm2w-*_}S7tDc8IZbkB)DJ42>LWAp4
z=a1+8dY)XlsCo%Ur9y33%+t$}@IFW%z|~f7cYgJM|BmCTj95e9V$1OC(+S^u4wItv
z7$GpwRdz9cP$Qdim*UB!*<^>Td16xAq)yNHs$m-oH@YV8Cn_7nQCva(Ex5Q(oo{{*
z&J+tBWn47(4*l_BcGOY*8<5RwaT_sBy>TJJ4vZ(f8qdqu{u<*ozMV%)a@lTQShFef
z-m5|zvvgkPGHU}-`Mqa|EYpR%0Aphk+8bS2qG$+(U$1qxOqV?7>;cb%#0ulFcprAt
z)*%2;qpN&Nz|wfeEAmV>0H1jJ^0fnr=D~A~fD7%3
z4;$D#HMu}K;r7g!2H+%x^$}}#-i^@qZP$_i#=Df&4zfV`YP_@8`(o=B;@4J_(7`F?
z=XnOv?bvzFE9@?!xa|&t!{Oz23fY>Dk~nR<24&1iy(>eH0viL3|9N7VZ=rp#RsRoJ
zgpTfJNlcm0%CE^(8ook&G@YRHFCz}Zrbnll*T=9OZ!6u~FzHuU8bGkNS)*n2UHZS}
zzARWhnR(U@9`A^)TF3!=#O&}e7KVu=J4ZA#_@
zASUxv)HZ7XsP24(9Pw;)xor%Zl+i1@4F|m>%Au00mw;U0G>QNRjeT>}C)TsHlYD|A
z4X5sq>iV}>gy6w^&IM)u0jNAn%Gu)CDAEHq;zot_5Y1g@`Td{sDD|6F@O~V7q6S3E
zH~4=^$9yGhV|uxNd*f&c2|Xn^2VZmi?srSEY}A1{zHD$q@vp9`Tm~PNe|_-q9jw}}
zw6}(nhI(~6vH<6|Tz3TdV=c^?*^Wmw*YRcg_1%dhvd&@R9YfGi{Ndh_07Z|a
z+=_Ie35hJ02&b56FSQYxt2$s_LVS#lcJe=tO<{)2{$DFkVK-#Eo)J@6416Fz0yx`9
zRMYLkE7P6E^z__OuFt}z?(XTd<#!}LKNN{@q))s5lH$&t7VO`_y44KsZ!efS2o(w+
z6Jl6C#;5oFC4p}iNBQ)#zr3i35le34*p@&abVJo;g!XXZ@MRKFvc
z&SU2;ea0Kad-n1|y0*&@_hbpqxgaCWVZ5kc;zk&)w*`^!i#k8O(;Q^`U1mc(Ekk?<
zDNV-O^d>5g(VlLLrZGVFVxTf@j9eh$stTe?1y}jvJqnt<_ReaxR+CMYMk;cu@{zA<`
zME%n}MRC%f%QBoEyK=E>4ZYgwgbDZZ0K>T`HKN3(hJ$bymF2?`9+k-dKuB*R^LIOfeq)mh~jdGF2SJwo@_
zhrW>9%_HIx=#5tLuiRhjp1$_Qk;RxD61;jeZ))u5@zWk8qQ5+47ONS=j}}d%
zR7|SoLP>o-4w4c*wy$OMvO67a9_f8ZP6ZY+4U;7d7BbIRv
ztmKfq`&mpUb7T1Eumq4ix}QvFKP{|U9Pyb>^TR;n7jmEHW_$1lg5Ryz2XYI^Q=TpF
zs<`*z1pThu3pOWXbtW}_bmyxZM4f@6NFi##rtoo{Kl^>(&vkB?n*vD@N;yxO@4U_{
zc;i{loqH0q9}f5Y}TuIMNDc7l}Ozl^b=Pb!+nFSH_4
zq5{GGpKp)L&p**Zk$hG1IS<|-mTmnBr#=~z_2l;;DaSC+yRpqPrI+&r4}Ce#xS-%~
zkt>85$%5Kx$hz;YXdhVhNDef?PrNi9E2rTGucqOeQ+d{VaG+gY-1AjB`ukntIbN9B
zke;u?ma=PZ16JCtN90li^JUW*V_v%0VX53`&DWcJ2GO+04xo
zHLy2YkwFt-THrez+L{~ixJssjXas5RGtNE<_}Ra!$URc2Y!r4X(~qFCF+x=BJYy6j
z{KC1e-;6~5+}*#07J|_Knt92*#(m@?q;w$WkrroGE
z6n;sHt^T4ZkH;J_+UIvPViGCugQ8@^thw5dwDTVTj
zG09Vmj4YYNo}AKuHoDm}eVjP1u%i;1hi~nc(jMU2?dbl~vRuyb<;ud8zd0ACrRH2Y
z1~UF2apA*kG~T#^?s-!dCv*kW1KhWIhT;>^-^rEeacb%GGD%VJN?J_e1H3?B6A4_}?i}m`hmX&9HZ`)747bc{3AJ2~Cd*5AX`TTn
zHbM73>2>gY4T!(fi+?PB&b`7jLx>Z&=2V
zg4DL4VA-dJNx(#&7Q(XKcOQ?Ty%SR?lEF|Splbk%8!PZ$_Q8QVD)DUBJ>|*|Z$CbD
zojL1U_WjdoGP*q@q6n9q@b$}ds)r|oEjCV9bndm_T)_Re^uK#gH?1lQg&l~4fyXdDo9gB_|Nf-cq&OawItwB;
zsg-?yU-Qn?m`D`%T~I~G#no?z?#_v!JDKN=9i~sFMne*Jf0yh9(JX%d$XHcAFDll7
zMOs=f=U{rw-_PZR&J1ooNs@ny?jISO+D;tB0AVP!e?}4|4gUKqXM90zWE<;k?ggB
zPB)+(jcBphF^GZ06Zsdj8|ViWTK+Dj9w*(7Gf`SGnrUSN8H_^0HDP?;A^
zguCfBjBX^LlW=Fv(Uahi7)F5_(yQZhs)Q8Z07^fpd|f{jDbiFl6=vaPd^~|X<3A=G
zeBY1EU8;L{^@uUmM`z6ky)oT~eVcBZWJGp{7joPwEfdYDtqD)PvXys5BL>sU-Fd+2
zTH5dkB+&$W#pCUAOD}-qG4YR+AzY{Mrxo+2dBtoL7?n-}_pM+QHM!e~UHAYOEwT$;
zUINy0oAyXl_aNr+*3~m!HFT2rTNhl
z*o5zDmN)GxX5;bR>3;NXXY-$Kj(gE!L|SSrEg0D=$<$nQ_sC8s?LDa8qiI$M!#&AI
z926REv%7)8EZo7De^SYp2_XcXFY*cyU!)1AeVr9CQQgUSvrF=Ojg`l`z)^WpV$TWh
z6Q_CG|8kXR+TzElh@!5+)ej3$E>kU~K$p++MCyVK-4S9?oWA$E2{rt|kq5r3dXB!c!;#JY%%Q!eLRT7IY7^
zx_});J&w&R^qTIzi0AWwRM!xecY;li~i
zawd&1@sZ^x1SXhERa|7{IpIsQj@9&dfPW37Cs9Wdwj~?csr71>jQJVMsd9Z1A(nu5BMg`qe4w<;q>Tl=p1@dsb
z)IZd-ErCc0m}@&?7v-D=_D}Tk>_&%g@YJho9c7|i>j&w+ytscUjH|!r4BF#jiin8h
zsWG|i&(C#P^}GEXp$A}J1kew@%xhtQALYxft6Je0kWvSLu@?oAp`k@k0&xsQtb>9+
z>y4i)Q}v3k?Do;QLjO)@YEK8N|E#KuJE~_@NeD%^io>-xzqZU++2ne}_uPyz5_VSy
zxo3~P4`gL!;US>mF*?9NeUMLHun)T+jjRi(Er%K0(3CLpnsrU*c_uuxup-hdc!!%nooS)+xXyO6t-+v
zH7pk^%Jl}?eHh#^&3ze-(|R1!@Ae}1;?Un8@>si~=jX{=pW3VK^OI7GrR;YRddN7j
zKf;9CU-l2z=TR>2SCEGGRQ^2iG&SEa*$e#$(WNW;lJk4ZDnnZ{BAANputYGoX77vK
zCPmJnO$;@<&OJ%5H*H5tG;FKLEZ%ui)oa=YQ&j*cT->Cjq|NbSviQeZ0Ux;{gg=l*
zNjDsJk(~p$vyze$1W0ZIIo)ia3J>uIkTW9C4Hy9?p1TBO3oHQyFN^~;F%Aa}p;thw
zept6;n36j{XC}ui)VX5Zop+0r*)Iq`#7D;@d4ibkeT0UO9;`ROAt*@xYHwBy!1#b*
zLI{x8L<5^e62P%x8UaAM8nkU+1}MIEply2CLloFxS`R8b(7mD<31Cfot80d
z3GmEM8vwSLnBCAHI#C8Rzlk|5?|+t0G8SF{(%_k8W%p%sMvQ&Nb!FsW0o??LfB+dl
z^R*6M-MOS+6#TXbZFv8<80m
z4Q{WzkpF}ScCWo$Vig#$B7>_{-YD{<`nPIchLi=Zy6niwOH`}7;@~NNXI%jrtht(?
zy*Nyf6yV)ps;!#~&CM6*pg%UyNJl2;b3%l*%Lem!2sFt*#x#Rj@A>{+%Fd3p0P0oq
z8G!nX0i^DH-|sHs*Dn`X+jRi)_HQ+07qI}X7-3E90bKtL0NCS!?d^u8QleWMFgvRY
z0Gq}De!dCxd;ucyjqOYo90VrpTLc7Wfz(F#$@au>p$6Npy*V=NVnWElVK>zZ5Uqzn
z%u5WKqhb1{K-VEeI=g&}bXuFr@Pc2PA4TqQV>C=zTUr`9no1-ZsMRiQr2AD}UZ=F0
z!V}s*gD6#m%;iUr&}v8ywE3*%6&(F)j1KKN4sFzke`?VL3@tD{%(tWR+-Rp7MlPkF
zYn+~<=wR!5Z`Sl8KGsf$1z7~x
zbeNWb)p!wFoNO@A-NnsSOs}fD1D)PYj#hg&0W`9qv5^D77QsW5DehzXQ;Pv|`DAXa
zrluy#kxytqo;e(}2%p>~-=6>oe}FMZi;ay%#bDOHJOqLvT>$(T1aP3@4DXjZi420M
zQDI1V2sP+z4p85m2H9ysUd8~^t;7@ud2Aq|vyJfB0JgI*0$croSxA)(IM|`U(jqa-a^P#9mV=xv~@o2#sFRyA7F@3d1aXs31uK
z>w{#WvHC~c*GZ9V+>BWZbU5_UyDw_qbjVe63J86LxTR)(xcfEseNA|@x`RUdg(plBK706~wim0>Zx5p;JdZ}5
z>`}OyI4OcmqIIG9^*H6iF|aiVcIISh+E-tH-{ns0HDx*u)IVGSC{d|uyEz!@6nLja
z@Y~O?j|Z7K-{d4a!P>1WDcuJeFu~w=!z{*lx+GGsDgD!z!QyaWD*>fiS`LokYap_|
zl9Qkh52*lhl@p}~-GD^_WG4;)ayA1X?ZNS^+#(e|ca_?zZ}!oe82;#7CUtleT5?HN
zMs~(P2coIQ>2VZ@i=951Y%P8SPl|u}2Q$%ju$4D9Q1aEnVL~L}OoPRdIbN5#tCDBH
z6TVo*<9s=S$YnL!0{AD6YkdUHUd8GW;CNA1T8)W=ar3|0(=~o11I!{A1FXY5phvU=
z8{)7%&T@R{dvggB=3d{opQ}R>yZ=qjNL!Uoslq}Bplg@5w<7>TeQB{ZklyOyK6&3U
zIPL9&gZHpVIUx{0_aOrv@?rW2V7eTS*A;;J2aS+Wqhvjxv#bQ(6NW%Asf5Xah)wru
z?|bD&Pgj_60S)Ti@=GUyYP5BU+RAoVncd>Cj2T1^aUUWy+{+(S}q4Pr=;d_OU{GC!&V?IesFuWzXIl5
zsc?+f`@9S~!A?z`(Xo?wg@9$PRTmAaNQ3}?{6?3RjiX7sxJquJs*i2d=wdK;X}`GT
z$IVALh!wJjiDXPyjW->c$ld;kgsra6Ah)%Z%W9hKQ}b`4c6KFvDROdhPxlwN0WN#A
z0Ez(|TKCz4A87KDoE!m+iXqTWUhf9Zf)Q*rIR@4|6;(d3boZZvE-U&oYZaN?=`%BZ=fD%D_oI2OiG>m>S(?UkVVSfeK!H
zNp_BNS+G1qJ4{Xju6M%tA1nYEDqenmevR8MDM0?)I5(e`SzYnXerM?bOVB!~W8{1N
zv+4UHSP~+g7h>5@KTOS!o!KxKN^_jqAov5hoE5NBpZWn3i6`t-yKY@Ks;vSV(7|SQ
z3@y+%fVnj_HGFQ{>D3RHvqlDdv~q}H$AA~}ti_BSK7{qFhL#5
zffT-V;e8-{0SGxX4;|fNgi*NPPteMItQ=cl$nFF#T1W@dctnpK@|CEe6w2K_m@)LAh}!Ixw@Eu2{M9NdW_?0@)xWAO`d$?P@!{
z%*&!$*y+KQD}_ZCIQQ?vHW18`0BT%ifq_h2vQB8+%$@jmL;J{`;hGK(KPsfOEwU(`
zSI6aU8Y)RXAF3^MK&1KIk8<{_E!c`M=LS=D0{lLb;NW1$<9#m;lAPU{_zxd~KqwgR
zUQ{78)5njZBLv7~&tAOfs&$r;YGje}@_c;eKDIeJ%#|dvy1P9HO+{^$>>2Rl*rOoH>xD{w?q;|CC+O49NqM
zSnfv8>(Y5nwc2`
zgYa<+b7?~WAgLGvW%WcZYxP19!U0)*JsRS$^o!T&`m7irS3(WwI(3@S1fAxD)>e^AAyTwyqi;JAg;$(B$U^+lnV@!f)v(CIutEe#BRXEL`
zEv&Ji5#2+88gTNoi}llX`EjZE>^E*vdZPPO#cgyr=slmVd88%d7*PWhL=>l|^=V_(
z+$+(4dj-XA5oMiYax
zVA7;@wwMm+jS+fjat9sOAL3&}u)N+4Q$L2>2z;Q7f4HV1PdQ1>ccJZlQ+Q(Z=*;Hp
z00TTu(XtNZ57J(~Uc`AXF31}5*Y^C25J{R3N(hXqL&mQ#Sho$sH$khRo
zDyLZCuTx2B8{$RA_hj3O>o3fN_kCx-7KZ{^j`Vbcq;3J%Ekv>V>>zy#iDyCdhaCyP
zKnetQceyjd5%7aCNjZD$PU5=b&f@5B<)q+C1U#gR1U|-?c^x~PL{qYCKROV7$iLl)?D9yk=_`dTB)TdwF4J{&@2-$WOfqgvz$?YG^_Bw#p|kn&mLDa`
zDzXf|B2EM>WKfv|!peT15;fYzS}_*(*Dhiv@m;!=x+G;!zr(
zGcqtoFljs|XJ-!J5Oo_p34qAXz1=Agl9j^DOO5XhKbTv}>8}dK>e(P~>gnwj1JN${
z!!9@qmSY73q&#-ITkjI$ts49WtANf}mQni~@CoYke3}wq`ll??r2r?4vfDzFOuNaRWj1PGBFmo;q9tZRKLNT(@LJR0+yj2Mb
zt`5(@hH>oBZyOr_Tc8aRBjO8^ZEe<;Jk$BY0Vn+)jSbt{cI>#mx*#Mm)arcAFM+*E
zLp5RHD^{7^0=En}dSGxKFj9I_xBYxWSZycbK+s@rSYoRIM
zr~M0+EZSa2kdim#1rHdCr&2%i*Ks#6D+;pM;fB|)R~=~L!%s4#v(X}W9`M$PPgs-+&2W-p
zft+1q>q~Hn5|uL4eD_H(GTlLQ1nJ}Hq=?21F4SLy>`w;B23-z|m3lgJnssA(EjzmL
zw8d24GP*s=`N;kX@3T)6RU=~)z1yQz+7A7IxBDjGq)19ioimq8N@ANnfrCZPU`Jhu
zu`rERDXe_l9D0oDrIs-%!cQBVMmHY-CrvpirglGbbi)TT3}|elLWGaYn<7Z^C&6XXd-*7BfJ0s$ZxH
zIAz(h-Irc{ID8S6nW{%DLT=a4&|l;TYEE#6!%Tv>nf~(HPNht#=@EKgD0)n3^$o+Y6q>g5gyiQ-F5z&hlK0T%;POcuG9!?o*tPz?CF1_
zuM-NVy0K*3W~a-0igGO>&au~cw9p(SjUen-jv{?~#T*pG#V*oxdJ|?jw*6ePavKQ*
z!L@-48V!LJtBB9coH(t1Ak4`$qvwCCGtitoI8@`bnxiP6dy{l^b)Dnh@N13cnHhSE
zY1<)%wS&_HKYqbioaqcc&&X7?J1Zl?Qk+^=opz=_dD`^(??p@V2e;p`fy9vvkEh(>
znSwFjQ`zh13xk4Z=BxV4^W6i2Yma5py2=9U4W_Wrl)-6#X09Lp6c!kcz!p8p_m9Sk
z+a%8qO2{&XBeHM|+n=n8>yd4KGi>jTVSjgDx7h)g(aksJ^80kQLVE7d=Rwt>m*{1i
zQ_+bHi?1FwddpSZbzc`l{P*WcgO6pZvh{!P3O;ebU6o~vFp+zp!fAQ&kd4rnh3+ad`{na}=Hb0^G`Cu0#yicBPwg}8J8^@H
zc$yT9uWtQMj?>#~&U}N^7}(B8pPQlAkBXjJ#Arc{2Qhya=AB-_!f0--B)aSy4Uq$$r!l9F2cw}nGn3Ozp(-zFKHehX?w
zV8jcz*#E##-xBK6OR6hBkxcc2OrF<4IS_$}%dO~R3jNstmRjk=q@Y*6z6<%5R5!Gs
zw1w1l_S|8fR-1t7AOT!jQ{3&r20pn?#s
z>&)6*)xc=Xo1?<|LDNE95X%*tu~LGxTfw`~qS(0H*PyI8dbfvPkDm}9tL4A_3eE#5
z0q?O~<~(AOn*Y3#%dt8W+!j(lQ&~{q8l!&S5VT3O;O*uMg!0?$4Tqby=RwjP8FnrL
zqY-xZk{Xtu##xOPo(o5unD8%7u$fDw`k)BI(bTvsD@s=FYAy5ey|P<87-V)3;3AoZ
z=R!Yvf-}ut)n*dc1#YwjR``8&ZV5k)G>ZhR%{K;r(|Hu??^BAymp*Ak@CS=f(D(bO
zw`@^tI0)23?#Sm!oAk$bca_u7ZcNH9TIjvY*YH6h8#^w*enD~|iwFO^(lqA=S}e1p
zE#CZGP@)nYt@Q_Qzb|G}qEQ1uH$+^#F3&NF=MgK%gU&xQNsd3h5-OuzX{>zoRmD}g
zGx$GZKwgVO%Z0G%RbB>nC-mW?#_R^kG2TkL*hckKaR1uj%>93XK!(Gy)T9Tf
z<9cc0nAuIcXc5&)RkxgXJfeCo{$<&fpxb
zX9^!Z`GS6MGw`7P*K^6$X93irw6T*FJn<^588UbTpHB`X>K%~mHb}}Y-;x^aYtV81
zFg!c#am4gw0q0Ay>Sx$M`)8xk9s7(Qgp2DrPhS+$e6bY-$4$C#iVoG57$K)R{UTAn
zQ0#KdIdoTew!tMsUq>_ZHcsCcShFX7_Cs+g9K^We5Ryk5a)NGmBcUY@Q9ayMO?Uqk
z{K5SRfCzjo^%DwgfT#E@NQn{Lu-xpbAJ5VFk$O@w*xpwl(1z4@F)9V4%t5IoWUAx6@k3r+RH}pOI@*%(xT8(gQaw4JT>Caq~
zC+YqiN|M7e7Ff9qR^@%=z)U;Sk-*FFe}W`#|2N!X*}c9yt!v&jebH?i34G|Y$wQ%W
zzjJakCN@%RW%Cv^0q}!)qlCr&+2b@I|q22tFla==N
z;P&gP+wk^GrrdE5Nzis>9<+f!WoRkq(F*i=Ks?<=MnOhW1|sW}`pou=Wd4eRQpqz0
zrt|!B0o@X-MGPwA{kmXH9((jrEfUl`$~VGz3#>THsJLjx_4nGtlVSjp1Oeh1l^&H>
zGp~8OB?|0&CX1TM_K2T^lYM&Oq4Rh^
zeQQ2sc1M3j;&qb>RGp~U98wB<5zCafnlij-)o>#*w2@Q|2SK-`NxOF9Hcn`WRJHXo
z{{HDK1DXec=*e8;9R{Yc@cX(jzM#0Eu-10#e0p4!c=e=Oqz*4v-A5^_ksiSy(O|tC
zKseklZ=&cyXWe=JBVB+W)X;b!24H6FX4Z9wP7M|r8i1+Pnn
zTb?V6!pjLVq-)OyNHa0H)7nf^Nv)PG{xY=eYhjuICvo)b9&7(tMjhsR%9HjfP8PRY
zXsFNzhU2erWW6l4Q0t6i>aS2%9*)GDcA_uKwMk$%zN7$E@1)u3#QU1-OwR0QV(K
zpMNa6AF-4dQXlcI?F!IAEY)>he>OrVZ!T!va}J(#tAn7hSq)|96KRh^{dLAe3kN^v
zyd`xx|Jkd-ZaIeX)|%`5GC0QQ!?QB;5ifVoL(E9+Tl1Wl!cnb6?w^e+-K_e{Z}YM`
zu#&s97QgH&K?WJ&2E
zVe5|?8aUB#Q);ZV1dDgbc+nKZ&$h;?XmLlUhA}p$M7{d#+MZt}bT}4)Vc2k1M@o-d
z8+3zil
znQjv72J^RGA5KhFpc6DDX`jw(R#7rK;cK84)*$}g)UjsQpMGCcvu1yRf%EX;J-gkP
zV&BN_2y2AmJtBQP^FD$h(+P|<^JDrb_84v~BY+)%Ak}p857I&S&rQ?=elH~5d*}!6
z7Q@u%Hz%KkrCMMdOkhy}wAxTz)5x9)PV7cGR}p|~0lD<7sNY|yMxa^!6Anj<-XOcX
zONi)g_IS7+I*vyf+*2Oo*%X6L*1#US2e@JFXYcd(7(4T1yg7a<`&C3K!Ob?m2`w^=
zxv&2Z+$8#GUFgn3nc~(f=6jhQ?N$e8%AtDLI39UANf*xs`-ti>X_NV^2sJ!b#xP>D
z^+U-a4!@i-e{?a>o~@zF5j*`=uAlmog2bwZ;W#)-BjQYn{|mLE{IBQO|6G+Qb!PnT
zf|h49oBjJgOkyn0Ag=Jv
zgCqlf)0CY^wIbhF~tSWTiE5|e@i!QbMQ2*_hPR(YnB$JO@n
z6@2;G%s;pydYCs;mQSolgB(oURW1KYi^3+8StASWa`aJZwEHl3EH!ZQJ$?$~!(b1J-H6&&QT6}1eYNn>l@V@tRhSbg}#E_Fz&f6e^vdfi
zg&OP|z=J6a>{O4S?Ow*RmrEkb#cmRj?zXiTjyVZzAeMh*Yz81aK@e1j`6t1rs5q-C
za8CKhU0>s$N?<77^XE-l>2f`5ET9xbVUy~-oeQH-=h*rEv9N*%t&;)+Ui9@P7Vh1w
zf7>-V%ufiVds58b(wt$FIU2U-!oY9x7iuAIlFnqFf9?ivDYfXVZHic?C
zL@v}d5ad_M4)GT&ZQmQCqP>`81-QnbQM*tOC=X>wX%v<$y@|=qQr_MR-R1t~-IU&p
zACu4|h?p;LR4Z&?csCFyYmN-7K$GQf{8cPoc-?*mVQw)BW&M1&Cb#O4mRI&sZ5MPi%B3&KDM@(IoN|aD
zi>p-IZ(ZEU-ZUPOc*7U2tPW6>`QADcpUua-?OWxovZenrQfl;zY@uyN1aX2br6VkL
zA=~%s>DxYh!qUv*6D?;7TOkx$+LmJp#ORHt6_2A@Es347&jr=hTsFT!oJaUz3dk3K
z<;>J1t8NZ)jM#1PRrry(f
zr#|H#OPGaMBIPSy9d2ad{816+{#<@8sI)V`I74(HLoYWm)=L8=+Ht0;80~arE4qIQ
zJt&jbje@v^E40+57f#WHt3EvmkJPbPZ_en7P
z{p$+C0hma_bp7SQ0zg7uU$X!tWPx`SC4_yWUILq{(3iz8B!-4CTP&t1QVnsu=;|ym
zz`y0=iNo2Y*=V#)a9K}0?{&Wu{;HW+=8@P)<^s-iUhtU7Y@-K3W|Xm<;O8T(hlf4`
zsbsv@$LO;!6y8*cMg=s;2XQ~1>e9CwwN_d73e*qTiQGQ7z0i3pd*@p?(CCGcRIPF!
zj=JuLXwls!A1qDI#No&zo5McInvr180EKzm{8CMiiQ#c)2N@z2yKK?y?zm(=Y^r`!
z_~p+q^vDpn1=?_j#lwi1YEhdz;ZsL@y}Yn0f27-BMG{Epg1&IN%z{
z?<24+Kt>}i-<$*3?~~BZWa(t0@1pMVD1zK;KIa-3?T5RCv7UQ*CnjqCl1wazjcN=b
zY9%diNh0nmHNMEV6({5F{(v;&xb_Qfw$7Un#qScnUSqI(vU&oa<;r4-l+Z`D_HqwE
z=C>`k`5vkXj}vT&^H=3XRFG-87uZ)wbSIs33Pb!80`8&&0$qn&&
z^jBk$5j|ly%`ph{dzN=`2wVbUh%^N9MCE;KLR@sd+^OhiC3W}6jG@aOSw)sZO?`D_
zZ-hbw$S-ZIk~6B!fyZhU)rOFDleBaoW&ZQ-j34;a
z8J@O~_);UyY(n08Ka4L=N8TpHz3vbqW75sefvxs^<*~(HnlONbefEHS@O!s2AqTvm
zSLDl>(-I{K=szexUJxco9)2&%?CI|VP9RXZwEbPa9vlQAcR;3bleZ$A3MsV;V!a`Z
zU@Jyx#bX!M`hy7JOt?d)!>tXJ;+A$^8$ckqVfZwzq%d|x5VvKtgiid@ru_mugcHf*
zd`aXEY+R|8nt6cwCga-T5li=Lh)P{(iYfX9OhH~C+~8nf+xGt2(WQbfRlWLvR9iF`
z%mRYyXRrc>e#d&}UE2zqx9K7IW3B(0k5T>2$N00RPXelxq6~yn!%_rzl#1QxFq6uO
zW!V<89QDuddrh!()Oo084D^YDUHL2N{M1KG<6?rn>PqQT?gOJWo$UMgq_D~mvlo1=|axBI0^q6?U)^5JT=
zYBkx5lO?)e@^}y+!tpk^M8dXYryS9q!4ugX72WBeGww{OME^W(r-(t<;N(m4L7y%Y
z|BZ&j-w0T=nVE4)e+8qZz+I7ZE)<>Hr^HSA@e2t64>{WzQ~$o)T>{>Xjhx2qk)+FD
zQwL|!Lh8=JmC@Q#BnVOj06_aC*h&LUSc&rf)QJ{k9v6Jvh85$f6%ps=r)4TJm3u`fyq%+c`n@;0rco4NS4%(r5C$A;XHeZY11=77x5R27P
zi2Y{-d4wJ%gqrs;{10IfrLFR1rRdN+CdbMoX2?NjH0JrGCtp55W4aEt0451PFN4i9{{;~dH=)r)X%6jj@BRH#4;o6o>^1Ow7ed>A1i=^Ntz<)
zqxGxo`QP|`ugk-_8wLqgj-x?^2O<9taaM~M8OgN%@iUxP_J3fs0OJYE!ejOLya^&s9G=Bm~+I|?4eD%5bb>qQqk#C+(yCR?)^aP&l4?)Hd8aqH1^3x`D$&ejhhSja9
z1^N#bfQWiYmPX#KC?ZtwJvAzO64RZX^0lv-HC0u`ZBkWUE{$Mf&7Cf(u9?n)IzxK_jc6?rPAZC-MN{(v^-LF_XSOf%2TQyP(g
zAQwEcj{XMI31-`mOE9V{CgbzEP96(^ntr3P_t2ve45xg5g>C?Zjx{q5fnxsAZ13y-
zlcMNG7sF|2f2N$l9ji$!36`hLaQ;9$|7LTmPi(Mi_Et&C
zZX!u@OJ73ZETIO$=jOtwzsv!cKaSB&v$O1P_5-Vlz<~afxK3+?q=(%G7rdUe5M;Wc
zns|7hk5?JyqCZ9Urg1;xD38&(DM7E({=Re52$sEns8$KzN%3|UV|IJw64RY
zJ=%hPiYJsfP`#=*P~0#?X_GZy{fh3t*8i5+?xMW{NYch3IK4?Uxp*)eCj+
zWtkg3v_gTF_`DEqf1$CsUn;3^rycrI2}U`YeZ%tleJd(?oubfdivy&cwIYE8}I6|Qr995AyjGTnh$+gK&jnaGK`sT=y$U<$ToyO4AjdXcc8{;;~jfQH8aeasa1uX=RWyT6C6eG1n>
zufDLoN|Fn(4yofHsaXv}#pOH=ab_W!DV_AJ3SyL1eka%^X?&r8mgnMiw1)d1bEcKa
zs(rvv+JJ=Ht@?2^@;KOw_5Hn7^6w2=5g1=wEpY|hE4m6X=U>DEc(-G1MFoSFoQuaq
zUWTj9&En5A5sp`^7t%x38`C5KA8T#gG75`IQ7U_U4rj(T@ghaU?-5Xg@IX~bYFp+5
z4u41gh6W@;0b-?6&vxI0Zi~QSX}eTfc{`Dp2QvMI(EiSZ!5IrFr+27JwQf0mYL)VQ
z3<5t?(1d>Y-TVILcmMveU40=cq}n9nD?e)pB=_uAZP~r;lq-+w2i#41a>Q)?&&jGC
z1@b!|`e`9csxTh&{#~10f
zFH;!sFibKWnT4veGiYu-qmZ4A?B}EBp`p7r<#jI__5?ijWq|St3N&7_ayJY_A#^2ol8~01_g>1)Q*>{c8+TMdy88BS
z)|0jb1;j~^`Rs5DPf71&^y|Z#h6APo{GSzAJ5cY-93vUBJ7NOaj7Qj`(_KX;13UUS
zRMgYh_~V`M5Z#yB18*FH8g?*2%8WQxWbIqTE65ZNXn5Fn4e-^f?=xAHvZ>kdGOT$w6_jP4qQ$pa~(}|GqB&AtShqs*sC1HAs`*Y?f@YP;N;3BpU*2=YH0*8WXjf`B&iy941BdBZ&pD~ZAA>0
z7foLm&Lh*U;7l#<$8WlAXYq=YJ>y8SpG~-#w$bNASOSBZ{BX8pz=GLbT`Ij*o2`Dr
z_TMg2e)8u$L25G*SQ-fv!I^srQMu(0(8rqx12;D_y+58JVcEW(_{C75Bt_U?Ct+}T
zQPIHgZ0myf>XZ~DA_RqjR)ez{{{|9iLD{}K9AZOJfb#Pd1l&zUHi?AIwe2<*bU
ze1w0nb56M};*I?=paFesn!4&~i2-l0GfFie0DA{pKUVzTp}z~Z$bc8&70+WaeRFSj{%DRglLsF*xl*Q5h#zkhD=rxoN-lJ}kuSHei=ubMoi1OJuK5Wml
z*-=o*wlAW}vYLAJhNQOXzmnRoX}V(pulajH0qQ^abH3|OGO!SyH8YE5;szG?C0}Jp
zeByCS>uryfLprxoWFQ??%CIkD+d_8!SyCW={{gVWAb2f!_%D2@8o_hGS%U4y0&cPU
z)UnT)D4ga70U9NP`!;dDn$&nJ&w)=bV$f~2vjAZVAkH3gP~r>++V~KCIq{MICsq#y
zwRZ4$hF|q<@yn-=>G&r6$cxhy_Sf-=AwTkid6U&cYu?O<+OMOCu+3vKWQ~Lg70Uke
z<~M&La=z5B=4xYDu_jJuhAGYY0PX1HcpxquZSY8R;~`aUq@v`W%uYrxu>$aa;P)2od9v500GnBCy8n@S(5u_p4uTX4_SpVwi$WYu
zAG0FsN}AeNHRb-j8O~!e{KLX@qaOz~ep;`8^D<3I_EBNBHdYTPAJHC3g4oe|0M|m=P=T!2|jaG9^f6kOaDBbF24i6ceWLNalT^sumFW<-s@AA2oT*25sFuDN9xo%gdb-mnX|&w?p#IbX4`iyk}@FjG5AGh+twblaCd!`78sJ|eAnIf&LH7*#!37U
z-U8(w%0sJ5L$6Fpy}QAApKc!LjUTz++$S%#Q>)i~X~^!4ywo(bA!(JPd}a1}@9(Uu!HeJy;ct1Ic4XpPs*
zDv|aG1xGh=cgGe=dKj|0SlViV)<$uKD*{e;Vz<+GY;|;-0VDgtDWJvfsW>W4(0y?=
zs4#)c&l|PntgqkIU9{fF6Pk5)xiW4WW;d1VJ5sCVIToua6!KJ81u*Im1*1So!^CL|tnfYT!N`;%peSWp&@rP>
zghCSn-fhj#k3pgH`UGGWz?dbWYuOeq`7h$2I%b7Xv2in-(Q5sZPl>^5<{AXs$29m+
zZ{@^70_N)!b6nD8mGw|A)oiM(^(#Mfyef8-_xV`unq$*gFyJ;tv7(hwta15L_e_)`
zf{z!!4js%#mZnsCJN-(nUqd+N>w?R#E)(wa!pwa{(^qZe4O99x3qbAw@EHQCbC=!$
zOgSKBya1^4Mz6EeX+dl%VfpVLxwOg*#ga~*JbkK+A3QobivBSPHf--y#oG^avHpEe
zJ{-AdnKZ^hsny7(Um_{yAh24%KOytsCw{Ai@_ZezDy_yW4t2kSX)Ii_Df#65;NchL
zh2EB4^_sO>>hE|oX|RuD@*Gn(!(uUQcz4Jo3q?I>ps1*so*x^muHM++Jr&mhsMJdA
z7ue*yqMn}opt*jg#s;8@_S;s^Nds7PBmceMYcDKZ>1y4S);WM$>u;#drdI
z)x4KYLLr^5WUe-+D=z8$KimSlEHahdGN>XhJ}4W&wi*Eytvg*s-b3h(R-b#x$?o^v
zWACM+f~|RdO}XdK4^QzC?~{=90zH=oA*|n{6Bn+{>hBpF)10<={xU(2Y>=OSM60;X
z$ve4q`E(pE=@gw^rs`2eIfZeO4}Vo%-#8P~M)jTZhFe%wUz+A?=0_&UJ^(aOkII+lCC)yEA;|6f9i%ylxYe^WQ!H0_xfc*!<@E9#8
zN+@_{9Rl-jfUJkd-OX7oQ2D9ZZ@SW~b)x5)cc%kLYS&FyuNxwrYC8&mhF$|~XqVmT
zIOS|ecF+oG^4DE`%Gd!rJ1y5DXRV@yim|*nt!Hn6{(?pRL
z9=qKuB0|FTksAU25ZIgAv3Twi6Zg1I=vfpbCnR(OYe6jJ$$5Kw`+1vDF^x0Wqhz4j
z@9uUMyaI48-T)foS3pexG+!&_4ZuT$Q9WTYRe&$Zs#_xpQ2i<1q|@Y$-ADXBmz=e=
zwS?&i$M+Rs1{dC&bMuK4tL~BHa17zX;n`O*3APmu+v5GU=idhER4N8r+H5zBYID;^&i0zVEKAl3n!u(BDhFLyFX4~WP%b#CdQEUs
z>-?d*dhqyE3p$_HTs<@M15ok-Gw^Wg1P3zXlr%w+)^4RYkO{VQFq&gGx
zIXwWp9@EwUq$17oHe=>iz-9R2b!HcF+I(}Gth1rj;IjmN9MB$NR#sO2JOSjDfNC2f
zpmNa(3sd?R6+O6Ic^iTKb{)TcCL0;r{Y3-3@iBMEf({aW>Z4
zh^&VhYOC&2?a;UAciJ32w>Ig;+vA5!m#_JsbJd)Z_wi>>W}ccAeRM1ms;TgQ21R3=
z=40)-2Y;P%rR(7Zjc3$w6*^xG1lBYih~tgqt1L`Ws%#uRuD@Kq$_2DR;5_?;ss8!K!*Vgv>T<^zEzKz%VC`Scug
zD+mHQKm3)&CTDwdNkU%Spcer(J3CBbPp4t8*X!5uu?f(66TIBahDoPEp~dP&Xur1{
zaFo5ZyBKrLjcL0_Y&zJ-9W`-?NLS_S8)y~zSGdgFk+}*Xo`g$KD=R_@JblyMsxaN*
ztQWNsUq>pcAB`SPb?YQO-S*W2SPF>3RX$2*kD&nI}DWjoHRP&m2u-Fuq1
zCsvFb!qq?Xr(9k>iL>`5l{1c*RPm|X-tKN(f<=MK&~wig=#F2%;?jo2bi_F6aCmei6rBR1|0Jr7e+uF$_!|~y%<~qtu)L#!wFz3
z5yz}?v~YKu02(^j)$ekH1`X1{n`qU$70-yXvZq;(1Kupmy9(?#n4DY!BCY#oW(Z&k
z62f#6Z?7+4-UDdvl>z!C2xX+4oSt}icz|{sI206EEr8(!9Lhc5-pxRq1!VbNgP;kx)Q7M5IHSp_EcVO1eR$JBAJ^@urpTl8&LfMH&XA
zq(!=8s3D&{-uL(SeEu(ZGt4<>&f06Qz4qGIwJy~>ZXmkW?k=DWwDth$8>LJ)m?n~d
z(p}tdIs4fNwFa|v7;yhQkgEy$-d2C-kR0(uh;RSR@1`+JpC2-f)gtDk2U_XWrq;8f
zuWBcqYw=CTY5msCgFay%{28Y_BpA`Xr@wYhOiK3aW*cY5r9|oV>yPrzRSOg&JLDuj
zG5S<4wn&6&)zpx#Wy$F~7h+)^z{BR_FuydDOhJrxo?N$C)mqb)qDIe
zbf5yIw;`V*a^6@x)RdKoZ+oeurzk&^CBG~f7Enf!snSzaqM=7EHvu0$T%NWR`^Do-
zoP9Cfct&h!2kD)C)bt
zrufu9$$O=9LJaiMrg_9`mFqBux2?HlW48MGahOI9j;8vXr>YwIA11cz2uXML-?gFa
z!=>H2Nmr%0^I>ezcp*KMF#9vtmCabs@CNHzFS%Aa%Pi!q_*mGuFrl&`TZTjO``!MX
zSNr)n@=@+Up{wMAg&Cq&b6j5=@Nl3!uf(~vTEkI0o?Oa{o
z<6S)e(yAG6bKl{rSLV~HN5&-dre~;SarL)^`5Rz*%fY8N^M0SgkExQoMEgSFw=4a5
zx68`}waF*eA|tac%VbhF2!Ww(amA24rNd(e`Q8BzG_{+nUG8g`Z*bSlqx$PSw|WQq
z`EgD*wgk^FXH^%))Ob7BxE#y0W-I07RANr)v-atVA!||2KClEKC{Hk$_FK~7Ywe5q
zK)p3rvCE>>uqma(Cx4`2;t}2klIQly2e%v_icbp}unk*o)XU4dGhZT4f!p&FF&wX
zf|vD+`xQ`}(V@2Ni|c^^zM*O4@q}t?GZgh7{z^KgShP8p@V(hI
z6u&TfghXP5lM5%bPSu(2e<>72&MC0#l>gu}_ZjuCFV?*2m|9eNKO3ou++7b0Z}lPF
zA8d%)^;2TtWB*KG2I~=ZVX-?KoqNEpjU@Bi@d`T+N^VB>Yvw^15kZbQYojHmVmEWQ
z5Z?8uwUb+)A5q)4vg*di<-%Rv!4=T6`z6yy_j;o84t||8b|W_xW~q&bUXd|#hC8&(4!G;*F2YXPc*=)W6%ZC^v&Crcs^<;~9`O|CBCmg%Y945QDpTs9|Tg)`=q
zb{kc8W_(lcUMe95i#jq|{0fA|Cbo
zUt=>)dpQ2@9ys|PuxNkR?8~?-ZI-{)rXmIjd!-jCa*k-3*~tNq!<(S<
zm}6U%VcTXW9BU?{`MH>fqj?a2f8Uk0l?2@FaJDuI{z$0Jjm#>wAC>7|_=Bezjd(A%
zR@ocRngho2IUkWck}J?Qnyi1rQEPKG``)!?OFFsV|K-hi3AOr>VMoC
zEQ^`VmMr?N!Y*0%+RhO<%#bl*OVTNht=MDB=_u`Ke@HxB6*c*Ph*rpC;qC~h57EBS
zCR0ORYGNC+aJ5B;r%JBfy89PDl(JNpsMovJYP~HuKLwqA0y~V_vi~|Fp?VM;OFuFp
z%LUt{NpIn~T$H)?iJ!U0kpjHtdwk|NAOLPSlW;fn%Ot;u8^#r9+^~#n?Osg+Z4`|9LWN8k$`~&
zp1#3BdUif+<<}Qy;c17}-L)mERMFFoj}g(	J{FIL*E~(Wm{Y=T=i+orSp(5|O|4
zUItvszLlRp*t5el^r?kiXj%+jU4MF>D{!DHC)Sp|EwGTu!pvtwHIN(2{*G&vkevEn
zdxg~F(QW8>A+4qMVwSr8aA~(wZRGWVIZ1#lTR6~)v@U0zDt-JT_&zz+)jFO>14c9Qh?R%e
zS=db=PI3MrHZH!=X*Kj)zIm1H0J^VUo07A4v9MN);>&)l`X;VcIsDKmXu_G4~3Nl$kK-)E2@ixqPL}%sBI7xOfqz`(*B_P1(Do
z1CU}ldgfmlId2}nmLD3erwp#8-WtnRu+3{d7Sm2broK^Jo-a%xcQBC;kh=5sq`Ifv
zkhA5hLe)+lQYZDZ2#8-LHu3{+&{3rr&8OG
ze@|^VL@P3O7e0MtqT8WA?Qc*=qd@MfDQLHODWPW+D#>2-z+Iat-(vp3+VRZ3qDK-R?5lUrG!!}W#Yst2fTbmgKyR;
zZ$w@8@1(oCa<^R8NWNUU4nA|b(!!-qe*0&wb
zzWY%Jm(65SgOLzZq<;1+35?gK9jn?#H$>BX&PJt2ZE|jSWb~j<)l_D=5lnRzpL&ts
z{e%1m6B&igH5XJOQKn_`teH;rk4_!^Zgiaf-Usa`nqzvbp4{A!}0GjsiD}InBKJezq|?85I}iX^*Z*
z{jMKjNguX}-$4Z5e}}~=IFFh>J~*HB!-=GNbs9k}3^gOQa`?D;!j&NL_Bavap3H*$
zaD*n0;{q9tj}tdq{+PIA-!S7*YeT_{;bVTio8RK&Mt+~>CgbSqpfY{K44#7_SM|oU
z(}J!m?_4<*UMfwR5hS&T4pk0
zSIGB*1MlSxF_vu|p4~vA{q2*d7mfxk6&#NU+CEe4L=_c1)%U*3D3Xz!6HnN<#l?eF
z_er?!=&*m3F(ai!)GF%sYq`2#jCeg`{&J2A^)I;*A6>Gox5bC-{h`CeR63KR3zaIU
z*}pZ{qF`+2-yeTXZnKb~Xy-3i`dZ>!(
z0#9Zm(zpwOz4DIo@P#xtLD;vV3uvd%X_a^|ez)QTH@L~HV!x+Z}mmRG$+&k=W
zS$6RGP=;jOFE7&f*f_t(m!`mN3!c9h56xQ9o@{m_`efu6$<%TW-FLjvgr1lpTq2Q`
zPkFnQv49?(N{o*j{OEZRQte~I^&8WIjvFfRb3Uug^kk*d4rjLV>i)9Gzd_MP#dYZkC9Ry?uaY1Rs7>
z9p&eLddM|WHj1dWKBn5G5PLN%8)G@Uph{1T7s-CAkG3tUL
zYFkhJH+^(wbCR*C-rxyeI}?rGt%XZJz@=O3PzqZ+a~$1vBOZba-TuN#$ZU(1nXCUH
zbLJ)jE*^$AQ1ka@P!Ra~@4YAG@A32hng(x}zWqV?sC)YCHjMsPh7LigwBPeEo#i&^
zzTx%39o=yFMGL*2dls$oBliM-BYx^MDgi6E-T&}zp&5fwBnngES)I*Z%R#qVg(cTL#q!PDol8!G21uxyd{GwRqRLV@o?SW$Kjwl7c2Rb9{BX8I%k{Znyu=Sm*
zrH#s?Hx?JHa(3wXj~6YIX+6b)E3#x_4##oXFHLcpPk-h#lv;#k9=mRBU>Ea%r`
z_1+qe&O{dz4$m4sJNjqn2@bJV+}u7cSWZ>Epo9X0G@4UpeW`exSQUZeTxjK2FM$wigAto2Fw{Y?z^UoQS`Z4y1B2oKj;5s}ubq<_z+gKWj!WHmQQ
zv>1ym@rc)*x)G&({Uk0L3BbctX&-br@kdWn-@A-ycF-DyrfoLvI`@Q1)RyI7Vma(B
ztH?Mr&j@@kX{FeK2P^IIrvyVU!!jbDTU*~?zL9x=|NCb?0r7m=i7zDxN$2`glKM&Scd5P;??2nRe?=ygnExkKnBexE4kn
zVu$&XaElNO+%)K<3l|Su?Ua7K8Q{DR0+o7Llb?{+(R*=g*IcKAx16DO(o%v(+MxGT
zvOJ^0nTlF>!Uq=81C)A)6wgkHz0*spDmw#D@HQU~@pE({?c7+KBF&y#&uh={VXs}B
zei;uv#(I%IX3R}iGSdWU{dXVGi4_Ql!jkl7@%lJj>=%x@y&4bqv@tzTFDpC@UAsZ0
zwFQ7G)FK2mXV8;Qt;Ll2~YS
zefHmFC?wfj|8iIqfV>rq20SL*BPX)s-ziUb=en|XB_0y#hJ1ms-&Hs5A;_rkTIO-$
zQ7s&+e=s$VBqnl%qBFDinvZ?06RP)N2*&W}nzWWopp+e}pXcvJSBsOF^c`FnV(aab
z%Xx`VJZ`XlNKaIOR+4x_ryz2gXA9Fs@0F7`9%k_I_^@EVaeap>g~8ziT%)$sJwe_)
zl1P=we|yh68KN+gxTZWQ7HfLhsD$)br3!7ky)=677up8j@q;7ywt3q3#Ny+hzw5w<=*OeUI&y=Jm3zTlXZeB%T{zG!=;d|zvPKJkx7mPuM
zhcQUu#*bF&iMIyi-)e86$`OeRO4HaWe>$)0t6s4KnPOCeb(-F527(Jj5Xbbz?CU2|
z$DASRe^Xpw*{mU^Ku%ryh^9NNx;~y|c&B$s39vi`GUPWGS*t5}G7mGAi{w9@MUZ8i=-uK8q>3TwUErShsQZ}w~croc5BS%y81HW8^?mN<8%s5#`
z^tloGuR#>unbLPSXZKPXF+ZY$Vw+z{zHu0*3$?ezjeiH)&W$5Emakz?sZz6&h1vsG
z`||*<{7CZ^wt>-0WR=hS6V42gj?e4j^iQ9}uShjb*DN_x)NFlP;QFXf+4lSQ>4Uk8
zE-J|-YdyE_=4&zk4r5YiYD3Rh$l1|J_^Srr9b%K-)plM?q1v;@Wz;uh
zE&JZdwuZxNO-Q)oR9HsSF8NH#_rU$@d+q581H#qIuRe*MmdV_nw&E92z6!{@c#a;v
zs~YI1w0;1_!B-@*QaieReT)mwqkz_(g97I&9`tR6&EvIreyuP
z0+Z<|)O6WospoL%AJUwzmPpIE>7%P_KR?Sg$XB*dMw)pYYB(RA<>zEu31kk{NU@jM
z8lJICElLC*B+j6@Y`BRu32?5SFSdOdsm0o4$uAr&PqJK9`W~-u@vAbd@Ot<}euMW&
zdu)3ibFs%zSHo%Rx8fF~tHS`5+x3C4t=EHG->OKo>IPTre>6{Dvf!%j&IcBQBTMI=
zQ^dq9!I=FqqN{Gtu5qvTilTN)cE0Nr&4t{Bi}vk~lCK@Ho}aL6{tMr%{%#-3D_7>gZJl6e_4mv-7f09R&x}!vH1NYoZ-I@Y
zT@tPIj1Z>IvdW8U+;qN9X5EgMNf9y4%W?ePY7H-Pxf5*_w(|ZeEBKe^VCgtd9-;HY
zPiuUe@hSln+r>F}nKJG&z07yM-se;2zcH#GF>1}GtV9gn4kyqaLyFA4uaVL|CS8f~
z6i?b0_eO7NUDlo%MH*6^&uCdRN7Owd(H_I8aummENv8rjpHR1AX$pBpJyxPMTb+G0a)~uk
zF7pO$W#j7VN?t=ISh@=>Iwek}EVCG8rTz;@L8zL#j5<9Vd{|pY@#-*#8TW@d-ugdRMom4Tdp3oYv{PxMIXRLenR$*x?xyj6sB*f)oEd
zFhvX+6O)zPE<9wUNxz}qrMws`am}2)c-zOk64Lhbmm5z=r(_&v1-jm%_}6-eGw=JC
z=h=_!HhZQvk3GYpPJC}RuJ4i^{bGM&Jw9cln*WBfyR{`f$M1S@n!yjTJw;n?*{8PW
zeaO2U4;8p^D?d+%C#iGgu4#=Y4XFo2&IPimsY5qtl$fr6ME95-4ouM=EPt&dbJ)M_
z7n9Izqh*J$Pp4P5;@=67)K7+S;o==I@JPc~bhqNP;bfoVt}%QC_%ryM-o5E#exZ^n
z<&8a67xt7-St+aI_s=5sSKV0d)TQHKqg
z14q=RtJ?mn3+3{f!@f2)g%&wV+b{ZVkXnqpRC-CPzyDe=a%c=fr`mu1>)Ld^P7&N`
zsaP^#B_!x{1B&bKi`#*|Cz4a_8y-12)$MpJNmK0ht#Vt7)Jy5ZSg9rR^^7zjO$kvX
zc1YKx?I3dbR_7jFKEZyj;<5l5G3m+@E!o>YaXzn`D+w^a-yQV2l1LS9j{3yWM5vv^
znE+3tdr})&=(B~4=^B4t_6QdbpGjo7a>0JEhqrH}6BHj6=87v@L)v1sd(XVyArTJG
zFiDc99=taSVcwESxfRQH6Dq0v@#Qr1&JsO((3=t9GTFE|xFZ9wu>15S_4`ajb{}Xy
zXvJ~_kRc#NuG%$@^LL>ocOjbnR>A|TExos6tFEL!Tu}_Xuk`l#XE4kY1pVxK5CPSk+zb8yZG4LZ$!~%O8Cn
z&b04f>h|581p|5L9s>_pcP?5IKo
z7?D!B@6y1Bd_$+%E4|e%DpU+Khor6QZkCANEq2vgi?CNjc)fde^8OAe&pkUc)XY)q
zHC@?C`m_gmgxcW){993bkA_Q}Ilrc`roM1>WB-^gS)=)O*HOmA>Un%XtQSdKskm+0
z_uQJ90f-)A5SNFNt!l1bEUOfPil59XR1R^IjlSF*+Zc%TrMIn%G5u#=V<@X#&&oQ1
zaO)|UN}!?ceZFD@(y6|Iz7Iol<|8C&qkLBV2|+Rwze9(wdk6YfX3x@}aIEqMbpN&@;>vjwTk111Tslw8=nyn!p`NiuXL{-3G{
z`PUz7TSA=n&e2I*xKlSEhb|3EH2LdVRVvPyb7xN|#XaH%))gPBSBVGGxW{}nGT}De
zr=iN)aH-z)VY{E+NF-&9j&?1puolXBy}MvyS?W30+p$Zj-F9GVm&!fQfvD=4fFr&p
z?5+HqUIw$rn3?(aJQfW7_oqYHqs7Lid05~U@SY=FysKj+2q=QYq?yxk&P9Qo({h%%
zYY$f4(AX5XgO4&fFx!{xixIc(W%D+fZL{^wxl_K6(*oYP{h>&E_;i6uPHXLMLsnTg^%5dYhU$VI{1)JT|55)fN3C6%a;e9xij!ZFf&A
zrrJTO+~));6SnhSp#D
zUnK&s>(iy9Pt)reLEMTsDJ>80QJk{Pg`znGNoT^nl~sq!=DzA{=eu5zlclOlLwYW{
zf=&Y%u1m1T&doM9oS=qwgcTpF2YTw)g%+fQXJFx$dayy3f_#vXirhYb#rQ!-s{p)F{v=?(*Aq^+7|}NRz41eDbGCM?;9j
zbluf?U3Vqr6N!`xS0trxzSy3r3q_TxtiZhvFpX+wP>O
z8u|3N_nm9#RlfeyscLMd5javp^Mm?-2}v2EFr$QE$0z!m$#c?Ke<81%FPD<+uP#*v
ztt_flL;6{?BP%lkQrEF2n14@
zDqINzA+qlzp0#KDZaPDyCx}%A{c)UqC%sGEy(A35%VT;Uv0ij~x3*s6*cn`5-gq>y
zds{#-(<0$}e!)%V65R&`8E3z?JG3oHuB4l}Hf``x;pi7q?-&wntrIlXW=mYgp7_5a
zF7_01ys3G%Y48*0LOLoE(+3^!xxtD;@kV(|bn&KYvCVhOP}`EVG=AagD1s%4-6w@a
zOI*eEMMN^AU?tnxwXE-25G!d$u-Ih?CM7&Oey@}-?tOePOBNcS7pT~i?FjQaJ{c@$
zCXhg1bi8;cUhJm#XJ2)i8Q0}4wyDu2^ZV}+_W9})lNuY5j{GG*H(faG{h9wxR%z2U
z)tCPcPQi?5i>*=VYf!xxh%|Y8q}8&_gQ|P3y9>-e{7yE)`SpOVrKg)T`^hTCy~D#m
z)s|CFr~^yY$C@TiRCH8BEp}$gJe|Ya@duG7^lYet1VDX48UObnLXrks
zBTxZJ`i@?$HKm-`xDRw44Q0512#8iiJyU|i|#H&qK|OG3)_
z%*atamF{W3va+JD9C-iQl37Cd+Y4aRf|X=`Ygu=9A<(8B=%LJ-ETdqH;Im4jjyrW{
z`YLC1RJWpLW@g}c7qp<~j4EW73;v|HR@|*KL9q5u2ejdOOH1<3Ny%(lKO39k;h4lG
zi2yA)0m>|56-HtjXnTYX`Q;N)7n!f{JHBnsQcR-Pe6Fck?Ag@vcT5T6;{NVmq#50Q
zx1;|Y8$$x{rx-*LU?}V6T`7Td3)E|o1WM3ZmC^Gv^ln+`#ZqzUAu&cHAI@@8%C?hg
zzJ=r+D5ykh&^terRHFge{>oahH0E|gjWwRqqFMs}VV2MZQ(|2pD^BD8EUNn$1>NuL
z1pU72H}g_ZEtOLQ?FIA!qV5BLAR445IfR6si+WxFNMPi8zUIaRPk;dj16tyF-*Y7;
z1#Fdn?qQx2k!P=OrfkJG5!l_$JRtK
zweP1XLq9P9->RuMlu-eiA5aV5NYPhbXGT#ia1$#lR?rQb0bt1isty6ZDLhvvY^c=%
zGP8w0yXx8`*I}5b`3iPLcm5jvGwWW<*}^gamVW@<{ZVK;pvd6}kgbUZ84`ZtKnv@4
zl*G+%0L}pd8Udi0|HAQwGhFNypdtWFcGr&|+9z!Q94Q+?@dKd3z5{(1G5}Zxzagi*
zd_>KBZ7>N183euJQLua0y}7%!2<{(0Z<|u2I4ko?f@UI
z-RQ!yJq2SD7ES{CA){@6eyYDv(0hDpNEa}+8~~PawVppgaUoa=D$nrr(`bMBMV{_;
z(0FC^0KmD^&6S7i&aV#8a-1VT{1x-ZVJuBYF!WW;)EW(*d0AJ}|Hg_U%KvO&*BwsY
ze6dyB)zkBcifRzZi8MGY-2rHH3?mdMnvDA+z_Sq%6L;;_E$)q5rGv2s9skM78rCQz
zB_-=`Z!S>~P|ubHHS_*h79dgM4_pHUR~yNaLIUZYIbRqW3Z@4nJ(vLExDgsLtE7-2
zo(9^lk7mBS3-*5Q=Sv}gWCC6Z{o`^^)L5&J*BI{>z{>$2d!*a9Uc7wK(!t{d+X3(s1yxQB-l!bU_i04`|kk9^ojYuPOk;jtU?HA^S~qXtV6S-
z6BBy@8h>*%2Mee)@WWb8DIRm`0SsY4u6%;iggQ94=>|a0lC7TZZXIv$onOxlymt9d
z_vYmRt`K}P6Bws#KJ@`L#HIXPh>I%<2p4gC9$Q>pUn@*0rHl4;1Y)71u(Uuv;5Tra
zJfP8~=Qa;L-knVVIM9&vEH(t2#-y{$3XPuQ#jz#mxz4fooR4pO)p!1lh64^gD?q_~
zH{yamxKK
zW=;evVd!)D+)zkdYp3)ch=wzXS*Z5A4>fBbXCVt2yeGCp1y@qSb2+?ekT(;|)2;74
zv?p3^iqqz&!eO&z##&q`By*Zl=sqKDUj=Ky#XBp-n=m!y;wInM`dsZV%IKw}8iw*CruD}Y<}?D2;G
z2WQqf1pD0bGb&K?VFR%!q^(Wj&6_vXcJmY;2M6CKTtd}lg7mP?Xn82%Ia0`vp)D^x
z(p%e`=>PHJNe*oa8BCRYSB3kQ)(haw{yPuG?m1T*fgZhOPMg)sv~Ifp(Udtik(Mbh
z(NL4wH^x#$W2D&*`AMzfLTfvOC3L#2rn|-lHldS~ZbrCfGyn5(X_Vd={wWo0PiiP~
zk{YvWdyPK&6grXJ>N=gRG@+r3SWFePFMf}Fro~?jvpO!cntXoqDu)BVnxAsYNxWz}
z#e1Rz_SFdx+IS_l0thADI|07|HUG71Z2-At-00$XQ(8u5w8~rw4S2>Y(7PYt@;6){
zz|TiFsU*4)9ayyA0LCAWN+?<}4T92e0DxNYNE$IYptaS}(SZuzKrfC509_-gggXIp
z9OwRh#5jkp$59`X$5vtfYd}u|1B-E=^kAn7?)0Y%2XHesXY067tRxT$@&Qct>Ud1)
zgPB>fB%>F+-0N%)Xyv33G039W2SC^^Ej_1Y-D00c59m1c94vkJI39ha4e*OaMLhNk
zq6rr+GBTxql3kqjoe&e*hoa)C|78EVZ5=3G7CyYK8ZcoYY2s0SSvVp+L}|!YhDnVX
zXuZqPw$7>KHIdsc4|sbIc}Zs3Qfh+kZE%R6x@s{R|CjroBGpD#_w6%P=3OWz2DtTk
z3F^U-bi}seWQVMeiP)>V^t$LtP_N}*k=1o14wgaMa)&~gVaYlOzMGu*#-gg$yx4IL
z4--@JpP59zV$~O2sRPp>ruR&XN)Lv7eUzp;{<)YI_PvJ{cXe7oC@F*zae2zb1eVfQ
z1_m@>odqgqt*PQZ7u?33_W(z)VJ+GEwY+=|5NZI-v+iRzI1nC#Bj6J7{0v`QGN~vq
zfGEYaM%P^sq*W6kP*yaMHyv+H+y{hBAk@-5*WkQ4l69BX=a`5<^Z*U`HV{+@0!vK*
zz`2c4J?7Bqk|Gf60(S=sG!_t^G9Sr&38VvqfZWNi7A!B&;=dE9y}eOVT8|Xj<4!ZF
z0qTNVlcfWGx0i+>foj;TofUH3Ap_hTv)!4RypI;hqgB!C<1sMJLO|?MjgF0uMl(7i
z2gvJq{U%qGJ{rjPx&cu!HJR4vOn9aV3sg>N%!gC^z^W||z}c1y&B8z6_}MNc>WJ9!qDImy~j
zru9Cfo-uIzQnt&L*&sLY{zrVow!oGarbN7H7$j7Kx-@E(EazLE>kEyGp?Y5Qq*9U(
zk!6FGH8aKILKJ5vY~L!#*@cOTlVG7diE;9<_R5;Z{zm-Ttyq=C3c51T<-Ht58igxt
z|A<(;mx&)D1_`g!{{RcLf}
z41p7eBxebImn+4YHhkeFpv2-E&ttGk4WHC18-6+C)z(tC{uglzNV01HGV_l08;|Xj
zzZEVUd%s`wPt=wOL4~LF3S=rek1`Le%_GE^90fhZnHI`VYI^)rHw(59K5y`coldAYBe(|u@z6S0dD`5|^2`#?B(#$<
zO!<|_J!g)$KeOIxnNceRnXg`_dfC~)FV4Cc6;tK&;c*YxK8S*yi^FzlofG);^r8^h
zbb)|13zTRxG?LQ4aDBq&N#BoTLQgln?x!L~N_rJ67=ywED%VwygWKv`RCr{{>ykm7
z%wkV9g3H3K)ga$~&X|orzEw6rvm!ViQsXqpU9dwZ3dQxgL@MJl0ThF85i&Y?8oEk5
zr9quqwl6HPFK>3U)k4A(4{ofYBa;DDo2Y9j#$Q&C?XS$ut*dK8xO!=}J!=;8v#&!D
z13^8s=%^vHc!#%2jj8|ig4J+(I(o?U-~%;}C5B$)y${{4d*ZXj-yW+@%f2?@)v6>C
zkJHznDljQ-ZK>qFh8O_u7D(Lzv%3{A+nCfs+TV0L@4J^(yFtC}>&}9O)z<&V{gx3Y
zbBU{=7?^JFYk)+SY&nUNsq&WvLJv&Avs%!rNusB5!P!#Pv3lB2dv{#<08(9L)zIJ6
zOFh9{)22b=FM*B}hht76{hm8ddR^c{ZS~db)BhbL8ux*N#GFt)|J54~JdSM(`!ucO
zLxnICwyKi5oVybS!^f874jxt{u>>XUM=yRc7Nd4_pC85vOGh6*ZFgo;^4{~NXZ~^;
zC&G>@v7-!gSdlq!%~mVg5|`ywr&7IgC9Iblz=j21`niBVid4q3DQSFXQ_@~Joo^1`
zI&z6#^UgHC7A65#v$ElEd=NFHg=M=R-D22dG5%~l3JLyuTS+xW;CCLrUXtsYP)k^M
z<($Fjjkf;q@KEw3E9Vp-yS7kgmsYxI7ztNCEP7wZ&F+`U^!LZAY2zj@0{mY+E3uA6
zA-;c1`1*y=&(^CVFRlM73+jKIUnVdD>SkpN#dZQm$mXaMg%2$CrJbF)W|uhJut`yS
zLg_F6fcL8DHrz~N81}YG&C_(!Ji{hObNyD>yH@kCrh1%xOLk#E!jCD%S7>#|Lkk?*
z%X{f9YWc4gdY~-RnO;Pp6H5cjRtquLj_WDj+x0QVk8AiZ!Bh`c!qiU}A{dySP_G@{
zx>yNZ+?eD|hY3WmIooArp|jttiEjypj+ZR!XlW)-P6kG%e#fBCdg+bajp36kFrHX2
z@K`>(6!PA%NrH+JxyJ38BQ64|6y#V;)Dud7E9Pr2>YD!rrFYhD5$dR|zEIHU^03?K
zpKEP|6&<;HfnNwLU4)x~*ErX3XFz}TPc-YvJAr>VtN@hRLd?2~PJTUqq~?_gz+IL|
z)^#{!`0QN4$Q-iZsapp>f14~ltC1qZAxpAi_
z!uGdaU!+(lPQ;q-$aiw0oaMdR>WE}FV$hu=H~X$=>0!(K!s9i%-G2)v1q*;3$=X6O
zaA^+p?@tRPtwfwX8uT=I1O~_0bM^d-Jl?KIf&kQTWo=>K5fCW8<(fUV)pw8omm%__
zAMQ57awsEyW0Yccn;Y0PFA~XnEU`b*ytnLP0r@AN78%|==Z#i->za6@+15z{r7`Xr
z;spuT(739G2L8#Dee)AdKPRbumg-Qy*e~DBoQ@BKKP`$zL~`6j`L`G@jrWCtN2)Nr
z=L0;D(47)*z!fo@Pfo+zLg1$9ioLl_=Q@Hf=9k<_F%HZ_^uXSPH+j)SgzIjel^fYaocd5{zWbvka^Wtli?|SK3$Pq$9
z=^+t$*R1NK?%bD*w!`OsRo+YktzI8z_qJ>%n_M%^xyi`frb6_l8U3l>sGggBzdb}K
zVIemH6^K_J^O&uOUKlKXYI&7+)U`sXUHzL$RX1(;T-JP|f7ko^ddz0_khSgFJ9?DC
z?77LXkIk(YMV9f)!L;j^!ia!?)(wTdSI&Y5)0$C;^+nZxlnVMNVI|*OxDh|Lda$Fg
z&Gvds5}&*Ap)K#HD|s&`?+6`ZZr6%^-&`3Gyfc(xoO;g`WttyDv6Vz;lppOY;lRr)Kn
z(kQq~6Y`2TM$|3Pc6Rs;AFerXC)GHRi71JFulA5iLSC4GEqdw?F?UZd$sP@8l6~4n
z;4K@fnq}~ALh3%w!CZckU#ZZ)L$H8P`QLQdxoQ?ew~qH$b>ucu(NG@~MAUAPT^>qL
zd#86tg)Gkh)BR7y$IeW8+w+6@42~@lIP_q|^(o&!&K9CFDq1bbTOghAjBX?fGvdzX
zL>-2gE6qB5F{VEDYscM3myZZnUWF*)9Jggr>k8c19zG3;sK=G?6H|Gf^;Do(>XPjd
zfWza;sBDbau=wqZMQTiX0Gxv=7fA=x8b;k=DbGNR3G&N~#Ef`fU#sMN
zx+kj57w;JP{goTNs3201w`}_`$4=wj^>7G-+9^oGG^Irb-$KT6e0J*5h^M
zL*<4QXNxwypRFUA%31{n-%pp_-~H?xpxH%1^BiPkuy{anxgH*h$CQ@KELVv5nCl}`
z7xL4bNArqG?|;Bp)3v~tj}x)(lm+RRO*~J$L%KYzHEC}XCY6C=kJO_7Wsm4ipj-SB
z%3Eo?>pXjNNhF?#Wa!HeUy}ZRxg!=9nVZ^-d}ST+5%Gv(b*6s4uy#5>tBFXS2?E?>
zrXHL0wJpKVym5E$T=n%0vt7JqD}N60JQqcJVDOa*cNDep_9Yam@KSwjJjqe;L1Jx$
zt!0)kN0|(!AsO-&&XN`C(iPY*3Uwul3UUM5+S2Dzy^G^iZ6V{P?Ft-7rKxF`#v-Dr
zW840rx*A3vNbe4fTqi`m*yi-vXl3MZ^&J2|DJbf=Xq38UHZy$Oja*sZ?i-lmwY2N&
z7gfT!e+r9sO#NbLk@;qxxqESL7?o;cFEOeMR~KxdM~AOdHdDu{o&FztB&7T7e`^RU
zZw5i*TpPuJKAW=~u|0`}z5I`lXcA#)r6uLty&Z^f+L~xVGko)2+qBV5Vguhbwn$n9
zdM;RdHaO-CkPIwTig~x+6SkP~7>iGK(ta)cWH2MdBVmm`Vw!hk8%*T2{pGQ+wGYCw
zj^~=9SO{8Dvi%d2g+$@-lo0)YvQ<4+r`1!1lHc>!ovvrjo;WuozoMXjspRSpP32tUM|Nh=LR+b4_Rx#%ogJuN^Mn
z)Tman8bm>~-btk4BzmH2L8Wz%jZy-cN|B9}|fXNYWOpq;a0sI1t(NXkMbt|#Wj
zkn^(oTBbBu9`^p1qiVD^jW(MZscMi%#o7j|MPg{r3~Gu@Z{;htyyn#F{r%gY7jUi)
zCkKiFmu7WtS`7lVeyi+&R+aDbO4bu!!d|#NYWr+@cdXJzj7{AHs<|mBcYn?9_dssj
z9YQLq_R(*srG2#4hhYX9z8;+4`uiyN?awi%#H;gnG>vZPrlsw7fLV$*4B%fAblb-h
zroP4{4K!3)Qk;}(LNPFqj>iWwKi@fPUG(AG(bj)t2;ns1E}#;Q?QGyB=ldL0D-v}S
zOO5k~>i)L3p9mEySEDADd6_?^t`$wU>|xd%`X>Sno{}M=*>f6v+R7cWHMfue|B7n4
z0KTf~eb8id=^w+umOrQ7gNz;|It^^H6HDg4Dl5a=lnAtjG)w+Jw2=HkLvM*i*tAqg
zSALS%wO6;;�@bPM&TyuQHA87PT6SFsXrBiIUkNF5O%7n~avrNc*CjGprlP8_3Uh
za=NFSQpy6K3o<(smfmAy_tNyMsp)+d;-m`=rSsY^@}~X2cu?F~w^hMzAV%kEC038@b=F_Gyls2$8kFrZl*wo)%&cqg1R@O|%v3Tnf?Om5ogB~3j@^AeC=PmiE`(OxLOpLx|A!CY}-_zhYVQVf;X
z6_x5iC|*>HEZkOOp}puGe7Q+v{Utd$5-;N8_}WB;GyME&l#h_d`C!KW_NAfKQyvk)
zj@$v>kDnb7iIjrPL5arlgf!Pwl@P)2iNWxt)hQ84LcV%$FY=&4Dro|JY@!#F`Lhnr
zOU{HRSsM%6=q~#wXwgZ*;If0UVUymiIbuO@^DW&p@JBtgZGC(y3lP%G;4v?^0$zfP
z)>tbXh1Oc|U>63e{xZQ~_#)8w
z2r8IL%JAzD`nfBLTW7>R!-2=Yw*B7Yt+X_8Nr3S0qaWTk4|^TN_Xm5t)G>T-Un~ZE
zL3P+;9yvFbNtkPC#EqoVH1Tj|`ySmj`;oq_zmG2a&xiuy3B8cd7WA{Rj@nk;@E^y!
zta#TCv&HY>Y1_@_Jk0x{B^&GEO49XNSr2lF-o;HVccvYCD})w^
z?kE-CK}HnXRh97qZ;ZYS?)}*VNn(J>@ct8f5E;=YHk>i~Pn(2eUZh6c$z^FBpWg+p
zOO1#8w(7*&TqdNaIC^>=VGQXyO@Ba!+Y?1yiTz$0`|GeGKisfhJm<$nUSgR{#*eig
zj2sJ3-wQ%LQG|GxGIjr{K4u49@@0aT9Y@z^!c>WC8t{`?kM%Lg!t<0zjHHq4;vr;2
z0af`6L1cMV?i^?xbdo7wzxdggRb4bRrHEJhMG)#5hMLgQ={X=tFKSjrdeA7-d`aH3G
zG*`yGC8c~549ij&dVy3)q5H7*r+=rcH*9oyt9kD=vh5WZ+Ao3sSiJLjLZNvp!B=)e
zL!s$M5K+_;COfh4GRJB^(=_l*#j5{|`TEKeyU-yWw^EHC}kdm)A*)1n=wd
zE8sByXZ+NT3>uQX{V%stZ@9Z<>7X}spO@%A`}kt;6>feew=4S9_}q;!9JEU|=^*@k
zp&Uz5L!3O$CrzMx?}OR$+8jN1cYOO02K)_86nQJLq#38(AT_p>%{@DIGCUR9&_orC
zWiAXJ=PwX*H1Om#q{-1bm!4tU)yY{18BIX_`qY|4>&)@h+VQ970uQIw&-}rsKjE+t
z0scp(vLzwWabereo}Gl0{?-W3m*XMXJNGFSraV_L{gsuvu5y
zNJ(UEWs`73hF}23Or~R(R^YHx;f3mii-F~bPb8G#H>|04hp`<29UJ-#3gXrsL7*NqM9UQlERAk
z7lWf?LaePnq3J}hl#E$LH@naUg;S{&9X<=%5q%1y)(}e2c(L`_@$iYp-Km>{`kU#o5=
zNWcUUDv@tp4*3gSdoVv)oW#
z3BJA^N~ZzBvmdHuY4)FXu0YP*=sf2AiPc@Yp#=K$V@Ur!C7zBr?{-Go=#K9uI%hGM
zGVv46Wmt;CpR^Wmeu9Yp`VDswX=;?NPGi7X(R;s7|EZ86O-D@s%v-VB-+Mo-X;N6Y
z}P$SkAs0sR_%;)zA{a-{RwPV!(!RRrt~EZ
zI;AfjZ#)XybpsbvhA*2QShfX?76yY6BYSKH56asI7UnYAdLaZlbSyNFs^*zzORi9!&k^^gbUx
zc=vW(nUzkJsaB0e&8b#t+eC|LW5BCs9U;%3o*5cjv@|DkGdxD8bhF&coRChvG>s3XbhI-K&ph!m8Ulr3BE|pa3dMfEpha_TMmOC(-Bo=jo9;Q
zupF8Zq=1Ub*xE8qb(eSFw18KQ&+!M+ksv2So|2tY7w?Xp4>=TRWL3349&(RcT^&iBUohbjhdW{Io&4PK&~0r|laj}I4Ju+U
zt#Yyv`boOMqwN$g2_on;5LWGgeJTEotA`@gI%9`vyQEScoOB-w;)n&qQ>$tZ
zu^P^^GDWM2o-`u)o=2{|r_
zKu)Us&ou7+6UvQBnxVH`#2BGcP`xsSI@rSAzwxuY*-1Q;-WLf{dTiG-P@x_B`gdATeNDT9^R@cLsJhY^kW340-h3z5lQw!H;XH%ZycUSCdlu(<0
zrxnNs=koTD@!xOvP9pE$4ywk!+pDjrr!2RAC_Qj?(5dOZQeypWCshUNCX8s=&8A5d^PT5{$VfrH=fc0)6o0Amk8j-C
zK1GE9S@S%<jwN
z$N73xb>mXY45#OWP5xxTvq4Q|`2s~73;mYrpY0o(<3Hdwt^TElSu5@P=wj
z|9uJx4t`Iu~{`VChB`>Vt9~P!a^haW-JKtq8>1LTI
z_&|*S*(d63Y_7kTx^%z&FA#dvNB^b}Z@Q6ud)`!K)}SAY@xxR2LU#{C{r>_c_rK4*
z_+DVoOTlnBzXHuC|C^>&o&PO^9sh9EH&U#D1d|~$E%>`9ZXTF=7Z*WXc#yj$Omu);
z3l2tz)U(pz^@2Vpz2U)(yb>&s{$(?{vDBbd={H8=+3f{ni^(y2HV%DXn?PS9xe~0l
zzUgI)JNlD3CN_zjIofKLdAoKwl2?-^_53gxtdmmS0_@;-N+_H51Grgv=+CQcmTHsrT|e@%;6>
zf9+8_LFnIS=GRs1Uqo{s^V&5YevtS%BHW{I{v6vurmd@QjGVad*;>oO^v`lBx2dS6
z?~s!{51(Jy>3%oQ8CT3MeC?YvzjXDh&uzfMi8muav4y71%uZ8yS;5*U+6pW1B#Jwu
zGak+A*9k#YIoY4{TDmGbds6j4(@8Jt)7qZCF@w>ommvD-tv%grvP7P?AFg=igjiZr
zAM3pkKenY~KvTjDFtklP1_q?z0(6H?|2d_EkTj#!ovt8rHb9a03sa+fq==8-yk-)k
z@mcKUmN#SJX+GwwZ91mvgNrD0aJuz
z;7N#8D04Vsa8yY@JE;;c!M7reB(v0ntk1&pbx~l;MC%G;UoIOjQVfx`pr#L}|M)og
zE2o#lxzqIe*MGCVGmv62Q-)5~Zz>{-;<@E#YO(z!O4>&siQlx-z#~+YG_CvVd+5Qp
z&g<#R|v{%TNy|Slz#xgz~68O@^44&PofC$Gy{5;=e;LP4$EFDLZ~rIc81Lhc$NmZtg9_Wkbe
zt)vrKYU4)!Ou$#G8Tzy8DgG)oTbT@v;A{pz=Z(IMgamva;l)asG@f7c@O)Qn^N&bR
z0m3Zm+kGom92nJ!sc`RYQZhw*-0k1N@=vBvb#%)_kf|l^SLzdqH2q7$F3f_@UByr^gsq1W%5$hJ!ke_-?&_xj|IUhb=W$JXCeWe~5}L!)Q+LJdA!dCz!Gu8F;WCZAma7n8r}
zjpFYN`!Zq~9hJB>G)AREh&NYk&TMXhBVvkInQ)zmB8f~jMdz2)`}S`f-mlB~H(2Ym
zo{oQuKm@5=DlRBf
zpZJ^5e+_`T*&J2=u>2xi2;ceb953eT$AU)7rjkhXn)R#M$3@EF=3c3cQ-~xk5$N7k
zgw*aXWQ&;fzvr~*Rj8j7pIUxdw1OJmO$8k-EIJSgZU|IZY2s(raZAONDS!lZ%yWF8
z*1#GO;tRdnBlSE}W5-N+kN9ZWkAHM)nh~?~#p{rdVG?+rp8pxEQ7dkmip4oHh)x}}
zZ|r(iHsJrlzxn-AAO3f9xdK|zxT;59BUapv;7}0%jz9ImB36GR5YR}bq4eA*mp0x#
zOdErq&s$R~r8+R%XI6cPBK>4XJ_(#x?8Gn
z@^@usMxWxG8J;?=y<8nXvWPm_*>tE9D)HV(HT&aff2Xm5#py4_5>*~bK}Z1@rC3pUJ*lGQbes&Mvve?{J=|LKQiMa?)=U$
zOlvq5N#g2qtKsRvid6c-k!Ud;SNA|MhAJ*1snMRq*n_2YjNOwnI4IlTa9!VA8w
zen32thv(XzuQRItf@<7s{u)(R`9kckT`byju+@N9g^zpvkW4crNoZ7A<|D2(RY0(y
zXjDWNA;3{ni+pW8Z>}{`d;07o{3)-?%bC@N+minM_)oKMW*Hv2N7)X~l*ru?g+Lu;v3)1s=j$(c
zd@#K6B|sfDKiY;|ADz}~Jp={oakaqLVA?m+=rs~J(;hqAi(09^EcXq_$*Q8wLH%oc
z`xB!z?q8{O&5sONL$Fig!?d>fLabIZ+D4%(vt~qkv!)h@kfQ}Bdj*kSR;1xCSnh|X
zWaz5K=U>SW5;g7qS8B6w*%!W(DGGU4o3S%F$T;ux!8|$`tx}*!HK)~!sX&Y(z1-Z0
zmFUGDh;t2G{7Dlp-JCzmeyT2Ddo9a7fE!I;{|LiG_7>SxPA{g)g=XB2+moI*wH&%zlaI`w@wuAz9uTp@9`a=G-zV*z
zpB5t1JBq#?-*7}4d++!xfe%^GN$;8Lt_mU(R7yN^C9Zif+FqdFAaSclet;Opx?iX>
z`S)JwcrW??v*~-jks}tT!ut^%H7-ecZA`VuLG8ARB?j)cTm)w=(_uHA0VG?a-h9Mr
zS40c<)p$7{w8du&8RvDbylyTIxaQyXOD)uVoxXt?%==7rt2b`3bW-*mU#5O096dR5
z1AXOWoXr*pi|%fS_GKq80zdoiEmd+#5maK1dTNkH}%kf)}Bq6+603)+ZhgAw+|p9sSc
zhaJ_C#IQ@Od&R&r!1<|i^(2Bz%irl6R&6CxW==Ir1pz@uXG`~fs~Ux)*uC^YYi)n^
z(Q0Ta>!(?{8xQ8re-or_lsoY86g!*Ooa6v(AGc^>NUZ)#Y5aISGPxD$H|*q-dnP7p
zJ-my{WdYd*SL}9^y8!lKMyYlMXvwwN3QMI)*iYt!}hgvIvv
zbDpRQ3xsK1-(%o>-`&PhQa-XL*y-d)<^LwU8A+=y&
zAzyWY>v+{{#p`lb_3Qn8p1|SaP9qVk6N;e+^XrqirWTRMboo=uZ{$+oGdFKSkG*24
zv%9G)+^uM;Zk^=LE+htZ@4as$5ikyPcl{fYM=Ip%~0?z
zs9O8WFIJ-jbW6{kan@B+r}$*PXj&2(xaqkvMqx9ymfjV6GA+Fvn#GH-A^mqJc%-nmf@IH6$#0&&?;SJh|Qkd%D&Jh#s
ztpRUcoo?3D%XcGg>_SI2?^TbhiZKmOE{qQQ?_7?iC~}UpQ<37iZQ#yHJt!h04b%lq
zUopD;X1KK4G)8wZ*xQUNJ(zx#ORh&i(bzm=QV@nQw#)L=sf)W3X
zn5%!KXXe>7?D~?
zwMIyG=u2eF%Iw+74)`5+7vXMG>al!%f6E7M?MDJTXTdKFrnHcT)H_TRL<($i_l>wo5&
zCdYYKY${l{q=y`7pcyetIMB4{K;ShuAE+dAWn_C@KYG+?Rg-Kmn_;s#b)P2wC%>k~
z9@`t1OM;nF8@`>0g$Sp+JxgRx61286O9kiKq4GJh%8qx88&kSC$N~{=aeHCjv&ga6
zHnA4sM?IzanSOn0AoZz{^omf;~2EV#mjE>
zhbl>~E>P%Cs?^xe*7%WPq*%{;mNEy7{4SAdPs)d%Xust3gw2^(fhJ->4dPsHOg
zmsiGq@+Mqz^rmYunS1i0afhqs@uzX_g|(gDq6__*`Nk-2e|`S_=|2SJStD
z-u9}0xM;$v!^SW)q*17sbK|sD#n|YRMuhuWnd`SEjZRk7l|k%EGIF!oXNIR>lC
zrVF;@AFq^w;a+;Epd%%`bcA-x)KY}_n_NsBoOD+wZ9>7bt)O^*i|-TD*(_|c6~z%S
z4j*4i)ako^Ua<6vo5f?Q|rJqo(Xewju+CFN;rg6Cxbl_3NwoZi{Ftm%V&s?PSNc@)C}+PMMce6CUHIYD@7esn?k?K2|y+%tQr;
z)A}Qn!)dLX3prLrf#o~haZ(blsN8$PffV<`5vf?Z9ZFaeZRH;(rMJx*=hnbmJGYQV
zCNvvRCeJPPUamFIu@XyeaUzh{<>1kG1*|T8+aCJxa@u-Nd)~2ug*<{Y_YoY?pGvmL
zn^TIWs}n}#io$QbhzC5?1zInV7qlij7JsP4e6gRc7z`@esXWhB>~h4aB&_FbOR|za
zTx*rtOhbX2jZEJxEtq1o!mLA6RV{j5
z;dX0az#lz$zx)`^z<{c!Zy-j;G^?+4(}D;$UsBa1e4?^4#`c`v-r-oj@>D*toB7p(
zXhN9+%&0q^vgI+x#6|=L>2#d08;7^3;wlDQOJU+O*TeMsV`k$5E6b>%sSl~sm0`9Y<@II{yQ{ic2X3Q;zGmEE$B1Z_I;2$m}8b4o%lpGag00G
zAbHzuW79k`v7i;D;DTNoeQM)b==9-<&!x2u!YohmmnfX{Z~^?%KVNz5_>Z!Sfdlk9
z+t;as<$@)-`ZFYxw72gDJ5uhCZ8zU#hfMk}BzycRe{u;Ow^}DgE7k+&4eI%1e<5k@pMBKh2SJiC~n3%cjZ4|NVJ
znARrb`Z>-BJfJY&7i8RzlhxSMAZ{2@mpC2pIofFaxvAmNLlj+rF85j}le$uGAuRtX
z34K_JX^AeqMvS$9pxLOuR#qg@J5QYsm45dr{mjemJI}{CKaOtFDlcuNcpSH9h&00p
zesueMMu#YU2NZ^*^p0_1mf>@rV@C1vw$!YtTwAVKd^7ukYK&Dz^GEP6WOnS1#{}ON
z`?tIl5Q)ZpG}p2@kXJy&txqMKFoC7x!5t(;qra*}Vz;7g?~R^bQrw2Z4&A$t&aXq$
zuCqm~gvNHBVmTYOjmyDFm~kRxy!&FL*G9q(Me)r^F%O=S_6&0o@qY3!1H>|A=g2-)
zAY>1C;I`{|!HIZzpv`gLeCy;mH(bE-QYd|gw|lQ+U);y=yu)_{8}a26b1R4U+IG8^
z>`gZ?k&h5!?2JEYG6kBsN-vPB#~cLNn{v4FbTG*%a#x_n*f?$jsrR~F$2Jiwf!6Om
z8>6W+C(N)3GW-^Y2v?B~C`i9PnaI#-S5Nvd)b_RGiIOCE@e6g&!q%Joy7RI1&Jx|F
zN3D}kTIZEjcjjQe;ULML0towo%PUszyKauDSIRk(uDJwrubf|p9ls!R>49t)&g>U!
zk-sfKS-ZX^JiN%Wh_5+k3Dvk{z#P~Kq-}I)M*a1NzZua}J(wV3Nee^k;-$XmT-~(O
zB`!hZc-Ku$jxJ(WkNDu<=dE3%rLqCd2d{=ee$Zc56N@{=sC)e)?vIL&6so_~L=0y~
z?HZpBbq>$Oi%5iX>AybJkmN^7|
zERMM4J`YDy_rVJ<)*z_1F{R#CM5EtS3o!op5h$P0?XogIZZc@Mr{6*p>KyS>yf^P|
z;ADYXoDf@w{`L9HbbUn3S|6R#b%cpa5VJ&!QXwRL&&cV>wq7$=m4?8)y4>~!8^#7h
z!emlRLQrTZX-Ai2nsMS!unZdfscYwT;m5!Cn0H)45@-G%8~gQAvox4u0pW9sO?Zlx
znjwTIXVgdHDT+lo@4-hBQD0S>*9G{aMH(tI7Fm-phHAEpg7(YpWh6M!pv?)^3>g|_
zI);l`aOA`zbo<8YiRUi49!qA=jW;d)`&AnG=6AmajV3Qpow_Ep_@>KPo}mn98$d*O
zMsnSm>*m#vH)QC1wu{`G3<(^KKgzf;@FqJxjo>Z}8AuW&6?7m=TyjI0I9}pVuXdUe
zQZcf-v@CnBKjU#suQ^T!8Fkz39pa*77ksp!IVn*c%f&W2f}ny@)S~yQ7=Py6^I@eq
zrudL%zm03AkVE=T@(a9gx9;by9@by;L+tmKUN%gi7!lFV6kUh@=Ut2MGIURG7^y%q
z9gG>icjn@
z)u!+d;OMRKVvs3)zcQSCua}80Um(X9`fQ0{P4@KdG)_CoYpYXbEBRi8def%RLs0^`J87BF_
z(>a~xv%Cev`(?UeI2r3Ra#vWKT*U2Sim|tNg=SMc|G6XDaeeby>9rZ|Sh@X+^az2q
zf+LBOoy(I;{NJ!`k>0AnPkM;bCmwK0CeG6LMKXu8xh$+c6L7Lb*Icnfb739_5#7OYv--XB3*WK!qpNfsJF0MZ
zyASTgI^VH9DCSN{=7}lf=6ywc1j3D>x*h)bG)I#K1*5zBYUzK>NAk}W;>%st?ekrGQwgar28V}}mmi%)PPf^gwqG%I{q97BrwtPFLKgPT`%z_da9~N(*SW&_@|hz){Dp`%Vk+~liS<&J8VzGY
zBZ`hYH*3US*q(*0jeJ_v~bZ
zWKdTr=tMp4*F~ulR!8jj5e(0HnJgI?|Hjho;H?B?iNW~{N_N=RpvP(f6)|)AfW+v%
z?Xz{c$nEIiBf#Bw`v^|70Ecz8_bT^xo_>APrn+&tlY0v$&m^mR&y#H}hzeH|6B#yX
z#3Py*TK^(3=?v?gT|@2ehYGR_
z{3yjS96ez)neGb72FYL}o135Fs?=OeB<2ZxTh~Weo7dVPX`$e+Su#A^kOsI3Omg>qosFwltPuz;qocx;1QG3zLxZ*Sg*Rbt^cF*=I8bE&?|WdlJ_R$^x?B#atmL5pUh)s9^OO96z<>*rxeLv
z=WP!Ao)C0Q!%b-qRd^6hTAZZ%#rqgpiH!hZsAK{T6uWiDpjEW_ez0fMOyS8lGxAB&S-K6vTD&3`xe-Dm
z4x=%4Khlak&b)FIk;4s&RWoqS7a2Fs>9fQl-Il
zDA3e$G8ZXfc>}}CjuS2sggLVb=
zcQBVPI7It@wpi@>u_i;vb9^1y3h+{{D}`SQJv~tLaRJJeu)71#gn&Y&x{QEMQ*Kax
zh+fRZp_F(-9`7bBbx+AqWq95lE@bOj>zH{I7x~j{Fd-zHKaFxF@uzA=>(-apFqqR1
z9a-thKxRlN!1Y`^T&C
zdY>-40i|rsPu9#syU(q7jBl?#1hL5m)hs$ZhekYWc*E|aoeCp}Ai?4Jh!l#G{zgy$
z2^C!|LHHZI_?^)B6Dc#rG5mtYUvu~7$`z3!LxWtar(dD*?A|C}8s~9|!}R}j%~+vL
zP|SU)9y96=%?P&gNdj=~D#NZgl(aveSgQ`aF6|9pRn*7*UA#B>Ch
zeoDI#-$>kfio~-<=hEK}ie@QjrT6wpj{SaR5IjL+Zf&aVQOyOn
zyp`T>Mt&a&8*V(JSY5oNtKsoR0|&OjjaUP(VT9b`Kcd3zX?Q?LwNK3G7zBs4Ot=Z@
zk3;3p16dw~v>>FZK5ndDC8e--?AUpE+dSx8J-o?)&Xrck5F_BX_OF3$L?g@RQILy3
zphBI7Oo{$*NBN&TwlB#W$YZJhCy#wriSTpWvEzvz8Wjg2&nBin5U=tO#^+bQ3)4iP
zdVv9g99&21V6C`*D3f0zdP-m}me@e9@yM@5wu=wR0&riZQRmfJi9iNn%TyJ8Q~No7
zrJwTS2DQ@=9FY-CB)gCP-Ic1?kDu1MdOJx?u!sk>Y{*}=Y;$EWtho@*mj;C;-JtTw
zOx;S5SjD6JCdcmVVx;EE!!>s$h#rw`Zml+TS1Q6;=Fs+96C8ok;T;rgeDr;Z8oQF%
zR+{hOM0b5=OYQ#Q68UND{nyjq+M%BLINVK3&pv<`teO~4i&oPB^CCDTzGO4f-Ebb7pZ&5*Es$ysifiJE=
zeD?+Lb8C-WPq!w|AvuPvPsQSj+u8r_MMT%Zn^8|&<7WFl*xD~TE6R`StNZ*8>Al>`
zZGFtxl`y6X(bbG_AmCSk6eVEWKdGt67f2gyPK{FarD8_#4ZaFvSS~Ive%KXG$FUMz
z<94kc1e9J1th~QTK76?^1VauO&>qhfLISE-=HhL1GcMSgnXS_i2R4w<7EV4w1-bA_6&*l)hR?B*)Wq~QqiDntm*}PPYxo1NP@y
zHePt+;A6)1M#GS}*)UA){aP^@ycv=bQoO2Y-&iXQ>Oteeua8!0gmkBHDw9`bM?-6&c?i7*^+?`Vk
zFW8ft7Mon@XWc>itI5%KtkAjqeR~eEXGGHZnOKhvlz7hu3@NAJ8m^zqA2w;yz!_0E
zLG9jGEZ*HHTY`;g=RfsS4}fVi)jA=mu8=t4AQ||gj%p*(@Ku?yw3!uUN0NS%B0A&O
zR-eT2HST=B!u!RRjG4)!uOYn64sGtuBuzqfNti-$`^LsF9(d2$jRVqgSq9c4Jnu`U
z8+Gc}<3X#lA%5IY&fx+qvEx2drlORJuv{$3zCHuh3s#0L7VwGXu6%`94up~#y;xy9
zx&kRtVuH}7
z;NIUQozK#SWtP_P#CP2^ERg(ZhIqqlKJOQMXcrJ@rEH$Z=mZ?bpue0<)xGS2ct|g?i+@YPZUQ?xz
zX01!t6F$!Hh&Cmrz6{U25o&cNSQaZ8DDDn~|4{NG~70emYB6K-~S8%4xs?qBZmEG
zcd2>FBpetFW&-FHtwP2lK;#Mso|tdQo88aofd_a%d^}M!gBm7aS4qgpg?FbI0aI@r
ze0-!r)ob6|k1--HZ+U!n_jP6}0)8jd^TkAdYi$Hj1OZ)>hXrhJp9
z8!%c<1Av<8DyzgDi_1&Ue+e8+qaX~Z2*4m#rSQX`ljA8Vsog36Vx1E@xCjYZ*(c(5sG
z!)yKSrxuLhQ4A&v5R&j%cK|kx?a`_v=cjSh55lltup>acG5VAJe!AL*f?2mA7yx;e
z7hIOOMDB(J4k($JFhC}t)v2o(yaf2Mt`#&3D-^iNhVx*8`I_mwcesDacuk3=bb=Ur)-gN4I6Oj9v`fW5HWt}@QE#aO_^
zz$`msR)&g_VcjGq7SP?*X4lt0MJ8vq=O5flc8tqqR1>HiJ03otYIty53%5I&xsg1`d$;Ww+hUmQON`u*BW6E~qn*L?mI
zp9ZwP=qH`kC{ag4bsXf$pM(9s6@PyDpF;k*H;E%%AZLhJI55+_2-hiWIo6JeJu!&m
z&_bI<{t&35J%$t0rHpT~aWB%#7w9{1Y?_1kip9n_*I7m~Sx&+%7`|Z18v(lHfg8bD
z^u#8C+lm+%6w3qSVK~vOcQFHVNnUp_2;8zQv;xepH89}q07`^SLh=pFvNb@2V-wjo
zyXKd^o%lT(xmx9=Wo^y;Q0xFOV8ZZYe_t?EYnMlBaK6AUdNqi`TNFSF?d|OkVUW}{
zKNNC#p>AN(u263BzPGlfhPQVZGMw5Sj6*X5U~HUbBXA&qekgmoDgJo*{<#QbeI2mg
z;6V^beMZI@@L|@e3sB0DgNH+YLgWiTVGoJMbt6Wt#A=TFqp>l(ybH#Hh911fl)|8c
zZTCihTx4=GlUsOX
zMZmP1Dons2?!+pZz67sI`E;hSP8)q=ogrJ$i-Ot1D1WX_o{zE|W17*;2VFeuru4@g
z!zU!sOdN%rPfs|B5AM!47>XPGft&e>R}gS<9-K=8(-hseuJBLO{hh}X3SrX6j~|KdfWW@ZkYSDsAMp$#q|#%)XQbR
zMp1KBxe+G%cfNfTrbg3FL9O#7i+zxJ;^3nI`4o~IQY<9Ak<2-7p-p~YSo55M
z0h?@d4M6uu1zoX$+4Vz0fQNhb>>0B#&D1;qbn<#$maNY&HhK(p&&fF8Fn=}VJ)y=gs82GsnNB+jg(o%up+0~%!D{kgWS=v&
z^>4IBwjEtJ6HE_5p|u7Oj5V%D)F#6zd*?M0L-afJMHyh+KpJbOznl9jB>kAGhpe-&
zqVEQv2P7cURCHpklr8gU1&Tc`R_vKSD%N4esl;d5QZUc?_Unpfb4HxSPY#cA#Ax`D
zJJ*&u{XBE9b-u34XhQa@^U`W`Kf`|d^fN#;u7D8}2L*+pp<#&wL=LV6@aaZ@N4wz7
zVf(`aeR$ZGOHB(mf)qYCcAiI_gu!3Gih)fV$krpj-ad7x907~Lpi%a(Uuhuxz<;&^
zjv##eK3}Ef40|Xs7n+@&T|BQHb<^c4dNiZPSVPxDftnE@#^$P*Qg?-ueFH_n8d!0+
zCJPhuAkj-1D-Ln7J#1NW7Xf
ze>&ZJ#?r8e@aTj$%1Dg0c;>j_c4}}Dx-EiMCC89!4=<~g16-8b43_-&pr9}f{WOse
zPkXfoB`uwADg^Tr4#DWU3e3P6FKz5cz?2;mETKQVZYrmA+rd2^f+ZdS&U&%oD==oW2tD*{i
zN2)?-ANntHbGJ$vf0vU=k4dqIYkO>PznhSXU1z9?p8d-He&hAa+C}!YK2Lom%pOav
zzCsmqXISkqg$Ma8cV2%zU@#udH84
z8fo0W6HFbu?n?VvPMhPvk@?+=MJw9KmD=d&vxWX}OepKG#oidNfhjTk&;JC+#@_gj
zTy+q7Omc>Z&)t1mNip}Qbm}PY&#+3uD3r-iKKNP+o0wX|f(MfC0wa2C^ytiKlnPCHX%VnEYqNQSc+4s_
z0g@TvVQXYvw2gS{kAyw~Fr}wkqllp5-kK;N97+|5xH^LhR65OqP18{uI*UG_-97{p
z0gG-ZsN*vL#CN*F{D<`9R?kB*=(M8@m+j$G34pG^>=$fOI>48*g3?^@dbj)`!~o0k
z8wfyyv0rcVrwGg{G?`QjH3LCmvj&*4{&G2>low6pwqga?Q9TwcINOT_m$ji}0r30+
z>^sSdagf-}of;suik%Y;&YVq+ieq--u*W)32z7cegmKK2aD8elYECRL@pd8ntpt|D
zduu@yo8l!iywSRpW1kORWUA(z#5o70px;qKLYWkm+J`ALxF^S3ASyUaB(?2yw9u;&
zkNWsKMsreIf(*~YF=mtQyU1tU8zh3RYLCZ@iSwqrNVL!hKtE#Aa5?ACjC$h|5leGA
z7m=BaU96V9w1C?Ms9v-WN|%rKW33!V$;U!I&%Oi)`*)k_k+)nWzbY~HrqE*v3_gan
zyCh0p^hO2uh=YMNxhw0t8jfc(@@y2OrdC}3%CgO5&iRwN@qiF=aszrYDP9-(JJrlB
zJ$+~boWJ#eLeGG72=tuOG|$$34}ODTy(A@pS0h*&NL3Kdo7y~-^QC@n80LQ6pudGx
zQyun8L=eS#;Pv184i+QCK_U&4`rPwoqp&nMxTo3e+FK_DcQ^eBqTtfe|N7S1|FzPU
zbrk;X70Z27dyQ0B-MkEM{x#-;
zhElP>;%k(v#(e*Dk^jl+pCbS50#;rZ!~u<~yJ(yFnGc_y^}HKdqL`FVy1wRT{HHf3
z_*XLYyqd4=##~OwFIts%!ZU-HryNw9|BQQ%byM>FpJasf-vy^0bu&F_Y^8Yp%w|}P
zlwp-9`jPkjDmek}-{C_1_q*#HO_+6nD}*29TTr9cfq#+KL2>o}wUMV8{ksj}4QNT9
zmVufbY7(!Wfrksk$mje*<$t@j9MH1E$TSOM4o?a3lhZmh{@i(Y5i9+ArSz1kk<#P$
zywtuaERw9UdfV#HK!V%&Fw>mgd9=p~+@v1dSMK|%vaht}#p7x#4YX6;S4mDbFxYzd
z@9x;p=lnYsz{wP%!TL*)^
zN|7=|P|sN=biACK?nv-x=Ujp0309TLl=Spjz8E>@NbGN4XAGmHeHK)Itzy68>+5<`
z;t#K#C1@Yje;BR3Kcn~AR5RNf-*7tD!-c4>z4OS?{I-8Dz%tvYkCvLO<2kS>LU}%E
zaKD5>cuhB8>T}x{@r+AU;uG=Yy+Eg&9ym8W2K}EGh{lnLIg;O`rr_0^ofsH}f20t8
z9F`IBwfL>se=&SuRt|c{-&nbR6fkWa{&Wsgb4M(wkgN2IorP$;;Ttb~;vI?xVG<|n
zZvuVG?~LdJ75i3xNa`u%g*BdlNyi!NFh5n6e0onHoD!4Oxp@FxZHIs8&WrWLaW1@m
z^@wG>9uAIHgL-(Q_ABk-!q6xc9Rn`Og&89E7J@(Jp|b5}6LGHVDtp^lxfMA_Ao~!C
z%uwa2u18L{GkbtN_Xx-BQO6PubhslnqJ}_r2E9)ETNY_yJ4}CYYvqtJQ=3zfaM@5wVuABGc}rFXMgvspS5>K{ks4bYHWIJTBHceOsPUs$LcF4yohHyHP?{rAoJKdaK@?}p<7!k?mPGa}oN6?afi-m;I&+?K>d+QdY$#V9k
zUc4sZ9Tnz~2|Do3LjrH9R{Nfc5iWaHj<(cCX{Y=M_V1So6Eyx$3(&c%cnSOD7I$4w
z@{r_+!lb$}8abOq&}D|DVrb$%`ZL4(=bnc@d6ycAulY@`O6{go39IsA$NkYO#P+tz
zHzJ5KU;P29rc+-!AeYGbZSxQL`*?+ta;dn<;`!mlD>5OQ+Or%ThQO1-OC8YC8cE~r
z8o`{EPsG)2f2$)@<@J=VCqOwnk-|b>=Y`29QxCb?=6ON-E9DIINi4x
z74Kw=wwl^GO-cE}h2=Ii-atMLR}xA=gGp^6q@S&(VD^a
zBbv>cJrG2jein!R%$g|@YpGOE;p~of%G?@NHLTOY~Fy}7$K21ugcl(
z2?glEw+|}*oWBYhV=|IIk+syqiP3yYsuPLffd8q7Y@MO%I{9VtaWnPDSvpZ{qi1Yb(7U_-Mee}!yM^leCA?y5
z^esyd1zT!gduCA%ByHjxL}sTo0W_OvM5az09gJ*k(mye0a~KQSzuy_D02NH6g=3Sh
zFW1|ok9JD-I2kaJ%P1qPO144mkwr7iPZC6fHW|;m(TotPZ0cn9EI+@ReJQ7k-I
zF)nVdW5chaReoY9A<&Jq*WhC5reqB75BnSfYWX$eZ%y>G1s0iXOO~qkjAjE`R`g++
z=e2+$fP~(E@%C=~{xZwa^5%qoNN-poc*w?Q>09uP@9Bx;^4XM`|H%?Bba*kU`%$K0
zuu*ktayJ^^Q(C?yUF>8_8y?R{lBEPDjAV-DvoFY%^wDYFz8$laret+^iHX2zi
z!ddUPMz`6@`(eMdiL!3KsOnV4U9lm3<5AfU;@MY`s}&C4uIo|Of16*Lq>~7>dOjtc
zeg%#fY2*K@KqraRY)p*$o1oJ!I|OiP=3!A=vl)zi-2t_NF9Ap=B{!d)cyB?Vul@vu
z;aWwfFFmHb}KZ$Pj7P$ra#!rI_;WC1}Nry
z+AI&9T3x@a8uFYeRlF5)(3ovY-^cW%+#&38MIO!h=10h`HcfssrgSxdu|l(EbpOdy
zg`-U%e?iuH7hUjM_F`v@{`AbmPTa%yHD`5=y
z>ai+1De6HNpk1?|j4jVt=tT(X&I<>$D!2Tw-?A&x5=Rb8u!DCZr#!fPCl0j
z3#ZtM+l_A{U+T2@`y*|m()y#)zb|dh$Q5CzBK$vey=7EXZP+$!fJjJ4Nh1hIcbB4o
zGzde(NVjyyD2PZ42ty;?-7$=GcXyY>NDVQ-yj$<*ectcS_h;6cwf5Tk+ULH`K9A#~
z`uR-&Q-i3Gp`e8=>*YJBu?)-Emlvt?
zDUe2P=TGYBR}jV*RS14!$Ln$^~XXiAhvng=5pnY_VqmIH|r
zA;xP$MV33MpHrtgpi3(-B{w9lR~V8mqd{`Cr~6q4QTQgsCzt^$K~IL!Mt4KB|FBJZ
z)GMIBeDKa$p6IPhE5XG<;+Tt*PkV@gbdZ@`Na|RVnsAl2Ry8>TbRFb}i5X{WUk({8
z&{sw$uIWB9&2$u{>^WnCWTSk+z!|4K9)IOU_#I;ttStF_fVff#1lm?zP~MN4xmcXPOoBp3-uY9SGw;?sl(G?Mhp9k5`Wx@Ss3_!cn{EK>qlZV-=ubOuzc>H
zFMe)m^Vl5o0kk|YQV`B^aKcVWq@Z-*Qd?OE5Y8u7yFCWkbj3ebZz7Pa^DH&)p71d_EuJ;`41lH*%~t$6z4QZi13G`Q6SKNhI}mf$S%_2_
z=`8z6PdwvUBLDVnMIMa5Y08)@-bkPH!F*Au3Mz{-u4
zJ%yGZn{Cq*15~aIi$e0($PE8K!XZ!Nj$P0_Zl1vo>3`6zh)|@!awKQ
z-Hyrq8mp~iykbv
ze?6PzC{yu*!kUMjo=b+yVLMiPEep-hpuaaxOb30w)cOTaiFQ_-*%K&6K5{7cP9e|G
zRrb*4Vq;6W6P2l_Vjj=wM>|6hd#U2CZQGuE?}cj3GEA=4OV3))-;@>KGITsr-WdH{
zjd#Q!T=xVWO9Hmf+og%~e8#$EK@@IoF)9Px#C
zx>JrcJ92n<`R@6@cKab)pJh4AY0z-+2(zWdp*%i{0HeH*4ijh
zePOQ{w8V+vo(;u4bMsAgexw4hAk_>m;=R$$CrwLdQh*()+B*D*-N6_V+t_@nm8=yI
zxtx|)0lAHjFGN0GZS(%fzcr&ioN8Zft9Z|#0TJb`>~rcE
z;+LnYJF+U?I?=b{4}u!j;Q%V4^Y9MGD1X@Mcy6S}%XjTL@`X{31I*eOR*(NsOpBgv
z(NE8%VO#{7us6*t#`i@b`(q9WcEXhKhF+7z)hl=8v-&rxtXA46cLA$*r0S8db*hrn()z{2A##2j&t)Xbd>O~qO
zD;05nPW_$RpC|p0hnFFk;)%_EJe~v}kDjjR9lQrUdVe;a1uXrc1~}qBtz8U#XlJ=N
znL$EBzu{S2_Y_le{TgDn!_R;q#lc*DKWZ0xG_$oum%aT%-+a8c&Pset=*(L0$EEPF}2%gg2q4t(;YJE8@%-XL$+?3->p*%CTmY~{vd#88y
zV%HJ~mHm#+dWaQbnBTAUQYx$e>q%S5atnX1co9xE78gC&^VAHXs1JAL7p`YcY#X#j
zubnE~@e{abhf=p5?(!G#!j5b@PcWNVFJHcVc~fOC?YU&Mvj1DK@b^=|c$5kG6&5MN
zehkb&($}Z1LT0__)9i|&Yr(Cx&phlbo@NCuVO4A>K5e>&T`>}q*;`(AuDtXdaZ8T6
zOP*T5EpJv$gTKAdNg!Hqq$GVb{9$qvWWKX~|Ib|(=iBo%#Y2|lhs_?k
zFC94@4HsxUy`?rn9)Tz?vuV)eWXE#N_`;IRu|lg3R%_XI?5
z|DrkCo?Y5`Aub#}2Aq|#{wt{ynIhL?MOJZ;@rE{<&YDCP1X4g@;o#t4Ygk^_~A1Pqg!%iA02X}Hw{0COZO;rAR
zU<|Vdy1OTihJmk6R*0_vKJVbNf)Sn7XqO>3SwgJz*8H0sbx&Q^
zuXb?A>1BiJmk=z?S~(8kMg8c*l<4Zc48|8MVDn94VIoS(x77)ty~WxXBWk;n){D9q
zFK6G{8@_Pghvwf#o)@U|QpPD{f)pLi8}*rhd3rv@LT2RnmQUNyxqUxei24I5UwLEx_L=jsh6I0^FfRNyrNuD(@KKQ`
zyl%40psmi$1i&m>R)_BEPp_{$M2xM@v(2v9IV^d7k!)7wIjUX$5sJf-=|>VR^T#okB{thV)auk1&gW+hbLKLA*~Ju3uwY
zKVa-gacH&WLQvOKj##&
z7dP#+?nR07-A5|OgT~I8k2P|3_HjESZhnL1q1WAwd&ZlOTkHw7k))KA%8g+f9^Q{)
zl47!xtm2D;y&qv=0e=o-4dRQO1swI_=%5a!n~CDOcel**p7yVT!+3AbcW7OzTniKa
z{8Tz1R@h&f_9(5>S-#SV`jm#G@@<^vkd@V
zja|@|=|g1!wPdVv>HIVdmrW;OiX;=p1Lctn2FEUGNH_
z3U<}Smh?jCS{L37>=*xmTZGmiw9;lTol;a@Q_E`DOIu;amGX_WI!q;(w@SVN43DBX
zIbW5i6GOagv7}y9p_`KsXG&Vn>W%E79fz;-N7~6eML{!E?%Ecyn9t
z(r3!}?y5q%mQBBW13giX2{;_2$X6CuEyF9{{_n$25-UFN1NVFMD^Qx)X67dYh`8Tt
zbEWRSFWOuQJGR9>3M(NtsWm8#fX#YpHCFcIfpXqH-W3BfT0~I`{
z0zOP&jtY^|2pj#*`+5mc;0^@emEQm0#uT$UnC?
z4$3H}Co5&ta$B}z_}QcpHt1^iRx3Pg-i5j1vxa=4k!bz4hI;X1xCN>avgz(|CzqkX
zNkmW+Fb&9}J$|UwY-g_KLO_r$!QIhNvA1`4OPOtR%lq;}cjW9;h3yNg_W;6axCyS|
z^u2v3(v+2s1~1H$V9&6p&e&>DFMe~m4g(w4dC~VExrmoq#Kxa!=g*$o^0f!!0)9H<
z;d>#g2|nwYpQbSpDQPJ^ZE-eux2xp)4^RA}(qO+@b537nj*^lYCYoJ6+ZQzXoof#d
z&LZxE_V)K>f81)ftMBDOd^D0s%3kkZ(e$Y+9{Q#L2XKNoPl~!uwj6z*jW=fY+|}0>
z46kCdizc~*hE)>z!H)?2#V0Fixc5f-ZtI0sk>t>mnPK9ZLhxsUWKgc_sV!H=A$ugd
z?Cv~749eFWr4Qy(Cm~^i|L&QuxTEDHtW0gKU&^1qB8@2B!o9lCC<>DW(cG1b`A?OT
z$OP}whm=a)5=0q6_cG4M4_&@p_c`=NGP?aCbx}7YtVc_5E51`QS&_@bX>z
zRBh4zib_wY+P9pUPnLC;3-!!IG&FLJI2Ixqv2D5Gb8p{%{rU5I!v&rXx#+MG&u_Tk
zwnDS|hrQ)|(hcxkl!F05wUy^n&vR?+g2^Jy^qLyH*r>E@MUY+mRIZYqxnVyu*Yk1K
zJQ-D0F7k2y0?l%UY+86wO!QJWjc{vQSgye{o?V|5qoMgN1u4&%ADc=5IQgE_2Q&q+
z|3Jk(YhIZpK!qW6*5VAQ{ZW+`$hZoOgg@(>ty3VaeXo~KDA=Kgmc0da_|t;6T`{qK
zp{IQmfT_*Qa(T&%S*=$YY8vHhpR_w#_$V{;HOG}DO9&FEd<@S=iSBQgJ21*X=^xAN
z<*061cR#V+U#|GUb2F%YK+w#sd1S1I>m$&++_Gl6*Q^MKL
zG=)>1yEwPEaY!BC8S7;7-12%wJ-V?N$ya1&HjR)RibXdMn4w2M@O#!3wf%MS=+Y;b
z?)$uXlFNM_Eia{j;^0BwPL5Gf*b9HrAAduMj@F45K7Eue8*Bd!lOI%Sg^
zoAqcjIOi}5&R`0@*lH_4o#O0NPxsAfV!7E>vh*(Zd48fwODc-bv8P4W(l(FN>H_rv
z^Bw5NNIOY#j~{ujazPj}wxoN?p1}hfP=?AGY7twUO(W#50vfg^GH^X|Ucq
z$^5?I;jJLyfVJ^p4%LjNR|9oZz;Q{I{V2FAP|tvCvz(yN$W7n*P#ZB};q(Iq%ZzA$
z#Go3)Vy(sfo>mF9N;O%nm-}M<;%p%iZ{>7|no2S#-#c8tMFh$I{{6Uu(!gqCz9qlQ
zEK6BZ0oisyWpa8wTxGpAu7E7Q(?4ItpDi9>8k*xEB>_lY+GDKk#$#+2s^i4ovES#8Eb6H||D_kwnq_!0OcTGsfQ`);
z=fyBJ?Kx)C_#0qU10u!w*7}u;J(ocZ4c;8rhppER1mMznB?Pk8VF2GpaRuL9w+cxQ
z|0pT9a83Fx6c01Z5%H`x69&Vmmh*-)UplM!gHlXh=Z?_F>?a21%aZ9nVJkb=dyz2qqz}
z*kMm0lSMn&XuGqK*9AxezZE}AZ>?)rlAwlp&x`Gi&k!@y)zdAuA8U6+4nUH4zsNQd
z{eAz^>4t3@?f#|D^=P{z$=_=$iPWt`IwrBf>0>L5OAYQ4XYx_EG{=Wx`bf!Q?yGCbtY4J0M_RsJIL1d?YNhYPRlPcV
z1Jg9?U+vgkjnXhIOBcJ2vUm2}{Tf20R|y;K|~rDy29UNmr*_#+Rm?KvMO}EhoCPwebJW0_^bm4?XX-UtdnyKkl5A
zzPu?ME?`qeIfn$PpuUnO-P{U2TK0RA*=CuCZkX^GoDgLzyMcv{Woj^y$gY(-3iCl0
zX^eXi^G(c*%%u#>e3K?~?OBNrvqc|WWW)_`khyBcu|+f$V{c`5T$#x~CFgf-DI&o$
zS8OU8Acll@1!ldhLz7?5LP)i~?uEWTVuRuM;j+2ocJligpaSCha^|=$!y*O7>I;MS
zRvf#t^q)B?J%pZ~z}Wf&vuwH3Y+8DtW`a{cz2AtQjF|LKvG>%%l4Aq-BALv{T>eDA
zz1OexEem1wuwW{atZ~l!K({6mkzuH8TnFL^5OSs_Iwr!9|MJHjEh=!}%^B$W
z_XPQTje9SKyldcCt-c=8u>>!O4cHT|mngrdCJttWz&AGGEGIh^5fjw*R&&HkC9Y1I
z4o5eR!sAi-?;B-7WRa;ODrSzEi-KE;bMBNZb6!=zd8y3tT?#+RA3)fFzjU4tM3ic^PABTKmjFnZjGkchf2hddO01U=-!=f@SMf
zG1Q$JqZB_T{oZ5Fc)ZEz&G?VzicLD50~d9Hlh5tAzM67e@p7DbgBRkBqc9?t{IlA~
z?3Xx-a`>cj0rE)*Pp3*>Ggbe2y7o5?&dN!RKd7z*W0dNh3*|M3{d~A>y}ljF81(fr
zn{4X5g?jv1v_;v0}Y>djyH}1#arr-$LnHY)l``;c{50$bqLTA_b9~(4<
z4$^XpqUYXBUn-j1?5<5bHmGV_Ylw{CFbz=~xo~{lZ$*&x6^lGflwz(EKSTC%T&3=K
z>JOp8d9qo+kf`{%lz|^Aml5PPZ@WfiJAD^eX9N$5R9zSbzT`tx;OXsQ&uxq3N74ad
zye{)9az$JR!}&)-6iKfmQbwbuh(qWN{cjo%MR2tbH-dPDp>Aw}^dv;w%HTKal_ag+
zxdsaPE_2WBkOan@M!KK!m7dMV#pR!%qDTT3V+XqiCDyY8_vV_Y9e&F;8~q+mC(1Z7
z^+(hiM?*w+FSes=O2MC(dfw~aT-;~tlH;?Pj9gr-!MO*w7wyj04A@b&eMCbATF;IV
z9zWc&04&H*J9+c`x`g;Cw-mTBJk@aDeQL5tYz+OiDFSWz5Xdth(VxF3i!o`#NK!t7
zS8{3g=t5glu1-~#4H|+w@+WKz8KW*}kKTpI#oJe6sp%>N6;-x4sj=S!8bGg?q_$^M
zGqpma-BNO#qzXG+v1uvt9%Wt3f&LfmX^ZnMMGK;jHb(jiPoZ8Pr&n%Z`!9ft-WQva
zpoJ?UgXqcMQ4fW!WlpJh*?mWiSl!1ojVUL|M<*tQO8AG*X~%%no3lbG96@!M{oF{k%fl*O;iH|DMulxgiLbg}Y+!^vPSD$UMa0-GO0C+7#kW
zwKT=n61=}uH+9Wzg2a(vVsdNACP4>UhYprV
z>B_5O-Di#7Bvzh41-TqcwB|SA9C@{Zn&J$pBhJxhMEVkN2E=iRmnCL?eMZ(Pl~KWR
z<|DfLo5!EH92Zru``i|&&ywc2A^6%F`+4(tdPsVN(JB5R-Tu4N*PNFHR~ScLr(CfF
zX=YOk@bQmYzGcnjOsA`$FUURnub8)^-v8
z8;!U)!7=2`$D2VLh)i~yYt56gU}me+AX@9
zr%H29_$;l8?Xz7!;ErVZ+tPTJm?v8Ok|J12ZX!MSEeZ#-m@n{ByOR)dTRkI;0ka$&
zVNVX{n?yZONL
zU4O62|L{k$FE|h4*c#J1cjV1qK~xrDaiYzul{d$noljDta_yffU@;T&d(a*~l-@e<
zjNaY>C$1bnbR-r>iuXTBZQDc0eVY4XdY@^FA>s1f!A!Y_wfNywZY*>K`B9pq2f$&k
z0N`&aad9Xk*6`_3IQ(>!xRf{PK$<*?SJDv|@Hg#K;T!H5Rv0OyV#1NxZ-Sl$EGG|g
zFT^HY**#@PtK@kuxELCgD)U_s
z*t#MqNo~1*kj1g~d@dUCG3sV$B~H|hJo{4|EA*M4l7Gfqu3skU2hPG+;)@dWF2X$@
zZbSqgO&NX?v#
zcRQ0&nyK>vVG)~nst(e!x!b|qj5Uy|hqy=RsLAYUrVr@!Ghqi>8>PeH+PQkaj0Mrg
zyTW}v5wGVq@)P%Tg`H_i>1+;H!8|&Bp{(|)bz^NS*5JdyVWA8BGo87aq^G(whOi%u1>Ll)&eVt)f88CVZD0FQlIhD#dRO|IZZ}G
zDA3oHogdTE(wbJj4%(M@&p9fcsj%wex@TJ464UaaN?w>B}Pz;vyuK;@!1J}zr)
zf7=(ZO9nSmLR!--t9^`*HcxvlsD>*nJ=usMB8sB(CiOISpTUuZ(&Y$SfO
z51FZ**>+=fI5rGY#8U(P%0aZ5v>o%A`z}p(mW$`Nk7qaFSqK2C5!(!*OE)K@$
zN_Hz;jITMw3^O4p7v-xfjK=4$zZknoNbfLV(nL{k$y$$<{yuTj>og4Ue+U{rl2kD)
z^}1(j*XeRPJFI2@KD44L^KB<|ZMUhT>^|wc`P9vE8coK8l=S_#UET4IIjQ8#WiaGt
zXjrK^(N=LOF+f58?!c96>jS<}cGgppr*G{`b$vz-g1SCB!m2<6Pr=;j{vxYY%oLuG#L!~Wo+bXHwpfGja7DK%>!-ittFetzw(UR6h7nC#
zgTu4Y)Xc~A{s$-v?#3k1k6kQaS$6QvCZ^uMDHyd+*haySNmwbs&Mb$8W5kq4%Uy3o6BZm
z1xGvoQA@hwsqC;)ubhrEpTDV01D_0re4~-#m%ejPMcJCS!ozI8y7GrSm0rgM9w7wm
zd75Fw;vm*HD&CNy+8!Xe;Qx{IW|mC*(pkW`eYCoWYo(|-Vj()R`7`f^*zA_}<=mU<
z!=Yz?vmz(=_2^klb)m(WqryUtMNuvUm2X5NHaW1iIr7H{%g#ZUb4H9Ug|_(n`|C}
zM4i}zWcO8pK6vC1)F7WMI1!&t9B<4D1LhTx`K{y&itCRRMZ|6FQ2nKk$RoP}3^^
zWmc_6*bmdGzHWQ_?#n(dm5=XhJu`g~*r@<%3dn`9sHm>A|Jc1pQVhW_KBAtfhlzz|
zb0~K5TXJ4R9xvtpaNR-I+#&0e8xp~`w1>S_PuB~-5KlfM&|rw5`;CzxDowJ4@T}V6u#G
zctX;aC(llht>^3CMB4aTEQSf4`kdbY-S!jsp5>Q})8v=rvUpah!K^48j?y9y-x$RK
z?c%Tj)5`Bq=+5}?z<}%S%+ppJB#P~_^``^3%hGaQ-npS$=lx&m)T3xr&iVU1lo{qR
z<+3}cp!g*73%)Nv7V@g`nsl~x6AUmoypv8+9Y8I7z~|L03-JE!R%CVQIWC2MfF^sR
zZ-UQD43_>pUq3l^KD9A|zH4)ET#>u#CVd==m${Vafp>LvUnctvZcqQLH+{iR%9+>y5C|4PHph`*QDis{&p8dn&+Dt<7KNm)F)TyRqS(w>?v
z$L-+IAz}e@#n(1E4c1&2z)^~;iR$Mt$S5*O$^{+fy1!Y449eUr5fr~f{yyoELt8Nc
z5yN90ZY060+fWn4N`ntYS*i9{chq~mraQa=;WUf$VUni0cdL2bm5*XDUN?vxBc50H
znY$1!!DwO6ikc!4s5vhE_Y4ayAd3Z61KxcyjtfX{@o!+Ft;B|#XKF~tK8I2}k+j@f
z|5E>Q?g8>xxDHgWXFy~d*|lITnDbbI9c5Bs(x>(o6g+3Av$Y0N!x%D6
zZQl0BTcYCZ`6GYfFWiW`K=grs`nQJ1GFdVdJUdbgQu=g)-qAHV0^0<4$wzTM&bQe#
zA}!nW;k3rTR&xn>oZ3Fzr8jhQ;ogocAg>l4N8(ug621*BBkI^Y%ukwduM?E-Vp}cn
zCKK{&`pk=>?Vz?#3qi>HjSM?_R=M7(%4Ewl`YtbZZlWOGJ5cjn%S_X?<4zW!4$1O@
z?TeHv%CxU?`{cMb?r#D1y=~OEwqQzWC*$IpSEuBsYU~{`K*r1?rCiBoh-U
zD^us4WA!{qZz4+k=FqN3e$`Y2Xo;|@_-q9!w0S2oVnnoPVhpEC<1;4W(
zfz~M1^*pzC!NhcBo1HJaz9~tAwv$b`#!n`fE2;_N&x5VWgC%5}k+tYp3i%j_f9o)*yCVK7c?{n}~9>UXLre9@iz
zC1k}|u33yopirZ`Tm)0efV37;CcAZU$jY(emVle&-6@ah$|V_c4x70aOs9XG8-c1RS*ee>M*G)GRx~d^F~t-YZuMM5pLx|Se09KHXrVO~b$)QTfax1R5vsQw
z_Q6({nvPSvU-Q;ryi0ZSx7P|DW%=ye-7e)J-?;~8#f}FEm|)c2rKlA9OY=X%9U?W`
z_w1OA-0io8!W2>x1Hk
zZVzx$9!NIVJt&vO&|5!g%SPD|gU`2leYSYuv|rS50rP5~8}`?+J=UR=tp#R{P*_PM
z9~~&=Lmu(kizTWWr}smP)h;EXfNRJ;*Xhh0DNXUDs04dbDVlWQhGi`|id)|A${It2
zIlPUuI7*9r8qG4X-}KIuQ6-Ahc?=0#WPM
z?nen&zrcy8af$2FXHIQcO%)0uul$RGQxMs!ri&+UtHVja52U
z->h0M_A;aAVI^BCDp)6)4iiIXUGV!@{tO8VD=J7`c25@0GM=dchB@=bhC6uj0O$$5
zG6nUig5QZVN5(06HzYy#P+0D{}x|cT(t*P{ZY28;hFx88h
zm6cnZhu2bXKpLD^N@QQVd5=TPj6)5tUXK!sM0DQoF#GDuLvclM3p>8kJ>GP58XPHl
z4$^pXQSLI-MHQnoB+c#bn?0p*hWW;`xX7ALlV^EfMVcTRHsdz{{ui{$KX|`;0`ApV
zstGs6+wgGyX0+UWpN2@y-QEscri{EDfS(liZZdq|>z<_QA^yxU!o-5W;476vA#j(g
zs?e?03;%WTb%^h9RLH`22!_JU{D9?u43pVqv#I1z*XV;#Jm*MPV2?T-gq6Zb8gVvu
zY555qBV@^7W?l3p6o6HT`a3+^xAN`5=K)Q~AqtXxNmkDQCEfA$Mlmjj>mA+rk&Z-k
za;%BRMP1kBeHP|PGulY^(+6)reCRAzT)|MgiWq7Oiw@Xd+{%eG)MoK
zKHjXl-W*P+MjUf8n$N|%?7h_f;n}7GadrRgs<5o0qMTdI*9}boTBE22$LL;?;)^(l
za#ffiXEBOM0o*c1{s9<**BQe#X$5`1Vk@!fI+j}<_kBG*YV2&>*Co;vQ`$FU%a#$<
zNIcy_zWH|d@2oY($m0UZrmCu;1BD0{w(+KfYE#eWAJAZu!{1brzB-j{%dHjgKfdyF
z4^RQJ1GHG+kVTR%cHbWA159F6IKQtiZ0oG%5MWZN;leXQ;czqH3JXjJe9Q4P%Ouit
zrQ<3M2Mrmu;GInVc8%P#ErLehdg?P2WP>v!<6vK3CdJq=GxGnu6a5P(2weGbW^9mt
zS*}>s{Z>b$ZXWJ=cz*OBjS*ZCwM%y8P59gO_Vhpc|4wNhYxtIF2UGZ9sIxN(%|d?)AYx3}Zywfv|D
z=Z_rW1cdp+2e9&SUO$6C+F+F?n_F}
z83dq8e?T=T7~MeY3=EdVUnb2Pd3|{6@2fQEEAIG_N0MyGt;73t8=CKhoKCHl$n*a-
zwQTyiNpQTSSN`v&D8qZ8cTl7!m70XIyP{qo4$EWcSSk9>?&}BdY9A<#BRzKQ&+_`S
zm!bN8d;^sclrN*9nywf}X&KTQOIDKsc(O
z)aoD>dEv=A5sT!usVu4qH5yq2T%}qzR3IGaLd#2c6uqSDWTER}D>~KIcmlPU3^QtZ
z0IffkB;AeZWD$tI;`*L+0<}CFR{N*lW)Ezmx&CPlM&_BuxUa+2KA42ys%u3W9Xs4+
zbg)j+_zT_tE0zpLNen%{3QoS*KOxDuXyPNWjp*b@=tv*?mvaTd+Wf-yxFCl>K46b>
z^5$?MMN1hjcj}NjUJy<6I0638R@`%{SFZTD7w-P|X@4+@n*N1VgkMZ^b7aF_5h|8>
zpxKQ$Cc|wib-9#sRp=ixtwStOP+C87#}+Be3h*C03vJnx8zcd1aYv0g|
z>!ltrWADe!#f{0;LK^$yUB>^N1!$0nEQgnVxiC=rP=RIGhd5;YD`6cr2Dg7koBnav
z@D$gNACJ(ldNW_#9&Q+NeI5t^4#Kp_l)X_hw&!?4A`gO=%e;@lwfoq~l3_}TqZn|f
zS{wS7c76eaOHG!21UST#eN-aSQ8#7F3YKOgQTk)kZjjM3U0l(>Z}o@vf2B7dqkv1A+a^uu(m;8uloeJi(#>X*$wYYb@Oen=
z88^+}ml%8hr4=N5#ZW>K{;{hPBU>15)lwXI;_(mXiAZ2Q^U2`m#t>d9Zt`3DA0@Q<
zA3OD$kc&#EXjDV%?mY-}a<|<1vd{u2+_mNJJ0OdSfKA9r(sS~u8>At{p}AXJv2CUd
z2BuukSeN}9T8#555-|k8n8Id$rI-KAUYq9s=f0om-7fd(9*Y_d-9g`Qa6~@PxzEH|
z^9}^+LqYNV49iA#iVCtW*j-FYfn=wpEbzn!Sa1T-*+)9AkpvpI!8|VvVerN8_nR^Q
zX2-oEZl?Yv*T9MmL&V9@c25C8K
zrC+f%d3%18(t3SWL!hIftzy%4F}Y0jdeES4D98{mf#<(;o&WzWn?2X)Wh3Q{Qns${
zdypYCJd5nlgVzT{kT%_yjj14wR&2Z0$C}+B^*e3%t2zQ`{Q!v%<2THHDZ|y`)oq0?
z=vcPsRXx|=t(g8VwEDYaoaR^~W%}Z`0byVSLTmuJiS)4BWHJ-p+=dc8RuV~g
z(TM}@;sSpW%qVX2GgchaEx~O7gj|-)2FN;0#mSXFT26`XpGp6~RFFelVYMth;`+y0
z>&`P6(o2Ib5U(SJ1$4E7xf2q~{h?VVTekZ)$_|jjA^BQIeF>U{DQ#}g5tPYlc4rl`7*Y6o$5S^qS(&5#cOws
zNN}F4LZ0v!dF`)7)?QSGb3<<~KX$fEo>=sS;T`k+iwO<=F@B8J9?TS_N|CFkSX$lp
z|2%N~brD9N$8=>);~bgknD6^~FY{HdWJ12jwP+(g;wregIee{MNdnVtK3G^^_0|VT
zi)%_LM2oGod*&-#*7__@y6&)G;NiU)WbPn+LP^O;oagg@X5|SJr<;nNm1VD>f)ont
z!JX84s~BwzZlDwr{#w=iZIw&*@5)h3CT?vbPE(!ITJb(M;@8Fn%Zzw*vM+*QF@8iD
zcrY!q!U3p+-vAK_-Nl!xFuvtWVC#^U!(dSP1X`Uq(zjgch72R%ESYsyZnOW|hVrkl
zjV!WVc4ton98xld5E=(ui>ZIp(7ecvnNJfU|c0;>Hy94}8w%8KqD-f$B{nz(jfX4~%j@03Fh#|E0_k
zO#>ePXL^C1eqZ_C(b!h2Voz-;r^h8(@jac_3YDQ+!vimD&rHVu2seIHo35R!y_F0D
z0%2d1^eEVFryJ>^a$RDSX(mchOwu=dizPh2w7~{kCUIgwI~4xcZ5sOjIFnKVZh!Y4
za9~Nw_q`1MLnP8ahQ@qqVL-NK$ypJ+0OhajZ=K*2WgB90Im9R^t@7Dj`!avG={%F6
zcO`8YrR^ajh6bRBsLh&@z6`&2br!sz%9rD)u|$CzjZ|O%^Ot3+Z-F#pO))PgxGXE|
z-=_zjtHXAm87O9c-ZEMMfZbqf{-P`wvh*5Et`Q7AG+Xak0_$WEay-o2d*O(!A`DJh
z+|b4KALtIl0f>X20;$iJ*qof=s{b`JWtI<=ciB3L+xqBIvWQ1;zTj}KDcB{+sA2N0
zu=my!H?)M>xPTVDd4Vq(hk4;78R-z>hiCfNIJJ{T>I=l{-ANRME1|K9>X5jUk4SK`oVd$ARfggk!O8kNcU(&iv2_xXEo#_u*u30LIR
zJUS^}8}z?ldPuWhtjAfI#$v}yL~nSpM_f&R%|hKY*!|O%_{iToD>ds+n}+jt{@h+y
zm#8Vf0cF}m@}1dgM$K%~ov|1JxERKOBBOuo2>&u{K$MXfsiT;(dX>34pDkt(?jUIX
zMzWFptL%9t&75(4q>Ng`Nz44eM4~otpAU1e+2sn-@W6$1qJ?zmtY?b&sHV6wtmtIm
ze_Z#=CnTPr1mQb!zEj=SJ!;5-gj$(wE-eK&B^1Yk3x`g_+
VAYb6g+7
zpWxrLKOwNo8Ge1?YPP_Im2@|?_`2IH*oCtl*&h_aNJffa62uLEY#q68Z7Z`#n{SGL
zpOifqK-pGhRURAt{97^mGX7d2)Togx9`2D}X?iL;&Uf-H5-$saI;knvA&ax`2UZsH
zqek7L8H+RUUw7%RBNQb3AVcoOO-KsD)OO}HDF97ba8KLd`e>&*s)hwPa%bCS#V!#j
zwVhk(N|=#{%x(d&==~Axe|AUg)_~=;!H{9_G0wu+niCA3HqOOM)4^YA+JX^TC-e9knPcw>@?;7ZI^-DS^Ir;v_g$q
z+dFo#n=YN)=8@TGcaAuvhJlaZ7Cyyx#SrH(C24qGpu_~IG!f@W?iX)n_5QtH8RODm
zQV}KHX|l)r@IkB>v$wa>#B;qnpZeN2OSz+~T{VUOXcvuw_sT(5v?4gJS
zVO1zHuKwmjtnUh^do}{N?vb^7{L}al%Fq@Nes)qK*Z)YbA^)Ha;~I!>sSomR9v=LE5eKI_q(ai*(d9^YW`
zu`)&B#cO8Ey+(m<^XL9^1B}SnU+J#2-Qge4SA5i%^6pTkSv`EW37^B^cM}lV_i5DF
z+~>?VnB4v9&{KX3RK(prR4iMv0LO9lKVKu|Dy+)QY2R~I@Z8fp$6tOOTJ<%lDHwM#
zQ^6Vp&czI+Xo=n4CK=c?N91aJVVa)wjxy;QAm-+Vfbx@wE6-DOB44IcmzkEf9?9J@
zSXh20LBwZkZ*1gAGirafQOT)h^PlKt4cDPvZC6M|cYXEg-Je!luf<
zaa%w7Tz5b3=$hOe;8*xi@%=!vUGFw2^V!dPmL!ur%uUe!!m|hs5k?snaQ4(X1qws!4USR@SG@ujmKoa}-$kAG72zfBSnc1FE-(U678t;Q%$F+#m
zKhIq{=TO+SbS8D39GC~i6?0hvky>R)I7U1QUyub0z&F?#DRroTS&7y&_
zym_zFbv8}to^Ux!xp1iEs8X`^@X+oQ@RS;7nRCB52!-#pVut1Mj01kJ~=$)e)BgYt^@W%22!;
z8rwI>4-K|x@LsrmWdhsS*FAQU#s!Nij4X_o^InAuTuElk2JtsjIE(f+Rv~}QU-1nU
zlacZEX?vG_!^0tw^08!==B%#2b8!bFcfMSXevQa_oN!seaw0E-UaPUc%tM&nwPeoj
zPKjD$qmA{f!ZzLLV~o*D2LQjXj9ARB=(Dv`;PcS-<7SC{zyd@v0ROxGik6qwqHq@>-4-`
zIAar^n4%@x`IT|jZl!RLjBGlCv-@Z(Z3{x@9nfH5tDeVr^_QT3!!>W;)nn+#%%-cI
z2fK6Xj?2`pA(CFiN7e@uu=D2`YfiGp)Cw}J!AP?0c;L+APK1h_?TV!Sy;SBpBfRR>
zFi}KzGYWa|G9j^33>R?Pi+I50g^mCM`fVD}(Dp+zz(@09hg2ZbMzVCmjLh8?C$kop
zul)Jb&AX?5PUPc?jK{{z)Etq6aHk12?-I@C4vw+szRg{(1ySOPPcMbBl^#<+TX76E
zY2~CU-v7~ErrctjBg
zSEp^Mtvl?`N2@#b1N^RSo$zW-=Y|<-A^8uD0DIziF4xQ=n$>)C$IEiGI^z@r|pW4m{iQ;+GV4bcsFQ?uHQ%Y+@R+jAm-l#1?X!?sLY)_Qpb(
z?9SCe>ps}i!Yw4P|6h9@7%D1t>X$)=j(pIM^G@P)9%_vI?BKuza86_Zj1vE26KoJ@
zb(UeOw5BLqTIUWG$UH{?P0&~#JVUoEDHWRZ(GsT!wqrR<^v355-eS+reGFGBLN*9N
zMZpj>Hb(45%sP2KSakMK*j(d1k%>B5WSAS1CpSVfO^1r)rg|^I5~vM^4tcC1s7Ws1
zt%H1dCiEjcu)LTY(a<`CJg{fU{ZFX8vBb6Lc+xUq)|c^*QgilEKR>2k?`N?}4OI-C
zvh3b&xDqsjkZz!(Jwj@%JiSdT!L61h1!P6VW*kU}prH2nnGk-dSw%ZXjU(f1A%Urr
z6PSc&T`vWP7lkxIXk^TzR*Y5&K-OJbgr^Vlym50`>i72cai-tG`Jvpp
z3b9)4O_t;pj86_?yVjbKYS#yVS3>JCcjN91lS~vJ!h>MpC<%PQi>-!W-<}DaHpyFA
z4fXWXn61yOLV7fwu#)m>_kE(Ffx+isor+Dpgd)ni%CO$)^3;OMXP#Gb&9
zG*j5`3*$V1bFEz8)KR2;R&^P5lxT{11A;}ooGS7?J}4Zc(NpvAn7qbi^pKRB(nMK9
zL5}n!A2E%L(ePR9w)(|0BmLV5^ZhD;Y#P0tf~uYirkuc-J=4vh0}g||gjFo2DdYi1
zLa-K@0u}kL>K!k3c=0F^$cMt$;G~@}uIw}wGBiA-LKUlf2Muzof)nUAZUy;#AxZt*15X9K3y99yMC^_z
z`ziO1`5oR$P_7&EJEan6@P@^ATFubXn(k?w+1~SaJ4?i9pKt9gj@Usyfx0@O-gT~7
z-K>8MAdR0L(2gTa5kNCe8yqRYdGIpT8K=FT$<{nQf|7zX6G#_eppMS(#Izq3%da-|
zgA&rjvryD6CwZ^?D^;cenuF0QOAXB?4{Tf(i_bPTpI&;S1B5Fe5YTi#0O&2R_x$u|
zV{6;`gGiuMw-vmm2F=s86zJhEDkjD!EsZ!;to6zj1(59Is+14_d;|)5w~g-CYem&d
z`~WWr&>h_0y6F}=)$GA6=_sYHKqU*7pLppBs@F>06j+M!|nNCaU#&>wFk$7
zXNMC1VTYTxvbKi38*LgJZ7nQ~V~NfAj3At##t+#*J;OK+v+BtWQH+&Ud&XN7XyO-yXuet+!@3XV
zi>3Dc*bnodEM(H&*LW65NY;%5|OX~>{?SyY*Mj$GbtjJW8;<1+vavPrLzS3
zT|JCq(i|}IH?9jw5_hWeB|H7XyXW{Qa9W6rU60e0#WiB2NJfG+m+t1M2AN(gzq-04@?3>K_M1ZZ>bhk%!0Z<5;
z45xIyP=~NCd!PbrHYaCiZ2+s=>i&KKYg2GveN9a~o-e!8;hf(K85ppS+5jdJdgB4C
zBsN?AD0|)f=dE~l`;W#$)OU7PMe3X0I#LM?P)Gz^Ex@(_YuO;se(9?}Wo*EwGHdkk
z8=+p@*m1J+)K`qnl@vSGYZj9rg;uoTS1^RPwn$>Qo@^f4NOZl3!WoS9q75&XcrTy2
z8SMl3Lb?yibV>H-EO}z40VEu$-m=R!^Y@G-NX}L{By1g+;D34sDFn5p!NASjlxMu9
z_=p%7&qvbaEG13R0#rU;er_(dFEi+;_~XTZ7a?B}dG-mO+02a(464eK1CdY%>5rvO
z!-B-EXWt$xte~(kZPZVBIWi`Yk53XFvz`4!uH5*1PbGQL2J~oK4K#M?GQoKG(pX$Y>%w$|-q{nI#_>Ug#`H`>=v%fDqo
zRP%>nBw)M3TC1Y2lR38ampFF(AdoN(r?Q
zIG05NwvN$l(Ku^)*`_MxQFfl~nj6Mi#>0ssymt+UiWk&T`Ju{nS^=@IiKWQ#l-|wi
zVoEOIjxSWA^y=_<^@qIqPBAD}O!O*mCJ@3m4@pdyfiZ~^?EGc<$j?T@Uw`JCQFYaG
z6pU;Kxy{&-BWJD?mu)MeG$k)+UDu&a&(5$jn1mh@`~G-%44H8}_!zET8xb*u*A;F3
z)OIRFvU+H>gy1A1ujN||aDru;PZxuN!6OL0ZopoM@Bq1{cfYb&w2h36#Nu!u2dL)^
zOH%-nwNS#6=JLS2Gl01S772M9Ujs;#s@E&CxjG{XQ;bMG!1Et$P7)~kEaP%t$G`de
zv!SA8p_&ukVf|D=nn&c$42LNW{&&|3FA?b~9Os|or;`hfJCBl#E-oDVF=4@Zz8Gah
z@t02+zq$K_jD-n!JWNPFM(CWo40K)N-k;X{VqIPEYu0-psTcWtr@FR%)`o7|oBk}h
z?`As2gjHc&*zW6&&E|1Jxs-6h%>?BqybtMvL4WyfoDz>Q`^o2z$ZStfPp7-}6DF$8
z9UM3$>}KpJ<5Pg5s40kmr>6rD)U=^#-JwFMp5CGENB3Rf>tf1BkK}jWe*zKOyBS&1
z?W;eYoYU3OW8f=b(x^{%iX2<=yQ%NmU;S+5@P0bNN;#~58>B+WUjT(t!qf3spOCAu
ztaE$=j5{zCuB}1Mo2$tx;l}J}dFN7+bb(u|Z=&GSIz9D8JYXStls2I)L)=nayk*H#
zjumkUXvHmfBZHB=n!GIa7jb5o5V?~xdNfy3-888
zQ@Iza4z_{YP_Cvg*~`-=kU{ClsQfmJK1kJw@yrZOHfV9%)-~;h1*-J|xpSgMs>I$u
zaD^yS|5)QluBz^@!vRE`GQRC`M=JS*;cXaK
z_xW(FtdAd70oe@w{v8t35{AqA$#zZ9;a~=&
zURYdojNsd>m@H;wuQiQ~D*%TCS8nFT7_?AUY{R(5lXOT`@;>pY
zIHh_e{3ti!_(w<;S}wtQ+RMHPgBsrRI3%I5FeV36E=&@4I+1aV*7(Mi1O`v4yJaY<
z6h96f-=WcGzBml~AIUKRx67dnBj29Zx4WuDO1ws0tXASoQ65-?k?wKmqz;0$JvBb3
z%j)*#$NliwN1m&W4-%s*Y2h4)Npm0Gg5H~WzTHFaBc>;D){
zSS0IpzB#sTzMS_a1$Pk}!u<&bUq8Ejz8n9!`UVOJ4}4Ca)!_{Iq<2frTSipz7QE3R
z{JZwJn_W8Ldxode6ecVjtWGJ)E6(Vj6Xog}2gIk(4`x_g@~ss@3I4#*@SEr@1Rm2H
z^K*7w22B(wq|2^_pbJDgFe2b5dVH0WDdDf!`Rwr^#OwPw;cZ&;Y-Hl4bcyvscd|jA
zZDjmL^N3d&rCQX@44*&5Z^%CT}dIt+*(l*x#nQx0F?{ianTpB?$RG(sE)VR6gSmaj}guN@~9A34#Ln3x6yS+
zx5jtaEm6lRCS&qNBnc&VN0?xr7d?@KoEd*4MTHr04y)9KUwZ4bFK_p!8hoy^*A-u8
zxjY^AtG`)|=y&VK_~o8VrN4)lfU~YJ&Z6Jt3}egr0T&8~<|l)fc9Hd6|&+;l$TUjN2e7sgI6aA>quwrlzK4a1dd>+PXV|(Zu1|(8R>x
zY@;_Sj+M#3kJ25XGYO(>omIqk$JqS&@I~fgj143(!MkGVqV24I`}{g$^L&n(^t>e>OTFdP7U|W=(*y*l{?{tYG72ZC^{%spVvAhqo}H)ho`+V(9Gq
z7LA@Lz<~CAaV=YfXA%Q$JL8S{XM!55|Ks)NXQXQVp6>ZF
zFX2&nWy&mY)O(@y(V9temxhb0vFUmP!EOKe$8os
zu|@g@djZ3(eEPF-tr}ZiRJlv=QiCaw*8Ns~E|R{~4zAzNH%Q2wyJSpnF1znZ)Byq1
zOU7B7Pu?!QJh`;no|#||blK(nZM;Ros1mll{g6Gsy|*wuoGy)UZX)1P!;!8L^6sJG
z7_N!D0xv&rf5Br7hcw4fporqTdZyp=i8UCgS^C(wCSLaEcFzQsck&Xe?Xk6>
zoj6OIZ$p{BI9?mJ!!7_1y0#Q6Ql?MFY6Z0ego1V#UQGavu-o+!(5gWba5ceS0!Y9i
zeoLT%T(%j)Y(DiiM>?4wVDNMp#^-Q=1qf0r#a-!tv%$n3c>@+bDLL8paE=+^Dh2}q
z9Kj#*UV<{fcw;%B`{8^53&rT<3!iXzRNBNcSXG-Oz;!v%DMUO~3%$S8*Vz>&9D{A0T$OKZm<3lLbL(->#w%0rdLE^NlZs2Wckz@X1UG
zR&RODIB`M2lo;xthR>VIcSHE56b6<4D#ox}R%YX*D6_CbqE1}MTrjt#Pp>JxaBbWk
zqQ+J_s$r*dY)Ojah<5k7tB}zJeiUb(BpQFMW@!(GHZ|QH0$n@`qTyiYbr|cBvg7lY
zC)NDW71m1i#gmR|*QV=*16ER9+d*hF2&!BPdz(vxCw9jf^hL>6e|mWo9T1C)u(X0k
z6)k{X)zGP|j>*HGkD7j$$`0YdN=#j!d65dKm14zJiB~8Gf89G;F4my~fwZuN1+{Xq
zW>5$wo&HD~pN+G#h^*|p7fLdK1e*|GkbEHkZfIy2$XBtr)jw{!r_Gj54i*l>Ap&^i
z07l0aAWpE|pQ5?By8|az=NOrvukYKW5el}~-F@O@XI}+~iaT+w8@>YI7yvnX+#gnF
z?0;HMk;Y@-qS5i!W
zGl%P|uKz@pjL@W<%+`j5ADIPFtUvX(4jPFd6c~dZFh_tx+655WI)e29Qxq|+?T+w*
zI{*`E}hJQ9?R;y+ql{u
zK*Gp~3NSo&0Di_hbaX#_PNy9|F3K1Fwt_>iZ$nVk1-xz}5t(F#ib^zUAnzQgEVU_Kmm=4NT^
z{%-L#<=PJ!w&?+Tv)xtrmg
zE+tCqq8|nEd7JRdYRNt{sR;gY^`|4t(ZEt<0`$O9hu1(DOFD&V9&iHed}L
z(7wI@TtM@D9ho&xatrKdL^!{;6<&pG$;f(lP`(Hupt=G(%2%`C!SZ71n7qZf1R`k0
zksh-kE9+VW2hM_HblOU5w94*nuu~3=$a@%?hfvUu1E5HTHO@FBMpM$ef%ym*I?z_@
z>J#yE&2ZI{!K;ht5leq
zbxj=TKgD0C5E?oi5(Dw#NuV@=P%|cLagRSwv2A=@;mArcm*XPx#XL|azOA$HhK^;@
zQ`t0ejN$3BNmH3s)4Kb;93Wwl><>{s7JU8jrvY?qq)j25-V5ncW)$%bcpt5VW}8~p
zXU#vn989M|54$2W_E*TtPqvu#GQ9bAiGmw77M6s687#|Z)B0Ub43-LMzZhF6aY6K>
zM1*)(Sp}L{V8o<<~g
z-M>&{YwO8R2T++uDii3!wfhCrF=}F>bT(Onddk?ws9ztJv6sE)j7FV(z~Hq;2;W5j
zKZXZU5k4KuA#8r*8gysuZ~z^S`4&=K$XoG?+?5B$Wu*nDQ7WU7$t(#%wL+7}?O<|9
zIO9alSl`jQ(nIcqfgg4^7Q6NWeV_dB{K_*rAVGzq2MAjB{r7HcM6`2+jp}zHW)Xlz
zd=EQf*8fbSPxtCTBF{o3*4JA}=PJy;>ql8lc%uGjPmri!Z4`lt>x2!P5DKa3Z>)li
zSd*`vv+4sZwz6uF7ls_xo5tUsg}EK;<(t5h^6ARlrPZl+h>006A^r`ob?X~#3j?rN
z5qS%5sn3lam8ur|20s=&tjjawB+(nW+_s!m)P%xlZcV76{D?VLxOIZ-$Ja>kDg4tn
zyhq6l(cfc1P0!d2tbIOr6QvkkuQHGfFmS$SY(R?pVEkiu
z+N80uuEv818F?19ww-e6)5nZ_gO*peV?})S-o>wDaPTprVwWxLA_Smd>u1bo4ZRck
zidAPlI9&OfmXk#Q#g`tTux{=iA`)Z{^6bVu}69j$z7i;
z@*4_Hcd<`4nz!3&@INKFRRGYwMM>MXC7VIkB!L7Wnsp+4^{gxQ4L#F+zm1P6+XUJL
zA-sPI9Xk)JOwa;X;$1Tdz^fDkcyeY{`@$Fu
zkfcs2z0ptLzj6xT%))$rfcmN3^bddZ|8fs~UBceM>~F=xT?3F^J&W)!nhPbfydpnz
zQg8Pxl?&7Ag4vIsDBrWTAL1Q4#+-nwMEUke+nj-c`(!=$%YPGV=qdn+9dOOUXsEW8
z*2TeC@KXQTJd4#mPKScvssb2l0P8j_N*MSr>=tj;eF?q;h>Gj{*s%
zUNmLqX%~G}C=?DF)-J(+xm_g9TIlPhD2J^zxv)`2@0VyzfxxcnzIvI`zymN7hu44W
z?BOm5sL0RvcCEGAu%C)_z%%my)%f?ZhWh|p-pjo@XZS~2&%*Ipl(V?q5oxei#JZQB
zHHco`KcBY70gy%jh=^Y3pDyEyvtg=6aMyMUKNRc#rI(KPcfTuCJ=3riGMfi-i(Od+6uo;hFa1BquW6`mu#bnQm@vMYI2ezDS|t1+|?4m
z)dfhEV;A--BdQ^~-SA#yeM9DOepJ23Y#rOa=-d?>NwM3%0U-SwfKm^>gQ(mR#@$NK
zW)>{`A^RU2V7fX}AOQYY;Bs%vdZlgcG275?l75j5>bv6UN(q%8MaBFdh~tJ)8}CwP
zu869h*W!@SRQ$k|aK5G8$0@0@mb@!mPH{~1WiC!mlWpy!!y4RovrFKAxoL*k_rEKv
z7DjWZ#C%MrYx8)oR2&d^(EsS+{NO3#`=NxXOx*o#&FhN%ZcPL17N9`$FADSXf7mI{
z{F(VqIV~V~wW*v!NNXsp(SQ%F`63O1V`~I|NAGBNeA$sWAV>=1|L-sVm{^s`oMk(4
z2L%m*3g^!Slj?`gNZ#EqGZ#q$^u2w#2Pr&7+gC0=I
zz%`4u{I+Uipt!Js|G|k8;hy$)rYu#OmnG|44`L8}6`4w{6mdp5TgdCTL<#;qM>$r#
zO}1!sEy8~H=1~&IZc-icj>F*dK8~r}GOFgukHFt;0NprniE97*hLgO%7J!FB#4(nw
z8&WdZ!L6>Rx3#cTjx(|BPf_<`&R;p=iC3T>2IPlz_x!Yp3gJ0V2=%kueMK@1<9~n2tO->
zom{VVYXT4YYr=@5#oky3(r#0+IiN^vFd03t-3d=Pr^vG
z_4;v;e7ZeT6sNUdTN=By2DqdePyiKgm|?2j>x$V`%bcw+yo#^jB)spddW%X@l)%C;gL<%MN3EC6
zA|k1N$8$va)=UBQ5XX%#k4d{ZGeE1IcQ&X>3f%VA35Nx*#(xDQL;d8e;t-S%0ji0M
z^|jSGXnnGGURf*b_<@M5`l*ajGf6SNdX<#t;pvA}f8`HM<%}Ce+QbD2l6t1LLc$2$
zp^|PekXkJiTb&K0I%{E(xz0NuzeE|m-l_`G!2yB!k6&Nj8AyhY;-3TgKqlBTzO}`-
z7Tg~A$gNGlOVJfZl7Bpo6g4FI(>^{#o3laT?G{_Fbl-*|H&XnXv}X
z3+bj97-{ZJ=bZZ|cZ@VI0M1WOCXv*HMEYD$2+1AXtOB~7TJ)az3rfvorH|cdQ$Sid
z%Ukbp`ArWUx!}1Qn2WWb`Gt3RliHz*%oKbwmd%SIKo;};tDVH8Ek=H$Lnbw)_!f~7
ztm+AaB`E>Cq-R3a#!WU(^6zCxK)E;2W%8>t1rF96@jekE)$K`foVuIVm?O{aMWX85
z7(4{Dsx`YzZk2HD5en`kKpwSgXg=%~S6XlHMA^mO=Is(5UgrGx-qjvv|KS&Adqvy<
zyh&(~?Eb9L=A3r<;ajgIXQ1&RvpJ9}3kxKNW{!@}Nb%WQW6Tu-`U0+fW+T_s^Fi
zrTMEQP{!fFnf%o8nru{#idChIi=nFtDbgsJ`^D2`6)6tH`=@lTXWw%5kdJVy$dc<0
z&pI5iC(?>}Td1-4wgf=ofJzPV
zsU{Nrznhel``@Vhd-_DAQ=p$0U8R9x5Bk1kKAVK9aiY_Sx=T{>$tggYy;TSKi06H-
zkacc+KZpkh`h-%IEINQPD>1*dmBG87{hE6>27#;=hC6ba@!AtQ&IPiyYUVfda1X6$x2{GCF76h_F0>N#kVx(&~SLTn$e
z^nU!hGs=J_(>XzNy^N9YvAapI;O<9|;kaP;7r&LKM4;1S>mBkf4t|?biQlLZuo~d(
zp5c{-gkizq$1AfO``ZeW41GZp0PRLWOt1`%I!g2u*NRjVa1r}VUZL^!vXk_&TYH84
zsbqV4{po@)wLK{cVmD59&b@c41F8#pLs)Y{&_uWQm}UDjQh1GagoH)xp%lvxe(NE^
zZS#$G5@H;b?BKM24gZEIp8GoZi=`?jr)SlQflKo=pc}wK34SVih%A^|yKg2~d9GUM
zx}C3$+4Cjo>z&?t^k8O3SpCY=qhxNgHU}YjfSaWeReVsU&kPUWlJ`BpZCI>!BZY#9
znQQgncSj}X2MI0X?U+!xVEgx-SOHF1B2O_lPk@Xvh
zxheBBU0E@6D2L}qS#DpurHI`sW19%KTjc3g>P
z{;_uI1-Bo5of^{n+Hq@mqV!!&mV4`|@Xn_|8CE}yhN%@H)mZwMcTe^A#6bDTd-e~w
z{Z1(*6Idy~s*o9*I0NS?W`VM=vSSMdz!v9og$&A}1xQa>49j^BzbalZ&AK0fqVj;l
zB2i7Cls`~(pU~4s)8dDlXB?NeyYH{=MG$jHfE;N7RHRoj0moEN92E3q6G0A^TfT>Y
zhVH4P9$1bIjlJX^Es5bZyT@9d7PGjY99=;~C^fr=diyKJN7q}%;;EF-gFA#j@pyUH
zZ#wp%GlM2)G*9hQtkluhCqk>v9}(3CkD*PV;47iT89KPIt|H;D**n>f|62;d865N@
z0Ls-@iXrGN)IUdA=g
z_~xoizh{^rJlEd+uEt9++DLl^%%m`q3YmjB^0(A$tyWA_jyU&4MZ!cGP2Y`Z3m{u2
zYyRhi@QDVrGQI2h4ih!_jc29}!zz(J+2i+*=}#`+g655PQemsWUr(0$yx(0v2?Lo0
z60ROxs_YJ;B)&L4&Ft53QIr7gYiejvAn8w_^6u!TcbN;PL8?{wGd7CBsZ(IC8FWK4
zavpZ=&3(U|opYyI=;5f+ud>GS0i~*kWD&qWdxG7z+WZV806j_QxbWAJ2%TfpHpz-=
z{7HO~g70+OWS$eo_&%=D4?XTkJ4&d;j81
z90-u^{6xAbf^L~+osQO$)SvZKf)_JSjMZgF!&LFSd5IRDD=&fC8`m0+)%smeN+sim
zq~&*LD?w`O9KW=(vr9CL>V*YKIVp@CV`vEvj=!m_f!iKx6TEn=%W(|0OhmX^iFaut
z(R2%1(de-`d;1Gvih_-GT`T6+Z_R>kOQKWpjijoUc%#CZ1!#v>UX%neLnxsCBELtA
zL;ae9&sD?*K+rCPz+wk>;*34vy|e;V!8F|Livr(qR{PRJ7fLVAWdCM5lMEGjUWf1t
zfAPGL7?_sQencjObCIdLH_$rvPEp>t`WnZCK_&$if8!ZEI1d}e}n*q(rXWnyUx2zM+iD{v~b1(a{#&AJo+X?fez;arP){Ky|4}cRoO_+Ua+trtK+j$wvJ;q3KnTh(7qSnnlxOL>Ar7KWf7?S*Z(!cB
z+GbTy54{b>>x1PkTmHp&L+X4g^t>`;p?j*Ri;4GZru-PzA5kOa{>6d$nE&Dvc=G@)
z(7TQ8P(X)+f$E2UtJGQ&m&<_bR*nRLz1}mDDca82=S6|y{hYyVXb_VWlrP`1MlDkA
zU;b<7Q1nJ#-S6^Iv&xLVb76X9C9-et1D_gS!)?L4e*s_N@Az3ELBsxF2wHR65FmZ|
zk_N94d3{K)cz9zS5SF&iNfg?Nsk}^po#20gmz|$h1k;g{m4exsh~wF{&)c%)k+vz}
z*lb(BY}q<(HBN`MsKD3T<^Fq=slN4QAG!m=9JH99gsAY}vWf8@cq#+)0pH~x=PsRxR_#aAA3{|kOD?G`;EsfR(8z5*#8G~2
z-}^Vl%Z2?H9DILo=kJfkhY3=_q5gpe9Y;(0^4#9p
zwKL}_pb-vML3fy|h^n$o_zOG{hw$gB@`dT1IDd#O^EK!tCA$uNKa`*Uv`I7gKkyv9
zH~)4UY^^Vf`&>bOT`Ww8pWpw6B5j!dTLYmPep!8ZJ>UqE@QBfIx?f{;H|W(42ZsKW
z7mSyR$T*mM0*pES8S>c101S(gyQ=QLvnz_D)V%*1G#WyevD&FEmGy^M&d*hZ7h*>N
z)4E6?@EVyBe*r?iNGMWGj);ISvv$tDh14WeIpUi*FdQTlrwmReOS$r{gme*MesDFs^RbG{wIJ0*3C
zd$`EMeGa`3-$3TVux;xsMbl}zm0RECpR1&Or*LsWu;gZRGFl)oO1Aq6{zvuVSo4T~
z`(^`}2(=b?9Xt^P73Mv+&E7@ya$-UgshB<}P>mCEjdG_GBiv2FLgWpbdFj0qN|Wa_
z5?;<{N#jO6!W$&>%7kU)9PqRc2UUEbQG}76k&7K|@MkqpDgBN?O?(3S`p*@78
z^nkZ{FvU;qq2SypTKXd)%Lx&r)vi=sYHkYa<||
zXB=Uxq4Y1yZYQ|#ci9X1j~Y`7T`Lj-b6u9{%&WS8yh;Ne@;^yb5~Zht_bFs-4-Axy
zG)Un8tM)c}{z*-t@WBCet%)CSHbjBTw_m;4q;T;SNsBDf|EnRCKRBy)CbkKZGMf+O
z|AT&hM@V1-Nd`Ig#-%&-2!?ZHf`bs9r|RZ~MIug;P85G7iqsNhK!-yEI1sos5^5w`
zjpQp982RujX>#m)-!e8g?arAQuD>6|Ft-=;7y0u|7aBnoRsi4Ab>jAbYO<
z?KQPPg!(_~G|}Si8bjJmKLd)o30IYaajb^KFTySb;<9-F)|oaTK_!Dj&zgT>aKb;U
z(wlMfu{fHs-YO@*Y|{mY_QN)RmSUQ@0wLgw!po?%f5k$Mq)6&~)ux8?|2C4vZ}qkl
zy=-Vs>Fm{xOAFBFedv@O-1Ptuo!7#LjC{R+z&!T}o%hRXM#aRJQc=l-+%SH**%rru
zs{|(&SQzC6@_k^mDA!SectOeqmFgP*3_-`sUc~rm!wPYjUD##+?WC76(j^S&Kd`h7bq3
zRxLL!yf<>W|8UiFilFN~jJdz&l*OwU^=R8xfD`9O@fU%FmODjoxXg>
zSYou-JFTy1-m%5oSGgRQx$0i)IL~^e_7x94j^cG2MiBs-5r*mEXR_%$>AN`V9KH6h
z=CFt-9c$jsLUFkqARqTH>{!>G1P8p%FhDT)cK6*^C{)h&PBA|qGGA*yMoPaqK7WOP
zQr60Qxj&UlkPO_N!frjd;embtE@mdYNSb5&T-_BEm}<=}TC4?U$ix8GA4`-#U`%|E
zp9vS~=?20?eBTq2E)BL!wNE1>AB1dLcccrsAXMDFk9`k%u0CRGT69H>C_fve;egb6
zl2L)Ya|MC^eDg_I#&6r>#*9oX`I;@;vW0Y?e!gyRx`zj9xgufxv|jI9{&?Mn85CgNtN^P%S){z0t||5Nfj$lSYmX>k;m-kVz8^D`s@sT
z1`jvdpbc8W9mRP}Ewly*-Wlx~figB9!1s1AKysQ!$C@eM;a6==6NY2u+-i
zSH@~x32d^yGk8SJx}O1-WMj6RN(ihbAMQGCGVFh|Y9?O@(9pYkDzSRdu}U4Vf6B@@
z13dfxugfZcj5VZbZUVR2p
zAhi(bJLm8TY}na4l&W8+C4@UM<2gVhGE(7>#c-Qc&S-%WmX#+fhWj%kBv~*f
zVPc_m@v8~jOe%fJwmy8Z<9m~ws3to!5Z&qzY@Zrm^A(mPnz27BE
z#y<4Mfefxvi1hM9c=DcxT;0mU~-pHHQBOl~;f
zCGo!P^G{CtOKfCS0*n
z2jdWNGnh-$mO2hVCkG3R
zx1e~Ivp3%xN>KnD^~BCOwveuidd3A#LVG{^zUwU1WM22S_XqWH
zoW^pC!{}Y6SSt2)8aA1+I2hm=b`zv|JUoWi6g1lLaWcbPJ@lhdXZAo7J+42Jo0Mig>@oR2_(^$3lMt
zY3;1Q0vdV(2W-|A$G4)Kl7w5O#X4u)_DZ>!mJfH*sLnQ$d)j07VPF?
zJJTgT5{3}T~u*
z7#Wr~f)TOlE|l4{0IUTFApIkRWU4@!L8%c?M`FfW4xk+;i$nIxU-}6UQmVv`XjP>E
zfTnoitM&}Ff3_?D7y4|SQcID=fUj0+5^eT+8TW|c6aT@ek2m<{d
zfxVGN+H1$+;^WNN|4e=zV1nh?|NqY^{BkdbeW!b%&3AhKytf4TtXbnb0>JwI5x8?q
z`C%nG`^H|lAFm?IHkdzkh-LBKwz}6Hwhna7xaFCMY`ZCR`*4nCeYPiCSPP=
zGKxSiP*Vi~ifh7sD60QVDK_LM+$tYWjT4#H@vXV@7_!UOClEVL~fO#d#DP2ai+?i)}<{Q>X`S0BH&n$C})Iuqob9;U$DxLrdm
zi08SeiHg+R_FvnrTdX!J9+>#)11GF>$6Eyt*kir}IiA&RHMEbwuwRV{b84jk*-`u7
z3={zQ0>;E`hvO=>-uIvZ`~~>Ow9e9#y@$4Ow&nlP45Qqsk6g%=a$J$dj9*xLEoVfNj6e8V8P%AZjRmq^8wVkc9=Bd
z`KIPjaj??OA(>5|CZ1f{@OkkO0j4&+Xd}%3d6zIOYkqjG#g9|g9Iou?*gVk95rM&O
z!dM4Fg3&PVt
zW^V-*9f2ffrw6wp)+jCL_$@5ILp*2k(qQ{H{A&OZLE{==@-SFNHs&~KTwpU#f>7x2
zQYN3OmhybCU!?^S6Xi?ge)1mLpDwiruI0ud`AW@p6c=7Y1LT+$AXSWLW)lFdT6qr+
zJt+RUw3&>>dj_c~MTpg{$vX#ATcaE^q$)``$GBCMr55JSy9Ql=XDR%W5h(DS^EJ|M
zCTGH_#)0~k!CMvAw1iK>!P)`nMg5~XRFKa_l83|pqU$Z=s@lTsVJs921d#?20qO2i
z8tLwmZb@lW1eBC60Ridm25D(&knZm8_t~Cv?!6!0{|A4^gPXnAUTe-L#u(3-v6a`P
zUXu#jnS4$Ot<-Z|^8VGSs5hTh^%f@fl2&i&v}RwcJnT}GC6X}VwSEvnberBZ;K^lU
z-7TbZ&KlI&H~b7M(GkC3yr6???Q3wd@KQP_0&ky~q(^K9R3##)ogzU4n}elDhomjN
z8VCzSGTsx0^;Aw4<&TddU92~xIG=rV+vzxyZftElH`6h_pDI|rPawy%7aX0{vS;hw
zzfotsxwa&e=AG8Iw!gN|>u9yW_u0jRBt*XO5#f9Qh;Gp?j~62{Y}pg79!rsN3$Fi2
z`2Bo<2&QB;GZ1wXl3>t>&rJ*^P3aWS+*zSC{Ne4!9!ntu2SL5HFU;SngFTtbYB#JcT)GCZB?IaQ
zTo|TA0_Of@MN0*UxBD4`=Y=ITy;O5(zb8ruj%zX)ACq(SW;oH4p_)^?Q_jqpWMZ9b
zTpIh{I48N#U%1EM(I|Fq4ff+*IL~aieoY4O^@h6Gjh4^yR6eA0(?Sqmqdb+q6p!!9
z>dFf((2*_ap0(MHODQ%EihZD!tO1xFRU_`PVR*vs!~=XB*Zg+3P>iXLj$gw_wsRlq
z@~#n(yf1UPo-0*YJLxiX7}9h11o!c|5!N?UvNw4_c$Js$vfZ5uQP}oe87#G#d9iWd
z>_@FWetWUyeKxW?s^eI}B|c6C?bC^C!6Cc*>BX~$?ib(2HHnj24-+Z^>^-#V&&4M6
z*KS-@K6_SWyD3bWq@}61UBZT3(sZuOiRap-;iY!%tb4L$Vqg*N=F7zQmZh?iL~OG5
z;>~?a??3<2cw?lfsK8qoh!=CSYNjawf1Nu#@R$jQ{Vm5mo@BS*i3L~+ue~OERtuHM
zrdp9(ud{GpOS%60?oSVgYlO5vrRUz~9bF@`xM9dcGPc$rGPZqGF`v|P*X#P@tI=Ct
zSxlEq%ufgowXQ|1H&)a$xaQ>=&X4%~?YllpAoLmzX(my8g+tQx$1z2r$2lm9uWuo0
zry>tt-!}29kHAgv=RBEE|MaHKY(j9D?v{AMjb};i`T8R%GDtbAF5eBJxGpO~0w#?*syPI
z9kb50eyXKmi``zalTLv|cU+k$?kD4a@rHf+2fw_Pa;>s5#-PbRC}uDTzyCOV
zR+(}iqk~#8{pPWj#oWJ5yCUkp0PO5v05&W1dQFns?}CCzONvITL++Q!PucDpZv<7c
zXk?DxE+o#C`nk_Lcw}GbG^D}$^qE2e!gRO0|~?r
zS?vdCj}#+HA_e~ipZ)`mUi)5D&QnVZD&Ty4`gV#AfkQmI_rIVI$b^0h>OW|{7FpuH
zq*BEr(AVGp2oLX`m6esFi_2WeuohF623S^1Z)`kZV`H=1nw&^-lMoes38QHRZ<_Cl
zt(7cdP
z?34ht`5&kJE1XJi?mwU&K>>U%BnaG7UEN|^)8&sRO2){Zp+Az+nW`|Va35I-qV;C8=
zt665BZlfAoFJ^K0p(1A;q*-@fWGmp9^<)+*xHRhuz<>zCpn;IbS*OJxXM2DDCK9!3
z;Z44)bEet3xkvc;{&}^hcM((UqeVtDv$G$Ii3
zKuAKfkwQLo8eYfle$I|0T8oMYw|0~kD4Zwx2Svm{uFr2GD`NCimpuygDdX2zqpzsq
zAN&M?Gm^{Gt`OA*Su=qb&li9o3Osh0av?jh-%AMao@>fcewOrpc;isw;(XRfdR-1{McuYk`(
zF=kFT#TsNbt4To#3I9#}=H_N%T3RV(pA6dY@wtIS
zCK>YyybM$nY7o1V2N2vsq@}B#$+%V_&5a-;j|cbZ*M0Am`-vtcWmIf^5wB1%%`h-g
zicyPkt}-zwo9Z$%0{#Bq^(k{Oy{dy9ZiwIXzKS
zPiKC0!j#WZ*51jtJ>Df(qk~8
z-7r=V8yJaM&s<;FO4Y$(ZpOW5fO)lGU3h5b(<0T$56d(2UdfmgK$G$Igs?6RmafkR58fL=t
z%1>zg@K<>bmy?Eu+e6)!0tU1mE}EG3srT7gO=NpplAQS@f3G6fRysLivu(2Tuf+uYo|dHc34-05d|2^_ZG`T6unkbi_<
zzrH)1t4Tsi>IGgg-hWEWr~-RosGn<9T2k`XQ*v^jMjuor_jAXNLJu&&VBJrj57W5e
zS^pjq1q^J>OuU;S1yh+Ctsd|xzzj%)&}w30BD6~NWwu4egy?k^jBS-yRA7F|8(ve)
zP)A(9!T;R-{*3S!!Moj5+JBx)2}wUwh;R&?{dH4IsLp|O=Z{$)gWIXy?Ck9CwF5|B
zpLyL*zsARJ=Cew~b4NpXgJ|`!*H|A0!4)#(yeuawuOXZ~&Mz()ug^4DmEZ5mIdll1G{S6h=cG7y-NMl2Yk&Ht>mjdpA)e%hTLA;mWseffoa=NrbB
z12HkNMvZeg)B%v}O-2P)j(6rD*T#~^JAQ;8dZQ9a!EpS{%#8U|ErExJ2hyK1tDX$`
zG+5O0i;Hq4riw_l=ey!CJZHq>4jvC;l9J8F%x{=+X%^}(PtBmpL5gBFL3gf+2WKI;
z1Fy;~Cq_ywVi$OpW_*YiwXaUG%!abB-9;yY&!AQ>^#?-u&Al=b!X*=i-e)6+UaddIVVbT;$9
z9;uaCN
zKdNJPxEcP2?zA1A7sXr1gIZ#>yqO06l;^N
z2o*eWl&kIh(V8jliEnchRa4hH8lCtuiwn;xRy$%kiFFhrtO)fD3?>d#bQ#-~a&vRD
zRg1i#FxhLT%KHu_S5MDU{cR%smA;hBrhG4wmhaV}4Csh&LG!$q6hhL>?%p2gDne8W
zilFFVa6NvjR%*Vop8EN9Dm9tfYc)drDrH3>S~{_Nf0uk-Hp16xY;0`Pn$Y9?@1Lqx
ziU%SAubjyAo+CoDx@eL(ynSOzN(x66O87^JD=;o<_Hy09+j*O_VKexiO=N20L*
z!|VAr+_eD2y@%f@5<~oAdo%=W2HQo6%kw>Hn5t_>EW?ok5+qcVF2R?)IMXBY)YnZPFx1t{g9TBiJcMr#`)Cp@L#<%W1
zwQu&tyideCZhB>5kqi;3J&eM9y!?@8{Y%)h(F}UXP+Z--OZc$n>hcUKl5a4I*IQ&H
z&3n2`a{a~)MI1fSVc9lzEF+jqtv;G`L4pTnJ^k|e=A(
z{KR(^GwFGIV>YFL16jq%5pCWz^E=x4Wp`K=b>wfD2cnjjk0?z(x=XmqMNjxp>RD75
zd!c9GW9bbU(OC;kuVU=%C;CCcCMLlw@
z_NT31jgCf+26LQSx7G`>t6~zcK9n^hR@1=KFSNv;UWh>2Y6p*-(R|$-@E8rCy}FOb
zif|Tzk~1fQ@7*Ro68MW5f@c|&u6~dU)FMy{99&^_W~-M4g4YeK!R8QB@#V2HI%{j|
z@~SHJStzMJQF)Kw3;8|SS&X$}blx`x6PwOrll~}Hix>mx^sulnbUICAGdvO!uueE-
zhTemep8ji86gK#>NhI*PZEtVuw1tqGkK})VkANo<4ZbGO4nbJEky(tItu2x==SpW)
zbMOT52Z_aWpnUEao_vMt38T+Lg6ld^jr*RG35taus47~SS4$CC31qBc@head2dM2=
ztW;kq5X7!!{$kpu%p^i3+Z}43x|3tOS7cH}#^P8y51`7vp#v5fuKT?rh}Hroxu
z+pZ2|JX>0|W``?Hkk5WWpkX(ViHky~^_<^@Wn>5Y|IKu8jmy`OL4L16Z}A3_6HNK8yS
z{D=MRG}2b}-b2VP3%?Tu9336UZO<*;CWaVjH#s95zJ){CIr3GM@cf>LjYHw<5KaC&
zF2Oe=|9&P~cUm`X72F*BB(YGkZydI%zsxc+G9v2E&dx$3A`DN;>c4MJGC}u0x7?f5
zA-O(QCOwEP;I&=6583=Zr=Ix+=O0^OoH9IPW7UI?oolqh
zE4|9G)QQ?NWwi+J-yob
zi;R1po$jv)I6FIkS?yi11CNHxKddPh>l0OMOh(<$p;QLj4fo3vqkZQZgTsOOwosbZ
z0AUg^DM)!8PHDF`^gzbQKSxufq~*jCBshRL=s>7{PEH;g7S;sY$|6_!j#)c@A>^q6
znJ7CO{p$8p!iarkvMu-8p!+qwqmqw6Y4&s2libCJqY4LZH{}b}N*Ty%!2ogWwUH+v
z%$1-aLb!8jyB%PmXL(Ajh;MsxzKYuwmg+xb3`{Q%;72X~I
z+qn$Qs?xzZY3`ky@MB0v-7ICVbRWm$pbkz_c!QAo{^A5116MI>GdL~
z!KXMylmGY7F474Z(^=2*sE2USjRe_y#O+_!kZGbh$ewt3L5H>|}t%=u#K
z7yP+hc{Er08-fDP@^;RqYrypEj;!^!uH1&@gqXx@HvVL$iq>;8O75QqD~|}{55FsY
zG&_vQGLPoMCX!~uR9nCokK?^+2USsBWAb(g@Fk;J-_{xC3
z5SJYJl-suL=PW_w$$Dg#zW!2wUJ6kjpPC=-(aFfbLt??^xlD#++kSdCnWsE4r*Ao!
zf)uOzR1W7byVDl_@YJq1LY#p`C%E#f&kO$B?w=|d8xkIFKdk2^j-=$kD4bNy6x+p0
z!}-9O3*ev?}b-hX}oNcfa<1pJi9UT(NBD3?jA8EuN5j5a`A4mZ8}zOK{P&h
zWMjxO6?0zJTw+A+z_$Dqt6BubV}1oY4sUMs9<4;7;u&Qe|
zs;DKM$n~C6UUwm@%3awz*)&G>!6>?BJn=Ks{pd`#OC&438Z9Y=$#Z2(@;uYAVpq_I
z(hhY`_pCr-HpJ@fvjyZ0T&G_0aV%}=F)UF;G)pQhcVvmJ&?a1{vqF9AAP?VI;bqA3
z-?}EFI2@
zZP91#H)-&|j5NV|($t_#NNuwB32ou?Uu;6C!ilW30ljVUY6j6S9*^0Np1v(%yV;E({mt
zIl=Y^wOkQx-59FtT%lc_U9QWISG|;G3e&xqS_DT!BCI`av6vKdiZrVnG^!ngtE;&I
z31}^wa?dio@Pddb0tSIj;6j7LVe=!?o+}fT|HQttyE_%yZg6vc$j8TbyHw`Rfk{cO
zqjT8$j-6rYqwj(X*U#-peW9T9_*+s?GA=F@InK?Xhg;kC<}az-^^ww&(^zbq)J5F)
zFf*?&seNQsyOQ}wc2D>Sd!CVyC#=@0#T##lbRft7O8V4od-w*KAT8HuGfIkkxGcIT
zRu5~%nS>|JYJT6#4oO(#Q-R|fRG%hG#zR}Vyw$cYrrrz_G`=o^^{?eg>4x@pp9Y7l
z>4{^BW(!B0**$s45B4oq#AA`;$JZ<*4XqV<>#i=jkk*Ftj39qG_+g--{RA}AWJu!T
zNK@qC)yNU2T4+LoG{D=>oyhQWz3a%i_*|0&4gLhE`;~R)G%WzI6#LJ^o8)A3r=Ic2
z;VGanAF=m633dLHCJPvja5&gi;IKRJ`Vh1higvU&DlaumJ18!r&3a`uj6P1X+m$pd
zzQH$pdTM>_QZ;q(&9s)&`yFbELB1PnjB+2c%b5Gq;P07Q_*&czGX>JL9=PvI_~P`N
z12G;3(OKH`zN`|~0*jtDtF*$C6~QOk8#Uc@9?MSYDRjCeR^62Lb|1yjO)-aS-OjK%
zY@S2DHJ_*)+^tMV083YotEHw06c+Y9POTc}K?-XK!l$o7zGCt}HaOd?fX#@=u5x08
zE%CMbIy*1#7*N
z@kb<_oH6k{E^^>=3E&4*JTHUiBI&djA$2}yXODvNaXZN4{6=zbx*E||-IY7^)FSZv
z0IJ^uDB}lsEZeR0H8UaurkCt?b~yfS%8mPEKJ*)($E4h(G4t+fR$d<@*^i!{PGFck
zTA=4M%iucugdhEv~(rxT&Yo07axq8CF$a*dQ#PHGZ){lsYwCBr9
zjv>h!0eU47MgIYep#UqdN0`E@6zZeEr6k+04Iwd`4R!WKKtD&Z>50bA{!uGWB)Jy>
zf~}{9aX}(YG3<%~)Fg9cuYC_&{BOeyFAE5{U6?U!qgt^>+2K*y?zQG*SJ4xT<7wJg
zRe@@m`_Neb)^=0X6oXZxZVS!%7hDph-!`)zCmghcHs^f@=ExUyISOuXI6g?{=r_!2
zV(;x<`#f%Ub1G4m*?E%*X_b2qrB)z$n*g`DGC}$p{aRg}x>oHae({cU|W3(y@aNFcT0S@sCQr6pO{TsmMFmKMB>~S>y>m)(ke#B
z%IPX{=_a3Y603qz{3b28YaQJczKZ7|vDUXLbNv-mlk9+s?gfd3v-}9}=jH8r05$J!
zluZ=^B?Ca@9aL1K?Avm+1;fK~N4E@63wf*&y;lj%wu-3DEA
zv$Ij`*3S_3U(i9k*~_nw-Ff$+K*;P#3SH}OfL29J3RjT1e^mQ=IXFu;D1PA$sZp(g57aF
zKyQ{}3aWSl>SMQuT6@)qvI`strd;+El
zggUcLUyvleu?NFoHUzr!(psNOEIF^vbq)Wt=|LQN7o_C9(TXQ(ahe{#C
z+T=&?BlC5G-&)+%oh2-h)n9rkjPT7U6NEhq=nO2g(8^3!IpyBE*7+h%BNzj2@
zd!WxX>0C&F#3GzsPIB0H)m*r_Brn+G9;S9pi+U7Se2T4()N(Ym+{{X=hv0V4Uj0Ze
zbkN!@4E;UG)O#yN<)3;3DPasoZNX3x;3P44q{K$6pnO_-xvh
qo|VVBfhwMogcW*2k!ldF
za9B(gf``HSy`j*cBV9i2Arb-yLtwK?i^Qy(h!9}sU%KuebQuZ!MlxyB!h=Pm?wj@|jQa|1@sE!C>fK?r*pTFn&5y7WN
zWU7_P8Zq>nmbmkB7tRfEL3)36;T|}dvpSPQwH#ZMcTHt$?xZ8S_-@9l#-AVE=$@$!
zk|rr?DvrrSXtEf${c2mfD!nwgn6Ii3uWcJt6ws=Tb~CbJ|K{jPo2UgjPk702>@1uI
zU1N#PfPO!RX5i#(DVdu7k_XRPyi*l-@UYA*+wj5lb+xMv1Wg9`78DtU2%5}zApMcY
zokc^32{!v2*0YH2?ng>W
z_zPjN@k<(;Yj+TA5)!wAxt$Zh`-f?5R&J|^IrlaiAWuLk#aI=Oc<
zyCve0vt0n|q|h(!*;yZKNTmiqY|(badTUGcdc
zzXI_Z1%SED?z~v-*(MhPnJV!pGVGeIIhwrjl#0p^P}%LDH&9gq)$!(b1^@OXpU^1#
zjqgU>F=MmLDTYo}m0zqmpl~IjbGrd9wPn
z0L4UKAFm=LSfDlksp2f|-mX}ARDIM5->gGN*q3=K)o-tB9*#>qTd|C0Q7G~yM_L&b
z9>mbh5ikv_3`iLSrDA+ER#&sx=uO3gHyyRM_czera9H-3A+?{fQ$F)|XR0GzY`XP_
z4duu1h5gDz<|F|h7iRdqTc~Y0HN%f}Nr%JY^(c3@8o#g|Q)p@LwuJY2=D8{+?PZ<0
z?DmG;;VMa;pT+2RyqM#mjvk7QARgSnH8blpapwrJ7{)H6q*YZtKF!x{Jia)Z0yrfN
zG6BG+-r%B*-kiS}Z4{pfLIKzFomSx069E54;02U{vgl>q&u@T3W@}@kCuknPHkK6N
zDRDRh36Xygxd-m_To>DvJ`j-ztd^j0>`E)-kb)u)R^AH@!W}K9VQ~y?!}@!~_zQpw
z`xElS0JHiVPR{^|MF(7wsHo`A!NG7+@d)@IWLQ)v_wVaNV7iY@|C6!E5uiD6?23jB
zbZxw#ZU8c^Du^LK*)ou+=m)>_yUa=(^84F&%B80QuyE3(6AXa~yornqi>nb3o0a}F
zRG@IqP18D9iG1XYpYm+
zQI9^OD&NWz+sk)xG`6?1;$JxY=vd)E8pvK`5GE3-%c1V
zrJ{lj9Dq;=^KDeZV%M>89eQxHZ-)GZ&1{U8S9i%tqgFf(jUH5IjjM26@`?4qYS+c1
zdq;HkgJhng}
zSZ~}C&wGf&*tFXWu=W##yJ*MR@c#j}0eJoY!?MLMz|CLC-A5AW?I7OM&tUfd0XTIk
z)47wqC6mF-@RSG8BM1tPpZE2^CIIpwp0u|FmOH1zx+th7FvH7I%XvYlhCxUu1yD6q
zs|Z}}{d*+H==I>#iP>5n36|+udK&TUx|pVaF)`Ukqm}+c8w**E6^z*en5o1g4Bf|5X&wXjfzADd8rOD@E;4g~=G6Xdm(K6L~lpC~Fj`fr%7M)&$
zQv*_dnVHgc7*UB7TN_8BLBXW6)T__QjPk?x-K#e$IcC#a8?V$_9(2?_s1Um1Di{(G
zu;hhC+1n0fX2OW;(O&)BvekUSl|Dt}Zr1lsLngA&mE0{34}cX%O;#HZ
zHv)O0%$Dy%wJB0Im@OEGie=uYIk?KU#+`7c2jWx&T8;Aey~gv6sC-RRQwbnHxR!6b
zBz1m%4s7{fT>yUdcEdej|6?HK=`rIXnm%Co$-q(;DNafRdjAvPpXqS}zVW)nBzRn^
z9=Ja-0)pfGZ0d>(BrEId=7>{C5GoI0;d=i$5Im{3yZ|&Gm?AJ!ra^t26QFx`xN)-E
zK_@05F%RJb8r9S$i(=Nmp;MiEFp&-i`Z&%7u!%}ak^-2FN#(ogl_Bhkii+y$x?W|T
z;pf*@M~u*(o*st
zcHM@Z-hIPAG6$rp|$vWf2kpRsBh-F&&ZOIh57O6{jODp~R**S@a0q7(EGi48kt
z%ad?VlpvCsy&ZQRs>l0JemLvgG5$RksL6Jlf8=^roi=t@CA2XGj0rqG-ndXXaO1Xe
zuZ;Wwr?O0`JH?e2*UMT^OINlb9a{xZy8~%}53v00$Ip3yihd0V29WsF4h7^&dI~x1pd+m+r1ySS*+#$A9
z9p2+N90U?Tdu-kp{%>Sn}sO|N?$0o28^St)7MvJai+
z15qn?B#vUHr!i^Vx!a}+y{M_Q`CUARxHp$S;{U<-LHrpS9;&_)L2Ir#98Oj;PBX#78K$xR6p+@hx6p*sw%}zLx98)Z}YeuB_eF73-~LS#MMb
z2&65w>a#zlx1lm-tLqthIK6kmyBpCX+QT}XV=J&W+mOE*@FhkepL;5EOM1J$h2;5V
zxR|3S5-;{xIBl1y>Z&`yC%X1oyNjPok>+DNjc>aP?#3_-WGq^gJQ(`KeT1XT^gw?R
zySKuO=1hS=`|xqeRc>85YiYtRW|~|d!~9}stX1LjL}(O)+))RVU1_|#C%emBFv{}}
zIWo^CG<+r6Sd4iu@5%M=1Kh)~IAPv6Y{e@fm{kdSndp2+EIhsL-#;uBwTKRpV9ac;
zWf8^kyyi;P+O67-Cl)8+yv1qEiKTL(fe0rD)f)JglRq
zi^93l39|=aei%FM%2bk7x${<>2={gbFGr@*$E;`6nvV1P1}(KmJnDU7PS
zako^^yGKD-*$z#P3CR0o@^#OldV>;7`*`Z=a%p}D#pU<>{4|R@_ojNO4nFheb%PFi
z7w3rRxze}x_B67)lAm2nU=UIgPid}*jnD>weoVUR?dwp?2}LxBOG>7|;Y}~=`|<%3
zVSue%h;IcQSC(>KDzJGVMYdl4O$_ANwPPq{@7%vX51Z!r?5samlWWp(?g8WgbBa2v
zDPB+$e=iR}_PmA~6sv95*3pp+I3EI@EW9}bB}O}h;3!zPP=`R?sT5qeceG4tew5?8
z)(G_iLitg5afk;d{yLx#0gYE2;7)}QuhDztazOz;<#rTjU@``ve3o9ef~sRF1=GLs
z#msevtTm+>dwb@ry!|)LR5hFI
zPkJ+HcVtQYs-INtw{dm5Y*lnuA89OcX?fTRs*8VI)rwym^Itk;ty3#(csN|cb51E_iO@1PI|jEtGlT8uR=L}K
zzcoD_Ogi4^Ojqd-?bBLfY4FhN;@TI}9C_Gmx)k=TQ>4pLY1Y>A7bZjut-tUC!Qa%>
z^me6>v}3EeVN}?wgO_6r9?Gkw?sp!1@o@8iEO&V8*G_1r$80n!DDmpm^y=M{6UVyk
z>xO)Mxe2@h!~QsqLHJ@rrw3A8n*r)4qt6&@qnUVw2^JgzPgs4}EL#Bp|S
zH^(`kEQsQEVuT2-4X1h{<@z9$M1dbHI8C3dI##6sAl(EJpkDk|)S}v@vz6RzMYR&4)(>0)CLAkXul!FK9=tEKL?VoN(6CyS?
zyP82JCA;M)*+IdZL{~3EBSnD1b?l0|#=)`7cXnSxvzJcOw6FY2LbT5LB`b=^7yArOrcQ0hDJq+g6a=WXQAtr_yHR$
z=6?i)nWUo%tPwu>p})YAxpuM(Fbz+S(iTK@C$$dDiD8liyvIQ}C
z!p|ozC;6P$EbkjHNE`pIIa1Fs7sqn2(DG9zteF~>EVvgsDiKJ$mD@UAOj~ZINoD7n
z7znPqX{g!m!*k?v_t!sn&F1-kG?1@i+euZ&pzA2iNYU-N8A+|F!l;~UWX(Yn2U@AB
zmPfNn-CDlCJwsGKTniV*Xq5yM}xcHJ*YfZ6=>>Eod@%4
zKl0sq+iSc%#Ic3T#Z{Z1XB!z{&KJ>qQmmn&$!ur3E|j-8dF&YG&c`2JuXwo5V;!s=
z_?%&`O$oAVx?pE~{4(D~n9Ih)rS&;rxwHuQ+{X*I1~yV9q8k8>!W=fB$3{#mfEYDP
ztxMi3#x48Ke=B~y;gLA@WUg-{RZ!VcntBx56hYTK(3Mh}$9_r=xL4lQ5e>S|&*h=<
zM~kmGxnjjO6J;PX|GYS`-$yx0j0v(}+JD~mw=C7w2BrjxoVDE6yGPXuNwDJ)T|ek#
z(Vsj?RoYWXR3f6fThWj8m&i6K5FAJ2%i^6p^XVoMV*3wyk)=Y~lC$w3eG!*7HA$?jR!G
z8ARU-<@{$xsedu8hTH*ByK9-xI}p)FlY|cG@^+Ad)Tk|jh7
z;q&$o-Mcgbi~qAr@c#;${_8LBVJ4<WtB*~g*
zd7542^~zOJRHkeFZZXqyD@yY$>Tk5GW4hQ}y8rMv?0H|aGgBQ~DX6Jt<{qu`PKtsj
zFDUj`hYHsUSBRmAz`I9YqibNwkCDA9zMt_$DvfHlGOvtW@uGTDb~Tl-DUSU^7^G0_f+Y6}*yv&g@ori02?H~4q
znIM06;yiIYdb>bff8a63G{X39n}CS{R9Drpi@fnX$(6H-Y$FsOQ#(DB3h9;(C)M^{
zbEs$+U76o?l}xJrIo*=6_sELO>3L`9Q`X9`V|nb1UPvB6C*5f;%AF|jf9o5846t&_
zuL$&bfYK19T#3i17-G-LEtt>aK64dDV<>+j_AHE@aDhrpZufoQ#O)q)*0F_afj)
zBBfHRtflUx&OdlS!_J2rb%nK<@48P|
z1KJKkqLXH&{^K(q8Zkz{wWhc*r(_8+sO21%0}p#s57ifWEuHKoVP(;b1}sVozDrRF
z6TZCh5GvzW7_C%zMD&ynoqC3z2eg>$g`lYO4v*rFRiISHqKK_^=uT3d*5t+|HHEZ%
z%4v&iu+1h<2qUWsdmb^6AtH8lwHbH7g#X(^)9_(;y#0%IGK44)vBT2d9=fY9)KnLR
zdYPs#i`^7yWs1(`W0w&ZBI@uZgbhiFOsefFVW_ZZ45OsxsNUUVRhK_)^Md
zl&%&zVDM%@OLjrbcUgs?kG6MviX&9y@+N4NFvM|-?k!g3-BA5OMrwgfy0<*AQioO+
z`ftd4d)J8Ex1HZ9-wj3BD58wiQb_8VD7U@Xb@R7piak4xEpcnyxTAQKo%-C8@>m#B>p$!pI3l7UFmMbYx&})
z*xe&NtRlY$Vzdu%HOxgBO)j+-^TU{a|9)IN<-N<=%vKdyRfhG9
zghXrdbBG1p{pSgWs46a0Rfn7!3%zNZ_Ccy)xOlSSDIIat3Pka`sHRT%nd3z-jt$}?1VJ)DY~NB`Hx*i-_4`1Ee^Pq
z=bL%4r{VysM3+sr4UrDm3cubj`h!d
zQye*!SLbi-=DP7csX*ZN1zOR
zuWw#|BM>5<+|KBgA}u#s+mw7@8%BLAecrl)Qye2xHL`HwdiX%H)D>o{ygjv$0oi|d
zk8(h?nB7H5!6`v;A)6t6xya_Z)`>sq4g^CR*~0bjh0(-bOX>NYiAk&(1rpzxA}Iz&
z@^35a+SwcCQ;5B!Oc+(HHDtjIOr*4r*32gJ!t4^3w?Gkz9wg2FZ$HUMs*18w&{(xI
zg?NnBL?Z`uQKcZX`yc%LEc|HM3tfdg(zlKsS39%i34b3(N0HuD{{70QnQFzKlIeXG
zo@H*xIwnjby&ff0C(HVRb8xsSSLmU#5`N`Y{+j5@Qk*$6Tt%)>IlM0gX;97icd&~;
zg=T|ni2)&IoE}dB8714_h!ScsY3p4*mQZ{bB*%*J=$d&Rao!2)S{ZsM#FP>K;~&T7
zMuUc5$X57PZ$Z=z<9m%RzKH(K$fZ!cD_Yx&62^<0Nskp`iBpV~LOI*W7sN=jz-y%q6R`
z1C}X8bCH>l+Itt0G2o)Bimt)Bo?;+yy|H;+Zq@)doIE1zoBU;n;X~={
z3}y`3nl&;?l7F5v^`v9@FZcDaBeJ)YC(apbwgZwWeX%WlFR0jB!at0ID*P>|!U-RO
z4*OZhy`&XY*9jE=E*o&J$xiR8kjkdO0zz7Pyw&$BZY{80M4)x4w9CH~gBQu*YpUIHGM
zLTkTckcy2J2;RdJq_vL=tjs80lGIbJV9140IBoWiP;2deRk@=}72Qnwnv#a5Yg7^q
zG4G9-)&Ws6^ebT3|E~=q_740IgC~LPL95<*>o-AThc-3ynWO$$c=(
ziUhQ%XI>s1EHD_Gk+R0K#Kgc5fH`|O5V|x>J74D-2m}$U?Ug68HDD!*e-a`iMP9x{
z0{PL4U1vZ@pr@-@n1U%P#2JVRIcniQf4(G>hywaW;qi@W7%e2Dq|6j`!v$ol{(E}R
zP-hHF2jeDC7oN9KpKt)f&<%8Lx>^Y(f`5S_Y3qrG%FWQBot9P?{3p^SO(99Dnyc%B
zDW-2EOH}o}%qM*A*&ohO&M{Xo
z1$wv_LWzPZ!Y}n4hK{Ig7AVWaR4gqjv+?(WOaNoDeniT{J
zfN^~P{xnl5=QY6Zqm}kqvkb)!bCx-7uUlj-FNK`spP*|R7Da#R5a#u*ZHVC+X1DJZ
zmBGyUyA%lHvQV%de5*8`@8`kEJ|*>>&vJ4IP=#HK1ezZMU?yEbW_S*B>S7_fD_;uH-N^5kpyc{K{Dz5L=ry%5g1}h?N=~~(~K7uLM|4>y?X~R6J@(`wF@tY
z^^tl%x>`|S7T{=59e8iwQ#Ok~%wReJu3YkRa=$t|pZ^25M5aGL0wQY;xPt{?0g7CA
z%>Q@9N={xL!HPqZO|$D%S-df9TEe~iiv(0_qq}NqYPi7bBdi`GhM&-Ch=O_;F&qoZ
z3w&PJDBuF9d=o!~@IG0}DVr?xcLBZQ7CSF!*g@#onJC}`qth0_J*N#0APWag7zcH{
z^A}QElS<-aXr@Lkw-mOY=0>T^`VNSevm03=wL*zp(P}0adi2AsLN&DFQmQ
z+xQa>s~R!b{qr=y_J2hLFzvm&<#81Ypp+WABB>L*Y|P<@1M9F@4_U&|Q>0;i%2DsJ
zvdXad*42c(g7sGBkle+-)I{
PLVITz-pp=e37J8xNLR&r-jK2tL1EJ!shXz)&k zcSoKvigmP?X@OmIsarXMF#-;18D=oFw6tJED*^_$=T=wm{sa~p0TG#u-@$M z3iFBiM$+}`U*QOMh}A-Xslnjy&6_uWCkfpH#x!NQyyW!W0qDU1xSZbF>Vcz3>mjoI zYL{E8ujymO=d}Bb*e768f|xmW;$dVALCn@UAFL)0nnnYl3VR&*>sR52s05tFqDgBW zS7$^J&|1dM79;lzd*ZwPo*TbI3^*S=Z^UhmbP(PA{v#+QJRzY6OWPHu=I_CYp=1ED zLN3hsZ8{@i+GQ$dp%j@M$#1tksq@Zemsi7pzF_{dsw63A8a3IF`Es30=?AqPvGY2H zxb40FkF&Q9in5LWg~0%o6a}P31f)SqT2WCzK)Sm-mQIyYQo2F9ySqCU>26qZ>0Xv| z?eqMeGv~~lnRn*>!xmFfcs7)2j^J0*hw0n;c5FX?{<^3_Vf@*ybrs3>a0|4F7Ll?v6T+Dqc~ zFj72-q#DxIQF!OvxMb;?3t6MH)TBn)FjCClE})c}SDknfIvj7wb+#gj6^#4|JE)b?aGVtzO-X2=@Hi6$8T zX86*S6Q#mPQyvc&g!YjH%oG}gve4$31wgZ#e?Qv0)PH~@hW=H z?FqYL8B`_hTfy`TAAH=j=CJ-eOIwBW^R?UsT_+*>7Uk|9VO{1$f_~oSL?e^XuZnu= z^`&rNL)Wac*9FEYjas|=z_S2ISybf5({4btuTj8=g&ST)#>VylUmNiJ(gJK6OqAhL z{ZF&}PI!PcU_IJbw@#c#8U7YsR|U<+UV+j7>B|@Fz1iw)Abmo`h9pU6n_yd-YNxJr z$%<21Q1jllB?Ib!RB9`RMpjMBn!S`s3%W4Mw-z@rmcP;Up2ginVRZgzN&W$KWRA8} zKb8>Ua`i{p=IS&e*~U9v9D|>G;xEhC6TIs1#%#qZ5{+smxJN@xsjU8hgnBbebG5{* zA6SD?2>OBxi~@zY<|*c*p$;U@9{mSb7uK1GthfR35`w^(+JTNr66|rRVABVc0;6oATkw0+D-{CYhkV?{`LD9~x{ zwST@&UwhY-=FFI+KN4Fo6d>m|b_n@T3m_LdqN+*S`7pLVrqc3Po4}Koq+Ci*JSB!| zr?Xdvn0D*;4!Mki$4OkwPNh_Dv4$#Uc*_IFGjh_e_Ol#x4N|fMM`(SZ&7rVBD&&%O zzTSl(e{zg}MVkEa%sDd?BW3L@0C6!@%UADBmSZ=tm)uXLJVz$C0(g#KDPO;HLd6by zdlxHpt?LRn{jz{)VysX-251F=GFzp}LiLxPO1&x2{;`8P8s#nq!6E?N!Cp^#MO|36 z`3$^VaxyaW%k;?|m-Q%RG?Z~1xExUmFR&JX%npj0KtY2C^>7rwMdKpfmU zs@V<~vHLk^8yq{z+oOE2?(_+|#2BPJ!DTw)1BjESE-c*Kgs(1wo{*EfcORmt6e7oS zwx3ZxjXTQaIN-eFG;o+PDm?-0SVXE4B|v=(==zG!mtbJz&J4A!V>!X+#DN7lM=mp9 zT}4_N*B_6{2oNJ&7E}1&c6WCP#`pL4Wz^K-*B5;yeTd79ss6KHn*aGHw&P9A<^HX_ zUDi0~)4?t|n~=O~e+IsM3zlVPUea<)@uC%@-+cBwN+j{)_U4)suT)>i6`s9uy=B*C zbD|S;6iF}=X}&${d%pvC{XS6bd21CL-y&}Ux`tj@_QEy9d|t*aONdT8wcd0PZ{e({T*{)bHj71@2@JDsbAOiSqvt_6hqm`J)n!UVGSunN*run*wdITl z^1;J<;RE6uLth>=R~OI2rUzS33gaaHq8+QAbh;+Q!jYhH z8%8<$(m(M_8L=}ssMB*5I``7B!<2%kvUK4w%|GLcp@_lIy<0P z03<35$n;;0lT*5^JqG#&)Q*l;1FR^{z#08a*!>L<6S^M{@*;rz$H&)~gy$BlkFICC zy+A?@1*twB?W@{eH{3vbK!M_lfx8%FAS8iZpTnRB9}T2eiGivSn3{CK;J%zh2^Y{% zMJFoIsm{S1rH)JBv3zv059b3%vN3oKpeiH~W&v)EM8J_54W$DI?IrNP*n&&^1w-74 zHRhFmP`=q~#y_uLvmdQA;1q0+*pJ)f4yPAS`(v#vhQLA?oSx3A845F|=M6{|O0_yd z_is3hwOu2c4VyaknYtK5vnRw1Ls|BfEL|y1^*MI9|9l)W}LCd$* zKdYM-25Kc$X}2Zk1i2auJm*n?A-`6QM@Y`DUc`?T4#3z>S7)H548*UWos0Yn_YoT6Vl}j*yE}b{A7!xz zN_2V_mSDhe9~OJv1fq-^CFYZ?K*bcJ?YjnpeFAeGBuO!#8ZqD}Lxp<0h5Jna$=}80 zB`QA;_rMexKAO$diK&+B z7A=p}+-DHl1=w)JX|dPfEl+;Fd2B>PJLoM?`UMUS4wPXHlnoI{Nm6g$ej7n36zSW- zmICr&9NgSdpQt<{37M3Jfjdt4V(D(3{g$YiSzaoyY+wBa3{czsT^lk~C#MPxMkfi! zrmE^RS$>^{>Q3i<{H{Yo(0Wz~q5-E|JYB#2xof8%KEa=f;w)2>_^!akB)y#_DXYOw zd2(rr=I>My{N~uY)V|uIk|au6H)c<0uDO=0#Kt{dV=hxV6&?Dp@z{ZDMcU2_aS7~S zTAjXY(rAiFp<7rVb$My%cepuY)G#cfaT$AJLClK6BchfM~iw}6p!dkPsDpwp^u9ukOJpu3in)!t@lSiGoSEmbHv z7`;ddAeU6m`8>)UmB~st8)=sd4z{n#nor-0qfBvQ>ogq;k7I;|7U2iDo*{~}9g(%L z=rKA{k%S~Fuj~AN{iA}#sDUxQ**EsDquvHm;hmVa{hZa$ea4((8&qaBDQ;BE-?Mfo zjcU$s)SbngiPL4b$G zC>KC;T7a76kq5$;SW9QUFUlT1NAeA%VA07W52V~8P+2tXOvMdt;Gf*?Ghmf40M3k$ z!jx20PykY4AuIqK(g$1WJC#Hd8lenultOjG-N2LymLsvS?z%cZbaZrgp!&E>@O$67 zO&E3geP>5yj<05A+oeDumjFmLSPwWarGwbx`;t9<`J&8VG4mwkpG(tBhwHSSd#p0z zQ?8he+UR$kXC0ds<-4?tGE?I4DQrBSqnUg&nHdz$cB-zVfX!v=w1Z#Ag>9fJO6L|) zh%l}U&Z=ZSF(v)5jt^x{TiK(1{1JXVR#Q_WO63Nxg@Jv%EXG*jHSaYd)jiSsKu~j3 z6GM_;Iwm!&i0`kx@;--yhVMK6Iz8Xt93Htz%@QJAiMOY#_mv+qR#}QOIavv6K}d#8 z>ay@pOlpeKMkR)aZum@NHT=XRV8z2-J;%J;e6k5n%XM6#8S)xZo-_77GO&GvuJHK* zO~~&sCoX*zXqO5?^2|jpQtF@m4s%b69~!V2u{k0bUd8d4mVY9{R8S~`5bA!xlrk{e z+R;76=qS;?Y*S)1NGc~McVvwQQzTO?R2$b{jrMUbagZtst5IE>PgKN7U^RN968*b% zi0OKXt4KFh`}olZznCV8)(GC8`))hN{2d=`mJApnyf5X-?Hu_ zD@U^P@n8;aD{f@!T9QmPc_R6c`e#%K80;Ra)=;YPv6?$V0q+IlIy}j~+Z?MVTmEhr z?$7?_4Q@(A;_l_@g9G0BMD1i+^$6$Fd@1rbaS|3uHd#SC(%p32k;M>Mg`(1YNW+3( zWA-ya+lL}wHf2ILDm8|RYZlQJ;lj`;_%$zXcE}*S18yUCxG-nU9U5faHb(DOgAjVf zWUrZR_p|HZj5@Ifzokq_^<`p38SiwT<>}o`$|s*Y->%+adsypE!^0B|gl5E7Dht;e zWxGZxB9WmG{A6Q;5(9;cwXsB2rb2uB__@E4o=-5pR?2b@?FtP3Fy64SfDrt?ZmPY; zYBaeu`ShdBo3G&t|F|2vezawuR`&@4bMvmTUm>WR88d=UQEgr7vE_9szyi@nA7W#z zg?M4V+*_l4F*|0oFg1PRTuQ8{pLe+a4a!$3L?Si1c_r#(I_TCOcKM@J? zq|7hwcV)zFAz(ulE)6)S@6&XM?37`lTeV-l7MtAC@Lz`bbDB>vhbz&inv(9Nm_jXA zt!Q%W@acI|40+}pzYY4iBNysj9QBGa-Az-$y#uBMF+GGW>h%XKHSB6<>BMO4>I2wU zOIj-!)h*;p-CIHo1y|R0XRNFjnR@L%xaWO}_yR1`N&UX}g~w*zmZgc0)|AWNy{gLZ z#sepi_NNeB*i2BlH|vS$@0+!Lla`I`yNIFwcxVs`2;PG~4CU47sfS}r6Pf%?^5vhc z0)H*V6pHVPIR0Il)->cvh~tCMfo!}qP+OoP4#~*+fe92;j52~eR^n?OB`mul0CQXP z5Ndq=4$S+woVkp_0oyX?ZbLe@#+DF$zOhH=tEH4t&!)M9NOj>JJ42(qzhKt<1MeG2 zAqVA;gSN@~TPBIaqD}8FrC$ovHcDxRc%44TzZLee{AC;`=!A*pMLHL7GOf>dV`(+M z(HockIYlTM{QvlF)+D7#%x!sI>UZ9to8A>o_vVe~{dDtEN0aHBvvs)D&1Go$jZ6N+ znax!CyWie~3_KI}k9{E4)cnN-w}|{k@8wo`_sEO9N97X~Y3Ake2w}r#l8X(JrXf@V z(?h9ik{0oTk!#jsht;=REyj}PzxDe)5ZZxG;Fz5k;4*`U#gwUr-g$GElOOw_q4}5S zyKme>!uHu#Q}pX?H=09VZ-f|OyImWTOE-R)tYv4Mta%Bmak({xBWLxRt`?h4Otd6j zJU`yxK69YZ9He%bcs61jy53$o4{lx>$sK!TfY)-m|Gk*73Qu{j&ogLSE$ntn;rn)~ zQ5AXba{9Vy!Cb<}kog;-M~$`nF|T?IUOy%vklXZPjxK9H`8z1=^@O6Tjyh)}KS=KhcG6%` z4T?0mMm*RM;!fCGM7I9Eo$H=F_v~EoeaHo;inw;RxObeA*dJnD&2b^}{_<*pZMxb< z2D!S|?T|73`wei-CwYyzB}7tWJ3BAa@FuFKdj?Nk?H9TaF9}Io2oL>c^J7f9_3Pbn z$zOT75O{%!Mm6S|Qq3s6QWy|1b#r5dX61IoqqiSyXtWsT{uOpt&N1U)c*XzGUa^wD z+MLL(=Z3IgOvZSv?hmInrMf?Pm7QmtKZqC_l_H%8w_%*m{eE@(1s?wQA!YSv6M2Etr#r z6~5Ry(wHP*J5-4bk+IKOF9BApTZV{j1eJgEKJ0Hw*G7tdHOx&eQWA(-^+0N z_U1$VUy4JlS1LPc46FjVRo&6sc5y{l9A>u_E2@YAgv_GKMuQ7+I!1>rhfqSZM`;-5 zqDos`68$_}`yxaE)pq(@EIW=h;T<*n>J%lZ={1dT>PVNT#V2nii!eS*6&(l3n!R!Hs*Gx!co8cZvQ*vnGPWsbgJv zMQEX!FRSVe zs=L+GjPI{vRF(hzvE8E2LrT3-B%$z=-R9#bAL?_^252w(autA((uxi%8nDAjDcfBew-!r1Gjr?k>Mx5O@Y;GCe!KQ6*428B z{amVA=MzCv(q?Hj>3Z3{+(p_)-y(hFGg2g0?L{ADKauNc_}#jt$uMT28K_Sd@upFL z_WW?ZHq=1l22$ab~zam*9874254S;P9-$(;6b3Kc=P+5L$b-pP@TzdU1mYBBmdMMJ4Z3Gsq8 zo9~E(n|^m>DpzA}o@7f8kZ|}{teSos+6d+Oi_AUa+8q7nEs1-_##!}6_XO;$@~n{C z{>`N^Ga<7In>tp~Xr-d2+s4b`)pc;@p&K_^@O}`r?Pcm5q}k4_UrRIIYj@hiSUk2o zaE4+#t?KFXnk$dpD-Ar)XO}E7Auz4@%SqhjN1xTC7uD#-2P+g4LFUZfS)5s~{c`OErcZobKU4*%-D zrPMKop}qr6s`L-kK82>~&j6{?J>=v#Dty;Qao8S}r}1RiafYE9UrcmVGkJ1-fg|Q8xshkwF87g>bK)Yl@Yk3T^5659 zMVGkE|0)+p)@!!`0q62N<(*A4+T_*tL#fU4?K?cXd>^l4AGlT*NTnz_JgA5*olYW` zuKm#7Q-aXv#vvWg>sVMNmLRv+6loT`mdP4ns9HLfTbH=6DP9CI%}HJbzpXhw^}TnI zur+^VhL*uELgR%gaV%IbH@E+tekw^>%bb+)D_qGA87-p@~62ap@5mYl}CA|(rw$MJ+h*=8K$d{tYY zzw;R;Lp95wyVhBD-i(Pdf-xf2_DK-`Az79xei&}z`(`;?tNyZ7fuzm&5SCHDX_0el zo7V1P#BH3?7o^bu(x{D0VZ(jpG9NPTFhs>KQ%N`erizE*c5atX`Buy*u~$=qi=7#N zXWc%tr1%w`7w0lgTdmDz9Ur-%%=lE^a0Gb4PL>VfNzG?M^GhQe5bc<1dNN`QymsD6ZD=&pjQe2+b7M>?HW5FnFkS zrQzY+nWhj2g>Tc^&$Q2q)))>{MweHi4Sp8 zf`M7WwgtLag=vcWFb~8Rei@IFJ6_FYwfI_USy8#r)2#QS;T{{cgrA3ja-8y@epXaI9*s|& z`R2f!m^X*RzM%Q08j2aP&TPt zU2}K}IQ#K7&9@!jYV}je`#o8*W-W`~m`TaUxHk#iH`Df2CeKUu7O=O^gv>SEa!De2 ze)1ygqhMKu>&LKFNg)xZ|2VaO*iq37!9z`AL-b}qZMR3i>1*(OK(_yRgPH?NMo_-TjIkA#25G zn|qBvk;QQ_8y`lo9ulAK-1_evvjr&W;NHKkbF`}DA6rR4UmRX77TUapA_u61sL=e! zzYEO{@!oA(URe&jym`u-wVg8E@U1IZAGRceY4SSf;cWAanD@fp0WwbBWvu#lWL4!+Yq%PBywVRxxUXiaG3#T`BamQl79Us|02_0Nw%~88 zxD@CeD3#9@MWfj_=^CaCj*5P99yKX4+tS(3D;y(=S~A%a=a<(;F9mY&*PuD*ysW+o zq^IUScFL0`8%2B_M~mT`<}Kl$MIYde*ohSQVl47se~gtPB@zjKejC2X%u9F}yvVPJ zAOU4Njj<*3d{ir_<#&a{-urJpcjbOa_ogM%Yjrx}os_A;#?yiae36L~%{&ghWi!*! z&e65SIDfQG1AMC-lt=5Lrfn}YzT4;?BS-Cj{$jT$cdCik4;g&-`H#r^x(}Qgl3dnr z_`f=>#aK*x`CTYg{>pnSv63r!If?)LaEvy7tQY51o9LZ4DbY!{DyC=l0l5^?;Mey( zquqtQdziF!4%Zy2F5)zkY1Whk9RdT_j$tdN$hGRZ`CN1-r=X(?ghH(VrgAgp!q<^+ zOhU80c^=bvN3Tj#+sbQ%ZwAsrg{53XfccRD?*7Zg@ z5j={d3d_4qoF9u|g=Kgo-`dS!D`8j)R$t@BGVv$yeL>E zJt*xJ$0H#O2}^y-&CGkmUk`5Dw;DmkA{&9_f6TaOzgeaIrNJaer(SW@N&mv)5C z_p;q)A0CBG=8hI6cgL^~6mByjg{ox<4d80kJ1^y77W9V(jNgew#BhGzY&)53 zjC-nxoX3;{#QkS_l>$>$y!?0EQs!1?K zl7v>ht}ltNMxa}m4Z1pB%wv1xj=@tO!CfB#KDLfY2b~*#aci4D-zp%2kk0&#R>xgm z2K``x10dc|8V1vsDNvm&sGEdfcQyu(@l5!<#gyA4@ae?UUmd~MJYEmkX^J4WlT+h- zcVdg?rt;(q@7$7)26IpquI*+rw{O5$onTAb=PeQTn%!IWjLl63v-V26y({g8rU*wa z|DRtuY50|O3}ON#sFlTm*c&K{mjgu=Bl_&@ilPnWlVu(&ijg5JLvZ!^<%{K?8U@DH zv<|v>b_d?mP~~xHALBkraK=onYy*ZGZQPa^L;l3B&2B|Ew&Jzw%pEBV1dzx7UK1Do5!)!g> z32I9OVh)OZb^iQE{{6cp$obQg8#nE|ZpChqj+C?2Sx&v_2Y1@<1SUo7us=N#9l9%d zCSNVDsU|oTAfz14GN9jI2H~Sk;-Q(fOvl>F-8>BEdHHu^xk{l(+jqF< z>r62G>(U|jt;)ySYD|K^1sLem==Pp3a(X1C(elOD*#`JT#P}V&5$4M{Tncwl*TMB@M^FHe3Du`)eQ& zaN3ku^kdn$(2M@HDghldyN^#s(qjFoOtqHYd;KAah4et0vIYBIy0=gr@dt4n_}n+P z=>7HNU;eoylx;sI_g^xF z`06i#Rg;scFBWHWWLTrgTb(6VIB_TQ)(yOFWN0z3A)A*g8i+4+eh$_>6^D&A#p>@z z$_uc)+!(}-M}Fli$EDEW_EBjo+Cpq-T~CxXK0~R6`upDRBg30N$+uD64!YVxd@bwWmKZN=5urd$E#bcL4C}Q z(whT5wb0v;*qP-o&nBuq9DC0RWg8)K@nk$3;ydJHeio5*;XS#6DVvenCI!}a6%8G_ zhMoi)4*9hQU2ZCGH1V{QEPAihTnkov!7*`@8$N^#eji+ZiDR-Go z!yrx=GoM<%{u$RC&mh8d9IOwIsmQ))umm2-jTW_e0zZQ?hZ$KI2%c z18`1vNcW_Hb|34>g?X+bcU+0#%$u@p_CbGM87}+fW6du8}di4(MFm+Z<_>}A> zZGx;BhNZ|=Om`u|Vv^GhLA7z7$MWeXynm(WPOah7KHpuQYE<>hKcVW8qRCb zLyY_mr{cl5)Q;`1s>7WPRljtwCWTwmNJ%wZj_*D8d?O2DT}Srp~sq}U$NP)2>!LSMP2wa zFmL;4s%*P*YiDaDjp$}}^88fckqX$4%ath%jxLgg6TAVIw2XcXW82u=`m&B!n})F2 zv{BdO>^`^D-$q)p&d!n#SB1n$GJJ~V7ZNuO!mUiocuAqNec}z5R-VhS`|*OrLs`9z ziKzLWH@+5Z<3>$cQFHQ$*ZjnS;LCN+Tcv%dfxI5iqE*9(y_>kXziD@H^7==fB!~aD z|6XIeNOUy?Ij=A{inY>8mYs%_Ytv+2rZi$Jine5Ym-laG^VPv}KY!+4hq0i7{bc`W zZZ)OgJIvMXvTtoBMNi*Fl*o95LIr_(HzSp@&xu4TC)s2$fw#uiTvlDNC1;yWM#+J2 zB4P%qTVr87cS$U~zde}8={Z^MOId!HLUyqwBNFAzJPzsKfhOV6jIZP!jumBNdF;+y zlBtj1E84IaOI9*F=E=(2*?#TMxHp#~uRFq7=w4}$cdOYHN}k}|<-eperngT4a!!XXK9E_yTGUm++f;G z0>hAcUMoog`}K#o?6#i*AK9Bd8wl{huA=DQ5Xom9sqyngvM|b(hd0b~t69w3utG`H zH}#R8M=Vq|m&IWti00`#j+jAP(PV<_Q_JJ%wXI`V9?OHY2d#EcA0I`!)ORZpqd7vi(6{g&7WOGZDaw>6`A-a~H z9{yINMzoTEAb^F zA)TDn;d_5CvM$2o*!}LIQp7>Yz~1zz?nnWT``9LiJ$z{T!W3p$MQ>QXzpzbv_QJS7ke(Nr+Q|P2a($zR~P+;+wwqA12X@|H| zOwP+a6QX&2(ao7RDlTjk)E$NN{aNLXUOeg{YXD3e!{!{#HpR!G8MBW4+ybzT1=zSw z?(e*;Hdnca?SD=br6$kU^Qf{nIt<>v%_QO?Y4myqzYpG+1}zTiZr28H8nAIDo_29= z(Irl@+^+@YSk};V7Zv8v(!Q~t3A9U)hdqQb^&RWR%RZ+6Zh6^Vb2`e&y<82wLk#1f z`!93!{*cZzyx)R9QheWtP-4Cs9L!|5Ny_J#tuRrPM&XwUK$?5mGAzWq*Ee+;5L~mW_R2Zd1 zwiPx1@LSh+hGIG79c(Vf+!?Oc6zZCv59c3es(bb+(&-=G96Z(bsHi_!cGbtVEQ~*+ z9qvv$vquzuDpRRvOCR$9CkkY9<&)$Ca`rxFWBk2gRz@_)!vM)f3Vs*JA{#J<>GGH= zU?1(-=`LWKD8#UHS;W21Bce7>@$m)%VaQ_l}QVD@ru=wy2Y`L?49O zoQo9{hXM=!%4S{2=+?=Md2lt#yP6Os(UMvfnPf($e{P9GXu0)|bmS|DgZqPLW(lF<saLtKaIJ4GY+EVgxjb5KX=w#El~97e(*>r~W5^zVQ>56r%U zS5EnPU^@}Lh(|s1jzO4Ffteg-agd@zE+%)EV%1jsH=4jXvACVFR+ZW370yFoR9rDqO3sfD$`|8g<>Jg48S1Q&(QCZs>;?QE|5?W)iiK!h=lYf zCl`ojZg5t8Ls;<$bX-R~Gs$VgVC*tKB?dO6-JL}H=-AC>|3XgLX5H64lS+G1g40V% z_e&H{?Yrpw!>E}}x|m7X%>Q{%X&M4R_1FzFJ{-!C4r*%5EzYI^D`G_%k>kx)2&2kg zE!Cg0f`G*UuG$!`IIeRd1sfOw{YzF__jzZqqjd;utdUslkK6g2UH8$3^ZGvG&W)5Z zvY74~5J4SM@3|FP4eQH@;&U<0zh?d59{iP+FuxD0e-%)xLB3VVCf&!zz zwoh$Xu~4sf@sWA_#sqoa%HjG=mZu9T`zCI)0Y$v^U%~3@uT5aIMs<$Uw7=ff-5XXCJcBPG1L@n&Z~Fy$0mz2Ey`{S#a#T z1)=6G=$Vo8x!4!PzS6?YAs!m5@$|)Bx^9J~ijbGPrhh2W?ovc&eeLgLDL~Vs)G#+c zhY>t93+iItq^acM;`)E1(?3jNua*OZ@3G=XQ{194uQkb`Y~zSHOXG?Do4Dq!2M}KJ zEC4<3Uag_WKq_V^1bHo4AQMz{-5Y;Y8~Yl#3Rv#WPC`Yxy#|opx1f*j?B_eN&9quZ`Lw}ZhJY|(B+*HA>>7ljx>rn?k_hmCn)x;{B@UJ{#W6XAt3qN zn#%ASad311AtXoi`x{oK?W9hj(4AE!;RT?DF^vb$KRM{rC35~L5X1R8<;UBp*WnH( zn$!ip9SB^G053c9*olHPdae+NqHS06*^Bb}fto`j*pFHJl>*6)88DsnZT$b!O6-}S zNe_{KyxJ|zfKnz8R+M24Z1#~BvVH>hJ6s5^-W?LaUi+?!#cJSj*sfH&KcTOt~N9!APAiyM_gKM zz0tMQmtLF!%{MZb!s>PY76I2%b+A3EC3oEYG#%5^wb!{)9GG@1i5>W z+>*}*bKZJHasVOP)U3#=&9+~nH)wps>`m1oO7OAR@S217{GmXNYBVKygc`bs$U&2*s=s;|sU;wSjyy?9f5&sE9!vc0 z3GsbvW!vE`rG3fa*>L2^VkoGO+Gy2-ZI>f#J#u~M#C&B8 zU#oC3b3x|))!lSWrFrvf=CE?WiWA%jeyEXfYuLHjRbadeQD+FrnUbFC`{x88=smSZ zek8%>$}2V!q-X$5a=G?joOW0fG>ual$~0VO^M|nl;4>_eOH4}Zd-J8)g4fENAue6^ zol}7H#J6O+WZa-<|z5O%+a(&!)ALWx}=&+kQpW_8^EH(i=k4}0$OYSF#XU0d}HrF;Fobo$71 zL%4yUe$-^iff6bVsP9GP`a?z3URV1zsxc>MMr7a*EjeMAoCur#86MNs=V|}^icj^; zAJWXXC7%&_v6d;ve5D3^blMkEAiTSonOIL~KFRd_@QP|^u{AG=9%pNI#h7kb�Ho z+vF?F*eSZQ^N~?nO6?MvWfY&Ig`D?V@u$+0bA5#40k*Y!y)W#xFWO^sy0;LI{Lj_j2({XldEqPH%NYgUDh-A)B0_rLky|?72G@@9RJ4IOzOy#Qw zwRc2j8y0Ia9|td)g#dvg^>-xp7Qvu;sJ&kM^?Lw{mmA)a03l|YnNPU$w%qsCCo-C8 z;z>4W{}MEsf11MqJ>%K_tQW?K&4)?-;S#p9ej8Nt&b^$<+UKYth^j|e$APntlX`XX&-t~~(<0C7= ztWz$aS}qV5WTEbPI>zZ#+2b{5=5#QG*DTNQ@|)G*3-nELb#Jzq{Z*hO1%GiY@;I3Q zdYJotFs{mgm+UOL&CH42*yDxmY7;HXs96sm7>^>_`-jkQOh`K`P(UXIe ziT){v1I;P}WLv}Agf%3Vzo|Vlm_q74vQ!Yosr)8=iU@@F-~8^{nvnB_-?w;Hh8M$j z_qEd<{?qiJnRCgxynm-D_Wui7=Hs}&(C5dM+0bhyOc-31HuawIoL3>{oyaHv5}Fs)rfXPJXnfwb$jYVr0bAN`(bYuRh6<{}q5&{$c&1$L_LpqH2qTmjyk$C?%_}Iq2 zdY4jnw5Q4b%FZRsvI61Z6_CzwUzIM?aQx{^&0M~Oe<1oUpNXNfVO6Ib>XrjnH{)_4wh%YVE7!V;oR7C>ryfF4cP94u zdv_Hv;{=yq|vNT z?l}1pUaGj@j-fh{FyiMHL{Bz8Xv@5deD|-)yki0fc9dA^E4;B!KTGq7 zx{sQ7zx~SWvZ>IUH_G)3SUc>vI3v3!ItZ^~QQe$ul|904n&s=bf;*YA7{3!Xm&lfPNwk8B%w)@mY&=LR9m!61Jv95I}FZ||6oXldZ29}Iu}lycXv|+LLm`U zpOm&67k`rpa4GO(P*~ZWf0|WY?7Mk&?k)}Zq8k0gZX>-rvH!Gd__Y5uZtOwzyrv*l zlPB2m)|uL8*kL|DRek|gVT1AYb0qA4da`F-KOa5NregD{sdN!o3ZG>D=i900XxvcR zKJcCIFI|fZS3+d?t@YjHLT0ajf!j*y=iyZ4NlI}v9?h!~;bw{xdL8oLA$dNfs09yo z6+b3(-N`sE%;w!89cw$>c#J`F%*Z-x@W9rLM{KtbW@X8mDJuUmkAx<3pex{VSW!Ry zr#VAibyXZCL;+uet*rE;XzUB{q!^U2Fpy>A2ESmQTM+c=k?`CV^sE}Y0dPgi+wutf zz%HReZzrLqv5pu1iX15 ztHZ(qH%V>T^TeD|)ZRfd7e?b#!#!4LVPqepbCx*;R7<$vR!fh4l0tA0TVpLhI%RGg zdWq3vm;?Vk?8l1xfA`_+FE;N;jQJ0ymNRf9a9@}ARBb8b>SS4$ud>e=LD(K)p*q^L)5tcJAUGc&xQ zW)j&S(mmEC6Vhky7(Z%WerD~b;i3;%s(*$}O%&>RV`@}gg5z&-dFlrNU@RK>UL02D zA#|TwqGMF@8C<-oOizZ+c!V5baT22=~YeF<}3Rmlb<>fEk3hEOhieAG|#jss(Agq`vv3UgIC;iqHBPGLY zqpYx|64m2ka)lN`iTzm#$e-T^JfsdXcGAj{F!@uQP3`kq21wRm7R>WWy@yzi19Y3u zC_Y2!-uc{%jSL~?O=)K^wRiU<+7H_9*qlwca^+tkdb(Nsm!9X_*(nR^1r0snX_`*_ zsG!K%sLwq1;Mo0sF0=eApPKQlHWqoa8WsXh-dS>x9f1dMXf5-}QvPAFkldar z`!*JhSV$JZR|ba+_V6(6SdJ_ctTR%`R6Vo|sWg3XR4e`TpXM8krDa>Odzz-I{7h#?Wj89gb+|Y*h}uR~c5bFx zPf+J6Pdf<@o|YBpQWJr`89oZ`_`* zITdL-uIlzYf#s83iGckJrpv!a^>Ht2kVa$G{KMH;40U ztbiQlX&Dw`CHVILVF~)OAcT0B|AO>CEr1%0(VCYy1XbuPG(LWETOZ{Fj^X>(ugc?j zZSu^LObH60r{(nf0v85{R;(GuqttXi7*^}#MP6_<5B@xl{`*Jl6;OVvRII=H%iVsl zTkEX$K~XmTy{$qB8=MsFjYHC8)6Bp?)OAl^slL-Dv-6E~oo20{QoT2j!5I!+wXaQ) za{o^GT;4;Q?zgzfwg)?&OwF{@Q!Z`N-55xFlO_L7jXJWQ zpsoXBNa~Xl$?#^pJoV?l3{qbe=<@zpZ9L+dUt2hQgq|Q#5=nMalV{eKkbV6nEK@^zI|qqZq-rz~GnjX4_o4NPcY z`ghL+p8sPa@|Ig^?QZSng@NLIY>XvuzFEFV_g_*G{schn<0r&`l8rYN?%*$JpOhuF zJi`FC?N%P{oms4CHTMo@2SK8~R z6B0PTo87FM?wS+lW5DDVUI@VTAFQFzri2412hGtB5KBt#$2;g7mVyI78XU`2rtew* zKNv#7dCI>Sg7F5HM=nqXp+V>7!$XPp^e(HOV4EnjHES;NIBbnn(4nC~1Tda$`1-|s z|AZ@IM*HReFUsCBD9+`116?4vy9S3OXz)O=;4TS^OK^9G#oZwU2rdBvixb>kgS!WJ zcfT*^oL|-b@V_7KR&5nU?ZUh>J>5@tKQoWziR7BbT{VDyfsz&Z&e)H3T*7T3?*l8o zlANMVwZiUNs63=-o;>rqgm%e-egevIe_aeK#Cph_NRmy@?m%X=vBM-^Ubz{Xo)PtE zPNH$qYxfT%GMa=oI3=}5%Lv&yvVJM>3rRv&`LK+*zS0DiU&fe@)aV#%%udEq`wvFw zzeg{hPg^`6faoc9TMp~~+1N>56ds@>lusL8+(faCE~0or0A9bWY=%F(ea_1J3RYg9 zPxz4b4M7;l9G0I}U0q)DDgs%Juib7vl@)FKnC2VmwJ1__ded z532iaekn7$xkvI+W2tR@j&d}g3q-g-e@qE*3NJ~1zWF~{*fw=XITsFT^uTBYejSkS zJ{#o6UTx3BbMyMc@+Ap|A_8O)7VCMt}OHhDLgwk#iMFtb&Z5tMhXY za`$rB;{Ti5`-M%)D_LB*iUEdO1WrD3mwe-@MSYHY)^2tndfg?BZ?ecBUoA3dR2aS2 z`8qjG;g_V%8l<|r{a*2s_JgqC)ye?-Z)^Ui7~__VU@Y>y7PX|J>p}~CA|?O7;#aF@ z#%-?FiksEY{=TpM1?N0Ks}F$*{@oy9__tWsrhl-k!~~}teUzqJqH-fX@c0a`gx$(D zL3!pWgIQGkU1C9irBJM>$d}F>J>B9*|7;~6$qUIrFSc(W-g3*e% zvXZKbvz`b;;u>S@5pk*}wJjyk&L6!Yq>Im2{w))##y}uG=gd1u?{nV8EJBq#p2m_i zJYm0P@ywKvht=-O@RkQIN6V6eB|)$mU?33TY`Af#v+q45^v2Vn1^F%BGZg=W9I^7U zG?}wu?5mmhUsLp?GY7CGb9&Dw^h*wYp0nrwH95#uuT?^2Q)PCp4AcgxX%>1WgFeo= z@+WxmalJ6eqfWlIsMinzMY$(jLXGIZj0IG{L}veXZd zFJ)A^>G^vLJ8RZmo~ro(EK#if@V7J(hJT)JCpeao^)JA;0~+EVEOdzu0xn zSF9~;g5~I!Hns!}=y`f(N>zsIaZlQ~Oz8Mb82_zbvuq zs#F~XF%+$%e@Q`sz}E4`Nk>graM;TKN*SgKYN&5$fB}$$&ex|0i$BJSuhe0YNJ@44 zm;IkCJD!W%{~x6g9?*{~Io4oj%f8%W@Ac-n1;F_~4llky1(*|#)KCBuDE40Gh|0YZ zYH>hh9h#MT2y_RCq?AwzaX}x!jJL=Q5F5SlbTFb-TI&JrY|hx5=??452R1?n%)VVG znNb#iPm$9Sm|X;AN$^&IHfXCQ>&RZMIyfHj1v3#Akhl0!4U#cbD;G*|nAD3+(UC}w?rR}H_Gr!N$J%O4TuJ2?d8U4u8aHf#fNQ!F+mh!tF z#3KL${b>p9+we#A`V|=mD6S8OllhKf?^_?(M_Y*exl0?JehSvwbS@4P5W2U$XY(rY zs0+Y-cwWVxgLuma_jbvps4VEniB7l;RI)G)7#@??+3aS|?*q21wE+$S$$+;`2Yx*+ zknRJ91j(lbcuXiYW{YF|KcdI{9vXO3)89DUj6h|(Bs6y>n0{E`>sWKM1U3Zbq5^E* zn$g}2psgO?>*QGE6Y|A$=qNr{+bp-};d+!X|`T8nLHW{zwdeTzzgEwbLdT)tSBu1{MoKm)I81nibfL&Mi>#s7s zJPqyW@B23N!}-{3ovABW++}B8Nz)}N(uJ4>yARpA_#Dj!Op{QR8+k!JAb+F+MyqYC z0#35HF;a=m)12C5P-l!jPI`CofD_fjoaE}L%0*0E?CyvxxqhXHTg&i2ddUbewu^a+ z#0&fU0>yAdnsE@99Y|Rs01C%0Ij&gmj_||K*{T#$HhpjD@7?k!X2}Ef*8l}1WoN2O zq|+j3g(%^loc!aY&05hx#io-+a&BdB;=Ha7RG-MrAX&7wr~uN0siqIK;w^0Dz-n{F z77kM}(mbo2lv$rWhbOOQzilE1{rX>x1~Oa#ut|h0xQdLaxW_w=C4pmJwgf=g=Y?_C zY#3m!!wQPu&=Coxo=rdT{1$~C#?4;Bm;H(4krWY&a92{F^$X66c7$vFazFtsKzo@= zlrb}ro6S=NScb(#kj9smf!1y`52yNPsKsXul+ivY{me54kt)?zH?;KIAqBnIzr{LW z$B5Q>u1_tM5AHe{uM`~Fv@uNs5x1)p5k~~ zlU+DpCdDSA=-C8P8nr#@SCS1p6mT6=%sgV~B)&{`*-XxQ{(OfN{uaD9{P3i%EUUxae&ESNbQBa)Kfct{^*85h^;J7saazD=T^fYJGo6YG#K9UDsi|6 zmTQ%a#q{oc7)oE^(qp}9woGi8$J~A7cW$TNE>`;jN;x6Sfr=3cLnJ8@-aS z!TJe>h6(Q*v?w!(XRth;4Y0+!AJIv)6cOX^;RU5tjfJ>WCgk@?rwx~+cPu1T# za#yw(NAX@T{8hSvenH=S*EB2Vn_5+BlI=UwME%>X4~mKbX>C~%F9So{94;S-+z$&a zRUxrUb|xT%W1*(DmOzjNx{Be#X7t>)6}LoFNiB{sGH@(}aO`0J;sF~}o=6l`u^Ij2=})YR3)7l~6MwlwqZ0W@1!Ms~TmZG;6A!_j-&Dt>I;$ zeRdfbMK`fOLN7k!b<-^1;b6yo7=zOP>|puX@#d7__a1&q0uH7k1AS3tv^B~gf|UvI z3}7zg7_}j2L*(LC99YMQ6G2A|co}_ALj><84;Nhf41)Hx~L2VbS%LfE0>#Uu!p z9yPp7OZlTdLFT4Jow?&a-QLW|#bcmUp*szl9Q^>X* zHue<%lr^KUVJ#}B#WF0uWrx-WKFMS5pl?0AU$+2~B7O?E49Y?Sa80I1Zv4A$+&o}k zhO?0_L$NILZU4CI3wWwzMyO6erV&@imX`BtC%S7(+}OtUK6Ea?V_PXn;-YV6q-#v> z>CTuys&9a&EIa~+5IfFZMH=7DtW@M4kr7;wjsjdn#2?3td}*kZI-Y27-S*>@R*J9Y zt0~)1=Cpd+(I&^EjXBA?&6xJ%G*;`@u5nd3-#Z1bRcy_ z1bdWb^1VeOjHvR_|K=Qtxr2HK68^Sy!Xgn*QCUQ1n8KaGaK-rB4=$y4NT=V;ADLIwIHP_N;%Mv z!Oo6j1M3(hs3WL`Cf7YNN_S|jgCW%%VEIb%p4Nz`GHu~EpQac0lyjl|1@HU%TK*L8 z;goc-*W5)|Fxogszbu?h-nCy5kM=PjKKuTXiSqXd`I;M-5C89(t&(lK(?wQl=wTIu zD9Chp1jpQPIayaCoy}xpK9WxeZB#HswhGg9{uL)mL1kWFsKg)XA{@mkim_Xw>7>yn z_<151Z=xxa;+>b>L|t1{3f9TN6CVSEH~Vir9r3Y16R@al_s4m)F{Un{jUjw{F|jF) zp<)Xhfsd#uhhj4;`f`%m0+x6r)wYpUou(BAJq~H}9^-=kwECm12tQYvl}|7+gdZB* zirtKO9fmhRpojhY^< zIP9F$ZeiUP?K{^JKPLhe%5OAcUEZUon%H(!;P)!kYUun4&GftOSP?y%)9&TMZOsUs zY%ZL=xa;`_oQVX!aB_OyJ{MqGVSXh+re%-D7K7OVes)a&C^m5yti6XiE-UP0H#R~> z(h|?TcQku`Pe^7&zPZ-F%GWCJ`=si8L$gcRwo2Ig{5k=jiS z38{=n%+-qtwC|6cclcWvArDL+98b`=n_37YRmn^3noVpJ%>W#+Ll7;IK_Q_+{I5&c zaLK@c%20qjJupmkj}NnhYGZmqBeSP&LSdm6n1%s#V=h}(`erKS(@l~bix<1J5U=wj zspYsp^4iM|=9T5>S{l6&gD+T@UpX7(cES})H3Ano>OjGM|dy< zk=;|l4Wnbb8*c4b0|P3)EH?Ae@htXsaPGf^ zlP!HlsyB6zEj^B@o3g*zPiLMzJ2JUJxLdnDAUsi>O*I}NVAK$;m0mj8<%%G)th>K1 z{IPJ@3i6FrkU!bH)ce|I2>1T!XV{O4b%96^YtH$YBj4`=zp&45zH(YEAe|$6f6aXM zO$_y>MR+q!CTai4zW$Qb)0>EM0bD!!r~bS%bii;FxX@TkEWcm~41Sqa%sF1&A@Y%f zwy+nk#e=iJG7|f>zspyq%3v_`EA!ndk=3&qDGqlR9LNRI^g1Mw+}Ggub2INr1rd{v z88HiOzUXIW_2)Cbi{xp-)qoEBy|OPm-ddP{j-RPmy7WLGD=#0$U_qfjW6y29hUGV< z6mjkhxS*4nv{tN3C7&wdIfPyxUY4qwyOKQ`XGQ2w6sD&bx2|XGW3&40-vUH8UBZ&1 zQ|<<)wAS(>s-FpH`!-8s?rAXWeY$u{XYLwQn}W^6qOBr27Ddu{?GS(!15Ar)n2bN5 z>6|dGK?py)Sp)`zuE`}lkb$QyRlf3bR>%xx8m5}&a>p}y#lF2n2~HIuy!kU z>CRO_V2Qg^r$8qQe^g{*VE28x3v-h6391i@_Nh?^KKDBEG=g6EU4u$9J|01!#39M* z2L{Xrmdxwrk?XbSm1E&SKjye50pe4i9vT~Yvhef6DJ4lCzVQ7my^x;Qr0a`ujyv9x zt_QbPF+a%(BvLS`$1=gw9?dGO_@Az z=R}CddNG51m-G%gmRgqsRN*?}9-{I=t9M7M}7L$J)J`Lz~i9wiIiM9KPxeTa*oCRgG1uDOcrfAE2G zO5iONc?NE?Q^eM1bQlz~9(K0nSO;=^w!*=la_Rv)zl&9gm^Q;g?gYg-21g=K<4OBz z_Y)g_qlJNL+6#>*2w6gnb>o>o*DLBh`IU?YuRRz*><_o=uw1jPW}viW<8;>Y9ShmE zKMxCPkPb=1vsp2^3tT1jy^n9-{$UZb_h!TJ_Q6`RISzhJaU^WksswTDOdYO_5?%Zb>K7{|!oT9@kp3@@1DMpTbQ^bx}c$Vo}Oo*!gt|9bP zbl&2RWF~KftNvMNbe*}`$J%Jo9(%82!hgaH;f~kaKl@Yc&sHv*GS);ROeVp!)vK1K zzvr4fywK`HL(_gHK1k-~nM5#eCv#tl1b}8m zGf+>r$I9h9Wgn$bNF=zs>S@gTWQP4q`htVwl60@pFPJBu=R{4%iY3q4Gg0>`=&PQ_ zt2G{&-VQK#tkQ*?_Wt(yy4n_0j-WY4y*FNC?J2krU{mWXTD#D&+BZ%Ee9WQAKRYKT6BES|)!u9urr7()^$!InQ2J-= zxC1YTcrhKmFR#jTK|SP72+AJ5a@@RQ-0v@N$JP1Y&V%y7iXJ1omF< z`7{(d?mJkF)wl3mjDbkv1cDO~uHK$;oW<6$VartbKdu~g&bpR~E2(=;+G=|$*CWsTX+e;0;11#%4d0Rc$!0vYbW(QA`oL9hIiy4MbY?ecsf<}!7I{LodIHSo z7djV<^LN7V2m%~PoVUn`;q0!Vg}@)v z+ihaSG|Mk-Afz$XF>DQKArM2ev8E7u63LtScsEuc*5ptO)eOTzsYF9*?7Gt*6(3%o z4tnwGMtq94Fa>W>jOH${A?_Ye%MDzqwYEjQr;)}0Bq^El78B!2>Lb=d$nZ3tRSlhg z47JjGJ71b}s&K5(vu3`ses}#QOWBAfw>N)%ptw^aIH zU^$|_9Z8XRKW?)UU3Gw?eUJWPBK}ny3GZJ~5P3X7_~7}M zsfFmC_l)BlCe(W6=23pdaN{8E$5}7lBx*?r>M7yLL{(-icBXRvW((_e_KZMtY&lEz zhD`Ow-SK;eE2%}WgO>yZ!8@g?^LZd^dmr8-gQ1Q!G1lhwfm5xnB) zC1G$e*IALf!+y#x~2obVsRpW4Yy6 zY4|p6`BNGPa9n9m5nKb~+c30eutIpB#UPVB`es`WBT-%_-~Dm#jhg;td=UdbA@qUd z;PFjyOs~J+GaQIa8-j&MNbw4Epj`*|k?;v!>gqH}?moMNEeMvOv*S&LOK>{6$r`bt zSkixCo_uu|3MEX+kQC4(1!ZMSZ8->ushk@VYQ~zzuWmE0kLh1orxLwnNq7?Qveenh zu+r5#$C6p{=CYy;({kFa(`l(r=KhjbLYk|VS}XUctpZP}c6tOtsC+An#HfaXF1&ck z*?h{pDv!{VK!zGL)Wk@wR(pMl|7D$LM2c$S%%s(%zK>Kq*i*Wf#oV+T)18K?AKCv> zQ$zJ5Q4_mrL^v5BpK0MMdy*yA&ZjdlDQk?gT+9m0GP+g3T$=N%oK4!c1fcI)ttABq zsUuL$XRn20(ezlpoBgF$yQ98(hV3uO;n2D&l@TcYxj$V+36^oew`o%3?k%*?jGvO# z|7F0ySP*qSwR9UlWrkXmd8m>yRl1x0Wdp{+t{`7z;b75Z;nMSCnVAeMF0OAw1@bIk z8<yhORr(T$d{7cues!EeiTG@bS7F6_)HL-iYG^OOAP(k2aAz~X zfSMhls18VLSPqX18N@o#{aRiz_*Fr54<3m)GG$leRT%^3k^U?ccn_4yGS`=t+I}gJ z3jdoK5MD-j+WVjcq`K8%ohf$a!pNsmEVCd{+vGM1Alf#)IJiT z8R!aUo(SMFyfc^;*L0Hr&`^+e_Syfq0NB{>Y1pRDFrx=F9E$CJxgzAIa$=hs_=9iX zB0MQnWq;uyc7BX>&&qi_OyP9QU=t=KR-LYF3J%Nsl(#fIdESX;;rK3V+$U44=L=vB zqpCh@7wMf~lD1u>KHAg$sE zC}jwT)o^(vT!eO1Mg%c%lMnV@-N524s8k`JI=hompD8fV;xRHzlmFKLQjG4UrGbU7 z@0S_T%9Z+>k8OjRnt=|h1qog@+~hKX;RSy@-G1k9BSg)DDwms#-ii0(W{W7oCoTIH zRgDiN>_?$w8Vxyi0fsEostc}WEZJkH=j0f?*l$SXCLj*@7cvryfTIfku%F#^7~pFe zqL$x^37$o$aF*G&PwBv7&2%I<4zElyc)mTo&tWd*Bs5Oj~r+b|7X`k%+@PPSS4*Qa{EwQLr1Ox zdk`#hv;;^hN{qT^b{F!M6ovD6{yNwkRWgW<5h*s2$TF$3=dyyQn-p`V%*62jHI(V4Jj0yOFwNR_ga>7zA>~zOPEh6xj~|1OI>OyZypu7=r8;aI*lh5^=pZ zjVOnr&6fh9zvG1EVPhDPUBk2N?3}oTdpJl#H$Sn!!XI{I1Ts-mqkO_cEgI70t1M@C_=9Wdtgmi-2I z4+fwzwi$`k08*Ljz14>IAs_B-pAZN6mNG-^!ID0lXy4J3;fPsTxKPJpo_*)OAby?a zN*54FCkeYA{k?Y!^Ydf>&Vo!w4=##AZegf^sc4c3J3>ty_J-|&8yes;8BD#wxJu4a zhThSXBV8Ip#_Obc<_}69{BdgpTG(&;g`K<6^s>F z;V6JlXzx>aGFxt=b`q!$kyl>jID#{Jc>nqmt^~ivTxEnJKBP+YdOAvgJ0wM6pL{yl z^Y6Q)5QQi>y2xYwPoIOs(cj>aNh#ZHMEYmwv)@VfA)QPKWh8HV13^^|Aw$$yrQGN< zIR3Yn`Q=Y36eGA{qJUf!qff(LOBPSO+IO(*;@bpK)P|(sTVu2Gp7MyaE{=a|tJaZ= z9q(JEx)X8XXGzpTsZ#-m#7s0dM-fMr@GLc0KFGhTahqbjhp5SR-A+ZGE-L@a8Pl>; z;Jp?Ys8yrWE>6$*%FvqHa+V*_<+J4_f=Zec&LOX3c9y;6|}+hUtNY9pvZB<-+au* zB8?%WEK?G<%Zglm4L3?RNvNsAv4i7yUeucH6O#WSG0iVe%V4ZYpK{s9>}{|UIy@+? zzb@2sfze2z5Gu7WO!V_ZFkW6tryG6+OQ&-UQ&!HTJHwc;rsp6 z*3+JtMi7YCU3`|!6z8tWdW}dA?8iHJz1Eq4XeaoQ;_)MA_ab5lWsZ65&!KeN5{Xok zPoauK9pQ(_p;o#SsHV}vz!C;-W#yI61}C5M81!#_z{8sxrE3Bdv?dYPbEpHiZw6Pw z+@8)$UPi3|@Bn~G4LzDg18PJ;n|1cWq>Dl|Me7;djn>#_J|0-!04iU6Bp&~LZqL?A zB*g!bM94a-cbwTC$@Vkmy-*YQoZMn5AbUwO>vD1=_GWjU46b znd?~DdR^}CB)6DIY7YUKoN_sxZhMb9eB(E~uByZ_E|p-W6YJ2Is!S170~|lv=?+&k z7C0Z8RzcSg^izs-%;o5Yx7h`VkYhWg@L~q--`tFoq-OK*Q~H(KQCx94|8;DO15@SF zV83Q*@mafKtYmkTJ$1UsT>@XhRaN!0f^T1E>9&5A*t^}9nl;&v2t$DTP1_wQ>yMTvN8`iIjPnUw%)gK4Y&6dkCwn}DWEHkvC;hr1uRWoH5o*!DyX_s4U zv9<(1;Ei})5#~2FlQgchiko3*E^AaGzELnW*l*;?Hn)3i#rr+{+GjbW$g?^yvrODE z+H-wae_)}g;^wwuXQ33iMbi^6UiTjTG*iw`oN`m@#t8+KHAAzq79#)VeUl_nMr$=I z>?0)TD7#Of4(dtNqT_=Wvos0^icMDw&8+QSk1~f#Z%|tvo@$d$^tH)m=i`d6xc}~N zjf|rDTDg2ZH~do^`#ewK%?Qj(0o|*q8QTBjM7lWLUh4x1IBF|x-W>xtTNv?vubAaN zQ#{TtDnW-f_}}nfpTPKOVrFT0&tE`jP-k=!>4tWH_22ky1u+?y{JB3D>dad`5ZVIc!_zhCpSC*7IGVHp=qtv2kpCuog|b%!~w;+ zag&yLwc_5dPsBM_U~N~Pk!hXng4L}=Ar=JrE%%5JR>mn=zNCQlUmPH9l!gZ{)}acg z`{q9pLBO>VFLSLt!rGfC`Yv>D6tB}WP~T`I@uXHWaLbL87Y>duq4`ae|AS77Y*`=9ugXK|Zr#?HA0$H5rpap(0al^OH<54kEQM2`~N5*bv5Fu50LM>y8IOZvUl^UwjS2Mt7 z(N_6u2y-W{KK5o2Q^Dy=@T)>O;~CE)4xRl10urHp!pOY|6g>fVF6tD7)mEXy?gp>^ z;0x=^inbMtcGl;54C@qlb3LVw9fsze=pnni!NMD_ZRvv#-UQFL9v{i|%CQ@_6eOBK zMO$Rz420r70@wB?whC`^J&Ed~W zB?YYt*aPX$6~|Susj!@-jQUc!k$kH7x8hs;1sd+N{#uewV!jkUhh>L*Q}#BDYB?zb z4^aBH*ss!G`F{>$;#$MC#SXCitg$P;;Y$ZAW$=f{ZgU4Uz;VOD@?le(TFs5@jYIRd zuYz);gifbnxl_?=7adS5D=SZWDSRm3zc2r@f4(&&3f$>FpfgjJ`H1F=NCCkDRGqpx zM)(8R412!c!VA*Sz}YsPBD{!i!~(tFg1~NSs zT%e(5_33$9^?*+!6ulSaRLzPiI-kJM?TwEU#-;OabLMcQ-hwvMTbA?FunA(KsYd)W zl!%It+~~4;L=h+Smkd@#YTK&l6+B1U$ynwKV;nmx*@fBbDtTFb!NNXza-m^efj^`k zdZ#H$@*%;8k|PFUBkzZJU64>EtYqd+E}T4AL`dg+u1YC(LN1CQ*w@`VzQ?_`pV2B( zZjHuYN38jj)%s-d!B#&`r4L6*sO=I94E7T(WZ>?di%r0Mbqfc#(6_UAafZlwR95~O zj8C8;l8GUYUg$}L<Z|YJ23Pkh@Y3cwyO0yak1nA+eRnO>lws_s207|gm z>#qQ%RY^c5P1k)B|I4%pQ10PM-P_RBA(+$czuY+yMl-h+SMUp`aMkYWC3V2^8NMy= zo;$US$6vrH-~Ec>a2|2;G;(jZp@LvTHi`hFO2LU*8lb2~A)#M^E*a1AccfW*v-G<5 zE=$+f5)7xfJV~m^Jns(|whKhZj>XM;yUX7~R2;RVh29r`^f!JFo;+-8#ieCL~Ftpcf83f9A%Q!;`Ju8wE3_Qh=ALy-?8M=P#@Wn;=TP8YU%dFqtNPk7EIJo5#w+_D#(Tzvb*RmCCL+ zN+evU+P~xTCr`h|V8H9IJ{5;vRzaU+JlWTWL{uCras~&z0MRKoz`2}GYuV~=^u@7U zJ5I>+Q1S7Rur{5*(#xm3qM)G2&CUG^I3Ks2><}zsV)MVn00pB@E6nOx6&uL4vJW3e zc^1Q?-k`jo@=V))@dbK8qOpF8p}h!euC2YB>h>Ey>yzCSK($DfECmtK;8u1f$2&{l z{q33F%-=Z78N;KI`}YNoju3vs4=(@)SN92CHlx%T}6GEAED1p02Xr zrH+xTP$;k2?P)!m+g*I?`?X}_59t{Z(?fguW9u}d4YWJsDTY97fSkp|x$Oj8hRXMg z8J*wVJSC4IG0d?aJXPpayg4HC5V&LJ3xr=!mi8Kmq~JYtOdDYlcG0D>@i9zTXDSbP zZbZ+Hg5^5S=D7YFz52TRsPd*1W;vHI(&yT-zW2?VZuNS%|{f)?%H%-!)^i(xGIS zDvF?^0*|1hYjwQbTnDI7*)Levt^yK3zmA@kJ@%Q=NH~!)A!o>d{GMRb!`a};>1q4< zh>&fY6|i&#HKVxP@ORQg6H^pp)(_=ZQi2$?gjC$jh&oy~25vwqSn%`FMmSgp|NJuU z^fYqy#;xbXp)6{LIJooNmi4^;kz!f1bcu8VVGY1tgI-VdT6b%}Tb1{z%976z$Ql{F z9wXkAvYc0lCuTjJg`SXAaat=OX7US)%#ihpi+|Nt5eq82z11w=ImK3Ynj=aY0i-l^ zz|qVOn6lX#udq#ABoTu}tjt7*#|M}m2b;mJ$OYr4v(agBx6jyn+&FTb#DSeEAX2%E1Oms1|o zKD$0TBs%NM=@CeuCJf)PTq!O`eo9JexIb!)i;oZR_lJFvt#$)c2^p12>Gma zg*j;93}*W%6uS*iR_zYQ3&HOl5*PJWEn5Vuw`bQ1l2>S$IVfUQX5n6I zA$WT5aVdOaD`)=HUit+k9G|k-FEHEMn6r#Enf#@2moqOH6pqlpD}~AWoPSs;)XYE_)=#e{a9i^&;W;ycm^-|k* zWeth~V!U=oi}f;k6apTc-jA1RN%t#&?)KB&KI_X?_yGv++B6<=R{>VgQG<+(jD(3v zxK|6PbN&$Y;@cWZ4FS|1sVFJiHhCNF$+3WWxCGz%s@fW>?Os}oKv`x!=kK2a0`Rrp=#c%MS~4<%d#!zJzy129M1#-RcJu|M zhW;&3P$fZ>AzDG*ufCnn;CICe210-Oso5{|ax92-c-F9>W0TY_P3_ zmA>2UC5|r(Gi0o=p4BCTsVms<_QqI`yrzfXs~7-0P%bx5yTxebl>GwUq11FS_6;NY zKll~R5AKZ1A+!N?DBxAl@O5;P3Q!exwR{Hj1z|vdvM?-b%MEn~*fqBAgkK-CKKWh>_$993^L!=*4AIYUK%~G+5|qS zy#t~S)l1ar2nVoKIZZYw)B3-t6DEX-t?UjII6Okoe{q0s`h->LBA>9?A0SGV@>C9xK>QYgmD&Rr8CFny0nytafOQvVXb2hzHK+MmO7XIwz|-tS ze^O&+gWTB3W$+UN>S;acuFGNE*7F~9Qw6;|f@UDMn@M$#R|`Wa)WD)meLliykl`wz zF%&uGBbw0(%7HC=PT4&B@xp1m!_Us3cn{zK2=j}x0A3a-a}&6fh62}%0DBzY+uM6n z_x@_ZHZCb?8>t>pO(XAR{eu5kTN%3#{5Av%1$0mXv}XPRP@%)7zdc5p?6*9MVqW>^ z0hjCmmGy5%&t+xZPXgV4M*9y=|M?e9$N~R6$hbW7ZA_`4li4(S(J5JZci|`$K-Hq2jHY6qVA59Q#>Rcb6f&^FDx#wT&>T~<= zi|;*Z|GqRdmIwa$My#N|P&^B`PqEFNj*Jkl(xf^ALt0#~A)$Ry->kcdmV`PcCMczSIu!)buv%Zt!$B7 z*9C;aXuHwy!_?&8e;)JCmEt~})DAujc>2zL@G`TPwpj=r>ft%+Sp8HXz;peLT*Yg@ zcH*f~DM9S11#d?(D_x3o_4L&ivx|2no8|8n2gj{kbtW^sq)5eY+b9>})nQ7b37Hv) zptqqTPWwioea%cIu&djar!U3KzCmK;ggl-d*sQr;X93t)wGF-gqHn{+GRK!a6)PAV z6qTt08TFVK;Hix^B``b%3K36+yCDqu?Qf0#)>LnpwrZV7vY403(#HZN`$|dKs^w%Q zwCYY>T#n=Y7~8 zt+-dy^i|&Q^HXN9`-CV%7iB9Zmwp;TLZ64I@tlCIf{o4l^P;^-Lkr7e{nEqjKETBm z^Q6hki>z!bN?c5ZBSA22W_e9Xv5%%vn>XLbN)gfN4vt%6<&tDwHkLC0UphgyBLmty zb|i^|!r8D26d=pQiqzaBf2n8OU`Gmcw4re|u>>M;Q&tFggZs zRa~mIjz$*zuM|rFswXICKy%Yqp^wqJG*^*-8C08}Io#z*gh%ioDEv2-{o31@Ej z--q{&brxRD2awF{qSk36&q7UwzYO_vfeB_il$p@r#1ybN=+$W&$O1}9zD_pi0W z__Mu??1?znKL^$|EwSvrNS{!4sz9O;_j;Ar*Tk$;I(c|y^W$XmRnyoF`bOS1_}=>k z)f0v1v>A+f5L$2Yzvd*i7XZ};>!2}PT)8dgkA7%M5)-SAnFH(GLF+8ed9^)i-%Yx% z9769rt4#hmE|6Rw^z(*0!%>t7qlSbL|^M3sTyw-;E zg&<`5-cIfKG9$wW{FtJFyfK7%C1B5L&7ErA9{FZ^JM8q&+9AFVkRirgT}zc9-dyRv znm4n-SHnieF-nI?-lvLtgAoAMc92L@ zKCj0^+M`e`M$a^j|4>%6^BEtX*-$gb{#VZj*PFCozYIeoZ_tdoOhKSM?jlBrj;;s! zve^TIiuk1&*9*;T%EqJi{iGJYu^2ev7h`AX!Jro08(?X-?d+TO_{jkdJYCIU7D@$_ z+0Pn-4(zrs^0OrT&RHM0 z*Lj>8OGsf;5!PynTiaRtFgu?F(10t=yN>Nlj)^6Ousf;W-_*Aw9pVBBQ(VJG{Qak9 zj5j>#PVSF>@^GhOd$`7eu10g%5I|zZ!J0-=9N^d;6;zcmjx>K`!ZFAIaBym4mVRUK z2iFa{k&;K7H-Y}SiNEB4oQt}GK* zP+BR4DG4itA zkGF_U2aq^-$6qm0Xd;7<%#L2P z_~=WXqgkREABcdeU=@z(l@1CG2QEkYUOa#a7=Vb+L{JjY#;6t_z@S!keTCJ&Ac|^> zA^jnv;?Kix5wVRDmA5IM!WiDsZHt4K;<5FAnCZFP!7=>T%q1zBHWhH zAPHD4DiQ*wgynVUd{{}VN3e-Xs=M)MLj1RY^;R;;V^2>t5~pg48S=OpF#~{i7|l?$ z&KN_(!G>!nL=>Szar{DS1^soUA?#Hp z6J6SJwqNq!FJIk6ml8^e?urUCuIZJXPYJZOeINin;0QVDI(~CvG#|giKSrKdFY@lQ z9l$>8?FG2|2{l6_eXJ7%pwbj){1^7$uTDX6d)fljA6$T!d# zxlgLEEtler{d#R+eb@#$Rlvg*c~*PpNR`9w-T_o|WyKnp^N2>d1m3e5zXxNvyr`KI+405<-e zz~sngJ?})>$0KX!2agv=E;dyNwLF44abB@b*ZD`9JeB==2>Q{rfv=^+q+uU{+$+EV zaF84@E36U4iyAv%6IwVP;zznFh~T#YCmNJ%hRJLjfX$0?G)o$FA} zA{i=YCad?N0uNaRl)l&-!0*3EnMd6>En&U(=;b?-eoXgf?dJ zwtOyDxn%=gw3PAO$sGs$yW|%s)2O6+OX@9t)7R?^x-+EnAYovO87_vbmv1F= zeW+O@I%C}s1%u@lIOSbehM^O67&aO%sNZVn^`~%)7_7d;f_>vj71!=lTgy%^$E`7p zDXt-%_W63Qiwk=fmoL6~SBB!EfkZVykak*Y4jziV&8eTl4#>nD52Z4>V1MVP+^iDdy#A{WqPrqNz#zblfDQ`A$1MYb#!o_fOA^^)> z-l%?trAt>h;)!{{YZ(Y&Fz9e(rVjP8zrxaE%LQSymKQ&7okNEfeX@-yh~vG}UKO-q z#(Ni(pP_+VIWf@vL=VJw{4(7(FU;)Yi{!Rm|pCf8I2w7+1qZ}MyZ z!KGzLut&$|GaFr$QEdBSlq9ezusDAJ~bkd#jkwZ&d>{ zSC&NX2fP-h+MN8s)~~F!A|dps*n-w}YYJZnRBpM-68Q{))Q?c;c;f+N`gI`g zx7}&%$Hm3^5j<*s1=g+nc4(-NKY7=TbdjFOS>)hTbhpJPrHCwc;3{^{&*>R6z!97G zdrQhhkIQCcTJC6m@~~sqel1e!NxmN0mskX^gL6xyLB4mZm#?SfqUIMyFf5lF>uE0P zu{>c2M|56+`dZjiu5FHpdI8+>jfr;)VB$vt8ml1C3(NA6d@qFA*2YiKSFs+Y)-hxx zo3aQ5vb#H;m~FP>p?~~<4D5%{iH{0Dl^Z39&N{FFP)T|0FF?D%ZAyA4u!#AEN1=O* zhnwc!iCa@|ibTGV$N7$2XE@nAy}mZDQqdHSVKi4Zo*ith@vAb{i*F%M@TI_O@ehjU zLuzvHC;|v6i{i=`wyvz<_yE%hA6KYIc7I#hkD=%r&;hTM^$Gex{Vv56oNXjgI_d*- zZLCqcr}NnmnR_zX?Ck70=Y{K~>rCWu=K(IVAuxYTnYuJ`9>E|YP%yN{ZJ@iV-Kc54 zy&25#9&*Pj2{3KLW1_x|Ep-Tk89Z5*3<_7j;fLq!uL_Zpdv(+aG_NnNUS4K2x_wLh zwL8<=IJyXLC&b{Zcp4^rch}WKy??PhvJQot(rV7`__Ar)x_o5-C3-VrEjb>`?DkwS z?Bwxu{X-zh_*EP74hqmb=kqzg;W+KbraFYNtsX~m=)X8=s892v@1N#7UFZFUuiW0| zQ0~1S+J;>$UkTki+63T51PFj5>wT#Lo1Mnkk?P&=W{viBze8cO93zPJ_JDoU(r2U7 zfkV=wNn2-zqO-}}jKldAE5VP9Riq>xK|GB(e|CW8aW1t8^_{k;||DC#oQ1FKWP5?_tM9Ew?aaR z!;$uI#nxICDNVR}3yUWGSrCuUqkO<9n+SE|a9uXg9z}#oWr9JWmZ%Lxdm(cyf`(j( zEn4sg41$8WMV@AWcUp8K z2hxF4aQ={Um#nO4o5Ck6q-M?z!m5!^RJfzT<^`5? z_kK@W24>jb?-sHrq3$Y=5vAau{lC6!ivCgQq`GLW;D^`GK11#tuN%mB{8II~4oKpj zH5VnD{7Yf}-z)^j^trHo7W;d)>mY;o%z`!r!EBN*=FOAS()w)Pp$Wn_1)tU=g2fv zZ0&dyrRK-or%|2l)16b#qQl08UylA$w(R-Wo zh~_{bbvelOuDx@jQM8KHSiUuYxW|cE)|wnzGNQ~HW23$1tq>N)#tdL2TXqhAc3a`@ zfX~rmbWX=)w)RS+T*NT)d8kiCGsn%1Tz_^F9TlcYh7$9zsFeN)q^Ups?tm2DK0eu*QBnMa z56K{{B+~YGUi|FB^8aSDb+YE436ErsWI6{2iIyEaZ2Z z_N@8J<0%s=(*KRqzS9%KdKWCF{X7xFyFNnBaAtS>*3&3pFe0ts8eU{2CMjvC09?13 z_vxEO%@_JAmA<_NsuKl9@fzq*o?2r`83qvSb({XIroJs+9{KCLx+1hrps)s%P{_Wg z>O}(e&aS9m3Q(QDFi9_8BqT_yv^^8}SMEM2Wsgg#)&#%HOfxw>xhx!S<8EemwT~@^w*|}vqV0k+eDmI5c}DqWr2_5CEp7CtLN)#o5zW(`T69l@1TIDELHy}2<4yg zy2k0!Qsgppg`*M*a7HLo6jo#QuWAt~aSaAsgY>d$fK>6@TSxFCAu(fAs64!b+ozfS z6k|nygZ}dMS%dvjK>e=luBNV?FIr%D>sT711N(x1Gtj=Q$Dcz(`L3+N1e_geV}S`U ziJ~+;+GZcPphT;dA9L1)p2PM|t};S5wy+DA$I*!=zt|uon4d(kUfr&bVEg7Q6J{9VO+Nsn^^`hRMIH zpvx)hpgIH}H%9bKk_E(fK~%1)#%K5{lr?N{!ee7PmgHp>x9btLn34--k&a* zckOqy+@uO$XSMr7zka_zhY2yct=Y)@?RPPiysYwA-rkp)%^N)M|L+qGZIPTG4jO?^9++}Qx#a3aw zF_N!V#D?Qfd~vITm30t`_^3BUEWFxCplz(L*&8o1tWm(2b*%VPS2 zbe+|XOcz}}5|i~}q)?(E>G++6#h>W4RP5DNobm!%(x)m=8qU5WRzP%g?0ZHYh-=K< zNRXrL53trSl$DP{TvRcY;~%l_OS7}yWK&PPX1zm1L=!!E#MS{H zBn)NVEQ8# z-uu*ux`;8L3 zl?^sK4rYSVm~jB1v(SupyM0y8`I|%VgNAZ%k}|Sg1m1&au50KNNr!)1MqgGC9jaP~ z>F;J{)Pb)V`u+XG#z-#C(?dT!3CJfW`aiE+ha0E?xu~doQ+adVw*8S`hS@2OJwNH}R4AUM6-j)PBijO! zGC$_Gh9~$f6z1vlRS$19YWGzqHYh?krh?p|i*#KfO#iCK_6COSfIAmKm`Dh{O8SYI z4x(hXG(Jr##Cf=YBTx47&aZtfDhlP}c1%?E#>6rSI=sH`EM-Q-|Fuv5J>Iw3wOl$&1K;~~X|>`$5Ggr`L`7w< zFt)x}3Z~pPiX@rE6WBlR4ASd}C~){p@VW>)aIA`cD*+@KF8XGTHZqgWDLkC$9gTeaQXlF` zfp_nncluSVOI$UYuiP)j#m{q(EPa_jY72jRV)mJn^yKTvMp<9#knSD!s=Vulh!>iS zX+H+=EM9VzdZXqN+jR6oQ6|^fAYX zuM!TQr!wph-n_nMI31(ARR%vP7 zBK;isxQu<;^Sjn6xP!$h1g)4*gQz~!Oj%>iWbA&ahW-ghCMc+cg!8DXIkKMGjxFc; z$arrHx%zyAfuyb!0_-RUnQ@J|#OXENW&|Ta7cNlv3>=i8rv$oxwW-M5iBk7M;AXt1 z_MKx`%Gl^TsBxvU7(B1Q;EYzP8Pp3db281{Rh^eZ}g|6BqO zwWAp$l<{&m(?W+m3r@PwC!(hXRz{zD0e}rbd+h1qAN@`G5;-QD0~AyVg8KK&A7Q^j z=n#ccf@AhssBT4(#r+yZw~_!60{-i~7d`*Zcq2cg=47)k`L+W-ph%k1Uj-J6=L7Ga zkLjojjbmd_>q~>kGR#JaosAZ#a#n>r$XOi(#J3rf6~11>mwOZy#<59_(VLSt-X#u;IbNbR{;{l)U#ujr`KEtsUvJM$8Vj<~VYem_ZZ4F_HY z!?%OTuosY3{hq~_@sYE#wd`-jYC5WxJH{7q+UZIP|2Oew&Wa>@a;VVyxXZ1q+{a0W zAN?)4YC{sVXcVgNGj9-xiQzXEJ{+R}g#O@As242J|L}cX>!yO)OlCyHTyAhxa)G75Q-Vj;H5ozZ=pa8 zsLouQ=9-9UYu;_Rwoys^^VioX2YTnR9VCtgglRvEw&0;U-_Q7{2Qz<%cmoPSaQoy6 z3|<5mqbafhGZfNbdtJxX22IBrIJmyfJ?}f%oq;KujDG-K*qDG|sV!vv{Fj0+`ezCZ z#9UbMs9XV`RF<3geb4^5yA1dPVveX6R z56_tF=2LqLmh4DFzS;Y01AJEJ4fAUp=3^)0hIWU=`)0i*RLS~7>vI484Q+1&4y+(q zk~x@yH+<%X@#PO)^s7U3>B3j|#>R!-z=qOT>tD66)(G54RE}2Viw)^f4|I3#O8igV zPoE=|SmS^)5Rw%ZqCK8ucZuDnKkS4%mOI9+4NX~Y9K@>?EGWM>K+O^9?52Sh_PA|8gd5zfu*#M62Jxm(Bv*Uadm0K$*->&+4 zI+?mJ<_tR`K8w`~^vdWIH1rM*k;t=cv(3{=xu8GsqwDjCM;(`(V5Y2S9n{Z>L_P5! zD<>Q1Nnp0OI%${^N z|A$HOZOD|J9C9^1=Iu?4HT>SjsAue%Ndba$7Fs9h#uJ zpf0lhs(>tbgKpkGJn{2s-Qf2AF=v7m&B1(mw+N#`Q!XTH?*8vJOxHYBih*d9M zzdptx#4sX8Y+&iI$A;jk6Y~|_{XRkmQeeTAx)E+T z;-e#~mYGNc3lgGzqpgD(&tRy0kJ=0L9E$Zq1p7Mg9*Nh2g=C z*T5d_2$pDARk2CeC*$#y2D2y4&ABVE8xneb9cobsNni}hnC{3n9WoPf6$10G8Xk%D z7;gR^tK1n5o2yhwZw2UjIo$3&@00aK(yixyQ$#a5dNFc{7frewa~!SV{`+DRLU?&8 zD|~oCnQB95F*eG7IojxJV5qHr5!EsFH2SK+X8Az)xP%=1yka-wh8bS8aqWZgOVom2VN#!7_sqNuQQFa=^Tg$C)00V-$tY2RR09Q`tG zM2+h(W;ZlWH%zn>v~|O7+{cN@qNuD)HlGubwOCT+C6&l900GEvvt*f=FPlh9*@Jp!Ays;Yf3!#P;Ft&>;J zRUIl;Ma{Snb0AMw$ORqQ{!3i4t#+DW5nKK^CGlrOOAK~?a~Mad9`E!uKPEre4=ibr zh!g(BuqGbqzoYY z^8PJgf>Y{X8=Nim-0g$9S)qsphSD;M(vteoC%dA`Epo%*@;Gf$C$(+$!QruOUwf(- zR2bRl7RRfhj$9FKWkb44LQTcz>frMqz)@8W=nnud54n6UYoqmtD|?iiV8Syz2mBsZ1szRFS>)0evg zHg}Q&YRm=Bf#ZDaKd<3vXtimN>QPN514MQbCi7$V|6eaa6IZJ~OC~E78_V1SmO0tI>fq z{WX&Ni*rUfwsy5@BI+~TK_DKqa@CH*bN90p5_k!Lnf-Rh&6Rnk3>TQR8qY^({mfhc z8nBK<%SR1WrGX-iMwI_VEa$A(!sag#=S__B)`OvAeQ6o3aNl4zK#N+>^Rvw6D395m zu;XKmr5EFJyb+ZUM?_Oug7cL>`<hmPeL)uXM2ACT%92NkoFZ- zv8vVe_@VW)4IO=|!&JVJhh0RNH61Cr=3Na=@YOpxBN(n$&OF@dz=giPUiVrzfUj#P z(0?WK5L35k`u? zOO^3{g~xMla7MGZuO-@$JnhD5VSeD&IF33`k!PQ@2w|P^UGi+2!TrXq*WD#E zK>;t6)UvFe#n0N64Uawpx|S7!y_!q2cDc{9bbU!9Ikd_0-_fWua4M>V0~)6uPC4bSFwkygQ_-X<`b@- z`+qX7lPO-sF~T>Z>`q(x;XyJM{0!eT$(c6)Q?w$cW^~@hVV;RMgB5=SUkx99`n2|S zq|#uH6sb^KI>Ci&gEMGen%owex`o^Pr5}4mV4JZaVD(|QqD66>7v+`j8~&by#n=(G zyOUvhmC@O7^`AM`qs)f#6mA@-Os$?euRl4L_i#Hqdrm3-4{GGV4GfW9uYVE}lE`5x zb}jMRXkTLu%XYtJ#ob@^3CEsIM%KsLllcb;xoA&YtF9+_aD?mKGCB?+=*?3+4UzxN zN|FLeb0_%Ux0ImGU9$A646JA3%hZ1qVo1(<)mJ(iwfC(r?g1c#;&>pbk#6^444wJz z&i9{IyrsI`&rV;43(?r5%ydBm7U~hKyDgbFegPjY zqYgvi{H{(50z;!FEUiV?$ zHR-`i-#|lKSYY3AV8d}Eb##3DWLNy8kzfxUvSeUa@;NyOqFWCBIg17!>VI)b_Oj4T$YRk%AdNQ|B?k{{zhkHd#DoY0ce`p?Z;U8+a{WN`C8piECi2tP+a&!;!Eb&VT~^j z$oU{L?(L)~ld^m~Zjp<=NlEDSmZJqmBWFwj_G~@-EAq+s&6t0L!gsNgE_zv%FAr*I zb-Z8wKaxJs8hW%&5Z(YrY)#G^0Z!+p(|^pKfAsA2tibaa!*J;iy)v`;UlKa9t3ChYV-5rC~&0mi|=Dx4mK!-^=f0Pe962*l`U~*4x%$f<`_{CMF-8+C#yUR z0`CW)rW-JwUV%i9b}CrKFD>HHh2oKDI%uUQWlix75vjLKUk>TF;ga!hof`QRYnuV^ zROVcZk`w7ttZ4S4`$9`=m*dY|A;`$>I=0^1k&PLiR2j`$T2ja;zW-w_V3c}|l=tuP z%|?_e(R{A)(J~b`;t-anSr=l2iYhO%Ef9Ph+y!2qWo6U_8pM+uuG8+0*$nUf8L%V% z6d0xZk|fCQOl!(mW0Q3{aY(0Bq%Lu(4Byr(C}oVJ&kFkpM_tzOTt4)!)g!lLuGI$S z(0afKdRtu@3LCma%Ex4JGIRzUnX)Tp8xA%TQWN-V2bQKm=i1r7A`;Dl3w?^oWlXUj ztRKXHh1c%?M4)#!1e*qmvrMfLv$<%b%UuJ=DEDKv{jp6R3k_S-#GRvu)yWMmXIY}U z=}|ZIzwB;S^2{^v-R(nxAa~}-rnnjzKU00xx#YYv?aKS#uc9pcFR@%UC!|foAG1qQ z^4B48XWtuq9>|9KR?8S_;ky^z(fgeoy8gx=Q58(pnRsP{cT`!k(qkOuPyGJ+9dxEC zvGKV1O`uGKfUX1uPRyC0r>NZgct1qvS#XDO8= zCq0&t4bCtm^ytKz5n07{&(qE`NpI!rwKm6}txvmnvaxtmXS;gZlhS6rY8qLlSW3f= zW>t}bTn;m%Mq%spdsTY@^bRjBr$l+fLl%qeJWR1B!~uqe-My*~3{U5&#v*5fk(Gck z6V?m(b>HhA?rCk@{4!ig5V%Y}klTblDaa{uZu=%Ah&MYgcXMnevM5WO@1b&BNXd!j zhsFu*C+u_^t{Lr$5?HY%%lnAZ=k9%ZKYj&Z%SjejTKu1@V|07`f5Uh6d)_M^9N+Cm zvvdMQoU>ru$O>*NUc9{QMW02LKL)lZUWTNp^K>>BT+fX#+*)$KL74i8TJtCT-i?5tm9#b`i>I+B`gKly*h;lj1z`wf8IHOM^2{nb!bo^90@c zIZswWW3~Xy(j3E_m30JsA$AvlucvceD0*Jn!q>ss98spYY%G9Sh!vwPP}i2_1PK|D znzbD8dVEM=Qe8ZWAKJVK(+B*`B-=__NMdoA#N*XgQJ)$;o$T z+MkIi2+Y>J-l`Inn9xEv&D^YN^DU;zWb)DCVi=)Y#nDG+*KX%eQrWbKV=}?TrF#V zJPaNixmlbYh9MT{;&wh$MVf#~+AjEfpoXLKmo6H3msAnci^URis}`G4z)I$#|9$uA zL6uxq@_!MTraf+2taQGv2EOP|3eq?j13q82ktEc;l4wg& z+lcXj*wI$9^<1Bdb34g!;TrWo%O^=uSL8Ji*!)()s4J>2=uXOOYk;@Vodn^~&PF-~ zt8|i5y9Wr@{3*dpEGonPe*27c>BVE`1{f}gl93dLyJHkLFfEpB+hOA11mP>|F63&T zL%-SC^#gI-LJXd)7R8phN~O#LM123eH}NiFts~!`Ojbj>WFf2BJ=+QeguZB42d3c{ zSJi?11J|@Iq&a!15Xx@7>Mhh1w6Ur2{I`V97_Uk6h8%y^y=;Kw;2T;E7F}kqq)Ca~!hEdwS;%}K*4z-)b)||(oeOOPP#ho7M zKSDoTb1_ys`Dv)zY-{Qe3rKmG$hFvzzf;OV)4X6^N`R_^47u);i{A)=UG{1)dB( z*%{8I{P9E;e_BliJ0mbiSC0g&cBkGiY6U)CbF#d5>dPA~`dg+-0h}MB7qjH%GrOK} z#2Xw{S*?+I?4GhesGp!4iQ>M!IT1|nSxeDRA|IscrHv3)M4h>tUuFKsWD3YNePmOm ziA-#GZ~8`g#Mo_IeWe2)PwDAssR0}8!(ZH%qE8=Wwl(l}FS%;lvs0n2 z=WD!u+Z%(KWSH3t0s^4A?fmK>JBg+N(;RL98vVSDK|alutQRZ~Ku-=4`-f@FK!&Ab z*C|D!rXA=3#qt>IcFoxD#&HfjqfEd> zoH1)KoGCq$AOrg$ZectQGWf7rK0nbC^%Z=Zx!}nk5wh>5^H<=fbc6d%6f4F-32v@D zMkssJFXUyjhTfu`m;(oi*cDvUUUB1`(a6CMh{47vo>)fo3ORH}#0{8wVC=b>$Z%=S z=-=w(jq~UXw!=yM`d|7uza;rrD;d|BY5Z)Hq;5fyx82|U`)CrZQh_xkGE;^GXo=1r zma&t^#_i;T`+rziY8(thbz3^y$CrPP{cEqB*@)wgc2GX2F8+})k#R@f?tM5WiIL~D zYq8hk98=mOK6CI6Gb-a-~Y7p8h%{^3wM z=@6vp)6ox3Fjk(ywS4i9OxJA^Txt1S;@~Bd8WVX8?)&ae9kW&;t}tKvuNC$Ud6b$S zOII$vkc|<+|6#qbXVFfZpL5#-kr_D+ZzT8V(yZ5?%G6RI&SK|7cf3@X9GWmCtL5nC ze3x0?ovIM)i`I|MGv*|c+QEVqtA58t71})N%0+iM^Lw-|S-JFNq2$dQFlj_egEKmq zsFL$k?XjAExk`W6jn7f1zACYE)6-F#9ur6LdvKuaE3ysJ7?m$R)F_LrL{o(Ju~l(9 z3-Tg$5?sgv$HDNCW{<@7`5~cPbI$%sds!8V%td{E zeUABc!}!=(IXr~&8OXN~Fe~}47&r#)lXhAb&UA?m`iAUV2{bgY+iM9-U+j z&F=gjVZYc}58&l>Wqkl|y3qa~n`A6+dft7@NUO1~r81rr1eUoPS z1r6q#kpuVTPUk~+jV&2Qve0{G)uVGFWmXRon0cojBBB0nsU&Vm zBN7~aG(`_0roIY2Qfk0zhH4LNVqqe_ud{Z;Ie)HCfMU@mJDmP?2n6nBOk8G{hI>xD zCcbUP>NqIeQmKa74|{CtM8XX^cIp*X*bxl{2Hs3>q8P-oA~6-m7#y&AFwGxIZbYo3 z!257}y7}ZcNw!xhEO#=iK_&5gf3rv?$XD&dmYu9j+KqDEcy%u2=R9TSV_HYXqnUI~6ipNYnLPN_;E$)Kd)^ zH}nbk-^T<;E8mLA&V`pzZ@W#$ez22P;piW-!$B`%rHh2BYlpSv-;{1y$GVK9CQ@U7 zB^1LbMFC^9)Pru3kb~PAVh<0*ZNr0Azm7MqF#fmaBCYRz{t_FzAL|Rg#}!C=^C%iJ z?Lb;S%MtepVTKBf8KSW~MZhFo!K>!4u5C7)MMpx)6J%E{L&}^E%N~b$49o6es`Awe zTy24F@;OtqNNRE?aZ%cUUUER%^ZCB2<3q57* z0LrML1ov`#ZIb?lLk4RthKLKOV|Ft=YK_FurQ*sT-d+y;ellmZHvYik(XE`kSzpHS zVjYZXFtg*angV%u#*WaXw>CjtVST{%kcEXY(j?wW%1gUqDzP?W zD=jk)X}8bjj3(-s9aW552d8)7#ki?+1aW#|wvgyWl!dDsFHsGGO` zb%PsBu|g0llt68p<`1pwuVGiESIF3y|P(vD!ai_SNZYkz!BSjVuuEtWs zXI8OKsEf1swv70WvqTWbo^^eN)-DbSDu(3`|KSdoh2~9$!wvw0AMt`V=n1NtUBM^1 z&@eC$z#RCg?H`-zLT_K+t*tHbY~g@^4l)%%hlhvOoJ4ABYX5|iC}uQHXjK;Fb=33* zh79w*XNGbKJlJcS)Q}v)<6gm4UPWs>VEt?;fk(OJLDFxGt!l4oF6MzmDQhZnx|u(U zZ;pwFGUxVR?KW?_XVw3DWfU1)+4r3koT;iccw2lZQIt?Gu;9>ASqnt{*=MJ`jt&uY z;d=^z{in8UwYd1x)YR1dUb>Tgt-9(9Qm{%&DZ%WH>?K{tZ-f52x0_tGj?cNXh4422~Lg0A0mbLYncWnghFQx&S+F?o3A}w{#aMLiUAve ziHi%@nI{!<5AZ55*k?owHXRr9A-Y1h)T-jZ#T7n@*v3jz^JnggZ%CyQT6E0xSv+KS zyy^$=$n~~?xj~2R0|AcS+9n#zlLYZgJkR7fa4bO+8^^5=Gc?v32Oz$sIFS4F1 zCv@8z$pzS~p!Azu!!EZ6>KvB2-^0V?OUIdlPZkN@!$Xnzov;yc+si5_ytkVxrxX;- zaB6?(zsW59g#7w29&~?HRwXMZm*;i9<_})9f`x-CIYMVO>who&bRhxdxbn9Ppl|wn zT-kEMji6VvVQgUSs$9$!Oe;!nPA4ruGzC?p%FhY@Ogtp$7@)1(d+sEN z_f3{ilQQuRd3%zMcG>36z5=&(lWDc177OTZK=b6bJsG2BK@=@?Z8Wa*em7TaDPkG#~7FfPsOKZgE{ z34`kH3hA))#J0|hc28G#T9m>i7)kZfbEfsV{9pg2ed1W`!I{aIJ8+#lz5u&Ymj|jy zL8uYM;il>D)m|ch8z@C%H2jG*^tu}r~NJzM|w+ArSbxh>Tpm&0Q_NiPJ z#Sn8^i<+8J@i?uSA5Q%)2Q@@z2fd^|-X6Q%9H`<*ZNBZ%TC0zKM_{H>CO@0i7zq)l zHI%5QCwEor6*7ZXIRyhl%;2DGp+ahRwMmag#pX9ki8uNqpS@Hog=R-Q5l4ec6Zav!Cfj!ro4yVRb?~u|QjT?3pG3L39R39l6$5TNXXo{d$tn zOG5PQX5TDN=5Rr|I4J<59f-f_#?$Z!$w)5?g z=0}ATKyn2|4GptO6uS<$uJ_05JdU-{$ONsmDL-&?C)L+;f*Nf5rhdyGuC)n3?#`{A zA8(;X(s>fH10SPMiMcVz$;ZFEgCXE`rdV8Dgc`}<`w5V`RNKunZ*6b;0E}Y~fQcXc z_fM@MyzvnsX?}8wgwf;PF# z-98}{e)2NH5#R^EKJ`%X@$qE{`6Wrm5v88laS{>nIF3YN(=vjA332*p`DEtriHR1C z@MmXdXoPwgr6OP;&tH{hSZb@VI27-T+4ntgxkM{@R8=3io0&Pk@actNm%zv%jtMgJ zRBQF-p+!UAoO*6QJ3!L7=65ax2=bS*GKT3wMQTdQuMW$N@0>R}-@&62Nvfz|Ff%g? zK3(*X2zV+qJa^N^tgfuQ5jF97U*zbL=6bCMBm?w6nCR%}(}!Daes@a&l~9|Tn}W|b z)0~bgDf}K+$BqC$+s_1%mlWUIqulvpAArne!kW{5DV2VzWks)dP&**)Y;pI;l0?SoNC31^ zdPeoxBfj(JyS>X^v*(f zF4N?+Ul-t7YDBnVIRhZy1Ry|E`J$K>uXCk8j;n8+J)dP|JBNp3-b9Vrpt+cgnx4K? z2Lfyy<*WN89SKokQYGM|F-p zQ{&?~A_=LKnn9kHdFv5RmOuBY1aP!b@$uX(c;w{DnfxAugM%9zT_J@t-B5tN({NZe zX2rn6lXiEnFA))ScjpA!9l@zkxB>`z00&jq&`{LDfi?xt9PHx90mv8S>wN zC!eX%O9muhEL&t85I2)qf+X%?iWPd7|JMsZ>eH-K#UrwA%1Yqu;$m{T)IcKSJ9E=< zbK`Cp_+SCW?{%i&?=M)@{zwl9v_iIUCZKyil9CD^LmQVAoAX|9&y1G%OH)Ib29Zv# zR7vwnNt#j}P~-73fiz{$K$(&%JG)_D!v3dHH#zw)fHq$3CL7h74WQN5)#=ul!ZCsB zDY&?BwY9Z_LqbwTf`kFXA}J+RV>A7+?4;vnN?swIJ1z#FR>c{%EAi|XNDPtkDEaw2rw^|^%|u>5 zQn#0WxSr;Sozo1`F5U*bX&!of5H7@s9rs&kJj%B=^{L`UC;%PJx87CH89By^v z2pjnC3}P}Hw#urhVd1kFn(zKgw4}RbR9za{f<{)t7Xi#z;nqpKpuzizRv$Msa$$SV z3TG2=c{$_C>M9|ZEzWt{9hrdV2?apaF60L;21z(MIbmUAkDllPy$SW42zsCVpAW2hzn>jM78DdzsGL3a??XK z>N+hYf^O;HPN_XVBTI;(ddeFdY;D3si);B3F_W$JommwYL2n!#npIK8Y#K1Yo`Wc% z3W{hz)=YqDnQ;g9n8l#^7vRm_IMwQ=`;O$iG3uL~6n0-8C% zp!Ejqxa{WMC<9H+%r394*aZyUc+jx1v3mf3eQ|M7Tug_*4KSFRUF{6LnP)(y+MmIC zg-V(K9XZiF0`@5jc* z{GV^?2mI4Z-XVMpLIuL}^83GB-Sm+?Hb2UCN5V1R)oNiEda;J|`N_;f+R3d+gV9Px}TOiH}S{c-4 z&8xwP$@%y@r&;66(Ujm4Lq%Sj8fDVVKqKeZ#MDBSE}b(XeYkvW!yq^*4^*~W8#Jch ztQtR&Rf`W7nDWX*JOBOtuD|s>LOgVEqeP%zP#H!o+w59dYw%Vf1$9{ffL zP~E`-&PHEf-`Uf%5b&Ob)|7h6coKct%xLK71M~CAfYe+KQYWN5;!8@#kqM>IsTW>s zgrH;L;Pe6!mChf>%+C~{Mted)#R`l%0`DOZ4x1^Ow}SyN7yu*zUSEm3vsJ)VFj33I z7Z|j7YCRNLzU?~}5P)qJpA<9e@x(LuZ8x#2{pOR~DMeD{n+<3^qKDSJyHol^K9E2z zlRx?E{tbj{Hu?oq;NTi4Sm<}#_OJZvw2bX_FrvP?ka zWO&w2?w$Y;cUW4SwCbf~o8vg2uwiZX98F z5M3`c9LeXQJ5ug~1YbOQY-W#zsag^~O=eF+UW{+8UX8m)ZoTi}lNO}eh!$HMg^R$B z6o*%QPqsW1wGhOmn*b<#-nMk$8)b?FiLI=+49d#)S9!t&6F24@w~cS^?ONI1o$g?U za-Yk-)=Wn)>O75Yd&A9ao>_^gIPg4f;AsVbz!IJblPr@a2fSr-6f8XHi^^Vk<0r7fl2f&> zA(R!^$@KL+=C7wPvK2l$QNy|X9mKGAXmNG~K~JPEyj9`ov6cr} z?qq|VHq?)wKT`qj$aEg3J%GBq-26PWbfmVTD60Ki?~V+=)>}vH$udJE;$Wf4mxnvm zE@@+*8kr!yyk@9$h@QNMQW*l42hH}7H=IU&b?ZeWEl{j(DS_n=XeckjKT$!$H{A~H&|7cyG z-!1gHg9TP9_bw$UtwHZ$NM2t6UuwW)`?3vOm)pa@9m*7+bi$?euH7a580}7Cl?)+&t$wXW+7I^-Cy<6>ozVOw3wf zXw5k@hg5@Anm#^Z0Dz}GW} zN1+$^h9hq7#mo1_24`Hn4NlaV(y{v!{n-ok2QWw6O)>2WZDZl*meo$H=(zC=IR>vB}}q$k(fbRpCq z-a(xrh)8$$S))1XC`|q4m405gvh-xw;$T1&A9@Sx0RQRwaAl$X$WB}I6qonq%NIE? zb2dPqISm*gtn)SifV?F?y2vzz%;V^@|0x;iRJ$0Y3hQvh7tpo34g75Qby<4k9eirt z09sl7AP`5fLdVN~n)T@Z^+@d_KRs?v+Y(N_#vk}y;_*4ouMe2AxC~Rj7F)2UakMHs zEmrS|r2H6`Dr32U`qrny`|yVFa$F<)Utzx3v!it#TG9Bt$;5p?3aCz~Qs0*x^;>xTBxM79gf2!<~Wi}f1;pc|s7BByVf zFVf^Iawn6gg51xvPscm+geCZsWL01I%U@2i>yHA+8^qH=ZD{jjnN)|ajAm>4qRu@} zaSe}5%@dv*1XwK6=}3&DEofGd@oQc`YX-nbu^PPkL2Oq7po_^D7j0WSi1uZU4l=H& zw`^w}?3*o*i8#->W5X-_eq?2co;;Ddld+MlZ@`Cg+Q_@|segEc58EQfNjtAIRKDqA zip}6m2Hg^+H`MiO+pUE}-@l#UNvwM#%WJF>b^rA3+CCJb({lRKCR3D9GmM!bGbnTI zhzqD5db;zIqTn%Z0LWsn*W)cECb|7Q+S*VEm{yn>nunAx@=2F-62ahrNx)4 z&UVg$bioHz)4IY^AR=5Q_o&Vf+nT$VJ9V!6@w5QCc2m`kkW0MJlhaXCHybXdDjn(q z+6(B)6U|=U;zm|2Q&?RK6!%>t0r>_XQS5&5(f!?D-}`Sa)TuL}-nAT#7?b~BSzj3x zSFnV;1h?Q4G(ZS$!98dQ9^5rJ1a}Pt50F5R;O;E$vI($2a0%}2?kuqHkbCcYKi;W@ zsx68+Gd(liJ>6f==Z(f<1wo3EwDiFMA3^ObkcekfD(CAc-!-tV%gR5a5~~MAh*cp& zgiN7o>%4~o5z7$&w5B{^o3V4FGQSwXGm`o}UM_SLWFGrP#2DGG+XPrYJsDwyw{zg} z>naKEy19GtHN6UO3Ow`i_ZL~WmE%LZPnd>Loy)?31-i+tV}x^~)jeRnazjMVV^G2} z=u@MS?=5PA)1zCnAEW%?NfNaCrJaAhWd-wEt;J9qd%DnDpszzM3l1Klb-M2)(|#7U zmKKkczeFo=1&(e{KiUoC1++!cY;<-S?toz}jlXjgaKdJA7=qDQ%*SmpEAwWi=9Nrk z5ip+C#yk9t2RDs3egON2<4Y(3(*!28|CO~jIWDQtJNfXz@tu+|FK^5Nx7K7jM6^mK zzq$faG4qIJX`W%8wnXb&u0P8M=0$sHANZBB2d|X;il{}zzfVzRQ+@&cUWyE#g803% z7Rr0=k%XU_!7#Sfev0!z(J+19XqVbr#p(D;L&oaiq=Z@)_M@d&iFB!qyDk*kW^l`` zrUv>Yl@6aJ+cXfcJ8bw(u;!CTES6i2(My-H;NQaj##pO(r5P3Oq(4jbr2c)p+$JNd zhY!x|NEdI8tLq++_8|Y1V95K!6T43DyqZT3=KrAE zgV+r7x7niSDcJ3rV8gfe&z7{+97q+4!84jWF3#7l_qVVCJ)RGkT%zYD*jvt{!Fp?UH*{UZeyG7pS&llfX`0rN)1fkH!YskL2-jeBJ%B zz53Fi5v}(>_&Dr3kV@BJK$iO@+LS6$H#m3s&w~{HfP)I|mp$k#xh|`KdM|s!t0*85 zma~?UcI$7^`VSGqb&%P@xo=6vG(Ai(*G2#W^b|0NSHK{gE*UdhRI!&nyEjw#m(N@R zp2eF9^^;~z%s*s)&!i&7nqKg8z}NVRpc>TWZ8zax(jqlUR47-9uCvze|M-8Y-?MB1 z^EZv=cYz+4PFdJ|W7aN~V%#Ws$1GcIi+6ad7&cEehxByMXUCUIeUlQue>w`shA{X) zJEv&(H3WgzVVZQbj)P)}L9`mISSjANbK1jOM za7XTl%&>c~efVIfil?()#%pIWWHkUwVk8Q@`=&eJeX|3k^d-p}p|e8?3BuDq4S`MZ z*?8-b`sZ$=l=E@^^}lSSeAg8AZ2u7lpcC=`W!))L3rD}NKgBD7kECW#M+v5An;_r; z@T*r6=x`YCloHMTSS*A5PDu_a3wLZj2x~l)v966Rzwx5YcqbC0{gx8kO<^@sxjYuo zrz7$Zf*_~VCha5t4D?=m4b(*D8C;jo0=hw(>YAi8w#R%Poi8)utNSo3zQc3C6x^Ab z;I~Q98gPqo9E5AI(AK}X59r02Ge}&dU`s~N;oTa!LDqhZ)U)6U@se&t1P~D zI0b7aAF_ziwK+H@#5K%Zo|GVyVWX4ejS-f8XTbu6ye_BqV&EUveu+@9q zedF9-_nx}u%_r4LUDSAXsEgOR+J*A4fM9Ftokr+?WNHtk`k?!^*+K3pUv}>xn23l{e;Q;8xvefG~g<5tgrApLT-{_#07{O*&w$^FCkg_72At9WC z{o?J>A8P7<@Bs%*PW8d^z!(xl$pU$~Z^`^=>G5y$&aY9crCw+|5|?BiP0hTMM|t|Y zkBzGy)%!;TZhFt;hcL^7rONQqhKDRIx3v7QlF^A(EDkcf z5uaMJVma!>t#5kVXP6|68LUbt;M)~a^!Rn-)I-psp;a9TOdaQ4dl&NEt4T=r6YJA% z0Y09bxy2gDgrSheN}AjOT)gK0^cio4E;MKE(F*(m!1#rKA;&a=C@4p z1rp=7uVWR-8UH~mo?6O&{y~%ZJFruLbt=CGwf^+Vut4(|_TNM@kdgc_W;PA|NSCIv z=|b}-LK*XLe}896HLgrzoERh6giO18`W?F_GWL@-ggSeJwB2TbysehKt4M7@9KXw>=$87 zcgf^lMKv|ys^b;YG}OO@cY(d@ea;HhWqVmn>?c)g0ZxhbvmeAynD@i;*H*qq$9Zsf zMTUoEf#wAM5>zN<2smaauuajd%jVJbg^4HP8)9c9s8@ z`G;s{iMZ@Xp(xQbzYa9d9km-&86ozMnUxJ|W=nGbAgYA(JY2O(c=M?>_I6Q_sUN0g zqi_>FP7<8eMu`F-{A1%<1M9l3&emVowRo;(vCh}~6}BGb50&IyOic}I}hV~h2& zS!H5wgM4EJVXa~JaV{Op-UGWYt1Bdtw&4t38*EQY%5>m4CWd?+1s7;J zRCo3zi@uWYF)-jdt|TP*6cwv~Q(uTgQ{=WtIz%A2#61H90fe zLMY$T5z3axbgi0y@sUO-uiX2QQhSn8dRnLd89u1$5v5~61e%7QYir*J`OYc-?9j-MnE;b-T`i%(Q%@aRuq9`e*D+5al_?vDp=6ei% zCF3Mhf>vZ@ zpTorbgIdXuQ+u=tZ!VI%24R<^(-UuY|MEl#j$GzTd`m7FGN*RP2JmHE$?Ceb3`r+* zOMd6_;(DRCWoHuDcGo>zhvA0lEh&n%J{^PJV1^93U^rmOuVAM-b&t?m{<%Wz?&~9m zw`^#-0bR4VtjIA8zbT`I6Q~i??zoR8njeWzOn3(XhYLlVNVaEYc3gCkT_?tuQO})!0`${rzqjnr8Ma zp`lM_ljM_qqhZTit5(%QBZ@j`!)Eo<4thcbnb*I{VO|aEoZ|MDHs;Qqd|QUgw%Fq~czbkBfaPn4O+qa48NlwUXj7V+w5k%TP<4lXAs69t zrK=4P8{8N>^m6}5ZbUg7b}>jHBJ2MLio70GFz9=}`n>Jd2^C~3df{GOz5U?xBkNIQ zD9rOXry+|xt5UH4R|&-FAUd3D$B~*)9|EVK7GKN&=E`jETKEO*6y^D_zg!|tl7JuI z=W1$4liNBVG)~c%JM=;1xApH9Af$FPg~4&9Xfk@sJR;t zk^SHc{Kal(QWghj1JDUj7+1dFo#YM_vCL*Qui;v)sn7842L$^q&YWIb03f^0poU8) zkN7z+`c><(_tHP8uC{%B5(Zl()P@}rk zz`g13s4?xNNvr(PzfAm&*|UuWkumuM%?4aVUXDMQQJ;!3|sEho`d+EMZ%u z?1pkuc?j^go=1BIaSWNqZ>6JOVg~$lY!TsG^NlBPtg(biM>{B7%}X02T{G%eZh{b%JqEgjl5%g%VRS6(PTV@ElUNCn(2ppuO!sHx1R4f0X^ojLyd z_Mb`sypgFOtK#GK7M`nMqI2!zulJrX!Ar?bY~@}C#71PoWXTWWpT)qS{Dr}W=urZ9 zuAl5fgWDU&n`-R(4qbM-?#72?8XNbAo94PAOB}%x69J~Z%gH-4JP|;tZKIpufy=|6nv-X>no#NB;P6gY)imO1chBx@ZF5pZR-Mhj4d~ zvC8QKhk1lOr9m)Tf0dzI*i@AElyuQ7hlBZ_AM8$vOaZzxUuv`rUI?}zl2vHQ=%2%` zzjk0cR3|i8VDh70Lr-T!jh!z#vC5!jeo^ycZE#wZ1sEdguoB-Tg-4)59of7?^55yJ zx)?3#F(QK2e~La|KU%1bv(&+&byd}kjB|&#NdRGncyepg z^UE34A!$H(>iK5eB7^lo);lTq{qf6~!V8`OTgII%H*WFJ)yM(a=jkJK^y&(gvO>>9 zSgB{K1p;^pT0U8BrnZX_WDRC}vE7qS5sjQPsr&Ei%*5%B})7ea; z(Mi@F)5^z)L9#bsB(u1TS12?`+WCKu1h9;AShCKFwrgK45C&v;@lpmwXnls-hlkys z!5*!Oi8eHgi#iM4i*DD}57Wh*u~i&NS}tw+${ep~IAqx} zo1HaaK@y;PT1JYmGf10aD4@frs^0h7esrWEheW4`fg$9WrRg+(cG`c`0h^8 zb^h(K*0?#zNW6DkHO|jp$k#n@x!=z~Rr0`4BY#eP^8AQuSOk8CyX51~HD<-G-rB%X zsYL!B{=4F4dIMUn?}}TYB?Wx*I$q~$T4atFt(*4UxtP;0!8F~-j&r(p+JY~U_RKr7 zZr`k)d%>ExK1$myr37%Wvnw+)V(mZR`Lrh`k~8EmVkEC6-0o!5+b{d1?nm;UoI2x43s2^LSXLgX>gxF%zkk7_{EV=Dbp;(jhpP_m@zDR)Uo@Ys1cdYu=Z>&m|VD_d? zxlGJ)`(5rn&y6)kl%Z*}0mX?=XPRD<*jw9rx$V#{h+?&I7Ldao=PZ>e+M^odFdxEr z*CrcV)(_(0DwVW$u>h|A-#IC^)2P11Z;=Pfv82e{KeWQ?ETY45!#?zG!Z{?^ZkVtz%PBKIkl5&b>KJYrnZ^Iqlzn z0NuE*+ho3|jri2w_48pU#M+%K;4XlAM5HEYq=UNam+5!w(VS_!$Vg?ArShTs?Q>LT zkFoQ)?kumv(5#*DcD#ph*|W2a&-M>=Q8tzAd7p}@6O5?s^}-Vsu1@!AF`Ks^TT^(v z(R6j22h-ZW_a4K)y6YE>3gmi}?8DCE7u(JbJQsWe-x+1RNiQ#_5QA^ zO0;Lu&VhGhm40`pWIZBF3BJe2()ZB)`}Km`k5-#2+s8IO;()ET8Zj_EDmVW50QE}G z$ne^1uKdY!s^4-MVX`vHr%VJz|FBIh@&=HH}lpt?nHzKm52V$h(zY!Rd zZMyz4K;J*2@ihl@dL-k$dYBoz(-tN-0o831uqSS|>vE97y6EaGNmtNaladdWJhR3hK)+TM?>FjJ%^bm3?11IeB@8N@_TGD|M~OU ze_QGAqW2Z7!>&O&tgN-&G8_pw%n{txgE45B>u5-oTn#+i&Q*GJFZrHGF+a;>F&WV4dwlHwIP0TR8Dz< zbJCrkJiz~@u`OIn=9;2xFu)SH8xp))4lvD0J|KNL=vDR%tscbx%<3LAr z6s`4pqLcXDCEEenlpsphgBOO>vb>B}5DwoEZP)B{ddI=i%{m zJp%2b+PCfX;O9K^gOg6%^wu9Son~xN^+HHk#?L`Im-6Blt&Ln$!&E`qVxmDywH?1V z(>!r0x1xzAeCS#-kq!SxhDR(>`b&7LQoV3JjoOUU{GO{X1bw|q;hU%&KUim+y76)j z;N7lia1%o35NSpi?hwbJju953=j##8pdHJxBe<{7Wb9{Wva@sA-gNTRHh+k%4l~#1 z%j{uE;5S{}#-XcLp^(M7eZ~YEuQGdJbUUKT!$Ez8AUfX`chZNVlPIFRtThcC-F;=h z#Er~c-{D`O6^=@f3jN*H||K$a?vyxy8+#BC6`HjTM6@4&9N zOc_e;0YL|m@|_q`EO?^8V%x)aNd1Wx=QLql7M(64Z)z}?;3qbqJCpN{sal{%Xm)%M zH1znK-2M4OZkqS_VK{n0A+X_9m}q6LOj#?|gNL0k$?u#19f$=P$V4?RFj32`JwjjC z+nPWWlY0xR+jRmaN$o3t^Ke6dFz-!F*XthE*hp@ade7Uj>_q`FOb@SlxqXV7v^LtI z0bCaY*e-kr((Q2gY$P4(__sPAo-gAM4UYn@l!(F{oqu&jxx8gPLB9sA_)Yc;69kgP z6ap(=t*$P;FCa2hK^V<3ZjHORZgPAkov7hMb)p(a)RbWeP=$M=fKE0-jA<+ag{F>3 zV=UREyAgZxU_1Ebb&4%N+)$pKOLx#NV|Ms?zwJo1?=duJ=Ka^`WYM`Ep)QL5UqHe*sIBLLK8+QsA$hB?g26 zl0%8D7-4JDeB*O*MxBt?T%W^ypEDUMR~r^FCcMDHKJPj_@JkR(d9xl+D;OC&5w57s zZZVr;OqrR*HG1HsXg~^DIfE*K-rj|hH+j>c()GSd{aT=z#`q-^CdDq$jl;!d8ushe zD~&+qaOG23D|9sM>)yiykrZw8L~T4|6r0f+O}c~=dY*EGH{8c5mzW?iF)Hho;~012 zf^73-ow|Ke;P#H_DbF04{Z{$oiQ?e*TC4e|mv3RiA7tt9IK6%1&IIEx*ra<=o`Jvy zEeBKC5Wd`K^h zbS~*7jldvkZ-ZM;@Qx}c7B3U?8RBg zUGj`uEOUk*Wf9gR8IP2o*&Z3vI#G2L2r+$J$dH4d6B}g(ih~XRAb^liU^=A9&pAt9 zVq%P?EGFx*Nfu3L_Zqa9FZT)Aj2XRRo3nF%{RX@s+x61l_z}B3=1A&|M$j(2P zOss!osDH2vcI+P~DxT3(|8<^`Z;n*|5nBvMwJr_*4))|XpH^XT7G39yiW=Z2o=((4 zHyjyVRxtTisjF{6lT;lPX5mUlx&^&BzFdu(+c7E+mEQDE3TMk@lpNQ69NpRicW&Fg zHXBb-;qv=Mi-cnLZKn1GiuhlY_B-CjnPlgA37U=qh}0&;SC{|TWL3P;{5V$JNtAB- zV?aerSNzxU<;p6wb6@6p_@R$1m}Un9>x!?rgSCkbeyi;VVe|c+GTz<>5Q-TT96+>( z^*TpWt#dBW)o=)KTFr=N6J#Zh> z@F35+Zs(xs&ICTgN9yLombRaG`;~M8Wu4k6nwKRI`jy|8l7y@M3kiuQ61{FFsr_Ma5U=VD|kXt-nT4|ZW4N`Zcb4!W=;o{s~o2uW0`vpy!TFgpf=N%!X( z;yRbsE6w5H+L`7D!hks+d~z-7-`S>~=BH5%gY$eC{tF8l1w=}lrc<@%9o>=U$(#<# z#tZW$=saZna`>Z=3CX)#n_?9lbn}j26oF@aw6zW%8s_m5dKTPJC_$s zWY`eQ{-Ea419K46*tEEKnOBvKqz@5{4tneO>WWv-vg*PQ@CfJXFnhlv?)EjsM~|ZU zLqGIv&cAlpcf$u84u$qiQ<52T?f0&eLJ?b%-+0VzyqG zxBkcs?ShS%_kV?PeR9F+8Y6$vY`NV(-;$CUDFqub?f;5E3b&{3I^BOE_>8#;p;Bo;A3A z*UXv=FbYp7w(FoImuj{r<27`$_1SM=m>{%}~{^c%%w*1Gx&W8HzAf+B?iwZg6&cc~9nLzI>$q`nuY4nrWe*d6`9_ zMipxDW=r0GilF>{_P!p30J%>O+s=H%YOFn1qOx?~%>;q^T1u!ZUxM^9{HU4NweO4% z7shWBN5=XK?qas<=gta9kP${-Y|Y9ymXRXKu~ zaR=rp`s&GrOK(p-Q^7RK`gT`|fXj}^ocmX3aS{IMD%|1sM)d%vm$k4#n-jr+_(E9I z7pJZnwpiVGdsac6&p1KB_XQ&QyrML$v>@R?k(du%V%;g|k!a6Pq4j06IxQy*fLR!Q z6!W(*mMf;lz?A=1=S7lh7BXt}78BDjIwBTZQm&YzLry3t<;zPiui|R&2j`7#qC96e zDGG)6y<=S@PAcI&XwNTYOij_XSZCvkXWm{MP#Z64*MrtZ$Ou1y##%fW{*kb1J;$uLXe^k(JUl#!LOhn(Zn?{Q?eO4}k2u4oLXS`V)YARHbYHisT$pxL@Nqw|fnXgFoF! z8?`46uh`=n(+#SZ`C2iNSHOZi5b9cAj;89p+4iAbm*>mOI`Ot~PujgaqGD#%7eshO zSRxhQGZ*1MBDk>@Imv?NaXHHye>7+gWm-?jp4{Vf9adHJx(Ya$^~O)=Wrv3GqE^4M zd6sJ~sj8M6Dp=wMVLZn%x!WmQTNbH%c-w!W_bKCI=};FdMnuqe7v^>I_~2Ijjy%h1ssO`z4)+lzS2 zP7zi;;sLA;!IaZMA?tb}5urj#{M2x>)mGsumt7j2(xg4^Vy2|~YrKU&Y@J_Emsbj` z_-Ocx(5N3Bk&XhZeon$_+95wPm^oM`Z4>R%03Nlwrkc_LoplhNiknl{w<+5jZ#02> zZFI3E!;Mj$h=?D|Wh-WV|9r6xgJ6x%p1`dRDqTGe&M9I?5wl{NXQ;bd3Kf5-`=ejK zLr$=>aG0GX82H|>`66q#{;UC6cjjfv7fj3bFveD#?Rp2kPspw&d<>T23U8^MbSqbC zv=%M=XQc)Xf1+9lCFJ!y>{KQz#-?^8g~r6UvC%usKWE=`80a$iGbKPT>@d*FBnUZV zYbDRiejC~-+l~Po6s*V|wuE@FS)~=T4>@#zq=l*`2`IvBQ+s*p{jW{?2Y98B+Ls@u zsFY-wc$1VR&0?gjW%N{z3C9-=hPqJAsp-9m&o@`?rlX`CC-47xocv4y^4WEz13Anb zp}xLMHsqEFVeJhRmCCNN^Ooxk!l)Qm%C&c^3e?297|s1g0UIR*zYS?RLzwa^^#Djs0OSJP#s~#Yz+t>rj$D-w)-go%{S$tKP zi|`F*WhrNq^Y8Jfq&Q)IQgxnBp{}Y|5`A}XLX!}|XYx!62+1F{Og}vdtBD0WCwGa#extX@6#h_J>yo_Y<5h?A4XaT-@NtO-T;X1q;=`s=25_hwp)a?>Ku3J=6B0Waj^9Z{_}v+8i)1$O^38_& zw%5xxqF)ltCj9+F+0rCT;-S-fu81BVeLS`a$pF*&9#?9}cA&aMGd~_9YHK?&?W!EE zBJ$T`Fn>38qXVWWx1~~2LfcxEN{_ESHV+c6D`2}j7F5AMApVq^evxjKCVV|A#=(U76 zP#>G|ME_|R!vSQs*}XK0iyEN$Xn~QTjQXK`qoLY~mDP@bY0b&9Y9fSPJNzwpNY3C! zbs5TEG$Lm}j&SM;ja(F5MRCU;eO=yUwmnx;>{8f|WGmzu*ezX^`p-AmLzP%d<#^g6 z8`*hQV3VtHO43CmZ|hfR!BU#Klm4QrwJgFjS^m7DbtZuaDkT(^NODI3DYsrQW>TVm zb2_AUTRHMXSBIC_A}of5O#iF~^M|qiqOZH4i@*EfP}gJnd{&vpyCCi9K#zlw^p|J? zuJxpbdNLZn`<_f#ge?_{;lB=WejpsJupm`7C|QtNnIrY!0$$16F0SpoDGppJXSFRC z*dg)#X5aR?&?DPIB;AwrX-k{`%Ppn;m3r+9_k5d8&q*e&!W2js6lehou^UI7M)IenH)_R zR&Rcd(7)}k^yK;G*-?qenuX*l_2ze@=(l)m9ks?O>we{eg1>BJfxGrKpNm5W_y zCnLVN(iXlu$BFE2aQ$SxvmNTa-tue0@bE(HNh}ezLjRdi4U5=}*05%lD;h<*)ujl@ z8*i-xAqiX{L0SNFzql*a(x$4Zg|y64Betb3XEpYaG-||rzdv`!I!+8KLaqD4IJFSC z{nRBqP5S?~Sa61^aI$+1F@FHp%As9Lmj}{x@<6Mu6&t`>?oaFGKqU^6)*q3Jb{vFe z(0>liRM(Y@!1}de>PaUyzFhJEMBacQ^8J_9kp9y$ljPZNjr(u$VcaBd@F)u-&ubk`Q^@c zRjcDU33&1IOSGN&4s?`dl8`U52*Z2-oU~#s^Z%?9_GE3OM@j%g8vUw+&Ja{ diff --git a/fhir-persistence/pom.xml b/fhir-persistence/pom.xml index 6d22adfc846..0ba50f2d2cf 100644 --- a/fhir-persistence/pom.xml +++ b/fhir-persistence/pom.xml @@ -36,6 +36,10 @@ fhir-search ${project.version} + + com.google.code.gson + gson + ${project.groupId} fhir-model diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java new file mode 100644 index 00000000000..caa35512fca --- /dev/null +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java @@ -0,0 +1,73 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.helper; + +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.logging.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import com.ibm.fhir.persistence.index.RemoteIndexMessage; + +/** + * Utility methods supporting the fhir-remote-index consumer + */ +public class RemoteIndexSupport { + private static final Logger logger = Logger.getLogger(RemoteIndexSupport.class.getName()); + private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + + /** + * Get an instance of Gson configured to support serialization/deserialization of + * remote index messages (sent through Kafka as strings) + * @return + */ + public static Gson getGson() { + Gson gson = new GsonBuilder() + .registerTypeAdapter(Instant.class, (JsonSerializer) (value, type, context) -> + new JsonPrimitive(formatter.format(value)) + ) + .registerTypeAdapter(Instant.class, (JsonDeserializer) (jsonElement, type, context) -> + formatter.parse(jsonElement.getAsString(), Instant::from) + ) + .create(); + + return gson; + } + + /** + * Unmarshall the JSON payload parameter as a RemoteIndexMessage + * @param jsonPayload + * @return + */ + public static RemoteIndexMessage unmarshall(String jsonPayload) { + try { + Gson gson = getGson(); + return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + } catch (Throwable t) { + // We need to sink this error to avoid poison messages from + // blocking the queues. + // TODO. Perhaps push this to a dedicated error topic + logger.severe("Not a RemoteIndexMessage. Ignoring: '" + jsonPayload + "'"); + } + return null; + + } + + /** + * Marshall the RemoteIndexMessage to a JSON string + * @param message + * @return + */ + public static String marshallToString(RemoteIndexMessage message) { + Gson gson = getGson(); + return gson.toJson(message); + } +} diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java index 8cfbdf248fb..663cd5337b3 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java @@ -12,6 +12,7 @@ public class RemoteIndexMessage { private String tenantId; private int messageVersion; + private String instanceIdentifier; private SearchParametersTransport data; @Override @@ -65,5 +66,19 @@ public int getMessageVersion() { public void setMessageVersion(int messageVersion) { this.messageVersion = messageVersion; } + + /** + * @return the instanceIdentifier + */ + public String getInstanceIdentifier() { + return instanceIdentifier; + } + + /** + * @param instanceIdentifier the instanceIdentifier to set + */ + public void setInstanceIdentifier(String instanceIdentifier) { + this.instanceIdentifier = instanceIdentifier; + } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java index cd5954b151a..dcd762c7d8a 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.index; import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; @@ -563,19 +564,28 @@ public void setParameterHash(String parameterHash) { } /** - * @return the lastUpdated + * @return the lastUpdated (UTC) */ public Instant getLastUpdated() { return lastUpdated; } /** - * @param lastUpdated the lastUpdated to set + * @param lastUpdated the lastUpdated to set. */ public void setLastUpdated(Instant lastUpdated) { this.lastUpdated = lastUpdated; } + /** + * Convenience function to get the lastUpdated time as an Instant. All our times are + * always UTC. + * @return + */ + public Instant getLastUpdatedInstant() { + return Instant.from(lastUpdated.atOffset(ZoneOffset.UTC)); + } + /** * @return the refValues */ diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java similarity index 90% rename from fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java rename to fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java index c3257eb9fce..01fbb5e39bf 100644 --- a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/MessageSerializationTest.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package com.ibm.fhir.remote.index; +package com.ibm.fhir.persistence.helper; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -67,16 +67,16 @@ public void testRoundtrip() throws Exception { adapter.tokenValue("token-param", valueSystem, valueCode, compositeId); sent.setData(adapter.build()); - final String payload = marshallToString(sent); + final String payload = RemoteIndexSupport.marshallToString(sent); // Now unmarshall the payload and check everything matches - RemoteIndexMessage rcvd = unmarshallPayload(payload); + RemoteIndexMessage rcvd = RemoteIndexSupport.unmarshall(payload); assertNotNull(rcvd); assertEquals(rcvd.getMessageVersion(), RemoteIndexConstants.MESSAGE_VERSION); SearchParametersTransport data = rcvd.getData(); assertNotNull(data); assertEquals(data.getParameterHash(), parameterHash); - assertEquals(data.getLastUpdated(), lastUpdated); + assertEquals(data.getLastUpdatedInstant(), lastUpdated); assertEquals(data.getLogicalResourceId(), logicalResourceId); assertEquals(data.getResourceType(), resourceType); assertEquals(data.getLogicalId(), logicalId); @@ -115,17 +115,14 @@ public void testRoundtrip() throws Exception { assertEquals(data.getTokenValues().get(0).getValueCode(), valueCode); } - /** - * Marshall the message to a string - * @param message - * @return - */ - private String marshallToString(RemoteIndexMessage message) { - final Gson gson = new Gson(); - return gson.toJson(message); - } - private RemoteIndexMessage unmarshallPayload(String jsonPayload) throws Exception { - Gson gson = new Gson(); - return gson.fromJson(jsonPayload, RemoteIndexMessage.class); + @Test + public void testInstant() { + Gson gson = RemoteIndexSupport.getGson(); + Instant x = Instant.now(); + String value = gson.toJson(x); + + // now try and convert the other way + Instant x2 = gson.fromJson(value, Instant.class); + assertEquals(x, x2); } } diff --git a/fhir-remote-index/README.md b/fhir-remote-index/README.md index ffbcf835e71..0214b77a358 100644 --- a/fhir-remote-index/README.md +++ b/fhir-remote-index/README.md @@ -47,6 +47,7 @@ To enable remote indexing of search parameters, add the following `remoteIndexSe ... "remoteIndexService": { "type": "kafka", + "instanceIdenfier": "a-random-uuid-value", "kafka": { "mode": "ACTIVE", "topicName": "FHIR_REMOTE_INDEX", @@ -78,7 +79,8 @@ java -Djava.util.logging.config.file=logging.properties \ --database-properties database.properties \ --kafka-properties kafka.properties \ --topic-name FHIR_REMOTE_INDEX \ - --consumer-count 3 + --consumer-count 3 \ + --instance-identifier "a-random-uuid-value" ``` Logging uses standard `java.util.logging` (JUL) and can be configured as follows: @@ -157,6 +159,7 @@ Note: Citus configuration is the same as PostgreSQL. | --db-type {type} | The type of database. One of `postgresql`, `derby`, `db2` or `citus`. | | --database-properties {properties-file} | A Java properties file containing connection details for the downstream IBM FHIR Server database. | | --topic-name {topic} | The name of the Kafka topic to consume. Default `FHIR_REMOTE_INDEX`. | +| --instance-identifier {uuid} | Each IBM FHIR Server cluster should be allocated a unique instance identifier. This identifier is added to each message sent over Kafka. The consumer will ignore messages unless they include the same instance identifier value. This helps to ensure that messages are processed from only intended sources. | | --consumer-group {grp} | Override the default Kafka consumer group (`group.id` value) for this application. Default `remote-index-service-cg`. | | --schema-type {type} | Set the schema type. One of `PLAIN` or `DISTRIBUTED`. Default is `PLAIN`. The schema type `DISTRIBUTED` is for use with Citus databases. | | --max-ready-time-ms {milliseconds} | The maximum number of milliseconds to wait for the database to contain the correct data for a particular set of consumed messages. Should be slightly longer than the configured Liberty transaction timeout value. | @@ -169,4 +172,4 @@ Because messages are sent to Kafka before the transaction is committed, it is po 1. If the resource version doesn't yet exist in the database, the consumer will pause and wait for the transaction to be committed. The consumer will only wait up to the maximum transaction timeout window, at which point it will assume the transaction has failed and the message will be discarded. 2. If the resource version matches, but the lastUpdated time does not match, it assumes the message came from an IBM FHIR Server which failed before the transaction was committed, but the request was processed successfully by another server. The message will be discarded because there will be another message waiting in the queue from the second attempt. -3. If the resource version in the database already exceeds the version in the message, the message will be discarded because the information is already out of date. There will be another message waiting in the queue containing the search parameter values from the most recent resource. \ No newline at end of file +3. If the resource version in the database already exceeds the version in the message, the message will be discarded because the information is already out of date. There will be another message waiting in the queue containing the search parameter values from the most recent resource. diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index 9ca6f29b6d1..a7bce7b6362 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -27,12 +27,15 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.PartitionInfo; +import com.ibm.fhir.core.util.LogSupport; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.derby.DerbyPropertyAdapter; +import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; @@ -89,6 +92,9 @@ public class Main { private IDatabaseTranslator translator; private IConnectionProvider connectionProvider; private DbType dbType = DbType.POSTGRESQL; + + // Make sure we process messages sent from only the FHIR servers we are configured for + private String instanceIdentifier; /** * Parse the given command line arguments @@ -127,6 +133,13 @@ public void parseArgs(String[] args) { throw new IllegalArgumentException("Missing value for --topic-name"); } break; + case "--instance-identifier": + if (a < args.length && !args[a].startsWith("--")) { + instanceIdentifier = args[a++]; + } else { + throw new IllegalArgumentException("Missing value for --instance-identifier"); + } + break; case "--consumer-group": if (a < args.length && !args[a].startsWith("--")) { consumerGroup = args[a++]; @@ -205,7 +218,7 @@ private String getSchemaName() throws FHIRPersistenceException { public void run() throws FHIRPersistenceException { dumpProperties("kafka", kafkaProperties); dumpProperties("database", databaseProperties); - configureForPostgres(); + configureDatabaseAccess(); initIdentityCache(); // Keep track of how many consumers are still running. If too many fail, @@ -298,6 +311,21 @@ private KafkaConsumer buildConsumer() { return consumer; } + /** + * Set up the database connection + */ + private void configureDatabaseAccess() { + switch (this.dbType) { + case POSTGRESQL: + case CITUS: + configureForPostgres(); + case DERBY: + configureForDerby(); + default: + throw new IllegalArgumentException("Database type not supported: " + this.dbType); + } + } + /** * Set things up to talk to a PostgreSQL database */ @@ -313,6 +341,24 @@ private void configureForPostgres() { connectionProvider = new JdbcConnectionProvider(translator, propertyAdapter); } + /** + * Set things up to talk to a Derby database. Note that the in-memory + * instance of Derby supports only a single JVM and so the FHIR server + * instance would need to be stopped before running this fhir-remote-index + * application. Therefore, this is useful only for development work. + */ + private void configureForDerby() { + this.translator = new DerbyTranslator(); + try { + Class.forName(translator.getDriverClassName()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException(e); + } + + DerbyPropertyAdapter propertyAdapter = new DerbyPropertyAdapter(databaseProperties); + connectionProvider = new JdbcConnectionProvider(translator, propertyAdapter); + } + /** * Instantiate a new message handler for use by a consumer thread. Each handler gets * its own database connection. @@ -331,15 +377,15 @@ private IMessageHandler buildHandler() throws FHIRPersistenceException { switch (schemaType) { case SHARDED: - return new ShardedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + return new ShardedPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs); case PLAIN: if (dbType == DbType.DERBY) { - return new PlainDerbyMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + return new PlainDerbyMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs); } else { - return new PlainPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + return new PlainPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs); } case DISTRIBUTED: - return new DistributedPostgresMessageHandler(c, getSchemaName(), identityCache, maxReadyTimeMs); + return new DistributedPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs); default: throw new FHIRPersistenceException("Schema type not supported: " + schemaType.name()); } @@ -395,6 +441,9 @@ protected void dumpProperties(String which, Properties p) { if (key.toLowerCase().contains("password")) { value = "[*******]"; } + // kill any passwords embedded within a more complex value string + value = LogSupport.hidePassword(value); + if (first) { first = false; } else { @@ -405,7 +454,7 @@ protected void dumpProperties(String which, Properties p) { buffer.append("\"").append(value).append("\""); } buffer.append("}"); - logger.info(which + ": " + buffer.toString()); + logger.fine(which + ": " + buffer.toString()); } } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index 4a7f54bc09e..b542af4deb6 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -7,15 +7,16 @@ package com.ibm.fhir.remote.index.database; import java.security.SecureRandom; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import com.google.gson.Gson; import com.ibm.fhir.database.utils.thread.ThreadHandler; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.helper.RemoteIndexSupport; import com.ibm.fhir.persistence.index.DateParameter; import com.ibm.fhir.persistence.index.LocationParameter; import com.ibm.fhir.persistence.index.NumberParameter; @@ -37,6 +38,7 @@ */ public abstract class BaseMessageHandler implements IMessageHandler { private final Logger logger = Logger.getLogger(BaseMessageHandler.class.getName()); + private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; private static final int MIN_SUPPORTED_MESSAGE_VERSION = 1; // If we fail 10 times due to deadlocks, then something is seriously wrong @@ -45,11 +47,18 @@ public abstract class BaseMessageHandler implements IMessageHandler { private final long maxReadyWaitMs; + // Process messages only from a known origin + private final String instanceIdentifier; + /** * Protected constructor * @param maxReadyWaitMs the max time in ms to wait for the upstream transaction to make the data ready */ - protected BaseMessageHandler(long maxReadyWaitMs) { + protected BaseMessageHandler(String instanceIdentifier, long maxReadyWaitMs) { + if (instanceIdentifier == null || instanceIdentifier.isEmpty()) { + throw new IllegalArgumentException("Must specify an instanceIdentifier value"); + } + this.instanceIdentifier = instanceIdentifier; this.maxReadyWaitMs = maxReadyWaitMs; } @@ -60,10 +69,16 @@ public void process(List messages) throws FHIRPersistenceException { if (logger.isLoggable(Level.FINEST)) { logger.finest("Processing message payload: " + payload); } - RemoteIndexMessage message = unmarshall(payload); + RemoteIndexMessage message = RemoteIndexSupport.unmarshall(payload); if (message != null) { if (message.getMessageVersion() >= MIN_SUPPORTED_MESSAGE_VERSION) { - unmarshalled.add(message); + // check to make sure that the instanceIdentifier matches our configuration. This protects us + // from messages accidentally sent over the same topic from another instance + if (this.instanceIdentifier.equals(message.getInstanceIdentifier())) { + unmarshalled.add(message); + } else { + logger.warning("Message from unknown origin, ignoring payload=[" + payload + "]"); + } } else { logger.warning("Message version [" + message.getMessageVersion() + "] not supported, ignoring payload=[" + payload + "]"); } @@ -136,24 +151,6 @@ private void processWithRetry(List messages) throws FHIRPers */ protected abstract void pushBatch() throws FHIRPersistenceException; - /** - * Unmarshall the json payload string into a RemoteIndexMessage - * @param payload - * @return - */ - private RemoteIndexMessage unmarshall(String jsonPayload) { - Gson gson = new Gson(); - try { - return gson.fromJson(jsonPayload, RemoteIndexMessage.class); - } catch (Throwable t) { - // We need to sink this error to avoid poison messages from - // blocking the queues. - // TODO. Perhaps push this to a dedicated error topic - logger.severe("Not a RemoteIndexMessage. Ignoring: '" + jsonPayload + "'"); - } - return null; - } - /** * Process the list of messages * @param messages diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java index 041082adc7c..dd679de3fc2 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java @@ -28,13 +28,15 @@ public class DistributedPostgresMessageHandler extends PlainMessageHandler { /** * Public constructor + * + * @param instanceIdentifier * @param connection * @param schemaName * @param cache * @param maxReadyTimeMs */ - public DistributedPostgresMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs); + public DistributedPostgresMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(instanceIdentifier, new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs); } @Override diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java index 3c7ba2c9234..ad5e36052b6 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java @@ -26,13 +26,14 @@ public class PlainDerbyMessageHandler extends PlainMessageHandler { /** * Public constructor + * @param instanceIdentifier * @param connection * @param schemaName * @param cache * @param maxReadyTimeMs */ - public PlainDerbyMessageHandler(Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(new DerbyTranslator(), connection, schemaName, cache, maxReadyTimeMs); + public PlainDerbyMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(instanceIdentifier, new DerbyTranslator(), connection, schemaName, cache, maxReadyTimeMs); } @Override diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java index fbdf152c39d..5db820a8209 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java @@ -119,13 +119,15 @@ public abstract class PlainMessageHandler extends BaseMessageHandler { /** * Public constructor * + * @param instanceIdentifier + * @param translator * @param connection * @param schemaName * @param cache * @param maxReadyTimeMs */ - public PlainMessageHandler(IDatabaseTranslator translator, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { - super(maxReadyTimeMs); + public PlainMessageHandler(String instanceIdentifier, IDatabaseTranslator translator, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) { + super(instanceIdentifier, maxReadyTimeMs); this.translator = translator; this.connection = connection; this.schemaName = schemaName; @@ -1114,7 +1116,7 @@ protected void checkReady(List messages, List getMessages(long logicalResourceId) { RemoteIndexMessage sent = new RemoteIndexMessage(); sent.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); + sent.setInstanceIdentifier(instanceIdentifier); // Create an Observation resource with a few parameters SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(OBSERVATION, OBSERVATION_LOGICAL_ID, logicalResourceId, @@ -127,23 +129,12 @@ private List getMessages(long logicalResourceId) { adapter.tagValue("tag-param", valueSystem, valueCode, WHOLE_SYSTEM); sent.setData(adapter.build()); - final String payload = marshallToString(sent); - + final String payload = RemoteIndexSupport.marshallToString(sent); final List result = new ArrayList<>(); result.add(payload); return result; } - /** - * Marshall the message to a string - * @param message - * @return - */ - private String marshallToString(RemoteIndexMessage message) { - final Gson gson = new Gson(); - return gson.toJson(message); - } - @Test public void testFill() throws Exception { final long logicalResourceId; @@ -154,7 +145,7 @@ public void testFill() throws Exception { try (Connection c = connectionProvider.getConnection()) { try { - PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(c, SCHEMA_NAME, identityCache, 1000L); + PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(instanceIdentifier, c, SCHEMA_NAME, identityCache, 1000L); handler.process(getMessages(logicalResourceId)); checkData(c, logicalResourceId); c.commit(); diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java index 385a3d671ec..2d96cc375ce 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java @@ -19,8 +19,8 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; -import com.google.gson.Gson; import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.persistence.helper.RemoteIndexSupport; import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.index.IndexProviderResponse; import com.ibm.fhir.persistence.index.RemoteIndexConstants; @@ -41,6 +41,7 @@ public class FHIRRemoteIndexKafkaService extends FHIRRemoteIndexService { private String topicName = null; private Producer producer; private KafkaPropertyAdapter.Mode mode; + private String instanceIdentifier; /** * Default constructor @@ -55,6 +56,7 @@ public FHIRRemoteIndexKafkaService() { public void init(KafkaPropertyAdapter properties) { this.mode = properties.getMode(); this.topicName = properties.getTopicName(); + this.instanceIdentifier = properties.getInstanceIdentifier(); Properties kafkaProps = new Properties(); properties.putPropertiesTo(kafkaProps); @@ -91,17 +93,6 @@ public void shutdown() { } } - /** - * Render the data value to a JSON string which is the wire format we - * use for remote indexing messages - * @param message - * @return - */ - public String marshallToString(RemoteIndexMessage message) { - final Gson gson = new Gson(); - return gson.toJson(message); - } - @Override public IndexProviderResponse submit(final RemoteIndexData data) { // We rely on the default Kafka partitioner, which in our case will @@ -112,7 +103,7 @@ public IndexProviderResponse submit(final RemoteIndexData data) { msg.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); msg.setTenantId(tenantId); msg.setData(data.getSearchParameters()); - final String message = marshallToString(msg); + final String message = RemoteIndexSupport.marshallToString(msg); if (this.mode == Mode.ACTIVE) { ProducerRecord record = new ProducerRecord(topicName, data.getPartitionKey(), message); diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java index 97b3900a97e..6db10342a61 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java @@ -16,6 +16,7 @@ public class KafkaPropertyAdapter { private final Properties properties; private final String topicName; + private final String instanceIdentifier; private final Mode mode; public static enum Mode { @@ -25,10 +26,14 @@ public static enum Mode { /** * Public constructor + * + * @param instanceIdentifier * @param topicName * @param properties + * @param mode */ - public KafkaPropertyAdapter(String topicName, Properties properties, Mode mode) { + public KafkaPropertyAdapter(String instanceIdentifier, String topicName, Properties properties, Mode mode) { + this.instanceIdentifier = instanceIdentifier; this.topicName = topicName; this.properties = properties; this.mode = mode; @@ -61,4 +66,11 @@ public String getTopicName() { public Mode getMode() { return mode; } + + /** + * @return the instanceIdentifier + */ + public String getInstanceIdentifier() { + return instanceIdentifier; + } } diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java index 9e456f3f8c5..98404ebd189 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java @@ -26,6 +26,7 @@ import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TLS_ENABLED; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE_PW; +import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_REMOTE_INDEX_SERVICE_TYPE; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_REGISTRY_RESOURCE_PROVIDER_ENABLED; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_RESOLVE_FUNCTION_ENABLED; @@ -231,6 +232,7 @@ public void contextInitialized(ServletContextEvent event) { if (remoteIndexServiceType != null) { if ("kafka".equals(remoteIndexServiceType)) { String topicName = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME, DEFAULT_KAFKA_INDEX_SERVICE_TOPICNAME); + String instanceIdentifier = fhirConfig.getStringProperty(PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER); String mode = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_MODE, "active"); // Gather up the Kafka connection properties for the async index service @@ -247,7 +249,7 @@ public void contextInitialized(ServletContextEvent event) { log.info("Initializing Kafka async indexing service."); FHIRRemoteIndexKafkaService s = new FHIRRemoteIndexKafkaService(); - s.init(new KafkaPropertyAdapter(topicName, kafkaProps, KafkaPropertyAdapter.Mode.valueOf(mode))); + s.init(new KafkaPropertyAdapter(instanceIdentifier, topicName, kafkaProps, KafkaPropertyAdapter.Mode.valueOf(mode))); // Now the service is ready, we can publish it remoteIndexService = s; FHIRRemoteIndexService.setServiceInstance(remoteIndexService); From 583903ae7bf3422875d32eee49316459441b4f72 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Tue, 21 Jun 2022 08:30:41 +0100 Subject: [PATCH 38/40] issue #3713 use distributed add_any_resource function for citus (#3718) * issue #3713 use distributed add_any_resource function for citus Signed-off-by: Robin Arnold * issue #3437 fix usage of logicalResourceId and resourceId for remote index Signed-off-by: Robin Arnold --- .../database/utils/citus/CitusAdapter.java | 19 +- .../utils/common/PreparedStatementHelper.java | 36 ++++ .../jdbc/citus/CitusResourceDAO.java | 150 ++++++++++++++ .../jdbc/dao/impl/ResourceDAOImpl.java | 8 +- .../jdbc/derby/DerbyResourceDAO.java | 12 +- .../fhir/persistence/jdbc/dto/Resource.java | 40 ++-- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 26 +-- .../jdbc/postgres/PostgresResourceDAO.java | 121 +++++------ .../postgres/PostgresResourceNoProcDAO.java | 8 +- .../java/com/ibm/fhir/schema/app/Main.java | 9 +- .../build/DistributedSchemaAdapter.java | 5 - .../schema/control/FhirSchemaGenerator.java | 78 +++++++ .../main/resources/citus/add_any_resource.sql | 190 ++++++++++++++++++ .../citus/add_logical_resource_ident.sql | 80 ++++++++ 14 files changed, 671 insertions(+), 111 deletions(-) create mode 100644 fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql create mode 100644 fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java index bda0eac3005..82257514882 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -200,10 +200,12 @@ public void distributeFunction(String schemaName, String functionName, int distr throw new IllegalArgumentException("invalid distributeByParamNumber value: " + distributeByParamNumber); } // Need to get the signature text first in order to build the create_distribution_function - // statement + // statement. Note the cast to ::regprocedure will return a string like this: + // "fhirdata.add_logical_resource_ident(integer,character varying)" + // which can be passed in to the Citus create_distributed_function procedure final String objectName = DataDefinitionUtil.getQualifiedName(schemaName, functionName); final String SELECT = - "SELECT p.oid::regproc || '(' || pg_get_function_identity_arguments(p.oid) || ')' " + + "SELECT p.oid::regprocedure " + " FROM pg_catalog.pg_proc p " + " WHERE p.oid::regproc::text = LOWER(?)"; @@ -216,14 +218,23 @@ public void distributeFunction(String schemaName, String functionName, int distr if (rs.next()) { functionSig = rs.getString(1); } + + if (rs.next()) { + final String fn = DataDefinitionUtil.getQualifiedName(schemaName, functionName); + logger.severe("Overloaded function signature: " + fn + " " + functionSig); + functionSig = rs.getString(1); + logger.severe("Overloaded function signature: " + fn + " " + functionSig); + throw new IllegalStateException("Overloading not supported for function '" + fn + "'"); + } } if (functionSig != null) { - final String DISTRIBUTE = "SELECT create_distributed_function(?, ?)"; + logger.info("Distributing function: " + functionSig); + final String DISTRIBUTE = "SELECT create_distributed_function(?::regprocedure, ?::text)"; try (PreparedStatement ps = c.prepareStatement(DISTRIBUTE)) { ps.setString(1, functionSig); ps.setString(2, "$" + distributeByParamNumber); - ps.executeQuery(DISTRIBUTE); + ps.execute(); } } else { logger.warning("No matching function found for '" + objectName + "'"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java index 7be7e257cf1..d281b44af66 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java @@ -6,6 +6,8 @@ package com.ibm.fhir.database.utils.common; +import java.io.InputStream; +import java.sql.CallableStatement; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; @@ -101,6 +103,23 @@ public PreparedStatementHelper setString(String value) throws SQLException { return this; } + /** + * Set the (possibly null) InputStream value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setBinaryStream(InputStream value) throws SQLException { + if (value != null) { + ps.setBinaryStream(index, value); + } else { + ps.setNull(index, Types.BINARY); + } + index++; + return this; + } + /** * Set the (possibly null) int value at the current position * and increment the position by 1 @@ -118,6 +137,23 @@ public PreparedStatementHelper setTimestamp(Timestamp value) throws SQLException return this; } + /** + * Register an OUT parameter, assuming the delegate is a CallableStatement + * @param parameterType from {@link java.sql.Types} + * @return the parameter index of the OUT parameter + * @throws SQLException + */ + public int registerOutParameter(int parameterType) throws SQLException { + int idx = index++; + if (ps instanceof CallableStatement) { + CallableStatement cs = (CallableStatement)ps; + cs.registerOutParameter(idx, parameterType); + } else { + throw new IllegalStateException("Delegate is not a CallableStatement"); + } + return idx; + } + /** * Add a new batch entry based on the current state of the {@link PreparedStatement}. * Note that we don't return this on purpose...because addBatch should be last in diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java index fc2c352abe4..76be6e2c4fb 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -6,22 +6,39 @@ package com.ibm.fhir.persistence.jdbc.citus; +import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Types; import java.util.List; +import java.util.Objects; +import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.persistence.InteractionStatus; import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; +import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.JDBCIdentityCacheImpl; +import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterVisitorBatchDAO; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceDAO; @@ -33,6 +50,14 @@ public class CitusResourceDAO extends PostgresResourceDAO { private static final String CLASSNAME = CitusResourceDAO.class.getName(); private static final Logger log = Logger.getLogger(CLASSNAME); + // @formatter:off + // 0 1 + // 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // @formatter:on + // Don't forget that we must account for IN and OUT parameters. + private static final String SQL_INSERT_WITH_PARAMETERS = "{ CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) }"; + private static final String SQL_LOGICAL_RESOURCE_IDENT = "{ CALL %s.add_logical_resource_ident(?,?,?) }"; + // Read the current version of the resource (even if the resource has been deleted) private static final String SQL_READ = "" + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " @@ -171,4 +196,129 @@ public Resource versionRead(String logicalId, String resourceType, int versionId } + @Override + public Resource insert(Resource resource, List parameters, String parameterHashB64, + ParameterDAO parameterDao, Integer ifNoneMatch) + throws FHIRPersistenceException { + final String METHODNAME = "insert(Resource, List"; + log.entering(CLASSNAME, METHODNAME); + + final Connection connection = getConnection(); // do not close + long dbCallStartTime; + double dbCallDuration; + + try { + // Just make sure this resource type is known to the database before we + // hit the procedure + Integer resourceTypeId = getResourceTypeId(resource.getResourceType()); + Objects.requireNonNull(resourceTypeId); + + // For Citus, we first make a call to establish the logical_resource_ident record + long logicalResourceId = createOrLockLogicalResourceIdent(resourceTypeId, resource.getLogicalId()); + + final String stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); + try (CallableStatement stmt = connection.prepareCall(stmtString)) { + PreparedStatementHelper psh = new PreparedStatementHelper(stmt); + + psh.setLong(logicalResourceId); + psh.setInt(resourceTypeId); + psh.setString(resource.getResourceType()); + psh.setString(resource.getLogicalId()); + psh.setBinaryStream(resource.getDataStream() != null ? resource.getDataStream().inputStream() : null); + psh.setTimestamp(resource.getLastUpdated()); + psh.setString(resource.isDeleted() ? "Y": "N"); + psh.setString(UUID.randomUUID().toString()); + psh.setInt(resource.getVersionId()); + psh.setString(parameterHashB64); + psh.setInt(ifNoneMatch); + psh.setString(resource.getResourcePayloadKey()); + + final int oldParameterHashIndex = psh.registerOutParameter(Types.VARCHAR); + final int interactionStatusIndex = psh.registerOutParameter(Types.INTEGER); + final int ifNoneMatchVersionIndex = psh.registerOutParameter(Types.INTEGER); + + dbCallStartTime = System.nanoTime(); + stmt.execute(); + dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; + + resource.setLogicalResourceId(logicalResourceId); + if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status + // no change, so skip parameter updates + resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); + resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version + } else { + resource.setInteractionStatus(InteractionStatus.MODIFIED); + + // Parameter time + // To keep things simple for the postgresql use-case, we just use a visitor to + // handle inserts of parameters directly in the resource parameter tables. + // Note we don't get any parameters for the resource soft-delete operation + // Bypass the parameter insert here if we have the remoteIndexService configured + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + final String currentParameterHash = stmt.getString(oldParameterHashIndex); + if (remoteIndexService == null + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentParameterHash))) { + // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: + JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); + try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getLogicalResourceId(), 100, + identityCache, getResourceReferenceDAO(), getTransactionData())) { + for (ExtractedParameterValue p: parameters) { + p.accept(pvd); + } + } + } + } + if (log.isLoggable(Level.FINE)) { + log.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); + } + } + } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { + throw e; + } catch(SQLIntegrityConstraintViolationException e) { + FHIRPersistenceFKVException fx = new FHIRPersistenceFKVException("Encountered FK violation while inserting Resource."); + throw severe(log, fx, e); + } catch(SQLException e) { + if (FHIRDAOConstants.SQLSTATE_WRONG_VERSION.equals(e.getSQLState())) { + // this is just a concurrency update, so there's no need to log the SQLException here + throw new FHIRPersistenceVersionIdMismatchException("Encountered version id mismatch while inserting Resource"); + } else { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("SQLException encountered while inserting Resource."); + throw severe(log, fx, e); + } + } catch(Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure inserting Resource."); + throw severe(log, fx, e); + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + + return resource; + } + + /** + * Call the ADD_LOGICAL_RESOURCE_IDENT procedure to create or lock (select for update) + * the logical_resource_ident record. For Citus we run this step first because this + * function is distributed by the logical_id parameter. + * @param resourceTypeId + * @param logicalId + * @return + * @throws SQLException + */ + protected long createOrLockLogicalResourceIdent(int resourceTypeId, String logicalId) throws SQLException { + long logicalResourceId; + + final String stmtString = String.format(SQL_LOGICAL_RESOURCE_IDENT, getSchemaName()); + try (CallableStatement cs = getConnection().prepareCall(stmtString)) { + PreparedStatementHelper psh = new PreparedStatementHelper(cs); + psh.setInt(resourceTypeId); + psh.setString(logicalId); + int idxLogicalResourceId = psh.registerOutParameter(Types.BIGINT); + cs.execute(); + logicalResourceId = cs.getLong(idxLogicalResourceId); + } + + // At this point the logical_resource_ident record will be locked for update + return logicalResourceId; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index 5833512dc9d..7645db9f9ad 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -279,7 +279,7 @@ protected Resource createDTO(ResultSet resultSet, boolean hasResourceTypeId) thr if (payloadData != null) { resource.setDataStream(new InputOutputByteStream(payloadData, payloadData.length)); } - resource.setId(resultSet.getLong(IDX_RESOURCE_ID)); + resource.setResourceId(resultSet.getLong(IDX_RESOURCE_ID)); resource.setLogicalResourceId(resultSet.getLong(IDX_LOGICAL_RESOURCE_ID)); resource.setLastUpdated(resultSet.getTimestamp(IDX_LAST_UPDATED, CalendarHelper.getCalendarForUTC())); resource.setLogicalId(resultSet.getString(IDX_LOGICAL_ID)); @@ -541,7 +541,7 @@ public Resource insert(Resource resource, List paramete long latestTime = System.nanoTime(); double dbCallDuration = (latestTime-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(10)); + resource.setLogicalResourceId(stmt.getLong(10)); final long versionedResourceRowId = stmt.getLong(11); final String currentHash = stmt.getString(12); final int interactionStatus = stmt.getInt(13); @@ -580,7 +580,7 @@ public Resource insert(Resource resource, List paramete || !parameterHashB64.equals(currentHash))) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(cache, this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", resource.getResourceType(), true, - resource.getId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { + resource.getLogicalResourceId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { for (ExtractedParameterValue p: parameters) { p.accept(pvd); } @@ -591,7 +591,7 @@ public Resource insert(Resource resource, List paramete latestTime = System.nanoTime(); double totalDuration = (latestTime - dbCallStartTime) / 1e6; double paramInsertDuration = (latestTime-paramInsertStartTime)/1e6; - log.fine("Successfully inserted Resource. id=" + resource.getId() + " total=" + totalDuration + "ms, proc=" + dbCallDuration + "ms, param=" + paramInsertDuration + "ms"); + log.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " total=" + totalDuration + "ms, proc=" + dbCallDuration + "ms, param=" + paramInsertDuration + "ms"); } } } catch (FHIRPersistenceDBConnectException | diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index 9e13a2f1714..b157878c475 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -108,7 +108,7 @@ public Resource insert(Resource resource, List paramete AtomicInteger outInteractionStatus = new AtomicInteger(); AtomicInteger outIfNoneMatchVersion = new AtomicInteger(); - long resourceId = this.storeResource(resource.getResourceType(), + long logicalResourceId = this.storeResource(resource.getResourceType(), parameters, resource.getLogicalId(), resource.getDataStream() != null ? resource.getDataStream().inputStream() : null, @@ -132,11 +132,11 @@ public Resource insert(Resource resource, List paramete resource.setIfNoneMatchVersion(outIfNoneMatchVersion.get()); } else { resource.setInteractionStatus(InteractionStatus.MODIFIED); - resource.setId(resourceId); + resource.setLogicalResourceId(logicalResourceId); } if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; @@ -199,7 +199,7 @@ public Resource insert(Resource resource, List paramete * @param resourcePayloadKey * @param outInteractionStatus * @param outIfNoneMatchVersion - * @return the resource_id for the entry we created + * @return the logical_resource_id for the entry we created * @throws Exception */ public long storeResource(String tablePrefix, List parameters, @@ -299,7 +299,7 @@ public long storeResource(String tablePrefix, List para // insert the logical_resource_ident record (which we now do our locking on) final String INS_IDENT = "INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?, ?, ?)"; if (logger.isLoggable(Level.FINEST)) { - logger.finest("Creating new logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Creating new logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id + " => logical_resource_id=" + v_logical_resource_id); } try (PreparedStatement stmt = conn.prepareStatement(INS_IDENT)) { @@ -577,7 +577,7 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { } logger.exiting(CLASSNAME, METHODNAME); - return v_resource_id; + return v_logical_resource_id; } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java index 7cbd092be82..6602ad59dbd 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2021 + * (C) Copyright IBM Corp. 2017, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,14 +17,14 @@ public class Resource { /** - * This is the _RESOURCES.RESOURCE_ID column + * This is the _RESOURCES.RESOURCE_ID column. It is unique for a specific version + * of a resource. It is not used during create/update interactions. */ - private long id; + private long resourceId; /** - * This is the _LOGICAL_RESOURCES.LOGICAL_RESOURCE_ID column. It is only - * set when this DTO is used to read table data. It is not set when the DTO is - * used to insert/update. + * This is the _LOGICAL_RESOURCES.LOGICAL_RESOURCE_ID column. It is used during + * create/update interactions as well as read interactions */ private long logicalResourceId; @@ -114,18 +114,34 @@ public Integer getIfNoneMatchVersion() { return this.ifNoneMatchVersion; } - public long getId() { - return id; + /** + * Getter for the database xx_resources.resource_id value + * @return + */ + public long getResourceId() { + return resourceId; } - public void setId(long id) { - this.id = id; + /** + * Setter for the database xx_resources.resource_id value + * @param id + */ + public void setResourceId(long id) { + this.resourceId = id; } + /** + * Getter for the logical_resources.logical_resource_id value + * @return + */ public long getLogicalResourceId() { return logicalResourceId; } - + + /** + * Setter for the logical_resources.logical_resource_id value + * @param logicalResourceId + */ public void setLogicalResourceId(long logicalResourceId) { this.logicalResourceId = logicalResourceId; } @@ -156,7 +172,7 @@ public void setDeleted(boolean deleted) { @Override public String toString() { - return "Resource [id=" + id + ", logicalResourceId=" + logicalResourceId + ", logicalId=" + logicalId + + return "Resource [id=" + resourceId + ", logicalResourceId=" + logicalResourceId + ", logicalId=" + logicalId + ", versionId=" + versionId + ", resourceType=" + resourceType + ", lastUpdated=" + lastUpdated + ", deleted=" + deleted + "]"; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 80be6df1200..c3da80886fe 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -421,12 +421,12 @@ public SingleResourceResult create(FHIRPersistenceContex ExtractedSearchParameters searchParameters = this.extractSearchParameters(updatedResource, resourceDTO); resourceDao.insert(resourceDTO, searchParameters.getParameters(), searchParameters.getParameterHashB64(), parameterDao, context.getIfNoneMatch()); if (log.isLoggable(Level.FINE)) { - log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getLogicalResourceId(), resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); } SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() @@ -674,17 +674,17 @@ public SingleResourceResult update(FHIRPersistenceContex if (log.isLoggable(Level.FINE)) { if (resourceDTO.getInteractionStatus() == InteractionStatus.IF_NONE_MATCH_EXISTED) { - log.fine("If-None-Match: Existing FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("If-None-Match: Existing FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } else { - log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } } // If configured, send the extracted parameters to the remote indexing service if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { - sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getId(), + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getLogicalResourceId(), resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); } @@ -886,7 +886,7 @@ private List newSearchForIncludeReso List allIncludeResources = new ArrayList<>(); // Used for de-duplication - Set allResourceIds = resourceDTOList.stream().map(r -> r.getId()).collect(Collectors.toSet()); + Set allResourceIds = resourceDTOList.stream().map(r -> r.getResourceId()).collect(Collectors.toSet()); // This is a map of iterations to query results. The query results is a map of // search resource type to returned logical resource IDs. The logical resource IDs @@ -909,7 +909,7 @@ private List newSearchForIncludeReso baseLogicalResourceIds, queryResultMap, resourceDao, 1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(includeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(includeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(includeResources); @@ -931,7 +931,7 @@ private List newSearchForIncludeReso baseLogicalResourceIds, queryResultMap, resourceDao, 1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(revincludeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(revincludeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(revincludeResources); @@ -975,7 +975,7 @@ private List newSearchForIncludeReso SearchConstants.INCLUDE, queryIds, queryResultMap, resourceDao, i+1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(includeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(includeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(includeResources); @@ -1000,7 +1000,7 @@ private List newSearchForIncludeReso SearchConstants.REVINCLUDE, queryIds, queryResultMap, resourceDao, i+1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(revincludeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(revincludeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(revincludeResources); @@ -1053,7 +1053,7 @@ private List runIncludeQuery(Class includeDTOs = - resourceDao.search(includeQuery).stream().filter(r -> !allResourceIds.contains(r.getId())).collect(Collectors.toList()); + resourceDao.search(includeQuery).stream().filter(r -> !allResourceIds.contains(r.getResourceId())).collect(Collectors.toList()); // Add query result to map. // The logical resource IDs are pulled from the returned DTOs and saved in a @@ -1146,7 +1146,7 @@ public void delete(FHIRPersistenceContext context, Class resourceDao.insert(resourceDTO, null, null, null, IF_NONE_MATCH_NULL); if (log.isLoggable(Level.FINE)) { - log.fine("Deleted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Deleted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } } catch(FHIRPersistenceException e) { @@ -1484,7 +1484,7 @@ protected List buildSortedResourceDT // Store each ResourceDTO in its proper position in the returned sorted list. for (com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTO : resourceDTOList) { - sortIndex = idPositionMap.get(resourceDTO.getId()); + sortIndex = idPositionMap.get(resourceDTO.getResourceId()); sortedResourceDTOs[sortIndex] = resourceDTO; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index bf39a82b4d4..8c1fa2700e8 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -107,7 +107,6 @@ public Resource insert(Resource resource, List paramete logger.entering(CLASSNAME, METHODNAME); final Connection connection = getConnection(); // do not close - CallableStatement stmt = null; String stmtString = null; Timestamp lastUpdated; long dbCallStartTime; @@ -126,70 +125,72 @@ public Resource insert(Resource resource, List paramete } else { stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); } - stmt = connection.prepareCall(stmtString); - int arg = 1; - if (getFlavor().getSchemaType() == SchemaType.SHARDED) { - stmt.setShort(arg++, shardKey); - } - stmt.setString(arg++, resource.getResourceType()); - stmt.setString(arg++, resource.getLogicalId()); - - if (resource.getDataStream() != null) { - stmt.setBinaryStream(arg++, resource.getDataStream().inputStream()); - } else { - // payload was offloaded to another data store - stmt.setNull(arg++, Types.BINARY); - } - - lastUpdated = resource.getLastUpdated(); - stmt.setTimestamp(arg++, lastUpdated, CalendarHelper.getCalendarForUTC()); - stmt.setString(arg++, resource.isDeleted() ? "Y": "N"); - stmt.setString(arg++, UUID.randomUUID().toString()); - stmt.setInt(arg++, resource.getVersionId()); - stmt.setString(arg++, parameterHashB64); - setInt(stmt, arg++, ifNoneMatch); - setString(stmt, arg++, resource.getResourcePayloadKey()); - - // TODO use a helper function which can return the arg index to help clean up the syntax - stmt.registerOutParameter(arg, Types.BIGINT); final int resourceIdIndex = arg++; - stmt.registerOutParameter(arg, Types.VARCHAR); final int oldParameterHashIndex = arg++; - stmt.registerOutParameter(arg, Types.INTEGER); final int interactionStatusIndex = arg++; - stmt.registerOutParameter(arg, Types.INTEGER); final int ifNoneMatchVersionIndex = arg++; - - dbCallStartTime = System.nanoTime(); - stmt.execute(); - dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(resourceIdIndex)); - if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status - // no change, so skip parameter updates - resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); - resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version - } else { - resource.setInteractionStatus(InteractionStatus.MODIFIED); + try (CallableStatement stmt = connection.prepareCall(stmtString)) { + int arg = 1; + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { + stmt.setShort(arg++, shardKey); + } + stmt.setString(arg++, resource.getResourceType()); + stmt.setString(arg++, resource.getLogicalId()); + + if (resource.getDataStream() != null) { + stmt.setBinaryStream(arg++, resource.getDataStream().inputStream()); + } else { + // payload was offloaded to another data store + stmt.setNull(arg++, Types.BINARY); + } + + lastUpdated = resource.getLastUpdated(); + stmt.setTimestamp(arg++, lastUpdated, CalendarHelper.getCalendarForUTC()); + stmt.setString(arg++, resource.isDeleted() ? "Y": "N"); + stmt.setString(arg++, UUID.randomUUID().toString()); + stmt.setInt(arg++, resource.getVersionId()); + stmt.setString(arg++, parameterHashB64); + setInt(stmt, arg++, ifNoneMatch); + setString(stmt, arg++, resource.getResourcePayloadKey()); + + // TODO use a helper function which can return the arg index to help clean up the syntax + stmt.registerOutParameter(arg, Types.BIGINT); final int logicalResourceIdIndex = arg++; + stmt.registerOutParameter(arg, Types.VARCHAR); final int oldParameterHashIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int interactionStatusIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int ifNoneMatchVersionIndex = arg++; + + dbCallStartTime = System.nanoTime(); + stmt.execute(); + dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; - // Parameter time - // To keep things simple for the postgresql use-case, we just use a visitor to - // handle inserts of parameters directly in the resource parameter tables. - // Note we don't get any parameters for the resource soft-delete operation - // Bypass the parameter insert here if we have the remoteIndexService configured - FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); - final String currentParameterHash = stmt.getString(oldParameterHashIndex); - if (remoteIndexService == null - && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() - || !parameterHashB64.equals(currentParameterHash))) { - // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: - JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); - try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getId(), 100, - identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { - p.accept(pvd); + resource.setLogicalResourceId(stmt.getLong(logicalResourceIdIndex)); + if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status + // no change, so skip parameter updates + resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); + resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version + } else { + resource.setInteractionStatus(InteractionStatus.MODIFIED); + + // Parameter time + // To keep things simple for the postgresql use-case, we just use a visitor to + // handle inserts of parameters directly in the resource parameter tables. + // Note we don't get any parameters for the resource soft-delete operation + // Bypass the parameter insert here if we have the remoteIndexService configured + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + final String currentParameterHash = stmt.getString(oldParameterHashIndex); + if (remoteIndexService == null + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentParameterHash))) { + // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: + JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); + try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getLogicalResourceId(), 100, + identityCache, getResourceReferenceDAO(), getTransactionData())) { + for (ExtractedParameterValue p: parameters) { + p.accept(pvd); + } } } } - } - if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + if (logger.isLoggable(Level.FINE)) { + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); + } } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java index 34fde9eeb23..bff777ad2ec 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java @@ -100,7 +100,7 @@ public Resource insert(Resource resource, List paramete AtomicInteger outInteractionStatus = new AtomicInteger(); AtomicInteger outIfNoneMatchVersion = new AtomicInteger(); - long resourceId = this.storeResource(resource.getResourceType(), + long logicalResourceId = this.storeResource(resource.getResourceType(), parameters, resource.getLogicalId(), resource.getDataStream().inputStream(), @@ -125,11 +125,11 @@ public Resource insert(Resource resource, List paramete resource.setIfNoneMatchVersion(outIfNoneMatchVersion.get()); } else { resource.setInteractionStatus(InteractionStatus.MODIFIED); - resource.setId(resourceId); + resource.setLogicalResourceId(logicalResourceId); } if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; @@ -459,7 +459,7 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { } logger.exiting(CLASSNAME, METHODNAME); - return v_resource_id; + return v_logical_resource_id; } /** diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 2d738a0821e..e443fecd44d 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -533,7 +533,7 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { gen.buildDatabaseSpecificArtifactsPostgres(pdm); break; case CITUS: - gen.buildDatabaseSpecificArtifactsPostgres(pdm); + gen.buildDatabaseSpecificArtifactsCitus(pdm); break; default: throw new IllegalStateException("Unsupported db type: " + dbType); @@ -764,8 +764,11 @@ protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) { } /** - * Apply any table distribution rules in one transaction and then add all the - * FK constraints that are needed + * Apply any table distribution rules then add all the + * FK constraints that are needed. Applying all the distribution rules + * in one transaction causes issues with Citus/PostgreSQL (out of shared memory + * errors) so instead we provide a function to allow the visitor to break + * things up into smaller transactions. * @param pdm */ private void applyDistributionRules(PhysicalDataModel pdm, SchemaType schemaType) { diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java index a56e0b41908..237fc96a4e1 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java @@ -79,9 +79,4 @@ public void applyDistributionRules(String schemaName, String tableName, Distribu DistributionContext dc = createContext(distributionType, distributionColumnName); databaseAdapter.applyDistributionRules(schemaName, tableName, dc); } - - @Override - public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { - // NOP for now - } } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java index 5ffd17e61c4..2dd8904f59b 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java @@ -142,6 +142,7 @@ public class FhirSchemaGenerator { private static final String ADD_PARAMETER_NAME = "ADD_PARAMETER_NAME"; private static final String ADD_RESOURCE_TYPE = "ADD_RESOURCE_TYPE"; private static final String ADD_ANY_RESOURCE = "ADD_ANY_RESOURCE"; + private static final String ADD_LOGICAL_RESOURCE_IDENT = "ADD_LOGICAL_RESOURCE_IDENT"; // Special procedure for Citus database support private static final String ADD_LOGICAL_RESOURCE = "ADD_LOGICAL_RESOURCE"; @@ -545,6 +546,83 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) { fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); } + /** + * Add stored procedures/functions for Citus (largely based on PostgreSQL, but some functions are distributed + * based on a parameter to make them more efficient. + * @implNote https://docs.microsoft.com/en-us/azure/postgresql/hyperscale/reference-functions#create_distributed_function + * @param model + */ + public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) { + // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name. + final String ROOT_DIR = "postgres/"; + final String CITUS_ROOT_DIR = "citus/"; + FunctionDef fd = model.addFunction(this.schemaName, + ADD_CODE_SYSTEM, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null), + Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete), + procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_PARAMETER_NAME, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ADD_RESOURCE_TYPE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase() + + ".sql", null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // We currently only support functions with PostgreSQL, although this is really just a procedure + final String deleteResourceParametersScript; + final String addAnyResourceScript; + final String eraseResourceScript; + final String schemaTypeSuffix = getSchemaTypeSuffix(); + addAnyResourceScript = CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + schemaTypeSuffix; + deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql"; + eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql"; + final String addLogicalResourceIdentScript = CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE_IDENT.toLowerCase() + ".sql"; + + FunctionDef deleteResourceParameters = model.addFunction(this.schemaName, + DELETE_RESOURCE_PARAMETERS, + FhirSchemaVersion.V0020.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, deleteResourceParametersScript, null), + Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), + procedurePrivileges, 2); // distributed by p_logical_resource_id + deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // For Citus, we use an additional function to create/lock the logical_resource_ident record + // Function is distributed by p_logical_id (parameter 2) + fd = model.addFunction(this.schemaName, + ADD_LOGICAL_RESOURCE_IDENT, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addLogicalResourceIdentScript, null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges, 2); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + // Function is distributed by p_logical_resource_id (parameter 1) + fd = model.addFunction(this.schemaName, + ADD_ANY_RESOURCE, + FhirSchemaVersion.V0001.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addAnyResourceScript, null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges, 1); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + + fd = model.addFunction(this.schemaName, + ERASE_RESOURCE, + FhirSchemaVersion.V0013.vid(), + () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, eraseResourceScript, null), + Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges); + fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP); + } + /** * Get the suffix to select the appropriate procedure/function script * for the schema type diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql new file mode 100644 index 00000000000..5c719b387cf --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql @@ -0,0 +1,190 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2020, 2022 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to add a resource version and its associated parameters. These +-- parameters only ever point to the latest version of a resource, never to +-- previous versions, which are kept to support history queries. +-- From V0027, we now use a logical_resource_ident table for locking. Records +-- can be created in this table either by this procedure, or as part of +-- reference parameter processing. +-- +-- This variant is for use with the Citus distributed variant of the schema. +-- This function is distributed by logical_resource_id (the first parameter) +-- because all SQL/DML it executes uses logical_resource_id. The +-- logical_resource_ident record must already exist and be locked for update +-- before this function is called. +-- +-- Because this function is distributed all object names must be fully +-- qualified. +-- +-- implNote - Conventions: +-- p_... prefix used to represent input parameters +-- v_... prefix used to represent declared variables +-- t_... prefix used to represent temp variables +-- o_... prefix used to represent output parameters +-- Parameters: +-- p_logical_resource_id: the logical_resource_ident primary key value for the resource +-- p_resource_type_id: the resource_type_id from resource_types +-- p_resource_type: the resource type name +-- p_logical_id: the logical id given to the resource by the FHIR server +-- p_payload: the BLOB (of JSON) which is the resource content +-- p_last_updated the last_updated time given by the FHIR server +-- p_is_deleted: the soft delete flag +-- p_version_id: the intended new version id of the resource (matching the JSON payload) +-- p_parameter_hash_b64 the Base64 encoded hash of parameter values +-- p_if_none_match the encoded If-None-Match value +-- o_current_parameter_hash: Base64 current parameter hash if existing resource +-- o_interaction_status: output indicating whether a change was made or IfNoneMatch hit +-- o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch) +-- Exceptions: +-- SQLSTATE 99001: on version conflict (concurrency) +-- SQLSTATE 99002: missing expected row (data integrity) +-- SQLSTATE 99004: delete a currently deleted resource (data integrity) +-- ---------------------------------------------------------------------------- + ( IN p_logical_resource_id BIGINT, + IN p_resource_type_id INT, + IN p_resource_type VARCHAR( 36), + IN p_logical_id VARCHAR(255), + IN p_payload BYTEA, + IN p_last_updated TIMESTAMP, + IN p_is_deleted CHAR( 1), + IN p_source_key VARCHAR( 64), + IN p_version INT, + IN p_parameter_hash_b64 VARCHAR( 44), + IN p_if_none_match INT, + IN p_resource_payload_key VARCHAR( 36), + OUT o_current_parameter_hash VARCHAR( 44), + OUT o_interaction_status INT, + OUT o_if_none_match_version INT) + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + t_logical_resource_id BIGINT := NULL; + v_current_resource_id BIGINT := NULL; + v_resource_id BIGINT := NULL; + v_currently_deleted CHAR(1) := NULL; + v_new_resource INT := 0; + v_duplicate INT := 0; + v_current_version INT := 0; + v_ghost_resource INT := 0; + v_change_type CHAR(1) := NULL; + +BEGIN + -- default value unless we hit If-None-Match + o_interaction_status := 0; + + -- LOADED ON: {{DATE}} + v_schema_name := '{{SCHEMA_NAME}}'; + + -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later) + SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id; + + -- Read the record from logical_resources to see if this is an existing resource + SELECT logical_resource_id, parameter_hash, is_deleted + INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted + FROM {{SCHEMA_NAME}}.logical_resources + WHERE logical_resource_id = p_logical_resource_id; + IF (t_logical_resource_id IS NULL) + THEN + v_new_resource := 1; + -- we already own the lock on the ident record, so we can safely create + -- the corresponding records in the logical_resources and resource-type-specific + -- xx_logical_resources tables + INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) + VALUES (p_logical_resource_id, p_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING; + + EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) ' + || ' VALUES ($1, $2, $3, $4, $5, $6)' USING p_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id; + + -- Since the resource did not previously exist, make sure o_current_parameter_hash is null + o_current_parameter_hash := NULL; + ELSE + -- as this is an existing resource, we need to know the current resource id. + -- This is only available at the resource-specific logical_resources level + EXECUTE + 'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources ' + || ' WHERE logical_resource_id = $1 ' + INTO v_current_resource_id, v_current_version USING p_logical_resource_id; + + IF v_current_resource_id IS NULL OR v_current_version IS NULL + THEN + -- our concurrency protection means that this shouldn't happen + RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002'; + END IF; + + -- If-None-Match does not apply if the resource is currently deleted + IF v_currently_deleted = 'N' AND p_if_none_match = 0 + THEN + -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the + -- connection with a fatal error, so instead we use an out parameter to + -- indicate the match + o_interaction_status := 1; + o_if_none_match_version := v_current_version; + RETURN; + END IF; + + -- Concurrency check: + -- the version parameter we've been given (which is also embedded in the JSON payload) must be + -- one greater than the current version, otherwise we've hit a concurrent update race condition + IF p_version != v_current_version + 1 + THEN + RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001'; + END IF; + + -- Prevent creating a new deletion marker if the resource is currently deleted + IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y' + THEN + RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004'; + END IF; + + IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash + THEN + -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure) + -- TODO patch parameter sets instead of all delete/all insert. + EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)' + USING p_resource_type, p_logical_resource_id; + END IF; -- end if check parameter hash + END IF; -- end if existing resource + + -- create the new resource version entry in xx_resources + EXECUTE + 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) ' + || ' VALUES ($1, $2, $3, $4, $5, $6, $7)' + USING v_resource_id, p_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key; + + IF v_new_resource = 0 THEN + -- As this is an existing logical resource, we need to update the xx_logical_resource values to match + -- the values of the current resource. For new resources, these are added by the insert so we don't + -- need to update them here. + EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5' + USING v_resource_id, p_is_deleted, p_last_updated, p_version, p_logical_resource_id; + + -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level + EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4' + USING p_is_deleted, p_last_updated, p_parameter_hash_b64, p_logical_resource_id; + END IF; + + -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event + -- related to resources changes (issue-1955) + IF p_is_deleted = 'Y' + THEN + v_change_type := 'D'; + ELSE + IF v_new_resource = 0 AND v_currently_deleted = 'N' + THEN + v_change_type := 'U'; + ELSE + v_change_type := 'C'; + END IF; + END IF; + + INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type) + VALUES (v_resource_id, p_last_updated, p_resource_type_id, p_logical_resource_id, p_version, v_change_type); + +END $$; diff --git a/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql new file mode 100644 index 00000000000..4ea33574c96 --- /dev/null +++ b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql @@ -0,0 +1,80 @@ +------------------------------------------------------------------------------- +-- (C) Copyright IBM Corp. 2022 +-- +-- SPDX-License-Identifier: Apache-2.0 +------------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- Procedure to either create or select for update a logical_resource_ident +-- record. For Citus, this part of the "add_any_resource" logic is split +-- off into its own function here which allows us to distribute the function +-- on logical_id, which is used in all the SQL/DML executed by this +-- function. This allows Citus to push execution of the entire function +-- down to the worker node. +-- +-- implNote - Conventions: +-- p_... prefix used to represent input parameters +-- v_... prefix used to represent declared variables +-- t_... prefix used to represent temp variables +-- o_... prefix used to represent output parameters +-- Parameters: +-- p_resource_type_id: the resource type id from resource_types +-- p_logical_id: the logical id given to the resource by the FHIR server +-- o_logical_resource_id: output field returning the newly assigned logical_resource_id value +-- +-- ---------------------------------------------------------------------------- + ( IN p_resource_type_id INT, + IN p_logical_id VARCHAR(64), + OUT o_logical_resource_id BIGINT) + LANGUAGE plpgsql + AS $$ + + DECLARE + v_schema_name VARCHAR(128); + v_logical_resource_id BIGINT := NULL; + t_logical_resource_id BIGINT := NULL; + + -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. + lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(1024)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE; + +BEGIN + + -- LOADED ON: {{DATE}} + v_schema_name := '{{SCHEMA_NAME}}'; + + -- Get a lock on the logical resource identity record + OPEN lock_cur(t_resource_type_id := p_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO v_logical_resource_id; + CLOSE lock_cur; + + -- Create the resource ident record if we don't have it already + IF v_logical_resource_id IS NULL + THEN + -- allocate the new logical_resource_id value + SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id; + + -- remember that we have a concurrent system...so there is a possibility + -- that another thread snuck in before us and created the ident record. To + -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn + -- around and read again to check that the logical_resource_id in the table + -- matches the value we tried to insert. + INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id) + VALUES (p_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING; + + -- Do a read so that we can verify that *we* did the insert + OPEN lock_cur(t_resource_type_id := p_resource_type_id, t_logical_id := p_logical_id); + FETCH lock_cur INTO t_logical_resource_id; + CLOSE lock_cur; + + IF v_logical_resource_id != t_logical_resource_id + THEN + -- logical_resource_ident record was created by another thread...so use that id instead + v_logical_resource_id := t_logical_resource_id; + END IF; + END IF; + + -- Hand back the id of the logical resource we created earlier. In the new R4 schema + -- only the logical_resource_id is the target of any FK, so there's no need to return + -- the resource_id (which is now private to the _resources tables). + o_logical_resource_id := v_logical_resource_id; +END $$; From 6c8d712da9e04af48f42e8454077827f01179d81 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Tue, 21 Jun 2022 10:08:58 +0100 Subject: [PATCH 39/40] issue #3437 inject instanceIdentifier when sending remote index payload Signed-off-by: Robin Arnold --- fhir-remote-index/pom.xml | 15 ++++++++++----- .../java/com/ibm/fhir/remote/index/app/Main.java | 2 ++ .../remote/index/database/BaseMessageHandler.java | 5 ++++- .../index/kafka/FHIRRemoteIndexKafkaService.java | 1 + 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml index f8271ea09bc..aa9e079b7a6 100644 --- a/fhir-remote-index/pom.xml +++ b/fhir-remote-index/pom.xml @@ -51,18 +51,28 @@ com.google.code.gson gson + org.apache.derby derby + true + + + org.apache.derby + derbytools + true com.ibm.db2 jcc + true org.postgresql postgresql + true + org.apache.kafka kafka-clients @@ -76,11 +86,6 @@ testng test - - org.apache.derby - derbytools - test - ${project.groupId} fhir-persistence-schema diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java index a7bce7b6362..03e82148cae 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java @@ -319,8 +319,10 @@ private void configureDatabaseAccess() { case POSTGRESQL: case CITUS: configureForPostgres(); + break; case DERBY: configureForDerby(); + break; default: throw new IllegalArgumentException("Database type not supported: " + this.dbType); } diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java index b542af4deb6..059153ccfdf 100644 --- a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java +++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java @@ -84,7 +84,10 @@ public void process(List messages) throws FHIRPersistenceException { } } } - processWithRetry(unmarshalled); + + if (unmarshalled.size() > 0) { + processWithRetry(unmarshalled); + } } /** diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java index 2d96cc375ce..2a2a2a21e84 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java @@ -101,6 +101,7 @@ public IndexProviderResponse submit(final RemoteIndexData data) { final String tenantId = FHIRRequestContext.get().getTenantId(); RemoteIndexMessage msg = new RemoteIndexMessage(); msg.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION); + msg.setInstanceIdentifier(this.instanceIdentifier); msg.setTenantId(tenantId); msg.setData(data.getSearchParameters()); final String message = RemoteIndexSupport.marshallToString(msg); From 59e3c2477030fab5ebe143762d6d6c0b4b6ca610 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Tue, 21 Jun 2022 14:37:13 +0100 Subject: [PATCH 40/40] issue #3437 more schema design documentation Signed-off-by: Robin Arnold --- fhir-persistence-schema/docs/SchemaDesign.md | 60 +++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/fhir-persistence-schema/docs/SchemaDesign.md b/fhir-persistence-schema/docs/SchemaDesign.md index e464cc8cd27..6b138646f94 100644 --- a/fhir-persistence-schema/docs/SchemaDesign.md +++ b/fhir-persistence-schema/docs/SchemaDesign.md @@ -491,8 +491,17 @@ The DDL for most objects (like tables) is specified once. Changes to the table s The schema update utility first reads the VERSION_HISTORY table, loading all records for the target schema (e.g. FHIRDATA). The utility only applies changes which have a version number greater than the currently recorded version. Once the DDL has been applied successfully, the version number is updated in VERSION_HISTORY. This makes the processed idempotent. Subsequent runs of the schema update utility only apply changes which have a greater version id value than the most recently stored value for each object. +### Schema Version V0027 +A new `logical_resource_ident` table has been added. This table is now the primary owner of the `(resource_type_id, logical_id) to (logical_resource_id)` mapping. During ingestion, the SELECT FOR UPDATE lock is now obtained on this record instead of the `logical_resources` record. This change supports more data efficient distribution when using Citus, and more efficient handling of reference search parameter values in general. +Schema version V0027 changes the way reference search parameter values are stored. Prior to V0027, reference parameters were treated as tokens and stored in the `common_token_values` table, with the `xx_resource_token_refs` providing the many-to-many mapping between the logical resource record in `xx_logical_resources` and `common_token_values`. + +As of schema version V0027, the reference mapping is now stored in `xx_ref_values` with the normalized referenced value stored in the `logical_resource_ident` table. This is useful, because it reduces the size of the `common_token_values` table, allowing it to be treated as a REFERENCE table when using the distributed schema in Citus. This arrangement also makes it more efficient to store local references (references between resources both stored within the same IBM FHIR Server database). Each logical_resource_ident record includes a foreign key referencing the resource type. Where the reference is an external reference (perhaps a URL pointing to another FHIR server), the target of the resource may not be known, in which case the `logical_resource_ident` record is assigned the `resource_type_id` for the `Resource` resource type. + +During schema migration, the schema tool will check to see if the `logical_resource_ident` table is empty, and if so, will populate the table with each record from the `logical_resources` table. + +Following migration, these changes require the search parameter data to be reindexed before any FHIR searches are run. ## Managing Resource Tables @@ -517,6 +526,7 @@ VISIONPRESCRIPTION_DATE_VALUES VISIONPRESCRIPTION_STR_VALUES VISIONPRESCRIPTION_PROFILES VISIONPRESCRIPTION_RESOURCE_TOKEN_REFS +VISIONPRESCRIPTION_REF_VALUES VISIONPRESCRIPTION_TAGS VISIONPRESCRIPTION_SECURITY VISIONPRESCRIPTION_QUANTITY_VALUES @@ -695,6 +705,8 @@ Foreign-key constraints: "fk_logical_resource_ident_rtid" FOREIGN KEY (resource_type_id) REFERENCES fhirdata.resource_types(resource_type_id) ``` +Note that the `logical_id` type is defined as `VARCHAR(1024)` which is much larger than the 64 characters required for a FHIR `Resource.id`. This is because this column must also accommodate external reference values, which are typically full URLS and therefore much longer. + The query to obtain the logical_resource_id is: ``` @@ -706,7 +718,7 @@ The query to obtain the logical_resource_id is: Because the table is distributed by `logical_id` and the value is given in the query, Citus can route the query to a single target node. -The `resource_type_id` value comes from the `resource_types` table and is fixed when the schema is first installed. This value is not guaranteed to be same across databases and schemas, so it must always be read from the `resource_types` table. For Citus, the `resource_types` table is distributed as a REFERENCE table which means there is a complete copy of the records on every node. It is therefore possible to join to a reference table without needing a common distribution key. For example: +The `resource_type_id` value comes from the `resource_types` table and is fixed when the schema is first installed. This value is not guaranteed to be same across databases and schemas, so it must always be read from the `resource_types` table for a given schema. For Citus, the `resource_types` table is distributed as a REFERENCE table which means there is a complete copy of the records on every node. It is therefore possible to join to a reference table without needing a common distribution key. For example: ``` SELECT lri.logical_resource_id @@ -717,7 +729,51 @@ The `resource_type_id` value comes from the `resource_types` table and is fixed ``` -## References +### Schema Differences + +In the standard `PLAIN` schema variant, tables may use IDENTITY columns to automate the generation of primary key values. IDENTITY columns are not supported in Citus so when the schema type is `DISTRIBUTED` (which is required for Citus), the primary key values are obtained from the sequence `fhir_sequence` instead. This impacts the common_token_values table, whose definition includes the following: +``` + .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) +``` + +Also, if an index on a DISTRIBUTED table does not include the distribution column (typically `logical_resource_id`), the index cannot be declared UNIQUE. Instead, a non-unique index is defined. For example, the `logical_resources` definition includes: +``` + .addUniqueIndex("UNQ_" + LOGICAL_RESOURCES, RESOURCE_TYPE_ID, LOGICAL_ID) +``` + +For the `DISTRIBUTED` schema type variant, uniqueness of the {resource_type_id, logical_id} tuple can no longer be enforced by the above unique index on `logical_resources`, because `logical_resources` is distributed by `logical_resource_id` and this column is not part of the index definition. Instead, the IBM FHIR Server relies on the new `logical_resource_ident` table to manage uniqueness of this tuple, and the `resource_type_id` and `logical_id` columns in `logical_resources` are just denormalized copies of the data. + +### Distributed Procedures/Functions + +The following description uses the term stored procedure for simplicity even though some implementations use stored functions. + +For the `PLAIN` schema type variant, the IBM FHIR Server uses a stored procedure called `add_any_resource` to create or update the database `logical_resources`, `xx_logical_resources` and `xx_resources` records (where `xx` represents the resource type name). Using a stored procedure improves ingestion performance by reducing the number of database round-trips required to execute the required logic. + +For the `DISTRIBUTED` schema type variant, the logic has been split into two procedures as follows: + +1. `add_logical_resource_ident(resource_type_id, logical_id)` - contains the logic to create a new logical_resource_ident record, or if one exists already, obtain a lock on the row by executing a SELECT FOR UPDATE. This procedure is therefore now responsible for allocating a new `logical_resource_id` value for all new logical resources; +2. `add_any_resource(logical_resource_id, ...)` - contains the remaining logic to create/update the `logical_resources`, `xx_logical_resources` and `xx_resources` records. + +Because the `logical_resource_id` value is no longer generated inside `add_any_resource`, it is now passed as a parameter to this procedure. This approach has some significant benefits with Citus, which allows stored procedures and functions to be distributed by one of their parameters. The `add_logical_resource_ident` includes SQL and DML statements involving only the`logical_resource_ident` table, and all statements use `logical_id` which is the distribution column for that table. This allows us to also distribute the procedure by the `logical_id` parameter value, allowing the database to optimize how the procedure is executed. + +Similarly, all SQL and DML statements within the `add_any_resource` procedure use `logical_resource_id`, so the `add_any_resource` procedure is distributed by the `logical_resource_id` parameter to provide the same benefit at runtime. + +### Schema Tool Changes to Support Citus + +When the database type is given as citus (`--db-type citus`), the schema-tool applies changes in this order: + +1. Create Tables (without any foreign key constraints) +2. Apply table distribution rules for all REFERENCE tables +3. Apply table distribution rules for all DISTRIBUTED tables +4. Add foreign key constraints +5. Add/replace stored functions +6. Apply stored function distribution rules + +The foreign key constraints have to be added after the tables are distributed. Attempting to distribute tables after the foreign key constraints have been applied leads to errors. + +The distribution step can take some time due to the amount of DDL Citus must execute for each table. + +## References and Additional Reading - [Git Issue: Document the schema migration process on the project wiki #270](https://github.com/IBM/FHIR/issues/270) - [Db2 11.5: Extent sizes in table spaces](https://www.ibm.com/support/knowledgecenter/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/c0004964.html) - [Db2 11.5: Altering table spaces](https://www.ibm.com/support/producthub/db2/docs/content/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/t0005096.html)