From c970ada563edcb0f0130137beae2ef5f7b33a4ef Mon Sep 17 00:00:00 2001 From: Troy Biesterfeld Date: Wed, 16 Jun 2021 14:10:38 -0500 Subject: [PATCH] Issue #2155 - Use hash to determine if search parameters need update Signed-off-by: Troy Biesterfeld --- .../jdbc/dao/ReindexResourceDAO.java | 41 +- .../persistence/jdbc/dao/api/ResourceDAO.java | 4 +- .../jdbc/dao/api/ResourceIndexRecord.java | 20 +- .../jdbc/dao/impl/ResourceDAOImpl.java | 11 +- .../jdbc/derby/DerbyResourceDAO.java | 14 +- .../jdbc/dto/CompositeParmVal.java | 7 + .../persistence/jdbc/dto/DateParmVal.java | 15 +- .../jdbc/dto/ExtractedParameterValue.java | 53 ++- .../persistence/jdbc/dto/LocationParmVal.java | 10 + .../persistence/jdbc/dto/NumberParmVal.java | 15 +- .../persistence/jdbc/dto/QuantityParmVal.java | 12 + .../jdbc/dto/ReferenceParmVal.java | 11 + .../persistence/jdbc/dto/StringParmVal.java | 20 +- .../persistence/jdbc/dto/TokenParmVal.java | 9 + .../fhir/persistence/jdbc/dto/UriParmVal.java | 9 +- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 398 ++++++++++-------- .../postgres/PostgresReindexResourceDAO.java | 3 +- .../jdbc/postgres/PostgresResourceDAO.java | 12 +- .../postgres/PostgresResourceNoProcDAO.java | 12 +- .../jdbc/util/ExtractedSearchParameters.java | 46 ++ .../util/JDBCParameterBuildingVisitor.java | 96 ++++- .../jdbc/util/ParameterHashUtil.java | 82 ++++ .../jdbc/test/util/ParameterHashUtilTest.java | 188 +++++++++ 23 files changed, 852 insertions(+), 236 deletions(-) create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java create mode 100644 fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java create mode 100644 fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java index 0fdcd950b4b..ff0b8b9e250 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java @@ -14,7 +14,6 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.Instant; -import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -33,6 +32,7 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceDAOImpl; import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; /** * DAO used to contain the logic required to reindex a given resource @@ -111,6 +111,7 @@ public ReindexResourceDAO(Connection connection, IDatabaseTranslator translator, * Getter for the translator currently held by this DAO * @return */ + @Override protected IDatabaseTranslator getTranslator() { return this.translator; } @@ -169,7 +170,8 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind } ResultSet rs = stmt.executeQuery(); if (rs.next()) { - result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); + // TODO: Update this when PARAMETERS_HASH column exists in the DB table + result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4), null); } } catch (SQLException x) { logger.log(Level.SEVERE, select, x); @@ -270,7 +272,7 @@ public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp, Integer r * @param logicalResourceId the logical resource id * @throws Exception */ - public void updateParameters(String tablePrefix, List parameters, String logicalId, long logicalResourceId) throws Exception { + public void updateParameters(String tablePrefix, ExtractedSearchParameters parameters, String logicalId, long logicalResourceId) throws Exception { final String METHODNAME = "updateParameters() for " + tablePrefix + "/" + logicalId; logger.entering(CLASSNAME, METHODNAME); @@ -295,14 +297,13 @@ public void updateParameters(String tablePrefix, List p deleteFromParameterTable(connection, "str_values", logicalResourceId); deleteFromParameterTable(connection, "date_values", logicalResourceId); - - if (parameters != null && !parameters.isEmpty()) { + if (!parameters.getParameters().isEmpty()) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); // Check if this is multitenant boolean isMultitenant = this.getFlavor().isMultitenant(); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", tablePrefix, isMultitenant, logicalResourceId, 100, identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { + for (ExtractedParameterValue p: parameters.getParameters()) { p.accept(pvd); } } catch (SQLException x) { @@ -310,6 +311,11 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { throw translator.translate(x); } } + + // Update the parameters hash + // TODO: Enable this when column exists + //updateParametersHash(connection, logicalResourceId, parameters.getHash()); + logger.exiting(CLASSNAME, METHODNAME); } @@ -335,4 +341,27 @@ protected void deleteFromParameterTable(Connection conn, String tableName, long throw translator.translate(x); } } + + /** + * Updates the parameters hash in the LOGICAL_RESOURCES table. + * @param conn the connection + * @param logicalResourceId the logical resource ID + * @param hash the parameters hash + * @throws SQLException + */ + protected void updateParametersHash(Connection conn, long logicalResourceId, String hash) throws SQLException { + final String SQL = "UPDATE logical_resources SET parameters_hash = ? WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(SQL)) { + // bind parameters + stmt.setString(1, hash); + stmt.setLong(2, logicalResourceId); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Update parameters_hash [" + hash + "] for logicalResourceId [" + logicalResourceId + "]"); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SQL, x); + throw translator.translate(x); + } + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java index af9fc60eaf2..ac40e697771 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java @@ -14,10 +14,10 @@ import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; -import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.SqlQueryData; /** @@ -224,6 +224,6 @@ List search(String sqlSelect) * @throws FHIRPersistenceVersionIdMismatchException * @throws FHIRPersistenceException */ - Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + Resource insert(Resource resource, ExtractedSearchParameters parameters, ParameterDAO parameterDao) throws FHIRPersistenceException; } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java index 5b54d7dc95a..385a46d5b34 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceIndexRecord.java @@ -30,11 +30,15 @@ public class ResourceIndexRecord { // Deletion flag for the resource. Set when we read the resource private boolean deleted; - public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId) { + // SHA-256 hash of the search parameters + private String parametersHash; + + public ResourceIndexRecord(long logicalResourceId, int resourceTypeId, String logicalId, long transactionId, String parametersHash) { this.logicalResourceId = logicalResourceId; this.resourceTypeId = resourceTypeId; this.logicalId = logicalId; this.transactionId = transactionId; + this.parametersHash = parametersHash; } /** @@ -92,4 +96,18 @@ public boolean isDeleted() { public void setDeleted(boolean deleted) { this.deleted = deleted; } + + /** + * @return the parametersHash + */ + public String getParametersHash() { + return parametersHash; + } + + /** + * @param parametersHash the parametersHash to set + */ + public void setParametersHash(String parametersHash) { + this.parametersHash = parametersHash; + } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index 92ce5ff4a10..3cdcd006756 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -48,6 +48,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCacheUpdater; import com.ibm.fhir.persistence.jdbc.util.SqlQueryData; @@ -539,9 +540,9 @@ protected Integer getResourceTypeIdFromCaches(String resourceType) { } @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, ExtractedSearchParameters parameters, ParameterDAO parameterDao) throws FHIRPersistenceException { - final String METHODNAME = "insert(Resource, List"; + final String METHODNAME = "insert(Resource, ExtractedSearchParameters"; log.entering(CLASSNAME, METHODNAME); final Connection connection = getConnection(); // do not close @@ -588,6 +589,8 @@ public Resource insert(Resource resource, List paramete stmt.registerOutParameter(7, Types.BIGINT); stmt.registerOutParameter(8, Types.BIGINT); + // TODO: Issue #2155: Update the "parameters hash" in the LOGICAL_RESOURCES table + stmt.execute(); long latestTime = System.nanoTime(); double dbCallDuration = (latestTime-dbCallStartTime)/1e6; @@ -614,11 +617,11 @@ public Resource insert(Resource resource, List paramete // Parameter time // TODO FHIR_ADMIN schema name needs to come from the configuration/context long paramInsertStartTime = latestTime; - if (parameters != null) { + if (!parameters.getParameters().isEmpty()) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(cache, this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", resource.getResourceType(), true, resource.getId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { - for (ExtractedParameterValue p: parameters) { + for (ExtractedParameterValue p: parameters.getParameters()) { p.accept(pvd); } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index 99582c1edd0..2c84992d8a4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -15,7 +15,6 @@ import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Timestamp; -import java.util.List; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -40,6 +39,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** @@ -89,7 +89,7 @@ public DerbyResourceDAO(Connection connection, String schemaName, FHIRDbFlavor f * @throws FHIRPersistenceVersionIdMismatchException */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, ExtractedSearchParameters parameters, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert"; logger.entering(CLASSNAME, METHODNAME); @@ -198,7 +198,7 @@ public Resource insert(Resource resource, List paramet * @return the resource_id for the entry we created * @throws Exception */ - public long storeResource(String tablePrefix, List parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, + public long storeResource(String tablePrefix, ExtractedSearchParameters parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, String p_source_key, Integer p_version, Connection conn, ParameterDAO parameterDao) throws Exception { final String METHODNAME = "storeResource() for " + tablePrefix + " resource"; @@ -364,6 +364,8 @@ public long storeResource(String tablePrefix, List para logger.finest("Created " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); } } + + // TODO: Issue #2155: Update the "parameters hash" in the LOGICAL_RESOURCES table } } @@ -471,12 +473,14 @@ public long storeResource(String tablePrefix, List para logger.finest("Updated logical_resources: " + v_resource_type + "/" + p_logical_id); } } + + // TODO: Issue #2155: Update the "parameters hash" in the LOGICAL_RESOURCES table } // To keep things simple for the Derby use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + if (!parameters.getParameters().isEmpty()) { // Derby doesn't support partitioned multi-tenancy, so we disable it on the DAO: if (logger.isLoggable(Level.FINEST)) { logger.finest("Storing parameters for: " + v_resource_type + "/" + p_logical_id); @@ -484,7 +488,7 @@ public long storeResource(String tablePrefix, List para JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(conn, null, tablePrefix, false, v_logical_resource_id, 100, identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { + for (ExtractedParameterValue p: parameters.getParameters()) { p.accept(pvd); } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java index 90c4522977a..21557b9bf47 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/CompositeParmVal.java @@ -10,6 +10,7 @@ import java.util.List; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. @@ -30,10 +31,16 @@ public CompositeParmVal() { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), parameterHashUtil.getParametersHash(component)); + } + /** * @return get the list of components in this composite parameter */ diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java index 227993960db..75b3c03ea53 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/DateParmVal.java @@ -9,6 +9,7 @@ import java.sql.Timestamp; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_DATE_VALUES tables. @@ -18,9 +19,6 @@ public class DateParmVal extends ExtractedParameterValue { private Timestamp valueDateStart; private Timestamp valueDateEnd; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - public enum TimeType { YEAR, YEAR_MONTH, @@ -53,13 +51,22 @@ public void setValueDateEnd(Timestamp valueDateEnd) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(valueDateStart); + sb.append("|").append(valueDateEnd); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } + @Override public String toString() { return "DateParmVal [resourceType=" + getResourceType() + ", name=" + getName() - + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + ", base=" + base + "]"; + + ", valueDateStart=" + valueDateStart + ", valueDateEnd=" + valueDateEnd + "]"; } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java index 597d44dad43..9841935b859 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ExtractedParameterValue.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dto; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * A search parameter value extracted from a resource and ready to store / index for search @@ -22,9 +23,13 @@ public abstract class ExtractedParameterValue { // The resource type associated with this parameter private String resourceType; - // The bass resource name + // The SearchParameter base type. If "Resource", then this is a Resource-level attribute private String base; + // URL and version of search parameter + private String url; + private String version; + /** * Protected constructor */ @@ -93,4 +98,50 @@ public String getName() { public void setName(String name) { this.name = name; } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param url the url to set + */ + public void setUrl(String url) { + this.url = url; + } + + /** + * @return the version + */ + public String getVersion() { + return version; + } + + /** + * @param version the version to set + */ + public void setVersion(String version) { + this.version = version; + } + + /** + * Gets the hash header. + * @return the hash header + */ + protected String getHashHeader() { + StringBuilder sb = new StringBuilder(name); + sb.append("|").append(url).append("|").append(version); + return sb.toString(); + } + + /** + * Gets the hash representation of the parameter. + * This should be generated from the search parameter (code, url, version) and the extracted value. + * @param the parameter hash utility to use for generating hashes + * @return the hash + */ + public abstract String getHash(ParameterHashUtil parameterHashUtil); } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java index da67655aedc..fb929b4fba7 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/LocationParmVal.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dto; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_LATLNG_VALUES tables. @@ -42,7 +43,16 @@ public void setValueLatitude(Double valueLatitude) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(valueLongitude); + sb.append("|").append(valueLatitude); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java index 4897bb48f3e..3395d625356 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/NumberParmVal.java @@ -9,6 +9,7 @@ import java.math.BigDecimal; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_NUMBER_VALUES tables. @@ -19,15 +20,13 @@ public class NumberParmVal extends ExtractedParameterValue { private BigDecimal valueNumberLow; private BigDecimal valueNumberHigh; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - /** * Public constructor */ public NumberParmVal() { super(); } + public BigDecimal getValueNumber() { return valueNumber; } @@ -55,7 +54,17 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(valueNumber); + sb.append("|").append(valueNumberLow); + sb.append("|").append(valueNumberHigh); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java index 8098760e231..645321228e2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/QuantityParmVal.java @@ -10,6 +10,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_QUANTITY_VALUES tables. @@ -83,4 +84,15 @@ public void setValueNumberHigh(BigDecimal valueNumberHigh) { public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(); + sb.append(valueNumber); + sb.append("|").append(valueNumberLow); + sb.append("|").append(valueNumberHigh); + sb.append("|").append(getValueSystem()); + sb.append("|").append(getValueCode()); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java index 41a2d7a3e70..bfe66810364 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dto; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; import com.ibm.fhir.search.SearchConstants.Type; import com.ibm.fhir.search.util.ReferenceValue; @@ -48,7 +49,17 @@ public Type getType() { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(refValue.getTargetResourceType()); + sb.append("|").append(refValue.getValue()); + sb.append("|").append(refValue.getType()); + sb.append("|").append(refValue.getVersion()); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java index 71ea95a89e7..bd23b591c1b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/StringParmVal.java @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dto; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_STR_VALUES tables. @@ -16,9 +17,6 @@ public class StringParmVal extends ExtractedParameterValue { // The string value of this extracted parameter private String valueString; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute - private String base; - /** * Public constructor */ @@ -37,21 +35,13 @@ public void setValueString(String valueString) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } - /** - * @return the base - */ - public String getBase() { - return base; - } - - /** - * @param base the base to set - */ - public void setBase(String base) { - this.base = base; + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), valueString); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java index 6090f67aaa7..ba9be88b1ad 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/TokenParmVal.java @@ -8,6 +8,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * This class defines the Data Transfer Object representing a row in the X_TOKEN_VALUES tables. @@ -52,7 +53,15 @@ public void setValueCode(String valueCode) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + StringBuilder sb = new StringBuilder(getValueSystem()); + sb.append("|").append(valueCode); + return parameterHashUtil.getNameValueHash(getHashHeader(), sb.toString()); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java index adda8f18f3f..5f088a5fbab 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/UriParmVal.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2019 + * (C) Copyright IBM Corp. 2017, 2021 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ package com.ibm.fhir.persistence.jdbc.dto; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; /** * Note used @@ -34,7 +35,13 @@ public void setValueString(String valueString) { /** * We know our type, so we can call the correct method on the visitor */ + @Override public void accept(ExtractedParameterValueVisitor visitor) throws FHIRPersistenceException { // visitor.visit(this); } + + @Override + public String getHash(ParameterHashUtil parameterHashUtil) { + return parameterHashUtil.getNameValueHash(getHashHeader(), valueString); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 47fdcfc0436..ccc85494218 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -141,9 +141,11 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.util.CodeSystemsCache; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.JDBCParameterBuildingVisitor; import com.ibm.fhir.persistence.jdbc.util.JDBCQueryBuilder; import com.ibm.fhir.persistence.jdbc.util.NewQueryBuilder; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; import com.ibm.fhir.persistence.jdbc.util.ParameterNamesCache; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; import com.ibm.fhir.persistence.jdbc.util.SqlQueryData; @@ -215,6 +217,8 @@ public class FHIRPersistenceJDBCImpl implements FHIRPersistence, SchemaNameSuppl // Use the optimized query builder when supported for the search request private final boolean optQueryBuilderEnabled; + private final ParameterHashUtil parameterHashUtil; + /** * Constructor for use when running as web application in WLP. * @throws Exception @@ -265,6 +269,8 @@ public FHIRPersistenceJDBCImpl(FHIRPersistenceJDBCCache cache) throws Exception this.transactionAdapter = new FHIRUserTransactionAdapter(userTransaction, trxSynchRegistry, cache, TXN_DATA_KEY); + this.parameterHashUtil = new ParameterHashUtil(); + log.exiting(CLASSNAME, METHODNAME); } @@ -326,6 +332,9 @@ public FHIRPersistenceJDBCImpl(Properties configProps, IConnectionProvider cp, F // Always want to be testing with the new query builder this.optQueryBuilderEnabled = true; + // Utility for generating hash of search parameters + this.parameterHashUtil = new ParameterHashUtil(); + log.exiting(CLASSNAME, METHODNAME); } @@ -411,7 +420,8 @@ public SingleResourceResult create(FHIRPersistenceContex // Persist the Resource DTO. resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterDao); + ExtractedSearchParameters parameters = this.extractSearchParameters(updatedResource, resourceDTO); + resourceDao.insert(resourceDTO, parameters, parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -597,7 +607,8 @@ public SingleResourceResult update(FHIRPersistenceContex // Persist the Resource DTO. resourceDao.setPersistenceContext(context); - resourceDao.insert(resourceDTO, this.extractSearchParameters(updatedResource, resourceDTO), parameterDao); + ExtractedSearchParameters parameters = this.extractSearchParameters(updatedResource, resourceDTO); + resourceDao.insert(resourceDTO, parameters, parameterDao); if (log.isLoggable(Level.FINE)) { log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + ", version=" + resourceDTO.getVersionId()); @@ -1878,136 +1889,199 @@ private TransactionSynchronizationRegistry getTrxSynchRegistry() throws FHIRPers * @return list of extracted search parameters * @throws Exception */ - private List extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTOx) + private ExtractedSearchParameters extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTOx) throws Exception { final String METHODNAME = "extractSearchParameters"; log.entering(CLASSNAME, METHODNAME); Map> map; String code; + String url; + String version; String type; String expression; - List allParameters = new ArrayList<>(); + ExtractedSearchParameters extractedParameters = new ExtractedSearchParameters(); try { - if (fhirResource == null) { - return allParameters; - } - map = SearchUtil.extractParameterValues(fhirResource); + if (fhirResource != null) { - for (Entry> entry : map.entrySet()) { - SearchParameter sp = entry.getKey(); - code = sp.getCode().getValue(); - final boolean wholeSystemParam = isWholeSystem(sp); + map = SearchUtil.extractParameterValues(fhirResource); - // As not to inject any other special handling logic, this is a simple inline check to see if - // _id or _lastUpdated are used, and ignore those extracted values. - if (SPECIAL_HANDLING.contains(code)) { - continue; - } - type = sp.getType().getValue(); - expression = sp.getExpression().getValue(); + for (Entry> entry : map.entrySet()) { + SearchParameter sp = entry.getKey(); + code = sp.getCode().getValue(); + url = sp.getUrl().getValue(); + version = sp.getVersion() != null ? sp.getVersion().getValue(): null; + final boolean wholeSystemParam = isWholeSystem(sp); - if (log.isLoggable(Level.FINE)) { - log.fine("Processing SearchParameter resource: " + fhirResource.getClass().getSimpleName() + ", code: " + code + ", type: " + type + ", expression: " + expression); - } + // As not to inject any other special handling logic, this is a simple inline check to see if + // _id or _lastUpdated are used, and ignore those extracted values. + if (SPECIAL_HANDLING.contains(code)) { + continue; + } + type = sp.getType().getValue(); + expression = sp.getExpression().getValue(); - List values = entry.getValue(); - - if (SearchParamType.COMPOSITE.equals(sp.getType())) { - List components = sp.getComponent(); - FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); - - for (FHIRPathNode value : values) { - Visitable fhirNode; - EvaluationContext context; - if (value.isResourceNode()) { - fhirNode = value.asResourceNode().resource(); - context = new EvaluationContext((Resource) fhirNode); - } else if (value.isElementNode()) { - fhirNode = value.asElementNode().element(); - context = new EvaluationContext((Element) fhirNode); - } else { - throw new IllegalStateException("Composite parameter expression must select one or more FHIR elements"); - } + if (log.isLoggable(Level.FINE)) { + log.fine("Processing SearchParameter resource: " + fhirResource.getClass().getSimpleName() + + ", code: " + code + ", url: " + url + ", version: " + version + ", type: " + type + ", expression: " + expression); + } - CompositeParmVal p = new CompositeParmVal(); - p.setName(code); - p.setResourceType(fhirResource.getClass().getSimpleName()); - - for (int i = 0; i < components.size(); i++) { - Component component = components.get(i); - Collection nodes = evaluator.evaluate(context, component.getExpression().getValue()); - if (nodes.isEmpty()){ - if (log.isLoggable(Level.FINER)) { - log.finer("Component expression '" + component.getExpression().getValue() + "' resulted in 0 nodes; " - + "skipping composite parameter '" + code + "'."); - } - continue; + List values = entry.getValue(); + + if (SearchParamType.COMPOSITE.equals(sp.getType())) { + List components = sp.getComponent(); + FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); + + for (FHIRPathNode value : values) { + Visitable fhirNode; + EvaluationContext context; + if (value.isResourceNode()) { + fhirNode = value.asResourceNode().resource(); + context = new EvaluationContext((Resource) fhirNode); + } else if (value.isElementNode()) { + fhirNode = value.asElementNode().element(); + context = new EvaluationContext((Element) fhirNode); + } else { + throw new IllegalStateException("Composite parameter expression must select one or more FHIR elements"); } - // Alternative: consider pulling the search parameter from the FHIRRegistry instead so we can use versioned references. - // Of course, that would require adding extension-search-params to the Registry which would require the Registry to be tenant-aware. -// SearchParameter compSP = FHIRRegistry.getInstance().getResource(component.getDefinition().getValue(), SearchParameter.class); - SearchParameter compSP = SearchUtil.getSearchParameter(p.getResourceType(), component.getDefinition()); - JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(p.getResourceType(), compSP); - FHIRPathNode node = nodes.iterator().next(); - if (nodes.size() > 1 ) { - // TODO: support component expressions that result in multiple nodes - // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. - // For example, if a composite has two components and each one's expression results in two nodes - // ([Code1,Code2] and [Quantity1,Quantity2]) and each node results in a single ExtractedParameterValue, - // then we need to generate CompositeParmVal objects for [Code1,Quantity1], [Code1,Quantity2], - // [Code2,Quantity1], and [Code2,Quantity2]. - // Assumption: this should be rare. - if (log.isLoggable(Level.FINE)) { - log.fine("Component expression '" + component.getExpression().getValue() + "' resulted in multiple nodes; " - + "proceeding with randomly chosen node '" + node.path() + "' for search parameter '" + code + "'."); + CompositeParmVal p = new CompositeParmVal(); + p.setName(code); + p.setUrl(url); + p.setVersion(version); + p.setResourceType(fhirResource.getClass().getSimpleName()); + + for (int i = 0; i < components.size(); i++) { + Component component = components.get(i); + Collection nodes = evaluator.evaluate(context, component.getExpression().getValue()); + if (nodes.isEmpty()){ + if (log.isLoggable(Level.FINER)) { + log.finer("Component expression '" + component.getExpression().getValue() + "' resulted in 0 nodes; " + + "skipping composite parameter '" + code + "'."); + } + continue; } - } - try { - if (node.isElementNode()) { - // parameterBuilder aggregates the results for later retrieval - node.asElementNode().element().accept(parameterBuilder); - // retrieve the list of parameters built from all the FHIRPathElementNode values - List parameters = parameterBuilder.getResult(); - if (parameters.isEmpty()){ - if (log.isLoggable(Level.FINE)) { - log.fine("Selected element '" + node.path() + "' resulted in 0 extracted parameter values; " - + "skipping composite parameter '" + code + "'."); - } - continue; + // Alternative: consider pulling the search parameter from the FHIRRegistry instead so we can use versioned references. + // Of course, that would require adding extension-search-params to the Registry which would require the Registry to be tenant-aware. + // SearchParameter compSP = FHIRRegistry.getInstance().getResource(component.getDefinition().getValue(), SearchParameter.class); + SearchParameter compSP = SearchUtil.getSearchParameter(p.getResourceType(), component.getDefinition()); + JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(p.getResourceType(), compSP); + FHIRPathNode node = nodes.iterator().next(); + if (nodes.size() > 1 ) { + // TODO: support component expressions that result in multiple nodes + // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. + // For example, if a composite has two components and each one's expression results in two nodes + // ([Code1,Code2] and [Quantity1,Quantity2]) and each node results in a single ExtractedParameterValue, + // then we need to generate CompositeParmVal objects for [Code1,Quantity1], [Code1,Quantity2], + // [Code2,Quantity1], and [Code2,Quantity2]. + // Assumption: this should be rare. + if (log.isLoggable(Level.FINE)) { + log.fine("Component expression '" + component.getExpression().getValue() + "' resulted in multiple nodes; " + + "proceeding with randomly chosen node '" + node.path() + "' for search parameter '" + code + "'."); } + } + + try { + if (node.isElementNode()) { + // parameterBuilder aggregates the results for later retrieval + node.asElementNode().element().accept(parameterBuilder); + // retrieve the list of parameters built from all the FHIRPathElementNode values + List parameters = parameterBuilder.getResult(); + if (parameters.isEmpty()){ + if (log.isLoggable(Level.FINE)) { + log.fine("Selected element '" + node.path() + "' resulted in 0 extracted parameter values; " + + "skipping composite parameter '" + code + "'."); + } + continue; + } + + if (parameters.size() > 1) { + // TODO: support component searchParms that lead to multiple ExtractedParameterValues + // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. + // For example: + // If a composite has two components and each results in two extracted parameters ([A,B] and [1,2] respectively) + // then we need to generate CompositeParmVal objects for [A,1], [A,2], [B,1], and [B,2] + // Assumption: this should only be common for Quantity search parameters with both a coded unit and a display unit, + // and in these cases, the coded unit is almost always the preferred value for search. + if (log.isLoggable(Level.FINE)) { + log.fine("Selected element '" + node.path() + "' resulted in multiple extracted parameter values; " + + "proceeding with the first extracted value for composite parameter '" + code + "'."); + } + } + ExtractedParameterValue componentParam = parameters.get(0); + // override the component parameter name with the composite parameter name + componentParam.setName(SearchUtil.makeCompositeSubCode(code, componentParam.getName())); + componentParam.setUrl(url); + componentParam.setVersion(version); + // componentParam.setBase(p.getBase()); TODO not needed? + p.addComponent(componentParam); + } else if (node.isSystemValue()){ + ExtractedParameterValue primitiveParam = processPrimitiveValue(node.asSystemValue()); + primitiveParam.setName(code); + primitiveParam.setUrl(url); + primitiveParam.setVersion(version); + primitiveParam.setResourceType(fhirResource.getClass().getSimpleName()); - if (parameters.size() > 1) { - // TODO: support component searchParms that lead to multiple ExtractedParameterValues - // On the current schema, this means creating a different CompositeParmValue for each ordered set of component values. - // For example: - // If a composite has two components and each results in two extracted parameters ([A,B] and [1,2] respectively) - // then we need to generate CompositeParmVal objects for [A,1], [A,2], [B,1], and [B,2] - // Assumption: this should only be common for Quantity search parameters with both a coded unit and a display unit, - // and in these cases, the coded unit is almost always the preferred value for search. if (log.isLoggable(Level.FINE)) { - log.fine("Selected element '" + node.path() + "' resulted in multiple extracted parameter values; " - + "proceeding with the first extracted value for composite parameter '" + code + "'."); + log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } + p.addComponent(primitiveParam); + } else { + // log and continue + String msg = "Unable to extract value from '" + value.path() + + "'; search parameter value extraction can only be performed on Elements and primitive values."; + if (log.isLoggable(Level.FINE)) { + log.fine(msg); + } + addWarning(IssueType.INVALID, msg); + continue; + } + } catch (IllegalArgumentException e) { + // log and continue with the other parameters + StringBuilder msg = new StringBuilder("Skipped search parameter '" + code + "'"); + if (sp.getId() != null) { + msg.append(" with id '" + sp.getId() + "'"); } - ExtractedParameterValue componentParam = parameters.get(0); - // override the component parameter name with the composite parameter name - componentParam.setName(SearchUtil.makeCompositeSubCode(code, componentParam.getName())); - // componentParam.setBase(p.getBase()); TODO not needed? - p.addComponent(componentParam); - } else if (node.isSystemValue()){ - ExtractedParameterValue primitiveParam = processPrimitiveValue(node.asSystemValue()); - primitiveParam.setName(code); - primitiveParam.setResourceType(fhirResource.getClass().getSimpleName()); + msg.append(" for resource type " + fhirResource.getClass().getSimpleName()); + // just use the message...no need for the whole stack trace + msg.append(" due to \n" + e.getMessage()); + if (log.isLoggable(Level.FINE)) { + log.fine(msg.toString()); + } + addWarning(IssueType.INVALID, msg.toString()); + } + } + if (components.size() == p.getComponent().size()) { + // only add the parameter if all of the components are present and accounted for + extractedParameters.getParameters().add(p); + } + } + } else { // ! SearchParamType.COMPOSITE.equals(sp.getType()) + JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(fhirResource.getClass().getSimpleName(), sp); + for (FHIRPathNode value : values) { + + try { + if (value.isElementNode()) { + // parameterBuilder aggregates the results for later retrieval + value.asElementNode().element().accept(parameterBuilder); + } else if (value.isSystemValue()){ + ExtractedParameterValue p = processPrimitiveValue(value.asSystemValue()); + p.setName(code); + p.setUrl(url); + p.setVersion(version); + p.setResourceType(fhirResource.getClass().getSimpleName()); + + if (wholeSystemParam) { + p.setWholeSystem(true); + } + extractedParameters.getParameters().add(p); if (log.isLoggable(Level.FINE)) { log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } - p.addComponent(primitiveParam); } else { // log and continue String msg = "Unable to extract value from '" + value.path() + @@ -2020,7 +2094,7 @@ private List extractSearchParameters(Resource fhirResou } } catch (IllegalArgumentException e) { // log and continue with the other parameters - StringBuilder msg = new StringBuilder("Skipped search parameter '" + code + "'"); + StringBuilder msg = new StringBuilder("Skipping search parameter '" + code + "'"); if (sp.getId() != null) { msg.append(" with id '" + sp.getId() + "'"); } @@ -2033,79 +2107,35 @@ private List extractSearchParameters(Resource fhirResou addWarning(IssueType.INVALID, msg.toString()); } } - if (components.size() == p.getComponent().size()) { - // only add the parameter if all of the components are present and accounted for - allParameters.add(p); - } - } - } else { // ! SearchParamType.COMPOSITE.equals(sp.getType()) - JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(fhirResource.getClass().getSimpleName(), sp); - - for (FHIRPathNode value : values) { - - try { - if (value.isElementNode()) { - // parameterBuilder aggregates the results for later retrieval - value.asElementNode().element().accept(parameterBuilder); - } else if (value.isSystemValue()){ - ExtractedParameterValue p = processPrimitiveValue(value.asSystemValue()); - p.setName(code); - p.setResourceType(fhirResource.getClass().getSimpleName()); - - if (wholeSystemParam) { - p.setWholeSystem(true); - } - allParameters.add(p); - if (log.isLoggable(Level.FINE)) { - log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); - } - } else { - // log and continue - String msg = "Unable to extract value from '" + value.path() + - "'; search parameter value extraction can only be performed on Elements and primitive values."; - if (log.isLoggable(Level.FINE)) { - log.fine(msg); - } - addWarning(IssueType.INVALID, msg); - continue; + // retrieve the list of parameters built from all the FHIRPathElementNode values + List parameters = parameterBuilder.getResult(); + for (ExtractedParameterValue p : parameters) { + if (wholeSystemParam) { + p.setWholeSystem(true); } - } catch (IllegalArgumentException e) { - // log and continue with the other parameters - StringBuilder msg = new StringBuilder("Skipping search parameter '" + code + "'"); - if (sp.getId() != null) { - msg.append(" with id '" + sp.getId() + "'"); - } - msg.append(" for resource type " + fhirResource.getClass().getSimpleName()); - // just use the message...no need for the whole stack trace - msg.append(" due to \n" + e.getMessage()); + extractedParameters.getParameters().add(p); if (log.isLoggable(Level.FINE)) { - log.fine(msg.toString()); + log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } - addWarning(IssueType.INVALID, msg.toString()); - } - } - // retrieve the list of parameters built from all the FHIRPathElementNode values - List parameters = parameterBuilder.getResult(); - for (ExtractedParameterValue p : parameters) { - if (wholeSystemParam) { - p.setWholeSystem(true); - } - allParameters.add(p); - if (log.isLoggable(Level.FINE)) { - log.fine("Extracted Parameter '" + p.getName() + "' from Resource."); } } } + + // Augment the extracted parameter list with special values we use to represent compartment relationships. + // These references are stored as tokens and are used by the search query builder + // for compartment-based searches + addCompartmentParams(extractedParameters.getParameters(), fhirResource); } - // Augment the extracted parameter list with special values we use to represent compartment relationships. - // These references are stored as tokens and are used by the search query builder - // for compartment-based searches - addCompartmentParams(allParameters, fhirResource); + // Generate the hash which is used to quickly determine whether the extracted parameters + // are different than the extracted parameters that currently exist in the database + extractedParameters.generateHash(parameterHashUtil); + } finally { log.exiting(CLASSNAME, METHODNAME); } - return allParameters; + + return extractedParameters; } /** @@ -2531,7 +2561,7 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper } else { // Skip this particular resource because it has been deleted if (log.isLoggable(Level.FINE)) { - log.info("Skipping reindex for deleted FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); + log.fine("Skipping reindex for deleted FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); } rir.setDeleted(true); } @@ -2573,11 +2603,11 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper /** * Update the parameters for the resource described by the given DTO * @param - * @param rir - * @param resourceTypeClass - * @param existingResourceDTO - * @param reindexDAO - * @param operationOutcomeResult + * @param rir the resource index record + * @param resourceTypeClass the resource type class + * @param existingResourceDTO the existing resource DTO + * @param reindexDAO the reindex resource DAO + * @param operationOutcomeResult the operation outcome result * @throws Exception */ public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, @@ -2585,19 +2615,29 @@ public void updateParameters(ResourceIndexRecord rir, Class if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); - // Extract parameters from the resource payload we just read and store them, replacing - // the existing set - reindexDAO.updateParameters(rir.getResourceType(), this.extractSearchParameters(existingResource, existingResourceDTO), rir.getLogicalId(), rir.getLogicalResourceId()); + // Extract parameters from the resource payload. + ExtractedSearchParameters parameters = this.extractSearchParameters(existingResource, existingResourceDTO); + + // Compare the hash of the extracted parameters with the hash in the index record + // If hash in the index record is not null and it matches the hash of the extracted parameters, then no need to replace the + // extracted search parameters in the database tables for this resource, which helps with performance during reindex + if (rir.getParametersHash() != null && rir.getParametersHash().equals(parameters.getHash())) { + if (log.isLoggable(Level.FINE)) { + log.fine("Skipping update of unchanged parameters for FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); + } + } else { + // Replace the existing set + reindexDAO.updateParameters(rir.getResourceType(), parameters, rir.getLogicalId(), rir.getLogicalResourceId()); - // Use an OperationOutcome Issue to let the caller know that some work was performed - final String diag = "Processed " + rir.getResourceType() + "/" + rir.getLogicalId(); - operationOutcomeResult.issue(Issue.builder().code(IssueType.INFORMATIONAL).severity(IssueSeverity.INFORMATION).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); + // Use an OperationOutcome Issue to let the caller know that some work was performed + final String diag = "Processed " + rir.getResourceType() + "/" + rir.getLogicalId(); + operationOutcomeResult.issue(Issue.builder().code(IssueType.INFORMATIONAL).severity(IssueSeverity.INFORMATION).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); + } } else { // Reasonable to assume that this resource was deleted because we can't read it final String diag = "Failed to read resource: " + rir.getResourceType() + "/" + rir.getLogicalId(); operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(string(diag)).build()); } - } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java index 2a6bcaa962a..574833c99c5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresReindexResourceDAO.java @@ -157,7 +157,8 @@ public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexT stmt.execute(); ResultSet rs = stmt.getResultSet(); if (rs.next()) { - result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); + // TODO: Update this when PARAMETERS_HASH column exists in the DB table + result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4), null); } } catch (SQLException x) { logger.log(Level.SEVERE, update, x); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 1035d090c08..1d7301bdd70 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -16,7 +16,6 @@ import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Timestamp; import java.sql.Types; -import java.util.List; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -40,6 +39,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** @@ -76,9 +76,9 @@ public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavo * @throws FHIRPersistenceException */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, ExtractedSearchParameters parameters, ParameterDAO parameterDao) throws FHIRPersistenceException { - final String METHODNAME = "insert(Resource, List"; + final String METHODNAME = "insert(Resource, ExtractedSearchParameters, ParameterDAO"; logger.entering(CLASSNAME, METHODNAME); final Connection connection = getConnection(); // do not close @@ -127,17 +127,19 @@ public Resource insert(Resource resource, List paramete // To keep things simple for the postgresql use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + if (!parameters.getParameters().isEmpty()) { // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getId(), 100, identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { + for (ExtractedParameterValue p: parameters.getParameters()) { p.accept(pvd); } } } + // TODO: Issue #2155: Update the "parameters hash" in the LOGICAL_RESOURCES table + if (logger.isLoggable(Level.FINE)) { logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java index 31f245c9a5f..62dc526c7a4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java @@ -17,7 +17,6 @@ import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Timestamp; import java.sql.Types; -import java.util.List; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,6 +40,7 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.ResourceTypesCache; /** @@ -84,7 +84,7 @@ public PostgresResourceNoProcDAO(Connection connection, String schemaName, FHIRD * @throws FHIRPersistenceVersionIdMismatchException */ @Override - public Resource insert(Resource resource, List parameters, ParameterDAO parameterDao) + public Resource insert(Resource resource, ExtractedSearchParameters parameters, ParameterDAO parameterDao) throws FHIRPersistenceException { final String METHODNAME = "insert"; logger.entering(CLASSNAME, METHODNAME); @@ -193,7 +193,7 @@ public Resource insert(Resource resource, List paramete * @return the resource_id for the entry we created * @throws Exception */ - public long storeResource(String tablePrefix, List parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, + public long storeResource(String tablePrefix, ExtractedSearchParameters parameters, String p_logical_id, InputStream p_payload, Timestamp p_last_updated, boolean p_is_deleted, String p_source_key, Integer p_version, Connection conn, ParameterDAO parameterDao) throws Exception { final String METHODNAME = "storeResource() for " + tablePrefix + " resource"; @@ -383,13 +383,15 @@ public long storeResource(String tablePrefix, List para stmt.executeUpdate(); } + // TODO: Issue #2155: Update the "parameters hash" in the LOGICAL_RESOURCES table + // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null) { + if (!parameters.getParameters().isEmpty()) { // PostgreSQL doesn't support partitioned multi-tenancy, so we disable it on the DAO: JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(conn, null, tablePrefix, false, v_logical_resource_id, 100, identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { + for (ExtractedParameterValue p: parameters.getParameters()) { p.accept(pvd); } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java new file mode 100644 index 00000000000..eb6086aaeb2 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ExtractedSearchParameters.java @@ -0,0 +1,46 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; + +/** + * Contains a list of extracted search parameters and a hash representation. + */ +public class ExtractedSearchParameters { + + private List parameters = new ArrayList<>(); + private String hash = null; + + /** + * Gets the parameters. + * @return the parameters + */ + public List getParameters() { + return parameters; + } + + /** + * Generates the hash representation of the parameters. + * @param the parameter hash utility to use for generating hashes + */ + public void generateHash(ParameterHashUtil parameterHashUtil) { + hash = parameterHashUtil.getParametersHash(parameters); + } + + /** + * Gets the already-generated hash representation of the parameters. + * @return the hash + */ + public String getHash() { + return hash; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java index 9e4f1658f32..6a1eecf0397 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java @@ -94,9 +94,11 @@ public class JDBCParameterBuildingVisitor extends DefaultVisitor { ZonedDateTime.parse("9999-12-31T23:59:59.999999Z").toInstant()); private final String resourceType; - // We only need the SearchParameter type and code, so just store those directly as members + // We only need the SearchParameter code, type, url, and version, so just store those directly as members private final String searchParamCode; private final SearchParamType searchParamType; + private final String searchParamUrl; + private final String searchParamVersion; /** * The result of the visit(s) @@ -105,15 +107,17 @@ public class JDBCParameterBuildingVisitor extends DefaultVisitor { /** * Public constructor - * @param resourceType - * @param searchParameter + * @param resourceType the resource type + * @param searchParameter the search parameter */ public JDBCParameterBuildingVisitor(String resourceType, SearchParameter searchParameter) { super(false); this.resourceType = resourceType; this.searchParamCode = searchParameter.getCode().getValue(); this.searchParamType = searchParameter.getType(); - this.result = new ArrayList<>(); + this.searchParamUrl = searchParameter.getUrl().getValue(); + this.searchParamVersion = searchParameter.getVersion() != null ? searchParameter.getVersion().getValue(): null; + this.result = new ArrayList<>(); } /** @@ -146,6 +150,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueSystem("http://terminology.hl7.org/CodeSystem/special-values"); if (_boolean.getValue()) { p.setValueCode("true"); @@ -166,6 +172,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi StringParmVal p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(canonical.getValue()); result.add(p); } @@ -181,6 +189,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); String system = ModelSupport.getSystem(code); setTokenValues(p, system != null ? Uri.of(system) : null, code.getValue()); result.add(p); @@ -197,6 +207,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setDateValues(p, date); result.add(p); } @@ -212,6 +224,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setDateValues(p, dateTime); result.add(p); } @@ -227,6 +241,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi NumberParmVal p = new NumberParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); BigDecimal value = decimal.getValue(); p.setValueNumber(value); p.setValueNumberLow(NumberParmBehaviorUtil.generateLowerBound(value)); @@ -245,6 +261,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(id.getValue()); result.add(p); } @@ -260,6 +278,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); Timestamp t = generateTimestamp(instant.getValue().toInstant()); p.setValueDateStart(t); p.setValueDateEnd(t); @@ -277,6 +297,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, com.ibm.fhi NumberParmVal p = new NumberParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); BigDecimal value = new BigDecimal(integer.getValue()); p.setValueNumber(value); p.setValueNumberLow(value); @@ -294,12 +316,16 @@ public boolean visit(String elementName, int elementIndex, com.ibm.fhir.model.ty p.setResourceType(resourceType); p.setValueString(value.getValue()); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } else if (TOKEN.equals(searchParamType)) { TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setValueCode(SearchUtil.normalizeForSearch(value.getValue())); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } else { throw invalidComboException(searchParamType, value); @@ -321,12 +347,16 @@ public boolean visit(String elementName, int elementIndex, Uri uri) { TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(uri.getValue()); result.add(p); } else { StringParmVal p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(uri.getValue()); result.add(p); } @@ -350,6 +380,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(aLine.getValue()); result.add(p); } @@ -357,6 +389,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getCity().getValue()); result.add(p); } @@ -364,6 +398,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getDistrict().getValue()); result.add(p); } @@ -371,6 +407,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getState().getValue()); result.add(p); } @@ -378,6 +416,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getCountry().getValue()); result.add(p); } @@ -385,6 +425,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getPostalCode().getValue()); result.add(p); } @@ -392,6 +434,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Address add p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(address.getText().getValue()); result.add(p); } @@ -411,6 +455,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, CodeableCon TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(SearchUtil.normalizeForSearch(codeableConcept.getText().getValue())); result.add(p); } @@ -426,6 +472,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Coding codi TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, coding.getSystem(), coding.getCode().getValue()); result.add(p); if (coding.getDisplay() != null && coding.getDisplay().hasValue()) { @@ -433,6 +481,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Coding codi p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.TEXT_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueCode(SearchUtil.normalizeForSearch(coding.getDisplay().getValue())); result.add(p); } @@ -449,6 +499,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, ContactPoin TokenParmVal telecom = new TokenParmVal(); telecom.setResourceType(resourceType); telecom.setName(searchParamCode); + telecom.setUrl(searchParamUrl); + telecom.setVersion(searchParamVersion); telecom.setValueCode(contactPoint.getValue().getValue()); result.add(telecom); } @@ -466,6 +518,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(humanName.getFamily().getValue()); result.add(p); } @@ -473,6 +527,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(given.getValue()); result.add(p); } @@ -480,6 +536,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(prefix.getValue()); result.add(p); } @@ -487,6 +545,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(suffix.getValue()); result.add(p); } @@ -494,6 +554,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, HumanName h p = new StringParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueString(humanName.getText().getValue()); result.add(p); } @@ -509,6 +571,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Money money QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(money.getValue().getValue()); if (money.getCurrency() != null) { p.setValueCode(money.getCurrency().getValue()); @@ -530,6 +594,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Period peri DateParmVal p = new DateParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); if (period.getStart() == null || period.getStart().getValue() == null) { p.setValueDateStart(SMALLEST_TIMESTAMP); } else { @@ -564,6 +630,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -581,6 +649,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -594,6 +664,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Quantity qu QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); p.setValueNumber(value); p.setValueNumberLow(valueLow); p.setValueNumberHigh(valueHigh); @@ -612,6 +684,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Range range QuantityParmVal p = new QuantityParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); if (range.getLow() != null && range.getLow().getValue() != null && range.getLow().getValue().getValue() != null) { if (range.getLow().getSystem() != null) { @@ -656,6 +730,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, identifier.getSystem(), identifier.getValue().getValue()); result.add(p); if (identifier.getType() != null) { @@ -665,11 +741,15 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier CompositeParmVal cp = new CompositeParmVal(); cp.setResourceType(resourceType); cp.setName(searchParamCode + SearchConstants.OF_TYPE_MODIFIER_SUFFIX); + cp.setUrl(searchParamUrl); + cp.setVersion(searchParamVersion); // type p = new TokenParmVal(); p.setResourceType(cp.getResourceType()); p.setName(SearchUtil.makeCompositeSubCode(cp.getName(), SearchConstants.OF_TYPE_MODIFIER_COMPONENT_TYPE)); + p.setUrl(cp.getUrl()); + p.setVersion(cp.getVersion()); setTokenValues(p, typeCoding.getSystem(), typeCoding.getCode().getValue()); cp.addComponent(p); @@ -677,6 +757,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Identifier p = new TokenParmVal(); p.setResourceType(cp.getResourceType()); p.setName(SearchUtil.makeCompositeSubCode(cp.getName(), SearchConstants.OF_TYPE_MODIFIER_COMPONENT_VALUE)); + p.setUrl(cp.getUrl()); + p.setVersion(cp.getVersion()); p.setValueCode(identifier.getValue().getValue()); cp.addComponent(p); @@ -705,6 +787,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r p.setResourceType(resourceType); p.setRefValue(refValue); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); result.add(p); } Identifier identifier = reference.getIdentifier(); @@ -712,6 +796,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r TokenParmVal p = new TokenParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode + SearchConstants.IDENTIFIER_MODIFIER_SUFFIX); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); setTokenValues(p, identifier.getSystem(), identifier.getValue().getValue()); result.add(p); } @@ -752,6 +838,8 @@ public boolean visit(java.lang.String elementName, int elementIndex, Location.Po LocationParmVal p = new LocationParmVal(); p.setResourceType(resourceType); p.setName(searchParamCode); + p.setUrl(searchParamUrl); + p.setVersion(searchParamVersion); // The following code ensures that the lat/lon is only added when there is a pair. boolean add = false; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java new file mode 100644 index 00000000000..a423b42fa26 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterHashUtil.java @@ -0,0 +1,82 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.util; + + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; + +/** + * Utility methods for generating SHA-256 hash for search parameters. + */ +public class ParameterHashUtil { + + private static final Charset UTF_8 = StandardCharsets.UTF_8; + private static final String SHA_256 = "SHA-256"; + private final MessageDigest digest; + + public ParameterHashUtil() { + try { + digest = MessageDigest.getInstance(SHA_256); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("MessageDigest not found: " + SHA_256, e); + } + } + + /** + * Gets a SHA-256 hash for a list of ExtractedParameterValues. + * This is used to quickly determine if the search parameters changed + * from the existing values in the database, which can then be used to + * avoid deleting and re-inserting the search parameters into the database. + * @param parameters extracted search parameters + * @return the hash + */ + public String getParametersHash(List parameterValues) { + // Sort hashes to make it deterministic + List sortedList = new ArrayList<>(parameterValues.size()); + for (ExtractedParameterValue parameterValue : parameterValues) { + sortedList.add(parameterValue.getHash(this)); + } + sortedList.sort(Comparator.comparing(String::toString)); + + StringBuilder sb = new StringBuilder("|"); + for (String hash : sortedList) { + sb.append(hash).append("|"); + } + + byte[] hashBytes = digest.digest(sb.toString().getBytes(UTF_8)); + return bytesToHex(hashBytes); + } + + /** + * Gets a SHA-256 hash for a name-value pair. + * @param name the name + * @param value the value + * @return the hash + */ + public String getNameValueHash(String name, String value) { + StringBuilder sb = new StringBuilder(name); + sb.append("=[").append(value).append("]"); + byte[] hashBytes = digest.digest(sb.toString().getBytes(UTF_8)); + return bytesToHex(hashBytes); + } + + private String bytesToHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +} diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java new file mode 100644 index 00000000000..cff47059716 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterHashUtilTest.java @@ -0,0 +1,188 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.Arrays; + +import org.testng.annotations.Test; + +import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; +import com.ibm.fhir.persistence.jdbc.dto.DateParmVal; +import com.ibm.fhir.persistence.jdbc.dto.LocationParmVal; +import com.ibm.fhir.persistence.jdbc.dto.NumberParmVal; +import com.ibm.fhir.persistence.jdbc.dto.QuantityParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; +import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; +import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; +import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; +import com.ibm.fhir.persistence.jdbc.util.ParameterHashUtil; +import com.ibm.fhir.persistence.jdbc.util.type.NumberParmBehaviorUtil; +import com.ibm.fhir.search.date.DateTimeHandler; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + +/** + * Utility to do testing of the parameter hash utility. + */ +public class ParameterHashUtilTest { + + @Test + public void testEmptyExtractedParameters() throws Exception { + ParameterHashUtil util = new ParameterHashUtil(); + + ExtractedSearchParameters esp1 = new ExtractedSearchParameters(); + ExtractedSearchParameters esp2 = new ExtractedSearchParameters(); + + // Hashes not generated yet + assertNull(esp1.getHash()); + assertNull(esp2.getHash()); + + // Generate hashes + esp1.generateHash(util); + esp2.generateHash(util); + + + String hash1 = esp1.getHash(); + String hash2 = esp2.getHash(); + assertNotNull(hash1); + assertNotNull(hash2); + assertEquals(hash1, hash2); + } + + @Test + public void testExtractedParametersDifferentOrders() throws Exception { + ParameterHashUtil util = new ParameterHashUtil(); + Instant instant = Instant.now(); + + // Define some search parameter values + StringParmVal p1 = new StringParmVal(); + p1.setResourceType("Patient"); + p1.setName("code1"); + p1.setUrl("url1"); + p1.setVersion("version1"); + p1.setValueString("valueString1"); + + TokenParmVal p2 = new TokenParmVal(); + p2.setResourceType("Patient"); + p2.setName("code2"); + p2.setUrl("url2"); + p2.setVersion("version2"); + p2.setValueSystem("valueSystem2"); + p2.setValueCode("valueCode2"); + + ReferenceParmVal p3 = new ReferenceParmVal(); + p3.setResourceType("Patient"); + p3.setName("code3"); + p3.setUrl("url3"); + p3.setVersion("version3"); + p3.setRefValue(new ReferenceValue("Patient", "value3", ReferenceType.LOGICAL, 1)); + + QuantityParmVal p4 = new QuantityParmVal(); + p4.setResourceType("Patient"); + p4.setName("code4"); + p4.setUrl("url4"); + p4.setVersion("version4"); + p4.setValueNumber(new BigDecimal(4)); + p4.setValueNumberLow(new BigDecimal(3.9)); + p4.setValueNumberHigh(new BigDecimal(4.1)); + p4.setValueCode("valueCode4"); + p4.setValueSystem("valueSystem4"); + + NumberParmVal p5 = new NumberParmVal(); + p5.setResourceType("Patient"); + p5.setName("code5"); + p5.setUrl("url5"); + p5.setVersion("version5"); + BigDecimal value5 = new BigDecimal(5); + p5.setValueNumber(value5); + p5.setValueNumberLow(NumberParmBehaviorUtil.generateLowerBound(value5)); + p5.setValueNumberHigh(NumberParmBehaviorUtil.generateUpperBound(value5)); + + LocationParmVal p6 = new LocationParmVal(); + p6.setResourceType("Patient"); + p6.setName("code6"); + p6.setUrl("url6"); + p6.setVersion("version6"); + p6.setValueLatitude(6.6); + p6.setValueLongitude(60.6); + + DateParmVal p7 = new DateParmVal(); + p7.setResourceType("Patient"); + p7.setName("code7"); + p7.setUrl("url7"); + p7.setVersion("version7"); + p7.setValueDateStart(DateTimeHandler.generateTimestamp(instant)); + p7.setValueDateEnd(DateTimeHandler.generateTimestamp(instant.plusSeconds(1))); + + CompositeParmVal p8a = new CompositeParmVal(); + p8a.setResourceType("Patient"); + p8a.setName("code8"); + p8a.setUrl("url8"); + p8a.setVersion("version8"); + p8a.addComponent(p1); + p8a.addComponent(p5); + + CompositeParmVal p8b = new CompositeParmVal(); + p8b.setResourceType("Patient"); + p8b.setName("code8"); + p8b.setUrl("url8"); + p8b.setVersion("version8"); + p8b.addComponent(p5); + p8b.addComponent(p1); + + CompositeParmVal p8diff = new CompositeParmVal(); + p8diff.setResourceType("Patient"); + p8diff.setName("code8"); + p8diff.setUrl("url8"); + p8diff.setVersion("version8"); + p8diff.addComponent(p4); + p8diff.addComponent(p6); + + // Add the search parameters in different orders for esp1 and esp2, but should still result in same hash + // But for esp3 and esp4, the values are different, so they should not match the others + ExtractedSearchParameters esp1 = new ExtractedSearchParameters(); + esp1.getParameters().addAll(Arrays.asList(p1, p2, p3, p4, p5, p6, p7, p8a)); + ExtractedSearchParameters esp2 = new ExtractedSearchParameters(); + esp2.getParameters().addAll(Arrays.asList(p8b, p6, p4, p2, p1, p3, p5, p7)); + ExtractedSearchParameters esp3 = new ExtractedSearchParameters(); + esp3.getParameters().addAll(Arrays.asList(p1, p2, p3, p4, p5, p6, p7, p8diff)); + ExtractedSearchParameters esp4 = new ExtractedSearchParameters(); + + // Hashes not generated yet + assertNull(esp1.getHash()); + assertNull(esp2.getHash()); + assertNull(esp3.getHash()); + assertNull(esp4.getHash()); + + // Generate hashes + esp1.generateHash(util); + esp2.generateHash(util); + esp3.generateHash(util); + esp4.generateHash(util); + + String hash1 = esp1.getHash(); + String hash2 = esp2.getHash(); + String hash3 = esp3.getHash(); + String hash4 = esp4.getHash(); + assertNotNull(hash1); + assertNotNull(hash2); + assertNotNull(hash3); + assertNotNull(hash4); + assertEquals(hash1, hash2); + assertNotEquals(hash1, hash3); + assertNotEquals(hash1, hash4); + assertNotEquals(hash3, hash4); + } +}