From d8629367780a831192e6c0e5801b41b18e03097b Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Wed, 18 Nov 2020 17:56:49 -0500 Subject: [PATCH 1/8] issue #1708 add custom compartment reference params for faster compartment searches Signed-off-by: Robin Arnold --- .../dao/impl/ParameterVisitorBatchDAO.java | 143 ++-------- .../jdbc/dto/ReferenceParmVal.java | 37 ++- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 57 ++++ .../util/JDBCParameterBuildingVisitor.java | 45 ++- .../test/util/ParameterExtractionTest.java | 139 +++++----- .../search/compartment/CompartmentUtil.java | 146 +++++++--- .../compartment/ResourceCompartmentCache.java | 64 +++++ .../reference/value/CompartmentReference.java | 74 +++++ .../ibm/fhir/search/util/ReferenceUtil.java | 251 +++++++++++++++++ .../ibm/fhir/search/util/ReferenceValue.java | 74 +++++ .../com/ibm/fhir/search/util/SearchUtil.java | 66 ++++- .../compartment/CompartmentUtilTest.java | 47 +++- .../search/reference/ReferenceUtilTest.java | 257 ++++++++++++++++++ 13 files changed, 1137 insertions(+), 263 deletions(-) create mode 100644 fhir-search/src/main/java/com/ibm/fhir/search/compartment/ResourceCompartmentCache.java create mode 100644 fhir-search/src/main/java/com/ibm/fhir/search/reference/value/CompartmentReference.java create mode 100644 fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java create mode 100644 fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceValue.java create mode 100644 fhir-search/src/test/java/com/ibm/fhir/search/reference/ReferenceUtilTest.java diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index 9d4803932f7..7597a67ebfb 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -9,8 +9,6 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UTC; import java.math.BigDecimal; -import java.net.MalformedURLException; -import java.net.URL; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -24,7 +22,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; @@ -41,6 +38,8 @@ import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.schema.control.FhirSchemaConstants; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; /** * Batch insert into the parameter values tables. Avoids having to create one stored procedure @@ -51,11 +50,6 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, AutoCloseable { private static final Logger logger = Logger.getLogger(ParameterVisitorBatchDAO.class.getName()); - private static final String HISTORY = "_history"; - private static final String HTTP = "http:"; - private static final String HTTPS = "https:"; - private static final String URN = "urn:"; - // the connection to use for the inserts private final Connection connection; @@ -118,9 +112,6 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, // The common cache for all our identity lookup needs private final JDBCIdentityCache identityCache; - // The server base "https://example.com:9443/" extracted the first time we need it - private String serverBase; - // If not null, we stash certain parameter data here for insertion later private final ParameterTransactionDataImpl transactionData; @@ -746,58 +737,9 @@ private boolean isBase(ExtractedParameterValue param) { return "Resource".equals(param.getBase()); } - /** - * Get the leading part of the url e.g. https://example.com - * @return - */ - private String getServerUrl() throws FHIRPersistenceException { - - if (this.serverBase != null) { - return this.serverBase; - } - - String uri = FHIRRequestContext.get().getOriginalRequestUri(); - - // request URI is not set for all unit-tests, so we need to take that into account - if (uri == null) { - return null; - } - - try { - StringBuilder result = new StringBuilder(); - URL url = new URL(uri); - - result.append(url.getProtocol()); - result.append("://"); - result.append(url.getHost()); - - if (url.getPort() != -1) { - result.append(":"); - result.append(url.getPort()); - } - - // https://example.com:9443/ - result.append("/"); - - final String endpoint = "fhir-server/api/v4/"; - if (uri.contains(endpoint)) { - result.append(endpoint); - } - - // Cache the result so we don't have to compute it over and over - this.serverBase = result.toString(); - return this.serverBase; - } catch (MalformedURLException x) { - // not very likely at this point - logger.severe("Malformed server URL: " + uri); - throw new FHIRPersistenceException("Server URL is malformed!"); - } - } - @Override public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { - String valueString = rpv.getValueString(); - if (valueString == null || valueString.isEmpty()) { + if (rpv.getRefValue() == null) { return; } @@ -809,66 +751,31 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { throw new FHIRPersistenceException("Resource type not found in cache: '" + resourceType + "'"); } - final String base = getServerUrl(); - ResourceTokenValueRec rec; - if (base != null && valueString.startsWith(base)) { - // - relative reference https://example.com/Patient/123 - // Because this reference is to a local FHIR resource (inside this server), we need use the correct - // resource type name (assigned as the code system) - // - https://localhost:9443/fhir-server/api/v4/Patient/1234 - // - https://example.com/Patient/1234 - // - https://example.com/Patient/1234/_history/2 - valueString = valueString.substring(base.length()); - - // Patient/1234 - // Patient/1234/_history/2 - String[] tokens = valueString.split("/"); - if (tokens.length > 1) { - String refResourceType = tokens[0]; - String refLogicalId = tokens[1]; - Integer refVersion = null; - if (tokens.length == 4 && HISTORY.equals(tokens[2])) { - // versioned reference - refVersion = Integer.parseInt(tokens[3]); - } - // Store a token value configured as a reference to another resource - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, refResourceType, refLogicalId, refVersion); - } else { - // stored as a token with the default system - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, TokenParmVal.DEFAULT_TOKEN_SYSTEM, valueString); - } - } else if (valueString.startsWith(HTTP) || valueString.startsWith(HTTPS) || valueString.startsWith(URN)) { - // - absolute URL ==> http://some.system/a/fhir/resource/path - // - absolute URI ==> urn:uuid:53fefa32-1111-2222-3333-55ee120877b7 - // stored as a token with the default system - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, TokenParmVal.DEFAULT_TOKEN_SYSTEM, valueString); - } else if (valueString.startsWith("#")) { - // - Internal ==> #fragmentid1 - // stored as a token value with the default system - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, TokenParmVal.DEFAULT_TOKEN_SYSTEM, valueString); - } else { - // - Relative ==> Patient/1234 - // - Relative ==> Patient/1234/_history/2 - String[] tokens = valueString.split("/"); - if (tokens.length > 1) { - String refResourceType = tokens[0]; - String refLogicalId = tokens[1]; - Integer refVersion = null; - if (tokens.length == 4 && HISTORY.equals(tokens[2])) { - // versioned reference - refVersion = Integer.parseInt(tokens[3]); - } + // The ReferenceValue has already been processed to convert the reference to + // the required standard form, ready for insertion as a token value. + ReferenceValue refValue = rpv.getRefValue(); - // Store a token value configured as a reference to another resource - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, refResourceType, refLogicalId, refVersion); + // Ignore references containing only a "display" element (apparently supported by the spec, + // but contains nothing useful to store because there's no searchable value). + String refResourceType = refValue.getTargetResourceType(); + String refLogicalId = refValue.getValue(); + Integer refVersion = refValue.getVersion(); + ResourceTokenValueRec rec; - } else { - // SearchReferenceTest system integration tests require support for arbitrary reference strings - // - Relative ==> 1234 - final String codeSystem = TokenParmVal.DEFAULT_TOKEN_SYSTEM; - rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, codeSystem, valueString); - } + if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { + // protect against code regression. Invalid/improper references should be + // filtered out already. + logger.warning("Invalid reference parameter type: " + resourceType + "." + rpv.getName() + " type=" + refValue.getType().name()); + throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); + } + + if (refResourceType != null) { + // Store a token value configured as a reference to another resource + rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, refResourceType, refLogicalId, refVersion); + } else { + // stored as a token with the default system + rec = new ResourceTokenValueRec(parameterNameId, resourceType, resourceTypeId, logicalResourceId, TokenParmVal.DEFAULT_TOKEN_SYSTEM, refLogicalId); } if (this.transactionData != null) { diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java index e8f49e4c328..a08d2abfef1 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java @@ -8,6 +8,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.search.SearchConstants.Type; +import com.ibm.fhir.search.util.ReferenceValue; /** * DTO representing external and local reference parameters @@ -21,11 +22,14 @@ public class ReferenceParmVal implements ExtractedParameterValue { private String name; // The reference value - private String valueString; + //private String valueString; // The SearchParameter base type. If "Resource", then this is a Resource-level attribute private String base; + // The value of the reference after it has been processed to determine target resource type, version etc. + private ReferenceValue refValue; + /** * Public constructor */ @@ -41,19 +45,40 @@ public String getName() { return name; } - public String getValueString() { - return valueString; - } +// public String getValueString() { +// return valueString; +// } + +// public void setValueString(String valueString) { +// this.valueString = valueString; +// } - public void setValueString(String valueString) { - this.valueString = valueString; + /** + * Get the refValue + * @return + */ + public ReferenceValue getRefValue() { + return this.refValue; } + /** + * Set the refValue + * @param refValue + */ + public void setRefValue(ReferenceValue refValue) { + this.refValue = refValue; + } + /** + * Get the reference type of the parameter (the origin, not the target of the reference) + */ public String getResourceType() { return resourceType; } + /** + * Set the reference type of the parameter (the origin, not the target of the reference) + */ public void setResourceType(String resourceType) { this.resourceType = resourceType; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 714719e4271..1d0279123d9 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -120,6 +120,7 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.NumberParmVal; import com.ibm.fhir.persistence.jdbc.dto.QuantityParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; @@ -138,9 +139,14 @@ import com.ibm.fhir.search.SearchConstants; import com.ibm.fhir.search.SearchConstants.Modifier; import com.ibm.fhir.search.SummaryValueSet; +import com.ibm.fhir.search.compartment.CompartmentUtil; import com.ibm.fhir.search.context.FHIRSearchContext; import com.ibm.fhir.search.date.DateTimeHandler; +import com.ibm.fhir.search.exception.FHIRSearchException; import com.ibm.fhir.search.parameters.QueryParameter; +import com.ibm.fhir.search.reference.value.CompartmentReference; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; import com.ibm.fhir.search.util.SearchUtil; /** @@ -1227,6 +1233,21 @@ private TransactionSynchronizationRegistry getTrxSynchRegistry() throws FHIRPers } } + private List extractCompartmentValues(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTO) throws Exception { + + List result = null; + + Map> compartmentRefParams = CompartmentUtil.getCompartmentParamsForResourceType(fhirResource.getClass().getSimpleName()); + + if (!compartmentRefParams.isEmpty()) { + //result = SearchUtil.extractCompartmentParameterValues(fhirResource, compartmentRefParams); + } else { + result = Collections.emptyList(); + } + + return result; + } + /** * Extracts search parameters for the passed FHIR Resource. * @param fhirResource - Some FHIR Resource @@ -1442,12 +1463,48 @@ private List extractSearchParameters(Resource fhirResou } } } + + // Augment the extracted parameter list with special values we use to represent compartment relationships. + // These references are stored as tokens and are used by the search query builder + // for compartment-based searches + addCompartmentParams(allParameters, fhirResource); } finally { log.exiting(CLASSNAME, METHODNAME); } return allParameters; } + /** + * Augment the given list with additional reference values + * @param allParameters + */ + protected void addCompartmentParams(List allParameters, Resource fhirResource) throws FHIRSearchException { + final String resourceType = fhirResource.getClass().getSimpleName(); + Map> compartmentRefParams = CompartmentUtil.getCompartmentParamsForResourceType(resourceType); + Map> compartmentMap = SearchUtil.extractCompartmentParameterValues(fhirResource, compartmentRefParams); + + for (Map.Entry> entry: compartmentMap.entrySet()) { + final String compartmentName = entry.getKey(); + final String parameterName = CompartmentUtil.makeCompartmentParamName(compartmentName); + + // Create a reference parameter value for each CompartmentReference extracted from the resource + for (CompartmentReference compartmentRef: entry.getValue()) { + ReferenceParmVal pv = new ReferenceParmVal(); + pv.setName(parameterName); + pv.setResourceType(resourceType); + + // ReferenceType doesn't really matter here, but LITERAL_RELATIVE is appropriate + ReferenceValue rv = new ReferenceValue(compartmentName, compartmentRef.getReferenceResourceValue(), ReferenceType.LITERAL_RELATIVE, null); + pv.setRefValue(rv); + + if (log.isLoggable(Level.FINE)) { + log.fine("Adding compartment reference parameter: [" + resourceType + "] "+ parameterName + " = " + rv.getTargetResourceType() + "/" + rv.getValue()); + } + allParameters.add(pv); + } + } + } + /** * Create a Parameter DTO from the primitive value. * Note: this method only sets the value; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java index 70b5ac553a8..d9a816f66dd 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java @@ -57,6 +57,10 @@ import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; import com.ibm.fhir.persistence.jdbc.util.type.NumberParmBehaviorUtil; import com.ibm.fhir.search.date.DateTimeHandler; +import com.ibm.fhir.search.exception.FHIRSearchException; +import com.ibm.fhir.search.util.ReferenceUtil; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; /** * This class is the JDBC persistence layer implementation for transforming @@ -597,24 +601,35 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r if (!REFERENCE.equals(searchParamType)) { throw invalidComboException(searchParamType, reference); } - if (reference.getReference() != null) { - ReferenceParmVal p = new ReferenceParmVal(); - p.setName(searchParamCode); - p.setValueString(reference.getReference().getValue()); - result.add(p); - } - // Make sure we process the identifier if there is one. - Identifier identifier = reference.getIdentifier(); - if (reference.getIdentifier() != null) { - TokenParmVal p = new TokenParmVal(); - p.setName(searchParamCode); - if (identifier.getSystem() != null) { - p.setValueSystem(identifier.getSystem().getValue()); + // TODO pass in the bundle if we want to support "a relative URL, which is relative to + // the Service Base URL, or, if processing a resource from a bundle, which is relative + // to the base URL implied by the Bundle.entry.fullUrl (see Resolving References in Bundles)" + try { + final String baseUrl = ReferenceUtil.getBaseUrl(null); + ReferenceValue refValue = ReferenceUtil.createReferenceValueFrom(reference, baseUrl); + if (refValue.getType() != ReferenceType.INVALID && refValue.getType() != ReferenceType.DISPLAY_ONLY) { + ReferenceParmVal p = new ReferenceParmVal(); + p.setRefValue(refValue); + p.setName(searchParamCode); + result.add(p); } - p.setValueCode(identifier.getValue().getValue()); - result.add(p); + } catch (FHIRSearchException x) { + // Log the error, but skip it because we're not supposed to throw exceptions here + log.log(Level.WARNING, "Error processing reference", x); } + + // Make sure we process the identifier if there is one. +// Identifier identifier = reference.getIdentifier(); +// if (reference.getIdentifier() != null) { +// TokenParmVal p = new TokenParmVal(); +// p.setName(searchParamCode); +// if (identifier.getSystem() != null) { +// p.setValueSystem(identifier.getSystem().getValue()); +// } +// p.setValueCode(identifier.getValue().getValue()); +// result.add(p); +// } return false; } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java index 1f01669bac3..55af4b1168a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java @@ -73,7 +73,9 @@ public class ParameterExtractionTest { private static final String SAMPLE_STRING = "test"; private static final String SAMPLE_URI = "http://example.com"; private static final String SAMPLE_UNIT = "s"; - private static final String SAMPLE_REF = "abc"; + private static final String SAMPLE_REF_RESOURCE_TYPE = "Patient"; + private static final String SAMPLE_REF_ID = "abc"; + private static final String SAMPLE_REF = SAMPLE_REF_RESOURCE_TYPE + "/" + SAMPLE_REF_ID; private static final String SAMPLE_DATE_START = "2016-01-01T00:00:00.000000Z"; private static final String SAMPLE_DATE_END = "2016-01-02T00:00:00.000000Z"; private static final String UNITSOFMEASURE = "http://unitsofmeasure.org"; @@ -86,7 +88,7 @@ public class ParameterExtractionTest { .appendFraction(ChronoField.MICRO_OF_SECOND, 6, 6, true) .appendPattern("XXX") .toFormatter(); - + private static final SearchParameter.Builder searchParamBuilder = SearchParameter.builder() .url(Uri.of("http://ibm.com/fhir/test")) .name(string("test-param")) @@ -101,12 +103,12 @@ public class ParameterExtractionTest { private static final SearchParameter uriSearchParam = searchParamBuilder.type(SearchParamType.URI).build(); private static final SearchParameter stringSearchParam = searchParamBuilder.type(SearchParamType.STRING).build(); private static final SearchParameter tokenSearchParam = searchParamBuilder.type(SearchParamType.TOKEN).build(); - + @BeforeClass public void setSystemTimeZone() { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); } - + @Test public void testBoolean() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -114,40 +116,40 @@ public void testBoolean() throws FHIRPersistenceProcessorException { List params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), "true"); - + assertNullValueReturnsNoParameters(tokenSearchParam, com.ibm.fhir.model.type.Boolean.builder()); } - + @Test public void testBoolean_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, com.ibm.fhir.model.type.Boolean.builder()); } - + @Test public void testCanonical() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder; Canonical canonical = Canonical.of(SAMPLE_URI); List params; - + parameterBuilder = new JDBCParameterBuildingVisitor(referenceSearchParam); canonical.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((StringParmVal) params.get(0)).getValueString(), SAMPLE_URI); - + parameterBuilder = new JDBCParameterBuildingVisitor(uriSearchParam); canonical.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((StringParmVal) params.get(0)).getValueString(), SAMPLE_URI); } - + @Test public void testCanonical_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(referenceSearchParam, Canonical.builder()); assertNullValueReturnsNoParameters(uriSearchParam, Canonical.builder()); } - + @Test public void testCode() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -156,12 +158,12 @@ public void testCode() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), SAMPLE_STRING); } - + @Test public void testCode_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, Code.builder()); } - + @Test public void testDate() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -173,7 +175,7 @@ public void testDate() throws FHIRPersistenceProcessorException { assertEquals(timestampToString(dateParam.getValueDateStart()), SAMPLE_DATE_START); } } - + @Test public void testDate_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(dateSearchParam, Date.builder()); @@ -188,12 +190,12 @@ public void testDateTime() throws FHIRPersistenceProcessorException { assertEquals(timestampToString(((DateParmVal) param).getValueDateStart()), "2016-01-01T06:10:10.100000Z"); } } - + @Test public void testDateTime_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(dateSearchParam, DateTime.builder()); } - + @Test public void testDecimal() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(numberSearchParam); @@ -204,12 +206,12 @@ public void testDecimal() throws FHIRPersistenceProcessorException { assertEquals(((NumberParmVal) params.get(0)).getValueNumberLow().doubleValue(), 99.985); assertEquals(((NumberParmVal) params.get(0)).getValueNumberHigh().doubleValue(), 99.995); } - + @Test public void testDecimal_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(numberSearchParam, Decimal.builder()); } - + @Test public void testId() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -218,12 +220,12 @@ public void testId() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), "x"); } - + @Test public void testId_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, Id.builder()); } - + @Test public void testInstant() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -233,12 +235,12 @@ public void testInstant() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(timestampToString(((DateParmVal) params.get(0)).getValueDateStart()), TIMESTAMP_FORMATTER.format(now.getValue())); } - + @Test public void testInstant_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(dateSearchParam, Instant.builder()); } - + @Test public void testInteger() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(numberSearchParam); @@ -247,70 +249,70 @@ public void testInteger() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((NumberParmVal) params.get(0)).getValueNumber().intValue(), 13); } - + @Test public void testInteger_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(numberSearchParam, Integer.builder()); } - + @Test public void testString() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder; com.ibm.fhir.model.type.String stringVal = string(SAMPLE_STRING); List params; - + parameterBuilder = new JDBCParameterBuildingVisitor(stringSearchParam); stringVal.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((StringParmVal) params.get(0)).getValueString(), SAMPLE_STRING); - + parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); stringVal.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), SAMPLE_STRING); } - + @Test public void testString_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(stringSearchParam, com.ibm.fhir.model.type.String.builder()); assertNullValueReturnsNoParameters(tokenSearchParam, com.ibm.fhir.model.type.String.builder()); } - + @Test public void testUri() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder; Uri uri = Uri.of(SAMPLE_URI); List params; - + parameterBuilder = new JDBCParameterBuildingVisitor(referenceSearchParam); uri.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), SAMPLE_URI); - + parameterBuilder = new JDBCParameterBuildingVisitor(uriSearchParam); uri.accept(parameterBuilder); params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((StringParmVal) params.get(0)).getValueString(), SAMPLE_URI); } - + @Test public void testUri_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(referenceSearchParam, Uri.builder()); assertNullValueReturnsNoParameters(uriSearchParam, Uri.builder()); } - + private void assertNullValueReturnsNoParameters(SearchParameter sp, Element.Builder builder) { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(sp); builder.extension(SAMPLE_EXTENSION).build().accept(parameterBuilder); List params = parameterBuilder.getResult(); assertEquals(params.size(), 0, "Number of extracted parameters"); } - - + + @Test public void testAddress() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(stringSearchParam); @@ -330,12 +332,12 @@ public void testAddress() throws FHIRPersistenceProcessorException { assertEquals(((StringParmVal) params.get(3)).getValueString(), "27703"); assertEquals(((StringParmVal) params.get(4)).getValueString(), "4025 S. Miami Blvd., Durham, NC 27703"); } - + @Test public void testAddress_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(stringSearchParam, Address.builder()); } - + @Test public void testAge() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -353,12 +355,12 @@ public void testAge() throws FHIRPersistenceProcessorException { assertEquals(quantParam.getValueSystem(), UNITSOFMEASURE); assertEquals(quantParam.getValueCode(), "a"); } - + @Test public void testAge_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(quantitySearchParam, Age.builder()); } - + @Test public void testCodeableConcept() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -377,12 +379,12 @@ public void testCodeableConcept() throws FHIRPersistenceProcessorException { assertEquals(((TokenParmVal) params.get(2)).getValueCode(), "c"); assertEquals(((TokenParmVal) params.get(2)).getValueSystem(), SAMPLE_URI); } - + @Test public void testCodeableConcept_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, CodeableConcept.builder()); } - + @Test public void testCoding() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -396,12 +398,12 @@ public void testCoding() throws FHIRPersistenceProcessorException { assertEquals(((TokenParmVal) params.get(0)).getValueCode(), SAMPLE_STRING); assertEquals(((TokenParmVal) params.get(0)).getValueSystem(), SAMPLE_URI); } - + @Test public void testCoding_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, Coding.builder()); } - + @Test public void testContactPoint() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -414,12 +416,12 @@ public void testContactPoint() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), "5558675309"); } - + @Test public void testContactPoint_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, ContactPoint.builder()); } - + @Test public void testDuration() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -435,12 +437,12 @@ public void testDuration() throws FHIRPersistenceProcessorException { assertEquals(((QuantityParmVal) params.get(0)).getValueSystem(), UNITSOFMEASURE); assertEquals(((QuantityParmVal) params.get(0)).getValueCode(), SAMPLE_UNIT); } - + @Test public void testDuration_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(quantitySearchParam, Duration.builder()); } - + @Test public void testHumanName() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(stringSearchParam); @@ -460,12 +462,12 @@ public void testHumanName() throws FHIRPersistenceProcessorException { assertEquals(((StringParmVal) params.get(3)).getValueString(), "III"); assertEquals(((StringParmVal) params.get(4)).getValueString(), "Dr. Nick"); } - + @Test public void testHumanName_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(stringSearchParam, HumanName.builder()); } - + @Test public void testIdentifier() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(tokenSearchParam); @@ -479,12 +481,12 @@ public void testIdentifier() throws FHIRPersistenceProcessorException { assertEquals(((TokenParmVal) params.get(0)).getValueSystem(), SAMPLE_URI); assertEquals(((TokenParmVal) params.get(0)).getValueCode(), "abc123"); } - + @Test public void testIdentifier_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(tokenSearchParam, Identifier.builder()); } - + @Test public void testMoney() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -498,12 +500,12 @@ public void testMoney() throws FHIRPersistenceProcessorException { assertEquals(((QuantityParmVal) params.get(0)).getValueCode(), "USD"); assertEquals(((QuantityParmVal) params.get(0)).getValueNumber().intValue(), 100); } - + @Test public void testMoney_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(quantitySearchParam, Money.builder()); } - + @Test public void testPeriod() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -517,7 +519,7 @@ public void testPeriod() throws FHIRPersistenceProcessorException { assertEquals(timestampToString(((DateParmVal) params.get(0)).getValueDateStart()), SAMPLE_DATE_START); assertEquals(timestampToString(((DateParmVal) params.get(0)).getValueDateEnd()), SAMPLE_DATE_END); } - + @Test public void testPeriod_nullStart() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -529,7 +531,7 @@ public void testPeriod_nullStart() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(timestampToString(((DateParmVal) params.get(0)).getValueDateEnd()), SAMPLE_DATE_END); } - + @Test public void testPeriod_nullEnd() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -541,12 +543,12 @@ public void testPeriod_nullEnd() throws FHIRPersistenceProcessorException { assertEquals(params.size(), 1, "Number of extracted parameters"); assertEquals(timestampToString(((DateParmVal) params.get(0)).getValueDateStart()), SAMPLE_DATE_START); } - + @Test public void testPeriod_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(dateSearchParam, Period.builder()); } - + @Test public void testQuantity() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -562,12 +564,12 @@ public void testQuantity() throws FHIRPersistenceProcessorException { assertEquals(((QuantityParmVal) params.get(0)).getValueSystem(), UNITSOFMEASURE); assertEquals(((QuantityParmVal) params.get(0)).getValueCode(), SAMPLE_UNIT); } - + @Test public void testQuantity_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(quantitySearchParam, Quantity.builder()); } - + @Test public void testRange() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -594,7 +596,7 @@ public void testRange() throws FHIRPersistenceProcessorException { assertNull(quantParam.getValueNumber()); assertEquals(quantParam.getValueNumberHigh(), BigDecimal.valueOf(2)); } - + @Test public void testRange_nullHigh() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -615,7 +617,7 @@ public void testRange_nullHigh() throws FHIRPersistenceProcessorException { assertNull(quantParam.getValueNumber()); assertNull(quantParam.getValueNumberHigh()); } - + @Test public void testRange_nullLow() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(quantitySearchParam); @@ -636,12 +638,12 @@ public void testRange_nullLow() throws FHIRPersistenceProcessorException { assertNull(quantParam.getValueNumber()); assertEquals(quantParam.getValueNumberHigh(), BigDecimal.valueOf(1)); } - + @Test public void testRange_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(quantitySearchParam, Range.builder()); } - + @Test public void testReference() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(referenceSearchParam); @@ -651,14 +653,15 @@ public void testReference() throws FHIRPersistenceProcessorException { .accept(parameterBuilder); List params = parameterBuilder.getResult(); assertEquals(params.size(), 1, "Number of extracted parameters"); - assertEquals(((ReferenceParmVal) params.get(0)).getValueString(), SAMPLE_REF); + assertEquals(((ReferenceParmVal) params.get(0)).getRefValue().getValue(), SAMPLE_REF_ID); + assertEquals(((ReferenceParmVal) params.get(0)).getRefValue().getTargetResourceType(), SAMPLE_REF_RESOURCE_TYPE); } - + @Test public void testReference_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(referenceSearchParam, Reference.builder()); } - + @Test public void testTimingBounds() throws FHIRPersistenceProcessorException { JDBCParameterBuildingVisitor parameterBuilder = new JDBCParameterBuildingVisitor(dateSearchParam); @@ -676,12 +679,12 @@ public void testTimingBounds() throws FHIRPersistenceProcessorException { assertEquals(timestampToString(((DateParmVal) param).getValueDateStart()), SAMPLE_DATE_START); assertEquals(timestampToString(((DateParmVal) param).getValueDateEnd()), SAMPLE_DATE_END); } - + @Test public void testTiming_null() throws FHIRPersistenceProcessorException { assertNullValueReturnsNoParameters(dateSearchParam, Timing.builder()); } - + // Timing doesn't currently extract from "events" // @Test // public void testTimingEvents() throws FHIRPersistenceProcessorException { @@ -697,7 +700,7 @@ public void testTiming_null() throws FHIRPersistenceProcessorException { // assertEquals(timestampToString(((DateParameter) param).getValueDateStart()), SAMPLE_DATE_START); // assertEquals(timestampToString(((DateParameter) param).getValueDateEnd()), SAMPLE_DATE_END); // } - + /** * Formats the given tstamp value as a string. Does not use Timestamp#toString() * because this adjusts the displayed string to local time diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/compartment/CompartmentUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/compartment/CompartmentUtil.java index 4c2ad4cc2bb..6b549a9515b 100644 --- a/fhir-search/src/main/java/com/ibm/fhir/search/compartment/CompartmentUtil.java +++ b/fhir-search/src/main/java/com/ibm/fhir/search/compartment/CompartmentUtil.java @@ -64,7 +64,7 @@ public class CompartmentUtil { public static final String FHIR_PATH_BUNDLE_ENTRY = "entry.children()"; public static final String RESOURCE = "/compartments.json"; - // List of compartmentDefintions. + // List of compartmentDefinitions. private static final Set compartmentDefinitions = new HashSet() { private static final long serialVersionUID = 7152515293380769882L; @@ -78,6 +78,18 @@ public class CompartmentUtil { } }; + // Map of Compartment name to CompartmentCache + private static final Map compartmentMap = new HashMap<>(); + + // Map of Resource type to ResourceCompartmentCache + private static final Map resourceCompartmentMap = new HashMap<>(); + + static { + // make one pass over the configuration to build both maps + buildMaps(compartmentMap, resourceCompartmentMap); + } + + /** * Loads the class in the classloader to initialize static members. Call this before using the class in order to * avoid a slight performance hit on first use. @@ -97,68 +109,83 @@ private CompartmentUtil() { // No Operation } - private static final Map compartmentMap = buildCompartmentMap(); + /** + * Builds an in-memory model of the Compartment map defined in compartments.json, for supporting compartment based + * FHIR searches. + * @implNote the maps being built are passed in as arguments to aid unit testing + * @param compMap map of compartment name to CompartmentCache + * @param resourceCompMap map of resource type name to ResourceCompartmentCache + * @throws IOException + */ + public static final void buildMaps(Map compMap, Map resourceCompMap) { + buildMaps(RESOURCE, compMap, resourceCompMap); + } /** * Builds an in-memory model of the Compartment map defined in compartments.json, for supporting compartment based * FHIR searches. - * - * @return a map of compartment caches + * @implNote the maps being built are passed in as arguments to aid unit testing + * @param the source resource to be read using getResourceAsStream + * @param compMap map of compartment name to CompartmentCache + * @param resourceCompMap map of resource type name to ResourceCompartmentCache * @throws IOException */ - public static final Map buildCompartmentMap() { - Map cachedCompartmentMap = compartmentMap; + public static final void buildMaps(String source, Map compMap, Map resourceCompMap) { + if (compMap == null) { + throw new IllegalArgumentException("compMap must not be null"); + } - if (cachedCompartmentMap == null) { - // If cachedCompartmentMap is empty, there is something else going on. + if (resourceCompMap == null) { + throw new IllegalArgumentException("resourceCompMap must not be null"); + } - cachedCompartmentMap = new HashMap<>(); + try (InputStreamReader reader = new InputStreamReader(CompartmentUtil.class.getResourceAsStream(source))) { + Bundle bundle = FHIRParser.parser(Format.JSON).parse(reader); - try (InputStreamReader reader = new InputStreamReader(CompartmentUtil.class.getResourceAsStream(RESOURCE))) { - Bundle bundle = FHIRParser.parser(Format.JSON).parse(reader); + FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); + EvaluationContext evaluationContext = new EvaluationContext(bundle); - FHIRPathEvaluator evaluator = FHIRPathEvaluator.evaluator(); - EvaluationContext evaluationContext = new EvaluationContext(bundle); + Collection result = evaluator.evaluate(evaluationContext, FHIR_PATH_BUNDLE_ENTRY); - Collection result = evaluator.evaluate(evaluationContext, FHIR_PATH_BUNDLE_ENTRY); + Iterator iter = result.iterator(); + while (iter.hasNext()) { + FHIRPathResourceNode node = iter.next().asResourceNode(); - Iterator iter = result.iterator(); - while (iter.hasNext()) { - FHIRPathResourceNode node = iter.next().asResourceNode(); + // Convert to Resource and lookup. + CompartmentDefinition compartmentDefinition = node.resource().as(CompartmentDefinition.class); + String compartmentName = compartmentDefinition.getCode().getValue(); - // Convert to Resource and lookup. - CompartmentDefinition compartmentDefinition = node.resource().as(CompartmentDefinition.class); + // The cached object (a smaller/lighter lookup resource) used for point lookups + CompartmentCache compartmentDefinitionCache = new CompartmentCache(); - // The cached object (a smaller/lighter lookup resource) used for point lookups - CompartmentCache compartmentDefinitionCache = new CompartmentCache(); + // Iterates over the resources embedded in the CompartmentDefinition. + for (Resource resource : compartmentDefinition.getResource()) { + String inclusionCode = resource.getCode().getValue(); + List params = resource.getParam(); + // Make sure to only add the valid resource types (at least with one inclusion) instead of all types. + if (!params.isEmpty()) { + compartmentDefinitionCache.add(inclusionCode, params); - // Iterates over the resources embedded in the CompartmentDefinition. - for (Resource resource : compartmentDefinition.getResource()) { - String inclusionCode = resource.getCode().getValue(); - List params = resource.getParam(); - // Make sure to only add the valid resource types (at least with one inclusion) instead of all types. - if (!params.isEmpty()) { - compartmentDefinitionCache.add(inclusionCode, params); + // Look up the ResourceCompartmentCache and create a new one if needed + ResourceCompartmentCache rcc = resourceCompMap.get(inclusionCode); + if (rcc == null) { + rcc = new ResourceCompartmentCache(); + resourceCompMap.put(inclusionCode, rcc); } - } - - String codeCacheName = compartmentDefinition.getCode().getValue(); - cachedCompartmentMap.put(codeCacheName, compartmentDefinitionCache); + // Add the mapping for this parameter to the target compartment name + rcc.add(params, compartmentName); + } } - // Make unmodifiable. - cachedCompartmentMap = Collections.unmodifiableMap(cachedCompartmentMap); - - } catch (FHIRException e) { - log.warning(String.format(PARSE_EXCEPTION, FROM_STREAM)); - } catch (IOException e1) { - log.warning(String.format(IO_EXCEPTION, FROM_STREAM)); + compMap.put(compartmentName, compartmentDefinitionCache); } - } - - return cachedCompartmentMap; + } catch (FHIRException e) { + log.warning(String.format(PARSE_EXCEPTION, FROM_STREAM)); + } catch (IOException e1) { + log.warning(String.format(IO_EXCEPTION, FROM_STREAM)); + } } /** @@ -242,4 +269,37 @@ public static void buildCompositeBundle(PrintStream out) throws FHIRGeneratorExc } -} + /** + * Get the map of parameter names used as compartment references for the + * given resource type. For example for CareTeam: + * participant -> {RelatedPerson, Patient} + * patient -> {Patient} + * encounter -> {Encounter} + * ... + * etc. + * @param resourceType the resource type name + * @return a map of parameter name to set of compartment names + */ + public static Map> getCompartmentParamsForResourceType(java.lang.String resourceType) { + ResourceCompartmentCache rcc = resourceCompartmentMap.get(resourceType); + if (rcc != null) { + return rcc.getCompartmentReferenceParams(); + } else { + return Collections.emptyMap(); + } + } + + /** + * Create the special parameter name used for references to the given + * compartment (e.g. Patient, RelatedPerson etc). + * @param compartmentName + * @return + */ + public static String makeCompartmentParamName(String compartmentName) { + final StringBuilder result = new StringBuilder(); + result.append("ibm-internal-") + .append(compartmentName) + .append("-Compartment"); + return result.toString(); + } +} \ No newline at end of file diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/compartment/ResourceCompartmentCache.java b/fhir-search/src/main/java/com/ibm/fhir/search/compartment/ResourceCompartmentCache.java new file mode 100644 index 00000000000..cd8ea66efa5 --- /dev/null +++ b/fhir-search/src/main/java/com/ibm/fhir/search/compartment/ResourceCompartmentCache.java @@ -0,0 +1,64 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.search.compartment; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Resource Compartment Cache is a localized class to cache the compartment information for + * a specific resource type + */ +public class ResourceCompartmentCache { + + // Map of parameter name to a set of compartment names + private Map> paramCompartmentMap = new HashMap<>(); + + /** + * constructor + */ + public ResourceCompartmentCache() { + super(); + } + + /** + * Add the parameters which point to the given compartment. Note that the + * same parameter can be used to point to more than compartment, e.g. for + * CareTeam, the participant parameter may refer to a Patient or a RelatedPerson. + * In the schema, we therefore have to store (unique) values for this parameter + * as both patient_compartment and relatedperson_compartment token references. + * @param params a list of model parameter names + */ + public void add(List params, java.lang.String compartmentName) { + if (params != null) { + // Fast Conversion to java.lang.String + List paramsAsStrings = params.stream().map(param -> param.getValue()).collect(Collectors.toList()); + + for (java.lang.String paramName: paramsAsStrings) { + Set compartmentNames = paramCompartmentMap.get(paramName); + if (compartmentNames == null) { + compartmentNames = new HashSet<>(); + paramCompartmentMap.put(paramName, compartmentNames); + } + compartmentNames.add(compartmentName); + } + } + } + + /** + * Getter for the set of parameters referencing compartments + * @return + */ + public Map> getCompartmentReferenceParams() { + return Collections.unmodifiableMap(this.paramCompartmentMap); + } +} diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/reference/value/CompartmentReference.java b/fhir-search/src/main/java/com/ibm/fhir/search/reference/value/CompartmentReference.java new file mode 100644 index 00000000000..988d27aed9b --- /dev/null +++ b/fhir-search/src/main/java/com/ibm/fhir/search/reference/value/CompartmentReference.java @@ -0,0 +1,74 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.search.reference.value; + + +/** + * Represents a reference to a resource compartment extracted by SearchUtil + */ +public class CompartmentReference { + + private final String parameterName; + + private final String referenceResourceType; + + private final String referenceResourceValue; + + /** + * Public constructor + * @param parameterName + * @param referenceResourceType + * @param referenceResourceValue + */ + public CompartmentReference(String parameterName, String referenceResourceType, String referenceResourceValue) { + this.parameterName = parameterName; + this.referenceResourceType = referenceResourceType; + this.referenceResourceValue = referenceResourceValue; + } + + @Override + public int hashCode() { + int prime = 31; + return parameterName.hashCode() + prime * (referenceResourceType.hashCode() + prime * referenceResourceValue.hashCode()); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof CompartmentReference) { + CompartmentReference that = (CompartmentReference)obj; + return this.parameterName.equals(that.parameterName) + && this.referenceResourceType.equals(that.referenceResourceType) + && this.referenceResourceValue.equals(that.referenceResourceValue); + } else { + return false; + } + } + + + /** + * @return the parameterName + */ + public String getParameterName() { + return parameterName; + } + + + /** + * @return the referenceResourceType + */ + public String getReferenceResourceType() { + return referenceResourceType; + } + + + /** + * @return the referenceResourceValue + */ + public String getReferenceResourceValue() { + return referenceResourceValue; + } +} diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java new file mode 100644 index 00000000000..15c69766e2d --- /dev/null +++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java @@ -0,0 +1,251 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.search.util; + +import java.util.Arrays; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.model.resource.Bundle; +import com.ibm.fhir.model.type.Reference; +import com.ibm.fhir.model.type.code.FHIRResourceType; +import com.ibm.fhir.search.exception.FHIRSearchException; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + +/** + * Encapsulates utility functions used when processing model Reference elements + * from a FHIR resource + */ +public class ReferenceUtil { + private static final Logger logger = Logger.getLogger(ReferenceUtil.class.getName()); + private static final String HISTORY = "_history"; + private static final String HTTP = "http:"; + private static final String HTTPS = "https:"; + private static final String URN = "urn:"; + + // A set of resource type names to assist computing the base URL string + private static final Set resourceTypes = Arrays.stream(FHIRResourceType.ValueSet.values()) + .map(FHIRResourceType.ValueSet::value) + .collect(Collectors.toSet()); + + + // The service base URL string cached after we compute it once. + private static volatile String serviceBase; + + /** + * Processes a Reference value from the FHIR model and interprets + * it according to https://www.hl7.org/fhir/references.html#2.3.0 + * + * @param ref + * @param fullUrl the server + * @return + */ + public static ReferenceValue createReferenceValueFrom(Reference ref, String baseUrl) { + String value; + String targetResourceType = null; + ReferenceType referenceType = ReferenceType.INVALID; + Integer version = null; + + if (ref.getReference() != null && ref.getReference().getValue() != null) { + // LITERAL REFERENCE + // * an absolute URL + // * relative URL, which is relative to the Service Base URL, or, if processing a resource + // from a bundle, which is relative to the base URL implied by the Bundle.entry.fullUrl + // Note that fragment (internal) references are not relevant here, because bundle + // processing will already have resolved them, replacing them with the relative values + value = ref.getReference().getValue(); + + if (baseUrl != null && value.startsWith(baseUrl)) { + // - relative reference https://example.com/Patient/123 + // Because this reference is to a local FHIR resource (inside this server), we need use the correct + // resource type name (assigned as the code system) + // - https://localhost:9443/fhir-server/api/v4/Patient/1234 + // - https://example.com/Patient/1234 + // - https://example.com/Patient/1234/_history/2 + referenceType = ReferenceType.LITERAL_RELATIVE; + value = value.substring(baseUrl.length()); + + // Patient/1234 + // Patient/1234/_history/2 + String[] tokens = value.split("/"); + if (tokens.length > 1) { + targetResourceType = tokens[0]; + value = tokens[1]; + if (tokens.length == 4 && HISTORY.equals(tokens[2])) { + // versioned reference + version = Integer.parseInt(tokens[3]); + } + } + } else if (value != null && value.startsWith(HTTP) || value.startsWith(HTTPS) || value.startsWith(URN)) { + // - Absolute reference. We only know the type if it is given by the type field + referenceType = ReferenceType.LITERAL_ABSOLUTE; + if (ref.getType() != null) { + targetResourceType = ref.getType().getValue(); + } + + } else { + // - Relative ==> Patient/1234 + // - Relative ==> Patient/1234/_history/2 + referenceType = ReferenceType.LITERAL_RELATIVE; + String[] tokens = value.split("/"); + if (tokens.length > 1) { + targetResourceType = tokens[0]; + value = tokens[1]; + if (tokens.length == 4 && HISTORY.equals(tokens[2])) { + // versioned reference + version = Integer.parseInt(tokens[3]); + } + } + } + } else if (ref.getIdentifier() != null && ref.getIdentifier().getValue() != null) { + // LOGICAL REFERENCE + value = ref.getIdentifier().getValue().getValue(); + referenceType = ReferenceType.LOGICAL; + if (ref.getType() != null) { + // We need + targetResourceType = ref.getType().getValue(); + } + } else if (ref.getDisplay() != null) { + // According to the spec this is still a valid reference. But because it + // doesn't refer to anything, we don't need to persist it + referenceType = ReferenceType.DISPLAY_ONLY; + value = null; + } else { + // model.Reference currently allows invalid references to be built, so for now we + // need to expect and handle this. + referenceType = ReferenceType.INVALID; + value = null; + } + + return new ReferenceValue(targetResourceType, value, referenceType, version); + } + + /** + * Extract the base URL from the bundle entry if one is given, otherwise + * use the service base URL. + * @param entry, can be null + * @return the base URL for use in interpreting references + * @throws FHIRSearchException + */ + public static String getBaseUrl(Bundle.Entry entry) throws FHIRSearchException { + if (entry != null) { + return getBaseUrlFromBundle(entry); + } else { + // return the cached value if we've already computed it + if (serviceBase != null) { + return serviceBase; + } + return getServiceBaseUrl(); + } + + } + + /** + * https://www.hl7.org/fhir/references.html#literal + * See: a relative URL, which is relative to the Service Base URL, or, if processing a + * resource from a bundle, which is relative to the base URL implied by the + * Bundle.entry.fullUrl (see Resolving References in Bundles) + * + * If the fullUrl looks like this + * "fullUrl": "https://localhost:9443/fhir-server/api/v4/Observation/17546b5a5a9-872ecfe4-cb5e-4f8c-a381-5b13df536f87" + * then the returned String will look like this: + * "https://localhost:9443/fhir-server/api/v4/" + * + * @param entry + * @return + */ + public static String getBaseUrlFromBundle(Bundle.Entry entry) throws FHIRSearchException { + String value = entry.getFullUrl().getValue(); + try { + return getServiceBaseUrl(value); + } catch (FHIRSearchException x) { + logger.warning("Bundle.entry.fullUrl is invalid: " + value); + throw x; + } + } + + /** + * Get the service base URL using the originalRequestUri currently set in the + * FHIRRequestContext + * @param bundle + * @return the base url + */ + public static String getServiceBaseUrl() throws FHIRSearchException { + String uri = FHIRRequestContext.get().getOriginalRequestUri(); + return getServiceBaseUrl(uri); + } + + /** + * Get the service base URL from the given uri + * @param uri + * @return + * @throws FHIRSearchException + */ + public static String getServiceBaseUrl(String uri) throws FHIRSearchException { + + // request URI is not set for all unit-tests, so we need to take that into account + if (uri == null) { + return null; + } + + if (!uri.startsWith(HTTP) && !uri.startsWith(HTTPS)) { + throw new FHIRSearchException("base URI expected to start with http(s):"); + } + + // Strip off any parameters + int pmark = uri.indexOf('?'); + if (pmark >= 0) { + uri = uri.substring(0, pmark); + } + + // We keep parsing /field1/field2/... in the uri until we hit a valid resource type name + int last = -1; + int posn = -1; + int start = -1; + + while (last < 0 && (posn = uri.indexOf('/', start+1)) >= 0) { + String fragment = uri.substring(start+1, posn); + if (resourceTypes.contains(fragment)) { + // we've hit the resource type in the URL, so it's time to stop + last = start+1; // so we include the last '/' in the + } else { + // try the next field + start = posn; + } + } + + if (last < 0) { + // Didn't see the resource-type anywhere in the URL, but need to + // make sure we test the final field which may not be terminated + // by a '/' + posn = uri.length(); + String fragment = uri.substring(start+1, posn); + if (resourceTypes.contains(fragment)) { + last = start+1; // skip the final resource-type field + } else { + last = posn; // the whole string + } + } + + if (last >= 0) { + String sb = uri.substring(0, last); + if (!sb.endsWith("/")) { + // make sure we always have a final "/" to make life easier downstream. + sb = sb + "/"; + } + serviceBase = sb; + } else { + // log locally, do not leak in exception...might contain server name/ip secrets + logger.severe("FHIRRequestContext.originalRequestUri is invalid: " + uri); + throw new FHIRSearchException("Invalid originalRequestUri in FHIRRequestContext. Details in log."); + } + + return serviceBase; + } +} diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceValue.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceValue.java new file mode 100644 index 00000000000..7ec711ce211 --- /dev/null +++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceValue.java @@ -0,0 +1,74 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.search.util; + + +/** + * A reference extracted from a FHIR resource, with processing applied to + * identify the reference type and target resource type + */ +public class ReferenceValue { + + public static enum ReferenceType { + LITERAL_RELATIVE, // Patient/123abc + LITERAL_ABSOLUTE, // http(s)://an.other.server/Patient/234def + LOGICAL, // e.g. SSN + DISPLAY_ONLY, // A Reference with only a display field + INVALID // Not a valid reference + } + + // The type of the resource this reference points to. Can be null + private final String targetResourceType; + + // The value of the reference + private final String value; + + // The reference type + private final ReferenceType type; + + // Option version number to support versioned references when we need this + private final Integer version; + + public ReferenceValue(String targetResourceType, String value, ReferenceType type, Integer version) { + this.targetResourceType = targetResourceType; + this.value = value; + this.type = type; + this.version = version; + } + + + /** + * @return the targetResourceType + */ + public String getTargetResourceType() { + return targetResourceType; + } + + + /** + * @return the value + */ + public String getValue() { + return value; + } + + + /** + * @return the type + */ + public ReferenceType getType() { + return type; + } + + + /** + * @return the version + */ + public Integer getVersion() { + return version; + } +} diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java index b67eff2de03..c037ddf0a7c 100644 --- a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java +++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java @@ -38,6 +38,7 @@ import com.ibm.fhir.model.resource.SearchParameter.Component; import com.ibm.fhir.model.type.Canonical; import com.ibm.fhir.model.type.Code; +import com.ibm.fhir.model.type.Reference; import com.ibm.fhir.model.type.code.ResourceType; import com.ibm.fhir.model.type.code.SearchComparator; import com.ibm.fhir.model.type.code.SearchModifierCode; @@ -66,8 +67,10 @@ import com.ibm.fhir.search.parameters.QueryParameterValue; import com.ibm.fhir.search.parameters.cache.TenantSpecificSearchParameterCache; import com.ibm.fhir.search.reference.ReferenceParameterHandler; +import com.ibm.fhir.search.reference.value.CompartmentReference; import com.ibm.fhir.search.sort.Sort; import com.ibm.fhir.search.uri.UriBuilder; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; /** * Search Utility
@@ -115,6 +118,9 @@ public class SearchUtil { private static final String SEARCH_PROPERTY_TYPE_REVINCLUDE = "_revinclude"; private static final String HAS_DELIMITER = SearchConstants.COLON_DELIMITER_STR + SearchConstants.HAS + SearchConstants.COLON_DELIMITER_STR; + // compartment parameter reference which can be ignore + private static final String COMPARTMENT_PARM_DEF = "{def}"; + // The functionality is split into a new class. private static final Sort sort = new Sort(); @@ -2134,4 +2140,62 @@ public static Set getSummaryTextElementNames(Class resourceType) { summaryTextList.add("text"); return Collections.unmodifiableSet(summaryTextList); } -} + + /** + * Extracts the parameter values defining compartment membership. + * @param fhirResource + * @param compartmentRefParams a map of parameter names to a set of compartment names (resource types) + * @return a map of compartment name to a set of unique compartment reference values + */ + public static Map> extractCompartmentParameterValues(Resource fhirResource, Map> compartmentRefParams) throws FHIRSearchException { + final Map> result = new HashMap<>(); + final String resourceType = fhirResource.getClass().getSimpleName(); + + // TODO, probably should use a Bundle.Entry value here if we are processing a bundle + final String baseUrl = ReferenceUtil.getBaseUrl(null); + + try { + EvaluationContext resourceContext = new FHIRPathEvaluator.EvaluationContext(fhirResource); + + // Extract any references we find matching parameters representing compartment membership. + // For example CareTeam.participant can be used to refer to a Patient or RelatedPerson resource: + // "participant": { "reference": "Patient/abc123" } + // "participant": { "reference": "RelatedPerson/abc456" } + for (Map.Entry> paramEntry : compartmentRefParams.entrySet()) { + final String searchParm = paramEntry.getKey(); + + // Ignore {def} which is used in the compartment definition where + // no other search parm is given (e.g. Encounter->Encounter). + if (!COMPARTMENT_PARM_DEF.equals(searchParm)) { + String expression = SearchUtil.getSearchParameter(resourceType, searchParm).getExpression().getValue(); + + if (log.isLoggable(Level.FINE)) { + log.fine("searchParam = [" + resourceType + "] " + searchParm + "; expression = " + expression); + } + Collection nodes = FHIRPathEvaluator.evaluator().evaluate(resourceContext, expression); + for (FHIRPathNode node : nodes) { + Reference reference = node.asElementNode().element().as(Reference.class); + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(reference, baseUrl); + if (rv.getType() != ReferenceType.DISPLAY_ONLY && rv.getType() != ReferenceType.INVALID) { + // Check that the target resource type of the reference matches one of the + // target resource types in the compartment definition. + final String compartmentName = rv.getTargetResourceType(); + if (paramEntry.getValue().contains(compartmentName)) { + // Add this reference to the set of references we're collecting for each compartment + CompartmentReference cref = new CompartmentReference(searchParm, compartmentName, rv.getValue()); + Set references = result.computeIfAbsent(compartmentName, k -> new HashSet<>()); + references.add(cref); + } + } + } + } + } + } catch (Exception e) { + final String msg = "Unexpected exception extracting compartment references " + + " for resource type " + resourceType; + log.log(Level.WARNING, msg, e); + throw SearchExceptionUtil.buildNewInvalidSearchException(msg); + } + return result; + } +} \ No newline at end of file diff --git a/fhir-search/src/test/java/com/ibm/fhir/search/compartment/CompartmentUtilTest.java b/fhir-search/src/test/java/com/ibm/fhir/search/compartment/CompartmentUtilTest.java index 772e2da7081..7854dda4a42 100644 --- a/fhir-search/src/test/java/com/ibm/fhir/search/compartment/CompartmentUtilTest.java +++ b/fhir-search/src/test/java/com/ibm/fhir/search/compartment/CompartmentUtilTest.java @@ -14,8 +14,10 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.testng.annotations.Test; @@ -24,12 +26,12 @@ /** * CompartmentUtil is tested in this class. - * + * * @author pbastide * */ public class CompartmentUtilTest extends BaseSearchTest { - + @Test() public void testInit() { CompartmentUtil.init(); @@ -38,7 +40,9 @@ public void testInit() { @Test() public void testBuildCompartmentMap() { - Map cache = CompartmentUtil.buildCompartmentMap(); + Map cache = new HashMap<>(); + Map resourceCompartmentCache = new HashMap<>(); + CompartmentUtil.buildMaps(cache, resourceCompartmentCache); // There should be 5 compartment definitions. // Detailed behavior of the individual cache are tested in the CompartmentCache. @@ -46,14 +50,6 @@ public void testBuildCompartmentMap() { } - @Test(expectedExceptions = { UnsupportedOperationException.class }, dependsOnMethods = { "testBuildCompartmentMap" }) - public void testBuildCompartmentMapWithModification() { - // Verifies the cache is unmodifiable. - Map cache = CompartmentUtil.buildCompartmentMap(); - cache.clear(); - - } - @Test(expectedExceptions = {}) public void testGetCompartmentResourceTypeExists() throws FHIRSearchException { // The Compartment Does not Exist Exists @@ -119,6 +115,33 @@ public void testBuildCompositeBundle() { e.printStackTrace(); fail(); } + } + @Test() + public void testParamListForExplanationOfBenefit() { + Map> pmap = CompartmentUtil.getCompartmentParamsForResourceType("ExplanationOfBenefit"); + assertNotNull(pmap); + + // EOB may belong to Patient, Encounter, Practitioner, RelatedPerson and Device compartments + assertEquals(pmap.size(), 10); + + assertTrue(pmap.get("procedure-udi").contains("Device")); + assertTrue(pmap.get("item-udi").contains("Device")); + assertTrue(pmap.get("detail-udi").contains("Device")); + assertTrue(pmap.get("subdetail-udi").contains("Device")); + assertTrue(pmap.get("enterer").contains("Practitioner")); + assertTrue(pmap.get("provider").contains("Practitioner")); + assertTrue(pmap.get("payee").contains("Practitioner")); + assertTrue(pmap.get("payee").contains("RelatedPerson")); + assertTrue(pmap.get("care-team").contains("Practitioner")); + assertTrue(pmap.get("patient").contains("Patient")); + assertTrue(pmap.get("encounter").contains("Encounter")); + } + + @Test() + public void testParamListForInvalidResource() { + Map> pmap = CompartmentUtil.getCompartmentParamsForResourceType("not-a-resource"); + assertNotNull(pmap); + assertTrue(pmap.isEmpty()); } -} +} \ No newline at end of file diff --git a/fhir-search/src/test/java/com/ibm/fhir/search/reference/ReferenceUtilTest.java b/fhir-search/src/test/java/com/ibm/fhir/search/reference/ReferenceUtilTest.java new file mode 100644 index 00000000000..fad3a2918ca --- /dev/null +++ b/fhir-search/src/test/java/com/ibm/fhir/search/reference/ReferenceUtilTest.java @@ -0,0 +1,257 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.search.reference; + +import static com.ibm.fhir.model.type.String.string; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +import org.testng.annotations.Test; + +import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.model.resource.Bundle; +import com.ibm.fhir.model.type.Identifier; +import com.ibm.fhir.model.type.Reference; +import com.ibm.fhir.model.type.Uri; +import com.ibm.fhir.search.exception.FHIRSearchException; +import com.ibm.fhir.search.util.ReferenceUtil; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + +/** + * Unit tests for {@link ReferenceUtil} + */ +public class ReferenceUtilTest { + + @Test + public void testLiteralRelativeShort() { + final String baseUrl = "http://example.com/"; + final String refString = "Patient/abc1234"; + Reference ref = Reference.builder() + .reference(string(refString)) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_RELATIVE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), "abc1234"); + } + + @Test + public void testLiteralRelativeLong() { + final String baseUrl = "http://example.com/"; + final String refString = baseUrl + "Patient/abc1234"; + Reference ref = Reference.builder() + .reference(string(refString)) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_RELATIVE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), "abc1234"); + } + + @Test + public void testLiteralRelativeShortVersion() { + final String baseUrl = "http://example.com/"; + final String refString = "Patient/abc1234/_history/1"; + Reference ref = Reference.builder() + .reference(string(refString)) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_RELATIVE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), "abc1234"); + assertNotNull(rv.getVersion()); + assertEquals(rv.getVersion().intValue(), 1); + } + + @Test + public void testLiteralRelativeLongVersion() { + final String baseUrl = "http://example.com/"; + final String refString = baseUrl + "Patient/abc1234/_history/1"; + Reference ref = Reference.builder() + .reference(string(refString)) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_RELATIVE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), "abc1234"); + assertNotNull(rv.getVersion()); + assertEquals(rv.getVersion().intValue(), 1); + } + + @Test + public void testLiteralAbsolute() { + final String baseUrl = "http://example.com/"; + final String refString = "http://an.other.system/Patient/abc1234"; + Reference ref = Reference.builder() + .reference(string(refString)) + .type(Uri.of("Patient")) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_ABSOLUTE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), refString); + } + + @Test + public void testLiteralAbsoluteVersion() { + final String baseUrl = "http://example.com/"; + final String refString = "http://an.other.system/Patient/abc1234/_history/1"; + Reference ref = Reference.builder() + .reference(string(refString)) + .type(Uri.of("Patient")) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_ABSOLUTE); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), refString); + } + + @Test + public void testLiteralAbsoluteNoType() { + final String baseUrl = "http://example.com/"; + final String refString = "http://an.other.system/Patient/abc1234"; + Reference ref = Reference.builder() + .reference(string(refString)) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LITERAL_ABSOLUTE); + assertNull(rv.getTargetResourceType()); + assertEquals(rv.getValue(), refString); + } + + @Test + public void testLogical() { + final String baseUrl = "http://example.com/"; + final String idString = "000-12-3456"; + Reference ref = Reference.builder() + .identifier(Identifier.builder().value(string(idString)).build()) + .type(Uri.of("Patient")) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.LOGICAL); + assertEquals(rv.getTargetResourceType(), "Patient"); + assertEquals(rv.getValue(), idString); + } + + @Test + public void testDisplayOnly() { + final String baseUrl = "http://example.com/"; + Reference ref = Reference.builder() + .display(string("display value")) + .build(); + + ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(ref, baseUrl); + assertEquals(rv.getType(), ReferenceType.DISPLAY_ONLY); + assertNull(rv.getValue()); + assertNull(rv.getTargetResourceType()); + assertNull(rv.getVersion()); + } + + @Test + public void testGetBaseUrlFromBundle() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4/Observation/17546b5a5a9-872ecfe4-cb5e-4f8c-a381-5b13df536f87"; + Bundle.Entry entry = Bundle.Entry.builder() + .fullUrl(Uri.of(fullUrl)) + .build(); + + final String baseUrl = ReferenceUtil.getBaseUrlFromBundle(entry); + assertNotNull(baseUrl); + assertEquals(baseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test(expectedExceptions = FHIRSearchException.class) + public void testGetBaseUrlFromBundleInvalid() throws FHIRSearchException { + final String fullUrl = "Observation/17546b5a5a9-872ecfe4-cb5e-4f8c-a381-5b13df536"; + Bundle.Entry entry = Bundle.Entry.builder() + .fullUrl(Uri.of(fullUrl)) + .build(); + + // should throw FHIRSearchException + ReferenceUtil.getBaseUrlFromBundle(entry); + } + + @Test(expectedExceptions = FHIRSearchException.class) + public void testGetBaseUrlFromBundleInvalid2() throws FHIRSearchException { + final String fullUrl = "17546b5a5a9-872ecfe4-cb5e-4f8c-a381-5b13df536"; + Bundle.Entry entry = Bundle.Entry.builder() + .fullUrl(Uri.of(fullUrl)) + .build(); + + // should throw FHIRSearchException + ReferenceUtil.getBaseUrlFromBundle(entry); + } + + @Test + public void testGetServiceBaseUrl() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4/Observation/17546b5a5a9-872ecfe4-cb5e-4f8c-a381-5b13df536f87"; + FHIRRequestContext context = new FHIRRequestContext(); + context.setOriginalRequestUri(fullUrl); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + assertEquals(serviceBaseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test + public void testGetServiceBaseUrl2() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4/Observation/"; + FHIRRequestContext context = new FHIRRequestContext(); + context.setOriginalRequestUri(fullUrl); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + assertEquals(serviceBaseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test + public void testGetServiceBaseUrl3() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4/Observation"; + FHIRRequestContext context = new FHIRRequestContext(); + context.setOriginalRequestUri(fullUrl); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + assertEquals(serviceBaseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test + public void testGetServiceBaseUrl4() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4/"; + FHIRRequestContext context = new FHIRRequestContext(); + context.setOriginalRequestUri(fullUrl); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + assertEquals(serviceBaseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test + public void testGetServiceBaseUrl5() throws FHIRSearchException { + final String fullUrl = "https://localhost:9443/fhir-server/api/v4"; + FHIRRequestContext context = new FHIRRequestContext(); + context.setOriginalRequestUri(fullUrl); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + + // Note the special case here...we tack on the final "/" for consistency + assertEquals(serviceBaseUrl, "https://localhost:9443/fhir-server/api/v4/"); + } + + @Test + public void testGetServiceBaseUrlNoOriginal() throws FHIRSearchException { + FHIRRequestContext context = new FHIRRequestContext(); + FHIRRequestContext.set(context); + final String serviceBaseUrl = ReferenceUtil.getServiceBaseUrl(); + assertNull(serviceBaseUrl); + } +} \ No newline at end of file From 2d89c8a62ca4c68cb3f20a8d4a765e8444eaa3d1 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Thu, 19 Nov 2020 17:01:09 -0500 Subject: [PATCH 2/8] issue #1708 use ibm-internal stored parameters for compartment searches Signed-off-by: Robin Arnold --- docs/src/pages/guides/FHIRServerUsersGuide.md | 17 +- .../ibm/fhir/config/FHIRConfiguration.java | 1 + .../json/ibm/basic/BasicCompartment.json | 48 +++ .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 6 +- .../test/JDBCSearchCompartmentTest.java | 68 +++ .../src/test/java/testng.xml | 1 + .../test/AbstractSearchCompartmentTest.java | 61 +++ .../extension-search-parameters.json | 403 ++++++++++++++++++ .../compartment/fhir-server-config.json | 8 + .../search/location/NearLocationHandler.java | 30 +- .../com/ibm/fhir/search/util/SearchUtil.java | 46 +- 11 files changed, 669 insertions(+), 20 deletions(-) create mode 100644 fhir-examples/src/main/resources/json/ibm/basic/BasicCompartment.json create mode 100644 fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java create mode 100644 fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractSearchCompartmentTest.java create mode 100644 fhir-persistence/src/test/resources/config/compartment/extension-search-parameters.json create mode 100644 fhir-persistence/src/test/resources/config/compartment/fhir-server-config.json diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 3d7cc301f04..a877adfdb1d 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -159,7 +159,7 @@ Configuration properties stored within a `fhir-server-config.json` file are stru Throughout this document, we use a path notation to refer to property names. For example, the name of the `defaultPrettyPrint` property in the preceding example would be `fhirServer/core/defaultPrettyPrint`. ## 3.3 Tenant-specific configuration properties -The FHIR server supports certain multi-tenant features. One such feature is the ability to set certain configuration properties on a per-tenant basis. +The IBM FHIR server supports certain multi-tenant features. One such feature is the ability to set certain configuration properties on a per-tenant basis. In general, the configuration properties for a particular tenant are stored in the `/usr/servers/fhir-server/config//fhir-server-config.json` file, where `` refers to the tenant's “short name” or tenant id. @@ -171,6 +171,20 @@ Search parameters are handled like a single configuration properly; providing a More information about multi-tenant support can be found in [Section 4.9 Multi-tenancy](#49-multi-tenancy). +## 3.3.1 Compartment Search Performance + +The IBM FHIR Server now supports the ability to compute and store compartment membership values during ingestion. Once stored, these values can help accelerate compartment-related search queries. To use this feature, update the IBM FHIR Server to the latest version and run a reindex operation. See the release notes for Release 4.5.0 for details. The reindex operation reprocesses the resources stored in the database, computing and storing the new compartment reference values. After the reindex operation has completed, add the following configuration element to the relevant tenant fhir-server-config.json file to allow the search queries to use the pre-computed values: + +``` + { + "fhirServer": { + "search": { + "useStoredCompartmentParam": true + } + } + } +``` + ## 3.4 Persistence layer configuration The IBM FHIR server allows deployers to select a persistence layer implementation that fits their needs. Currently, the server includes a JDBC persistence layer which supports Apache Derby, IBM Db2, and PostgreSQL. However, Apache Derby is not recommended for production usage. @@ -1848,6 +1862,7 @@ This section contains reference information about each of the configuration prop |`fhirServer/audit/serviceProperties/geoState`|string|The Geo State configure for CADF audit logging service.| |`fhirServer/audit/serviceProperties/geoCounty`|string|The Geo Country configure for CADF audit logging service.| |`fhirServer/search/useBoundingRadius`|boolean|True, the bounding area is a Radius, else the bounding area is a box.| +|`fhirServer/search/useStoredCompartmentParam`|boolean|False, Compute and store parameter to accelerate compartment searches. Requires reindex using latest IBM FHIR Server version before this feature is enabled | |`fhirServer/bulkdata/applicationName`| string|Fixed value, always set to fhir-bulkimportexport-webapp | |`fhirServer/bulkdata/moduleName`|string| Fixed value, always set to fhir-bulkimportexport.war | |`fhirServer/bulkdata/jobParameters/cos.bucket.name`|string|Object store bucket name | diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java index 92a4ae6e090..f2f07125eb4 100644 --- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java +++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java @@ -46,6 +46,7 @@ public class FHIRConfiguration { public static final String PROPERTY_FIELD_RESOURCES_SEARCH_PARAMETER_COMBINATIONS = "searchParameterCombinations"; public static final String PROPERTY_FIELD_RESOURCES_PROFILES = "profiles"; public static final String PROPERTY_FIELD_RESOURCES_PROFILES_AT_LEAST_ONE = "atLeastOne"; + public static final String PROPERTY_USE_STORED_COMPARTMENT_PARAM = "fhirServer/search/useStoredCompartmentParam"; // Auth and security properties public static final String PROPERTY_SECURITY_CORS = "fhirServer/security/cors"; diff --git a/fhir-examples/src/main/resources/json/ibm/basic/BasicCompartment.json b/fhir-examples/src/main/resources/json/ibm/basic/BasicCompartment.json new file mode 100644 index 00000000000..f0948d28370 --- /dev/null +++ b/fhir-examples/src/main/resources/json/ibm/basic/BasicCompartment.json @@ -0,0 +1,48 @@ +{ + "resourceType": "Basic", + "code": { + "text": "test" + }, + "author": { + "reference": "Patient/123" + }, + "extension": [ + { + "url": "http://example.org/uri", + "valueUri": "urn:uuid:53fefa32-1111-2222-3333-55ee120877b7" + }, + { + "url": "http://example.org/Reference", + "valueReference": { + "reference": "Basic/123" + } + }, + { + "url": "http://example.org/Reference-id", + "valueReference": { + "identifier": { + "system": "http://example.org/identifier", + "value": "123" + } + } + }, + { + "url": "http://example.org/Reference-relative", + "valueReference": { + "reference": "Patient/123" + } + }, + { + "url": "http://example.org/Reference-absolute", + "valueReference": { + "reference": "https://example.com/Patient/123" + } + }, + { + "url": "http://example.org/Reference-display", + "valueReference": { + "display": "text alternative for the resource" + } + } + ] +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 1d0279123d9..1d29b52efd5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -1475,7 +1475,11 @@ private List extractSearchParameters(Resource fhirResou } /** - * Augment the given list with additional reference values + * Augment the given allParameters list with ibm-internal parameters represents relationships + * of the fhirResource to its compartments. These parameter values are subsequently used + * to improve the performance of compartment-based FHIR search queries. See + * {@link CompartmentUtil#makeCompartmentParamName(String)} for details on how the + * parameter name is composed for each relationship. * @param allParameters */ protected void addCompartmentParams(List allParameters, Resource fhirResource) throws FHIRSearchException { diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java new file mode 100644 index 00000000000..c839284e5f1 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java @@ -0,0 +1,68 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.search.test; + +import java.util.Properties; + +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; +import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.persistence.FHIRPersistence; +import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; +import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.search.test.AbstractSearchCompartmentTest; + +/** + * JDBC unit-tests for compartment-based searches + */ +public class JDBCSearchCompartmentTest extends AbstractSearchCompartmentTest { + + private Properties testProps; + + private PoolConnectionProvider connectionPool; + + private FHIRPersistenceJDBCCache cache; + + public JDBCSearchCompartmentTest() throws Exception { + this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); + } + + @Override + public void bootstrapDatabase() throws Exception { + DerbyInitializer derbyInit; + String dbDriverName = this.testProps.getProperty("dbDriverName"); + if (dbDriverName != null && dbDriverName.contains("derby")) { + derbyInit = new DerbyInitializer(this.testProps); + IConnectionProvider cp = derbyInit.getConnectionProvider(false); + this.connectionPool = new PoolConnectionProvider(cp, 1); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new NameIdCache(), rrc); + } + } + + @Override + public FHIRPersistence getPersistenceImpl() throws Exception { + if (this.connectionPool == null) { + throw new IllegalStateException("Database not bootstrapped"); + } + return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + } + + @Override + protected void shutdownPools() throws Exception { + // Mark the pool as no longer in use. This allows the pool to check for + // lingering open connections/transactions. + if (this.connectionPool != null) { + this.connectionPool.close(); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/testng.xml b/fhir-persistence-jdbc/src/test/java/testng.xml index 8e4d160bd52..df785310fab 100644 --- a/fhir-persistence-jdbc/src/test/java/testng.xml +++ b/fhir-persistence-jdbc/src/test/java/testng.xml @@ -35,6 +35,7 @@ + diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractSearchCompartmentTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractSearchCompartmentTest.java new file mode 100644 index 00000000000..f59da97c6ad --- /dev/null +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/search/test/AbstractSearchCompartmentTest.java @@ -0,0 +1,61 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.search.test; + +import static org.testng.AssertJUnit.assertEquals; + +import java.util.List; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.exception.FHIRException; +import com.ibm.fhir.model.resource.Basic; +import com.ibm.fhir.model.resource.Resource; +import com.ibm.fhir.model.test.TestUtil; + +/** + * Unit tests for compartment-based searches + * @see https://hl7.org/fhir/r4/search.html + * GET [base]/Patient/[id]/[type]?parameter(s) + */ +public abstract class AbstractSearchCompartmentTest extends AbstractPLSearchTest { + + @Override + protected Basic getBasicResource() throws Exception { + return TestUtil.readExampleResource("json/ibm/basic/BasicCompartment.json"); + } + + @Override + protected void setTenant() throws Exception { + FHIRRequestContext.get().setTenantId("compartment"); + + // Need to set reference before storing the resource. The server-url + // is now used to determine if an absolute reference is local (can be served + // from this FHIR server). + createReference(); + } + + @BeforeClass + public void createReference() throws FHIRException { + String originalRequestUri = "https://example.com/Patient/123"; + FHIRRequestContext context = FHIRRequestContext.get(); + context.setOriginalRequestUri(originalRequestUri); + } + + @Test + public void testSearchCompartment() throws Exception { + // The saved Basic resource is a member of the compartment Patient/123 + // Check that we can find the resource with additional query parameters + // Note that "Reference-relative just happens to be another searchable + // parameter in the BasicCompartment.json resource + List results = runQueryTest("Patient", "123", + Basic.class, "Reference-relative", "Patient/123"); + assertEquals(1, results.size()); + } +} \ No newline at end of file diff --git a/fhir-persistence/src/test/resources/config/compartment/extension-search-parameters.json b/fhir-persistence/src/test/resources/config/compartment/extension-search-parameters.json new file mode 100644 index 00000000000..c46282c0710 --- /dev/null +++ b/fhir-persistence/src/test/resources/config/compartment/extension-search-parameters.json @@ -0,0 +1,403 @@ +{ + "resourceType": "Bundle", + "type": "collection", + "entry": [{ + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-boolean", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-boolean", + "url": "http://ibm.com/fhir/SearchParameter/Basic-boolean", + "name": "boolean", + "status": "active", + "description": "test param", + "code": "boolean", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/boolean').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/boolean']/f:valueBoolean", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-integer", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-integer", + "url": "http://ibm.com/fhir/SearchParameter/Basic-integer", + "name": "integer", + "status": "active", + "description": "test param", + "code": "integer", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/integer').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/integer']/f:valueInteger", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-string", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-string", + "url": "http://ibm.com/fhir/SearchParameter/Basic-string", + "name": "string", + "status": "active", + "description": "test param", + "code": "string", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/string').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/string']/f:valueString", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-code", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-code", + "url": "http://ibm.com/fhir/SearchParameter/Basic-code", + "name": "code", + "status": "active", + "description": "test param", + "code": "code", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/code').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/code']/f:valueCode", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-decimal", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-decimal", + "url": "http://ibm.com/fhir/SearchParameter/Basic-decimal", + "name": "decimal", + "status": "active", + "description": "test param", + "code": "decimal", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/decimal').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/decimal']/f:valueDecimal", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-uri", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-uri", + "url": "http://ibm.com/fhir/SearchParameter/Basic-uri", + "name": "uri", + "status": "active", + "description": "test param", + "code": "uri", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/uri').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/uri']/f:valueUri", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-instant", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-instant", + "url": "http://ibm.com/fhir/SearchParameter/Basic-instant", + "name": "instant", + "status": "active", + "description": "test param", + "code": "instant", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/instant').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/instant']/f:valueInstant", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-date", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-date", + "url": "http://ibm.com/fhir/SearchParameter/Basic-date", + "name": "date", + "status": "active", + "description": "test param", + "code": "date", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/date').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/date']/f:valueDate", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-dateTime", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-dateTime", + "url": "http://ibm.com/fhir/SearchParameter/Basic-dateTime", + "name": "dateTime", + "status": "active", + "description": "test param", + "code": "dateTime", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/dateTime').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/dateTime']/f:valueDateTime", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Period", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Period", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Period", + "name": "Period", + "status": "active", + "description": "test param", + "code": "Period", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Period').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Period']/f:valuePeriod", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Timing", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Timing", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Timing", + "name": "Timing", + "status": "active", + "description": "test param", + "code": "Timing", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Timing').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Timing']/f:valueTiming", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Quantity", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Quantity", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Quantity", + "name": "Quantity", + "status": "active", + "description": "test param", + "code": "Quantity", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Quantity').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Quantity']/f:valueQuantity", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Reference", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Reference", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Reference", + "name": "Reference", + "status": "active", + "description": "test param", + "code": "Reference", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Reference').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Reference']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Coding", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Coding", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Coding", + "name": "Coding", + "status": "active", + "description": "test param", + "code": "Coding", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Coding').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Coding']/f:valueCoding", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-CodeableConcept", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-CodeableConcept", + "url": "http://ibm.com/fhir/SearchParameter/Basic-CodeableConcept", + "name": "CodeableConcept", + "status": "active", + "description": "test param", + "code": "CodeableConcept", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/CodeableConcept').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/CodeableConcept']/f:valueCodeableConcept", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/fhir/SearchParameter/Basic-Identifier", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Identifier", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Identifier", + "name": "Identifier", + "status": "active", + "description": "test param", + "code": "Identifier", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Identifier').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Identifier']/f:valueIdentifier", + "xpathUsage": "normal" + } + }, + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-ContactPoint", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-ContactPoint", + "url": "http://ibm.com/fhir/SearchParameter/Basic-ContactPoint", + "name": "ContactPoint", + "status": "active", + "description": "test param", + "code": "ContactPoint", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/ContactPoint').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/ContactPoint']/f:valueContactPoint", + "xpathUsage": "normal" + } + }, + + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-Reference-id", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Reference-id", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Reference-id", + "name": "Reference-id", + "status": "active", + "description": "test param", + "code": "Reference-id", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Reference-id').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Reference-id']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-Reference-relative", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Reference-relative", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Reference-relative", + "name": "Reference-relative", + "status": "active", + "description": "test param", + "code": "Reference-relative", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Reference-relative').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Reference-relative']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-Reference-absolute", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Reference-absolute", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Reference-absolute", + "name": "Reference-absolute", + "status": "active", + "description": "test param", + "code": "Reference-absolute", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Reference-absolute').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Reference-absolute']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-Reference-display", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-Reference-display", + "url": "http://ibm.com/fhir/SearchParameter/Basic-Reference-display", + "name": "Reference-display", + "status": "active", + "description": "test param", + "code": "Reference-display", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/Reference-display').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/Reference-display']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-missing-Reference", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-missing-Reference", + "url": "http://ibm.com/fhir/SearchParameter/Basic-missing-Reference", + "name": "missing-Reference", + "status": "active", + "description": "test param", + "code": "missing-Reference", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/missing-Reference').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/missing-Reference']/f:valueReference", + "xpathUsage": "normal", + "target": ["Patient"] + } + }, + { + "fullUrl": "http://ibm.com/watsonhealth/fhir/SearchParameter/Basic-missing-uri", + "resource": { + "resourceType": "SearchParameter", + "id": "Basic-missing-uri", + "url": "http://ibm.com/fhir/SearchParameter/Basic-missing-uri", + "name": "missing-uri", + "status": "active", + "description": "test param", + "code": "missing-uri", + "base": ["Basic"], + "type": "reference", + "expression": "Basic.extension.where(url='http://example.org/missing-uri').value", + "xpath": "f:Basic/f:extension[@url='http://example.org/missing-uri']/f:valueUri", + "xpathUsage": "normal" + } + }] +} diff --git a/fhir-persistence/src/test/resources/config/compartment/fhir-server-config.json b/fhir-persistence/src/test/resources/config/compartment/fhir-server-config.json new file mode 100644 index 00000000000..8de9addbd0a --- /dev/null +++ b/fhir-persistence/src/test/resources/config/compartment/fhir-server-config.json @@ -0,0 +1,8 @@ +{ + "__comment": "FHIR Server configuration extension for the compartment test tenant", + "fhirServer": { + "search": { + "useStoredCompartmentParam": true + } + } +} diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/location/NearLocationHandler.java b/fhir-search/src/main/java/com/ibm/fhir/search/location/NearLocationHandler.java index 248ca1a69cd..df263ce2131 100644 --- a/fhir-search/src/main/java/com/ibm/fhir/search/location/NearLocationHandler.java +++ b/fhir-search/src/main/java/com/ibm/fhir/search/location/NearLocationHandler.java @@ -38,7 +38,7 @@ public class NearLocationHandler { private static final String CLASSNAME = NearLocationHandler.class.getName(); private static final Logger logger = Logger.getLogger(CLASSNAME); - // Radix for the Earth: + // Radix for the Earth: private static final double RADIUS_MERIDIAN = 6378.137; private static final double RADIUS_EQUATORIAL = 6356.7523; @@ -49,13 +49,13 @@ public class NearLocationHandler { public static final double DEFAULT_DISTANCE = 5.0; public static final String DEFAULT_UNIT = "km"; - // Turns on the use of the bounding raidus, if it's configured. + // Turns on the use of the bounding raidus, if it's configured. boolean boundingRadius = false; public NearLocationHandler() { try { PropertyGroup fhirConfig = FHIRConfiguration.getInstance().loadConfiguration(); - fhirConfig.getBooleanProperty(FHIRConfiguration.PROPERTY_SEARCH_BOUNDING_AREA_RADIUS_TYPE, + this.boundingRadius = fhirConfig.getBooleanProperty(FHIRConfiguration.PROPERTY_SEARCH_BOUNDING_AREA_RADIUS_TYPE, false); } catch (Exception e) { logger.fine("Issue loading the fhir configuration, defaulting to BoundingBox"); @@ -64,7 +64,7 @@ public NearLocationHandler() { /** * degrees to radians - * + * * @param deg * @return */ @@ -74,7 +74,7 @@ public double degree2radians(double deg) { /** * radians to degrees - * + * * @param rad * @return */ @@ -88,7 +88,7 @@ public double radians2degrees(double rad) { * WGS84 format * [latitude]|[longitude]|[distance]|[units] *
- * + * * @param latitude * @param longitude * @param distance @@ -118,7 +118,7 @@ public BoundingBox createBoundingBox(double latitude, double longitude, double d double minLongitude = longitude; double maxLongitude = longitude; - // If distance is not zero, we're going for boxed match. + // If distance is not zero, we're going for boxed match. if (distance != 0) { // Convert to Radians to do the ARC calculation // Based on https://stackoverflow.com/a/238558/1873438 @@ -127,14 +127,14 @@ public BoundingBox createBoundingBox(double latitude, double longitude, double d double lonRad = degree2radians(longitude); // build bounding box points - // The max/min ensures we don't loop infinitely over the pole, and are not taking silly. + // The max/min ensures we don't loop infinitely over the pole, and are not taking silly. // latitude parameters when calculated with the distance. double latMin = latRad - distance / RADIUS_EQUATORIAL; double latMax = latRad + distance / RADIUS_EQUATORIAL; double lonMin = lonRad - distance / (RADIUS_MERIDIAN * Math.cos(latRad)); double lonMax = lonRad + distance / (RADIUS_MERIDIAN * Math.cos(latRad)); - // Convert back to degrees, and minimize the box. + // Convert back to degrees, and minimize the box. minLatitude = Math.max(-90, radians2degrees(latMin)); maxLatitude = Math.min(90, radians2degrees(latMax)); minLongitude = Math.max(-180, radians2degrees(lonMin)); @@ -158,7 +158,7 @@ public BoundingBox createBoundingBox(double latitude, double longitude, double d /** * generates location positions for processing from parameters. - * + * * @param queryParameters * @return * @throws FHIRSearchException @@ -191,7 +191,7 @@ public List generateLocationPositionsFromParameters(List generateLocationPositionsFromParameters(List generateLocationPositionsFromParameters(List= 3) { unit = components[3]; } @@ -250,7 +250,7 @@ public List generateLocationPositionsFromParameters(List inclusionCriteria = - CompartmentUtil.getCompartmentResourceTypeInclusionCriteria(compartmentName, - resourceType.getSimpleName()); + List inclusionCriteria; + + if (useStoredCompartmentParam()) { + // issue #1708. When enabled, use the ibm-internal-... compartment parameter. This + // results in faster queries because only a single parameter is used to represent the + // compartment membership. + inclusionCriteria = Collections.singletonList(CompartmentUtil.makeCompartmentParamName(compartmentName)); + } else { + // pre #1708 behavior, which is the default + inclusionCriteria = + CompartmentUtil.getCompartmentResourceTypeInclusionCriteria(compartmentName, + resourceType.getSimpleName()); + } + for (String criteria : inclusionCriteria) { parameter = new QueryParameter(Type.REFERENCE, criteria, null, null, true); value = new QueryParameterValue(); From 77fd42777f134d15e78f19d60ffb2f235324bad5 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Sat, 21 Nov 2020 20:19:28 -0500 Subject: [PATCH 3/8] issue #1708 addressed code review comments and fixed reindexing as it is required for storing compartment references Signed-off-by: Robin Arnold --- docs/src/pages/guides/FHIRServerUsersGuide.md | 8 +- .../bucket/reindex/DriveReindexOperation.java | 58 +++++----- .../jdbc/dao/ReindexResourceDAO.java | 90 +++++++++++----- .../dao/impl/ParameterVisitorBatchDAO.java | 2 +- .../jdbc/dto/ReferenceParmVal.java | 11 -- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 99 ++++++++++------- .../PostgresReindexResourceDAO.java | 100 ++++++++++++++---- .../util/JDBCParameterBuildingVisitor.java | 12 --- .../scout/FHIRPersistenceScoutImpl.java | 100 +++++++----------- .../ibm/fhir/persistence/FHIRPersistence.java | 7 +- .../persistence/test/MockPersistenceImpl.java | 2 +- .../com/ibm/fhir/search/util/SearchUtil.java | 31 +++--- .../operation/spi/FHIRResourceHelpers.java | 3 +- .../ibm/fhir/server/util/FHIRRestHelper.java | 4 +- .../fhir/server/test/MockPersistenceImpl.java | 2 +- .../processor/DummyImportExportImplTest.java | 2 +- .../operation/reindex/ReindexOperation.java | 12 ++- .../src/main/resources/reindex.json | 14 ++- 18 files changed, 317 insertions(+), 240 deletions(-) diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 6d8a9c1ab6a..45873e807d9 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -173,7 +173,7 @@ More information about multi-tenant support can be found in [Section 4.9 Multi-t ## 3.3.1 Compartment Search Performance -The IBM FHIR Server now supports the ability to compute and store compartment membership values during ingestion. Once stored, these values can help accelerate compartment-related search queries. To use this feature, update the IBM FHIR Server to the latest version and run a reindex operation. See the release notes for Release 4.5.0 for details. The reindex operation reprocesses the resources stored in the database, computing and storing the new compartment reference values. After the reindex operation has completed, add the following configuration element to the relevant tenant fhir-server-config.json file to allow the search queries to use the pre-computed values: +The IBM FHIR Server now supports the ability to compute and store compartment membership values during ingestion. Once stored, these values can help accelerate compartment-related search queries. To use this feature, update the IBM FHIR Server at least version 4.5.1 and run a reindex operation as described in the release notes for Release 4.5.0. The reindex operation reprocesses the resources stored in the database, computing and storing the new compartment reference values. After the reindex operation has completed, add the `useStoredCompartmentParam` configuration element to the relevant tenant fhir-server-config.json file to allow the search queries to use the pre-computed values: ``` { @@ -186,9 +186,9 @@ The IBM FHIR Server now supports the ability to compute and store compartment me ``` ## 3.4 Persistence layer configuration -The IBM FHIR server allows deployers to select a persistence layer implementation that fits their needs. Currently, the server includes a JDBC persistence layer which supports Apache Derby, IBM Db2, and PostgreSQL. However, Apache Derby is not recommended for production usage. +The IBM FHIR Server allows deployers to select a persistence layer implementation that fits their needs. Currently, the server includes a JDBC persistence layer which supports Apache Derby, IBM Db2, and PostgreSQL. However, Apache Derby is not recommended for production usage. -The FHIR server is delivered with a default configuration that is already configured to use the JDBC persistence layer implementation with an Embedded Derby database. This provides the easiest out-of-the-box experience since it requires very little setup. The sections that follow in this chapter will focus on how to configure the JDBC persistence layer implementation with either Embedded Derby or Db2. +The IBM FHIR Server is delivered with a default configuration that is already configured to use the JDBC persistence layer implementation with an Embedded Derby database. This provides the easiest out-of-the-box experience since it requires very little setup. The sections that follow in this chapter will focus on how to configure the JDBC persistence layer implementation with either Embedded Derby or Db2. ### 3.4.1 Configuring the JDBC persistence layer #### 3.4.1.1 Database preparation @@ -1864,7 +1864,7 @@ This section contains reference information about each of the configuration prop |`fhirServer/audit/serviceProperties/geoState`|string|The Geo State configure for CADF audit logging service.| |`fhirServer/audit/serviceProperties/geoCounty`|string|The Geo Country configure for CADF audit logging service.| |`fhirServer/search/useBoundingRadius`|boolean|True, the bounding area is a Radius, else the bounding area is a box.| -|`fhirServer/search/useStoredCompartmentParam`|boolean|False, Compute and store parameter to accelerate compartment searches. Requires reindex using latest IBM FHIR Server version before this feature is enabled | +|`fhirServer/search/useStoredCompartmentParam`|boolean|False, Compute and store parameter to accelerate compartment searches. Requires reindex using at least IBM FHIR Server version 4.5.1 before this feature is enabled | |`fhirServer/bulkdata/applicationName`| string|Fixed value, always set to fhir-bulkimportexport-webapp | |`fhirServer/bulkdata/moduleName`|string| Fixed value, always set to fhir-bulkimportexport.war | |`fhirServer/bulkdata/jobParameters/cos.bucket.name`|string|Object store bucket name | diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/reindex/DriveReindexOperation.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/reindex/DriveReindexOperation.java index 9cc78cae192..23be986d0f8 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/reindex/DriveReindexOperation.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/reindex/DriveReindexOperation.java @@ -22,8 +22,8 @@ import com.ibm.fhir.model.resource.OperationOutcome; import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.resource.Parameters; -import com.ibm.fhir.model.resource.Resource; import com.ibm.fhir.model.resource.Parameters.Parameter; +import com.ibm.fhir.model.resource.Resource; /** * Drives the $reindex custom operation in parallel. Each thread keeps running @@ -31,30 +31,30 @@ */ public class DriveReindexOperation { private static final Logger logger = Logger.getLogger(DriveReindexOperation.class.getName()); - + // the maximum number of requests we permit private final int maxConcurrentRequests; // flag to indicate if we should be running private volatile boolean running = true; - + private volatile boolean active = false; // count of how many threads are currently running private AtomicInteger currentlyRunning = new AtomicInteger(); - + // thread pool for processing requests private final ExecutorService pool = Executors.newCachedThreadPool(); private final FHIRBucketClient fhirClient; - + private final String url = "$reindex"; - + // The serialized Parameters resource sent with each POST private final String requestBody; - + private Thread monitorThread; - + /** * Public constructor * @param client the FHIR client @@ -65,13 +65,13 @@ public DriveReindexOperation(FHIRBucketClient fhirClient, int maxConcurrentReque this.maxConcurrentRequests = maxConcurrentRequests; Parameters parameters = Parameters.builder() - .parameter(Parameter.builder().name(str("_tstamp")).value(str(tstampParam)).build()) - .parameter(Parameter.builder().name(str("_resourceCount")).value(intValue(resourceCountParam)).build()) + .parameter(Parameter.builder().name(str("tstamp")).value(str(tstampParam)).build()) + .parameter(Parameter.builder().name(str("resourceCount")).value(intValue(resourceCountParam)).build()) .build(); - + // Serialize into the requestBody string used by all the threads this.requestBody = FHIRBucketClientUtil.resourceToString(parameters); - + if (logger.isLoggable(Level.FINE)) { logger.fine("Reindex request parameters: " + requestBody); } @@ -85,7 +85,7 @@ public DriveReindexOperation(FHIRBucketClient fhirClient, int maxConcurrentReque private static com.ibm.fhir.model.type.String str(String str) { return com.ibm.fhir.model.type.String.of(str); } - + private static com.ibm.fhir.model.type.Integer intValue(int val) { return com.ibm.fhir.model.type.Integer.of(val); } @@ -97,7 +97,7 @@ public void init() { if (!running) { throw new IllegalStateException("Already shutdown"); } - + // Initiate the monitorThread. This will fill the pool // with worker threads, and monitor for completion or failure logger.info("Starting monitor thread"); @@ -106,7 +106,7 @@ public void init() { } /** - * The main monitor loop. + * The main monitor loop. */ public void monitorLoop() { while (this.running) { @@ -121,11 +121,11 @@ public void monitorLoop() { // should be OK now to fill the pool with workers logger.info("Test probe successful - filling worker pool"); this.active = true; - + for (int i=0; i callReindexOperation()); - + // Slow down the ramp-up so we don't hit a new server with // hundreds of requests in one go safeSleep(1000); @@ -140,7 +140,7 @@ public void monitorLoop() { logger.info("Waiting for current threads to complete before restart: " + currentThreadCount); safeSleep(5000); } - + } else { // active // worker threads are active, so sleep for a bit before we check again safeSleep(5000); @@ -165,11 +165,11 @@ protected void safeSleep(long ms) { */ public void signalStop() { this.running = false; - + // make sure the pool doesn't start new work pool.shutdown(); } - + /** * Wait until things are stopped */ @@ -177,13 +177,13 @@ public void waitForStop() { if (this.running) { signalStop(); } - + try { pool.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException x) { logger.warning("Wait for pool shutdown interrupted"); } - + try { // break any sleep inside the monitorThread this.monitorThread.interrupt(); @@ -205,25 +205,25 @@ private void callReindexOperation() { this.active = false; } } - + this.currentlyRunning.decrementAndGet(); } - + /** * Make one call to the FHIR server $reindex operation * @return true if the call was successful (200 OK) */ private boolean callOnce() { boolean result = false; - + // tell the FHIR Server to reindex a number of resources long start = System.nanoTime(); FhirServerResponse response = fhirClient.post(url, requestBody); long end = System.nanoTime(); - + double elapsed = (end - start) / 1e9; logger.info(String.format("called $reindex: %d %s [took %5.3f s]", response.getStatusCode(), response.getStatusMessage(), elapsed)); - + if (response.getStatusCode() == HttpStatus.SC_OK) { Resource resource = response.getResource(); if (resource != null) { @@ -243,7 +243,7 @@ private boolean callOnce() { // Stop as soon as we hit an error logger.severe("FHIR Server reindex operation returned an error: " + response.getStatusCode() + " " + response.getStatusMessage()); } - + return result; } @@ -257,7 +257,7 @@ private void checkResult(OperationOutcome result) { Issue one = issues.get(0); if ("Reindex complete".equals(one.getDiagnostics().getValue())) { logger.info("Reindex - all done"); - + // tell all the running threads they can stop now this.running = false; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java index d5578c83ff9..7816fc713a2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java @@ -12,20 +12,15 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.sql.Timestamp; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import javax.transaction.TransactionSynchronizationRegistry; -import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceDeletedException; import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceNotFoundException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; @@ -49,7 +44,7 @@ public class ReindexResourceDAO extends ResourceDAOImpl { // The translator specific to the database type we're working with private final IDatabaseTranslator translator; - + private final ParameterDAO parameterDao; /** @@ -93,7 +88,7 @@ public ReindexResourceDAO(Connection connection, IDatabaseTranslator translator, protected IDatabaseTranslator getTranslator() { return this.translator; } - + /** * Pick the next resource to process resource and lock it. Specializations for different * databases may use different techniques to optimize locking/concurrency control @@ -101,23 +96,50 @@ protected IDatabaseTranslator getTranslator() { * @return * @throws Exception */ - protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexTstamp) throws Exception { + protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexTstamp, Integer resourceTypeId, String logicalId) throws Exception { ResourceIndexRecord result = null; - + // no need to close Connection connection = getConnection(); IDatabaseTranslator translator = getTranslator(); - + // Derby can only do select for update with simple queries, so we need to select first, // then try and lock, but we also have to try and cover the race condition which can // occur here, using an optimistic locking pattern - final String SELECT = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " - + " FROM logical_resources lr " - + " WHERE lr.reindex_tstamp < ? " - + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " - ; - + String select; + + if (resourceTypeId != null && logicalId != null) { + // Just pick the requested resource + select = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.logical_id = ? " + + " AND lr.reindex_tstamp < ? " + ; + + } else if (resourceTypeId != null) { + // Limit to the given resource type + select = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.reindex_tstamp < ? " + + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " + ; + + } else if (resourceTypeId == null && logicalId == null) { + select = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.reindex_tstamp < ? " + + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " + ; + } else { + // programming error + throw new IllegalArgumentException("logicalId specified without a resourceType"); + } + // Randomly pick an offset, but if we get no rows, reduce the range // until we hit 0. This will ensure we get the last few rows more // quickly @@ -125,15 +147,25 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind do { // random offset in [0, offsetRange) int offset = random.nextInt(offsetRange); - try (PreparedStatement stmt = connection.prepareStatement(SELECT)) { - stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); - stmt.setInt(2, offset); + try (PreparedStatement stmt = connection.prepareStatement(select)) { + if (resourceTypeId != null && logicalId != null) { + stmt.setInt(1, resourceTypeId); + stmt.setString(2, logicalId); + stmt.setTimestamp(3, Timestamp.from(reindexTstamp)); + } else if (resourceTypeId != null) { + stmt.setInt(1, resourceTypeId); + stmt.setTimestamp(2, Timestamp.from(reindexTstamp)); + stmt.setInt(3, offset); + } else { + stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); + stmt.setInt(2, offset); + } ResultSet rs = stmt.executeQuery(); if (rs.next()) { result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); } } catch (SQLException x) { - logger.log(Level.SEVERE, SELECT, x); + logger.log(Level.SEVERE, select, x); throw translator.translate(x); } @@ -146,7 +178,7 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind + " reindex_txid = ? " + " WHERE logical_resource_id = ? " + " AND reindex_txid = ? "; // make sure we have the txid we selected above - + try (PreparedStatement stmt = connection.prepareStatement(UPDATE)) { stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); stmt.setLong(2, result.getTransactionId() + 1L); @@ -170,10 +202,10 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind offsetRange /= 2; } } while (offsetRange > 0 && result == null); - + return result; } - + /** * Get the resource record we want to reindex. This might take a few attempts, because * there could be hundreds of threads all trying to do the same thing, and we may see @@ -183,14 +215,14 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind * @return * @throws Exception */ - public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp) throws Exception { + public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp, Integer resourceTypeId, String logicalId) throws Exception { ResourceIndexRecord result = null; - + // no need to close Connection connection = getConnection(); // Get a resource which needs to be reindexed - result = getNextResource(this.random, reindexTstamp); + result = getNextResource(this.random, reindexTstamp, resourceTypeId, logicalId); if (result != null) { @@ -219,7 +251,7 @@ public ResourceIndexRecord getResourceToReindex(Instant reindexTstamp) throws Ex throw translator.translate(x); } } - + return result; } @@ -241,7 +273,7 @@ public void updateParameters(String tablePrefix, List p final String METHODNAME = "updateParameters() for " + tablePrefix + "/" + logicalId; logger.entering(CLASSNAME, METHODNAME); - + // no need to close Connection connection = getConnection(); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index 7597a67ebfb..2e8f050ec5c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -766,7 +766,7 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { // protect against code regression. Invalid/improper references should be // filtered out already. - logger.warning("Invalid reference parameter type: " + resourceType + "." + rpv.getName() + " type=" + refValue.getType().name()); + logger.warning("Invalid reference parameter type: '" + resourceType + "." + rpv.getName() + "' type=" + refValue.getType().name()); throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java index a08d2abfef1..0d07dc41d26 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ReferenceParmVal.java @@ -21,9 +21,6 @@ public class ReferenceParmVal implements ExtractedParameterValue { // The name of the parameter (key into PARAMETER_NAMES) private String name; - // The reference value - //private String valueString; - // The SearchParameter base type. If "Resource", then this is a Resource-level attribute private String base; @@ -45,14 +42,6 @@ public String getName() { return name; } -// public String getValueString() { -// return valueString; -// } - -// public void setValueString(String valueString) { -// this.valueString = valueString; -// } - /** * Get the refValue * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 1d29b52efd5..7af6e0f9c38 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -11,6 +11,7 @@ import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_JDBC_ENABLE_RESOURCE_TYPES_CACHE; import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_UPDATE_CREATE_ENABLED; import static com.ibm.fhir.model.type.String.string; +import static com.ibm.fhir.model.util.ModelSupport.getResourceType; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MAX_NUM_OF_COMPOSITE_COMPONENTS; import java.io.ByteArrayInputStream; @@ -1233,28 +1234,13 @@ private TransactionSynchronizationRegistry getTrxSynchRegistry() throws FHIRPers } } - private List extractCompartmentValues(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTO) throws Exception { - - List result = null; - - Map> compartmentRefParams = CompartmentUtil.getCompartmentParamsForResourceType(fhirResource.getClass().getSimpleName()); - - if (!compartmentRefParams.isEmpty()) { - //result = SearchUtil.extractCompartmentParameterValues(fhirResource, compartmentRefParams); - } else { - result = Collections.emptyList(); - } - - return result; - } - /** * Extracts search parameters for the passed FHIR Resource. * @param fhirResource - Some FHIR Resource * @param resourceDTO - A Resource DTO representation of the passed FHIR Resource. * @throws Exception */ - private List extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTO) + private List extractSearchParameters(Resource fhirResource, com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTOx) throws Exception { final String METHODNAME = "extractSearchParameters"; log.entering(CLASSNAME, METHODNAME); @@ -1275,14 +1261,14 @@ private List extractSearchParameters(Resource fhirResou // As not to inject any other special handling logic, this is a simple inline check to see if // _id or _lastUpdated are used, and ignore those extracted values. - if(SPECIAL_HANDLING.contains(code)) { + if (SPECIAL_HANDLING.contains(code)) { continue; } type = sp.getType().getValue(); expression = sp.getExpression().getValue(); if (log.isLoggable(Level.FINE)) { - log.fine("Processing SearchParameter code: " + code + ", type: " + type + ", expression: " + expression); + log.fine("Processing SearchParameter resource: " + fhirResource.getClass().getSimpleName() + ", code: " + code + ", type: " + type + ", expression: " + expression); } List values = entry.getValue(); @@ -1484,6 +1470,7 @@ private List extractSearchParameters(Resource fhirResou */ protected void addCompartmentParams(List allParameters, Resource fhirResource) throws FHIRSearchException { final String resourceType = fhirResource.getClass().getSimpleName(); + log.fine("Processing compartment parameters for resourceType: " + resourceType); Map> compartmentRefParams = CompartmentUtil.getCompartmentParamsForResourceType(resourceType); Map> compartmentMap = SearchUtil.extractCompartmentParameterValues(fhirResource, compartmentRefParams); @@ -1794,7 +1781,7 @@ public boolean isReindexSupported() { } @Override - public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp) + public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp, String resourceLogicalId) throws FHIRPersistenceException { final String METHODNAME = "reindex"; log.entering(CLASSNAME, METHODNAME); @@ -1802,13 +1789,13 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper int result = 0; if (log.isLoggable(Level.FINE)) { - log.fine("reindex _tstamp=" + tstamp.toString()); + log.fine("reindex tstamp=" + tstamp.toString()); } if (tstamp.isAfter(java.time.Instant.now())) { // protect against setting a future timestamp, which could otherwise // disable the ability to reindex anything - throw new FHIRPersistenceException("Reindex _tstamp cannot be in the future"); + throw new FHIRPersistenceException("Reindex tstamp cannot be in the future"); } try (Connection connection = openConnection()) { @@ -1817,8 +1804,27 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper ReindexResourceDAO reindexDAO = FHIRResourceDAOFactory.getReindexResourceDAO(connection, FhirSchemaConstants.FHIR_ADMIN, schemaNameSupplier.getSchemaForRequestContext(connection), connectionStrategy.getFlavor(), this.trxSynchRegistry, this.cache, parameterDao); // Obtain a resource we will reindex in this request/transaction. The record is locked as part // of its selection, so we avoid a lot of (but not all) deadlock issues + Integer resourceTypeId = null; + String resourceType = null; + String logicalId = null; + if (resourceLogicalId != null) { + // Restrict reindex to a specific resource type or resource e.g. "Patient" or "Patient/abc123" + String[] parts = resourceLogicalId.split("/"); + if (parts.length == 1) { + // Limit to resource type + resourceType = parts[0]; + } else if (parts.length == 2) { + // Limit to a single resource + resourceType = parts[0]; + logicalId = parts[1]; + } + + // Look up the resourceTypeId for the given resourceType + resourceTypeId = cache.getResourceTypeCache().getId(resourceType); + } + long start = System.nanoTime(); - ResourceIndexRecord rir = reindexDAO.getResourceToReindex(tstamp); + ResourceIndexRecord rir = reindexDAO.getResourceToReindex(tstamp, resourceTypeId, logicalId); long end = System.nanoTime(); if (log.isLoggable(Level.FINER)) { @@ -1836,21 +1842,9 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper // Read the current resource com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO = resourceDao.read(rir.getLogicalId(), rir.getResourceType()); - if (existingResourceDTO != null) { - List elements = Collections.emptyList(); - Resource existingResource = this.convertResourceDTO(existingResourceDTO, Resource.class, elements); - - reindexDAO.setPersistenceContext(context); - reindexDAO.updateParameters(rir.getResourceType(), this.extractSearchParameters(existingResource, existingResourceDTO), rir.getLogicalId(), rir.getLogicalResourceId()); - - // Use an OperationOutcome Issue to let the caller know that some work was performed - final String diag = "Processed " + rir.getResourceType() + "/" + rir.getLogicalId(); - operationOutcomeResult.issue(Issue.builder().code(IssueType.INFORMATIONAL).severity(IssueSeverity.INFORMATION).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); - } else { - // Reasonable to assume that this resource was deleted because we can't read it - final String diag = "Failed to read resource: " + rir.getResourceType() + "/" + rir.getLogicalId(); - operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); - } + Class resourceTypeClass = getResourceType(resourceType); + reindexDAO.setPersistenceContext(context); + updateParameters(rir, resourceTypeClass, existingResourceDTO, reindexDAO, operationOutcomeResult); } } catch(FHIRPersistenceFKVException e) { @@ -1886,6 +1880,37 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper return result; } + /** + * Update the parameters for the resource described by the given DTO + * @param + * @param rir + * @param resourceTypeClass + * @param existingResourceDTO + * @param reindexDAO + * @param operationOutcomeResult + * @throws Exception + */ + public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, + ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult) throws Exception { + if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { + List elements = Collections.emptyList(); + T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); + + // Extract parameters from the resource payload we just read and store them, replacing + // the existing set + reindexDAO.updateParameters(rir.getResourceType(), this.extractSearchParameters(existingResource, existingResourceDTO), rir.getLogicalId(), rir.getLogicalResourceId()); + + // Use an OperationOutcome Issue to let the caller know that some work was performed + final String diag = "Processed " + rir.getResourceType() + "/" + rir.getLogicalId(); + operationOutcomeResult.issue(Issue.builder().code(IssueType.INFORMATIONAL).severity(IssueSeverity.INFORMATION).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); + } else { + // Reasonable to assume that this resource was deleted because we can't read it + final String diag = "Failed to read resource: " + rir.getResourceType() + "/" + rir.getLogicalId(); + operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); + } + + } + @Override public String generateResourceId() { return logicalIdentityProvider.createNewIdentityValue(); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java index aab93389b18..19623dd61bd 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java @@ -58,15 +58,15 @@ public PostgresReindexResourceDAO(Connection connection, IDatabaseTranslator tra * @param cache * @param rrd */ - public PostgresReindexResourceDAO(Connection connection, IDatabaseTranslator translator, ParameterDAO parameterDao, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, + public PostgresReindexResourceDAO(Connection connection, IDatabaseTranslator translator, ParameterDAO parameterDao, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, ParameterTransactionDataImpl ptdi) { super(connection, translator, parameterDao, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); } - + @Override - public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexTstamp) throws Exception { + public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexTstamp, Integer resourceTypeId, String logicalId) throws Exception { ResourceIndexRecord result = null; - + // no need to close Connection connection = getConnection(); IDatabaseTranslator translator = getTranslator(); @@ -76,33 +76,87 @@ public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexT // by existing locks. The ORDER BY is included to persuade[force] Postgres to always // use the index instead of switching to a full tablescan when the distribution stats // confuse the optimizer. - final String UPDATE = "" - + " UPDATE logical_resources " - + " SET reindex_tstamp = ?," - + " reindex_txid = COALESCE(reindex_txid + 1, 1) " - + " WHERE logical_resource_id = ( " - + " SELECT lr.logical_resource_id " - + " FROM logical_resources lr " - + " WHERE lr.reindex_tstamp < ? " - + " ORDER BY lr.reindex_tstamp DESC " - + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " - ; - - try (PreparedStatement stmt = connection.prepareStatement(UPDATE)) { - stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); - stmt.setTimestamp(2, Timestamp.from(reindexTstamp)); + String update; + if (resourceTypeId != null && logicalId != null) { + // Limit to one resource + update = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?," + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.logical_id = ? " + + " AND lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + } else if (resourceTypeId != null) { + // Limit to one type of resource + update = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?, " + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + + } else if (resourceTypeId == null && logicalId == null) { + // Pick the next resource needing to be reindexed regardless of type + update = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?," + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + + } else { + // programming error + throw new IllegalArgumentException("logicalId specified without a resourceType"); + } + + try (PreparedStatement stmt = connection.prepareStatement(update)) { + if (resourceTypeId != null && logicalId != null) { + // specific resource + stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); + stmt.setInt(2, resourceTypeId); + stmt.setString(3, logicalId); + stmt.setTimestamp(4, Timestamp.from(reindexTstamp)); + } else if (resourceTypeId != null) { + // limit to resource type + stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); + stmt.setInt(2, resourceTypeId); + stmt.setTimestamp(3, Timestamp.from(reindexTstamp)); + } else { + // any resource type + stmt.setTimestamp(1, Timestamp.from(reindexTstamp)); + stmt.setTimestamp(2, Timestamp.from(reindexTstamp)); + } + stmt.execute(); ResultSet rs = stmt.getResultSet(); if (rs.next()) { result = new ResourceIndexRecord(rs.getLong(1), rs.getInt(2), rs.getString(3), rs.getLong(4)); } } catch (SQLException x) { - logger.log(Level.SEVERE, UPDATE, x); + logger.log(Level.SEVERE, update, x); throw translator.translate(x); } - + return result; } - } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java index d9a816f66dd..a593d1edfba 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/JDBCParameterBuildingVisitor.java @@ -618,18 +618,6 @@ public boolean visit(java.lang.String elementName, int elementIndex, Reference r // Log the error, but skip it because we're not supposed to throw exceptions here log.log(Level.WARNING, "Error processing reference", x); } - - // Make sure we process the identifier if there is one. -// Identifier identifier = reference.getIdentifier(); -// if (reference.getIdentifier() != null) { -// TokenParmVal p = new TokenParmVal(); -// p.setName(searchParamCode); -// if (identifier.getSystem() != null) { -// p.setValueSystem(identifier.getSystem().getValue()); -// } -// p.setValueCode(identifier.getValue().getValue()); -// result.add(p); -// } return false; } diff --git a/fhir-persistence-scout/src/main/java/com/ibm/fhir/persistence/scout/FHIRPersistenceScoutImpl.java b/fhir-persistence-scout/src/main/java/com/ibm/fhir/persistence/scout/FHIRPersistenceScoutImpl.java index cc56a1312ea..cedbdeca2e9 100644 --- a/fhir-persistence-scout/src/main/java/com/ibm/fhir/persistence/scout/FHIRPersistenceScoutImpl.java +++ b/fhir-persistence-scout/src/main/java/com/ibm/fhir/persistence/scout/FHIRPersistenceScoutImpl.java @@ -10,24 +10,16 @@ import static com.ibm.fhir.model.type.String.string; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.URL; -import java.sql.Connection; -import java.sql.SQLException; import java.sql.Timestamp; -import java.text.MessageFormat; import java.time.ZoneOffset; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.UUID; -import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -36,21 +28,15 @@ import javax.transaction.TransactionSynchronizationRegistry; import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.CqlSessionBuilder; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.ibm.fhir.config.FHIRConfigHelper; import com.ibm.fhir.config.FHIRConfiguration; -import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.config.PropertyGroup; import com.ibm.fhir.core.FHIRUtilities; -import com.ibm.fhir.database.utils.common.GetSequenceNextValueDAO; import com.ibm.fhir.model.format.Format; import com.ibm.fhir.model.generator.FHIRGenerator; import com.ibm.fhir.model.resource.OperationOutcome; +import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.resource.Resource; import com.ibm.fhir.model.resource.SearchParameter; -import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.resource.SearchParameter.Component; import com.ibm.fhir.model.type.CodeableConcept; import com.ibm.fhir.model.type.Element; @@ -72,16 +58,12 @@ import com.ibm.fhir.persistence.SingleResourceResult; import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.exception.FHIRPersistenceResourceDeletedException; -import com.ibm.fhir.persistence.scout.SearchParameters; import com.ibm.fhir.persistence.scout.SearchParameters.ParameterBlock; import com.ibm.fhir.persistence.scout.SearchParameters.StrValue; import com.ibm.fhir.persistence.scout.SearchParameters.StrValueList; import com.ibm.fhir.persistence.scout.SearchParameters.TokenValue; import com.ibm.fhir.persistence.scout.SearchParameters.TokenValueList; import com.ibm.fhir.persistence.scout.cql.DatasourceSessions; -import com.ibm.fhir.search.SearchConstants.Type; -import com.ibm.fhir.search.context.FHIRSearchContext; import com.ibm.fhir.search.date.DateTimeHandler; import com.ibm.fhir.search.util.SearchUtil; @@ -93,44 +75,44 @@ public class FHIRPersistenceScoutImpl implements FHIRPersistence { private static final Logger logger = Logger.getLogger(FHIRPersistenceScoutImpl.class.getName()); private static final String CLASSNAME = FHIRPersistenceScoutImpl.class.getName(); private static final Logger log = Logger.getLogger(CLASSNAME); - + public static final String TRX_SYNCH_REG_JNDI_NAME = "java:comp/TransactionSynchronizationRegistry"; - + // TODO. Shouldn't be necessary private static final int MAX_NUM_OF_COMPOSITE_COMPONENTS = 3; - + private TransactionSynchronizationRegistry trxSynchRegistry; - + private boolean updateCreateEnabled; - + private List supplementalIssues = new ArrayList<>(); - + /** - * Constructor for use when running as web application in WLP. - * @throws Exception + * Constructor for use when running as web application in WLP. + * @throws Exception */ public FHIRPersistenceScoutImpl() throws Exception { super(); final String METHODNAME = "FHIRPersistenceCloudantImpl()"; log.entering(CLASSNAME, METHODNAME); - + PropertyGroup fhirConfig = FHIRConfiguration.getInstance().loadConfiguration(); this.updateCreateEnabled = fhirConfig.getBooleanProperty(PROPERTY_UPDATE_CREATE_ENABLED, Boolean.TRUE); log.exiting(CLASSNAME, METHODNAME); } - + /** * Constructor for use when running standalone, outside of any web container. - * @throws Exception + * @throws Exception */ @SuppressWarnings("rawtypes") public FHIRPersistenceScoutImpl(Properties configProps) throws Exception { final String METHODNAME = "FHIRPersistenceCloudantImpl(Properties)"; log.entering(CLASSNAME, METHODNAME); - + this.updateCreateEnabled = Boolean.parseBoolean(configProps.getProperty("updateCreateEnabled")); - + log.exiting(CLASSNAME, METHODNAME); } @@ -156,10 +138,10 @@ public SingleResourceResult create(FHIRPersistenceContex try { ByteArrayOutputStream stream = new ByteArrayOutputStream(); String logicalId; - + // We need to update the meta in the resource, so we need a modifiable version Resource.Builder resultResourceBuilder = resource.toBuilder(); - + // This create() operation is only called by a REST create. If the given resource // contains an id, then for R4 we need to ignore it and replace it with our // system-generated value. For the update-or-create scenario, see update(). @@ -169,7 +151,7 @@ public SingleResourceResult create(FHIRPersistenceContex if (log.isLoggable(Level.FINE)) { log.fine("Creating new FHIR Resource of type '" + resource.getClass().getSimpleName() + "'"); } - + // Set the resource id and meta fields. Instant lastUpdated = Instant.now(ZoneOffset.UTC); resultResourceBuilder.id(logicalId); @@ -178,25 +160,25 @@ public SingleResourceResult create(FHIRPersistenceContex metaBuilder.versionId(Id.of(Integer.toString(newVersionNumber))); metaBuilder.lastUpdated(lastUpdated); resultResourceBuilder.meta(metaBuilder.build()); - + // rebuild the resource with updated meta @SuppressWarnings("unchecked") T updatedResource = (T) resultResourceBuilder.build(); - + // Create the parameter block we will populate with all the parameters // extracted from the resource. This parameter block gets serialized // and pushed into Redis Timestamp timestamp = FHIRUtilities.convertToTimestamp(lastUpdated.getValue()); ParameterBlock.Builder pb = ParameterBlock.newBuilder(); - + pb.setLogicalId(logicalId); pb.setVersionId(newVersionNumber); pb.setLastUpdated(timestamp.toInstant().toEpochMilli()); pb.setResourceType(updatedResource.getClass().getSimpleName()); - + // Extract parameters into the ParameterBlock - - + + // Serialize and compress the Resource GZIPOutputStream zipStream = new GZIPOutputStream(stream); FHIRGenerator.generator( Format.JSON, false).generate(updatedResource, zipStream); @@ -207,18 +189,18 @@ public SingleResourceResult create(FHIRPersistenceContex // Save the data List supplementalIssues = new ArrayList<>(); persist(pb.build(), payload); - + SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .resource(updatedResource); - + // Add supplemental issues to the OperationOutcome if (!supplementalIssues.isEmpty()) { resultBuilder.outcome(OperationOutcome.builder() .issue(supplementalIssues) .build()); } - + return resultBuilder.build(); } catch (Throwable e) { FHIRPersistenceException fx = new FHIRPersistenceException("Unexpected error while performing a create operation."); @@ -237,7 +219,7 @@ public SingleResourceResult create(FHIRPersistenceContex */ protected void persist(ParameterBlock pb, byte[] payload) { try (CqlSession session = getCqlSession()) { - + } } @@ -282,17 +264,17 @@ public boolean isTransactional() { @Override public OperationOutcome getHealth() throws FHIRPersistenceException { - + StrValue.Builder builder = StrValue.newBuilder(); builder.setStrValue("hello"); - + StrValueList.Builder slBuilder = StrValueList.newBuilder(); slBuilder.addStringValues(builder.build()); - + SearchParameters.ParameterBlock.Builder pb = SearchParameters.ParameterBlock.newBuilder(); pb.putStringValues("strParam", slBuilder.build()); - + // Let's try creating a token TokenValue.Builder tokenBuilder = TokenValue.newBuilder(); tokenBuilder.setCodeSystem("system"); @@ -301,12 +283,12 @@ public OperationOutcome getHealth() throws FHIRPersistenceException { tvListBuilder.addTokenValues(tokenBuilder.build()); pb.putTokenValues("token1", tvListBuilder.build()); - + // TODO Check that we can connect to Redis and Cassandra return buildOKOperationOutcome(); // return buildErrorOperationOutcome(); } - + private OperationOutcome buildOKOperationOutcome() { return FHIRUtil.buildOperationOutcome("All OK", IssueType.INFORMATIONAL, IssueSeverity.INFORMATION); } @@ -321,7 +303,7 @@ public FHIRPersistenceTransaction getTransaction() { // TODO Auto-generated method stub return null; } - + /** * Extracts search parameters for the passed FHIR Resource. @@ -422,7 +404,7 @@ private void extractSearchParameters(ParameterBlockBuilderHelper parameters, Res * Create a Parameter DTO from the primitive value. * Note: this method only sets the value; * caller is responsible for setting all other fields on the created Parameter. - builder, value.asSystemValue(), code, fhirResource.getClass().getSimpleName()); + builder, value.asSystemValue(), code, fhirResource.getClass().getSimpleName()); */ private void processPrimitiveValue(ParameterBlockBuilderHelper parameters, String name, FHIRPathSystemValue systemValue, String code, String resourceType) { @@ -435,12 +417,12 @@ private void processPrimitiveValue(ParameterBlockBuilderHelper parameters, Strin parameters.addTokenParam(name, "false", system); } } else if (systemValue.isTemporalValue()) { - + TemporalAccessor v = systemValue.asTemporalValue().temporal(); java.time.Instant inst = DateTimeHandler.generateValue(v); long t = DateTimeHandler.generateTimestamp(inst).getTime(); parameters.addDateParam(name, t, t); - + } else if (systemValue.isStringValue()) { parameters.addStrParam(name, systemValue.asStringValue().string()); } else if (systemValue.isNumberValue()) { @@ -463,14 +445,14 @@ private void addWarning(IssueType issueType, String message, String... expressio .expression(Arrays.stream(expression).map(com.ibm.fhir.model.type.String::string).collect(Collectors.toList())) .build()); } - + @Override public String generateResourceId() { return UUID.randomUUID().toString(); } - + @Override - public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, java.time.Instant tstamp) throws FHIRPersistenceException { + public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, java.time.Instant tstamp, String resourceLogicalId) throws FHIRPersistenceException { return 0; } } diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java index b78f1580bc2..c106433b8d7 100644 --- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java +++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java @@ -135,7 +135,7 @@ default boolean isDeleteSupported() { /** * Generates a resource ID. - * + * * @return resource ID */ String generateResourceId(); @@ -147,7 +147,7 @@ default boolean isDeleteSupported() { default boolean isReindexSupported() { return false; } - + /** * Initiates reindexing for resources not yet processed. Limits the number of resources * processed to resourceCount. The number processed is returned in the OperationOutcome. @@ -158,9 +158,10 @@ default boolean isReindexSupported() { * @param context the FHIRPersistenceContext instance associated with the current request. * @param operationOutcomeResult accumulate issues in this {@link Builder} * @param tstamp reindex any resources with an index_tstamp less than this. + * @param resourceLogicalId optional resourceType/logicalId value to reindex a specific resource * @return count of the number of resources reindexed by this call (0 or 1) * @throws FHIRPersistenceException */ - int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, Instant tstamp) + int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId) throws FHIRPersistenceException; } \ No newline at end of file diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java index b0a4bc03d0a..9301be3dc15 100644 --- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java +++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java @@ -72,7 +72,7 @@ public OperationOutcome getHealth() throws FHIRPersistenceException { } @Override - public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp) throws FHIRPersistenceException { + public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp, String resourceLogicalId) throws FHIRPersistenceException { return 0; } diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java index 4130b99119d..83232a38930 100644 --- a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java +++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java @@ -1281,24 +1281,7 @@ public static FHIRSearchContext parseQueryParameters(String compartmentName, Str * @return */ public static boolean useStoredCompartmentParam() { - boolean result = false; - try { - String tenantId = FHIRRequestContext.get().getTenantId(); - PropertyGroup fhirConfig = FHIRConfiguration.getInstance().loadConfigurationForTenant(tenantId); - if (fhirConfig == null) { - // fall back to default config (when unit tests don't provide config for a tenant) - fhirConfig = FHIRConfiguration.getInstance().loadConfiguration(); - } - - if (fhirConfig != null) { - result = fhirConfig.getBooleanProperty(FHIRConfiguration.PROPERTY_USE_STORED_COMPARTMENT_PARAM, false); - } - } catch (Exception e) { - log.log(Level.WARNING, "Issue loading the fhir configuration - assuming " + FHIRConfiguration.PROPERTY_USE_STORED_COMPARTMENT_PARAM - + " is false", e); - } - - return result; + return FHIRConfigHelper.getBooleanProperty(FHIRConfiguration.PROPERTY_USE_STORED_COMPARTMENT_PARAM, false); } /** @@ -2225,6 +2208,8 @@ public static Map> extractCompartmentParameter for (Map.Entry> paramEntry : compartmentRefParams.entrySet()) { final String searchParm = paramEntry.getKey(); + log.finest("searchParam = [" + resourceType + "] '" + searchParm + "'"); + // Ignore {def} which is used in the compartment definition where // no other search parm is given (e.g. Encounter->Encounter). if (!COMPARTMENT_PARM_DEF.equals(searchParm)) { @@ -2236,10 +2221,18 @@ public static Map> extractCompartmentParameter log.fine("searchParam = [" + resourceType + "] '" + searchParm + "'; expression = '" + expression + "'"); } Collection nodes = FHIRPathEvaluator.evaluator().evaluate(resourceContext, expression); + + if (log.isLoggable(Level.FINEST)) { + log.finest("Expression [" + expression + "], parameter-code [" + + searchParm + "], size [" + nodes.size() + "]"); + } + for (FHIRPathNode node : nodes) { Reference reference = node.asElementNode().element().as(Reference.class); ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(reference, baseUrl); if (rv.getType() != ReferenceType.DISPLAY_ONLY && rv.getType() != ReferenceType.INVALID) { + log.finest("reference value = [" + rv.getTargetResourceType() + "] '" + rv.getValue() + "'"); + // Check that the target resource type of the reference matches one of the // target resource types in the compartment definition. final String compartmentName = rv.getTargetResourceType(); @@ -2258,7 +2251,7 @@ public static Map> extractCompartmentParameter } } catch (Exception e) { final String msg = "Unexpected exception extracting compartment references " - + " for resource type " + resourceType; + + " for resource type '" + resourceType + "'"; log.log(Level.WARNING, msg, e); throw SearchExceptionUtil.buildNewInvalidSearchException(msg); } diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/operation/spi/FHIRResourceHelpers.java b/fhir-server/src/main/java/com/ibm/fhir/server/operation/spi/FHIRResourceHelpers.java index 7bf78d803d0..adcd8ea6361 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/operation/spi/FHIRResourceHelpers.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/operation/spi/FHIRResourceHelpers.java @@ -246,8 +246,9 @@ public Resource doInvoke(FHIROperationContext operationContext, String resourceT * @param operationContext * @param operationOutcomeResult * @param tstamp + * @param resourceLogicalId a reference to a resource e.g. "Patient/abc123". Can be null * @return number of resources reindexed (0 if no resources were found to reindex) * @throws Exception */ - public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp) throws Exception; + public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId) throws Exception; } diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java index f17fafb8288..87b95a0e7d8 100644 --- a/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java +++ b/fhir-server/src/main/java/com/ibm/fhir/server/util/FHIRRestHelper.java @@ -2668,7 +2668,7 @@ private void setOperationContextProperties(FHIROperationContext operationContext } @Override - public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp) throws Exception { + public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId) throws Exception { int result = 0; // handle some retries in case of deadlock exceptions final int TX_ATTEMPTS = 5; @@ -2678,7 +2678,7 @@ public int doReindex(FHIROperationContext operationContext, OperationOutcome.Bui txn.begin(); try { FHIRPersistenceContext persistenceContext = null; - result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp); + result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, resourceLogicalId); attempt = TX_ATTEMPTS; // end the retry loop } catch (FHIRPersistenceDataAccessException x) { if (x.isTransactionRetryable() && attempt < TX_ATTEMPTS) { diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java index 67a219b4a37..7747aa92577 100644 --- a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java +++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java @@ -116,7 +116,7 @@ public String generateResourceId() { } @Override - public int reindex(FHIRPersistenceContext context, Builder operationOutcomeResult, java.time.Instant tstamp) throws FHIRPersistenceException { + public int reindex(FHIRPersistenceContext context, Builder operationOutcomeResult, java.time.Instant tstamp, String resourceLogicalId) throws FHIRPersistenceException { return 0; } } diff --git a/operation/fhir-operation-bulkdata/src/test/java/com/ibm/fhir/operation/bulkdata/processor/DummyImportExportImplTest.java b/operation/fhir-operation-bulkdata/src/test/java/com/ibm/fhir/operation/bulkdata/processor/DummyImportExportImplTest.java index 3cd4c2848fb..71a7ddb9bc1 100644 --- a/operation/fhir-operation-bulkdata/src/test/java/com/ibm/fhir/operation/bulkdata/processor/DummyImportExportImplTest.java +++ b/operation/fhir-operation-bulkdata/src/test/java/com/ibm/fhir/operation/bulkdata/processor/DummyImportExportImplTest.java @@ -300,7 +300,7 @@ public FHIRPersistenceTransaction getTransaction() throws Exception { } @Override - public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder oob, java.time.Instant tstamp) throws Exception { + public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder oob, java.time.Instant tstamp, String resourceLogicalId) throws Exception { return 0; } }; diff --git a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java index d7c152c83c6..28b1c30c601 100644 --- a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java +++ b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java @@ -37,8 +37,9 @@ public class ReindexOperation extends AbstractOperation { private static final Logger logger = Logger.getLogger(ReindexOperation.class.getName()); - private static final String PARAM_TSTAMP = "_tstamp"; - private static final String PARAM_RESOURCE_COUNT = "_resourceCount"; + private static final String PARAM_TSTAMP = "tstamp"; + private static final String PARAM_RESOURCE_COUNT = "resourceCount"; + private static final String PARAM_RESOURCE_LOGICAL_ID = "resourceLogicalId"; static final DateTimeFormatter DAY_FORMAT = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd") @@ -73,6 +74,7 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class 0; i++) { - processed = resourceHelper.doReindex(operationContext, result, tstamp); + processed = resourceHelper.doReindex(operationContext, result, tstamp, resourceLogicalId); totalProcessed += processed; } diff --git a/operation/fhir-operation-reindex/src/main/resources/reindex.json b/operation/fhir-operation-reindex/src/main/resources/reindex.json index 5a62813962a..73b181597a7 100644 --- a/operation/fhir-operation-reindex/src/main/resources/reindex.json +++ b/operation/fhir-operation-reindex/src/main/resources/reindex.json @@ -18,7 +18,7 @@ "instance": false, "parameter": [ { - "name": "_resourceCount", + "name": "resourceCount", "use": "in", "min": 0, "max": "1", @@ -26,12 +26,20 @@ "type": "integer" }, { - "name": "_tstamp", + "name": "tstamp", "use": "in", "min": 0, "max": "1", "documentation": "Reindex any resource not previously reindexed before this timestamp. Format as a date YYYY-MM-DD or time YYYY-MM-DDTHH:MM:DDZ.", "type": "string" + }, + { + "name": "resourceLogicalId", + "use": "in", + "min": 0, + "max": "1", + "documentation": "Reindex only the specified resource or resources of the given resource type when no id is provided. Format as Patient/abc123 or Patient", + "type": "string" } ] -} +} \ No newline at end of file From d14f7758af996bbb04e978a92ef8235bf456d2d0 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Sun, 22 Nov 2020 12:37:55 -0500 Subject: [PATCH 4/8] issue #1708 updated docs per review comments Signed-off-by: Robin Arnold --- docs/src/pages/guides/FHIRServerUsersGuide.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 45873e807d9..9686b545201 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -45,7 +45,7 @@ This FHIR server is intended to be a common component for providing FHIR capabil ## 2.1 Installing a new server 0. Prereqs: The IBM FHIR Server requires Java 8 or higher and has been tested with OpenJDK 8, OpenJDK 11, and the IBM SDK, Java Technology Edition, Version 8. To install Java on your system, we recommend downloading and installing OpenJDK 8 from https://adoptopenjdk.net/. -1. To install the FHIR server, build or download the `fhir-install` zip installer (e.g. `fhir-server-distribution.zip` or `fhir-install-4.0.0-rc1-20191014-1610`). +1. To install the IBM FHIR Server, build or download the `fhir-install` zip installer (e.g. `fhir-server-distribution.zip` or `fhir-install-4.0.0-rc1-20191014-1610`). The Maven build creates the zip package under `fhir-install/target`. Alternatively, releases will be made available from the [Releases tab](https://github.com/ibm/fhir/releases). 2. Unzip the `.zip` package into a clean directory (referred to as `fhir-installer` here): @@ -55,17 +55,17 @@ The Maven build creates the zip package under `fhir-install/target`. Alternative unzip fhir-server-distribution.zip ``` -3. Determine an install location for the OpenLiberty server and the FHIR server webapp. Example: `/opt/ibm/fhir-server` +3. Determine an install location for the OpenLiberty server and the IBM FHIR Server webapp. Example: `/opt/ibm/fhir-server` 4. Run the `install.sh/.bat` script to install the server: ``` ./fhir-server-dist/install.sh /opt/ibm/fhir-server ``` - This step installs the OpenLiberty runtime and the FHIR server web application. The Liberty runtime is installed in a directory called `wlp` within the installation directory that you specify. For example, in the preceding command, the root directory of the Liberty server runtime would be `/opt/ibm/fhir-server/wlp`. + This step installs the OpenLiberty runtime and the IBM FHIR Server web application. The Liberty runtime is installed in a directory called `wlp` within the installation directory that you specify. For example, in the preceding command, the root directory of the Liberty server runtime would be `/opt/ibm/fhir-server/wlp`. 5. Configure the fhir-server's `server.xml` file as needed by completing the following steps: * Configure the ports that the server listen on. The server is installed with only port 9443 (HTTPS) enabled by default. To change the port numbers, modify the values in the `httpEndpoint` element. - * Configure a server keystore and truststore. The FHIR server is installed with a default keystore file that contains a single self-signed certificate for localhost. For production use, you must create and configure your own keystore and truststore files for the FHIR server deployment (that is, generate your own server certificate or obtain a trusted certificate, and then share the public key certificate with API consumers so that they can insert it into their client-side truststore). The keystore and truststore files are used along with the server's HTTPS endpoint and the FHIR server's client-certificate-based authentication protocol to secure the FHIR server's endpoint. For more information, see [Section 5.2 Keystores, truststores, and the FHIR server](#52-keystores-truststores-and-the-fhir-server). + * Configure a server keystore and truststore. The IBM FHIR Server is installed with a default keystore file that contains a single self-signed certificate for localhost. For production use, you must create and configure your own keystore and truststore files for the FHIR server deployment (that is, generate your own server certificate or obtain a trusted certificate, and then share the public key certificate with API consumers so that they can insert it into their client-side truststore). The keystore and truststore files are used along with the server's HTTPS endpoint and the FHIR server's client-certificate-based authentication protocol to secure the FHIR server's endpoint. For more information, see [Section 5.2 Keystores, truststores, and the FHIR server](#52-keystores-truststores-and-the-fhir-server). * Configure an appropriate user registry. The FHIR server is installed with a basic user registry that contains a single user named `fhiruser`. For production use, it's best to configure your own user registry. For more information about configuring user registries, see the [OpenLiberty documentation](https://openliberty.io/guides/security-intro.html#configuring-the-user-registry). To override the default fhiruser's password, one may set an Environment variable `FHIR_USER_PASSWORD` and for the fhiradmin's password one may set an Environment variable `FHIR_ADMIN_PASSWORD`. @@ -112,9 +112,9 @@ The Maven build creates the zip package under `fhir-install/target`. Alternative For more information about the capabilities of the implementation, see [Conformance](https://ibm.github.io/FHIR/Conformance). ## 2.2 Upgrading an existing server -The FHIR server does not include an upgrade installer. To upgrade a server to the next version, you can run the installer on a separate server, and then copy the resulting configuration files over to the existing server. +The IBM FHIR Server does not include an upgrade installer. To upgrade a server to the next version, you can run the installer on a separate server, and then copy the resulting configuration files over to the existing server. -To manage database updates over time, the FHIR server uses custom tools from the `fhir-database-utils` project. Through the use of a metadata table, the database utilities can detect the currently installed version of the database and apply any new changes that are needed to bring the database to the current level. +To manage database updates over time, the IBM FHIR Server uses custom tools from the `fhir-database-utils` project. Through the use of a metadata table, the database utilities can detect the currently installed version of the database and apply any new changes that are needed to bring the database to the current level. Complete the following steps to upgrade the server: @@ -131,7 +131,7 @@ The IBM FHIR Server includes a Docker image [ibmcom/ibm-fhir-server](https://hub Note, logging for the IBM FHIR Server docker image is to stderr and stdout, and is picked up by Logging agents. # 3 Configuration -This chapter contains information about the various ways in which the FHIR server can be configured by users. +This chapter contains information about the various ways in which the IBM FHIR Server can be configured by users. ## 3.1 Encoded passwords In the examples within the following sections, you'll see the default password `change-password`. In order to secure your server, these values should be changed. @@ -173,7 +173,7 @@ More information about multi-tenant support can be found in [Section 4.9 Multi-t ## 3.3.1 Compartment Search Performance -The IBM FHIR Server now supports the ability to compute and store compartment membership values during ingestion. Once stored, these values can help accelerate compartment-related search queries. To use this feature, update the IBM FHIR Server at least version 4.5.1 and run a reindex operation as described in the release notes for Release 4.5.0. The reindex operation reprocesses the resources stored in the database, computing and storing the new compartment reference values. After the reindex operation has completed, add the `useStoredCompartmentParam` configuration element to the relevant tenant fhir-server-config.json file to allow the search queries to use the pre-computed values: +The IBM FHIR Server supports the ability to compute and store compartment membership values during ingestion. Once stored, these values can help accelerate compartment-related search queries. To use this feature, update the IBM FHIR Server to at least version 4.5.1 and run a reindex operation as described in the [fhir-bucket](https://github.com/IBM/FHIR/tree/master/fhir-bucket) project [README](https://github.com/IBM/FHIR/blob/master/fhir-bucket/README.md). The reindex operation reprocesses the resources stored in the database, computing and storing the new compartment reference values. After the reindex operation has completed, add the `useStoredCompartmentParam` configuration element to the relevant tenant fhir-server-config.json file to allow the search queries to use the pre-computed values: ``` { @@ -184,6 +184,7 @@ The IBM FHIR Server now supports the ability to compute and store compartment me } } ``` +Note that this parameter only enables or disables the compartment search query optimization feature. The compartment membership values are always computed and stored during ingestion or reindexing, regardless of the setting of this value. After the reindex operation is complete, it is recommended to set `useStoredCompartmentParam` to true. No reindex is required if this value is subsequently set to false. ## 3.4 Persistence layer configuration The IBM FHIR Server allows deployers to select a persistence layer implementation that fits their needs. Currently, the server includes a JDBC persistence layer which supports Apache Derby, IBM Db2, and PostgreSQL. However, Apache Derby is not recommended for production usage. From cca3d4aa4982bdb2a156c41a1d22779f62eca4b3 Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Sun, 22 Nov 2020 19:00:02 -0500 Subject: [PATCH 5/8] issue #1708 updated copyright header Signed-off-by: Robin Arnold --- .../persistence/jdbc/test/util/ParameterExtractionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java index 55af4b1168a..b218837aab1 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterExtractionTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2020 * * SPDX-License-Identifier: Apache-2.0 */ From 74a614256d2cc4d1ee604fbd6ec967fe36b86a5d Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 23 Nov 2020 10:38:12 -0500 Subject: [PATCH 6/8] issue #1708 removed excess debug logging Signed-off-by: Robin Arnold --- .../src/main/java/com/ibm/fhir/search/util/SearchUtil.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java index 83232a38930..d3ecb8a3d60 100644 --- a/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java +++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/SearchUtil.java @@ -2208,8 +2208,6 @@ public static Map> extractCompartmentParameter for (Map.Entry> paramEntry : compartmentRefParams.entrySet()) { final String searchParm = paramEntry.getKey(); - log.finest("searchParam = [" + resourceType + "] '" + searchParm + "'"); - // Ignore {def} which is used in the compartment definition where // no other search parm is given (e.g. Encounter->Encounter). if (!COMPARTMENT_PARM_DEF.equals(searchParm)) { @@ -2231,7 +2229,6 @@ public static Map> extractCompartmentParameter Reference reference = node.asElementNode().element().as(Reference.class); ReferenceValue rv = ReferenceUtil.createReferenceValueFrom(reference, baseUrl); if (rv.getType() != ReferenceType.DISPLAY_ONLY && rv.getType() != ReferenceType.INVALID) { - log.finest("reference value = [" + rv.getTargetResourceType() + "] '" + rv.getValue() + "'"); // Check that the target resource type of the reference matches one of the // target resource types in the compartment definition. From 07f794b3780809ac687d95892c925c4b679ada5d Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 23 Nov 2020 13:35:29 -0500 Subject: [PATCH 7/8] issue #1742 clamp resource count to 1000 to avoid client read timeouts Signed-off-by: Robin Arnold --- fhir-bucket/README.md | 2 ++ .../com/ibm/fhir/operation/reindex/ReindexOperation.java | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/fhir-bucket/README.md b/fhir-bucket/README.md index 080cad07746..87998fb76c9 100644 --- a/fhir-bucket/README.md +++ b/fhir-bucket/README.md @@ -377,3 +377,5 @@ java \ ``` The format of the reindex timestamp can be a date `YYYY-MM-DD` representing midnight UTC on the given day, or an ISO timestamp `YYYY-MM-DDThh:mm:ssZ`. + +Values for `--reindex-resource-count` larger than 1000 will be clamped to 1000 to ensure that the `$reindex` server calls return within a reasonable time. diff --git a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java index 28b1c30c601..b1e68cd42bc 100644 --- a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java +++ b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java @@ -41,6 +41,9 @@ public class ReindexOperation extends AbstractOperation { private static final String PARAM_RESOURCE_COUNT = "resourceCount"; private static final String PARAM_RESOURCE_LOGICAL_ID = "resourceLogicalId"; + // The max number of resources we allow to be processed by one request + private static final int MAX_RESOURCE_COUNT = 1000; + static final DateTimeFormatter DAY_FORMAT = new DateTimeFormatterBuilder() .appendPattern("yyyy-MM-dd") .parseDefaulting(ChronoField.NANO_OF_DAY, 0) @@ -97,6 +100,10 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class MAX_RESOURCE_COUNT) { + logger.info("Clamping resourceCount " + val + " to max allowed: " + MAX_RESOURCE_COUNT); + val = MAX_RESOURCE_COUNT; + } resourceCount = val; } } else if (PARAM_RESOURCE_LOGICAL_ID.equals(parameter.getName().getValue())) { From 92bd1a75393fab6c1dbad6d357eaa936750d2c6d Mon Sep 17 00:00:00 2001 From: Robin Arnold Date: Mon, 23 Nov 2020 14:06:01 -0500 Subject: [PATCH 8/8] issue #1742 address review comments Signed-off-by: Robin Arnold --- .../jdbc/dao/ReindexResourceDAO.java | 50 ++++++----- .../jdbc/impl/FHIRPersistenceJDBCImpl.java | 3 +- .../PostgresReindexResourceDAO.java | 88 ++++++++++--------- 3 files changed, 74 insertions(+), 67 deletions(-) diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java index 7816fc713a2..6c3b31dd502 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/ReindexResourceDAO.java @@ -47,6 +47,29 @@ public class ReindexResourceDAO extends ResourceDAOImpl { private final ParameterDAO parameterDao; + private static final String PICK_SINGLE_RESOURCE = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.logical_id = ? " + + " AND lr.reindex_tstamp < ? " + ; + + private static final String PICK_SINGLE_RESOURCE_TYPE = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.reindex_tstamp < ? " + + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " + ; + + private static final String PICK_ANY_RESOURCE = "" + + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " + + " FROM logical_resources lr " + + " WHERE lr.reindex_tstamp < ? " + + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " + ; + /** * Public constructor * @param connection @@ -106,35 +129,16 @@ protected ResourceIndexRecord getNextResource(SecureRandom random, Instant reind // Derby can only do select for update with simple queries, so we need to select first, // then try and lock, but we also have to try and cover the race condition which can // occur here, using an optimistic locking pattern - String select; + final String select; if (resourceTypeId != null && logicalId != null) { // Just pick the requested resource - select = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " - + " FROM logical_resources lr " - + " WHERE lr.resource_type_id = ? " - + " AND lr.logical_id = ? " - + " AND lr.reindex_tstamp < ? " - ; - + select = PICK_SINGLE_RESOURCE; } else if (resourceTypeId != null) { // Limit to the given resource type - select = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " - + " FROM logical_resources lr " - + " WHERE lr.resource_type_id = ? " - + " AND lr.reindex_tstamp < ? " - + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " - ; - + select = PICK_SINGLE_RESOURCE_TYPE; } else if (resourceTypeId == null && logicalId == null) { - select = "" - + " SELECT lr.logical_resource_id, lr.resource_type_id, lr.logical_id, lr.reindex_txid " - + " FROM logical_resources lr " - + " WHERE lr.reindex_tstamp < ? " - + "OFFSET ? ROWS FETCH FIRST 1 ROWS ONLY " - ; + select = PICK_ANY_RESOURCE; } else { // programming error throw new IllegalArgumentException("logicalId specified without a resourceType"); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 7af6e0f9c38..eb7f310879d 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -1893,7 +1893,6 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult) throws Exception { if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { - List elements = Collections.emptyList(); T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); // Extract parameters from the resource payload we just read and store them, replacing @@ -1906,7 +1905,7 @@ public void updateParameters(ResourceIndexRecord rir, Class } else { // Reasonable to assume that this resource was deleted because we can't read it final String diag = "Failed to read resource: " + rir.getResourceType() + "/" + rir.getLogicalId(); - operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(com.ibm.fhir.model.type.String.of(diag)).build()); + operationOutcomeResult.issue(Issue.builder().code(IssueType.NOT_FOUND).severity(IssueSeverity.WARNING).diagnostics(string(diag)).build()); } } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java index 19623dd61bd..dfdd195c6c1 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgresql/PostgresReindexResourceDAO.java @@ -33,6 +33,48 @@ public class PostgresReindexResourceDAO extends ReindexResourceDAO { private static final Logger logger = Logger.getLogger(PostgresReindexResourceDAO.class.getName()); + private static final String PICK_SINGLE_RESOURCE = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?," + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.logical_id = ? " + + " AND lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + + private static final String PICK_SINGLE_RESOURCE_TYPE = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?, " + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.resource_type_id = ? " + + " AND lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + + private static final String PICK_ANY_RESOURCE = "" + + " UPDATE logical_resources " + + " SET reindex_tstamp = ?," + + " reindex_txid = COALESCE(reindex_txid + 1, 1) " + + " WHERE logical_resource_id = ( " + + " SELECT lr.logical_resource_id " + + " FROM logical_resources lr " + + " WHERE lr.reindex_tstamp < ? " + + " ORDER BY lr.reindex_tstamp DESC " + + " FOR UPDATE SKIP LOCKED LIMIT 1) " + + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " + ; + /** * Public constructor * @param connection @@ -76,54 +118,16 @@ public ResourceIndexRecord getNextResource(SecureRandom random, Instant reindexT // by existing locks. The ORDER BY is included to persuade[force] Postgres to always // use the index instead of switching to a full tablescan when the distribution stats // confuse the optimizer. - String update; + final String update; if (resourceTypeId != null && logicalId != null) { // Limit to one resource - update = "" - + " UPDATE logical_resources " - + " SET reindex_tstamp = ?," - + " reindex_txid = COALESCE(reindex_txid + 1, 1) " - + " WHERE logical_resource_id = ( " - + " SELECT lr.logical_resource_id " - + " FROM logical_resources lr " - + " WHERE lr.resource_type_id = ? " - + " AND lr.logical_id = ? " - + " AND lr.reindex_tstamp < ? " - + " ORDER BY lr.reindex_tstamp DESC " - + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " - ; + update = PICK_SINGLE_RESOURCE; } else if (resourceTypeId != null) { // Limit to one type of resource - update = "" - + " UPDATE logical_resources " - + " SET reindex_tstamp = ?, " - + " reindex_txid = COALESCE(reindex_txid + 1, 1) " - + " WHERE logical_resource_id = ( " - + " SELECT lr.logical_resource_id " - + " FROM logical_resources lr " - + " WHERE lr.resource_type_id = ? " - + " AND lr.reindex_tstamp < ? " - + " ORDER BY lr.reindex_tstamp DESC " - + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " - ; - + update = PICK_SINGLE_RESOURCE_TYPE; } else if (resourceTypeId == null && logicalId == null) { // Pick the next resource needing to be reindexed regardless of type - update = "" - + " UPDATE logical_resources " - + " SET reindex_tstamp = ?," - + " reindex_txid = COALESCE(reindex_txid + 1, 1) " - + " WHERE logical_resource_id = ( " - + " SELECT lr.logical_resource_id " - + " FROM logical_resources lr " - + " WHERE lr.reindex_tstamp < ? " - + " ORDER BY lr.reindex_tstamp DESC " - + " FOR UPDATE SKIP LOCKED LIMIT 1) " - + "RETURNING logical_resource_id, resource_type_id, logical_id, reindex_txid " - ; - + update = PICK_ANY_RESOURCE; } else { // programming error throw new IllegalArgumentException("logicalId specified without a resourceType");