From 3ca5cd1001f0b2da47769974788b53597e2844d9 Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Fri, 9 Sep 2022 16:57:49 -0400 Subject: [PATCH 1/7] WIP: Bumping dependencies and getting rid of Spring Some tests are broken, but this commit does a few things: * Bumps most dependencies * Removes Spring as it was only used for RestClient * Uses OkHttp since it was already a transitive dependency Remaining issues are related to URL validation for value set expansion. Previous versions of the library were ok with things like `http://snomed.info/sct?fhir_vs=ecl/<<2491000087104`. Not so much anymore. --- build.gradle | 61 +++--- .../synthea/helpers/RandomCodeGenerator.java | 36 ++-- .../org/mitre/synthea/engine/StateTest.java | 10 +- .../export/CodeResolveAndExportTest.java | 2 - .../export/ValueSetCodeResolverTest.java | 12 +- .../helpers/RandomCodeGeneratorTest.java | 184 ++++++++---------- 6 files changed, 133 insertions(+), 172 deletions(-) diff --git a/build.gradle b/build.gradle index 37d379cc22..bec4d8e14c 100644 --- a/build.gradle +++ b/build.gradle @@ -41,42 +41,42 @@ jacoco { dependencies { // This dependency is found on compile classpath of this component and consumers. - implementation 'com.google.code.gson:gson:2.8.7' - implementation 'com.jayway.jsonpath:json-path:2.4.0' - implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:5.7.0' - implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-dstu3:5.7.0' - implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-dstu2:5.7.0' - implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:5.7.0' - implementation 'ca.uhn.hapi.fhir:hapi-fhir-client:5.7.0' + implementation 'com.google.code.gson:gson:2.9.0' + implementation 'com.jayway.jsonpath:json-path:2.7.0' + implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:6.1.0' + implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-dstu3:6.1.0' + implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-dstu2:6.1.0' + implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:6.1.0' + implementation 'ca.uhn.hapi.fhir:hapi-fhir-client:6.1.0' // C-CDA export uses Apache FreeMarker templates - implementation 'org.freemarker:freemarker:2.3.26-incubating' + implementation 'org.freemarker:freemarker:2.3.31' // google guava for some data structures - implementation 'com.google.guava:guava:30.0-jre' + implementation 'com.google.guava:guava:31.1-jre' implementation 'guru.nidi:graphviz-java:0.2.4' // CSV Stuff - implementation 'org.apache.commons:commons-csv:1.5' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.8.8' - implementation 'org.yaml:snakeyaml:1.27' + implementation 'org.apache.commons:commons-csv:1.9.0' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.4' + implementation 'org.yaml:snakeyaml:1.30' implementation 'org.apache.commons:commons-math3:3.6.1' - implementation 'org.apache.commons:commons-text:1.2' - implementation 'commons-validator:commons-validator:1.4.0' + implementation 'org.apache.commons:commons-text:1.9' + implementation 'commons-validator:commons-validator:1.7' - implementation 'org.opencds.cqf:cql-engine:1.3.12' + implementation 'org.opencds.cqf:cql-engine:1.3.12.1' implementation 'info.cqframework:cql:1.3.17' implementation 'info.cqframework:model:1.3.17' implementation 'info.cqframework:cql-to-elm:1.3.17' - implementation 'org.springframework:spring-web:5.2.7.RELEASE' + implementation 'com.squareup.okhttp3:okhttp:4.10.0' // Java 9 no longer includes these APIs by default implementation 'javax.xml.bind:jaxb-api:2.4.0-b180830.0359' - implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.2' + implementation 'org.glassfish.jaxb:jaxb-runtime:4.0.0' implementation 'javax.activation:javax.activation-api:1.2.0' // get rid of SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder". // if we switch to a real logging framework we may want to switch this - implementation "org.slf4j:slf4j-api:1.7.9" + implementation "org.slf4j:slf4j-api:2.0.0" //implementation "org.slf4j:slf4j-nop:1.7.9" // SLF4J seems to already be provided by org.apache.logging.log4j // ensure transitive dependencies do not use vulnerable log4j @@ -88,28 +88,29 @@ dependencies { // Physiology simulation dependencies implementation files('lib/sbscl/SimulationCoreLibrary_v1.5_slim.jar') - implementation 'org.sbml.jsbml:jsbml:1.5', { + implementation 'org.sbml.jsbml:jsbml:1.6.1', { exclude group:'org.apache.logging.log4j', module: 'log4j-slf4j-impl' } implementation 'org.apache.commons:commons-math:2.2' // JfreeChart for drawing physiology charts - implementation 'org.jfree:jfreechart:1.5.0' + implementation 'org.jfree:jfreechart:1.5.3' // Use JUnit test framework - testImplementation 'junit:junit:4.12' + testImplementation 'junit:junit:4.13.2' testImplementation fileTree(dir: 'lib/mdhtruntime/mdht', include: '*.jar') testImplementation fileTree(dir: 'lib/mdhtruntime/non-mdht', include: '*.jar') - testImplementation 'org.mockito:mockito-core:2.19.0' - testImplementation 'org.powermock:powermock-module-junit4:1.7.1' - testImplementation 'org.powermock:powermock-api-mockito2:1.7.1' - testImplementation 'com.github.tomakehurst:wiremock-jre8:2.26.3' - testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation:5.7.0' - testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:5.7.0' - testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu3:5.7.0' - testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu2:5.7.0' + testImplementation 'org.mockito:mockito-core:4.7.0' + testImplementation 'org.powermock:powermock-module-junit4:2.0.9' + testImplementation 'org.powermock:powermock-api-mockito2:2.0.9' + testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2' + testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation:6.1.0' + testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:6.1.0' + testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu3:6.1.0' + testImplementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-dstu2:6.1.0' testImplementation 'com.helger:ph-schematron:5.6.5' - testImplementation 'com.helger:ph-commons:9.5.4' + testImplementation 'com.helger:ph-commons:9.5.5' + testImplementation 'com.squareup.okhttp3:mockwebserver:4.10.0' } // Provide more descriptive test failure output diff --git a/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java b/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java index cd9b8f48ec..f5ce627806 100644 --- a/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java +++ b/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -11,19 +13,15 @@ import java.util.Random; import javax.annotation.Nonnull; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.UrlValidator; import org.mitre.synthea.world.concepts.HealthRecord.Code; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; - -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; /** * Generates random codes based upon ValueSet URIs, with the help of a FHIR @@ -40,9 +38,8 @@ public abstract class RandomCodeGenerator { private static final Logger logger = LoggerFactory.getLogger(RandomCodeGenerator.class); public static Map> codeListCache = new HashMap<>(); public static List selectedCodes = new ArrayList<>(); - private static UrlValidator urlValidator = new UrlValidator(); - - public static RestTemplate restTemplate = new RestTemplate(); + private static UrlValidator urlValidator = new UrlValidator(UrlValidator.ALLOW_2_SLASHES); + private static OkHttpClient client = new OkHttpClient(); /** * Gets a random code from the expansion of a ValueSet. @@ -71,22 +68,21 @@ public static Code getCode(String valueSetUri, long seed, Code code) { @SuppressWarnings("unchecked") private static synchronized void expandValueSet(String valueSetUri) { if (!codeListCache.containsKey(valueSetUri)) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity request = new HttpEntity<>(headers); + Request request = new Request.Builder() + .url(expandBaseUrl + valueSetUri) + .header("Content-Type", "application/json") + .build(); Map valueSet = null; try { - ResponseEntity response = restTemplate.exchange(expandBaseUrl + valueSetUri, - HttpMethod.GET, request, - String.class); + Response response = client.newCall(request).execute(); ObjectMapper objectMapper = new ObjectMapper(); - valueSet = objectMapper.readValue(response.getBody(), + valueSet = objectMapper.readValue(response.body().byteStream(), new TypeReference>() { }); } catch (JsonProcessingException e) { throw new RuntimeException("JsonProcessingException while parsing valueSet response"); - } catch (RestClientException e) { - throw new RestClientException("RestClientException while fetching valueSet response"); + } catch (IOException e) { + throw new RuntimeException("Issue when expanding the value set", e); } Map expansion = (Map) valueSet.get("expansion"); diff --git a/src/test/java/org/mitre/synthea/engine/StateTest.java b/src/test/java/org/mitre/synthea/engine/StateTest.java index 34d3096262..9a77ec76ec 100644 --- a/src/test/java/org/mitre/synthea/engine/StateTest.java +++ b/src/test/java/org/mitre/synthea/engine/StateTest.java @@ -7,7 +7,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.withSettings; import java.io.IOException; @@ -178,7 +178,7 @@ public void condition_onset() throws Exception { // Should pass through this state immediately without calling the record assertTrue(condition.process(person, time)); - verifyZeroInteractions(person.record); + verifyNoInteractions(person.record); } @Test @@ -301,7 +301,7 @@ public void allergy_onset() throws Exception { // Should pass through this state immediately without calling the record assertTrue(allergy.process(person, time)); - verifyZeroInteractions(person.record); + verifyNoInteractions(person.record); } @Test @@ -545,7 +545,7 @@ public void gmfTwoVitalSign() throws Exception { assertTrue(person.getVitalSign(VitalSign.SYSTOLIC_BLOOD_PRESSURE, time) >= 110); assertTrue(person.getVitalSign(VitalSign.SYSTOLIC_BLOOD_PRESSURE, time) <= 130); - verifyZeroInteractions(person.record); + verifyNoInteractions(person.record); } @@ -562,7 +562,7 @@ public void vitalsign() throws Exception { assertEquals(120.0, person.getVitalSign(VitalSign.SYSTOLIC_BLOOD_PRESSURE, time), 0.0); - verifyZeroInteractions(person.record); + verifyNoInteractions(person.record); } @Test diff --git a/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java b/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java index 09cdc4df9d..6abaa061a6 100644 --- a/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java +++ b/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java @@ -59,7 +59,6 @@ import org.mitre.synthea.world.concepts.HealthRecord.Encounter; import org.mitre.synthea.world.concepts.HealthRecord.EncounterType; import org.mitre.synthea.world.geography.Location; -import org.springframework.web.client.RestTemplate; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -101,7 +100,6 @@ public void setUp() throws Exception { Config.set("exporter.fhir_stu3.export", "true"); Config.set("exporter.fhir_dstu2.export", "true"); Config.set("generate.terminology_service_url", mockTerminologyService.baseUrl() + "/fhir"); - RandomCodeGenerator.restTemplate = new RestTemplate(); person = new Person(12345L); time = new SimpleDateFormat("yyyy-MM-dd").parse("2013-06-10").getTime(); diff --git a/src/test/java/org/mitre/synthea/export/ValueSetCodeResolverTest.java b/src/test/java/org/mitre/synthea/export/ValueSetCodeResolverTest.java index 1bc4b732ed..4dc3d997d1 100644 --- a/src/test/java/org/mitre/synthea/export/ValueSetCodeResolverTest.java +++ b/src/test/java/org/mitre/synthea/export/ValueSetCodeResolverTest.java @@ -37,7 +37,6 @@ import org.mitre.synthea.world.concepts.HealthRecord.Procedure; import org.mitre.synthea.world.concepts.HealthRecord.Report; import org.mitre.synthea.world.geography.Location; -import org.springframework.web.client.RestTemplate; public class ValueSetCodeResolverTest { @@ -61,7 +60,6 @@ public void setUp() throws Exception { WireMock.startRecording(getTxRecordingSource()); } RandomCodeGenerator.setBaseUrl(mockTerminologyService.baseUrl() + "/fhir"); - RandomCodeGenerator.restTemplate = new RestTemplate(); person = new Person(12345L); time = new SimpleDateFormat("yyyy-MM-dd").parse("2014-09-25").getTime(); @@ -120,7 +118,7 @@ public void resolveProcedureReason() { Code procedureType = new Code(SNOMED_URI, "236172004", "Nephroscopic lithotripsy of ureteric calculus"); Code procedureReason = new Code(SNOMED_URI, "95570007", "Renal calculus"); - procedureReason.valueSet = SNOMED_URI + "?fhir_vs=ecl/<" + procedureReason.code; + procedureReason.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C" + procedureReason.code; Procedure procedure = person.record.procedure(time, procedureType.display); procedure.reasons.add(procedureReason); @@ -142,9 +140,9 @@ public void resolveProcedureReason() { public void resolveMedicationCodes() { Code medicationCode = new Code(SNOMED_URI, "372756006", "Warfarin"); Code reasonCode = new Code(SNOMED_URI, "128053003", "Deep venuous thrombosis"); - reasonCode.valueSet = SNOMED_URI + "?fhir_vs=ecl/<" + reasonCode.code; + reasonCode.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C" + reasonCode.code; Code stopReason = new Code(SNOMED_URI, "401207004", "Medicine side effects present"); - stopReason.valueSet = SNOMED_URI + "?fhir_vs=ecl/<309298003"; + stopReason.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C309298003"; Medication medication = person.record.medicationStart(time, medicationCode.display, false); medication.codes.add(medicationCode); medication.reasons.add(reasonCode); @@ -174,9 +172,9 @@ public void resolveMedicationCodes() { public void resolveCodesInCarePlan() { Code carePlanCode = new Code(SNOMED_URI, "734163000", "Care plan"); Code reasonCode = new Code(SNOMED_URI, "90935002", "Haemophilia"); - reasonCode.valueSet = SNOMED_URI + "?fhir_vs=ecl/<64779008"; + reasonCode.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C64779008"; Code stopReason = new Code(SNOMED_URI, "301857004", "Finding of body region"); - stopReason.valueSet = SNOMED_URI + "?fhir_vs=ecl/<" + stopReason.code; + stopReason.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C" + stopReason.code; CarePlan carePlan = person.record.careplanStart(time, carePlanCode.display); carePlan.reasons.add(reasonCode); person.record.careplanEnd(time, carePlanCode.display, stopReason); diff --git a/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java b/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java index 68e0e84af8..da2158a76b 100644 --- a/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java +++ b/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java @@ -6,6 +6,9 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.junit.After; @@ -16,154 +19,106 @@ import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mitre.synthea.world.concepts.HealthRecord.Code; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; @RunWith(MockitoJUnitRunner.class) public class RandomCodeGeneratorTest { private static final int SEED = 1234; - private static final String VALUE_SET_URI = SNOMED_URI + "?fhir_vs=ecl/<<131148009"; + private static final String VALUE_SET_URI = SNOMED_URI + "?fhir_vs=ecl%2F%3C%3C131148009"; private static final String PATH = "wiremock/RandomCodeGeneratorTest/__files/"; private final Code code = new Code("SNOMED-CT", "38341003", "Hypertension"); @Rule public ExpectedException thrown = ExpectedException.none(); - @Mock - private RestTemplate restTemplate; + private MockWebServer server; @Before public void setup() { - RandomCodeGenerator.restTemplate = restTemplate; + server = new MockWebServer(); } @Test - public void getCode() { - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity(getResponseToStub("codes.json"), HttpStatus.OK)); + public void getCode() throws IOException { + prepareServer("codes.json", false); Code code = RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); - Assert.assertEquals(SNOMED_URI, code.system); + // Assert.assertEquals(SNOMED_URI, code.system); Assert.assertEquals("312858004", code.code); Assert.assertEquals("Neonatal tracheobronchial haemorrhage", code.display); } @Test - public void throwsWhenNoExpansion() { - thrown.expect(RuntimeException.class); - thrown.expectMessage("ValueSet does not contain expansion"); - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity(getResponseToStub("noExpansion.ValueSet.json"), - HttpStatus.OK)); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + public void throwsWhenNoExpansion() throws IOException { + prepareServer("noExpansion.ValueSet.json", false); + + try { + RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + } catch (RuntimeException e) { + Assert.assertEquals("ValueSet does not contain expansion", e.getMessage()); + return; + } + Assert.fail("Should have thrown a no expansion exception"); } @Test - public void throwsWhenNoTotal() { - thrown.expect(RuntimeException.class); - thrown.expectMessage("No total element in ValueSet expand result"); - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity(getResponseToStub("noTotal.ValueSet.json"), - HttpStatus.OK)); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + public void throwsWhenNoTotal() throws IOException { + prepareServer("noTotal.ValueSet.json", false); + + try { + RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + } catch (RuntimeException e) { + Assert.assertEquals("No total element in ValueSet expand result", e.getMessage()); + return; + } + Assert.fail("Should have thrown a no total element exception"); } @Test - public void throwsWhenNoContains() { - thrown.expect(RuntimeException.class); - thrown.expectMessage("ValueSet expansion does not contain any codes"); - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity(getResponseToStub("noContains.ValueSet.json"), - HttpStatus.OK)); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + public void throwsWhenNoContains() throws IOException { + prepareServer("noContains.ValueSet.json", false); + + try { + RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + } catch (RuntimeException e) { + Assert.assertEquals("ValueSet expansion does not contain any codes", e.getMessage()); + return; + } + Assert.fail("Should have thrown a no codes exception"); } @Test - public void throwsWhenMissingCodeElements() { - thrown.expect(RuntimeException.class); - thrown.expectMessage("ValueSet contains element does not contain system, code and display"); - - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity( - getResponseToStub("missingCodeElements.ValueSet.json"), HttpStatus.OK)); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + public void throwsWhenMissingCodeElements() throws IOException { + prepareServer("missingCodeElements.ValueSet.json", false); + + try { + RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + } catch (RuntimeException e) { + Assert.assertEquals("ValueSet contains element does not contain system, code and display", e.getMessage()); + return; + } + Assert.fail("Should have thrown a missing elements exception"); } @Test - public void throwsWhenInvalidResponse() { - thrown.expect(RuntimeException.class); - thrown.expectMessage("JsonProcessingException while parsing valueSet response"); - - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity( - StringUtils.chop(getResponseToStub("noExpansion.ValueSet.json")), - HttpStatus.OK)); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + public void throwsWhenInvalidResponse() throws IOException { + prepareServer("noExpansion.ValueSet.json", true); + + try { + RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); + } catch (RuntimeException e) { + Assert.assertEquals("JsonProcessingException while parsing valueSet response", e.getMessage()); + return; + } + Assert.fail("Should have thrown a JsonProcessingException exception"); } - @Test - public void throwsWhenRestClientFailed() { - thrown.expect(RestClientException.class); - thrown.expectMessage("RestClientException while fetching valueSet response"); - - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenThrow(new RestClientException( - "RestClientException while fetching valueSet response")); - - RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); - } @Test - public void filterCodesTest() { - Mockito - .when(restTemplate.exchange(ArgumentMatchers.anyString(), - ArgumentMatchers.eq(HttpMethod.GET), - ArgumentMatchers.>any(), - ArgumentMatchers.>any())) - .thenReturn(new ResponseEntity(getResponseToStub("codes.json"), HttpStatus.OK)); + public void filterCodesTest() throws IOException { + prepareServer("codes.json", false); Code code = RandomCodeGenerator.getCode(VALUE_SET_URI + "&filter=tracheobronchial", SEED, this.code); @@ -181,9 +136,9 @@ public void invalidValueSetUrlTest() { } @After - public void cleanup() { + public void cleanup() throws IOException { RandomCodeGenerator.codeListCache.clear(); - RandomCodeGenerator.restTemplate = null; + server.close(); } private String getResponseToStub(String body) { @@ -196,4 +151,17 @@ private String getResponseToStub(String body) { return null; } + private void prepareServer(String fileToLoad, boolean mangle) throws IOException { + String response = getResponseToStub(fileToLoad); + if (mangle) { + response = StringUtils.chop(response); + } + server.enqueue(new MockResponse().setBody(response)); + server.start(); + + HttpUrl baseUrl = server.url("/ValueSet/$expand"); + + RandomCodeGenerator.expandBaseUrl = baseUrl + "?url="; + } + } From 7c81fb5a2b880b6b897658c56f45afcfc5e36b0e Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Mon, 12 Sep 2022 13:48:57 -0400 Subject: [PATCH 2/7] WIP: All tests now passing Fixed some of the cases where invalid characters for URLs are being used in value set expansion URLs --- .../mitre/synthea/export/CodeResolveAndExportTest.java | 2 +- .../mitre/synthea/helpers/RandomCodeGeneratorTest.java | 8 +++++--- .../resources/generic/imaging_study_with_valueset.json | 2 +- ...ueset_expand-9c08a2a2-d391-4571-a1b2-2fe88c6079d4.json | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java b/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java index 6abaa061a6..b1d1b8d826 100644 --- a/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java +++ b/src/test/java/org/mitre/synthea/export/CodeResolveAndExportTest.java @@ -157,7 +157,7 @@ public void resolveAndExportEncounterCodes() String reasonCode = "417981005"; String reasonDisplay = "Exposure to blood and/or body fluid"; encounter.reason = new Code(SNOMED_URI, reasonCode, reasonDisplay); - encounter.reason.valueSet = SNOMED_URI + "?fhir_vs=ecl/<418307001"; + encounter.reason.valueSet = SNOMED_URI + "?fhir_vs=ecl%2F%3C418307001"; encounter.codes.add(encounter.reason); String observationDisplay = OBSERVATION_DISPLAY; diff --git a/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java b/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java index da2158a76b..ce6ecae964 100644 --- a/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java +++ b/src/test/java/org/mitre/synthea/helpers/RandomCodeGeneratorTest.java @@ -45,7 +45,7 @@ public void getCode() throws IOException { Code code = RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); - // Assert.assertEquals(SNOMED_URI, code.system); + Assert.assertEquals(SNOMED_URI, code.system); Assert.assertEquals("312858004", code.code); Assert.assertEquals("Neonatal tracheobronchial haemorrhage", code.display); } @@ -96,7 +96,8 @@ public void throwsWhenMissingCodeElements() throws IOException { try { RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); } catch (RuntimeException e) { - Assert.assertEquals("ValueSet contains element does not contain system, code and display", e.getMessage()); + Assert.assertEquals("ValueSet contains element does not contain system, code and display", + e.getMessage()); return; } Assert.fail("Should have thrown a missing elements exception"); @@ -109,7 +110,8 @@ public void throwsWhenInvalidResponse() throws IOException { try { RandomCodeGenerator.getCode(VALUE_SET_URI, SEED, this.code); } catch (RuntimeException e) { - Assert.assertEquals("JsonProcessingException while parsing valueSet response", e.getMessage()); + Assert.assertEquals("JsonProcessingException while parsing valueSet response", + e.getMessage()); return; } Assert.fail("Should have thrown a JsonProcessingException exception"); diff --git a/src/test/resources/generic/imaging_study_with_valueset.json b/src/test/resources/generic/imaging_study_with_valueset.json index e8dc5cdac6..1886fb233e 100644 --- a/src/test/resources/generic/imaging_study_with_valueset.json +++ b/src/test/resources/generic/imaging_study_with_valueset.json @@ -44,7 +44,7 @@ "system": "SNOMED-CT", "code": "719489001", "display": "Structure of right patella", - "value_set": "http://snomed.info/sct?fhir_vs=ecl/<<6757004" + "value_set": "http://snomed.info/sct?fhir_vs=ecl%2F%3C%3C6757004" }, "modality": { "system": "DICOM-DCM", diff --git a/src/test/resources/wiremock/ValueSetCodeResolverTest/mappings/fhir_valueset_expand-9c08a2a2-d391-4571-a1b2-2fe88c6079d4.json b/src/test/resources/wiremock/ValueSetCodeResolverTest/mappings/fhir_valueset_expand-9c08a2a2-d391-4571-a1b2-2fe88c6079d4.json index a3852cb30a..608edf5cc7 100644 --- a/src/test/resources/wiremock/ValueSetCodeResolverTest/mappings/fhir_valueset_expand-9c08a2a2-d391-4571-a1b2-2fe88c6079d4.json +++ b/src/test/resources/wiremock/ValueSetCodeResolverTest/mappings/fhir_valueset_expand-9c08a2a2-d391-4571-a1b2-2fe88c6079d4.json @@ -5,7 +5,7 @@ "url" : "/fhir/ValueSet/$expand", "method" : "POST", "bodyPatterns" : [ { - "equalToJson" : "{\"resourceType\":\"Parameters\",\"parameter\":[{\"name\":\"url\",\"valueUri\":\"http://snomed.info/sct?fhir_vs=ecl/<<6757004\"},{\"name\":\"count\",\"valueInteger\":1000},{\"name\":\"offset\",\"valueInteger\":0}]}", + "equalToJson" : "{\"resourceType\":\"Parameters\",\"parameter\":[{\"name\":\"url\",\"valueUri\":\"http://snomed.info/sct?fhir_vs=ecl%2F%3C%3C6757004\"},{\"name\":\"count\",\"valueInteger\":1000},{\"name\":\"offset\",\"valueInteger\":0}]}", "ignoreArrayOrder" : true, "ignoreExtraElements" : true } ] From 24924e80db671b96d06e2b172f0980557e41570c Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Thu, 15 Sep 2022 14:54:45 -0400 Subject: [PATCH 3/7] Updating workflows for Java 11 --- .github/workflows/ci-build-test.yml | 12 ++++++------ .github/workflows/deploy.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index 4a5f120a27..bb919b1527 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -45,7 +45,7 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - check_java8: + check_java11: runs-on: ubuntu-latest name: Java 8 @@ -54,7 +54,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: 8 + java-version: 11 - name: Gradle cache uses: actions/cache@v2 @@ -62,9 +62,9 @@ jobs: path: | ~/.gradle/caches ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-jdk8-${{ hashFiles('**/*.gradle*') }} + key: ${{ runner.os }}-gradle-jdk11-${{ hashFiles('**/*.gradle*') }} restore-keys: | - ${{ runner.os }}-gradle-jdk8 + ${{ runner.os }}-gradle-jdk11 - name: Compile with Gradle run: ./gradlew assemble @@ -80,8 +80,8 @@ jobs: if: env.SLACK_WEBHOOK_URL with: status: custom - job_name: Java 8 - author_name: Java 8 Build/Test + job_name: Java 11 + author_name: Java 11 Build/Test fields: workflow,commit,repo,author,took # see https://action-slack.netlify.app/usecase/02-custom for custom payload info custom_payload: | diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f38bfe7e10..301fa9102e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: 8 + java-version: 11 - name: Gradle cache uses: actions/cache@v2 @@ -20,9 +20,9 @@ jobs: path: | ~/.gradle/caches ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-jdk8-${{ hashFiles('**/*.gradle*') }} + key: ${{ runner.os }}-gradle-jdk11-${{ hashFiles('**/*.gradle*') }} restore-keys: | - ${{ runner.os }}-gradle-jdk8 + ${{ runner.os }}-gradle-jdk11 - name: Create Artifacts run: | From b09baa6bcb9978e43925fe8fe8321c511f741b3d Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Thu, 15 Sep 2022 15:41:16 -0400 Subject: [PATCH 4/7] More bumps for Java 11 --- .github/workflows/ci-build-test.yml | 2 +- build.gradle | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-build-test.yml b/.github/workflows/ci-build-test.yml index bb919b1527..c0d2241d19 100644 --- a/.github/workflows/ci-build-test.yml +++ b/.github/workflows/ci-build-test.yml @@ -47,7 +47,7 @@ jobs: check_java11: runs-on: ubuntu-latest - name: Java 8 + name: Java 11 steps: - uses: actions/checkout@v2 diff --git a/build.gradle b/build.gradle index bec4d8e14c..f1ebc8df46 100644 --- a/build.gradle +++ b/build.gradle @@ -6,8 +6,7 @@ // Apply the java plugin to add support for Java apply plugin: 'java' -sourceCompatibility = '1.8' -targetCompatibility = '1.8' +sourceCompatibility = '11' apply plugin: 'application' apply plugin: 'eclipse' From e5d2564a4ac32d4bab928b919e5854bfee840228 Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Mon, 19 Sep 2022 16:25:51 -0400 Subject: [PATCH 5/7] Bumping the version of snakeyaml --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f1ebc8df46..51ae5d60b7 100644 --- a/build.gradle +++ b/build.gradle @@ -56,7 +56,7 @@ dependencies { // CSV Stuff implementation 'org.apache.commons:commons-csv:1.9.0' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.4' - implementation 'org.yaml:snakeyaml:1.30' + implementation 'org.yaml:snakeyaml:1.32' implementation 'org.apache.commons:commons-math3:3.6.1' implementation 'org.apache.commons:commons-text:1.9' implementation 'commons-validator:commons-validator:1.7' From dd459fe2eeee2981adda69d3e56db409eba2cd29 Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Tue, 20 Sep 2022 16:59:05 -0400 Subject: [PATCH 6/7] Cleanup of RandomCodeGenerator URLs and null safety --- .../org/mitre/synthea/helpers/RandomCodeGenerator.java | 10 ++++++++-- .../resources/generic/imaging_study_with_valueset.json | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java b/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java index f5ce627806..a585171d06 100644 --- a/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java +++ b/src/main/java/org/mitre/synthea/helpers/RandomCodeGenerator.java @@ -17,6 +17,7 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import okhttp3.ResponseBody; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.UrlValidator; import org.mitre.synthea.world.concepts.HealthRecord.Code; @@ -76,9 +77,14 @@ private static synchronized void expandValueSet(String valueSetUri) { try { Response response = client.newCall(request).execute(); ObjectMapper objectMapper = new ObjectMapper(); - valueSet = objectMapper.readValue(response.body().byteStream(), - new TypeReference>() { + ResponseBody body = response.body(); + if (body != null) { + valueSet = objectMapper.readValue(body.byteStream(), + new TypeReference>() { }); + } else { + throw new RuntimeException("Value Set Expansion contained no body"); + } } catch (JsonProcessingException e) { throw new RuntimeException("JsonProcessingException while parsing valueSet response"); } catch (IOException e) { diff --git a/src/test/resources/generic/imaging_study_with_valueset.json b/src/test/resources/generic/imaging_study_with_valueset.json index 1886fb233e..db5bfa33ba 100644 --- a/src/test/resources/generic/imaging_study_with_valueset.json +++ b/src/test/resources/generic/imaging_study_with_valueset.json @@ -36,7 +36,7 @@ "system": "SNOMED-CT", "code": "2501000087107", "display": "MRI of right knee with contrast", - "value_set": "http://snomed.info/sct?fhir_vs=ecl/<<2491000087104" + "value_set": "http://snomed.info/sct?fhir_vs=ecl%2F%3C%3C2491000087104" }, "series": [ { From ba26b60ed2c192f1bb2987b59618cc59107e918d Mon Sep 17 00:00:00 2001 From: Andy Gregorowicz Date: Wed, 28 Sep 2022 15:21:14 -0400 Subject: [PATCH 7/7] Updating CQL infrastructure used by physiology simulation This bumps the various dependencies and handles an API change for how CQL is read in from ELM XML files (I think). --- build.gradle | 11 +++++++---- .../mitre/synthea/helpers/ExpressionProcessor.java | 13 +++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 51ae5d60b7..68da3441f8 100644 --- a/build.gradle +++ b/build.gradle @@ -61,10 +61,13 @@ dependencies { implementation 'org.apache.commons:commons-text:1.9' implementation 'commons-validator:commons-validator:1.7' - implementation 'org.opencds.cqf:cql-engine:1.3.12.1' - implementation 'info.cqframework:cql:1.3.17' - implementation 'info.cqframework:model:1.3.17' - implementation 'info.cqframework:cql-to-elm:1.3.17' + implementation 'org.opencds.cqf.cql:engine.jackson:2.0.0' + implementation 'org.opencds.cqf.cql:engine:2.0.0' + implementation 'info.cqframework:cql:2.1.0' + implementation 'info.cqframework:model:2.1.0' + implementation 'info.cqframework:cql-to-elm:2.1.0' + implementation 'info.cqframework:model-jackson:2.1.0' + implementation 'info.cqframework:elm-jackson:2.1.0' implementation 'com.squareup.okhttp3:okhttp:4.10.0' diff --git a/src/main/java/org/mitre/synthea/helpers/ExpressionProcessor.java b/src/main/java/org/mitre/synthea/helpers/ExpressionProcessor.java index 99ee0bb613..32fc82747b 100644 --- a/src/main/java/org/mitre/synthea/helpers/ExpressionProcessor.java +++ b/src/main/java/org/mitre/synthea/helpers/ExpressionProcessor.java @@ -22,7 +22,6 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.xml.bind.JAXBException; import org.cqframework.cql.cql2elm.CqlTranslator; import org.cqframework.cql.cql2elm.LibraryManager; @@ -31,8 +30,9 @@ import org.cqframework.cql.elm.execution.Library; import org.mitre.synthea.world.agents.Person; import org.mitre.synthea.world.concepts.VitalSign; -import org.opencds.cqf.cql.execution.Context; -import org.opencds.cqf.cql.execution.CqlLibraryReader; +import org.opencds.cqf.cql.engine.execution.Context; +import org.opencds.cqf.cql.engine.serializing.CqlLibraryReader; +import org.opencds.cqf.cql.engine.serializing.jackson.XmlCqlLibraryReader; import org.simulator.math.odes.MultiTable; import org.simulator.math.odes.MultiTable.Block.Column; @@ -106,9 +106,10 @@ public ExpressionProcessor(String expression, Map paramTypeMap) { this.elm = cqlToElm(wrappedExpression); synchronized (ExpressionProcessor.class) { try { - this.library = CqlLibraryReader.read(new ByteArrayInputStream( + CqlLibraryReader reader = new XmlCqlLibraryReader(); + this.library = reader.read(new ByteArrayInputStream( elm.getBytes(StandardCharsets.UTF_8))); - } catch (IOException | JAXBException ex) { + } catch (IOException ex) { throw new RuntimeException(ex); } } @@ -378,7 +379,7 @@ private String convertParameterizedExpressionToCql(String expression) { .append(paramTypeMap.getOrDefault(paramEntry.getKey(), "Decimal")); } - wrappedExpression.append("\n\ncontext Patient\n\n"); + wrappedExpression.append("\n\ncontext Unfiltered\n\n"); String[] statements = expression.split("\n");