From 10a052823f35bb5611a348c97dbfca06b2f101ed Mon Sep 17 00:00:00 2001 From: Viacheslav Kolesnyk <94473337+viacheslavkol@users.noreply.github.com> Date: Tue, 21 May 2024 11:08:45 +0200 Subject: [PATCH 1/8] fix(marc-fields-order): Don't group fields with same tags together (#623) * fix(marc-fields-order): Don't group fields with same tags together Closes: MODSOURMAN-1191 --- .../java/org/folio/dao/util/MarcUtil.java | 87 ++++++++++--------- .../java/org/folio/dao/util/MarcUtilTest.java | 12 +-- .../parsedRecords/reorderedParsedRecord.json | 30 +++---- .../parsedRecords/reorderingResultRecord.json | 30 +++---- 4 files changed, 80 insertions(+), 79 deletions(-) diff --git a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java index d7ea0df51..bc5255d79 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java @@ -4,6 +4,10 @@ import static org.folio.services.util.AdditionalFieldsUtil.HR_ID_FROM_FIELD; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -11,16 +15,8 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; import java.util.LinkedList; -import java.util.Map; -import java.util.Queue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.marc4j.MarcException; @@ -144,60 +140,65 @@ private static String recordToTxtMarc(Record record) throws IOException { } /** - * Reorders MARC record fields + * Take field values from system modified record content while preserving incoming record content`s field order. + * Put system fields (001, 005) first, regardless of incoming record fields order. * - * @param sourceContent source parsed record - * @param targetContent target parsed record - * @return MARC txt + * @param sourceOrderContent content with incoming record fields order + * @param systemOrderContent system modified record content with reordered fields + * @return MARC record parsed content with desired fields order */ - public static String reorderMarcRecordFields(String sourceContent, String targetContent) { + public static String reorderMarcRecordFields(String sourceOrderContent, String systemOrderContent) { try { - var parsedContent = objectMapper.readTree(targetContent); + var parsedContent = objectMapper.readTree(systemOrderContent); var fieldsArrayNode = (ArrayNode) parsedContent.path(FIELDS); - var jsonNodesByTag = groupNodesByTag(fieldsArrayNode); - var sourceFields = getSourceFields(sourceContent); - var rearrangedArray = objectMapper.createArrayNode(); + var nodes = toNodeList(fieldsArrayNode); + var sourceOrderTags = getSourceFields(sourceOrderContent); + var reorderedFields = objectMapper.createArrayNode(); - var nodes001 = jsonNodesByTag.get(HR_ID_FROM_FIELD); - if (nodes001 != null && !nodes001.isEmpty()) { - rearrangedArray.addAll(nodes001); - jsonNodesByTag.remove(HR_ID_FROM_FIELD); + var node001 = removeAndGetNodeByTag(nodes, HR_ID_FROM_FIELD); + if (node001 != null && !node001.isEmpty()) { + reorderedFields.add(node001); } - var nodes005 = jsonNodesByTag.get(TAG_005); - if (nodes005 != null && !nodes005.isEmpty()) { - rearrangedArray.addAll(nodes005); - jsonNodesByTag.remove(TAG_005); + var node005 = removeAndGetNodeByTag(nodes, TAG_005); + if (node005 != null && !node005.isEmpty()) { + reorderedFields.add(node005); } - for (String tag : sourceFields) { - Queue nodes = jsonNodesByTag.get(tag); - if (nodes != null && !nodes.isEmpty()) { - rearrangedArray.addAll(nodes); - jsonNodesByTag.remove(tag); - } - + for (String tag : sourceOrderTags) { + var node = removeAndGetNodeByTag(nodes, tag); + if (node != null && !node.isEmpty()) { + reorderedFields.add(node); + } } - jsonNodesByTag.values().forEach(rearrangedArray::addAll); + reorderedFields.addAll(nodes); - ((ObjectNode) parsedContent).set(FIELDS, rearrangedArray); + ((ObjectNode) parsedContent).set(FIELDS, reorderedFields); return parsedContent.toString(); } catch (Exception e) { LOGGER.error("An error occurred while reordering Marc record fields: {}", e.getMessage(), e); - return targetContent; + return systemOrderContent; } } - private static Map> groupNodesByTag(ArrayNode fieldsArrayNode) { - var jsonNodesByTag = new LinkedHashMap>(); - for (JsonNode node : fieldsArrayNode) { - var tag = getTagFromNode(node); - jsonNodesByTag.putIfAbsent(tag, new LinkedList<>()); - jsonNodesByTag.get(tag).add(node); + private static List toNodeList(ArrayNode fieldsArrayNode) { + var nodes = new LinkedList(); + for (var node : fieldsArrayNode) { + nodes.add(node); + } + return nodes; + } + + private static JsonNode removeAndGetNodeByTag(List nodes, String tag) { + for (int i = 0; i < nodes.size(); i++) { + var nodeTag = getTagFromNode(nodes.get(i)); + if (nodeTag.equals(tag)) { + return nodes.remove(i); + } } - return jsonNodesByTag; + return null; } private static String getTagFromNode(JsonNode node) { diff --git a/mod-source-record-storage-server/src/test/java/org/folio/dao/util/MarcUtilTest.java b/mod-source-record-storage-server/src/test/java/org/folio/dao/util/MarcUtilTest.java index 81c368440..83b872d23 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/dao/util/MarcUtilTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/dao/util/MarcUtilTest.java @@ -71,14 +71,14 @@ public void shouldConvertMarcJsonToTxtMarc() throws IOException, MarcException { @Test public void shouldReorderMarcRecordFields() throws IOException, MarcException { - var reorderedRecordContent = readFileFromPath(PARSED_RECORD); - var sourceRecordContent = readFileFromPath(REORDERED_PARSED_RECORD); - var reorderingResultRecord = readFileFromPath(REORDERING_RESULT_RECORD); + var systemReorderedRecordContent = readFileFromPath(PARSED_RECORD); + var userOrderRecordContent = readFileFromPath(REORDERED_PARSED_RECORD); + var expectedOrderRecord = readFileFromPath(REORDERING_RESULT_RECORD); - var resultContent = MarcUtil.reorderMarcRecordFields(sourceRecordContent, reorderedRecordContent); + var actualOrderRecord = MarcUtil.reorderMarcRecordFields(userOrderRecordContent, systemReorderedRecordContent); - assertNotNull(resultContent); - assertEquals(formatContent(resultContent), formatContent(reorderingResultRecord)); + assertNotNull(actualOrderRecord); + assertEquals(formatContent(expectedOrderRecord), formatContent(actualOrderRecord)); } private static String readFileFromPath(String path) throws IOException { diff --git a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderedParsedRecord.json b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderedParsedRecord.json index 53a711df0..03c422f3f 100644 --- a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderedParsedRecord.json +++ b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderedParsedRecord.json @@ -34,21 +34,24 @@ "008":"120329s2011 sz a ob 001 0 eng d" }, { - "020":{ + "050":{ "subfields":[ { - "a":"2940447241 (electronic bk.)" + "a":"Z246" + }, + { + "b":".A43 2011" } ], "ind1":" ", - "ind2":" " + "ind2":"4" } }, { "020":{ "subfields":[ { - "a":"9782940447244 (electronic bk.)" + "a":"2940447241 (electronic bk.)" } ], "ind1":" ", @@ -56,31 +59,28 @@ } }, { - "050":{ + "082":{ "subfields":[ { - "a":"Z246" + "a":"686.22" }, { - "b":".A43 2011" + "2":"22" } ], - "ind1":" ", + "ind1":"0", "ind2":"4" } }, { - "082":{ + "020":{ "subfields":[ { - "a":"686.22" - }, - { - "2":"22" + "a":"9782940447244 (electronic bk.)" } ], - "ind1":"0", - "ind2":"4" + "ind1":" ", + "ind2":" " } }, { diff --git a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json index c710eca58..09fc0e0cd 100644 --- a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json +++ b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json @@ -34,21 +34,24 @@ "008": "120329s2011 sz a ob 001 0 eng d" }, { - "020": { + "050": { "subfields": [ { - "a": "2940447241 (electronic bk.)" + "a": "Z246" + }, + { + "b": ".A43 2011" } ], "ind1": " ", - "ind2": " " + "ind2": "4" } }, { "020": { "subfields": [ { - "a": "9782940447244 (electronic bk.)" + "a": "2940447241 (electronic bk.)" } ], "ind1": " ", @@ -56,31 +59,28 @@ } }, { - "050": { + "082": { "subfields": [ { - "a": "Z246" + "a": "686.22" }, { - "b": ".A43 2011" + "2": "22" } ], - "ind1": " ", + "ind1": "0", "ind2": "4" } }, { - "082": { + "020": { "subfields": [ { - "a": "686.22" - }, - { - "2": "22" + "a": "9782940447244 (electronic bk.)" } ], - "ind1": "0", - "ind2": "4" + "ind1": " ", + "ind2": " " } }, { From 51181e09f2a3aca217132d85f75ea9226cfa6153 Mon Sep 17 00:00:00 2001 From: Maksat <144414992+Maksat-Galymzhan@users.noreply.github.com> Date: Mon, 27 May 2024 19:18:32 +0500 Subject: [PATCH 2/8] MODDATAIMP-1052: Error appears when edit via quickMARC MARC Instance shared from Member tenant (#625) * MODDATAIMP-1052: removed IsNotEmpty check for externalHolder externalHrid --- .../services/util/AdditionalFieldsUtil.java | 2 +- .../services/AdditionalFieldsUtilTest.java | 99 +++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java index b158988c7..b97ec623b 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java @@ -674,7 +674,7 @@ public static boolean isFieldsFillingNeeded(Record record, JsonObject externalEn } private static boolean isValidIdAndHrid(String id, String hrid, String externalId, String externalHrid) { - return (isNotEmpty(externalId) && isNotEmpty(externalHrid)) && (id.equals(externalId) && !hrid.equals(externalHrid)); + return (isNotEmpty(externalId)) && (id.equals(externalId) && !hrid.equals(externalHrid)); } private static boolean isValidId(String id, String externalId) { diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java index fea73c9ba..837ce4c2f 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java @@ -2,6 +2,8 @@ import static org.folio.services.util.AdditionalFieldsUtil.*; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; import java.io.IOException; import java.util.List; @@ -480,4 +482,101 @@ public void caching() throws IOException { Assert.assertEquals(2, cacheStats.missCount()); Assert.assertEquals(2, cacheStats.loadCount()); } + + @Test + public void isFieldsFillingNeededTrue() { + String instanceId = UUID.randomUUID().toString(); + String instanceHrId = UUID.randomUUID().toString(); + Record srcRecord = new Record().withExternalIdsHolder(new ExternalIdsHolder() + .withInstanceId(instanceId) + .withInstanceHrid(UUID.randomUUID().toString())) + .withRecordType(Record.RecordType.MARC_BIB); + + JsonObject instanceJson = new JsonObject(); + instanceJson.put("id", instanceId); + instanceJson.put("hrid", instanceHrId); + + Assert.assertTrue(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson)); + + srcRecord.getExternalIdsHolder().setInstanceHrid(null); + Assert.assertTrue(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson)); + } + + @Test + public void isFieldsFillingNeededFalse() { + String instanceId = UUID.randomUUID().toString(); + String instanceHrId = UUID.randomUUID().toString(); + Record srcRecord = new Record().withExternalIdsHolder(new ExternalIdsHolder() + .withInstanceId(instanceId) + .withInstanceHrid(instanceHrId)) + .withRecordType(Record.RecordType.MARC_BIB); + + JsonObject instanceJson = new JsonObject(); + instanceJson.put("id", instanceId); + instanceJson.put("hrid", instanceHrId); + + assertFalse(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson)); + + srcRecord.getExternalIdsHolder().withInstanceId(instanceId); + instanceJson.put("id", UUID.randomUUID().toString()); + assertFalse(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson)); + + srcRecord.getExternalIdsHolder().withInstanceId(null).withInstanceHrid(null); + assertFalse(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson)); + } + + @Test(expected = Exception.class) + public void isFieldsFillingNeededForExternalHolderInstanceShouldThrowException() { + String instanceId = UUID.randomUUID().toString(); + String instanceHrId = UUID.randomUUID().toString(); + Record srcRecord = new Record().withExternalIdsHolder(new ExternalIdsHolder() + .withInstanceId(instanceId) + .withInstanceHrid(instanceHrId)) + .withRecordType(Record.RecordType.MARC_BIB); + + JsonObject instanceJson = new JsonObject(); + instanceJson.put("hrid", instanceHrId); + AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, instanceJson); + } + + @Test + public void isFieldsFillingNeededForHoldingsExternalHolder() { + String holdingId = UUID.randomUUID().toString(); + String holdingHrid = UUID.randomUUID().toString(); + Record srcRecord = new Record().withExternalIdsHolder(new ExternalIdsHolder().withHoldingsId(holdingId)) + .withRecordType(Record.RecordType.MARC_HOLDING); + + JsonObject jsonObject = new JsonObject(); + jsonObject.put("id", holdingId); + jsonObject.put("hrid", holdingHrid); + + Assert.assertTrue(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, jsonObject)); + + srcRecord.getExternalIdsHolder().setHoldingsHrid(holdingHrid); + + Assert.assertFalse(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, jsonObject)); + } + + @Test + public void isFieldsFillingNeededForUnknownRecordType() { + String entityId = UUID.randomUUID().toString(); + Record srcRecord = new Record().withRecordType(any()); + + JsonObject jsonObject = new JsonObject(); + jsonObject.put("id", entityId); + + Assert.assertFalse(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, jsonObject)); + } + + @Test + public void isFieldsFillingNeededTrueForMarcAuthority() { + String authorityId = UUID.randomUUID().toString(); + Record srcRecord = new Record().withExternalIdsHolder(new ExternalIdsHolder().withAuthorityId(authorityId)) + .withRecordType(Record.RecordType.MARC_AUTHORITY); + + JsonObject jsonObject = new JsonObject(); + jsonObject.put("id", authorityId); + + Assert.assertTrue(AdditionalFieldsUtil.isFieldsFillingNeeded(srcRecord, jsonObject)); + } } From 31a805e3e782a6a604acf8f952a8c004c7b2d265 Mon Sep 17 00:00:00 2001 From: Volodymyr Rohach Date: Tue, 28 May 2024 13:10:47 +0300 Subject: [PATCH 3/8] MODSOURCE-773: MARC Search omits suppressed from discovery records in default search. (#624) * MODSOURCE-773: MARC Search omits suppressed from discovery records in default search. * MODSOURCE-773: Useless condition removed. * MODSOURCE-773: NEWS updated. * MODSOURCE-773: TEMPORARY SCHEMA CHANGE. REMOVE AFTER TESTING ON RANCHER! * MODSOURCE-773: REVERTING CHANGE. * MODSOURCE-773: Updating raml-storage. * MODSOURCE-773: Updating raml-storage. * MODSOURCE-773: Compilation errors due to schema profile wrappers changes - fixed. * MODSOURCE-773: Compilation errors due to schema profile wrappers changes in tests - fixed . * MODSOURCE-773: Schema updated. --- NEWS.md | 1 + mod-source-record-storage-server/pom.xml | 2 +- .../services/RecordSearchParameters.java | 6 +-- .../AbstractPostProcessingEventHandler.java | 2 +- .../actions/AbstractDeleteEventHandler.java | 2 +- .../AbstractUpdateModifyEventHandler.java | 2 +- .../match/AbstractMarcMatchEventHandler.java | 3 +- .../rest/impl/SourceStorageStreamApiTest.java | 39 ++++++++++++++++++- .../MarcAuthorityDeleteEventHandlerTest.java | 2 +- .../MarcAuthorityMatchEventHandlerTest.java | 4 +- ...AuthorityUpdateModifyEventHandlerTest.java | 7 ++-- .../MarcBibUpdateModifyEventHandlerTest.java | 7 ++-- .../MarcHoldingsMatchEventHandlerTest.java | 4 +- ...cHoldingsUpdateModifyEventHandlerTest.java | 7 ++-- .../caches/JobProfileSnapshotCacheTest.java | 5 ++- ...thorityPostProcessingEventHandlerTest.java | 4 +- ...oldingsPostProcessingEventHandlerTest.java | 4 +- ...nstancePostProcessingEventHandlerTest.java | 7 +--- .../DataImportConsumersVerticleTest.java | 6 +-- ramls/raml-storage | 2 +- 20 files changed, 74 insertions(+), 42 deletions(-) diff --git a/NEWS.md b/NEWS.md index a0d53cba8..cd1dbeba7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ * [MODSOURCE-767](https://folio-org.atlassian.net/browse/MODSOURCE-767) Single record overlay creates duplicate OCLC#/035 * [MODSOURCE-756](https://issues.folio.org/browse/MODSOURCE-756) After setting an instance as marked for deletion it is no longer editable in quickmarc * [MODSOURCE-753](https://folio-org.atlassian.net/browse/MODSOURCE-753) Change SQL query parameters for MARC Search +* [MODSOURCE-773](https://folio-org.atlassian.net/browse/MODSOURCE-773) MARC Search omits suppressed from discovery records in default search ## 2024-03-20 5.8.0 * [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings diff --git a/mod-source-record-storage-server/pom.xml b/mod-source-record-storage-server/pom.xml index 10153217c..b9329db0b 100644 --- a/mod-source-record-storage-server/pom.xml +++ b/mod-source-record-storage-server/pom.xml @@ -182,7 +182,7 @@ org.folio data-import-processing-core - 4.2.0 + 4.3.0-SNAPSHOT io.vertx diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java index c112d1570..fdcc45af2 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java @@ -13,7 +13,7 @@ public class RecordSearchParameters { private String fieldsSearchExpression; private Record.RecordType recordType; private boolean deleted; - private boolean suppressedFromDiscovery; + private Boolean suppressedFromDiscovery; private Integer limit; private Integer offset; @@ -65,11 +65,11 @@ public void setDeleted(boolean deleted) { this.deleted = deleted; } - public boolean isSuppressedFromDiscovery() { + public Boolean isSuppressedFromDiscovery() { return suppressedFromDiscovery; } - public void setSuppressedFromDiscovery(boolean suppressedFromDiscovery) { + public void setSuppressedFromDiscovery(Boolean suppressedFromDiscovery) { this.suppressedFromDiscovery = suppressedFromDiscovery; } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/AbstractPostProcessingEventHandler.java b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/AbstractPostProcessingEventHandler.java index 61e54f77a..6c550f296 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/AbstractPostProcessingEventHandler.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/AbstractPostProcessingEventHandler.java @@ -44,7 +44,7 @@ import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalId; import static org.folio.dao.util.RecordDaoUtil.filterRecordByNotSnapshotId; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_INVENTORY_INSTANCE_UPDATED_READY_FOR_POST_PROCESSING; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.folio.rest.util.OkapiConnectionParams.OKAPI_TENANT_HEADER; import static org.folio.rest.util.OkapiConnectionParams.OKAPI_TOKEN_HEADER; import static org.folio.rest.util.OkapiConnectionParams.OKAPI_URL_HEADER; diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractDeleteEventHandler.java b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractDeleteEventHandler.java index 0aa802749..7cd92d8bc 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractDeleteEventHandler.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractDeleteEventHandler.java @@ -10,6 +10,7 @@ import org.folio.processing.events.services.handler.EventHandler; import org.folio.processing.exceptions.EventProcessingException; import org.folio.rest.jaxrs.model.ExternalIdsHolder; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper; import org.folio.rest.jaxrs.model.Record; import org.folio.rest.jooq.enums.RecordState; @@ -21,7 +22,6 @@ import static java.util.Objects.isNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.folio.ActionProfile.Action.DELETE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; /** * The abstraction handles the DELETE action diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractUpdateModifyEventHandler.java b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractUpdateModifyEventHandler.java index aa0b30cb6..fa37481c7 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractUpdateModifyEventHandler.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractUpdateModifyEventHandler.java @@ -38,7 +38,7 @@ import static java.util.Objects.nonNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.folio.ActionProfile.Action.UPDATE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import static org.folio.services.handlers.match.AbstractMarcMatchEventHandler.CENTRAL_TENANT_ID; import static org.folio.services.util.AdditionalFieldsUtil.HR_ID_FROM_FIELD; import static org.folio.services.util.AdditionalFieldsUtil.addControlledFieldToMarcRecord; diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/match/AbstractMarcMatchEventHandler.java b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/match/AbstractMarcMatchEventHandler.java index d7b222c46..ac9c86db6 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/match/AbstractMarcMatchEventHandler.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/handlers/match/AbstractMarcMatchEventHandler.java @@ -43,8 +43,7 @@ import static org.folio.dao.util.RecordDaoUtil.filterRecordByRecordId; import static org.folio.dao.util.RecordDaoUtil.filterRecordByState; import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MATCH_PROFILE; - +import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE; /** * Abstract handler for MARC-MARC matching/not-matching of Marc record by specific fields */ diff --git a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java index fc84e4a7a..5510b7da7 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java @@ -1124,7 +1124,7 @@ public void shouldReturnEmptyResponseOnSearchMarcRecordIdsWhenRecordWasSuppresse } @Test - public void shouldReturnIdOnResponseOnSearchMarcRecordIdsWhenRecordWasSuppressed(TestContext testContext) { + public void shouldReturnRecordOnSearchMarcRecordWhenRecordWasSuppressedAndSetSuppressFromDiscoveryNotSetInRequest(TestContext testContext) { // given Async async = testContext.async(); Record suppressedRecord = new Record() @@ -1143,7 +1143,42 @@ public void shouldReturnIdOnResponseOnSearchMarcRecordIdsWhenRecordWasSuppressed MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); searchRequest.setFieldsSearchExpression("001.value = '393893' and 005.value ^= '2014110' and 035.ind1 = '#'"); - searchRequest.setSuppressFromDiscovery(true); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + + @Test + public void shouldReturnRecordOnSearchMarcRecordWhenRecordWasNotSuppressedAndSetSuppressFromDiscoveryNotSetInRequest(TestContext testContext) { + // given + Async async = testContext.async(); + Record suppressedRecord = new Record() + .withId(marc_bib_record_2.getId()) + .withSnapshotId(snapshot_2.getJobExecutionId()) + .withRecordType(Record.RecordType.MARC_BIB) + .withRawRecord(marc_bib_record_2.getRawRecord()) + .withParsedRecord(marc_bib_record_2.getParsedRecord()) + .withMatchedId(marc_bib_record_2.getMatchedId()) + .withState(Record.State.ACTUAL) + .withAdditionalInfo(new AdditionalInfo().withSuppressDiscovery(false)) + .withExternalIdsHolder(marc_bib_record_2.getExternalIdsHolder()); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, suppressedRecord); + + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); + searchRequest.setFieldsSearchExpression("001.value = '393893' and 005.value ^= '2014110' and 035.ind1 = '#'"); // when ExtractableResponse response = RestAssured.given() .spec(spec) diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityDeleteEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityDeleteEventHandlerTest.java index 656cccc04..03fcd78fe 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityDeleteEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityDeleteEventHandlerTest.java @@ -31,7 +31,7 @@ import static org.folio.ActionProfile.Action.DELETE; import static org.folio.ActionProfile.Action.UPDATE; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_DELETED; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_AUTHORITY; @RunWith(VertxUnitRunner.class) diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityMatchEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityMatchEventHandlerTest.java index 86b9c126b..811d3d210 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityMatchEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityMatchEventHandlerTest.java @@ -47,8 +47,8 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_MATCHED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_NOT_MATCHED; import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MATCH_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_AUTHORITY; @RunWith(VertxUnitRunner.class) diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityUpdateModifyEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityUpdateModifyEventHandlerTest.java index dfd6e55fb..2452c0ec6 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityUpdateModifyEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcAuthorityUpdateModifyEventHandlerTest.java @@ -2,14 +2,13 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static org.folio.ActionProfile.Action.MODIFY; import static org.folio.DataImportEventTypes.DI_SRS_MARC_BIB_RECORD_CREATED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_UPDATED; import static org.folio.rest.jaxrs.model.EntityType.MARC_AUTHORITY; import static org.folio.rest.jaxrs.model.MappingDetail.MarcMappingOption.UPDATE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.JOB_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_BIB; import static org.folio.services.MarcBibUpdateModifyEventHandlerTest.getParsedContentWithoutLeaderAndDate; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcBibUpdateModifyEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcBibUpdateModifyEventHandlerTest.java index 0ee9d76d3..a0c57ce07 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcBibUpdateModifyEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcBibUpdateModifyEventHandlerTest.java @@ -14,9 +14,10 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_BIB_RECORD_UPDATED; import static org.folio.rest.jaxrs.model.EntityType.MARC_BIBLIOGRAPHIC; import static org.folio.rest.jaxrs.model.MappingDetail.MarcMappingOption.UPDATE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.JOB_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; + +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_BIB; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsMatchEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsMatchEventHandlerTest.java index 388f82f94..596ec23b1 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsMatchEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsMatchEventHandlerTest.java @@ -7,8 +7,8 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_HOLDINGS_RECORD_NOT_MATCHED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_HOLDING_RECORD_CREATED; import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MATCH_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_HOLDING; import java.io.IOException; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsUpdateModifyEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsUpdateModifyEventHandlerTest.java index a76da101d..9e0bee052 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsUpdateModifyEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/MarcHoldingsUpdateModifyEventHandlerTest.java @@ -2,15 +2,14 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static org.folio.ActionProfile.Action.MODIFY; import static org.folio.DataImportEventTypes.DI_SRS_MARC_BIB_RECORD_CREATED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_HOLDINGS_RECORD_MODIFIED_READY_FOR_POST_PROCESSING; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_HOLDINGS_RECORD_UPDATED; import static org.folio.rest.jaxrs.model.EntityType.MARC_HOLDINGS; import static org.folio.rest.jaxrs.model.MappingDetail.MarcMappingOption.UPDATE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.JOB_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_BIB; import static org.folio.services.MarcBibUpdateModifyEventHandlerTest.getParsedContentWithoutLeaderAndDate; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/caches/JobProfileSnapshotCacheTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/caches/JobProfileSnapshotCacheTest.java index 52836053c..47ad85c06 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/caches/JobProfileSnapshotCacheTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/caches/JobProfileSnapshotCacheTest.java @@ -27,8 +27,9 @@ import java.util.UUID; import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.JOB_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; + @RunWith(VertxUnitRunner.class) public class JobProfileSnapshotCacheTest { diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/AuthorityPostProcessingEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/AuthorityPostProcessingEventHandlerTest.java index 88d23f72f..1f1b0510e 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/AuthorityPostProcessingEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/AuthorityPostProcessingEventHandlerTest.java @@ -6,8 +6,8 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_INVENTORY_AUTHORITY_UPDATED_READY_FOR_POST_PROCESSING; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_CREATED; import static org.folio.rest.jaxrs.model.EntityType.AUTHORITY; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_AUTHORITY; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/HoldingsPostProcessingEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/HoldingsPostProcessingEventHandlerTest.java index ec677b18d..bfa9a5be8 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/HoldingsPostProcessingEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/HoldingsPostProcessingEventHandlerTest.java @@ -5,8 +5,8 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_HOLDING_RECORD_CREATED; import static org.folio.rest.jaxrs.model.EntityType.HOLDINGS; import static org.folio.rest.jaxrs.model.EntityType.MARC_HOLDINGS; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_HOLDING; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/InstancePostProcessingEventHandlerTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/InstancePostProcessingEventHandlerTest.java index 86152115f..6625fe1a4 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/InstancePostProcessingEventHandlerTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/handlers/InstancePostProcessingEventHandlerTest.java @@ -42,9 +42,6 @@ import org.mockito.MockitoAnnotations; import java.io.IOException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -58,8 +55,8 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_ORDER_CREATED_READY_FOR_POST_PROCESSING; import static org.folio.rest.jaxrs.model.EntityType.INSTANCE; import static org.folio.rest.jaxrs.model.EntityType.MARC_BIBLIOGRAPHIC; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_BIB; import static org.folio.services.handlers.InstancePostProcessingEventHandler.POST_PROCESSING_RESULT_EVENT; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; diff --git a/mod-source-record-storage-server/src/test/java/org/folio/verticle/consumers/DataImportConsumersVerticleTest.java b/mod-source-record-storage-server/src/test/java/org/folio/verticle/consumers/DataImportConsumersVerticleTest.java index 7e58349be..53ffd9e32 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/verticle/consumers/DataImportConsumersVerticleTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/verticle/consumers/DataImportConsumersVerticleTest.java @@ -7,7 +7,7 @@ import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_BIB_RECORD_CREATED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_BIB_RECORD_MODIFIED_READY_FOR_POST_PROCESSING; import static org.folio.rest.jaxrs.model.EntityType.MARC_BIBLIOGRAPHIC; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.folio.services.MarcBibUpdateModifyEventHandlerTest.getParsedContentWithoutLeaderAndDate; import static org.junit.Assert.assertEquals; @@ -17,8 +17,8 @@ import static org.folio.kafka.KafkaTopicNameHelper.getDefaultNameSpace; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_MARC_FOR_DELETE_RECEIVED; import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_SRS_MARC_AUTHORITY_RECORD_DELETED; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE; -import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.JOB_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; +import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; import static org.folio.rest.jaxrs.model.Record.RecordType.MARC_BIB; import static org.junit.Assert.assertNotNull; diff --git a/ramls/raml-storage b/ramls/raml-storage index d9a129bbf..8f35e1fa5 160000 --- a/ramls/raml-storage +++ b/ramls/raml-storage @@ -1 +1 @@ -Subproject commit d9a129bbf6d5e39d3cc4dc2cb2a2a3824b6a0a0a +Subproject commit 8f35e1fa5902bab4792a2b4a5511e9eb2a860aef From 755b9cffc4dfb64d88b739c025d3d5304ac78ae5 Mon Sep 17 00:00:00 2001 From: Volodymyr Rohach Date: Wed, 5 Jun 2024 17:45:40 +0300 Subject: [PATCH 4/8] MODSOURCE-773: Remove default searching via "deleted"="false". (#626) * MODSOURCE-773: MARC Search omits suppressed from discovery records in default search. * MODSOURCE-773: Useless condition removed. * MODSOURCE-773: NEWS updated. * MODSOURCE-773: TEMPORARY SCHEMA CHANGE. REMOVE AFTER TESTING ON RANCHER! * MODSOURCE-773: REVERTING CHANGE. * MODSOURCE-773: Updating raml-storage. * MODSOURCE-773: Updating raml-storage. * MODSOURCE-773: Compilation errors due to schema profile wrappers changes - fixed. * MODSOURCE-773: Compilation errors due to schema profile wrappers changes in tests - fixed . * MODSOURCE-773: Schema updated. * MODSOURCE-773: Deleted-parameter searching mechanism changed. * MODSOURCE-773: Schema updated. * MODSOURCE-773: Condition improved. --- .../org/folio/dao/util/RecordDaoUtil.java | 8 +++- .../services/RecordSearchParameters.java | 6 +-- .../rest/impl/SourceStorageStreamApiTest.java | 39 +++++++++++++++++++ ramls/raml-storage | 2 +- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java index f00d3eebc..3202f44b8 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java @@ -592,6 +592,10 @@ public static Condition filterRecordByUpdatedDateRange(Date updatedAfter, Date u */ public static Condition filterRecordByDeleted(Boolean deleted) { Condition condition = filterRecordByState(RecordState.ACTUAL.name()); + if (deleted == null) { + condition = condition.or(filterRecordByState(RecordState.DELETED.name())) + .or(filterRecordByState(RecordState.ACTUAL.name())); + } if (Boolean.TRUE.equals(deleted)) { condition = condition.or(filterRecordByState(RecordState.DELETED.name())) .or(filterRecordByState(RecordState.ACTUAL.name()).and(filterRecordByLeaderRecordStatus(DELETED_LEADER_RECORD_STATUS))); @@ -618,7 +622,7 @@ public static Condition filterRecordByExternalIdNonNull() { /** * Convert {@link List} of {@link String} to {@link List} or {@link OrderField} - * + *

* Relies on strong convention between dto property name and database column name. * Property name being lower camel case and column name being lower snake case of the property name. * @@ -629,7 +633,7 @@ public static Condition filterRecordByExternalIdNonNull() { @SuppressWarnings("squid:S1452") public static List> toRecordOrderFields(List orderBy, Boolean forOffset) { if (forOffset && orderBy.isEmpty()) { - return Arrays.asList(new OrderField[] {RECORDS_LB.ID.asc()}); + return Arrays.asList(new OrderField[]{RECORDS_LB.ID.asc()}); } return orderBy.stream() .map(order -> order.split(COMMA)) diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java index fdcc45af2..8aeff1174 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java @@ -12,7 +12,7 @@ public class RecordSearchParameters { private String leaderSearchExpression; private String fieldsSearchExpression; private Record.RecordType recordType; - private boolean deleted; + private Boolean deleted; private Boolean suppressedFromDiscovery; private Integer limit; private Integer offset; @@ -57,11 +57,11 @@ public void setRecordType(Record.RecordType recordType) { this.recordType = recordType; } - public boolean isDeleted() { + public Boolean isDeleted() { return deleted; } - public void setDeleted(boolean deleted) { + public void setDeleted(Boolean deleted) { this.deleted = deleted; } diff --git a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java index 5510b7da7..5744be9fe 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java @@ -1195,6 +1195,45 @@ public void shouldReturnRecordOnSearchMarcRecordWhenRecordWasNotSuppressedAndSet async.complete(); } + + @Test + public void shouldReturnRecordOnSearchMarcRecordWhenRecordWasDeletedAndDeletedNotSetInRequest(TestContext testContext) { + // given + postSnapshots(testContext, snapshot_2); + Response createParsed = RestAssured.given() + .spec(spec) + .body(marc_bib_record_2) + .when() + .post(SOURCE_STORAGE_RECORDS_PATH); + assertThat(createParsed.statusCode(), is(HttpStatus.SC_CREATED)); + Record parsed = createParsed.body().as(Record.class); + Async async = testContext.async(); + RestAssured.given() + .spec(spec) + .when() + .delete(SOURCE_STORAGE_RECORDS_PATH + "/" + parsed.getId()) + .then() + .statusCode(HttpStatus.SC_NO_CONTENT); + async.complete(); + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'd'"); + // when + async = testContext.async(); + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + @Test public void shouldReturnEmptyResponseOnSearchMarcRecordIdsWhenLimitIs0(TestContext testContext) { // given diff --git a/ramls/raml-storage b/ramls/raml-storage index 8f35e1fa5..195de7f9f 160000 --- a/ramls/raml-storage +++ b/ramls/raml-storage @@ -1 +1 @@ -Subproject commit 8f35e1fa5902bab4792a2b4a5511e9eb2a860aef +Subproject commit 195de7f9fd0f5ef76aac6542e11b0391a67e8af0 From dd267e7cf8ee89f71260e8ee343b3a84afea7478 Mon Sep 17 00:00:00 2001 From: Viacheslav Kolesnyk <94473337+viacheslavkol@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:35:32 +0200 Subject: [PATCH 5/8] fix(marc-fields-order): Reorder only 00X fields while preserving system order for all other (#627) - System algorithms reorder only 00X while also adding f.e. 035 so no need to reorder system records fields apart from 00X, which will also allow to preserve system generated 035 field order Closes: MODSOURCE-780 --- .../java/org/folio/dao/util/MarcUtil.java | 32 ++++++++++--- .../services/util/AdditionalFieldsUtil.java | 1 + .../parsedRecords/parsedRecord.json | 47 ++++++++++++------- .../parsedRecords/reorderingResultRecord.json | 11 +++++ 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java index bc5255d79..c295502cc 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java @@ -3,6 +3,7 @@ import static java.lang.String.format; import static org.folio.services.util.AdditionalFieldsUtil.HR_ID_FROM_FIELD; import static org.folio.services.util.AdditionalFieldsUtil.TAG_005; +import static org.folio.services.util.AdditionalFieldsUtil.TAG_00X_PREFIX; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -153,24 +154,30 @@ public static String reorderMarcRecordFields(String sourceOrderContent, String s var fieldsArrayNode = (ArrayNode) parsedContent.path(FIELDS); var nodes = toNodeList(fieldsArrayNode); + var nodes00X = removeAndGetNodesByTagPrefix(nodes, TAG_00X_PREFIX); var sourceOrderTags = getSourceFields(sourceOrderContent); var reorderedFields = objectMapper.createArrayNode(); - var node001 = removeAndGetNodeByTag(nodes, HR_ID_FROM_FIELD); + var node001 = removeAndGetNodeByTag(nodes00X, HR_ID_FROM_FIELD); if (node001 != null && !node001.isEmpty()) { reorderedFields.add(node001); } - var node005 = removeAndGetNodeByTag(nodes, TAG_005); + var node005 = removeAndGetNodeByTag(nodes00X, TAG_005); if (node005 != null && !node005.isEmpty()) { reorderedFields.add(node005); } for (String tag : sourceOrderTags) { - var node = removeAndGetNodeByTag(nodes, tag); - if (node != null && !node.isEmpty()) { - reorderedFields.add(node); - } + var nodeTag = tag; + //loop will add system generated fields that are absent in initial record, preserving their order, f.e. 035 + do { + var node = tag.startsWith(TAG_00X_PREFIX) ? removeAndGetNodeByTag(nodes00X, tag) : nodes.remove(0); + if (node != null && !node.isEmpty()) { + nodeTag = getTagFromNode(node); + reorderedFields.add(node); + } + } while (!tag.equals(nodeTag) && !nodes.isEmpty()); } reorderedFields.addAll(nodes); @@ -201,6 +208,19 @@ private static JsonNode removeAndGetNodeByTag(List nodes, String tag) return null; } + private static List removeAndGetNodesByTagPrefix(List nodes, String prefix) { + var startsWithNodes = new LinkedList(); + for (int i = 0; i < nodes.size(); i++) { + var nodeTag = getTagFromNode(nodes.get(i)); + if (nodeTag.startsWith(prefix)) { + startsWithNodes.add(nodes.get(i)); + } + } + + nodes.removeAll(startsWithNodes); + return startsWithNodes; + } + private static String getTagFromNode(JsonNode node) { return node.fieldNames().next(); } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java index b97ec623b..0d967bb14 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java @@ -56,6 +56,7 @@ */ public final class AdditionalFieldsUtil { + public static final String TAG_00X_PREFIX = "00"; public static final String TAG_005 = "005"; public static final String TAG_999 = "999"; public static final String TAG_035 = "035"; diff --git a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/parsedRecord.json b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/parsedRecord.json index 752f36761..9484ff805 100644 --- a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/parsedRecord.json +++ b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/parsedRecord.json @@ -5,10 +5,10 @@ "001":"ybp7406411" }, { - "005":"20120404100627.6" + "003":"NhCcYBP" }, { - "003":"NhCcYBP" + "005":"20120404100627.6" }, { "006":"m||||||||d|||||||" @@ -20,25 +20,14 @@ "008":"120329s2011 sz a ob 001 0 eng d" }, { - "020":{ - "subfields":[ - { - "a":"2940447241 (electronic bk.)" - } - ], - "ind1":" ", - "ind2":" " - } - }, - { - "020":{ - "subfields":[ + "035": { + "subfields": [ { - "a":"9782940447244 (electronic bk.)" + "a": "(OCoLC)63611770" } ], - "ind1":" ", - "ind2":" " + "ind1": " ", + "ind2": " " } }, { @@ -69,6 +58,17 @@ "ind2":"4" } }, + { + "020":{ + "subfields":[ + { + "a":"2940447241 (electronic bk.)" + } + ], + "ind1":" ", + "ind2":" " + } + }, { "082":{ "subfields":[ @@ -83,6 +83,17 @@ "ind2":"4" } }, + { + "020":{ + "subfields":[ + { + "a":"9782940447244 (electronic bk.)" + } + ], + "ind1":" ", + "ind2":" " + } + }, { "100":{ "subfields":[ diff --git a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json index 09fc0e0cd..c74eb2a7f 100644 --- a/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json +++ b/mod-source-record-storage-server/src/test/resources/mock/sourceRecords/parsedRecords/reorderingResultRecord.json @@ -7,6 +7,17 @@ { "005": "20120404100627.6" }, + { + "035": { + "subfields": [ + { + "a": "(OCoLC)63611770" + } + ], + "ind1": " ", + "ind2": " " + } + }, { "040": { "subfields": [ From 8f6b160ea0c2f0e650cc2b93b63d8c953fb262ae Mon Sep 17 00:00:00 2001 From: Maksat <144414992+Maksat-Galymzhan@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:11:06 +0500 Subject: [PATCH 6/8] MODSOURCE-763 SRS MARC query search won't allow you to create two search conditions for a single field (SPIKE) (#628) * MODSOURCE-763: Implementation of a new data searching approach --------- Co-authored-by: Aliaksandr-Fedasiuk --- mod-source-record-storage-server/pom.xml | 5 + .../main/java/org/folio/dao/RecordDao.java | 3 +- .../java/org/folio/dao/RecordDaoImpl.java | 306 +++++++++++------- .../org/folio/services/RecordServiceImpl.java | 9 +- .../util/parser/SearchExpressionParser.java | 4 +- .../operand/DateRangeBinaryOperand.java | 18 +- .../operand/IndicatorBinaryOperand.java | 9 +- .../lexeme/operand/PositionBinaryOperand.java | 8 +- .../lexeme/operand/PresenceBinaryOperand.java | 13 +- .../lexeme/operand/SubFieldBinaryOperand.java | 6 +- .../lexeme/operand/ValueBinaryOperand.java | 9 +- .../java/org/folio/dao/RecordDaoImplTest.java | 41 +-- .../rest/impl/SourceStorageStreamApiTest.java | 266 ++++++++++++--- .../SearchExpressionParserUnitTest.java | 46 +-- 14 files changed, 499 insertions(+), 244 deletions(-) diff --git a/mod-source-record-storage-server/pom.xml b/mod-source-record-storage-server/pom.xml index b9329db0b..ef5ae0c78 100644 --- a/mod-source-record-storage-server/pom.xml +++ b/mod-source-record-storage-server/pom.xml @@ -193,6 +193,11 @@ folio-kafka-wrapper 3.1.1 + + com.github.jsqlparser + jsqlparser + 4.9 + net.mguenther.kafka kafka-junit diff --git a/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java b/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java index 49412d6ee..3aaefd990 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java @@ -6,6 +6,7 @@ import java.util.function.Function; import io.vertx.sqlclient.Row; +import net.sf.jsqlparser.JSQLParserException; import org.folio.dao.util.IdType; import org.folio.dao.util.RecordType; import org.folio.rest.jaxrs.model.MarcBibCollection; @@ -128,7 +129,7 @@ Future getMatchedRecordsIdentifiers(MatchField mat * @param tenantId tenant id * @return {@link Flowable} of {@link Record id} */ - Flowable streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters, String tenantId); + Flowable streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters, String tenantId) throws JSQLParserException; /** * Searches for {@link Record} by id diff --git a/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java b/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java index 71f37e9d0..b50397b67 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java @@ -1,39 +1,5 @@ package org.folio.dao; -import static java.lang.String.format; -import static java.util.Collections.emptyList; -import static org.folio.dao.util.AdvisoryLockUtil.acquireLock; -import static org.folio.dao.util.ErrorRecordDaoUtil.ERROR_RECORD_CONTENT; -import static org.folio.dao.util.ParsedRecordDaoUtil.PARSED_RECORD_CONTENT; -import static org.folio.dao.util.RawRecordDaoUtil.RAW_RECORD_CONTENT; -import static org.folio.dao.util.RecordDaoUtil.RECORD_NOT_FOUND_TEMPLATE; -import static org.folio.dao.util.RecordDaoUtil.ensureRecordForeignKeys; -import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalIdNonNull; -import static org.folio.dao.util.RecordDaoUtil.filterRecordByState; -import static org.folio.dao.util.RecordDaoUtil.filterRecordByType; -import static org.folio.dao.util.RecordDaoUtil.getExternalHrid; -import static org.folio.dao.util.RecordDaoUtil.getExternalId; -import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_FOUND_TEMPLATE; -import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_STARTED_MESSAGE_TEMPLATE; -import static org.folio.rest.jooq.Tables.ERROR_RECORDS_LB; -import static org.folio.rest.jooq.Tables.MARC_RECORDS_LB; -import static org.folio.rest.jooq.Tables.MARC_RECORDS_TRACKING; -import static org.folio.rest.jooq.Tables.RAW_RECORDS_LB; -import static org.folio.rest.jooq.Tables.RECORDS_LB; -import static org.folio.rest.jooq.Tables.SNAPSHOTS_LB; -import static org.folio.rest.jooq.enums.RecordType.MARC_BIB; -import static org.folio.rest.util.QueryParamUtil.toRecordType; -import static org.jooq.impl.DSL.condition; -import static org.jooq.impl.DSL.countDistinct; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.inline; -import static org.jooq.impl.DSL.max; -import static org.jooq.impl.DSL.name; -import static org.jooq.impl.DSL.primaryKey; -import static org.jooq.impl.DSL.select; -import static org.jooq.impl.DSL.table; -import static org.jooq.impl.DSL.trueCondition; - import com.google.common.collect.Lists; import io.github.jklingsporn.vertx.jooq.classic.reactivepg.ReactiveClassicGenericQueryExecutor; import io.github.jklingsporn.vertx.jooq.shared.internal.QueryResult; @@ -45,24 +11,13 @@ import io.vertx.reactivex.pgclient.PgPool; import io.vertx.reactivex.sqlclient.SqlConnection; import io.vertx.sqlclient.Row; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import javax.ws.rs.BadRequestException; -import javax.ws.rs.NotFoundException; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.operators.conditional.AndExpression; +import net.sf.jsqlparser.expression.operators.conditional.OrExpression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.text.StrSubstitutor; import org.apache.commons.lang3.StringUtils; @@ -108,6 +63,7 @@ import org.folio.services.util.TypeConnection; import org.folio.services.util.parser.ParseFieldsResult; import org.folio.services.util.parser.ParseLeaderResult; +import org.jooq.CommonTableExpression; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Field; @@ -134,6 +90,50 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.NotFoundException; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static org.folio.dao.util.AdvisoryLockUtil.acquireLock; +import static org.folio.dao.util.ErrorRecordDaoUtil.ERROR_RECORD_CONTENT; +import static org.folio.dao.util.ParsedRecordDaoUtil.PARSED_RECORD_CONTENT; +import static org.folio.dao.util.RawRecordDaoUtil.RAW_RECORD_CONTENT; +import static org.folio.dao.util.RecordDaoUtil.RECORD_NOT_FOUND_TEMPLATE; +import static org.folio.dao.util.RecordDaoUtil.ensureRecordForeignKeys; +import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalIdNonNull; +import static org.folio.dao.util.RecordDaoUtil.filterRecordByState; +import static org.folio.dao.util.RecordDaoUtil.filterRecordByType; +import static org.folio.dao.util.RecordDaoUtil.getExternalHrid; +import static org.folio.dao.util.RecordDaoUtil.getExternalId; +import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_FOUND_TEMPLATE; +import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_STARTED_MESSAGE_TEMPLATE; +import static org.folio.rest.jooq.Tables.ERROR_RECORDS_LB; +import static org.folio.rest.jooq.Tables.MARC_RECORDS_LB; +import static org.folio.rest.jooq.Tables.MARC_RECORDS_TRACKING; +import static org.folio.rest.jooq.Tables.RAW_RECORDS_LB; +import static org.folio.rest.jooq.Tables.RECORDS_LB; +import static org.folio.rest.jooq.Tables.SNAPSHOTS_LB; +import static org.folio.rest.jooq.enums.RecordType.MARC_BIB; +import static org.folio.rest.util.QueryParamUtil.toRecordType; +import static org.jooq.impl.DSL.*; + @Component public class RecordDaoImpl implements RecordDao { @@ -166,7 +166,7 @@ public class RecordDaoImpl implements RecordDao { private static final Field COUNT_FIELD = field(name(COUNT), Integer.class); - private static final Field[] RECORD_FIELDS = new Field[] { + private static final Field[] RECORD_FIELDS = new Field[]{ RECORDS_LB.ID, RECORDS_LB.SNAPSHOT_ID, RECORDS_LB.MATCHED_ID, @@ -211,6 +211,9 @@ public class RecordDaoImpl implements RecordDao { "UNION " + "SELECT marc_id " + "FROM deleted_rows2"; + public static final String OR = " or "; + public static final String MARC_INDEXERS = "marc_indexers"; + public static final Field MARC_INDEXERS_MARC_ID = field(TABLE_FIELD_TEMPLATE, UUID.class, field(MARC_INDEXERS), field(MARC_ID)); private final PostgresClientFactory postgresClientFactory; @@ -406,11 +409,94 @@ public Flowable streamRecords(Condition condition, RecordType recordType .doAfterTerminate(tx::commit))); } + private static String buildCteDistinctCountCondition(Expression expression) { + StringBuilder combinedExpression = new StringBuilder(); + if (expression instanceof Parenthesis parenthesis) { + Expression innerExpression = parenthesis.getExpression(); + if (containsParenthesis(innerExpression)) { + combinedExpression.append(buildCteDistinctCountCondition(innerExpression)); + } else { + combinedExpression.append(countDistinct(DSL.case_().when(DSL.condition(expression.toString()), 1)).eq(1)); + } + } else if (expression instanceof BinaryExpression binaryExpression) { + Expression leftExpression = binaryExpression.getLeftExpression(); + Expression rightExpression = binaryExpression.getRightExpression(); + if (containsParenthesis(leftExpression)) { + combinedExpression.append("("); + combinedExpression.append(buildCteDistinctCountCondition(leftExpression)); + } + if (expression instanceof AndExpression) { + combinedExpression.append(" and "); + } else if (expression instanceof OrExpression) { + combinedExpression.append(OR); + } + if (containsParenthesis(rightExpression)) { + combinedExpression.append(buildCteDistinctCountCondition(rightExpression)); + combinedExpression.append(")"); + } + } + return combinedExpression.toString(); + } + + private static void parseExpression(Expression expr, List expressions) { + if (expr instanceof BinaryExpression binExpr) { + parseExpression(binExpr.getLeftExpression(), expressions); + parseExpression(binExpr.getRightExpression(), expressions); + } else if (expr instanceof Parenthesis parenthesis) { + if (containsParenthesis(parenthesis.getExpression())) parseExpression(parenthesis.getExpression(), expressions); + else expressions.add(parenthesis); + } + } + + private static boolean containsParenthesis(Expression expr) { + if (expr instanceof Parenthesis) { + return true; + } else if (expr instanceof BinaryExpression binExpr) { + return containsParenthesis(binExpr.getLeftExpression()) || containsParenthesis(binExpr.getRightExpression()); + } else { + return false; + } + } + + private String buildCteWhereCondition(String whereExpression) throws JSQLParserException { + List expressions = new ArrayList<>(); + StringBuilder cteWhereCondition = new StringBuilder(); + + Expression expr = CCJSqlParserUtil.parseCondExpression(whereExpression); + parseExpression(expr, expressions); + int i = 1; + for (Expression expression : expressions) { + cteWhereCondition.append(expression.toString()); + if (i < expressions.size()) cteWhereCondition.append(OR); + i++; + } + return cteWhereCondition.toString(); + } + @Override - public Flowable streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters, String tenantId) { + public Flowable streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, + RecordSearchParameters searchParameters, String tenantId) throws JSQLParserException { /* Building a search query */ - SelectJoinStep searchQuery = DSL.selectDistinct(RECORDS_LB.EXTERNAL_ID).from(RECORDS_LB); - appendJoin(searchQuery, parseLeaderResult, parseFieldsResult); + //TODO: adjust bracets in condtion statements + CommonTableExpression commonTableExpression = null; + if (parseFieldsResult.isEnabled()) { + String cteWhereExpression = buildCteWhereCondition(parseFieldsResult.getWhereExpression()); + + Expression expr = CCJSqlParserUtil.parseCondExpression(parseFieldsResult.getWhereExpression()); + String cteHavingExpression = buildCteDistinctCountCondition(expr); + + commonTableExpression = DSL.name(CTE).as( + DSL.selectDistinct(MARC_INDEXERS_MARC_ID) + .from(MARC_INDEXERS) + .join(MARC_RECORDS_TRACKING).on(MARC_RECORDS_TRACKING.MARC_ID.eq(MARC_INDEXERS_MARC_ID)) + .where(DSL.condition(cteWhereExpression, parseFieldsResult.getBindingParams().toArray())) + .groupBy(MARC_INDEXERS_MARC_ID) + .having(DSL.condition(cteHavingExpression, parseFieldsResult.getBindingParams().toArray())) + ); + } + + SelectJoinStep searchQuery = selectDistinct(RECORDS_LB.EXTERNAL_ID).from(RECORDS_LB); + appendJoin(searchQuery, parseLeaderResult); appendWhere(searchQuery, parseLeaderResult, parseFieldsResult, searchParameters); if (searchParameters.getOffset() != null) { searchQuery.offset(searchParameters.getOffset()); @@ -420,39 +506,32 @@ public Flowable streamMarcRecordIds(ParseLeaderResult parseLeaderResult, Pa } /* Building a count query */ SelectJoinStep countQuery = DSL.select(countDistinct(RECORDS_LB.EXTERNAL_ID)).from(RECORDS_LB); - appendJoin(countQuery, parseLeaderResult, parseFieldsResult); + appendJoin(countQuery, parseLeaderResult); appendWhere(countQuery, parseLeaderResult, parseFieldsResult, searchParameters); /* Join both in one query */ - String sql = DSL.select().from(searchQuery).rightJoin(countQuery).on(DSL.trueCondition()).getSQL(ParamType.INLINED); - + String sql = ""; + if (parseFieldsResult.isEnabled()) { + sql = DSL.with(commonTableExpression).select().from(searchQuery).rightJoin(countQuery).on(DSL.trueCondition()).getSQL(ParamType.INLINED); + } else { + sql = select().from(searchQuery).rightJoin(countQuery).on(DSL.trueCondition()).getSQL(ParamType.INLINED); + } + String finalSql = sql; + LOG.trace("streamMarcRecordIds:: SQL : {}", finalSql); return getCachedPool(tenantId) .rxGetConnection() .flatMapPublisher(conn -> conn.rxBegin() - .flatMapPublisher(tx -> conn.rxPrepare(sql) + .flatMapPublisher(tx -> conn.rxPrepare(finalSql) .flatMapPublisher(pq -> pq.createStream(10000) .toFlowable() - .filter(row -> !enableFallbackQuery || row.getInteger(COUNT) != 0) - .switchIfEmpty(streamMarcRecordIdsWithoutIndexersVersionUsage(conn, parseLeaderResult, parseFieldsResult, searchParameters)) .map(this::toRow)) .doAfterTerminate(tx::commit))); } - private void appendJoin(SelectJoinStep selectJoinStep, ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult) { + private void appendJoin(SelectJoinStep selectJoinStep, ParseLeaderResult parseLeaderResult) { if (parseLeaderResult.isEnabled() && !parseLeaderResult.isIndexedFieldsCriteriaOnly()) { Table marcIndexersLeader = table(name("marc_indexers_leader")); selectJoinStep.innerJoin(marcIndexersLeader).on(RECORDS_LB.ID.eq(field(TABLE_FIELD_TEMPLATE, UUID.class, marcIndexersLeader, name(MARC_ID)))); } - if (parseFieldsResult.isEnabled()) { - parseFieldsResult.getFieldsToJoin().forEach(fieldToJoin -> { - Table marcIndexers = table(name(MARC_INDEXERS_PARTITION_PREFIX + fieldToJoin)).as("i" + fieldToJoin); - Field marcIndexersMarcIdField = field(TABLE_FIELD_TEMPLATE, UUID.class, marcIndexers, name(MARC_ID)); - Field marcIndexersVersionField = field(TABLE_FIELD_TEMPLATE, Integer.class, marcIndexers, name(VERSION)); - selectJoinStep.innerJoin(marcIndexers).on(RECORDS_LB.ID.eq(field(TABLE_FIELD_TEMPLATE, UUID.class, marcIndexers, name(MARC_ID)))) - .innerJoin(MARC_RECORDS_TRACKING.as("mrt_"+ fieldToJoin)) // join to marc_records_tracking to return latest version - .on(marcIndexersMarcIdField.eq(MARC_RECORDS_TRACKING.as("mrt_"+ fieldToJoin).MARC_ID) - .and(marcIndexersVersionField.eq(MARC_RECORDS_TRACKING.as("mrt_"+ fieldToJoin).VERSION))); - }); - } } private void appendWhere(SelectJoinStep step, ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters) { @@ -463,7 +542,8 @@ private void appendWhere(SelectJoinStep step, ParseLeaderResult parseLeaderResul ? DSL.condition(parseLeaderResult.getWhereExpression(), parseLeaderResult.getBindingParams().toArray()) : DSL.noCondition(); Condition fieldsCondition = parseFieldsResult.isEnabled() - ? DSL.condition(parseFieldsResult.getWhereExpression(), parseFieldsResult.getBindingParams().toArray()) + ? exists(select(field("*")).from("cte") + .where("records_lb.id = cte.marc_id")) : DSL.noCondition(); step.where(leaderCondition) .and(fieldsCondition) @@ -481,7 +561,7 @@ private Flowable streamMarcRecordIdsWithoutInd } private String getAlternativeQuery(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters) { - SelectJoinStep> searchQuery = DSL.selectDistinct(RECORDS_LB.EXTERNAL_ID).from(RECORDS_LB); + SelectJoinStep> searchQuery = selectDistinct(RECORDS_LB.EXTERNAL_ID).from(RECORDS_LB); appendJoinAlternative(searchQuery, parseLeaderResult, parseFieldsResult); appendWhere(searchQuery, parseLeaderResult, parseFieldsResult, searchParameters); if (searchParameters.getOffset() != null) { @@ -1062,8 +1142,8 @@ public Future updateParsedRecords(RecordCollection r } }).map(Record::getParsedRecord) - .filter(parsedRecord -> Objects.nonNull(parsedRecord.getId())) - .collect(Collectors.toList()); + .filter(parsedRecord -> Objects.nonNull(parsedRecord.getId())) + .collect(Collectors.toList()); try (Connection connection = getConnection(tenantId)) { DSL.using(connection).transaction(ctx -> { @@ -1103,43 +1183,44 @@ public Future updateParsedRecords(RecordCollection r } catch (SQLException e) { LOG.warn("updateParsedRecords:: Failed to update records", e); blockingPromise.fail(e); - }}, - false, - result -> { - if (result.failed()) { - LOG.warn("updateParsedRecords:: Error during update of parsed records", result.cause()); - promise.fail(result.cause()); - } else { - LOG.debug("updateParsedRecords:: Parsed records update was successful"); - promise.complete(result.result()); - } - }); + } + }, + false, + result -> { + if (result.failed()) { + LOG.warn("updateParsedRecords:: Error during update of parsed records", result.cause()); + promise.fail(result.cause()); + } else { + LOG.debug("updateParsedRecords:: Parsed records update was successful"); + promise.complete(result.result()); + } + }); return promise.future(); } @Override public Future> getRecordByExternalId(String externalId, IdType idType, - String tenantId) { + String tenantId) { return getQueryExecutor(tenantId) - .transaction(txQE -> getRecordByExternalId(txQE, externalId, idType)); + .transaction(txQE -> getRecordByExternalId(txQE, externalId, idType)); } @Override public Future> getRecordByExternalId(ReactiveClassicGenericQueryExecutor txQE, - String externalId, IdType idType) { + String externalId, IdType idType) { Condition condition = RecordDaoUtil.getExternalIdCondition(externalId, idType) - .and(RECORDS_LB.STATE.eq(RecordState.ACTUAL) - .or(RECORDS_LB.STATE.eq(RecordState.DELETED))); + .and(RECORDS_LB.STATE.eq(RecordState.ACTUAL) + .or(RECORDS_LB.STATE.eq(RecordState.DELETED))); return txQE.findOneRow(dsl -> dsl.selectFrom(RECORDS_LB) - .where(condition) - .orderBy(RECORDS_LB.GENERATION.sort(SortOrder.DESC)) - .limit(1)) - .map(RecordDaoUtil::toOptionalRecord) - .compose(optionalRecord -> optionalRecord - .map(record -> lookupAssociatedRecords(txQE, record, false).map(Optional::of)) - .orElse(Future.failedFuture(new NotFoundException(format(RECORD_NOT_FOUND_BY_ID_TYPE, idType, externalId))))) - .onFailure(v -> txQE.rollback()); + .where(condition) + .orderBy(RECORDS_LB.GENERATION.sort(SortOrder.DESC)) + .limit(1)) + .map(RecordDaoUtil::toOptionalRecord) + .compose(optionalRecord -> optionalRecord + .map(record -> lookupAssociatedRecords(txQE, record, false).map(Optional::of)) + .orElse(Future.failedFuture(new NotFoundException(format(RECORD_NOT_FOUND_BY_ID_TYPE, idType, externalId))))) + .onFailure(v -> txQE.rollback()); } @Override @@ -1181,10 +1262,10 @@ public Future saveUpdatedRecord(ReactiveClassicGenericQueryExecutor txQE public Future updateSuppressFromDiscoveryForRecord(String id, IdType idType, Boolean suppress, String tenantId) { LOG.trace("updateSuppressFromDiscoveryForRecord:: Updating suppress from discovery with value {} for record with {} {} for tenant {}", suppress, idType, id, tenantId); return getQueryExecutor(tenantId).transaction(txQE -> getRecordByExternalId(txQE, id, idType) - .compose(optionalRecord -> optionalRecord - .map(record -> RecordDaoUtil.update(txQE, record.withAdditionalInfo(record.getAdditionalInfo().withSuppressDiscovery(suppress)))) - .orElse(Future.failedFuture(new NotFoundException(format(RECORD_NOT_FOUND_BY_ID_TYPE, idType, id)))))) - .map(u -> true); + .compose(optionalRecord -> optionalRecord + .map(record -> RecordDaoUtil.update(txQE, record.withAdditionalInfo(record.getAdditionalInfo().withSuppressDiscovery(suppress)))) + .orElse(Future.failedFuture(new NotFoundException(format(RECORD_NOT_FOUND_BY_ID_TYPE, idType, id)))))) + .map(u -> true); } @Override @@ -1448,19 +1529,19 @@ private Record validateParsedRecordId(Record record) { } private Field[] getRecordFields(Name prt) { - return (Field[]) ArrayUtils.addAll(RECORD_FIELDS, new Field[] { + return (Field[]) ArrayUtils.addAll(RECORD_FIELDS, new Field[]{ field(TABLE_FIELD_TEMPLATE, JSONB.class, prt, name(CONTENT)) }); } private Field[] getRecordFieldsWithCount(Name prt) { - return (Field[]) ArrayUtils.addAll(getRecordFields(prt), new Field[] { + return (Field[]) ArrayUtils.addAll(getRecordFields(prt), new Field[]{ COUNT_FIELD }); } private Field[] getAllRecordFields(Name prt) { - return (Field[]) ArrayUtils.addAll(RECORD_FIELDS, new Field[] { + return (Field[]) ArrayUtils.addAll(RECORD_FIELDS, new Field[]{ field(TABLE_FIELD_TEMPLATE, JSONB.class, prt, name(CONTENT)).as(PARSED_RECORD_CONTENT), RAW_RECORDS_LB.CONTENT.as(RAW_RECORD_CONTENT), ERROR_RECORDS_LB.CONTENT.as(ERROR_RECORD_CONTENT), @@ -1469,13 +1550,13 @@ private Field[] getAllRecordFields(Name prt) { } private Field[] getAllRecordFieldsWithCount(Name prt) { - return (Field[]) ArrayUtils.addAll(getAllRecordFields(prt), new Field[] { + return (Field[]) ArrayUtils.addAll(getAllRecordFields(prt), new Field[]{ COUNT_FIELD }); } private Field[] getStrippedParsedRecordWithCount(Name prt) { - return new Field[] { COUNT_FIELD, + return new Field[]{COUNT_FIELD, RECORDS_LB.ID, RECORDS_LB.EXTERNAL_ID, RECORDS_LB.STATE, RECORDS_LB.RECORD_TYPE, field(TABLE_FIELD_TEMPLATE, JSONB.class, prt, name(CONTENT)).as(PARSED_RECORD_CONTENT) @@ -1513,8 +1594,7 @@ private RecordCollection toRecordCollectionWithLimitCheck(QueryResult result, in // Validation to ignore records insertion to the returned recordCollection when limit equals zero if (limit == 0) { return new RecordCollection().withTotalRecords(asRow(result.unwrap()).getInteger(COUNT)); - } - else { + } else { return toRecordCollection(result); } } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java index 1b971d89a..1a91b8e99 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java @@ -38,6 +38,7 @@ import io.vertx.core.json.JsonObject; import io.vertx.pgclient.PgException; import io.vertx.sqlclient.Row; +import net.sf.jsqlparser.JSQLParserException; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -210,7 +211,13 @@ public Flowable streamMarcRecordIds(RecordSearchParameters searchParameters } ParseLeaderResult parseLeaderResult = SearchExpressionParser.parseLeaderSearchExpression(searchParameters.getLeaderSearchExpression()); ParseFieldsResult parseFieldsResult = SearchExpressionParser.parseFieldsSearchExpression(searchParameters.getFieldsSearchExpression()); - return recordDao.streamMarcRecordIds(parseLeaderResult, parseFieldsResult, searchParameters, tenantId); + return Flowable.defer(() -> { + try { + return recordDao.streamMarcRecordIds(parseLeaderResult, parseFieldsResult, searchParameters, tenantId); + } catch (JSQLParserException e) { + return Flowable.error(new RuntimeException("Error parsing expression", e)); + } + }); } @Override diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/SearchExpressionParser.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/SearchExpressionParser.java index f1f0a7b36..9dbedb1bb 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/SearchExpressionParser.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/SearchExpressionParser.java @@ -74,10 +74,10 @@ private static List getLexemes(String expression) { private static String processExpression(String expression, List lexemes) { if (expression.matches(MARC_FIELD.getSearchValue()) || expression.matches(LEADER_FIELD.getSearchValue())) { - String splitResult[] = expression.split(SPACE, 3); + String[] splitResult = expression.split(SPACE, 3); String leftOperand = splitResult[0]; String operator = splitResult[1]; - String rightSuffix[] = splitResult[2].split("'*'", 3); + String[] rightSuffix = splitResult[2].split("'*'", 3); String rightOperand = rightSuffix[1]; lexemes.add(BinaryOperandLexeme.of(leftOperand, operator, rightOperand)); return rightSuffix[2]; diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/DateRangeBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/DateRangeBinaryOperand.java index 23c715274..c410efd5a 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/DateRangeBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/DateRangeBinaryOperand.java @@ -50,18 +50,22 @@ public static boolean matches(String key) { @Override public String toSqlRepresentation() { - String iField = "\"i" + key.substring(0, key.indexOf('.')) + "\""; - StringBuilder builder = new StringBuilder("to_date(substring(").append(iField).append(".\"value\", 1, 8), '").append(DATE_PATTERN).append("')"); + String[] keyParts = getKey().split("\\."); + String field = keyParts[0]; + String fieldNumberToSearch = "\"field_no\" = '" + field + "'"; + + StringBuilder builder = new StringBuilder("( " + fieldNumberToSearch + " and to_date(substring(").append("value, 1, 8), '").append(DATE_PATTERN).append("')"); + if (BINARY_OPERATOR_EQUALS.equals(getOperator()) && !this.rangeSearch) { - return builder.append(" = ?").toString(); + return builder.append(" = ?)").toString(); } else if (BINARY_OPERATOR_NOT_EQUALS.equals(getOperator()) && !this.rangeSearch) { - return builder.append(" <> ?").toString(); + return builder.append(" <> ?)").toString(); } else if (BINARY_OPERATOR_FROM.equals(getOperator()) && !this.rangeSearch) { - return builder.append(" >= ?").toString(); + return builder.append(" >= ?)").toString(); } else if (BINARY_OPERATOR_TO.equals(getOperator()) && !this.rangeSearch) { - return builder.append(" <= ?").toString(); + return builder.append(" <= ?)").toString(); } else if (BINARY_OPERATOR_IN.equals(getOperator()) && this.rangeSearch) { - return builder.append(" between ? and ?").toString(); + return builder.append(" between ? and ?)").toString(); } throw new IllegalArgumentException(format("The given expression [%s %s '%s'] is not supported", key, operator.getSearchValue(), value)); } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/IndicatorBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/IndicatorBinaryOperand.java index f945602e3..eda76d3a4 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/IndicatorBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/IndicatorBinaryOperand.java @@ -37,13 +37,14 @@ public String toSqlRepresentation() { String[] keyParts = getKey().split("\\."); var field = keyParts[0]; var indicator = keyParts[1]; - var sqlRepresentation = "\"" + "i" + field + "\"" + "." + "\"" + indicator + "\""; + String fieldNumberToSearch = "\"field_no\" = '" + field+"'"; + var sqlRepresentation = "( " + fieldNumberToSearch + " and " + "\"" + indicator + "\""; if (BINARY_OPERATOR_LEFT_ANCHORED_EQUALS.equals(getOperator())) { - return sqlRepresentation + " like ?"; + return sqlRepresentation + " like ?)"; } else if (BINARY_OPERATOR_EQUALS.equals(getOperator())) { - return sqlRepresentation + " = ?"; + return sqlRepresentation + " = ?)"; } else if (BINARY_OPERATOR_NOT_EQUALS.equals(getOperator())) { - return sqlRepresentation + " <> ?"; + return sqlRepresentation + " <> ?)"; } else if (BINARY_OPERATOR_IS.equals(getOperator())) { return PresenceBinaryOperand.getSqlRepresentationForIndicator(field, indicator, value); } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PositionBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PositionBinaryOperand.java index 10691e832..54860d00f 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PositionBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PositionBinaryOperand.java @@ -32,12 +32,12 @@ public static boolean matches(String key) { @Override public String toSqlRepresentation() { - String iField = "\"" + "i" + field + "\""; - String prefix = "substring(" + iField + ".\"value\", " + startPosition + ", " + endPosition + ")"; + String fieldNumberToSearch = "\"field_no\" = '" + field+"'"; + String prefix = "( " + fieldNumberToSearch + " and substring(\"value\", " + startPosition + ", " + endPosition + ")"; if (BINARY_OPERATOR_EQUALS.equals(getOperator())) { - return prefix + " = ?"; + return prefix + " = ?)"; } else if (BINARY_OPERATOR_NOT_EQUALS.equals(getOperator())) { - return prefix + " <> ?"; + return prefix + " <> ?)"; } throw new IllegalArgumentException(format("Operator [%s] is not supported for the given Position operand", getOperator().getSearchValue())); } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PresenceBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PresenceBinaryOperand.java index d2ea21c56..6c1c9538a 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PresenceBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/PresenceBinaryOperand.java @@ -15,6 +15,7 @@ public class PresenceBinaryOperand { private static final String PRESENT = "present"; private static final String ABSENT = "absent"; + private static final String FIELD_NO = "( \"field_no\" = '"; private PresenceBinaryOperand() { } @@ -22,27 +23,27 @@ private PresenceBinaryOperand() { public static String getSqlRepresentationForMarcField(String field, String value) { validateValue(value); if (PRESENT.equals(value)) { - return "(id in (select marc_id from marc_indexers_" + field + "))"; + return FIELD_NO + field + "' and marc_indexers.marc_id in (select marc_id from marc_indexers_" + field + "))"; } else { - return "(id not in (select marc_id from marc_indexers_" + field + "))"; + return FIELD_NO + field + "' and marc_indexers.marc_id not in (select marc_id from marc_indexers_" + field + "))"; } } public static String getSqlRepresentationForSubField(String field, String subField, String value) { validateValue(value); if (PRESENT.equals(value)) { - return "(id in (select marc_id from marc_indexers_" + field + " where subfield_no = '" + subField + "')) "; + return FIELD_NO + field + "' and marc_indexers.marc_id in (select marc_id from marc_indexers_" + field + " where subfield_no = '" + subField + "'))"; } else { - return "(id not in (select marc_id from marc_indexers_" + field + " where subfield_no = '" + subField + "')) "; + return FIELD_NO + field + "' and marc_indexers.marc_id not in (select marc_id from marc_indexers_" + field + " where subfield_no = '" + subField + "'))"; } } public static String getSqlRepresentationForIndicator(String field, String indicator, String value) { validateValue(value); if (PRESENT.equals(value)) { - return "(id in (select marc_id from marc_indexers_" + field + " where " + indicator + " <> '#')) "; + return FIELD_NO + field + "' and marc_indexers.marc_id in (select marc_id from marc_indexers_" + field + " where " + indicator + " <> '#'))"; } else { - return "(id in (select marc_id from marc_indexers_" + field + " where " + indicator + " = '#')) "; + return FIELD_NO + field + "' and marc_indexers.marc_id in (select marc_id from marc_indexers_" + field + " where " + indicator + " = '#'))"; } } diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/SubFieldBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/SubFieldBinaryOperand.java index d838a4d46..ee8d4c8db 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/SubFieldBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/SubFieldBinaryOperand.java @@ -40,12 +40,12 @@ public static boolean matches(String key) { public String toSqlRepresentation() { String[] keyParts = getKey().split("\\."); String field = keyParts[0]; - String iField = "\"" + "i" + field + "\""; + String fieldNumberToSearch = "\"field_no\" = '" + field+"'"; String subField = keyParts[1]; StringBuilder stringBuilder = new StringBuilder() - .append("(").append(iField).append(".\"subfield_no\" = '").append(subField).append("'") + .append("( ").append(fieldNumberToSearch).append(" and \"subfield_no\" = '").append(subField).append("'") .append(" and ") - .append(iField).append(".\"value\" "); + .append("\"value\" "); if (BINARY_OPERATOR_LEFT_ANCHORED_EQUALS.equals(getOperator())) { return stringBuilder.append("like ?)").toString(); } else if (BINARY_OPERATOR_EQUALS.equals(getOperator())) { diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/ValueBinaryOperand.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/ValueBinaryOperand.java index b27b48775..83c7b72ee 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/ValueBinaryOperand.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/lexeme/operand/ValueBinaryOperand.java @@ -30,13 +30,14 @@ public static boolean matches(String key) { public String toSqlRepresentation() { StringBuilder stringBuilder = new StringBuilder(); String field = getKey().split("\\.")[0]; - String prefix = stringBuilder.append("\"").append("i").append(field).append("\".\"value\"").toString(); + String fieldNumberToSearch = "( \"field_no\" = '" + field + "'"; + String prefix = stringBuilder.append(fieldNumberToSearch).append(" and ").append("\"value\"").toString(); if (BINARY_OPERATOR_LEFT_ANCHORED_EQUALS.equals(getOperator())) { - return prefix + " like ?"; + return prefix + " like ?)"; } else if (BINARY_OPERATOR_EQUALS.equals(getOperator())) { - return prefix + " = ?"; + return prefix + " = ?)"; } else if (BINARY_OPERATOR_NOT_EQUALS.equals(getOperator())) { - return stringBuilder.append(" <> ?").toString(); + return stringBuilder.append(" <> ?)").toString(); } else if (BINARY_OPERATOR_IS.equals(getOperator())) { return PresenceBinaryOperand.getSqlRepresentationForMarcField(field, value); } diff --git a/mod-source-record-storage-server/src/test/java/org/folio/dao/RecordDaoImplTest.java b/mod-source-record-storage-server/src/test/java/org/folio/dao/RecordDaoImplTest.java index ca1accb41..47ebec140 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/dao/RecordDaoImplTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/dao/RecordDaoImplTest.java @@ -1,14 +1,10 @@ package org.folio.dao; import com.fasterxml.jackson.databind.ObjectMapper; -import io.reactivex.Flowable; import io.vertx.core.Future; -import io.vertx.core.json.JsonObject; import io.vertx.ext.unit.Async; import io.vertx.ext.unit.TestContext; import io.vertx.ext.unit.junit.VertxUnitRunner; -import io.vertx.reactivex.FlowableHelper; -import io.vertx.sqlclient.Row; import org.folio.TestMocks; import org.folio.TestUtil; import org.folio.dao.util.AdvisoryLockUtil; @@ -18,26 +14,20 @@ import org.folio.processing.value.MissingValue; import org.folio.processing.value.StringValue; import org.folio.rest.jaxrs.model.ExternalIdsHolder; -import org.folio.rest.jaxrs.model.MarcRecordSearchRequest; import org.folio.rest.jaxrs.model.ParsedRecord; import org.folio.rest.jaxrs.model.RawRecord; import org.folio.rest.jaxrs.model.Record; import org.folio.rest.jaxrs.model.Snapshot; import org.folio.services.AbstractLBServiceTest; -import org.folio.services.RecordSearchParameters; import org.folio.services.util.TypeConnection; -import org.folio.services.util.parser.ParseFieldsResult; -import org.folio.services.util.parser.ParseLeaderResult; -import org.folio.services.util.parser.SearchExpressionParser; + import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.util.ReflectionTestUtils; -import javax.ws.rs.DELETE; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -45,9 +35,7 @@ import static org.folio.dao.RecordDaoImpl.INDEXERS_DELETION_LOCK_NAMESPACE_ID; import static org.folio.rest.jaxrs.model.Record.State.ACTUAL; import static org.folio.rest.jaxrs.model.Record.State.DELETED; -import static org.folio.rest.jaxrs.model.Record.State.OLD; import static org.folio.rest.jooq.Tables.MARC_RECORDS_TRACKING; -import static org.folio.rest.jooq.Tables.RECORDS_LB; @RunWith(VertxUnitRunner.class) public class RecordDaoImplTest extends AbstractLBServiceTest { @@ -164,33 +152,6 @@ public void shouldReturnEmptyListIfValueFieldIsEmpty(TestContext context) { }); } - @Test - public void shouldReturnIdOnStreamMarcRecordIdsWhenThereIsNoTrackingRecordAndFallbackQueryEnabled(TestContext context) { - Async async = context.async(); - ReflectionTestUtils.setField(recordDao, ENABLE_FALLBACK_QUERY_FIELD, true); - MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest() - .withFieldsSearchExpression("001.value = '393893'"); - RecordSearchParameters searchParams = RecordSearchParameters.from(searchRequest); - ParseLeaderResult parseLeaderResult = SearchExpressionParser.parseLeaderSearchExpression(searchParams.getLeaderSearchExpression()); - ParseFieldsResult parseFieldsResult = SearchExpressionParser.parseFieldsSearchExpression(searchParams.getFieldsSearchExpression()); - ArrayList ids = new ArrayList<>(); - - Future> future = deleteTrackingRecordById(record.getId()) - .map(v -> recordDao.streamMarcRecordIds(parseLeaderResult, parseFieldsResult, searchParams, TENANT_ID)); - - future.onComplete(ar -> { - context.assertTrue(ar.succeeded()); - FlowableHelper.toReadStream(ar.result()) - .exceptionHandler(context::fail) - .handler(row -> ids.add(String.valueOf(row.getUUID(RECORDS_LB.EXTERNAL_ID.getName())))) - .endHandler(v -> { - context.assertEquals(1, ids.size()); - context.assertEquals(record.getExternalIdsHolder().getInstanceId(), ids.get(0)); - async.complete(); - }); - }); - } - @Test public void shouldReturnFalseWhenPreviousIndexersDeletionIsInProgress(TestContext context) { Async async = context.async(); diff --git a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java index 5744be9fe..df3dfdbbb 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageStreamApiTest.java @@ -1,22 +1,5 @@ package org.folio.rest.impl; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.io.InputStream; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Objects; -import java.util.Scanner; -import java.util.UUID; - import com.fasterxml.jackson.databind.ObjectMapper; import io.reactivex.BackpressureStrategy; import io.reactivex.Flowable; @@ -29,10 +12,6 @@ import io.vertx.ext.unit.TestContext; import io.vertx.ext.unit.junit.VertxUnitRunner; import org.apache.http.HttpStatus; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - import org.folio.TestUtil; import org.folio.dao.PostgresClientFactory; import org.folio.dao.util.ParsedRecordDaoUtil; @@ -47,6 +26,27 @@ import org.folio.rest.jaxrs.model.Record.RecordType; import org.folio.rest.jaxrs.model.Snapshot; import org.folio.rest.jaxrs.model.SourceRecord; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Scanner; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; @RunWith(VertxUnitRunner.class) public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { @@ -77,23 +77,23 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { } } - private static ParsedRecord invalidParsedRecord = new ParsedRecord() + private static final ParsedRecord invalidParsedRecord = new ParsedRecord() .withContent("Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur."); - private static ErrorRecord errorRecord = new ErrorRecord() + private static final ErrorRecord errorRecord = new ErrorRecord() .withDescription("Oops... something happened") .withContent("Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur."); - private static Snapshot snapshot_1 = new Snapshot() + private static final Snapshot snapshot_1 = new Snapshot() .withJobExecutionId(UUID.randomUUID().toString()) .withStatus(Snapshot.Status.PARSING_IN_PROGRESS); - private static Snapshot snapshot_2 = new Snapshot() + private static final Snapshot snapshot_2 = new Snapshot() .withJobExecutionId(UUID.randomUUID().toString()) .withStatus(Snapshot.Status.PARSING_IN_PROGRESS); - private static Snapshot snapshot_3 = new Snapshot() + private static final Snapshot snapshot_3 = new Snapshot() .withJobExecutionId(UUID.randomUUID().toString()) .withStatus(Snapshot.Status.PARSING_IN_PROGRESS); - private static Record marc_bib_record_1 = new Record() + private static final Record marc_bib_record_1 = new Record() .withId(FIRST_UUID) .withSnapshotId(snapshot_1.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -101,7 +101,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withMatchedId(FIRST_UUID) .withOrder(0) .withState(Record.State.ACTUAL); - private static Record marc_bib_record_2 = new Record() + private static final Record marc_bib_record_2 = new Record() .withId(SECOND_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -113,7 +113,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withExternalIdsHolder(new ExternalIdsHolder() .withInstanceId(UUID.randomUUID().toString()) .withInstanceHrid("12345")); - private static Record marc_bib_record_3 = new Record() + private static final Record marc_bib_record_3 = new Record() .withId(THIRD_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -121,7 +121,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withErrorRecord(errorRecord) .withMatchedId(THIRD_UUID) .withState(Record.State.ACTUAL); - private static Record marc_bib_record_4 = new Record() + private static final Record marc_bib_record_4 = new Record() .withId(FOURTH_UUID) .withSnapshotId(snapshot_1.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -133,7 +133,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withExternalIdsHolder(new ExternalIdsHolder() .withInstanceId(UUID.randomUUID().toString()) .withInstanceHrid("12345")); - private static Record marc_bib_record_5 = new Record() + private static final Record marc_bib_record_5 = new Record() .withId(FIFTH_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -142,7 +142,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withParsedRecord(invalidParsedRecord) .withOrder(101) .withState(Record.State.ACTUAL); - private static Record marc_bib_record_6 = new Record() + private static final Record marc_bib_record_6 = new Record() .withId(SIXTH_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(Record.RecordType.MARC_BIB) @@ -154,7 +154,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withExternalIdsHolder(new ExternalIdsHolder() .withInstanceId(UUID.randomUUID().toString()) .withInstanceHrid("12345")); - private static Record marc_auth_record_1 = new Record() + private static final Record marc_auth_record_1 = new Record() .withId(SEVENTH_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(RecordType.MARC_AUTHORITY) @@ -166,7 +166,7 @@ public class SourceStorageStreamApiTest extends AbstractRestVerticleTest { .withExternalIdsHolder(new ExternalIdsHolder() .withAuthorityId(UUID.randomUUID().toString()) .withAuthorityHrid("12345")); - private static Record marc_holdings_record_1 = new Record() + private static final Record marc_holdings_record_1 = new Record() .withId(EIGHTH_UUID) .withSnapshotId(snapshot_2.getJobExecutionId()) .withRecordType(RecordType.MARC_HOLDING) @@ -588,7 +588,7 @@ public void shouldReturnEmptyCollectionOnGetByRecordIdIfThereIsNoSuchRecord(Test InputStream response = RestAssured.given() .spec(spec) .when() - .get(SOURCE_STORAGE_STREAM_SOURCE_RECORDS_PATH + "?recordId=" + UUID.randomUUID().toString() + "&limit=1&offset=0") + .get(SOURCE_STORAGE_STREAM_SOURCE_RECORDS_PATH + "?recordId=" + UUID.randomUUID() + "&limit=1&offset=0") .then() .statusCode(HttpStatus.SC_OK) .extract().response().asInputStream(); @@ -1425,9 +1425,203 @@ public void shouldReturnIdOnSearchMarcRecordIdsWhenInstanceIdIsMissing(TestConte async.complete(); } + @Test + public void shouldProcessSearchQueryIfSearchNeededWithinOneField(TestContext testContext) { + // given + final Async async = testContext.async(); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, marc_bib_record_2); + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setFieldsSearchExpression("050.a ^= 'M3' and 050.b ^= '.M896'"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + + @Test + public void shouldProcessSearchQueryIfSearchNeededWithinOneFieldWithParenthesis(TestContext testContext) { + // given + final Async async = testContext.async(); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, marc_bib_record_2); + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setFieldsSearchExpression("(050.a ^= 'M3' and 050.b ^= '.M896') and 240.a ^= 'Works'"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + + @Test + public void shouldReturn400WithIncorrectRequest(TestContext testContext) { + // given + Async async = testContext.async(); + Record suppressedRecord = new Record() + .withId(marc_bib_record_2.getId()) + .withSnapshotId(snapshot_2.getJobExecutionId()) + .withRecordType(Record.RecordType.MARC_BIB) + .withRawRecord(marc_bib_record_2.getRawRecord()) + .withParsedRecord(marc_bib_record_2.getParsedRecord()) + .withMatchedId(marc_bib_record_2.getMatchedId()) + .withState(Record.State.ACTUAL) + .withAdditionalInfo(new AdditionalInfo().withSuppressDiscovery(true)) + .withExternalIdsHolder(marc_bib_record_2.getExternalIdsHolder()); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, suppressedRecord); + + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); + searchRequest.setFieldsSearchExpression("(035.a = '(OCoLC)63611770' and 036.ind1 = '1' or (245.a ^= 'Semantic web' and 005.value ^= '20141107')"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + // then + assertEquals(HttpStatus.SC_BAD_REQUEST, response.statusCode()); + assertEquals("The number of opened brackets should be equal to number of closed brackets [expression: marcFieldSearchExpression]", + response.body().asString()); + async.complete(); + } + + @Test + public void shouldReturnDataForDocumentationExample(TestContext testContext) { + // given + Async async = testContext.async(); + Record suppressedRecord = new Record() + .withId(marc_bib_record_2.getId()) + .withSnapshotId(snapshot_2.getJobExecutionId()) + .withRecordType(Record.RecordType.MARC_BIB) + .withRawRecord(marc_bib_record_2.getRawRecord()) + .withParsedRecord(marc_bib_record_2.getParsedRecord()) + .withMatchedId(marc_bib_record_2.getMatchedId()) + .withState(Record.State.ACTUAL) + .withAdditionalInfo(new AdditionalInfo().withSuppressDiscovery(true)) + .withExternalIdsHolder(marc_bib_record_2.getExternalIdsHolder()); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, suppressedRecord); + + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); + searchRequest.setFieldsSearchExpression("(035.a = '(OCoLC)63611770' and 036.ind1 = '1') or (245.a ^= 'Neue Ausgabe sämtlicher' and 005.value ^= '20141107')"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + + @Test + public void shouldReturnDataForNotEqualsOperator(TestContext testContext) { + // given + Async async = testContext.async(); + Record suppressedRecord = new Record() + .withId(marc_bib_record_2.getId()) + .withSnapshotId(snapshot_2.getJobExecutionId()) + .withRecordType(Record.RecordType.MARC_BIB) + .withRawRecord(marc_bib_record_2.getRawRecord()) + .withParsedRecord(marc_bib_record_2.getParsedRecord()) + .withMatchedId(marc_bib_record_2.getMatchedId()) + .withState(Record.State.ACTUAL) + .withAdditionalInfo(new AdditionalInfo().withSuppressDiscovery(true)) + .withExternalIdsHolder(marc_bib_record_2.getExternalIdsHolder()); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, suppressedRecord); + + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); + searchRequest.setFieldsSearchExpression("(035.a = '(OCoLC)63611770' and 948.ind1 not= '5')"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + + @Test + public void shouldReturnDataForOneFieldNoOperator(TestContext testContext) { + // given + Async async = testContext.async(); + Record suppressedRecord = new Record() + .withId(marc_bib_record_2.getId()) + .withSnapshotId(snapshot_2.getJobExecutionId()) + .withRecordType(Record.RecordType.MARC_BIB) + .withRawRecord(marc_bib_record_2.getRawRecord()) + .withParsedRecord(marc_bib_record_2.getParsedRecord()) + .withMatchedId(marc_bib_record_2.getMatchedId()) + .withState(Record.State.ACTUAL) + .withAdditionalInfo(new AdditionalInfo().withSuppressDiscovery(true)) + .withExternalIdsHolder(marc_bib_record_2.getExternalIdsHolder()); + postSnapshots(testContext, snapshot_2); + postRecords(testContext, suppressedRecord); + + MarcRecordSearchRequest searchRequest = new MarcRecordSearchRequest(); + searchRequest.setLeaderSearchExpression("p_05 = 'c' and p_06 = 'c' and p_07 = 'm'"); + searchRequest.setFieldsSearchExpression("(948.ind1 = '2' and 948.value = '20130128')" + + " and (948.ind1 = '2' and 948.value = '20141106') " + + " and (948.ind1 = '2' and 948.value = 'm') " + + " and (948.ind1 = '2' and 948.value = 'batch')"); + // when + ExtractableResponse response = RestAssured.given() + .spec(spec) + .body(searchRequest) + .when() + .post("/source-storage/stream/marc-record-identifiers") + .then() + .extract(); + JsonObject responseBody = new JsonObject(response.body().asString()); + // then + assertEquals(HttpStatus.SC_OK, response.statusCode()); + assertEquals(1, responseBody.getJsonArray("records").size()); + assertEquals(1, responseBody.getInteger("totalCount").intValue()); + async.complete(); + } + private Flowable flowableInputStreamScanner(InputStream inputStream) { return Flowable.create(subscriber -> { - try (Scanner scanner = new Scanner(inputStream, "UTF-8")) { + try (Scanner scanner = new Scanner(inputStream, StandardCharsets.UTF_8)) { while (scanner.hasNext()) { subscriber.onNext(scanner.nextLine()); } diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/SearchExpressionParserUnitTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/SearchExpressionParserUnitTest.java index 86827d54f..4f70c5802 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/SearchExpressionParserUnitTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/SearchExpressionParserUnitTest.java @@ -148,7 +148,7 @@ public void shouldParseFieldsSearchExpression_for_SubFieldOperand_EqualsOperator assertTrue(result.isEnabled()); assertEquals(singletonList("(OCoLC)63611770"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("035")), result.getFieldsToJoin()); - assertEquals("(\"i035\".\"subfield_no\" = 'a' and \"i035\".\"value\" = ?)", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and \"subfield_no\" = 'a' and \"value\" = ?)", result.getWhereExpression()); } @Test @@ -161,7 +161,7 @@ public void shouldParseFieldsSearchExpression_for_SubFieldOperand_LeftAnchoredEq assertTrue(result.isEnabled()); assertEquals(singletonList("(OCoLC)%"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("035")), result.getFieldsToJoin()); - assertEquals("(\"i035\".\"subfield_no\" = 'a' and \"i035\".\"value\" like ?)", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and \"subfield_no\" = 'a' and \"value\" like ?)", result.getWhereExpression()); } @Test @@ -174,7 +174,7 @@ public void shouldParseFieldsSearchExpression_for_SubFieldOperand_NotEqualsOpera assertTrue(result.isEnabled()); assertEquals(singletonList("(OCoLC)"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("035")), result.getFieldsToJoin()); - assertEquals("(\"i035\".\"subfield_no\" = 'a' and \"i035\".\"value\" <> ?)", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and \"subfield_no\" = 'a' and \"value\" <> ?)", result.getWhereExpression()); } @Test @@ -187,7 +187,7 @@ public void shouldParseFieldsSearchExpression_for_SubFieldOperand_IsPresentOpera assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id in (select marc_id from marc_indexers_035 where subfield_no = 'a')) ", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and marc_indexers.marc_id in (select marc_id from marc_indexers_035 where subfield_no = 'a'))", result.getWhereExpression()); } @Test @@ -200,7 +200,7 @@ public void shouldParseFieldsSearchExpression_for_SubFieldOperand_IsAbsentOperat assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id not in (select marc_id from marc_indexers_035 where subfield_no = 'z')) ", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and marc_indexers.marc_id not in (select marc_id from marc_indexers_035 where subfield_no = 'z'))", result.getWhereExpression()); } @Test @@ -213,7 +213,7 @@ public void shouldParseFieldsSearchExpression_for_IndicatorOperand_EqualsOperato assertTrue(result.isEnabled()); assertEquals(singletonList("1"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("036")), result.getFieldsToJoin()); - assertEquals("\"i036\".\"ind1\" = ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '036' and \"ind1\" = ?)", result.getWhereExpression()); } @Test @@ -226,7 +226,7 @@ public void shouldParseFieldsSearchExpression_for_IndicatorOperand_LeftAnchoredE assertTrue(result.isEnabled()); assertEquals(singletonList("1%"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("036")), result.getFieldsToJoin()); - assertEquals("\"i036\".\"ind1\" like ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '036' and \"ind1\" like ?)", result.getWhereExpression()); } @Test @@ -239,7 +239,7 @@ public void shouldParseFieldsSearchExpression_for_IndicatorOperand_NotEqualsOper assertTrue(result.isEnabled()); assertEquals(singletonList("1"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("036")), result.getFieldsToJoin()); - assertEquals("\"i036\".\"ind1\" <> ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '036' and \"ind1\" <> ?)", result.getWhereExpression()); } @Test @@ -252,7 +252,7 @@ public void shouldParseFieldsSearchExpression_for_ValueOperand_EqualsOperator() assertTrue(result.isEnabled()); assertEquals(singletonList("20141107001016.0"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("\"i005\".\"value\" = ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and \"value\" = ?)", result.getWhereExpression()); } @Test @@ -265,7 +265,7 @@ public void shouldParseFieldsSearchExpression_for_ValueOperand_LeftAnchoredEqual assertTrue(result.isEnabled()); assertEquals(singletonList("20141107%"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("\"i005\".\"value\" like ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and \"value\" like ?)", result.getWhereExpression()); } @Test @@ -278,7 +278,7 @@ public void shouldParseFieldsSearchExpression_for_ValueOperand_NotEqualsOperator assertTrue(result.isEnabled()); assertEquals(singletonList("20141107"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("\"i005\".\"value\" <> ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and \"value\" <> ?)", result.getWhereExpression()); } @Test @@ -291,7 +291,7 @@ public void shouldParseFieldsSearchExpression_for_ValueOperand_IsPresentOperator assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id in (select marc_id from marc_indexers_035))", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and marc_indexers.marc_id in (select marc_id from marc_indexers_035))", result.getWhereExpression()); } @Test @@ -304,7 +304,7 @@ public void shouldParseFieldsSearchExpression_for_ValueOperand_IsAbsentOperator( assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id not in (select marc_id from marc_indexers_035))", result.getWhereExpression()); + assertEquals("( \"field_no\" = '035' and marc_indexers.marc_id not in (select marc_id from marc_indexers_035))", result.getWhereExpression()); } @Test @@ -345,7 +345,7 @@ public void shouldParseFieldsSearchExpression_for_PositionOperand_EqualsOperator assertTrue(result.isEnabled()); assertEquals(singletonList("2014"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("substring(\"i005\".\"value\", 1, 4) = ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and substring(\"value\", 1, 4) = ?)", result.getWhereExpression()); } @Test @@ -358,7 +358,7 @@ public void shouldParseFieldsSearchExpression_for_PositionOperand_NotEqualsOpera assertTrue(result.isEnabled()); assertEquals(singletonList("2014"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("substring(\"i005\".\"value\", 1, 4) <> ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and substring(\"value\", 1, 4) <> ?)", result.getWhereExpression()); } @Test @@ -413,7 +413,7 @@ public void shouldParseFieldsSearchExpression_forDateRangeOperand_EqualsOperator assertTrue(result.isEnabled()); assertEquals(singletonList("201701025"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') = ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') = ?)", result.getWhereExpression()); } @Test @@ -426,7 +426,7 @@ public void shouldParseFieldsSearchExpression_forDateRangeOperand_NotEqualsOpera assertTrue(result.isEnabled()); assertEquals(singletonList("201701025"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') <> ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') <> ?)", result.getWhereExpression()); } @Test @@ -439,7 +439,7 @@ public void shouldParseFieldsSearchExpression_forDateRangeOperand_FromOperator() assertTrue(result.isEnabled()); assertEquals(singletonList("201701025"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') >= ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') >= ?)", result.getWhereExpression()); } @Test @@ -452,7 +452,7 @@ public void shouldParseFieldsSearchExpression_forDateRangeOperand_ToOperator() { assertTrue(result.isEnabled()); assertEquals(singletonList("201701025"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') <= ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') <= ?)", result.getWhereExpression()); } @Test @@ -465,7 +465,7 @@ public void shouldParseFieldsSearchExpression_forDateRangeOperand_InOperator() { assertTrue(result.isEnabled()); assertEquals(Arrays.asList("201701025", "20200213"), result.getBindingParams()); assertEquals(new HashSet<>(singletonList("005")), result.getFieldsToJoin()); - assertEquals("to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') between ? and ?", result.getWhereExpression()); + assertEquals("( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') between ? and ?)", result.getWhereExpression()); } @Test @@ -478,7 +478,7 @@ public void shouldParseFieldsSearchExpression_with_boolean_operators() { assertTrue(result.isEnabled()); assertEquals(asList("(OCoLC)63611770", "1", "1%", "20141107%", "abc", "20171128", "20200114"), result.getBindingParams()); assertEquals(new HashSet<>(asList("001", "035", "036", "005")), result.getFieldsToJoin()); - assertEquals("((\"i035\".\"subfield_no\" = 'a' and \"i035\".\"value\" = ?) and \"i036\".\"ind1\" <> ?) or (\"i036\".\"ind1\" like ? and \"i005\".\"value\" like ?) or (substring(\"i001\".\"value\", 2, 3) = ? and to_date(substring(\"i005\".\"value\", 1, 8), 'yyyymmdd') between ? and ?)", result.getWhereExpression()); + assertEquals("(( \"field_no\" = '035' and \"subfield_no\" = 'a' and \"value\" = ?) and ( \"field_no\" = '036' and \"ind1\" <> ?)) or (( \"field_no\" = '036' and \"ind1\" like ?) and ( \"field_no\" = '005' and \"value\" like ?)) or (( \"field_no\" = '001' and substring(\"value\", 2, 3) = ?) and ( \"field_no\" = '005' and to_date(substring(value, 1, 8), 'yyyymmdd') between ? and ?))", result.getWhereExpression()); } @Test @@ -491,7 +491,7 @@ public void shouldParseFieldsSearchExpression_for_IndicatorOperand_IsPresentOper assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id in (select marc_id from marc_indexers_050 where ind1 <> '#')) ", result.getWhereExpression()); + assertEquals("( \"field_no\" = '050' and marc_indexers.marc_id in (select marc_id from marc_indexers_050 where ind1 <> '#'))", result.getWhereExpression()); } @Test @@ -504,7 +504,7 @@ public void shouldParseFieldsSearchExpression_for_IndicatorOperand_IsAbsentOpera assertTrue(result.isEnabled()); assertEquals(emptyList(), result.getBindingParams()); assertEquals(emptySet(), result.getFieldsToJoin()); - assertEquals("(id in (select marc_id from marc_indexers_050 where ind2 = '#')) ", result.getWhereExpression()); + assertEquals("( \"field_no\" = '050' and marc_indexers.marc_id in (select marc_id from marc_indexers_050 where ind2 = '#'))", result.getWhereExpression()); } @Test From 2d065782e9e4dda582fdac60ff5e8be499df0dd5 Mon Sep 17 00:00:00 2001 From: Javokhir Abdullaev <101543142+JavokhirAbdullayev@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:18:51 +0500 Subject: [PATCH 7/8] MODINV-1044 Additional Requirements - Update Data Import logic to normalize OCLC 035 values (#630) * MODINV-1044 Additional Requirements - Update Data Import logic to normalize OCLC 035 values * Update AdditionalFieldsUtilTest.java --- NEWS.md | 1 + .../services/util/AdditionalFieldsUtil.java | 6 ++--- .../services/AdditionalFieldsUtilTest.java | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index cd1dbeba7..5885b4da9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ * [MODSOURCE-756](https://issues.folio.org/browse/MODSOURCE-756) After setting an instance as marked for deletion it is no longer editable in quickmarc * [MODSOURCE-753](https://folio-org.atlassian.net/browse/MODSOURCE-753) Change SQL query parameters for MARC Search * [MODSOURCE-773](https://folio-org.atlassian.net/browse/MODSOURCE-773) MARC Search omits suppressed from discovery records in default search +* [MODINV-1044](https://folio-org.atlassian.net/browse/MODINV-1044) Additional Requirements - Update Data Import logic to normalize OCLC 035 values ## 2024-03-20 5.8.0 * [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java index 0d967bb14..d906bd7a9 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java @@ -72,7 +72,7 @@ public final class AdditionalFieldsUtil { private static final char HR_ID_FIELD_IND = ' '; private static final String ANY_STRING = "*"; private static final String OCLC = "OCoLC"; - private static final String OCLC_PATTERN = "\\((" + OCLC + ")\\)((ocm|ocn)?0*|([a-zA-Z]+)0*)(\\d+\\w*)"; + private static final String OCLC_PATTERN = "\\((" + OCLC + ")\\)((ocm|ocn|on)?0*|([a-zA-Z]+)0*)(\\d+\\w*)"; public static final DateTimeFormatter dateTime005Formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss.S"); @@ -460,7 +460,7 @@ private static Set formatOclc(List subFields) { Pattern pattern = Pattern.compile(OCLC_PATTERN); for (Subfield subfield : subFields) { - String data = subfield.getData(); + String data = subfield.getData().replaceAll("[.\\s]", ""); var code = subfield.getCode(); Matcher matcher = pattern.matcher(data); @@ -469,7 +469,7 @@ private static Set formatOclc(List subFields) { String numericAndTrailing = matcher.group(5); // Numeric part and any characters that follow String prefix = matcher.group(2); // Entire prefix including letters and potentially leading zeros - if (prefix != null && (prefix.startsWith("ocm") || prefix.startsWith("ocn"))) { + if (prefix != null && (prefix.startsWith("ocm") || prefix.startsWith("ocn") || prefix.startsWith("on"))) { // If "ocm" or "ocn", strip entirely from the prefix processedSet.add(code + "&(" + oclcTag + ")" + numericAndTrailing); } else { diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java index 837ce4c2f..ba092539a 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java @@ -359,6 +359,28 @@ public void shouldReturnSubfieldIfOclcExist() { Assert.assertEquals(expectedSubfields.get(0), subfields.get(0)); } + @Test + public void shouldRemovePeriodsAndSpacesAfterNormalization() { + // given + var parsedContent = "{\"leader\":\"00120nam 22000731a 4500\",\"fields\":[{\"001\":\"in001\"}," + + "{\"035\":{\"subfields\":[{\"a\":\"(OCoLC)on. 607TST .001\"}],\"ind1\":\" \",\"ind2\":\" \"}}," + + "{\"500\":{\"subfields\":[{\"a\":\"data\"}],\"ind1\":\" \",\"ind2\":\" \"}}]}"; + + var expectedParsedContent = "{\"leader\":\"00098nam 22000611a 4500\",\"fields\":[{\"001\":\"in001\"}," + + "{\"035\":{\"subfields\":[{\"a\":\"(OCoLC)607TST001\"}],\"ind1\":\" \",\"ind2\":\" \"}}," + + "{\"500\":{\"subfields\":[{\"a\":\"data\"}],\"ind1\":\" \",\"ind2\":\" \"}}]}"; + ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedContent); + + Record record = new Record().withId(UUID.randomUUID().toString()) + .withParsedRecord(parsedRecord) + .withGeneration(0) + .withState(Record.State.ACTUAL) + .withExternalIdsHolder(new ExternalIdsHolder().withInstanceId("001").withInstanceHrid("in001")); + // when + AdditionalFieldsUtil.normalize035(record); + Assert.assertEquals(expectedParsedContent, parsedRecord.getContent()); + } + @Test public void shouldNotReturnSubfieldIfOclcNotExist() { // given From b1179d50adb8c2f81950a4031c98c819fdf9968f Mon Sep 17 00:00:00 2001 From: Dmytro Krutii Date: Wed, 17 Jul 2024 14:42:07 +0300 Subject: [PATCH 8/8] MODSOURMAN-1200 Find record by match id instead record id --- NEWS.md | 1 + .../org/folio/services/RecordServiceImpl.java | 2 +- .../org/folio/services/RecordServiceTest.java | 83 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5885b4da9..44e8e189c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ * [MODSOURCE-753](https://folio-org.atlassian.net/browse/MODSOURCE-753) Change SQL query parameters for MARC Search * [MODSOURCE-773](https://folio-org.atlassian.net/browse/MODSOURCE-773) MARC Search omits suppressed from discovery records in default search * [MODINV-1044](https://folio-org.atlassian.net/browse/MODINV-1044) Additional Requirements - Update Data Import logic to normalize OCLC 035 values +* [MODSOURMAN-1200](https://folio-org.atlassian.net/browse/MODSOURMAN-1200) Find record by match id on update generation ## 2024-03-20 5.8.0 * [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings diff --git a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java index 1a91b8e99..b47400c15 100644 --- a/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java +++ b/mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java @@ -182,7 +182,7 @@ public Future updateRecordGeneration(String matchedId, Record record, St } record.setId(UUID.randomUUID().toString()); - return recordDao.getRecordById(matchedId, tenantId) + return recordDao.getRecordByMatchedId(matchedId, tenantId) .map(r -> r.orElseThrow(() -> new NotFoundException(format(RECORD_WITH_GIVEN_MATCHED_ID_NOT_FOUND, matchedId)))) .compose(v -> saveRecord(record, tenantId)) .recover(throwable -> { diff --git a/mod-source-record-storage-server/src/test/java/org/folio/services/RecordServiceTest.java b/mod-source-record-storage-server/src/test/java/org/folio/services/RecordServiceTest.java index a70b68f7c..6de404c4b 100644 --- a/mod-source-record-storage-server/src/test/java/org/folio/services/RecordServiceTest.java +++ b/mod-source-record-storage-server/src/test/java/org/folio/services/RecordServiceTest.java @@ -605,6 +605,89 @@ public void shouldUpdateRecordGeneration(TestContext context) { }); } + @Test + public void shouldUpdateRecordGenerationByMatchId(TestContext context) { + var mock = TestMocks.getMarcBibRecord(); + var recordToSave = new Record() + .withId(UUID.randomUUID().toString()) + .withSnapshotId(mock.getSnapshotId()) + .withRecordType(mock.getRecordType()) + .withState(State.ACTUAL) + .withOrder(mock.getOrder()) + .withRawRecord(rawRecord) + .withParsedRecord(marcRecord) + .withAdditionalInfo(mock.getAdditionalInfo()) + .withExternalIdsHolder(new ExternalIdsHolder().withInstanceId(UUID.randomUUID().toString())) + .withMetadata(mock.getMetadata()); + + var async = context.async(); + + recordService.saveRecord(recordToSave, TENANT_ID).onComplete(savedRecord -> { + if (savedRecord.failed()) { + context.fail(savedRecord.cause()); + } + context.assertNotNull(savedRecord.result().getRawRecord()); + context.assertNotNull(savedRecord.result().getParsedRecord()); + context.assertEquals(savedRecord.result().getState(), State.ACTUAL); + compareRecords(context, recordToSave, savedRecord.result()); + + var matchedId = savedRecord.result().getMatchedId(); + var snapshot = new Snapshot().withJobExecutionId(UUID.randomUUID().toString()) + .withProcessingStartedDate(new Date()) + .withStatus(Snapshot.Status.PROCESSING_IN_PROGRESS); + + var parsedRecord = new ParsedRecord().withId(UUID.randomUUID().toString()) + .withContent(new JsonObject().put("leader", "01542ccm a2200361 4500") + .put("fields", new JsonArray().add(new JsonObject().put("999", new JsonObject() + .put("subfields", + new JsonArray().add(new JsonObject().put("s", matchedId))) + .put("ind1", "f") + .put("ind2", "f")))).encode()); + + var recordToUpdateGeneration = new Record() + .withId(UUID.randomUUID().toString()) + .withSnapshotId(snapshot.getJobExecutionId()) + .withRecordType(mock.getRecordType()) + .withState(State.ACTUAL) + .withOrder(mock.getOrder()) + .withRawRecord(mock.getRawRecord()) + .withParsedRecord(parsedRecord) + .withAdditionalInfo(mock.getAdditionalInfo()) + .withExternalIdsHolder(new ExternalIdsHolder().withInstanceId(UUID.randomUUID().toString())) + .withMetadata(mock.getMetadata()); + + SnapshotDaoUtil.save(postgresClientFactory.getQueryExecutor(TENANT_ID), snapshot).onComplete(snapshotSaved -> { + if (snapshotSaved.failed()) { + context.fail(snapshotSaved.cause()); + } + + recordService.updateRecordGeneration(matchedId, recordToUpdateGeneration, TENANT_ID).onComplete(recordToUpdateGenerationSaved -> { + context.assertTrue(recordToUpdateGenerationSaved.succeeded()); + context.assertEquals(recordToUpdateGenerationSaved.result().getMatchedId(), matchedId); + context.assertEquals(recordToUpdateGenerationSaved.result().getGeneration(), 1); + recordDao.getRecordByMatchedId(matchedId, TENANT_ID).onComplete(get -> { + if (get.failed()) { + context.fail(get.cause()); + } + context.assertTrue(get.result().isPresent()); + context.assertEquals(get.result().get().getGeneration(), 1); + context.assertEquals(get.result().get().getMatchedId(), matchedId); + context.assertNotEquals(get.result().get().getId(), matchedId); + context.assertEquals(get.result().get().getState(), State.ACTUAL); + recordDao.getRecordById(matchedId, TENANT_ID).onComplete(getRecord1 -> { + if (getRecord1.failed()) { + context.fail(get.cause()); + } + context.assertTrue(getRecord1.result().isPresent()); + context.assertEquals(getRecord1.result().get().getState(), State.OLD); + async.complete(); + }); + }); + }); + }); + }); + } + @Test public void shouldSaveMarcBibRecordWithMatchedIdFromRecordId(TestContext context) { Record original = TestMocks.getMarcBibRecord();