Merge branch 'master' of https://github.com/folio-org/mod-source-reco…

…rd-storage into MODSOURCE-785

NEWS.md

@@ -2,6 +2,9 @@
 * [MODSOURCE-767](https://folio-org.atlassian.net/browse/MODSOURCE-767) Single record overlay creates duplicate OCLC#/035
 * [MODSOURCE-756](https://issues.folio.org/browse/MODSOURCE-756) After setting an instance as marked for deletion it is no longer editable in quickmarc
 * [MODSOURCE-753](https://folio-org.atlassian.net/browse/MODSOURCE-753) Change SQL query parameters for MARC Search
+* [MODSOURCE-773](https://folio-org.atlassian.net/browse/MODSOURCE-773) MARC Search omits suppressed from discovery records in default search
+* [MODINV-1044](https://folio-org.atlassian.net/browse/MODINV-1044) Additional Requirements - Update Data Import logic to normalize OCLC 035 values
+* [MODSOURMAN-1200](https://folio-org.atlassian.net/browse/MODSOURMAN-1200) Find record by match id on update generation 
 
 ## 2024-03-20 5.8.0
 * [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings

mod-source-record-storage-server/pom.xml

@@ -182,7 +182,7 @@
     <dependency>
       <groupId>org.folio</groupId>
       <artifactId>data-import-processing-core</artifactId>
-      <version>4.2.0</version>
+      <version>4.3.0-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>io.vertx</groupId>
@@ -193,6 +193,11 @@
       <artifactId>folio-kafka-wrapper</artifactId>
       <version>3.1.1</version>
     </dependency>
+    <dependency>
+      <groupId>com.github.jsqlparser</groupId>
+      <artifactId>jsqlparser</artifactId>
+      <version>4.9</version>
+    </dependency>
     <dependency>
       <groupId>net.mguenther.kafka</groupId>
       <artifactId>kafka-junit</artifactId>

mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java

@@ -6,6 +6,7 @@
 import java.util.function.Function;
 
 import io.vertx.sqlclient.Row;
+import net.sf.jsqlparser.JSQLParserException;
 import org.folio.dao.util.IdType;
 import org.folio.dao.util.RecordType;
 import org.folio.rest.jaxrs.model.MarcBibCollection;
@@ -128,7 +129,7 @@ Future<RecordsIdentifiersCollection> getMatchedRecordsIdentifiers(MatchField mat
    * @param tenantId              tenant id
    * @return {@link Flowable} of {@link Record id}
    */
-  Flowable<Row> streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters, String tenantId);
+  Flowable<Row> streamMarcRecordIds(ParseLeaderResult parseLeaderResult, ParseFieldsResult parseFieldsResult, RecordSearchParameters searchParameters, String tenantId) throws JSQLParserException;
 
   /**
    * Searches for {@link Record} by id

mod-source-record-storage-server/src/main/java/org/folio/dao/util/MarcUtil.java

@@ -3,24 +3,21 @@
 import static java.lang.String.format;
 import static org.folio.services.util.AdditionalFieldsUtil.HR_ID_FROM_FIELD;
 import static org.folio.services.util.AdditionalFieldsUtil.TAG_005;
+import static org.folio.services.util.AdditionalFieldsUtil.TAG_00X_PREFIX;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.LinkedList;
-import java.util.Map;
-import java.util.Queue;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.marc4j.MarcException;
@@ -144,60 +141,84 @@ private static String recordToTxtMarc(Record record) throws IOException {
   }
 
   /**
-   * Reorders MARC record fields
+   * Take field values from system modified record content while preserving incoming record content`s field order.
+   * Put system fields (001, 005) first, regardless of incoming record fields order.
    *
-   * @param sourceContent source parsed record
-   * @param targetContent target parsed record
-   * @return MARC txt
+   * @param sourceOrderContent content with incoming record fields order
+   * @param systemOrderContent system modified record content with reordered fields
+   * @return MARC record parsed content with desired fields order
    */
-  public static String reorderMarcRecordFields(String sourceContent, String targetContent) {
+  public static String reorderMarcRecordFields(String sourceOrderContent, String systemOrderContent) {
     try {
-      var parsedContent = objectMapper.readTree(targetContent);
+      var parsedContent = objectMapper.readTree(systemOrderContent);
       var fieldsArrayNode = (ArrayNode) parsedContent.path(FIELDS);
 
-      var jsonNodesByTag = groupNodesByTag(fieldsArrayNode);
-      var sourceFields = getSourceFields(sourceContent);
-      var rearrangedArray = objectMapper.createArrayNode();
+      var nodes = toNodeList(fieldsArrayNode);
+      var nodes00X = removeAndGetNodesByTagPrefix(nodes, TAG_00X_PREFIX);
+      var sourceOrderTags = getSourceFields(sourceOrderContent);
+      var reorderedFields = objectMapper.createArrayNode();
 
-      var nodes001 = jsonNodesByTag.get(HR_ID_FROM_FIELD);
-      if (nodes001 != null && !nodes001.isEmpty()) {
-        rearrangedArray.addAll(nodes001);
-        jsonNodesByTag.remove(HR_ID_FROM_FIELD);
+      var node001 = removeAndGetNodeByTag(nodes00X, HR_ID_FROM_FIELD);
+      if (node001 != null && !node001.isEmpty()) {
+        reorderedFields.add(node001);
       }
 
-      var nodes005 = jsonNodesByTag.get(TAG_005);
-      if (nodes005 != null && !nodes005.isEmpty()) {
-        rearrangedArray.addAll(nodes005);
-        jsonNodesByTag.remove(TAG_005);
+      var node005 = removeAndGetNodeByTag(nodes00X, TAG_005);
+      if (node005 != null && !node005.isEmpty()) {
+        reorderedFields.add(node005);
       }
 
-      for (String tag : sourceFields) {
-          Queue<JsonNode> nodes = jsonNodesByTag.get(tag);
-          if (nodes != null && !nodes.isEmpty()) {
-            rearrangedArray.addAll(nodes);
-            jsonNodesByTag.remove(tag);
+      for (String tag : sourceOrderTags) {
+        var nodeTag = tag;
+        //loop will add system generated fields that are absent in initial record, preserving their order, f.e. 035
+        do {
+          var node = tag.startsWith(TAG_00X_PREFIX) ? removeAndGetNodeByTag(nodes00X, tag) : nodes.remove(0);
+          if (node != null && !node.isEmpty()) {
+            nodeTag = getTagFromNode(node);
+            reorderedFields.add(node);
           }
-
+        } while (!tag.equals(nodeTag) && !nodes.isEmpty());
       }
 
-      jsonNodesByTag.values().forEach(rearrangedArray::addAll);
+      reorderedFields.addAll(nodes);
 
-      ((ObjectNode) parsedContent).set(FIELDS, rearrangedArray);
+      ((ObjectNode) parsedContent).set(FIELDS, reorderedFields);
       return parsedContent.toString();
     } catch (Exception e) {
       LOGGER.error("An error occurred while reordering Marc record fields: {}", e.getMessage(), e);
-      return targetContent;
+      return systemOrderContent;
+    }
+  }
+
+  private static List<JsonNode> toNodeList(ArrayNode fieldsArrayNode) {
+    var nodes = new LinkedList<JsonNode>();
+    for (var node : fieldsArrayNode) {
+      nodes.add(node);
+    }
+    return nodes;
+  }
+
+  private static JsonNode removeAndGetNodeByTag(List<JsonNode> nodes, String tag) {
+    for (int i = 0; i < nodes.size(); i++) {
+      var nodeTag = getTagFromNode(nodes.get(i));
+      if (nodeTag.equals(tag)) {
+        return nodes.remove(i);
+      }
     }
+    return null;
   }
 
-  private static Map<String, Queue<JsonNode>> groupNodesByTag(ArrayNode fieldsArrayNode) {
-    var jsonNodesByTag = new LinkedHashMap<String, Queue<JsonNode>>();
-    for (JsonNode node : fieldsArrayNode) {
-      var tag = getTagFromNode(node);
-      jsonNodesByTag.putIfAbsent(tag, new LinkedList<>());
-      jsonNodesByTag.get(tag).add(node);
+  private static List<JsonNode> removeAndGetNodesByTagPrefix(List<JsonNode> nodes, String prefix) {
+    var startsWithNodes = new LinkedList<JsonNode>();
+    for (int i = 0; i < nodes.size(); i++) {
+      var nodeTag = getTagFromNode(nodes.get(i));
+      if (nodeTag.startsWith(prefix)) {
+        startsWithNodes.add(nodes.get(i));
+      }
     }
-    return jsonNodesByTag;
+
+    nodes.removeAll(startsWithNodes);
+    return startsWithNodes;
   }
 
   private static String getTagFromNode(JsonNode node) {

mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java

@@ -592,6 +592,10 @@ public static Condition filterRecordByUpdatedDateRange(Date updatedAfter, Date u
    */
   public static Condition filterRecordByDeleted(Boolean deleted) {
     Condition condition = filterRecordByState(RecordState.ACTUAL.name());
+    if (deleted == null) {
+      condition = condition.or(filterRecordByState(RecordState.DELETED.name()))
+        .or(filterRecordByState(RecordState.ACTUAL.name()));
+    }
     if (Boolean.TRUE.equals(deleted)) {
       condition = condition.or(filterRecordByState(RecordState.DELETED.name()))
         .or(filterRecordByState(RecordState.ACTUAL.name()).and(filterRecordByLeaderRecordStatus(DELETED_LEADER_RECORD_STATUS)));
@@ -618,7 +622,7 @@ public static Condition filterRecordByExternalIdNonNull() {
 
   /**
    * Convert {@link List} of {@link String} to {@link List} or {@link OrderField}
-   *
+   * <p>
    * Relies on strong convention between dto property name and database column name.
    * Property name being lower camel case and column name being lower snake case of the property name.
    *
@@ -629,7 +633,7 @@ public static Condition filterRecordByExternalIdNonNull() {
   @SuppressWarnings("squid:S1452")
   public static List<OrderField<?>> toRecordOrderFields(List<String> orderBy, Boolean forOffset) {
     if (forOffset && orderBy.isEmpty()) {
-      return Arrays.asList(new OrderField<?>[] {RECORDS_LB.ID.asc()});
+      return Arrays.asList(new OrderField<?>[]{RECORDS_LB.ID.asc()});
     }
     return orderBy.stream()
       .map(order -> order.split(COMMA))

mod-source-record-storage-server/src/main/java/org/folio/services/RecordSearchParameters.java

@@ -12,8 +12,8 @@ public class RecordSearchParameters {
   private String leaderSearchExpression;
   private String fieldsSearchExpression;
   private Record.RecordType recordType;
-  private boolean deleted;
-  private boolean suppressedFromDiscovery;
+  private Boolean deleted;
+  private Boolean suppressedFromDiscovery;
   private Integer limit;
   private Integer offset;
 
@@ -57,19 +57,19 @@ public void setRecordType(Record.RecordType recordType) {
     this.recordType = recordType;
   }
 
-  public boolean isDeleted() {
+  public Boolean isDeleted() {
     return deleted;
   }
 
-  public void setDeleted(boolean deleted) {
+  public void setDeleted(Boolean deleted) {
     this.deleted = deleted;
   }
 
-  public boolean isSuppressedFromDiscovery() {
+  public Boolean isSuppressedFromDiscovery() {
     return suppressedFromDiscovery;
   }
 
-  public void setSuppressedFromDiscovery(boolean suppressedFromDiscovery) {
+  public void setSuppressedFromDiscovery(Boolean suppressedFromDiscovery) {
     this.suppressedFromDiscovery = suppressedFromDiscovery;
   }
 

mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java

@@ -38,6 +38,7 @@
 import io.vertx.core.json.JsonObject;
 import io.vertx.pgclient.PgException;
 import io.vertx.sqlclient.Row;
+import net.sf.jsqlparser.JSQLParserException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -182,7 +183,7 @@ public Future<Record> updateRecordGeneration(String matchedId, Record record, St
     }
     record.setId(UUID.randomUUID().toString());
 
-    return recordDao.getRecordById(matchedId, tenantId)
+    return recordDao.getRecordByMatchedId(matchedId, tenantId)
       .map(r -> r.orElseThrow(() -> new NotFoundException(format(RECORD_WITH_GIVEN_MATCHED_ID_NOT_FOUND, matchedId))))
       .compose(v -> saveRecord(record, tenantId))
       .recover(throwable -> {
@@ -211,7 +212,13 @@ public Flowable<Row> streamMarcRecordIds(RecordSearchParameters searchParameters
     }
     ParseLeaderResult parseLeaderResult = SearchExpressionParser.parseLeaderSearchExpression(searchParameters.getLeaderSearchExpression());
     ParseFieldsResult parseFieldsResult = SearchExpressionParser.parseFieldsSearchExpression(searchParameters.getFieldsSearchExpression());
-    return recordDao.streamMarcRecordIds(parseLeaderResult, parseFieldsResult, searchParameters, tenantId);
+    return Flowable.defer(() -> {
+      try {
+        return recordDao.streamMarcRecordIds(parseLeaderResult, parseFieldsResult, searchParameters, tenantId);
+      } catch (JSQLParserException e) {
+        return Flowable.error(new RuntimeException("Error parsing expression", e));
+      }
+    });
   }
 
   @Override

mod-source-record-storage-server/src/main/java/org/folio/services/handlers/AbstractPostProcessingEventHandler.java

@@ -44,7 +44,7 @@
 import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalId;
 import static org.folio.dao.util.RecordDaoUtil.filterRecordByNotSnapshotId;
 import static org.folio.rest.jaxrs.model.DataImportEventTypes.DI_INVENTORY_INSTANCE_UPDATED_READY_FOR_POST_PROCESSING;
-import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MAPPING_PROFILE;
+import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE;
 import static org.folio.rest.util.OkapiConnectionParams.OKAPI_TENANT_HEADER;
 import static org.folio.rest.util.OkapiConnectionParams.OKAPI_TOKEN_HEADER;
 import static org.folio.rest.util.OkapiConnectionParams.OKAPI_URL_HEADER;

mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractDeleteEventHandler.java

@@ -10,6 +10,7 @@
 import org.folio.processing.events.services.handler.EventHandler;
 import org.folio.processing.exceptions.EventProcessingException;
 import org.folio.rest.jaxrs.model.ExternalIdsHolder;
+import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE;
 import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper;
 import org.folio.rest.jaxrs.model.Record;
 import org.folio.rest.jooq.enums.RecordState;
@@ -21,7 +22,6 @@
 import static java.util.Objects.isNull;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.folio.ActionProfile.Action.DELETE;
-import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE;
 
 /**
  * The abstraction handles the DELETE action

mod-source-record-storage-server/src/main/java/org/folio/services/handlers/actions/AbstractUpdateModifyEventHandler.java

@@ -38,7 +38,7 @@
 import static java.util.Objects.nonNull;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.folio.ActionProfile.Action.UPDATE;
-import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.ACTION_PROFILE;
+import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE;
 import static org.folio.services.handlers.match.AbstractMarcMatchEventHandler.CENTRAL_TENANT_ID;
 import static org.folio.services.util.AdditionalFieldsUtil.HR_ID_FROM_FIELD;
 import static org.folio.services.util.AdditionalFieldsUtil.addControlledFieldToMarcRecord;

mod-source-record-storage-server/src/main/java/org/folio/services/handlers/match/AbstractMarcMatchEventHandler.java

@@ -43,8 +43,7 @@
 import static org.folio.dao.util.RecordDaoUtil.filterRecordByRecordId;
 import static org.folio.dao.util.RecordDaoUtil.filterRecordByState;
 import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD;
-import static org.folio.rest.jaxrs.model.ProfileSnapshotWrapper.ContentType.MATCH_PROFILE;
-
+import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE;
 /**
  * Abstract handler for MARC-MARC matching/not-matching of Marc record by specific fields
  */

mod-source-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java

@@ -56,6 +56,7 @@
  */
 public final class AdditionalFieldsUtil {
 
+  public static final String TAG_00X_PREFIX = "00";
   public static final String TAG_005 = "005";
   public static final String TAG_999 = "999";
   public static final String TAG_035 = "035";
@@ -71,7 +72,7 @@ public final class AdditionalFieldsUtil {
   private static final char HR_ID_FIELD_IND = ' ';
   private static final String ANY_STRING = "*";
   private static final String OCLC = "OCoLC";
-  private static final String OCLC_PATTERN = "\\((" + OCLC + ")\\)((ocm|ocn)?0*|([a-zA-Z]+)0*)(\\d+\\w*)";
+  private static final String OCLC_PATTERN = "\\((" + OCLC + ")\\)((ocm|ocn|on)?0*|([a-zA-Z]+)0*)(\\d+\\w*)";
 
 
   public static final DateTimeFormatter dateTime005Formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss.S");
@@ -459,7 +460,7 @@ private static Set<String> formatOclc(List<Subfield> subFields) {
     Pattern pattern = Pattern.compile(OCLC_PATTERN);
 
     for (Subfield subfield : subFields) {
-      String data = subfield.getData();
+      String data = subfield.getData().replaceAll("[.\\s]", "");
       var code = subfield.getCode();
       Matcher matcher = pattern.matcher(data);
 
@@ -468,7 +469,7 @@ private static Set<String> formatOclc(List<Subfield> subFields) {
         String numericAndTrailing = matcher.group(5); // Numeric part and any characters that follow
         String prefix = matcher.group(2); // Entire prefix including letters and potentially leading zeros
 
-        if (prefix != null && (prefix.startsWith("ocm") || prefix.startsWith("ocn"))) {
+        if (prefix != null && (prefix.startsWith("ocm") || prefix.startsWith("ocn") || prefix.startsWith("on"))) {
           // If "ocm" or "ocn", strip entirely from the prefix
           processedSet.add(code + "&(" + oclcTag + ")" + numericAndTrailing);
         } else {
@@ -682,7 +683,7 @@ public static boolean isFieldsFillingNeeded(Record record, JsonObject externalEn
   }
 
   private static boolean isValidIdAndHrid(String id, String hrid, String externalId, String externalHrid) {
-    return (isNotEmpty(externalId) && isNotEmpty(externalHrid)) && (id.equals(externalId) && !hrid.equals(externalHrid));
+    return (isNotEmpty(externalId)) && (id.equals(externalId) && !hrid.equals(externalHrid));
   }
 
   private static boolean isValidId(String id, String externalId) {

mod-source-record-storage-server/src/main/java/org/folio/services/util/parser/SearchExpressionParser.java

@@ -74,10 +74,10 @@ private static List<Lexeme> getLexemes(String expression) {
 
   private static String processExpression(String expression, List<Lexeme> lexemes) {
     if (expression.matches(MARC_FIELD.getSearchValue()) || expression.matches(LEADER_FIELD.getSearchValue())) {
-      String splitResult[] = expression.split(SPACE, 3);
+      String[] splitResult = expression.split(SPACE, 3);
       String leftOperand = splitResult[0];
       String operator = splitResult[1];
-      String rightSuffix[] = splitResult[2].split("'*'", 3);
+      String[] rightSuffix = splitResult[2].split("'*'", 3);
       String rightOperand = rightSuffix[1];
       lexemes.add(BinaryOperandLexeme.of(leftOperand, operator, rightOperand));
       return rightSuffix[2];