Skip to content

Commit

Permalink
MODSOURCE-783 Extend MARC-MARC search query to account for qualifiers
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmytrokrutii
dmytrokrutii authored Jul 29, 2024
1 parent 2aa801b commit fb4dc58
Show file tree
Hide file tree
Showing 7 changed files with 340 additions and 26 deletions.
1 change: 1 addition & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
* [MODSOURMAN-1200](https://folio-org.atlassian.net/browse/MODSOURMAN-1200) Find record by match id on update generation
* [MODINV-1049](https://folio-org.atlassian.net/browse/MODINV-1049) Existing "035" field is not retained the original position in imported record
* [MODSOURCE-785](https://folio-org.atlassian.net/browse/MODSOURCE-785) Update 005 field when set MARC for deletion
* [MODSOURMAN-783](https://folio-org.atlassian.net/browse/MODSOURCE-783) Extend MARC-MARC search query to account for qualifiers

## 2024-03-20 5.8.0
* [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings
Expand Down
43 changes: 38 additions & 5 deletions mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,18 @@
import static org.folio.rest.jooq.Tables.SNAPSHOTS_LB;
import static org.folio.rest.jooq.enums.RecordType.MARC_BIB;
import static org.folio.rest.util.QueryParamUtil.toRecordType;
import static org.jooq.impl.DSL.*;
import static org.jooq.impl.DSL.condition;
import static org.jooq.impl.DSL.countDistinct;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.inline;
import static org.jooq.impl.DSL.max;
import static org.jooq.impl.DSL.name;
import static org.jooq.impl.DSL.primaryKey;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.table;
import static org.jooq.impl.DSL.trueCondition;
import static org.jooq.impl.DSL.selectDistinct;
import static org.jooq.impl.DSL.exists;

@Component
public class RecordDaoImpl implements RecordDao {
Expand All @@ -156,7 +167,9 @@ public class RecordDaoImpl implements RecordDao {
static final int INDEXERS_DELETION_LOCK_NAMESPACE_ID = "delete_marc_indexers".hashCode();

public static final String CONTROL_FIELD_CONDITION_TEMPLATE = "\"{partition}\".\"value\" in ({value})";
public static final String CONTROL_FIELD_CONDITION_TEMPLATE_WITH_QUALIFIER = "\"{partition}\".\"value\" IN ({value}) AND \"{partition}\".\"value\" LIKE {qualifier}";
public static final String DATA_FIELD_CONDITION_TEMPLATE = "\"{partition}\".\"value\" in ({value}) and \"{partition}\".\"ind1\" LIKE '{ind1}' and \"{partition}\".\"ind2\" LIKE '{ind2}' and \"{partition}\".\"subfield_no\" = '{subfield}'";
public static final String DATA_FIELD_CONDITION_TEMPLATE_WITH_QUALIFIER = "\"{partition}\".\"value\" IN ({value}) AND \"{partition}\".\"value\" LIKE {qualifier} AND \"{partition}\".\"ind1\" LIKE '{ind1}' AND \"{partition}\".\"ind2\" LIKE '{ind2}' AND \"{partition}\".\"subfield_no\" = '{subfield}'";
private static final String VALUE_IN_SINGLE_QUOTES = "'%s'";
private static final String RECORD_NOT_FOUND_BY_ID_TYPE = "Record with %s id: %s was not found";
private static final String INVALID_PARSED_RECORD_MESSAGE_TEMPLATE = "Record %s has invalid parsed record; %s";
Expand Down Expand Up @@ -352,18 +365,25 @@ public Future<List<Record>> getMatchedRecordsWithoutIndexersVersionUsage(MatchFi

private Condition getMatchedFieldCondition(MatchField matchedField, String partition) {
Map<String, String> params = new HashMap<>();
var qualifierSearch = false;
params.put("partition", partition);
params.put("value", getValueInSqlFormat(matchedField.getValue()));
if (matchedField.getQualifierMatch() != null) {
qualifierSearch = true;
params.put("qualifier", getSqlQualifier(matchedField.getQualifierMatch()));
}
String sql;
if (matchedField.isControlField()) {
String sql = StrSubstitutor.replace(CONTROL_FIELD_CONDITION_TEMPLATE, params, "{", "}");
return condition(sql);
sql = qualifierSearch ? StrSubstitutor.replace(CONTROL_FIELD_CONDITION_TEMPLATE_WITH_QUALIFIER, params, "{", "}")
: StrSubstitutor.replace(CONTROL_FIELD_CONDITION_TEMPLATE, params, "{", "}");
} else {
params.put("ind1", getSqlInd(matchedField.getInd1()));
params.put("ind2", getSqlInd(matchedField.getInd2()));
params.put("subfield", matchedField.getSubfield());
String sql = StrSubstitutor.replace(DATA_FIELD_CONDITION_TEMPLATE, params, "{", "}");
return condition(sql);
sql = qualifierSearch ? sql = StrSubstitutor.replace(DATA_FIELD_CONDITION_TEMPLATE_WITH_QUALIFIER, params, "{", "}")
: StrSubstitutor.replace(DATA_FIELD_CONDITION_TEMPLATE, params, "{", "}");
}
return condition(sql);
}

private String getSqlInd(String ind) {
Expand All @@ -372,6 +392,19 @@ private String getSqlInd(String ind) {
return ind;
}

private String getSqlQualifier(MatchField.QualifierMatch qualifierMatch) {
if (qualifierMatch == null) {
return null;
}
var value = qualifierMatch.value();

return switch (qualifierMatch.qualifier()) {
case BEGINS_WITH -> "'" + value + "%'";
case ENDS_WITH -> "'%" + value + "'";
case CONTAINS -> "'%" + value + "%'";
};
}

private String getValueInSqlFormat(Value value) {
if (Value.ValueType.STRING.equals(value.getType())) {
return format(VALUE_IN_SINGLE_QUOTES, value.getValue());
Expand Down
20 changes: 20 additions & 0 deletions mod-source-record-storage-server/src/main/java/org/folio/dao/util/MatchField.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.folio.dao.util;

import org.folio.processing.value.Value;
import org.folio.rest.jaxrs.model.Filter;
import org.marc4j.marc.impl.Verifier;

/**
Expand All @@ -15,6 +16,7 @@ public class MatchField {
private final String ind2;
private final String subfield;
private final Value value;
private final QualifierMatch qualifierMatch;
private final String fieldPath;

public MatchField(String tag, String ind1, String ind2, String subfield, Value value) {
Expand All @@ -24,6 +26,20 @@ public MatchField(String tag, String ind1, String ind2, String subfield, Value v
this.subfield = subfield;
this.value = value;
this.fieldPath = tag + ind1 + ind2 + subfield;
this.qualifierMatch = null;
}

public MatchField(String tag, String ind1, String ind2, String subfield, Value value, QualifierMatch qualifierMatch) {
this.tag = tag;
this.ind1 = ind1;
this.ind2 = ind2;
this.subfield = subfield;
this.value = value;
this.qualifierMatch = qualifierMatch;
this.fieldPath = tag + ind1 + ind2 + subfield;
}

public record QualifierMatch(Filter.Qualifier qualifier, String value) {
}

public String getTag() {
Expand All @@ -46,6 +62,10 @@ public Value getValue() {
return value;
}

public QualifierMatch getQualifierMatch() {
return qualifierMatch;
}

public boolean isControlField() {
return Verifier.isControlField(tag);
}
Expand Down
77 changes: 65 additions & 12 deletions mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public final class RecordDaoUtil {
public static final String RECORD_NOT_FOUND_TEMPLATE = "Record with id '%s' was not found";

private static final String COMMA = ",";
private static final String LIKE_OPERATOR = "%";
private static final List<String> DELETED_LEADER_RECORD_STATUS = Arrays.asList("d", "s", "x");

private RecordDaoUtil() {}
Expand All @@ -77,6 +78,18 @@ public static Condition getExternalIdsCondition(List<String> externalIds, IdType
return getIdCondition(idType, idField -> idField.in(toUUIDs(externalIds)));
}

/**
* Get {@link Condition} where in external list ids and {@link IdType} and match by qualifier value
*
* @param externalIds list of external id
* @param idType external id type
* @param qualifier qualifier type and value
* @return condition
*/
public static Condition getExternalIdsConditionWithQualifier(List<String> externalIds, IdType idType, MatchField.QualifierMatch qualifier) {
return getIdConditionWithQualifier(idType, idField -> idField.in(toUUIDs(externalIds)), qualifier);
}

/**
* Count query by {@link Condition}
*
Expand Down Expand Up @@ -478,6 +491,19 @@ public static Condition filterRecordByExternalHridValues(List<String> externalHr
return RECORDS_LB.EXTERNAL_HRID.in(externalHridValues);
}

/**
* Get {@link Condition} to filter by external entity hrid using specified values and match by qualifier value
*
* @param externalHridValues external entity hrid values to equal
* @param qualifier qualifier type and value
* @return condition
*/
public static Condition filterRecordByExternalHridValuesWithQualifier(List<String> externalHridValues, MatchField.QualifierMatch qualifier) {
var qualifierCondition = buildQualifierCondition(RECORDS_LB.EXTERNAL_HRID, qualifier);
var resultCondition = RECORDS_LB.EXTERNAL_HRID.in(externalHridValues);
return qualifierCondition != null ? resultCondition.and(qualifierCondition) : resultCondition;
}

/**
* Get {@link Condition} to filter by snapshotId id
*
Expand Down Expand Up @@ -734,20 +760,47 @@ private static Condition getRecordTypeCondition(RecordType recordType) {
}

private static Condition getIdCondition(IdType idType, Function<Field<UUID>, Condition> idFieldToConditionMapper) {
IdType idTypeToUse = idType;
RecordType recordType = null;
if (idType == IdType.HOLDINGS) {
idTypeToUse = IdType.EXTERNAL;
recordType = RecordType.MARC_HOLDING;
} else if (idType == IdType.INSTANCE) {
idTypeToUse = IdType.EXTERNAL;
recordType = RecordType.MARC_BIB;
} else if (idType == IdType.AUTHORITY) {
idTypeToUse = IdType.EXTERNAL;
recordType = RecordType.MARC_AUTHORITY;
}
IdType idTypeToUse = getIdType(idType);
RecordType recordType = getRecordType(idTypeToUse);
var idField = RECORDS_LB.field(LOWER_CAMEL.to(LOWER_UNDERSCORE, idTypeToUse.getIdField()), UUID.class);
return idFieldToConditionMapper.apply(idField).and(getRecordTypeCondition(recordType));
}

private static Condition getIdConditionWithQualifier(IdType idType, Function<Field<UUID>, Condition> idFieldToConditionMapper, MatchField.QualifierMatch qualifier) {
IdType idTypeToUse = getIdType(idType);
RecordType recordType = getRecordType(idType);
var idField = RECORDS_LB.field(LOWER_CAMEL.to(LOWER_UNDERSCORE, idTypeToUse.getIdField()), UUID.class);
var qualifierCondition = buildQualifierCondition(idField, qualifier);
var resultCondition = idFieldToConditionMapper.apply(idField).and(getRecordTypeCondition(recordType));
return qualifierCondition != null ? resultCondition.and(qualifierCondition) : resultCondition;
}

private static Condition buildQualifierCondition(Field field, MatchField.QualifierMatch qualifier) {
if (qualifier == null) {
return null;
}
var value = qualifier.value();
return switch (qualifier.qualifier()) {
case BEGINS_WITH -> field.like(value + LIKE_OPERATOR);
case ENDS_WITH -> field.like(LIKE_OPERATOR + value);
case CONTAINS -> field.like(LIKE_OPERATOR + value + LIKE_OPERATOR);
};
}

private static RecordType getRecordType(IdType idType) {
return switch (idType) {
case HOLDINGS -> RecordType.MARC_HOLDING;
case INSTANCE -> RecordType.MARC_BIB;
case AUTHORITY -> RecordType.MARC_AUTHORITY;
default -> null;
};
}

private static IdType getIdType(IdType idType) {
return switch (idType) {
case HOLDINGS, INSTANCE, AUTHORITY -> IdType.EXTERNAL;
default -> idType;
};
}

}
17 changes: 11 additions & 6 deletions mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
import static org.folio.dao.util.RecordDaoUtil.ensureRecordForeignKeys;
import static org.folio.dao.util.RecordDaoUtil.ensureRecordHasId;
import static org.folio.dao.util.RecordDaoUtil.ensureRecordHasSuppressDiscovery;
import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalHridValues;
import static org.folio.dao.util.RecordDaoUtil.filterRecordByExternalHridValuesWithQualifier;
import static org.folio.dao.util.RecordDaoUtil.filterRecordByState;
import static org.folio.dao.util.RecordDaoUtil.getExternalIdsCondition;
import static org.folio.dao.util.RecordDaoUtil.getExternalIdsConditionWithQualifier;
import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_FOUND_TEMPLATE;
import static org.folio.dao.util.SnapshotDaoUtil.SNAPSHOT_NOT_STARTED_MESSAGE_TEMPLATE;
import static org.folio.rest.util.QueryParamUtil.toRecordType;
Expand Down Expand Up @@ -465,7 +465,11 @@ private MatchField prepareMatchField(RecordMatchingDto recordMatchingDto) {
String ind1 = filter.getIndicator1() != null ? filter.getIndicator1() : StringUtils.EMPTY;
String ind2 = filter.getIndicator2() != null ? filter.getIndicator2() : StringUtils.EMPTY;
String subfield = filter.getSubfield() != null ? filter.getSubfield() : StringUtils.EMPTY;
return new MatchField(filter.getField(), ind1, ind2, subfield, ListValue.of(filter.getValues()));
MatchField.QualifierMatch qualifier = null;
if (filter.getQualifier() != null && filter.getQualifierValue() != null) {
qualifier = new MatchField.QualifierMatch(filter.getQualifier(), filter.getQualifierValue());
}
return new MatchField(filter.getField(), ind1, ind2, subfield, ListValue.of(filter.getValues()), qualifier);
}

private TypeConnection getTypeConnection(RecordMatchingDto.RecordType recordType) {
Expand All @@ -480,13 +484,14 @@ private Future<RecordsIdentifiersCollection> processDefaultMatchField(MatchField
RecordMatchingDto recordMatchingDto, String tenantId) {
Condition condition = filterRecordByState(Record.State.ACTUAL.value());
List<String> values = ((ListValue) matchField.getValue()).getValue();
var qualifier = matchField.getQualifierMatch();

if (matchField.isMatchedId()) {
condition = condition.and(getExternalIdsCondition(values, IdType.RECORD));
condition = condition.and(getExternalIdsConditionWithQualifier(values, IdType.RECORD, qualifier));
} else if (matchField.isExternalId()) {
condition = condition.and(getExternalIdsCondition(values, IdType.EXTERNAL));
condition = condition.and(getExternalIdsConditionWithQualifier(values, IdType.EXTERNAL, qualifier));
} else if (matchField.isExternalHrid()) {
condition = condition.and(filterRecordByExternalHridValues(values));
condition = condition.and(filterRecordByExternalHridValuesWithQualifier(values, qualifier));
}

return recordDao.getRecords(condition, typeConnection.getDbType(), Collections.emptyList(), recordMatchingDto.getOffset(),
Expand Down
Loading

0 comments on commit fb4dc58

Please sign in to comment.