Skip to content

Commit

Permalink
MODSOURCE-506: remove rawRecord field from source record
Browse files Browse the repository at this point in the history
  • Loading branch information
@yaroslav-epam
yaroslav-epam committed Nov 6, 2023
1 parent b4f17f6 commit 94ae701
Show file tree
Hide file tree
Showing 35 changed files with 93 additions and 118 deletions.
1 change: 0 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/dao/util/RecordDaoUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,6 @@ public static SourceRecord toSourceRecord(Record record) {
}
return sourceRecord
.withOrder(record.getOrder())
.withRawRecord(record.getRawRecord())
.withParsedRecord(record.getParsedRecord())
.withAdditionalInfo(record.getAdditionalInfo())
.withExternalIdsHolder(record.getExternalIdsHolder())
Expand Down
37 changes: 9 additions & 28 deletions mod-source-record-storage-server/src/test/java/org/folio/TestMocks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,21 @@
public class TestMocks {

private static final String SOURCE_RECORDS_FOLDER_PATH = "src/test/resources/mock/sourceRecords";

private static final String SNAPSHOT_PATH_TEMPLATE = "src/test/resources/mock/snapshots/%s.json";
private static final String RECORD_PATH_TEMPLATE = "src/test/resources/mock/records/%s.json";
private static final String RAW_RECORD_PATH_TEMPLATE = "src/test/resources/mock/rawRecords/%s.json";
private static final String ERROR_RECORD_PATH_TEMPLATE = "src/test/resources/mock/errorRecords/%s.json";

private static List<Snapshot> snapshots;

private static List<Record> records;

private static List<RawRecord> rawRecords;

private static List<ParsedRecord> parsedRecords;

private static List<ErrorRecord> errorRecords;

static {
List<SourceRecord> sourceRecords = readSourceRecords();
rawRecords = sourceRecords.stream().map(TestMocks::toRawRecord).collect(Collectors.toList());
parsedRecords = sourceRecords.stream().map(TestMocks::toParsedRecord).collect(Collectors.toList());
errorRecords = readErrorRecords(sourceRecords);
records = readRecords(sourceRecords);
Expand Down Expand Up @@ -102,14 +99,6 @@ public static ErrorRecord getErrorRecord(int index) {
return clone(errorRecords.get(index));
}

public static List<RawRecord> getRawRecords() {
return new ArrayList<>(rawRecords.stream().map(TestMocks::clone).collect(Collectors.toList()));
}

public static RawRecord getRawRecord(int index) {
return clone(rawRecords.get(index));
}

public static List<ParsedRecord> getParsedRecords() {
return new ArrayList<>(parsedRecords.stream().map(TestMocks::clone).collect(Collectors.toList()));
}
Expand All @@ -130,10 +119,6 @@ public static Optional<ErrorRecord> getErrorRecord(String id) {
return errorRecords.stream().map(TestMocks::clone).filter(er -> er.getId().equals(id)).findAny();
}

public static Optional<RawRecord> getRawRecord(String id) {
return rawRecords.stream().map(TestMocks::clone).filter(rr -> rr.getId().equals(id)).findAny();
}

public static Optional<ParsedRecord> getParsedRecord(String id) {
return parsedRecords.stream().map(TestMocks::clone).filter(pr -> pr.getId().equals(id)).findAny();
}
Expand All @@ -145,18 +130,13 @@ public static ParsedRecord normalizeContent(ParsedRecord parsedRecord) {
return parsedRecord;
}

private static RawRecord toRawRecord(SourceRecord sourceRecord) {
return sourceRecord.getRawRecord();
}

private static ParsedRecord toParsedRecord(SourceRecord sourceRecord) {
return sourceRecord.getParsedRecord();
}

private static List<SourceRecord> readSourceRecords() {
File sourceRecordsDirectory = new File(SOURCE_RECORDS_FOLDER_PATH);
String[] extensions = new String[]{ "json" };
return FileUtils.listFiles(sourceRecordsDirectory, extensions, false).stream()
return FileUtils.listFiles(sourceRecordsDirectory, new String[]{"json"}, false).stream()
.map(TestMocks::readSourceRecord)
.filter(sr -> sr.isPresent())
.map(sr -> sr.get())
Expand Down Expand Up @@ -209,18 +189,15 @@ private static Optional<Record> readRecord(SourceRecord sourceRecord) {
if (file.exists()) {
try {
Record record = new ObjectMapper().readValue(file, Record.class)
.withRawRecord(sourceRecord.getRawRecord())
.withRawRecord(readRawRecord(sourceRecord.getRecordId()))
.withParsedRecord(sourceRecord.getParsedRecord())
.withExternalIdsHolder(sourceRecord.getExternalIdsHolder())
.withAdditionalInfo(sourceRecord.getAdditionalInfo());
if (Objects.nonNull(sourceRecord.getMetadata())) {
record.withMetadata(sourceRecord.getMetadata());
}
Optional<ErrorRecord> errorRecord = errorRecords.stream()
.filter(er -> er.getId().equals(record.getId())).findAny();
if (errorRecord.isPresent()) {
record.withErrorRecord(errorRecord.get());
}
errorRecords.stream()
.filter(er -> er.getId().equals(record.getId())).findAny().ifPresent(record::withErrorRecord);
return Optional.of(record);
} catch (IOException e) {
e.printStackTrace();
Expand All @@ -229,6 +206,10 @@ private static Optional<Record> readRecord(SourceRecord sourceRecord) {
return Optional.empty();
}

private static RawRecord readRawRecord(String recordId) throws IOException {
return new ObjectMapper().readValue(new File(format(RAW_RECORD_PATH_TEMPLATE, recordId)), RawRecord.class);
}

private static List<ErrorRecord> readErrorRecords(List<SourceRecord> sourceRecords) {
return sourceRecords.stream()
.map(TestMocks::readErrorRecord)
Expand Down
9 changes: 5 additions & 4 deletions mod-source-record-storage-server/src/test/java/org/folio/dao/util/EdifactUtilTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
package org.folio.dao.util;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.xlate.edi.stream.EDIStreamException;
import org.folio.TestUtil;
import org.folio.rest.jaxrs.model.SourceRecord;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -21,6 +20,7 @@ public class EdifactUtilTest {

private static final String SOURCE_RECORD_PATH = "src/test/resources/mock/sourceRecords/e4cfe577-4015-46d8-a54d-7c9b34796955.json";
private static final String SOURCE_RECORD_WITH_IGNORED_CODES_PATH = "src/test/resources/mock/sourceRecords/4ca9d8ac-9de5-432a-83ee-15832f09e868.json";
private static final String RAW_EDIFACT_RECORD_CONTENT_SAMPLE_PATH = "src/test/resources/rawEdifactRecordContent.sample";

private static final String INVALID_RAW_EDIFACT_RECORD =
"UNA:+.? '" +
Expand Down Expand Up @@ -51,14 +51,15 @@ public class EdifactUtilTest {
private SourceRecord sourceRecord;

@Before
public void readSourceRecord() throws JsonParseException, JsonMappingException, IOException {
public void readSourceRecord() throws IOException {
File file = new File(SOURCE_RECORD_PATH);
sourceRecord = new ObjectMapper().readValue(file, SourceRecord.class);
}

@Test
public void shouldFormatEdifact() throws IOException, EDIStreamException {
String rawEdifact = sourceRecord.getRawRecord().getContent();
String rawEdifact = new ObjectMapper().readValue(
TestUtil.readFileFromPath(RAW_EDIFACT_RECORD_CONTENT_SAMPLE_PATH), String.class);
String formattedEdifact = EdifactUtil.formatEdifact(rawEdifact);
assertNotNull(formattedEdifact);
assertEquals(sourceRecord.getParsedRecord().getFormattedContent(), formattedEdifact);
Expand Down
23 changes: 10 additions & 13 deletions mod-source-record-storage-server/src/test/java/org/folio/dao/util/MarcUtilTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,39 @@
package org.folio.dao.util;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

import java.io.File;
import java.io.IOException;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.tomakehurst.wiremock.common.Json;

import org.folio.rest.jaxrs.model.SourceRecord;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;
import org.marc4j.MarcException;

import java.io.File;
import java.io.IOException;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

@RunWith(BlockJUnit4ClassRunner.class)
public class MarcUtilTest {

private static final String SOURCE_RECORD_PATH = "src/test/resources/mock/sourceRecords/d3cd3e1e-a18c-4f7c-b053-9aa50343394e.json";
private static final String RAW_MARC = "01743nai a2200409 i 4500001000800000005001700008006001900025007001500044008004100059010001800100035002400118042000800142043001200150040003200162049000800194074002300202086002100225100004300246245011300289250004200402264006500444300002300509310002400532336002600556337002600582338003600608490005400644500014400698504004800842588009200890610007200982710007301054773012901127830002601256856004001282922001101322\u001E5962418\u001E20200210110404.0\u001Em o d f \u001Ecr mn|||||||||\u001E200124c20189999dcu x w obb f0 2eng c\u001E \u001Fa 2020230732\u001E \u001Fa(OCoLC)on1137385866\u001E \u001Fapcc\u001E \u001Fan-us---\u001E \u001FaGPO\u001Fbeng\u001Ferda\u001Fepn\u001FcGPO\u001FdMvI\u001E \u001FaWWW\u001E \u001Fa0807-A-07 (online)\u001E0 \u001FaLC 14.23:98-888/\u001E1 \u001FaDavis, Christopher M.,\u001Fd1966-\u001Feauthor.\u001E10\u001Fa\"Fast-track\" or expedited procedures :\u001Fbtheir purpose, elements, and implications /\u001FcChristopher M. Davis.\u001E \u001Fa[Library of Congress public edition].\u001E 1\u001Fa[Washington, D.C.] :\u001FbCongressional Research Service,\u001Fc2018-\u001E \u001Fa1 online resource.\u001E \u001FaUpdated irregularly\u001E \u001Fatext\u001Fbtxt\u001F2rdacontent\u001E \u001Facomputer\u001Fbc\u001F2rdamedia\u001E \u001Faonline resource\u001Fbcr\u001F2rdacarrier\u001E1 \u001FaReport / Congressional Research Service ;\u001Fv98-888\u001E \u001FaThe CRS report home page provides access to all versions published since 2018 in accordance with P.L. 115-141; earliest version dated 2003.\u001E \u001FaReport includes bibliographical references.\u001E \u001FaDescription based on contents viewed on Jan. 21, 2020; title from CRS report home page.\u001E10\u001FaUnited States.\u001FbCongress\u001FxRules and practice\u001FxFast-track procedure.\u001E2 \u001FaLibrary of Congress.\u001FbCongressional Research Service,\u001Feissuing body.\u001E08\u001FiContained in (work):\u001FtCRS reports (Library of Congress. Congressional Research Service)\u001Fw(DLC) 2018231131\u001Fw(OCoLC)1052784408\u001E 0\u001FaCRS report ;\u001Fv98-888.\u001E40\u001Fuhttps://purl.fdlp.gov/GPO/gpo130975\u001E \u001FaCRSREP\u001E\u001D";

private SourceRecord sourceRecord;

@Before
public void readSourceRecord() throws JsonParseException, JsonMappingException, IOException {
public void readSourceRecord() throws IOException {
File file = new File(SOURCE_RECORD_PATH);
sourceRecord = new ObjectMapper().readValue(file, SourceRecord.class);
}

@Test
public void shouldConvertRawMarcToMarcJson() throws IOException, MarcException {
String rawMarc = sourceRecord.getRawRecord().getContent();
String marcJson = MarcUtil.rawMarcToMarcJson(rawMarc);
String marcJson = MarcUtil.rawMarcToMarcJson(RAW_MARC);
assertNotNull(marcJson);
assertEquals(rawMarc, MarcUtil.marcJsonToRawMarc(marcJson));
assertEquals(RAW_MARC, MarcUtil.marcJsonToRawMarc(marcJson));
}

@Test
Expand Down
19 changes: 12 additions & 7 deletions ...ource-record-storage-server/src/test/java/org/folio/rest/impl/TestMarcRecordsApiTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,33 @@
package org.folio.rest.impl;

import java.util.Collections;

import com.fasterxml.jackson.databind.ObjectMapper;
import io.restassured.RestAssured;
import io.vertx.ext.unit.junit.VertxUnitRunner;
import org.apache.http.HttpStatus;
import org.folio.TestMocks;
import org.folio.TestUtil;
import org.folio.rest.jaxrs.model.RawRecord;
import org.folio.rest.jaxrs.model.TestMarcRecordsCollection;
import org.junit.Test;
import org.junit.runner.RunWith;

import io.restassured.RestAssured;
import io.vertx.ext.unit.junit.VertxUnitRunner;
import java.io.IOException;
import java.util.Collections;
import java.util.List;

@RunWith(VertxUnitRunner.class)
public class TestMarcRecordsApiTest extends AbstractRestVerticleTest {

private static final String POPULATE_TEST_MARK_RECORDS_PATH = "/source-storage/populate-test-marc-records";

@Test
public void shouldReturnNoContentOnPostRecordCollectionPassedInBody() {
public void shouldReturnNoContentOnPostRecordCollectionPassedInBody() throws IOException {
RawRecord rawRecord = new RawRecord().withContent(
new ObjectMapper().readValue(TestUtil.readFileFromPath(RAW_MARC_RECORD_CONTENT_SAMPLE_PATH), String.class));

RestAssured.given()
.spec(spec)
.body(new TestMarcRecordsCollection()
.withRawRecords(TestMocks.getRawRecords()))
.withRawRecords(List.of(rawRecord)))
.when()
.post(POPULATE_TEST_MARK_RECORDS_PATH)
.then()
Expand Down
8 changes: 0 additions & 8 deletions mod-source-record-storage-server/src/test/java/org/folio/services/RecordServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1527,7 +1527,6 @@ private void getMarcSourceRecordById(TestContext context, Record expected) {
context.fail(get.cause());
}
context.assertTrue(get.result().isPresent());
context.assertNotNull(get.result().get().getRawRecord());
context.assertNotNull(get.result().get().getParsedRecord());
compareSourceRecords(context, RecordDaoUtil.toSourceRecord(expected), get.result().get());
async.complete();
Expand Down Expand Up @@ -1794,15 +1793,8 @@ private void compareSourceRecords(TestContext context, SourceRecord expected, So
context.assertEquals(expected.getSnapshotId(), actual.getSnapshotId());
context.assertEquals(expected.getRecordType(), actual.getRecordType());
context.assertEquals(expected.getOrder(), actual.getOrder());
if (Objects.nonNull(expected.getRawRecord())) {
compareRawRecords(context, expected.getRawRecord(), actual.getRawRecord());
} else {
context.assertNull(actual.getRawRecord());
}
if (Objects.nonNull(expected.getParsedRecord())) {
compareParsedRecords(context, expected.getParsedRecord(), actual.getParsedRecord());
} else {
context.assertNull(actual.getRawRecord());
}
if (Objects.nonNull(expected.getAdditionalInfo())) {
compareAdditionalInfo(context, expected.getAdditionalInfo(), actual.getAdditionalInfo());
Expand Down
4 changes: 4 additions & 0 deletions ...orage-server/src/test/resources/mock/rawRecords/0f0fe962-d502-4a4f-9e74-7732bec94ee8.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"id": "0f0fe962-d502-4a4f-9e74-7732bec94ee8",
"content": "{\"leader\":\"02042nma a2200349 c 4500\",\"fields\":[{\"001\":\"inst000000000019\"},{\"003\":\"DE-601\"},{\"005\":\"20180122150003.0\"},{\"007\":\"cu\\\\uuu---uuuuu\"},{\"008\":\"180122s2017\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\000\\\\0\\\\eng\\\\d\"},{\"020\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"9781450351430\"},{\"9\":\"978-1-4503-5143-0\"}]}},{\"035\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"(DE-599)GBV1011273942\"}]}},{\"035\":\"1011273942\"},{\"040\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"b\":\"ger\"},{\"c\":\"GBVCP\"},{\"e\":\"rda\"}]}},{\"041\":{\"ind1\":\"0\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"eng\"}]}},{\"110\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"ACM Workshop on Millimeter Wave Networks and Sensing Systems\"},{\"n\":\"1.\"},{\"d\":\"2017\"},{\"c\":\"Snowbird, Utah\"},{\"e\":\"VerfasserIn\"},{\"4\":\"aut\"}]}},{\"245\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"MobiCom'17\"},{\"n\":\"5\"},{\"p\":\"mmNets'17, October 16, 2017, Snowbird, UT, USA / general chairs: Haitham Hassanieh (University of Illinois at Urbana Champaign, USA), Xinyu Zhang (University of California San Diego, USA)\"}]}},{\"246\":{\"ind1\":\"3\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"1st First ACM Workshop Millimeter Wave Networks Sensing Systems\"}]}},{\"336\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Text\"},{\"b\":\"txt\"},{\"2\":\"rdacontent\"}]}},{\"337\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Computermedien\"},{\"b\":\"c\"},{\"2\":\"rdamedia\"}]}},{\"338\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Computerdisk\"},{\"b\":\"cd\"},{\"2\":\"rdacarrier\"}]}},{\"700\":{\"ind1\":\"1\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Hassanieh, Haitham\"},{\"e\":\"VeranstalterIn\"},{\"4\":\"orm\"}]}},{\"700\":{\"ind1\":\"1\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Zhang, Xinyu\"},{\"e\":\"VeranstalterIn\"},{\"4\":\"orm\"}]}},{\"710\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"MobiCom\"},{\"n\":\"23.\"},{\"d\":\"2017\"},{\"c\":\"Snowbird, Utah\"},{\"e\":\"VeranstalterIn\"},{\"4\":\"orm\"}]}},{\"710\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Association for Computing Machinery\"},{\"b\":\"Special Interest Group on Mobility of Systems Users, Data, and Computing\"},{\"e\":\"SponsorIn\"},{\"4\":\"spn\"},{\"0\":\"(DE-601)499677137\"},{\"0\":\"(DE-588)10113390-X\"}]}},{\"711\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"ACM Workshop on Millimeter Wave Networks and Sensing Systems\"},{\"n\":\"1\"},{\"d\":\"2017.10.16\"},{\"c\":\"Snowbird, Utah\"}]}},{\"711\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"mmNets\"},{\"n\":\"1\"},{\"d\":\"2017.10.16\"},{\"c\":\"Snowbird, Utah\"}]}},{\"711\":{\"ind1\":\"2\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"Annual International Conference on Mobile Computing and Networking (ACM MobiCom)\"},{\"n\":\"23\"},{\"d\":\"2017.10.16-20\"},{\"c\":\"Snowbird, Utah\"}]}},{\"773\":{\"ind1\":\"0\",\"ind2\":\"\\\\\",\"subfields\":[{\"w\":\"(DE-601)1011270897\"},{\"t\":\"MobiCom'17, MobiCom'17, proceedings and co-located workshops of the 23rd Annual International Conference on Mobile Computing and Networking : October 16-20, 2017, Snowbird, UT, USA, MobiCom, Snowbird, Utah. - New York, NY : ACM, Association for Computing Machinery\"}]}},{\"900\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"a\":\"GBV\"},{\"b\":\"TIB/UB Hannover <89>\"}]}},{\"954\":{\"ind1\":\"\\\\\",\"ind2\":\"\\\\\",\"subfields\":[{\"0\":\"TIB/UB Hannover <89>\"},{\"a\":\"70\"},{\"b\":\"1743063695\"},{\"c\":\"01\"},{\"x\":\"0089\"}]}},{\"999\":{\"ind1\":\"f\",\"ind2\":\"f\",\"subfields\":[{\"s\":\"0f0fe962-d502-4a4f-9e74-7732bec94ee8\"},{\"i\":\"6b4ae089-e1ee-431f-af83-e1133f8e3da0\"}]}}]}"
}
Loading

0 comments on commit 94ae701

Please sign in to comment.