diff --git a/build.gradle b/build.gradle index 9b74489f14..7fee9cd114 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ plugins { ext { // Move to gradle.properties (but versions.json generation build depends on this right now) - neo4jVersion = '4.4.41' + neo4jVersion = '4.4.40' publicDir = "${project.rootDir}" neo4jVersionEffective = project.hasProperty("neo4jVersionOverride") ? project.getProperty("neo4jVersionOverride") : neo4jVersion neo4jDockerVersion = project.hasProperty("neo4jDockerVersionOverride") ? project.getProperty("neo4jDockerVersionOverride") : neo4jVersion diff --git a/core/src/test/java/apoc/export/arrow/ArrowTest.java b/core/src/test/java/apoc/export/arrow/ArrowTest.java index f3fb0bd293..c750404a19 100644 --- a/core/src/test/java/apoc/export/arrow/ArrowTest.java +++ b/core/src/test/java/apoc/export/arrow/ArrowTest.java @@ -30,25 +30,17 @@ import apoc.util.JsonUtil; import apoc.util.TestUtil; import com.fasterxml.jackson.core.JsonProcessingException; -import java.io.File; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.LongStream; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.neo4j.configuration.GraphDatabaseSettings; -import org.neo4j.graphdb.Result; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; +import java.io.File; + + public class ArrowTest { private static File directory = new File("target/arrow import"); @@ -65,75 +57,11 @@ public class ArrowTest { directory.toPath().toAbsolutePath()) .withSetting(ApocSettings.apoc_export_file_enabled, true); - public static final List> EXPECTED = List.of( - new HashMap<>() { - { - put("name", "Adam"); - put("bffSince", null); - put("", null); - put("", 0L); - put("age", 42L); - put("labels", List.of("User")); - put("male", true); - put("", null); - put("kids", List.of("Sam", "Anna", "Grace")); - put( - "place", - Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D)); - put("", null); - put("since", null); - put( - "born", - LocalDateTime.parse("2015-05-18T19:32:24.000") - .atOffset(ZoneOffset.UTC) - .toZonedDateTime()); - } - }, - new HashMap<>() { - { - put("name", "Jim"); - put("bffSince", null); - put("", null); - put("", 1L); - put("age", 42L); - put("labels", List.of("User")); - put("male", null); - put("", null); - put("kids", null); - put("place", null); - put("", null); - put("since", null); - put("born", null); - } - }, - new HashMap<>() { - { - put("name", null); - put("bffSince", "P5M1DT12H"); - put("", 0L); - put("", 0L); - put("age", null); - put("labels", null); - put("male", null); - put("", "KNOWS"); - put("kids", null); - put("place", null); - put("", 1L); - put("since", 1993L); - put("born", null); - } - }); + @BeforeClass public static void beforeClass() { - db.executeTransactionally( - "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); - TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, Graphs.class, Meta.class); - } - - @AfterClass - public static void teardown() { - db.shutdown(); + initDbCommon(db); } @Before @@ -142,21 +70,13 @@ public void before() { apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); } - private byte[] extractByteArray(Result result) { - return result.columnAs("byteArray").next(); - } - - private String extractFileName(Result result) { - return result.columnAs("file").next(); + @AfterClass + public static void teardown() { + db.shutdown(); } - private T readValue(String json, Class clazz) { - if (json == null) return null; - try { - return JsonUtil.OBJECT_MAPPER.readValue(json, clazz); - } catch (JsonProcessingException e) { - return null; - } + private byte[] extractByteArray(Result result) { + return result.columnAs("byteArray").next(); } @Test @@ -215,7 +135,7 @@ public void testFileRoundtripArrowQuery() { String file = db.executeTransactionally( "CALL apoc.export.arrow.query('query_test.arrow', $query) YIELD file", Map.of("query", returnQuery), - this::extractFileName); + ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; @@ -251,22 +171,7 @@ public void testStreamRoundtripArrowGraph() { // then final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); - } - - private List> getActual(Result result) { - return result.stream() - .map(m -> (Map) m.get("value")) - .map(m -> { - final Map newMap = new HashMap(m); - newMap.put("place", readValue((String) m.get("place"), Map.class)); - return newMap; - }) - .collect(Collectors.toList()); + testLoadArrow(db, query, Map.of("byteArray", byteArray)); } @Test @@ -277,15 +182,11 @@ public void testFileRoundtripArrowGraph() { + "CALL apoc.export.arrow.graph('graph_test.arrow', graph) YIELD file " + "RETURN file", Map.of(), - this::extractFileName); + ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("file", file), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("file", file)); } @Test @@ -310,26 +211,18 @@ private void testStreamRoundtripAllCommon() { // then final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("byteArray", byteArray)); } @Test public void testFileRoundtripArrowAll() { // given - when String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), this::extractFileName); + "CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("file", file), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("file", file)); } @Test @@ -365,7 +258,7 @@ public void testFileVolumeArrowAll() { String file = db.executeTransactionally( "CALL apoc.export.arrow.query('volume_test.arrow', 'MATCH (n:ArrowNode) RETURN n.id AS id') YIELD file ", Map.of(), - this::extractFileName); + ArrowTestUtil::extractFileName); final List expected = LongStream.range(0, 10000).mapToObj(l -> l).collect(Collectors.toList()); diff --git a/core/src/test/java/apoc/util/s3/S3TestUtil.java b/core/src/test/java/apoc/util/s3/S3TestUtil.java index 3fde37e95b..9e1013022b 100644 --- a/core/src/test/java/apoc/util/s3/S3TestUtil.java +++ b/core/src/test/java/apoc/util/s3/S3TestUtil.java @@ -82,5 +82,14 @@ public static void assertS3KeyEventually(Runnable runnable) { v -> v, 30L, TimeUnit.SECONDS); + } + + public static String removeRegionFromUrl(S3Container s3Container, String url) { + return url.replace(s3Container.getEndpointConfiguration().getSigningRegion() + ".", ""); + } + + public static String putToS3AndGetUrl(S3Container s3Container, String filename) { + String url = s3Container.putFile(filename); + return removeRegionFromUrl(s3Container, url); } } diff --git a/extended-it/src/test/java/apoc/azure/ArrowAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/ArrowAzureStorageTest.java new file mode 100644 index 0000000000..c1ede51b82 --- /dev/null +++ b/extended-it/src/test/java/apoc/azure/ArrowAzureStorageTest.java @@ -0,0 +1,62 @@ +package apoc.azure; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +@Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = putToAzureStorageAndGetUrl("test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = putToAzureStorageAndGetUrl("test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java b/extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java new file mode 100644 index 0000000000..40bc46c1d5 --- /dev/null +++ b/extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java @@ -0,0 +1,74 @@ +package apoc.azure; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.UUID; + +public class AzureStorageBaseTest { + + public static GenericContainer azuriteContainer; + public static BlobContainerClient containerClient; + + @BeforeClass + public static void setUp() throws Exception { + DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite"); + azuriteContainer = new GenericContainer<>(azuriteImg) + .withExposedPorts(10000); + + azuriteContainer.start(); + + var accountName = "devstoreaccount1"; + var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + var blobEndpoint = "http://%s:%d/%s".formatted(azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName); + var connectionString = "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;" + .formatted(accountName, accountKey, blobEndpoint); + + containerClient = new BlobContainerClientBuilder() + .connectionString(connectionString) + .containerName("test-container") + .buildClient(); + containerClient.create(); + } + + @AfterClass + public static void teardown() { + azuriteContainer.close(); + } + + public static String putToAzureStorageAndGetUrl(String url) { + try { + File file = new File(url); + byte[] content = FileUtils.readFileToByteArray(file); + + var blobClient = getBlobClient(content); + BlobSasPermission permission = new BlobSasPermission().setReadPermission(true); + OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1); + String sasToken = blobClient.generateSas(new BlobServiceSasSignatureValues(expiryTime, permission), new Context("Azure-Storage-Log-String-To-Sign", "true")); + return blobClient.getBlobUrl() + "?" + sasToken; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static BlobClient getBlobClient(byte[] content) { + var blobName = "blob-" + UUID.randomUUID(); + var blobClient = containerClient.getBlobClient(blobName); + blobClient.upload(new ByteArrayInputStream(content)); + return blobClient; + } + +} diff --git a/extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java new file mode 100644 index 0000000000..095324015e --- /dev/null +++ b/extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java @@ -0,0 +1,51 @@ +package apoc.azure; + +import apoc.load.Gexf; +import apoc.util.TestUtil; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.GexfTestUtil.testImportGexfCommon; + +public class ImportAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToAzureStorageAndGetUrl(fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } + + @Test + public void testImportGexf() { + TestUtil.registerProcedure(db, Gexf.class); + + String filename = EXTENDED_RESOURCES_PATH + "gexf/data.gexf"; + String url = putToAzureStorageAndGetUrl(filename); + testImportGexfCommon(db, url); + } +} diff --git a/extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java new file mode 100644 index 0000000000..b853b7a5e6 --- /dev/null +++ b/extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java @@ -0,0 +1,74 @@ +package apoc.azure; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJsonExtended; +import apoc.load.Xml; +import apoc.load.xls.LoadXls; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.util.ExtendedITUtil.testLoadXmlCommon; + + +public class LoadAzureStorageTest extends AzureStorageBaseTest { + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @BeforeClass + public static void setUp() throws Exception { + AzureStorageBaseTest.setUp(); + + TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + + @Test + public void testLoadCsv() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } + +} diff --git a/extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java new file mode 100644 index 0000000000..03395ccc82 --- /dev/null +++ b/extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java @@ -0,0 +1,72 @@ +package apoc.azure; + +import apoc.export.parquet.ParquetTestUtil; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.export.parquet.ParquetTest.MAPPING_ALL; +import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon; +import static apoc.export.parquet.ParquetTestUtil.beforeCommon; +import static apoc.export.parquet.ParquetTestUtil.testImportAllCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; +import static apoc.util.TestUtil.testResult; + +public class ParquetAzureStorageTest extends AzureStorageBaseTest { + + private final String EXPORT_FILENAME = "test_all.parquet"; + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void beforeClass() { + beforeClassCommon(db); + } + + @Before + public void before() { + beforeCommon(db); + } + + @Test + @Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core") + public void testFileRoundtripParquetAll() { + // given - when + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME); + String file = db.executeTransactionally("CALL apoc.export.parquet.all($url) YIELD file", + Map.of("url", url), + ParquetTestUtil::extractFileName); + + // then + final String query = "CALL apoc.load.parquet($file, $config) YIELD value " + + "RETURN value"; + + testResult(db, query, Map.of("file", file, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + } + @Test + public void testLoadParquet() { + String query = "CALL apoc.load.parquet($url, $config) YIELD value " + + "RETURN value"; + + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME); + testResult(db, query, Map.of("url", url, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + } + + @Test + public void testImportParquet() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + EXPORT_FILENAME); + + Map params = Map.of("file", url, "config", MAPPING_ALL); + testImportAllCommon(db, params); + } +} diff --git a/extended-it/src/test/java/apoc/gc/ArrowGoogleCloudStorageTest.java b/extended-it/src/test/java/apoc/gc/ArrowGoogleCloudStorageTest.java new file mode 100644 index 0000000000..333f778f16 --- /dev/null +++ b/extended-it/src/test/java/apoc/gc/ArrowGoogleCloudStorageTest.java @@ -0,0 +1,65 @@ +package apoc.gc; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +@Ignore("This test won't work until the Google Cloud files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension(); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = gcsUrl(gcs, "test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = gcsUrl(gcs, "test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/extended-it/src/test/java/apoc/gc/ImportGoogleCloudStorageTest.java b/extended-it/src/test/java/apoc/gc/ImportGoogleCloudStorageTest.java new file mode 100644 index 0000000000..134e164688 --- /dev/null +++ b/extended-it/src/test/java/apoc/gc/ImportGoogleCloudStorageTest.java @@ -0,0 +1,55 @@ +package apoc.gc; + +import apoc.load.Gexf; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.util.GexfTestUtil.testImportGexfCommon; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +public class ImportGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test_all.arrow", "/folder/test_all.arrow") + .withMountedResourceFile("gexf/data.gexf", "/folder/data.gexf"); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + + gcs.start(); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String url = gcsUrl(gcs, "test_all.arrow"); + testImportCommon(db, url, MAPPING_ALL); + } + + @Test + public void testImportGexf() { + TestUtil.registerProcedure(db, Gexf.class); + + String url = gcsUrl(gcs, "data.gexf"); + testImportGexfCommon(db, url); + } + +} diff --git a/extended-it/src/test/java/apoc/gc/LoadGoogleCloudStorageTest.java b/extended-it/src/test/java/apoc/gc/LoadGoogleCloudStorageTest.java new file mode 100644 index 0000000000..cb0b380655 --- /dev/null +++ b/extended-it/src/test/java/apoc/gc/LoadGoogleCloudStorageTest.java @@ -0,0 +1,126 @@ +package apoc.gc; + +import apoc.load.LoadCsv; +import apoc.load.LoadHtml; +import apoc.load.LoadJsonExtended; +import apoc.load.Xml; +import apoc.load.xls.LoadXls; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import apoc.util.Util; +import apoc.xml.XmlTestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.Result; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static apoc.load.LoadCsvTest.assertRow; +import static apoc.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class LoadGoogleCloudStorageTest { + + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test.csv", "/folder/test.csv") + .withMountedResourceFile("map.json", "/folder/map.json") + .withMountedResourceFile("xml/books.xml", "/folder/books.xml") + .withMountedResourceFile("load_test.xlsx", "/folder/load_test.xlsx") + .withMountedResourceFile("wikipedia.html", "/folder/wikipedia.html"); + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + + @BeforeClass + public static void setUp() throws Exception { + gcs.start(); + TestUtil.registerProcedure(db, LoadCsv.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class); + } + + @AfterClass + public static void tearDown() { + gcs.close(); + db.shutdown(); + } + + @Test + public void testLoadCsv() { + String url = gcsUrl(gcs, "test.csv"); + + testResult(db, "CALL apoc.load.csv($url)", map("url", url), (r) -> { + assertRow(r, "Selma", "8", 0L); + assertRow(r, "Rana", "11", 1L); + assertRow(r, "Selina", "18", 2L); + assertFalse("It should be the last record", r.hasNext()); + }); + } + + @Test + public void testLoadJSON() { + String url = gcsUrl(gcs, "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = gcsUrl(gcs, "books.xml"); + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + @Test + public void testLoadXls() { + String url = gcsUrl(gcs, "load_test.xlsx"); + testResult(db, "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", map("url",url), // 'file:load_test.xlsx' + (r) -> { + assertXlsRow(r,0L,"String","Test","Boolean",true,"Integer",2L,"Float",1.5d,"Array",asList(1L,2L,3L)); + assertFalse("Should not have another row",r.hasNext()); + }); + } + + @Test + public void testLoadHtml() { + String url = gcsUrl(gcs, "wikipedia.html"); + + Map query = map("links", "a[href]"); + + testCall(db, "CALL apoc.load.html($url,$query)", + map("url", url, "query", query), + row -> { + final List> actual = (List) ((Map) row.get("value")).get("links"); + assertEquals(106, actual.size()); + assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); + }); + } + + static void assertXlsRow(Result r, long lineNo, Object...data) { + Map row = r.next(); + Map map = map(data); + assertEquals(map, row.get("map")); + Map stringMap = new LinkedHashMap<>(map.size()); + map.forEach((k,v) -> stringMap.put(k,v == null ? null : v.toString())); + assertEquals(new ArrayList<>(map.values()), row.get("list")); + assertEquals(lineNo, row.get("lineNo")); + } +} diff --git a/extended-it/src/test/java/apoc/gc/ParquetGoogleCloudStorageTest.java b/extended-it/src/test/java/apoc/gc/ParquetGoogleCloudStorageTest.java new file mode 100644 index 0000000000..eef1e224e5 --- /dev/null +++ b/extended-it/src/test/java/apoc/gc/ParquetGoogleCloudStorageTest.java @@ -0,0 +1,87 @@ +package apoc.gc; + +import apoc.export.parquet.ParquetTestUtil; +import apoc.util.GoogleCloudStorageContainerExtension; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.parquet.ParquetTest.MAPPING_ALL; +import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon; +import static apoc.export.parquet.ParquetTestUtil.testImportAllCommon; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; +import static apoc.util.TestUtil.testResult; + +public class ParquetGoogleCloudStorageTest { + + private static final String FILENAME = "test_all.parquet"; + + public static GoogleCloudStorageContainerExtension gcs; + + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + + beforeClassCommon(db); + + gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile(FILENAME, "/folder/" + FILENAME); + + gcs.start(); + } + + @AfterClass + public static void tearDown() { + gcs.close(); + db.shutdown(); + } + + @Test + @Ignore("This test won't work until the Google Cloud files will be correctly handled via FileUtils, placed in APOC Core") + public void testFileRoundtripParquetAll() { + // given - when + String url = gcsUrl(gcs, FILENAME); + String file = db.executeTransactionally("CALL apoc.export.parquet.all($url) YIELD file", + Map.of("url", url), + ParquetTestUtil::extractFileName); + + // then + final String query = "CALL apoc.load.parquet($file, $config) YIELD value " + + "RETURN value"; + + testResult(db, query, Map.of("file", file, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + } + + @Test + public void testLoadParquet() { + String query = "CALL apoc.load.parquet($url, $config) YIELD value " + + "RETURN value"; + + String url = gcsUrl(gcs, FILENAME); + testResult(db, query, Map.of("url", url, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + } + + @Test + public void testImportParquet() { + String url = gcsUrl(gcs, FILENAME); + + Map params = Map.of("file", url, "config", MAPPING_ALL); + testImportAllCommon(db, params); + } +} diff --git a/extended-it/src/test/java/apoc/s3/ArrowS3Test.java b/extended-it/src/test/java/apoc/s3/ArrowS3Test.java new file mode 100644 index 0000000000..bd46acb658 --- /dev/null +++ b/extended-it/src/test/java/apoc/s3/ArrowS3Test.java @@ -0,0 +1,60 @@ +package apoc.s3; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +public class ArrowS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = s3Container.getUrl("test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = s3Container.getUrl("test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/extended-it/src/test/java/apoc/s3/ImportS3Test.java b/extended-it/src/test/java/apoc/s3/ImportS3Test.java new file mode 100644 index 0000000000..1256ae3099 --- /dev/null +++ b/extended-it/src/test/java/apoc/s3/ImportS3Test.java @@ -0,0 +1,62 @@ +package apoc.s3; + +import apoc.load.Gexf; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.io.File; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.ARROW_BASE_FOLDER; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.GexfTestUtil.testImportGexfCommon; +import static apoc.util.s3.S3Util.putToS3AndGetUrl; + +public class ImportS3Test extends S3BaseTest { + private static File directory = new File(ARROW_BASE_FOLDER); + static { //noinspection ResultOfMethodCallIgnored + directory.mkdirs(); + } + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); + + @BeforeClass + public static void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToS3AndGetUrl(s3Container, fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } + + @Test + public void testImportGexf() { + TestUtil.registerProcedure(db, Gexf.class); + + String filename = EXTENDED_RESOURCES_PATH + "gexf/data.gexf"; + String url = putToS3AndGetUrl(s3Container, filename); + testImportGexfCommon(db, url); + } +} diff --git a/extended-it/src/test/java/apoc/s3/LoadS3Test.java b/extended-it/src/test/java/apoc/s3/LoadS3Test.java new file mode 100644 index 0000000000..5a91e5c1b6 --- /dev/null +++ b/extended-it/src/test/java/apoc/s3/LoadS3Test.java @@ -0,0 +1,73 @@ +package apoc.s3; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadJsonExtended; +import apoc.load.Xml; +import apoc.load.xls.LoadXls; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.util.ExtendedITUtil.testLoadXmlCommon; +import static apoc.util.s3.S3Util.putToS3AndGetUrl; + +public class LoadS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @Before + public void setUp() throws Exception { + TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + @Test + public void testLoadCsv() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } + +} diff --git a/extended-it/src/test/java/apoc/s3/ParquetS3Test.java b/extended-it/src/test/java/apoc/s3/ParquetS3Test.java new file mode 100644 index 0000000000..2c5374d196 --- /dev/null +++ b/extended-it/src/test/java/apoc/s3/ParquetS3Test.java @@ -0,0 +1,111 @@ +package apoc.s3; + +import apoc.export.parquet.ParquetTestUtil; +import apoc.util.collection.Iterators; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.io.File; +import java.util.Collections; +import java.util.Map; + +import static apoc.export.parquet.ParquetTest.MAPPING_ALL; +import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon; +import static apoc.export.parquet.ParquetTestUtil.beforeCommon; +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static apoc.util.s3.S3Util.putToS3AndGetUrl; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ParquetS3Test extends S3BaseTest { + + private final String EXPORT_FILENAME = "test_all.parquet"; + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void beforeClass() { + beforeClassCommon(db); + } + + @Before + public void before() { + beforeCommon(db); + } + + @Test + public void testFileRoundtripParquetAll() { + // given - when + String file = db.executeTransactionally("CALL apoc.export.parquet.all('test_all.parquet') YIELD file", + Map.of(), + ParquetTestUtil::extractFileName); + + // then + final String query = "CALL apoc.load.parquet($file, $config) YIELD value " + + "RETURN value"; + + testResult(db, query, Map.of("file", file, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + + deleteFile(file.replace("file://", "")); + } + + @Test + public void testFileRoundtripParquetAllFromS3Url() { + // given - when + String filename = exportToParquetFile(EXPORT_FILENAME); + String url = putToS3AndGetUrl(s3Container, filename); + + // then + final String query = "CALL apoc.load.parquet($url, $config) YIELD value " + + "RETURN value"; + + testResult(db, query, Map.of("url", url, "config", MAPPING_ALL), + ParquetTestUtil::roundtripLoadAllAssertions); + + deleteFile(filename); + } + + @Test + public void testImportParquetFromS3Url() { + String filename = exportToParquetFile(EXPORT_FILENAME); + String url = putToS3AndGetUrl(s3Container, filename); + + db.executeTransactionally("MATCH (n) DETACH DELETE n"); + Long count = db.executeTransactionally("MATCH (n) RETURN count(n) AS count", Collections.emptyMap(), + result -> Iterators.single(result.columnAs("count"))); + assertEquals(0L, count.longValue()); + + final String query = "CALL apoc.import.parquet($file, $config)"; + Map params = Map.of("file", url, "config", MAPPING_ALL); + testCall(db, query, params, + r -> { + assertEquals(4L, r.get("nodes")); + assertEquals(1L, r.get("relationships")); + }); + + deleteFile(filename); + } + + private String exportToParquetFile(String filename) { + String file = db.executeTransactionally("CALL apoc.export.parquet.all($filename) YIELD file", + Map.of("filename", filename), + ParquetTestUtil::extractFileName); + return file.replace("file://", ""); + } + + private void deleteFile(String filename) { + File fileToDelete = new File(filename); + if (fileToDelete.exists()) { + boolean deleted = fileToDelete.delete(); + assertTrue(deleted); + } + } +} diff --git a/extended-it/src/test/java/apoc/util/ExtendedITUtil.java b/extended-it/src/test/java/apoc/util/ExtendedITUtil.java new file mode 100644 index 0000000000..21c5b4d2b3 --- /dev/null +++ b/extended-it/src/test/java/apoc/util/ExtendedITUtil.java @@ -0,0 +1,32 @@ +package apoc.util; + +import apoc.xml.XmlTestUtils; +import org.junit.Assert; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.GraphDatabaseService; + +import static apoc.util.ExtendedTestUtil.RESOURCES_PATH; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + +public class ExtendedITUtil { + public static final String EXTENDED_PATH = "../extended/"; + public static final String EXTENDED_RESOURCES_PATH = EXTENDED_PATH + RESOURCES_PATH; + + public static void testLoadXmlCommon(GraphDatabaseService db, String url) { + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + public static void testLoadJsonCommon(GraphDatabaseService db, String url) { + testCall(db, "CALL apoc.load.jsonParams($url, null, null)", + map("url", url), + (row) -> { + assertEquals(map("foo",asList(1L,2L,3L)), row.get("value")); + }); + } +} diff --git a/extended-it/src/test/java/apoc/util/GoogleCloudStorageContainerExtension.java b/extended-it/src/test/java/apoc/util/GoogleCloudStorageContainerExtension.java new file mode 100644 index 0000000000..9845d96efe --- /dev/null +++ b/extended-it/src/test/java/apoc/util/GoogleCloudStorageContainerExtension.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.util; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; + +import static java.net.HttpURLConnection.HTTP_OK; + +import com.google.cloud.storage.Bucket; + +public class GoogleCloudStorageContainerExtension extends GenericContainer { + + public GoogleCloudStorageContainerExtension() { + super("fsouza/fake-gcs-server:latest"); + this.withCommand("-scheme http"); + + setWaitStrategy(new HttpWaitStrategy() + .forPath("/storage/v1/b") + .forPort(4443) + .forStatusCodeMatching(response -> response == HTTP_OK)); + + addExposedPort(4443); + } + + public GoogleCloudStorageContainerExtension withMountedResourceFile(String resourceFilePath, String gcsPath) { + this.withClasspathResourceMapping(resourceFilePath, "/data" + gcsPath, BindMode.READ_ONLY); + return this; + } + + public static String gcsUrl(GoogleCloudStorageContainerExtension gcs, String file) { + String path = "b/folder/o/%s?alt=media".formatted(file); + return String.format("http://%s:%d/storage/v1/%s", gcs.getContainerIpAddress(), gcs.getMappedPort(4443), path); + } +} \ No newline at end of file diff --git a/extended/src/test/java/apoc/load/GexfTest.java b/extended/src/test/java/apoc/load/GexfTest.java new file mode 100644 index 0000000000..bc7dd02e37 --- /dev/null +++ b/extended/src/test/java/apoc/load/GexfTest.java @@ -0,0 +1,111 @@ +package apoc.load; + +import apoc.util.TestUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.graphdb.Relationship; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.List; +import java.util.Map; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.util.ExtendedTestUtil.assertRelationship; +import static apoc.util.GexfTestUtil.testImportGexfCommon; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class GexfTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void setup() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + TestUtil.registerProcedure(db, Gexf.class); + } + + @After + public void tearDown() { + db.shutdown(); + } + + @Test + public void testLoadGexf() { + final String file = ClassLoader.getSystemResource("gexf/single-node.gexf").toString(); + testCall( + db, + "CALL apoc.load.gexf($file)", + Map.of("file", file), + (row) -> { + Map value = (Map) row.get("value"); + String expected = "{_type=gexf, _children=[{_type=graph, defaultedgetype=directed, _children=[{_type=nodes, _children=[{_type=node, _children=[{_type=attvalues, _children=[{_type=attvalue, for=0, value=http://gephi.org}]}], id=0, label=bar}]}]}], version=1.2}"; + assertEquals(expected, value.toString()); + }); + } + + @Test + public void testImportGexf() { + final String file = ClassLoader.getSystemResource("gexf/data.gexf").toString(); + testImportGexfCommon(db, file); + } + + @Test + public void testImportGexfWithStoreNodeIds() { + final String file = ClassLoader.getSystemResource("gexf/single-node.gexf").toString(); + TestUtil.testCall( + db, + "CALL apoc.import.gexf($file, {storeNodeIds: true})", + map("file", file), + (r) -> { + assertEquals("gexf", r.get("format")); + assertEquals(1L, r.get("nodes")); + }); + + Map props = TestUtil.singleResultFirstColumn(db, "MATCH (n) RETURN properties(n) AS props"); + assertEquals("http://gephi.org", props.get("0")); + assertTrue( props.containsKey("id") ); + } + + @Test + public void testImportGexfWithDefaultRelationshipTypeSourceAndTargetConfigs() { + String defaultRelType = "TEST_DEFAULT"; + final String file = ClassLoader.getSystemResource("gexf/single-rel.gexf").toString(); + + db.executeTransactionally("CREATE (:Foo {startId: 'start'})"); + db.executeTransactionally("CREATE (:Bar {endId: 'end'})"); + + TestUtil.testCall( + db, + "CALL apoc.import.gexf($file, {defaultRelationshipType: $defaultRelType, source: $source, target: $target})", + map("file", file, + "defaultRelType", defaultRelType, + "source", map("label", "Foo", "id", "startId"), + "target", map("label", "Bar", "id", "endId") + ), + (r) -> { + assertEquals("gexf", r.get("format")); + assertEquals(1L, r.get("relationships")); + }); + + TestUtil.testCall(db, "MATCH ()-[rel]->() RETURN rel", r -> { + Relationship rel = (Relationship) r.get("rel"); + assertRelationship(rel, defaultRelType, + Map.of(), + List.of("Foo"), + Map.of("startId", "start"), + List.of("Bar"), + Map.of("endId", "end") + ); + }); + } +} diff --git a/full-it/build.gradle b/full-it/build.gradle index d7936fe069..51a201e05f 100644 --- a/full-it/build.gradle +++ b/full-it/build.gradle @@ -13,6 +13,10 @@ dependencies { testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.770' testImplementation group: 'org.xmlunit', name: 'xmlunit-core', version: '2.9.1' + testImplementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1' + testImplementation group: 'org.apache.poi', name: 'poi', version: '5.1.0' + testImplementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.1.0' + testImplementation 'com.azure:azure-storage-blob:12.22.0' configurations.all { exclude group: 'org.slf4j', module: 'slf4j-nop' diff --git a/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java new file mode 100644 index 0000000000..ca536beee7 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java @@ -0,0 +1,62 @@ +package apoc.full.it.azure; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +@Ignore("This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = putToAzureStorageAndGetUrl("test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = putToAzureStorageAndGetUrl("test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java b/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java new file mode 100644 index 0000000000..a5bf9beba0 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java @@ -0,0 +1,75 @@ +package apoc.full.it.azure; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.UUID; + +public class AzureStorageBaseTest { + + public static GenericContainer azuriteContainer; + public static BlobContainerClient containerClient; + + @BeforeClass + public static void setUp() throws Exception { + DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite"); + azuriteContainer = new GenericContainer<>(azuriteImg) + .withExposedPorts(10000); + + azuriteContainer.start(); + + var accountName = "devstoreaccount1"; + var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + var blobEndpoint = String.format("http://%s:%d/%s", azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName); + var connectionString = String.format( + "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;", + accountName, accountKey, blobEndpoint); + + containerClient = new BlobContainerClientBuilder() + .connectionString(connectionString) + .containerName("test-container") + .buildClient(); + containerClient.create(); + } + + @AfterClass + public static void teardown() { + azuriteContainer.close(); + } + + public static String putToAzureStorageAndGetUrl(String url) { + try { + File file = new File(url); + byte[] content = FileUtils.readFileToByteArray(file); + + var blobClient = getBlobClient(content); + BlobSasPermission permission = new BlobSasPermission().setReadPermission(true); + OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1); + String sasToken = blobClient.generateSas(new BlobServiceSasSignatureValues(expiryTime, permission), new Context("Azure-Storage-Log-String-To-Sign", "true")); + return blobClient.getBlobUrl() + "?" + sasToken; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static BlobClient getBlobClient(byte[] content) { + var blobName = "blob-" + UUID.randomUUID(); + var blobClient = containerClient.getBlobClient(blobName); + blobClient.upload(new ByteArrayInputStream(content)); + return blobClient; + } + +} diff --git a/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java new file mode 100644 index 0000000000..6664b38f63 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java @@ -0,0 +1,39 @@ +package apoc.full.it.azure; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; + +public class ImportAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToAzureStorageAndGetUrl(fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java new file mode 100644 index 0000000000..36299df0ca --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java @@ -0,0 +1,77 @@ +package apoc.full.it.azure; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJsonExtended; +import apoc.load.Xml; +import apoc.load.xls.LoadXls; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.full.it.util.ExtendedITUtil.testLoadXmlCommon; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon; +import static apoc.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.util.ExtendedITUtil.testLoadXmlCommon; + + +public class LoadAzureStorageTest extends AzureStorageBaseTest { + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseInternalSettings.enable_experimental_cypher_versions, true); + + @BeforeClass + public static void setUp() throws Exception { + AzureStorageBaseTest.setUp(); + + TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJsonExtended.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + + @Test + public void testLoadCsv() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } + +} diff --git a/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java new file mode 100644 index 0000000000..f2461b9603 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java @@ -0,0 +1,64 @@ +package apoc.full.it.gc; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +@Ignore("This test won't work until the Google Cloud files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension(); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = gcsUrl(gcs, "test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = gcsUrl(gcs, "test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java new file mode 100644 index 0000000000..08e5555415 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java @@ -0,0 +1,47 @@ +package apoc.full.it.gc; + +import apoc.load.Gexf; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.util.GexfTestUtil.testImportGexfCommon; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +public class ImportGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test_all.arrow", "/folder/test_all.arrow") + .withMountedResourceFile("gexf/data.gexf", "/folder/data.gexf"); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + + gcs.start(); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String url = gcsUrl(gcs, "test_all.arrow"); + testImportCommon(db, url, MAPPING_ALL); + } + +} diff --git a/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java new file mode 100644 index 0000000000..3b8590aa76 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java @@ -0,0 +1,124 @@ +package apoc.full.it.gc; + +import apoc.load.LoadCsv; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadXls; +import apoc.load.Xml; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import apoc.util.Util; +import apoc.xml.XmlTestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.Result; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.load.LoadCsvTest.assertRow; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class LoadGoogleCloudStorageTest { + + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test.csv", "/folder/test.csv") + .withMountedResourceFile("map.json", "/folder/map.json") + .withMountedResourceFile("xml/books.xml", "/folder/books.xml") + .withMountedResourceFile("load_test.xlsx", "/folder/load_test.xlsx") + .withMountedResourceFile("wikipedia.html", "/folder/wikipedia.html"); + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + + @BeforeClass + public static void setUp() throws Exception { + gcs.start(); + TestUtil.registerProcedure(db, LoadCsv.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class); + } + + @AfterClass + public static void tearDown() { + gcs.close(); + db.shutdown(); + } + + @Test + public void testLoadCsv() { + String url = gcsUrl(gcs, "test.csv"); + + testResult(db, "CALL apoc.load.csv($url)", map("url", url), (r) -> { + assertRow(r, "Selma", "8", 0L); + assertRow(r, "Rana", "11", 1L); + assertRow(r, "Selina", "18", 2L); + assertFalse("It should be the last record", r.hasNext()); + }); + } + + @Test + public void testLoadJSON() { + String url = gcsUrl(gcs, "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = gcsUrl(gcs, "books.xml"); + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + @Test + public void testLoadXls() { + String url = gcsUrl(gcs, "load_test.xlsx"); + testResult(db, "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", map("url",url), // 'file:load_test.xlsx' + (r) -> { + assertXlsRow(r,0L,"String","Test","Boolean",true,"Integer",2L,"Float",1.5d,"Array",asList(1L,2L,3L)); + assertFalse("Should not have another row",r.hasNext()); + }); + } + + @Test + public void testLoadHtml() { + String url = gcsUrl(gcs, "wikipedia.html"); + + Map query = map("links", "a[href]"); + + testCall(db, "CALL apoc.load.html($url,$query)", + map("url", url, "query", query), + row -> { + final List> actual = (List) ((Map) row.get("value")).get("links"); + assertEquals(106, actual.size()); + assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); + }); + } + + static void assertXlsRow(Result r, long lineNo, Object...data) { + Map row = r.next(); + Map map = map(data); + assertEquals(map, row.get("map")); + Map stringMap = new LinkedHashMap<>(map.size()); + map.forEach((k,v) -> stringMap.put(k,v == null ? null : v.toString())); + assertEquals(new ArrayList<>(map.values()), row.get("list")); + assertEquals(lineNo, row.get("lineNo")); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java b/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java new file mode 100644 index 0000000000..7a312a8521 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java @@ -0,0 +1,59 @@ +package apoc.full.it.s3; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +public class ArrowS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = s3Container.getUrl("test_all.arrow"); + + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = s3Container.getUrl("test_all_import.arrow"); + String file = db.executeTransactionally("CALL apoc.export.arrow.all($url) YIELD file", + Map.of("url", url), + ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java b/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java new file mode 100644 index 0000000000..d926b81ee8 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java @@ -0,0 +1,50 @@ +package apoc.full.it.s3; + +import apoc.util.s3.S3BaseTest; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.io.File; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.ARROW_BASE_FOLDER; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.s3.S3TestUtil.putToS3AndGetUrl; + +public class ImportS3Test extends S3BaseTest { + private static File directory = new File(ARROW_BASE_FOLDER); + static { //noinspection ResultOfMethodCallIgnored + directory.mkdirs(); + } + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting(GraphDatabaseSettings.load_csv_file_url_root, directory.toPath().toAbsolutePath()); + + @BeforeClass + public static void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToS3AndGetUrl(s3Container, fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java b/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java new file mode 100644 index 0000000000..3441b19268 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java @@ -0,0 +1,70 @@ +package apoc.full.it.s3; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadXls; +import apoc.load.Xml; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.full.it.util.ExtendedITUtil.testLoadXmlCommon; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.LoadXlsTest.testLoadXlsCommon; +import static apoc.util.s3.S3TestUtil.putToS3AndGetUrl; + +public class LoadS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void setUp() throws Exception { + TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + @Test + public void testLoadCsv() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } + +} diff --git a/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java b/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java new file mode 100644 index 0000000000..0efa009b68 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java @@ -0,0 +1,32 @@ +package apoc.full.it.util; + +import apoc.xml.XmlTestUtils; +import org.junit.Assert; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.GraphDatabaseService; + +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + +public class ExtendedITUtil { + public static String RESOURCES_PATH = "src/test/resources/"; + public static final String EXTENDED_PATH = "../full/"; + public static final String EXTENDED_RESOURCES_PATH = EXTENDED_PATH + RESOURCES_PATH; + + public static void testLoadXmlCommon(GraphDatabaseService db, String url) { + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + public static void testLoadJsonCommon(GraphDatabaseService db, String url) { + testCall(db, "CALL apoc.load.jsonParams($url, null, null)", + map("url", url), + (row) -> { + assertEquals(map("foo",asList(1L,2L,3L)), row.get("value")); + }); + } +} diff --git a/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java b/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java new file mode 100644 index 0000000000..a29b20313a --- /dev/null +++ b/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java @@ -0,0 +1,210 @@ +package apoc.export.arrow; + +import apoc.graph.Graphs; +import apoc.load.LoadArrow; +import apoc.meta.Meta; +import apoc.util.JsonUtil; +import apoc.util.TestUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; +import org.neo4j.kernel.impl.util.ValueUtils; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.values.AnyValue; +import org.neo4j.values.storable.DurationValue; +import org.neo4j.values.storable.LocalDateTimeValue; +import org.neo4j.values.storable.PointValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ArrowTestUtil { + public static String ARROW_BASE_FOLDER = "target/arrowImport"; + + public static final List> EXPECTED = List.of( + new HashMap<>() { + { + put("name", "Adam"); + put("bffSince", null); + put("", null); + put("", 0L); + put("age", 42L); + put("labels", List.of("User")); + put("male", true); + put("", null); + put("kids", List.of("Sam", "Anna", "Grace")); + put( + "place", + Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D)); + put("", null); + put("since", null); + put( + "born", + LocalDateTime.parse("2015-05-18T19:32:24.000") + .atOffset(ZoneOffset.UTC) + .toZonedDateTime()); + } + }, + new HashMap<>() { + { + put("name", "Jim"); + put("bffSince", null); + put("", null); + put("", 1L); + put("age", 42L); + put("labels", List.of("User")); + put("male", null); + put("", null); + put("kids", null); + put("place", null); + put("", null); + put("since", null); + put("born", null); + } + }, + new HashMap<>() { + { + put("name", null); + put("bffSince", "P5M1DT12H"); + put("", 0L); + put("", 0L); + put("age", null); + put("labels", null); + put("male", null); + put("", "KNOWS"); + put("kids", null); + put("place", null); + put("", 1L); + put("since", 1993L); + put("born", null); + } + }); + + + public static void initDbCommon(GraphDatabaseService db) { + db.executeTransactionally( + "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); + TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, ImportArrow.class, Graphs.class, Meta.class); + } + + public static final Map MAPPING_ALL = Map.of("mapping", + Map.of("bffSince", "Duration", "place", "Point", "listInt", "LongArray", "born", "LocalDateTime") + ); + + public static void createNodesForImportTests(DbmsRule db) { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + } + + public static void testLoadArrow(GraphDatabaseService db, String query, Map params) { + db.executeTransactionally(query, params, result -> { + final List> actual = getActual(result); + assertEquals(EXPECTED, actual); + return null; + }); + } + + private static List> getActual(Result result) { + return result.stream() + .map(m -> (Map) m.get("value")) + .map(m -> { + final Map newMap = new HashMap(m); + newMap.put("place", readValue((String) m.get("place"), Map.class)); + return newMap; + }) + .collect(Collectors.toList()); + } + + private static T readValue(String json, Class clazz) { + if (json == null) return null; + try { + return JsonUtil.OBJECT_MAPPER.readValue(json, clazz); + } catch (JsonProcessingException e) { + return null; + } + } + + public static void testImportCommon(DbmsRule db, Object file, Map config) { + // then + Map params = Map.of("file", file, "config", config); + + // remove current data + db.executeTransactionally("MATCH (n) DETACH DELETE n"); + + final String query = "CALL apoc.import.arrow($file, $config)"; + testCall(db, query, params, + r -> { + assertEquals(4L, r.get("nodes")); + assertEquals(1L, r.get("relationships")); + }); + + testCall(db, "MATCH (start:User)-[rel:KNOWS]->(end:User) RETURN start, rel, end", r -> { + Node start = (Node) r.get("start"); + assertFirstUserNodeProps(start.getAllProperties()); + Node end = (Node) r.get("end"); + assertSecondUserNodeProps(end.getAllProperties()); + Relationship rel = (Relationship) r.get("rel"); + assertRelationshipProps(rel.getAllProperties()); + }); + + testResult(db, "MATCH (m:Another) RETURN m", r -> { + ResourceIterator m = r.columnAs("m"); + Node node = m.next(); + assertFirstAnotherNodeProps(node.getAllProperties()); + node = m.next(); + assertSecondAnotherNodeProps(node.getAllProperties()); + assertFalse(r.hasNext()); + }); + } + + public static String extractFileName(Result result) { + return result.columnAs("file").next(); + } + + public static byte[] extractByteArray(Result result) { + return result.columnAs("value").next(); + } + + public static void assertFirstUserNodeProps(Map props) { + assertEquals("Adam", props.get("name")); + assertEquals(42L, props.get("age")); + assertEquals( true, props.get("male")); + assertArrayEquals(new String[] { "Sam", "Anna", "Grace" }, (String[]) props.get("kids")); + Map latitude = Map.of("latitude", 13.1D, "longitude", 33.46789D, "height", 100.0D); + assertEquals(PointValue.fromMap(VirtualValues.map(latitude.keySet().toArray(new String[0]), latitude.values().stream().map(ValueUtils::of).toArray(AnyValue[]::new))), + props.get("place")); + assertEquals(LocalDateTimeValue.parse("2015-05-18T19:32:24.000").asObject(), props.get("born")); + } + + public static void assertSecondUserNodeProps(Map props) { + assertEquals( "Jim", props.get("name")); + assertEquals(42L, props.get("age")); + } + + public static void assertFirstAnotherNodeProps(Map map) { + assertEquals(1L, map.get("foo")); + assertArrayEquals(new long[] {1L, 2L}, (long[]) map.get("listInt")); + } + + public static void assertSecondAnotherNodeProps(Map map) { + assertEquals("Sam", map.get("bar")); + } + + public static void assertRelationshipProps(Map props) { + assertEquals(DurationValue.parse("P5M1DT12H"), props.get("bffSince")); + assertEquals(1993L, props.get("since")); + } +} diff --git a/full/src/test/java/apoc/export/arrow/ImportArrowTest.java b/full/src/test/java/apoc/export/arrow/ImportArrowTest.java index 59206a37c4..217a284a59 100644 --- a/full/src/test/java/apoc/export/arrow/ImportArrowTest.java +++ b/full/src/test/java/apoc/export/arrow/ImportArrowTest.java @@ -19,22 +19,21 @@ import org.junit.ClassRule; import org.junit.Test; import org.neo4j.configuration.GraphDatabaseSettings; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.ResourceIterator; -import org.neo4j.graphdb.Result; -import org.neo4j.kernel.impl.util.ValueUtils; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import org.neo4j.values.AnyValue; -import org.neo4j.values.storable.DurationValue; -import org.neo4j.values.storable.LocalDateTimeValue; -import org.neo4j.values.storable.PointValue; -import org.neo4j.values.virtual.VirtualValues; -public class ImportArrowTest { - private static File directory = new File("target/arrowImport"); +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.ARROW_BASE_FOLDER; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +public class ImportArrowTest { + private static File directory = new File(ARROW_BASE_FOLDER); static { //noinspection ResultOfMethodCallIgnored directory.mkdirs(); } @@ -58,8 +57,7 @@ public static void beforeClass() { public void before() { db.executeTransactionally("MATCH (n) DETACH DELETE n"); - db.executeTransactionally( - "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace', 'Qwe'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); + db.executeTransactionally("CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); @@ -68,100 +66,30 @@ public void before() { @Test public void testStreamRoundtripImportArrowAll() { - final byte[] bytes = - db.executeTransactionally("CALL apoc.export.arrow.stream.all", Map.of(), this::extractByteArray); + final byte[] bytes = db.executeTransactionally("CALL apoc.export.arrow.stream.all", + Map.of(), + ArrowTestUtil::extractByteArray); - testImportCommon(bytes, MAPPING_ALL); + testImportCommon(db, bytes, MAPPING_ALL); } @Test public void testFileRoundtripImportArrowAll() { - String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), this::extractFileName); - - testImportCommon(file, MAPPING_ALL); + String file = db.executeTransactionally("CALL apoc.export.arrow.all('test_all.arrow') YIELD file", + Map.of(), + ArrowTestUtil::extractFileName); + + testImportCommon(db, file, MAPPING_ALL); } @Test public void testFileRoundtripImportArrowAllWithSmallBatchSize() { - String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), this::extractFileName); + String file = db.executeTransactionally("CALL apoc.export.arrow.all('test_all.arrow') YIELD file", + Map.of(), + ArrowTestUtil::extractFileName); Map config = new HashMap<>(MAPPING_ALL); config.put("batchSize", 1); - testImportCommon(file, config); - } - - private void testImportCommon(Object file, Map config) { - // then - Map params = Map.of("file", file, "config", config); - - // remove current data - db.executeTransactionally("MATCH (n) DETACH DELETE n"); - - final String query = "CALL apoc.import.arrow($file, $config)"; - testCall(db, query, params, r -> { - assertEquals(4L, r.get("nodes")); - assertEquals(1L, r.get("relationships")); - }); - - testCall(db, "MATCH (start:User)-[rel:KNOWS]->(end:User) RETURN start, rel, end", r -> { - Node start = (Node) r.get("start"); - assertFirstUserNodeProps(start.getAllProperties()); - Node end = (Node) r.get("end"); - assertSecondUserNodeProps(end.getAllProperties()); - Relationship rel = (Relationship) r.get("rel"); - assertRelationshipProps(rel.getAllProperties()); - }); - - testResult(db, "MATCH (m:Another) RETURN m", r -> { - ResourceIterator m = r.columnAs("m"); - Node node = m.next(); - assertFirstAnotherNodeProps(node.getAllProperties()); - node = m.next(); - assertSecondAnotherNodeProps(node.getAllProperties()); - assertFalse(r.hasNext()); - }); - } - - private String extractFileName(Result result) { - return result.columnAs("file").next(); - } - - private byte[] extractByteArray(Result result) { - return result.columnAs("value").next(); - } - - private static void assertFirstUserNodeProps(Map props) { - assertEquals("Adam", props.get("name")); - assertEquals(42L, props.get("age")); - assertEquals(true, props.get("male")); - assertArrayEquals(new String[] {"Sam", "Anna", "Grace", "Qwe"}, (String[]) props.get("kids")); - Map latitude = Map.of("latitude", 13.1D, "longitude", 33.46789D, "height", 100.0D); - assertEquals( - PointValue.fromMap(VirtualValues.map( - latitude.keySet().toArray(new String[0]), - latitude.values().stream().map(ValueUtils::of).toArray(AnyValue[]::new))), - props.get("place")); - assertEquals(LocalDateTimeValue.parse("2015-05-18T19:32:24.000").asObject(), props.get("born")); - } - - private static void assertSecondUserNodeProps(Map props) { - assertEquals("Jim", props.get("name")); - assertEquals(42L, props.get("age")); - } - - private static void assertFirstAnotherNodeProps(Map map) { - assertEquals(1L, map.get("foo")); - assertArrayEquals(new long[] {1L, 2L}, (long[]) map.get("listInt")); - } - - private static void assertSecondAnotherNodeProps(Map map) { - assertEquals("Sam", map.get("bar")); - } - - private static void assertRelationshipProps(Map props) { - assertEquals(DurationValue.parse("P5M1DT12H"), props.get("bffSince")); - assertEquals(1993L, props.get("since")); + testImportCommon(db, file, config); } } diff --git a/full/src/test/java/apoc/export/csv/ExportXlsTest.java b/full/src/test/java/apoc/export/csv/ExportXlsTest.java index 4c34091a81..02a5d7b39e 100644 --- a/full/src/test/java/apoc/export/csv/ExportXlsTest.java +++ b/full/src/test/java/apoc/export/csv/ExportXlsTest.java @@ -253,7 +253,7 @@ public void testExportQueryXlsWithJoinedLabels() { "MATCH p = (u:User{name: 'Andrea'})-[r:COMPANY]->(c:Company{name: 'Larus'}) DELETE p"); } - private void assertResults( + public static void assertResults( String fileName, Map r, final String source, @@ -270,7 +270,7 @@ private void assertResults( assertTrue("Should get time greater than 0", ((long) r.get("time")) >= 0); } - private void assertResults(String fileName, Map r, final String source) { + public static void assertResults(String fileName, Map r, final String source) { assertResults(fileName, r, source, 8L, 2L, 6); } diff --git a/full/src/test/java/apoc/load/LoadCsvTest.java b/full/src/test/java/apoc/load/LoadCsvTest.java index 152b91fa28..cb634e8d2d 100644 --- a/full/src/test/java/apoc/load/LoadCsvTest.java +++ b/full/src/test/java/apoc/load/LoadCsvTest.java @@ -18,42 +18,49 @@ */ package apoc.load; -import static apoc.util.BinaryTestUtil.fileToBinary; -import static apoc.util.CompressionConfig.COMPRESSION; -import static apoc.util.MapUtil.map; -import static apoc.util.TestUtil.*; -import static java.util.Arrays.asList; -import static org.junit.Assert.*; -import static org.mockserver.integration.ClientAndServer.startClientAndServer; -import static org.mockserver.matchers.Times.exactly; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; - -import apoc.ApocSettings; import apoc.util.CompressionAlgo; import apoc.util.TestUtil; import apoc.util.Util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Paths; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.junit.*; import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; import org.mockserver.model.Header; import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.QueryExecutionException; import org.neo4j.graphdb.Result; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; +import org.neo4j.values.storable.*; import org.testcontainers.containers.GenericContainer; +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.time.ZoneId; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.util.BinaryTestUtil.fileToBinary; +import static apoc.util.CompressionConfig.COMPRESSION; +import static apoc.util.ExtendedTestUtil.assertMapEquals; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.*; +import static java.util.Arrays.asList; +import static org.junit.Assert.*; +import static org.mockserver.integration.ClientAndServer.startClientAndServer; +import static org.mockserver.matchers.Times.exactly; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; +import static org.neo4j.configuration.GraphDatabaseSettings.db_temporal_timezone; + public class LoadCsvTest { private static ClientAndServer mockServer; @@ -61,7 +68,8 @@ public class LoadCsvTest { private static final List> RESPONSE_BODY = List.of( Map.of("headFoo", "one", "headBar", "two"), Map.of("headFoo", "three", "headBar", "four"), - Map.of("headFoo", "five", "headBar", "six")); + Map.of("headFoo", "five", "headBar", "six") + ); @BeforeClass public static void startServer() { @@ -73,149 +81,168 @@ public static void stopServer() { mockServer.stop(); } - @After - public void cleanup() { - db.shutdown(); - } - @Rule public DbmsRule db = new ImpermanentDbmsRule() - .withSetting(ApocSettings.apoc_import_file_enabled, true) - .withSetting( - GraphDatabaseSettings.load_csv_file_url_root, - Paths.get(getUrlFileName("test.csv").toURI()).getParent()); + .withSetting(GraphDatabaseSettings.load_csv_file_url_root, Paths.get(getUrlFileName("test.csv").toURI()).getParent()); private GenericContainer httpServer; - public LoadCsvTest() throws URISyntaxException {} + public LoadCsvTest() throws URISyntaxException { + } - @Before - public void setUp() throws Exception { + @Before public void setUp() throws Exception { TestUtil.registerProcedure(db, LoadCsv.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); } - @Test - public void testLoadCsv() throws Exception { + @Test public void testLoadCsv() throws Exception { String url = "test.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' - this::commonAssertionsLoadCsv); + commonTestLoadCsv(db, url); } @Test public void testLoadCsvWithBinary() { - testResult( - db, - "CALL apoc.load.csvParams($file, null, null, $conf)", - map( - "file", - fileToBinary(new File(getUrlFileName("test.csv").getPath()), CompressionAlgo.DEFLATE.name()), - "conf", - map( - COMPRESSION, - CompressionAlgo.DEFLATE.name(), - "results", - List.of("map", "list", "stringMap", "strings"))), - this::commonAssertionsLoadCsv); - } - - private void commonAssertionsLoadCsv(Result r) { + testResult(db, "CALL apoc.load.csvParams($file, null, null, $conf)", + map("file", fileToBinary(new File(getUrlFileName("test.csv").getPath()), CompressionAlgo.DEFLATE.name()), + "conf", map(COMPRESSION, CompressionAlgo.DEFLATE.name(), "results", List.of("map", "list", "stringMap", "strings"))), + LoadCsvTest::commonAssertionsLoadCsv); + } + + @Test + public void testLoadCsvWithQuote() { + String url = "testQuote.csv"; + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' + r -> { + assertRow(r, 0L, "name", "Selma", "age", "8"); + assertRow(r, 1L, "name", "Rana", "age", "11"); + assertRow(r, 2L, "name", "Seli,na", "age", "18"); + assertFalse(r.hasNext()); + }); + } + + @Test + public void testLoadCsvWithQuoteAndIgnoreQuotations() { + String url = "testQuote.csv"; + testResult(db, "CALL apoc.load.csv($url,{ignoreQuotations: true, results:['list']})", map("url",url), // 'file:test.csv' + r -> { + Map row = r.next(); + assertEquals(List.of("Selma","8"), row.get("list")); + + row = r.next(); + assertEquals(List.of("Rana","11"), row.get("list")); + + row = r.next(); + assertEquals(List.of("Seli", "na", "18"), row.get("list")); + + assertFalse(r.hasNext()); + }); + } + + @Test + public void testLoadCsvWithMultiCharSeparator(){ + String url = "testMultiCharSep.csv"; + Map conf = map("results", List.of("map","list","stringMap","strings"), + "sep", "SEP"); + testResult(db, "CALL apoc.load.csv($url, $conf)", + map("url",url, "conf", conf), + LoadCsvTest::commonAssertionsLoadCsv); + } + + + private static void commonAssertionsLoadCsv(Result r) { assertRow(r, 0L, "name", "Selma", "age", "8"); assertRow(r, 1L, "name", "Rana", "age", "11"); assertRow(r, 2L, "name", "Selina", "age", "18"); assertFalse(r.hasNext()); } - /* - WITH 'file:///test.csv' AS url - CALL apoc.load.csv(url,) YIELD map AS m - RETURN m.col_1,m.col_2,m.col_3 - */ @Test - public void testLoadCsvWithEmptyColumns() throws Exception { + public void testLoadCsvWithNoneSeparator() { + String url = "test.csv"; + testResult(db, "CALL apoc.load.csv($url, {sep:'NONE'})", map("url",url), // 'file:test.csv' + r -> { + Object actualList = r.next().get("list"); + assertEquals(List.of("Selma,8"), actualList); + + actualList = r.next().get("list"); + assertEquals(List.of("Rana,11"), actualList); + + actualList = r.next().get("list"); + assertEquals(List.of("Selina,18"), actualList); + + assertFalse(r.hasNext()); + }); + } + + /* + WITH 'file:///test.csv' AS url +CALL apoc.load.csv(url,) YIELD map AS m +RETURN m.col_1,m.col_2,m.col_3 + */ + @Test public void testLoadCsvWithEmptyColumns() throws Exception { String url = "empty_columns.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{failOnError:false,mapping:{col_2:{type:'int'}}})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{failOnError:false,mapping:{col_2:{type:'int'}}})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); - assertEquals(map("col_1", "1", "col_2", null, "col_3", "1"), row.get("map")); + assertEquals(map("col_1", "1","col_2", null,"col_3", "1"), row.get("map")); row = r.next(); - assertEquals(map("col_1", "2", "col_2", 2L, "col_3", ""), row.get("map")); + assertEquals(map("col_1", "2","col_2", 2L,"col_3", ""), row.get("map")); row = r.next(); - assertEquals(map("col_1", "3", "col_2", 3L, "col_3", "3"), row.get("map")); + assertEquals(map("col_1", "3","col_2", 3L,"col_3", "3"), row.get("map")); assertEquals(false, r.hasNext()); }); - testResult( - db, - "CALL apoc.load.csv($url,{failOnError:false,nullValues:[''], mapping:{col_1:{type:'int'}}})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{failOnError:false,nullValues:[''], mapping:{col_1:{type:'int'}}})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); - assertEquals(map("col_1", 1L, "col_2", null, "col_3", "1"), row.get("map")); + assertEquals(map("col_1", 1L,"col_2", null,"col_3", "1"), row.get("map")); row = r.next(); - assertEquals(map("col_1", 2L, "col_2", "2", "col_3", null), row.get("map")); + assertEquals(map("col_1", 2L,"col_2", "2","col_3", null), row.get("map")); row = r.next(); - assertEquals(map("col_1", 3L, "col_2", "3", "col_3", "3"), row.get("map")); + assertEquals(map("col_1", 3L,"col_2", "3","col_3", "3"), row.get("map")); assertEquals(false, r.hasNext()); }); - testResult( - db, - "CALL apoc.load.csv($url,{failOnError:false,mapping:{col_3:{type:'int',nullValues:['']}}})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{failOnError:false,mapping:{col_3:{type:'int',nullValues:['']}}})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); - assertEquals(map("col_1", "1", "col_2", "", "col_3", 1L), row.get("map")); + assertEquals(map("col_1", "1","col_2", "","col_3", 1L), row.get("map")); row = r.next(); - assertEquals(map("col_1", "2", "col_2", "2", "col_3", null), row.get("map")); + assertEquals(map("col_1", "2","col_2", "2","col_3", null), row.get("map")); row = r.next(); - assertEquals(map("col_1", "3", "col_2", "3", "col_3", 3L), row.get("map")); + assertEquals(map("col_1", "3","col_2", "3","col_3", 3L), row.get("map")); assertEquals(false, r.hasNext()); }); } - static void assertRow(Result r, long lineNo, Object... data) { + static void assertRow(Result r, long lineNo, Object...data) { Map row = r.next(); Map map = map(data); assertEquals(map, row.get("map")); Map stringMap = new LinkedHashMap<>(map.size()); - map.forEach((k, v) -> stringMap.put(k, v == null ? null : v.toString())); + map.forEach((k,v) -> stringMap.put(k,v == null ? null : v.toString())); assertEquals(stringMap, row.get("stringMap")); assertEquals(new ArrayList<>(map.values()), row.get("list")); assertEquals(new ArrayList<>(stringMap.values()), row.get("strings")); assertEquals(lineNo, row.get("lineNo")); } - - static void assertRow(Result r, String name, String age, long lineNo) { + public static void assertRow(Result r, String name, String age, long lineNo) { Map row = r.next(); - assertEquals(map("name", name, "age", age), row.get("map")); + assertEquals(map("name", name,"age", age), row.get("map")); assertEquals(asList(name, age), row.get("list")); assertEquals(lineNo, row.get("lineNo")); } - @Test - public void testLoadCsvSkipLimit() throws Exception { + @Test public void testLoadCsvSkipLimit() throws Exception { String url = "test.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{skip:1,limit:1,results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{skip:1,limit:1,results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { assertRow(r, "Rana", "11", 1L); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvSkip() throws Exception { + @Test public void testLoadCsvSkip() throws Exception { String url = "test.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{skip:1,results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{skip:1,results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { assertRow(r, "Rana", "11", 1L); assertEquals(true, r.hasNext()); @@ -224,35 +251,24 @@ public void testLoadCsvSkip() throws Exception { }); } - @Test - public void testLoadCsvTabSeparator() throws Exception { + @Test public void testLoadCsvTabSeparator() throws Exception { String url = "test-tab.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{sep:'TAB',results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{sep:'TAB',results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Rana", "age", "11"); + assertRow(r, 0L,"name", "Rana", "age","11"); assertEquals(false, r.hasNext()); }); } @Test public void testLoadCsvEscape() { - String url = "test-escape.csv"; + URL url = getUrlFileName("test-escape.csv"); final List results = List.of("map", "list", "stringMap", "strings"); - testResult(db, "CALL apoc.load.csv($url, $config)", map("url", url, "config", map("results", results)), (r) -> { - assertRow(r, 0L, "name", "Naruto", "surname", "Uzumaki"); - assertRow(r, 1L, "name", "Minato", "surname", "Namikaze"); - assertFalse(r.hasNext()); - }); - testResult( - db, - "CALL apoc.load.csv($url,$config)", - map("url", url, "config", map("results", results, "escapeChar", "NONE")), + testResult(db, "CALL apoc.load.csv($url, $config)", + map("url", url.toString(), "config", map("results", results)), (r) -> { - assertRow(r, 0L, "name", "Narut\\o", "surname", "Uzu\\maki"); - assertRow(r, 1L, "name", "Minat\\o", "surname", "Nami\\kaze"); + assertRow(r, 0L,"name", "Narut\\o", "surname","Uzu\\maki"); + assertRow(r, 1L,"name", "Minat\\o", "surname","Nami\\kaze"); assertFalse(r.hasNext()); }); } @@ -275,38 +291,26 @@ public void testLoadCsvWithEscapedDelimiters() { (so the ',' is escaped) and then splits, so it parses as one column. */ - var e = assertThrows( - RuntimeException.class, - () -> testResult( - db, - "CALL apoc.load.csv($url, $config)", - map("url", url, "config", map("results", results)), - (r) -> { - // Consume the stream so it throws the exception - r.stream().toArray(); - })); - assertTrue( - e.getMessage() - .contains( - "Please check whether you included a delimiter before a column separator or forgot a column separator.")); - - testResult( - db, - "CALL apoc.load.csv($url,$config)", + var e = assertThrows(RuntimeException.class, () -> testResult(db, "CALL apoc.load.csv($url, $config)", + map("url", url, "config", map("results", results)), + (r) -> { + // Consume the stream so it throws the exception + r.stream().toArray(); + }) + ); + assertTrue(e.getMessage().contains("Please check whether you included a delimiter before a column separator or forgot a column separator.")); + + testResult(db, "CALL apoc.load.csv($url,$config)", map("url", url, "config", map("results", results, "escapeChar", "NONE")), (r) -> { - assertRow(r, 0L, "name", "Narut\\o\\", "surname", "Uzu\\maki"); + assertRow(r, 0L,"name", "Narut\\o\\", "surname","Uzu\\maki"); assertFalse(r.hasNext()); }); } - @Test - public void testLoadCsvNoHeader() throws Exception { + @Test public void testLoadCsvNoHeader() throws Exception { String url = "test-no-header.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{header:false,results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{header:false,results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); assertEquals(null, row.get("map")); @@ -315,40 +319,27 @@ public void testLoadCsvNoHeader() throws Exception { assertEquals(false, r.hasNext()); }); } - - @Test - public void testLoadCsvIgnoreFields() throws Exception { + @Test public void testLoadCsvIgnoreFields() throws Exception { String url = "test-tab.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{ignore:['age'],sep:'TAB',results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{ignore:['age'],sep:'TAB',results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Rana"); + assertRow(r,0L,"name","Rana"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvColonSeparator() throws Exception { + @Test public void testLoadCsvColonSeparator() throws Exception { String url = "test.dsv"; - testResult( - db, - "CALL apoc.load.csv($url,{sep:':',results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{sep:':',results:['map','list','stringMap','strings']})", map("url",url), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Rana", "age", "11"); + assertRow(r,0L,"name","Rana","age","11"); assertFalse(r.hasNext()); }); } - @Test - public void testPipeArraySeparator() throws Exception { + @Test public void testPipeArraySeparator() throws Exception { String url = "test-pipe-column.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", map("url",url), // 'file:test.csv' (r) -> { assertEquals(asList("Selma", asList("Soda")), r.next().get("list")); assertEquals(asList("Rana", asList("Tea", "Milk")), r.next().get("list")); @@ -356,13 +347,9 @@ public void testPipeArraySeparator() throws Exception { }); } - @Test - public void testWithSpacesInFileName() throws Exception { - String url = "test pipe column with spaces in filename.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", - map("url", url), // 'file:test.csv' + @Test public void testWithSpacesInFileName() throws Exception { + String url = "file:///test%20pipe%20column%20with%20spaces%20in%20filename.csv"; + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", map("url",url), // 'file:test.csv' (r) -> { assertEquals(asList("Selma", asList("Soda")), r.next().get("list")); assertEquals(asList("Rana", asList("Tea", "Milk")), r.next().get("list")); @@ -373,10 +360,8 @@ public void testWithSpacesInFileName() throws Exception { @Test public void testLoadCsvWithBom() { String url = "taxonomy.csv"; - testCall( - db, - "CALL apoc.load.csv($url) YIELD map return map['smth1'] as first, map['ceva'] as second", - map("url", url), + testCall(db, "CALL apoc.load.csv($url) YIELD map return map['smth1'] as first, map['ceva'] as second", + map("url",url), (row) -> { final Object first = row.get("first"); final Object second = row.get("second"); @@ -385,13 +370,9 @@ public void testLoadCsvWithBom() { }); } - @Test - public void testMapping() throws Exception { + @Test public void testMapping() throws Exception { String url = "test-mapping.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},age:{type:'int'},kids:{array:true,arraySep:':',type:'int'},pass:{ignore:true}}})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},age:{type:'int'},kids:{array:true,arraySep:':',type:'int'},pass:{ignore:true}}})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); assertEquals(map("name", "Michael", "age", 41L, "kids", asList(8L, 11L, 18L)), row.get("map")); @@ -403,20 +384,124 @@ public void testMapping() throws Exception { }); } + + @Test + public void testMappingWithManyTypes() { + ZoneId defaultTimezone = ZoneId.of(apocConfig().getString(db_temporal_timezone.name())); + + String url = "test-mapping-many-types.csv"; + Map mapping = map( + "localDateTimeField", map("type", "localDateTime"), + "localTimeField", map("type", "localTime"), + "timeField", map("type", "time"), + "dateField", map("type", "date"), + "dateTimeField", map("type", "dateTime"), + "durationField", map("type", "duration"), + "boolField", map("type", "boolean"), + "pointField", map("type", "point"), + "listDatesField", map("array", true, "arraySep", ",", "type", "date") + ); + testMappingWithManyTypesCommon(defaultTimezone, url, mapping); + } + + @Test + public void testMappingWithManyTypesWithOptionalData() { + ZoneId timezone = ZoneId.of("UTC-8"); + + String url = "test-mapping-many-types.csv"; + Map mapping = map( + "localDateTimeField", map("type", "localDateTime"), + "localTimeField", map("type", "localTime"), + "timeField", map("type", "time", "optionalData", map("timezone", timezone.getId())), + "dateField", map("type", "date"), + "dateTimeField", map("type", "dateTime", "optionalData", map("timezone", timezone.getId())), + "durationField", map("type", "duration"), + "boolField", map("type", "boolean"), + "pointField", map("type", "point"), + "listDatesField", map("array", true, "arraySep", ",", "type", "date") + ); + + testMappingWithManyTypesCommon(timezone, url, mapping); + } + + private void testMappingWithManyTypesCommon(ZoneId zoneId, String url, Map mapping) { + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:$mapping})", + map("url", url, "mapping", mapping), + (r) -> { + Map row = r.next(); + Map expectedFirstRow = map("localDateTimeField", LocalDateTimeValue.parse("1999").asObject(), + "localTimeField", LocalTimeValue.parse("10:01").asObject(), + "timeField", TimeValue.parse("10:01:00Z", () -> zoneId).asObject(), + "dateField", DateValue.parse("2024-01-18").asObject(), + "dateTimeField", DateTimeValue.parse("2024-01-18T14:22:59", () -> zoneId).asObject(), + "durationField", DurationValue.parse("P5M1.5D"), + "boolField", true, + "pointField", PointValue.parse("{x: 56.7, y: 12.78, crs: 'wgs-84'}"), + "listDatesField", asList(DateValue.parse("2000").asObject(), DateValue.parse("2001").asObject(), DateValue.parse("2002").asObject()) + ); + assertMapEquals(expectedFirstRow, (Map) row.get("map")); + + + Map expectedFirstStringRow = map("localDateTimeField", "1999", + "localTimeField","10:01", + "timeField", "10:01:00Z", + "dateField", "2024-01-18", + "dateTimeField", "2024-01-18T14:22:59", + "durationField", "P5M1.5D", + "boolField", "true", + "pointField", "{x: 56.7,y: 12.78, crs: 'wgs-84'}", + "listDatesField", "2000,2001,2002" + ); + assertMapEquals(expectedFirstStringRow, (Map) row.get("stringMap")); + assertEquals(Set.copyOf(expectedFirstRow.values()), Set.copyOf((List) row.get("list"))); + assertEquals(Set.copyOf(expectedFirstStringRow.values()), Set.copyOf((List) row.get("strings"))); + + row = r.next(); + + Map expectedSecondRow = map("localDateTimeField", LocalDateTimeValue.parse("2000").asObject(), + "localTimeField", LocalTimeValue.parse("11:01").asObject(), + "timeField", TimeValue.parse("11:01:00Z", () -> zoneId).asObject(), + "dateField", DateValue.parse("2023-01-18").asObject(), + "dateTimeField", DateTimeValue.parse("2023-01-18T14:22:59", () -> zoneId).asObject(), + "durationField", DurationValue.parse("P6M1.5D"), + "boolField", false, + "pointField", PointValue.parse("{x: 57.7, y: 11.78, crs: 'wgs-84'}"), + "listDatesField", asList(DateValue.parse("2000").asObject(), DateValue.parse("2001").asObject(), DateValue.parse("2002").asObject()) + ); + assertMapEquals(expectedSecondRow, (Map) row.get("map")); + + Map expectedSecondStringRow = map("localDateTimeField", "2000", + "localTimeField", "11:01", + "timeField", "11:01:00Z", + "dateField", "2023-01-18", + "dateTimeField", "2023-01-18T14:22:59", + "durationField", "P6M1.5D", + "boolField", "false", + "pointField", "{x: 57.7,y: 11.78, crs: 'wgs-84'}", + "listDatesField", "2000,2001,2002" + ); + assertMapEquals(expectedSecondStringRow, (Map) row.get("stringMap")); + + assertEquals(Set.copyOf(expectedSecondRow.values()), Set.copyOf((List) row.get("list"))); + assertEquals(Set.copyOf(expectedSecondStringRow.values()), Set.copyOf((List) row.get("strings"))); + + assertFalse(r.hasNext()); + }); + } + + + @Test public void testLoadCsvByUrl() throws Exception { - URL url = new URL( - "https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/3.1/src/test/resources/test.csv"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString()), + URL url = new URL("https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/3.1/src/test/resources/test.csv"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url", url.toString()), (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); + } @Test @@ -425,21 +510,25 @@ public void testLoadCsvWithUserPassInUrl() throws JsonProcessingException { String token = Util.encodeUserColonPassToBase64(userPass); new MockServerClient("localhost", 1080) - .when(request().withPath("/docs/csv").withHeader("Authorization", "Basic " + token), exactly(1)) - .respond(response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "text/csv; charset=utf-8"), - new Header("Cache-Control", "private, max-age=1000")) - .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) - .withDelay(TimeUnit.SECONDS, 1)); - - testResult( - db, - "CALL apoc.load.csv($url, {results:['map']}) YIELD map", + .when( + request() + .withPath("/docs/csv") + .withHeader("Authorization", "Basic " + token), + exactly(1)) + .respond( + response() + .withStatusCode(200) + .withHeaders( + new Header("Content-Type", "text/csv; charset=utf-8"), + new Header("Cache-Control", "private, max-age=1000")) + .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) + .withDelay(TimeUnit.SECONDS, 1) + ); + + testResult(db, "CALL apoc.load.csv($url, {results:['map']}) YIELD map", map("url", "http://" + userPass + "@localhost:1080/docs/csv"), - (row) -> assertEquals( - RESPONSE_BODY, row.stream().map(i -> i.get("map")).collect(Collectors.toList()))); + (row) -> assertEquals(RESPONSE_BODY, row.stream().map(i->i.get("map")).collect(Collectors.toList())) + ); } @Test @@ -454,26 +543,22 @@ public void testLoadCsvParamsWithUserPassInUrl() throws JsonProcessingException .withPath("/docs/csv") .withHeader("Authorization", "Basic " + token), exactly(1)) - .respond(response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "text/csv; charset=utf-8"), - new Header("Cache-Control", "private, max-age=100")) - .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) - .withDelay(TimeUnit.SECONDS, 1)); - - testResult( - db, - "CALL apoc.load.csvParams($url, $header, $payload, {results:['map','list','stringMap','strings']})", - map( - "url", - "http://" + userPass + "@localhost:1080/docs/csv", - "header", - map("method", "POST"), - "payload", - "{\"query\":\"pagecache\",\"version\":\"3.5\"}"), - (row) -> assertEquals( - RESPONSE_BODY, row.stream().map(i -> i.get("map")).collect(Collectors.toList()))); + .respond( + response() + .withStatusCode(200) + .withHeaders( + new Header("Content-Type", "text/csv; charset=utf-8"), + new Header("Cache-Control", "private, max-age=100")) + .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) + .withDelay(TimeUnit.SECONDS, 1) + ); + + testResult(db, "CALL apoc.load.csvParams($url, $header, $payload, {results:['map','list','stringMap','strings']})", + map("url", "http://" + userPass + "@localhost:1080/docs/csv", + "header", map("method", "POST"), + "payload", "{\"query\":\"pagecache\",\"version\":\"3.5\"}"), + (row) -> assertEquals(RESPONSE_BODY, row.stream().map(i->i.get("map")).collect(Collectors.toList())) + ); } @Test @@ -489,39 +574,34 @@ public void testLoadCsvParamsWithBasicAuth() throws JsonProcessingException { .withHeader("Authorization", "Basic " + token) .withHeader("Content-type", "application/json"), exactly(1)) - .respond(response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "text/csv; charset=utf-8"), - new Header("Cache-Control", "private, max-age=100")) - .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) - .withDelay(TimeUnit.SECONDS, 1)); - - testResult( - db, - "CALL apoc.load.csvParams($url, $header, $payload, {results:['map','list','stringMap','strings']})", - map( - "url", - "http://localhost:1080/docs/csv", - "header", - map("method", "POST", "Authorization", "Basic " + token, "Content-Type", "application/json"), - "payload", - "{\"query\":\"pagecache\",\"version\":\"3.5\"}"), - (row) -> assertEquals( - RESPONSE_BODY, row.stream().map(i -> i.get("map")).collect(Collectors.toList()))); + .respond( + response() + .withStatusCode(200) + .withHeaders( + new Header("Content-Type", "text/csv; charset=utf-8"), + new Header("Cache-Control", "private, max-age=100")) + .withBody(fromListOfMapToCsvString(RESPONSE_BODY)) + .withDelay(TimeUnit.SECONDS, 1) + ); + + testResult(db, "CALL apoc.load.csvParams($url, $header, $payload, {results:['map','list','stringMap','strings']})", + map("url", "http://localhost:1080/docs/csv", + "header", map("method", + "POST", "Authorization", "Basic " + token, + "Content-Type", "application/json"), + "payload", "{\"query\":\"pagecache\",\"version\":\"3.5\"}"), + (row) -> assertEquals(RESPONSE_BODY, row.stream().map(i->i.get("map")).collect(Collectors.toList())) + ); } @Test public void testLoadCsvByUrlRedirect() throws Exception { - URL url = new URL("http://bit.ly/2nXgHA2"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString()), + URL url = new URL("https://bit.ly/2nXgHA2"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url", url.toString()), (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } @@ -529,149 +609,149 @@ public void testLoadCsvByUrlRedirect() throws Exception { @Test public void testLoadCsvNoFailOnError() throws Exception { String url = getUrlFileName("test.csv").getPath(); - testResult( - db, - "CALL apoc.load.csv($url,{failOnError:false})", - map("url", url), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{failOnError:false})", map("url",url), // 'file:test.csv' (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); - assertEquals(asList("Selma", "8"), row.get("list")); - assertEquals(Util.map("name", "Selma", "age", "8"), row.get("map")); + assertEquals(asList("Selma","8"), row.get("list")); + assertEquals(Util.map("name","Selma","age","8"), row.get("map")); assertEquals(true, r.hasNext()); row = r.next(); assertEquals(1L, row.get("lineNo")); - assertEquals(asList("Rana", "11"), row.get("list")); - assertEquals(Util.map("name", "Rana", "age", "11"), row.get("map")); + assertEquals(asList("Rana","11"), row.get("list")); + assertEquals(Util.map("name","Rana","age","11"), row.get("map")); assertEquals(true, r.hasNext()); row = r.next(); assertEquals(2L, row.get("lineNo")); - assertEquals(asList("Selina", "18"), row.get("list")); - assertEquals(Util.map("name", "Selina", "age", "18"), row.get("map")); + assertEquals(asList("Selina","18"), row.get("list")); + assertEquals(Util.map("name","Selina","age","18"), row.get("map")); assertEquals(false, r.hasNext()); }); } @Test - public void testLoadCsvZip() throws Exception { + public void testIssue3156FailOnErrorFalse() { + String url = getUrlFileName("faulty.csv").getPath(); + testResult(db, "CALL apoc.load.csv($url, {failOnError: false})", map("url",url), + (r) -> { + Map row = r.next(); + assertEquals(0L, row.get("lineNo")); + assertEquals(List.of("Galata Tower", "", "Turkey", "67"), row.get("list")); + row = r.next(); + assertEquals(1L, row.get("lineNo")); + assertEquals(List.of("Belem Tower","Lisbon","","30"), row.get("list")); + row = r.next(); + assertEquals(3L, row.get("lineNo")); + assertEquals(List.of("","London","United Kingdom","96"), row.get("list")); + row = r.next(); + assertEquals(4L, row.get("lineNo")); + assertEquals(List.of("Leaning tower","Pisa","Italia","56"), row.get("list")); + row = r.next(); + assertEquals(5L, row.get("lineNo")); + assertEquals(List.of("Eiffel Tower","Paris","France","300"), row.get("list")); + assertFalse(r.hasNext()); + }); + } + + @Test + public void testIssue3156FailOnErrorTrue() { + String url = getUrlFileName("faulty.csv").getPath(); + try { + testResult(db, "CALL apoc.load.csv($url, {failOnError: true})", map("url", url), + Result::resultAsString); + } catch (RuntimeException e) { + String message = e.getMessage(); + assertTrue("Actual error message is: " + message, + message.contains(ERROR_WRONG_COL_SEPARATOR) + ); + } + + } + + @Test public void testLoadCsvZip() throws Exception { String url = "testload.zip"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url + "!csv/test.csv"), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTar() throws Exception { + @Test public void testLoadCsvTar() throws Exception { String url = "testload.tar"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url + "!csv/test.csv"), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTarGz() throws Exception { + @Test public void testLoadCsvTarGz() throws Exception { String url = "testload.tar.gz"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url + "!csv/test.csv"), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTgz() throws Exception { + @Test public void testLoadCsvTgz() throws Exception { String url = "testload.tgz"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url + "!csv/test.csv"), // 'file:test.csv' + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvZipByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar?raw=true"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString() + "!csv/test.csv"), // 'file:test.csv' + @Test public void testLoadCsvZipByUrl() throws Exception { + URL url = new URL("https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/3.4/src/test/resources/testload.zip?raw=true"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url.toString()+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTarByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar?raw=true"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString() + "!csv/test.csv"), // 'file:test.csv' + @Test public void testLoadCsvTarByUrl() throws Exception { + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tar?raw=true"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url.toString()+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTarGzByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar.gz?raw=true"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString() + "!csv/test.csv"), // 'file:test.csv' + @Test public void testLoadCsvTarGzByUrl() throws Exception { + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tar.gz?raw=true"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url.toString()+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } - @Test - public void testLoadCsvTgzByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tgz?raw=true"); - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url.toString() + "!csv/test.csv"), // 'file:test.csv' + @Test public void testLoadCsvTgzByUrl() throws Exception { + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tgz?raw=true"); + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url",url.toString()+"!csv/test.csv"), // 'file:test.csv' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); assertEquals(false, r.hasNext()); }); } @@ -679,15 +759,13 @@ public void testLoadCsvTgzByUrl() throws Exception { @Test(expected = QueryExecutionException.class) public void testLoadRedirectWithProtocolChange() { httpServer = new GenericContainer("alpine") - .withCommand( - "/bin/sh", - "-c", - "while true; do { echo -e 'HTTP/1.1 301 Moved Permanently\\r\\nLocation: file:/etc/passwd'; echo ; } | nc -l -p 8000; done") + .withCommand("/bin/sh", "-c", "while true; do { echo -e 'HTTP/1.1 301 Moved Permanently\\r\\nLocation: file:/etc/passwd'; echo ; } | nc -l -p 8000; done") .withExposedPorts(8000); httpServer.start(); String url = String.format("http://%s:%s", httpServer.getContainerIpAddress(), httpServer.getMappedPort(8000)); try { - testResult(db, "CALL apoc.load.csv($url)", map("url", url), (r) -> r.hasNext()); + testResult(db, "CALL apoc.load.csv($url)", map("url", url), + (r) -> r.hasNext()); } catch (QueryExecutionException e) { assertTrue(e.getMessage().contains("The redirect URI has a different protocol: file:/etc/passwd")); throw e; @@ -697,21 +775,16 @@ public void testLoadRedirectWithProtocolChange() { } @Ignore("long running test") - @Test - public void testWithEmptyQuoteChar() throws Exception { - // TODO: fix this test to not download 7 MB each time. + @Test public void testWithEmptyQuoteChar() throws Exception { + //TODO: fix this test to not download 7 MB each time. Assume.assumeFalse("skip this in CI it downloads 7.3 MB of data", TestUtil.isRunningInCI()); URL url = new URL("https://www.fhwa.dot.gov/bridge/nbi/2010/delimited/AL10.txt"); - testResult( - db, - "CALL apoc.load.csv($url, {quoteChar: '\0'})", - map("url", url.toString()), + testResult(db, "CALL apoc.load.csv($url, {quoteChar: '\0'})", map("url",url.toString()), (r) -> assertEquals(16018L, r.stream().count())); } - private static String fromListOfMapToCsvString(List> mapList) throws JsonProcessingException { - return new CsvMapper() - .writerFor(List.class) + private static String fromListOfMapToCsvString(List> mapList ) throws JsonProcessingException { + return new CsvMapper().writerFor(List.class) .with(CsvSchema.builder() .addColumn("headFoo") .addColumn("headBar") @@ -719,4 +792,9 @@ private static String fromListOfMapToCsvString(List> mapList .withHeader()) .writeValueAsString(mapList); } + + public static void commonTestLoadCsv(GraphDatabaseService db, String url) { + testResult(db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", map("url", url), // 'file:test.csv' + LoadCsvTest::commonAssertionsLoadCsv); + } } diff --git a/full/src/test/java/apoc/load/LoadHtmlTest.java b/full/src/test/java/apoc/load/LoadHtmlTest.java index e41909ea24..b11b4b7ae4 100644 --- a/full/src/test/java/apoc/load/LoadHtmlTest.java +++ b/full/src/test/java/apoc/load/LoadHtmlTest.java @@ -18,6 +18,30 @@ */ package apoc.load; +import apoc.util.TestUtil; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.jupiter.api.AfterAll; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.load.LoadHtml.INVALID_CONFIG_ERR; import static apoc.load.LoadHtml.KEY_ERROR; import static apoc.util.MapUtil.map; import static apoc.util.TestUtil.testCall; @@ -314,17 +338,8 @@ private void loadHtmlWithSelector(int expected, String selector) { @Test public void testQueryMetadataWithGetLinks() { - Map query = map("links", "a[href]"); - - testCall( - db, - "CALL apoc.load.html($url,$query)", - map("url", new File("src/test/resources/wikipedia.html").toURI().toString(), "query", query), - row -> { - final List> actual = (List) ((Map) row.get("value")).get("links"); - assertEquals(106, actual.size()); - assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); - }); + String url = new File("src/test/resources/wikipedia.html").toURI().toString(); + testLoadHtmlWithGetLinksCommon(db, url); } @Test @@ -649,6 +664,18 @@ private void testCallGeneratedJsWithBrowser(String browser) { }); } + public static void testLoadHtmlWithGetLinksCommon(GraphDatabaseService db, String url) { + Map query = map("links", "a[href]"); + + testCall(db, "CALL apoc.load.html($url,$query)", + map("url", url, "query", query), + row -> { + final List> actual = (List) ((Map) row.get("value")).get("links"); + assertEquals(106, actual.size()); + assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); + }); + } + public static void skipIfBrowserNotPresentOrCompatible(Runnable runnable) { try { runnable.run(); diff --git a/full/src/test/java/apoc/load/LoadXlsTest.java b/full/src/test/java/apoc/load/LoadXlsTest.java index d7a6cd49b5..fb06190404 100644 --- a/full/src/test/java/apoc/load/LoadXlsTest.java +++ b/full/src/test/java/apoc/load/LoadXlsTest.java @@ -18,16 +18,22 @@ */ package apoc.load; -import static apoc.util.MapUtil.map; -import static apoc.util.TestUtil.testCall; -import static apoc.util.TestUtil.testResult; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static org.junit.Assert.*; - -import apoc.ApocSettings; import apoc.util.TestUtil; import apoc.util.Util; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.jupiter.api.AfterAll; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.graphdb.Result; +import org.neo4j.internal.helpers.collection.Iterators; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + import java.net.URL; import java.time.LocalDate; import java.time.LocalDateTime; @@ -39,372 +45,240 @@ import java.util.Map; import java.util.Set; import java.util.function.BiFunction; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.neo4j.graphdb.QueryExecutionException; -import org.neo4j.graphdb.Result; -import org.neo4j.internal.helpers.collection.Iterators; -import org.neo4j.test.rule.DbmsRule; -import org.neo4j.test.rule.ImpermanentDbmsRule; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static org.junit.Assert.*; public class LoadXlsTest { - private static String loadTest = Thread.currentThread() - .getContextClassLoader() - .getResource("load_test.xlsx") - .getPath(); - private static String testDate = Thread.currentThread() - .getContextClassLoader() - .getResource("test_date.xlsx") - .getPath(); - private static String brokenHeader = Thread.currentThread() - .getContextClassLoader() - .getResource("brokenHeader.xls") - .getPath(); - private static String testColumnsAfterZ = Thread.currentThread() - .getContextClassLoader() - .getResource("testLoadXlsColumnsAfterZ.xlsx") - .getPath(); + private static String loadTest = Thread.currentThread().getContextClassLoader().getResource("load_test.xlsx").getPath(); + private static String testDate = Thread.currentThread().getContextClassLoader().getResource("test_date.xlsx").getPath(); + private static String brokenHeader = Thread.currentThread().getContextClassLoader().getResource("brokenHeader.xls").getPath(); + private static String testColumnsAfterZ = Thread.currentThread().getContextClassLoader().getResource("testLoadXlsColumnsAfterZ.xlsx").getPath(); @Rule - public DbmsRule db = new ImpermanentDbmsRule().withSetting(ApocSettings.apoc_import_file_enabled, true); + public DbmsRule db = new ImpermanentDbmsRule(); - @Before - public void setUp() throws Exception { + @Before public void setUp() throws Exception { TestUtil.registerProcedure(db, LoadXls.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); } + - @After - public void teardown() { - db.shutdown(); - } - - @Test - public void testLoadXls() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", - map("url", loadTest), // 'file:load_test.xlsx' - (r) -> { - assertRow( - r, - 0L, - "String", - "Test", - "Boolean", - true, - "Integer", - 2L, - "Float", - 1.5d, - "Array", - asList(1L, 2L, 3L)); - assertFalse("Should not have another row", r.hasNext()); - }); + @Test public void testLoadXls() throws Exception { + testLoadXlsCommon(db, loadTest); } - @Test - public void testLoadBrokenHeader() throws Exception { - BiFunction query = (sheet, header) -> db.executeTransactionally( + @Test public void testLoadBrokenHeader() throws Exception { + BiFunction query = (sheet,header) -> db.executeTransactionally( "CALL apoc.load.xls($url,$sheet,{header:$header}) yield map return count(*) as c", - map("header", header, "sheet", sheet, "url", brokenHeader), + map("header",header,"sheet",sheet,"url", brokenHeader ), result -> Iterators.single(result.columnAs("c"))); try { - query.apply("temp", true); + query.apply("temp",true); fail("Should fail with Header Error"); - } catch (QueryExecutionException qee) { - assertEquals( - "Failed to invoke procedure `apoc.load.xls`: Caused by: java.lang.IllegalStateException: Header at position 4 doesn't have a value", + } catch(QueryExecutionException qee) { + assertEquals("Failed to invoke procedure `apoc.load.xls`: Caused by: java.lang.IllegalStateException: Header at position 4 doesn't have a value", qee.getMessage()); } try { - query.apply("foo", false); + query.apply("foo",false); fail("Should fail with Sheet Error"); - } catch (QueryExecutionException qee) { - assertEquals( - "Failed to invoke procedure `apoc.load.xls`: Caused by: java.lang.IllegalStateException: Sheet foo not found", + } catch(QueryExecutionException qee) { + assertEquals("Failed to invoke procedure `apoc.load.xls`: Caused by: java.lang.IllegalStateException: Sheet foo not found", qee.getMessage()); } - assertEquals(2L, (long) query.apply("temp", false)); + assertEquals(2L, (long)query.apply("temp",false)); } - - @Test - public void testLoadXlsMany() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Many',{mapping:{Float:{type:'float'}, Array:{type:'int',array:true,arraySep:';'}}})", - map("url", loadTest), // 'file:load_test.xlsx' + @Test public void testLoadXlsMany() throws Exception { + testResult(db, "CALL apoc.load.xls($url,'Many',{mapping:{Float:{type:'float'}, Array:{type:'int',array:true,arraySep:';'}}})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { - assertRow(r, 0L, "String", "A", "Boolean", true, "Integer", 0L, "Float", 0d, "Array", emptyList()); - assertRow( - r, 1L, "String", "B", "Boolean", false, "Integer", 1L, "Float", 0.5d, "Array", asList(1L)); - assertRow( - r, - 2L, - "String", - "C", - "Boolean", - true, - "Integer", - 2L, - "Float", - 1.0d, - "Array", - asList(1L, 2L)); - assertRow( - r, - 3L, - "String", - "D", - "Boolean", - false, - "Integer", - 3L, - "Float", - 1.5d, - "Array", - asList(1L, 2L, 3L)); - assertRow( - r, - 4L, - "String", - "E", - "Boolean", - true, - "Integer", - 4L, - "Float", - 2.0d, - "Array", - asList(1L, 2L, 3L, 4L)); - assertFalse("Should not have another row", r.hasNext()); + assertRow(r,0L,"String","A","Boolean",true ,"Integer",0L,"Float",0d,"Array", emptyList()); + assertRow(r,1L,"String","B","Boolean",false,"Integer",1L,"Float",0.5d,"Array", asList(1L)); + assertRow(r,2L,"String","C","Boolean",true ,"Integer",2L,"Float",1.0d,"Array", asList(1L,2L)); + assertRow(r,3L,"String","D","Boolean",false,"Integer",3L,"Float",1.5d,"Array", asList(1L,2L,3L)); + assertRow(r,4L,"String","E","Boolean",true ,"Integer",4L,"Float",2.0d,"Array", asList(1L,2L,3L,4L)); + assertFalse("Should not have another row",r.hasNext()); }); } - @Test - public void testLoadXlsOffset() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Offset!B2:F3',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", - map("url", loadTest), // 'file:load_test.xlsx' + @Test public void testLoadXlsOffset() throws Exception { + testResult(db, "CALL apoc.load.xls($url,'Offset!B2:F3',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { - assertRow( - r, - 0L, - "String", - "Test", - "Boolean", - true, - "Integer", - 2L, - "Float", - 1.5d, - "Array", - asList(1L, 2L, 3L)); + assertRow(r,0L,"String","Test","Boolean",true,"Integer",2L,"Float",1.5d,"Array",asList(1L,2L,3L)); assertFalse(r.hasNext()); }); } - @Test - public void testLoadXlsNoHeaders() throws Exception { - testCall( - db, - "CALL apoc.load.xls($url,'NoHeader',{header:false})", - map("url", loadTest), // 'file:load_test.xlsx' + @Test public void testLoadXlsNoHeaders() throws Exception { + testCall(db, "CALL apoc.load.xls($url,'NoHeader',{header:false})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { - assertEquals(0L, r.get("lineNo")); - assertEquals(asList("Test", true, 2L, 1.5d, "1;2;3"), r.get("list")); + assertEquals(0L,r.get("lineNo")); + assertEquals(asList("Test",true,2L,1.5d,"1;2;3"),r.get("list")); }); } /* - WITH 'file:///load_test.xlsx' AS url - CALL apoc.load.xls(url,) YIELD map AS m - RETURN m.col_1,m.col_2,m.col_3 - */ - @Test - public void testLoadCsvWithEmptyColumns() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Empty',{failOnError:false,mapping:{col_2:{type:'int'}}})", - map("url", loadTest), // 'file:load_test.xlsx' + WITH 'file:///load_test.xlsx' AS url +CALL apoc.load.xls(url,) YIELD map AS m +RETURN m.col_1,m.col_2,m.col_3 + */ + @Test public void testLoadCsvWithEmptyColumns() throws Exception { + testResult(db, "CALL apoc.load.xls($url,'Empty',{failOnError:false,mapping:{col_2:{type:'int'}}})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { Map row = r.next(); - assertEquals(map("col_1", 1L, "col_2", null, "col_3", 1L), row.get("map")); + assertEquals(map("col_1", 1L,"col_2", null,"col_3", 1L), row.get("map")); row = r.next(); - assertEquals(map("col_1", 2L, "col_2", 2L, "col_3", ""), row.get("map")); + assertEquals(map("col_1", 2L,"col_2", 2L,"col_3", ""), row.get("map")); row = r.next(); - assertEquals(map("col_1", 3L, "col_2", 3L, "col_3", 3L), row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(map("col_1", 3L,"col_2", 3L,"col_3", 3L), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); - testResult( - db, - "CALL apoc.load.xls($url,'Empty',{failOnError:false,nullValues:[''], mapping:{col_1:{type:'int'}}})", - map("url", loadTest), // 'file:load_test.xlsx' + testResult(db, "CALL apoc.load.xls($url,'Empty',{failOnError:false,nullValues:[''], mapping:{col_1:{type:'int'}}})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { Map row = r.next(); - assertEquals(map("col_1", 1L, "col_2", null, "col_3", 1L), row.get("map")); + assertEquals(map("col_1", 1L,"col_2", null,"col_3", 1L), row.get("map")); row = r.next(); - assertEquals(map("col_1", 2L, "col_2", 2L, "col_3", null), row.get("map")); + assertEquals(map("col_1", 2L,"col_2", 2L,"col_3", null), row.get("map")); row = r.next(); - assertEquals(map("col_1", 3L, "col_2", 3L, "col_3", 3L), row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(map("col_1", 3L,"col_2", 3L,"col_3", 3L), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); - testResult( - db, - "CALL apoc.load.xls($url,'Empty',{failOnError:false,mapping:{col_3:{type:'int',nullValues:['']}}})", - map("url", loadTest), // 'file:load_test.xlsx' + testResult(db, "CALL apoc.load.xls($url,'Empty',{failOnError:false,mapping:{col_3:{type:'int',nullValues:['']}}})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { Map row = r.next(); - assertEquals(map("col_1", 1L, "col_2", "", "col_3", 1L), row.get("map")); + assertEquals(map("col_1", 1L,"col_2", "","col_3", 1L), row.get("map")); row = r.next(); - assertEquals(map("col_1", 2L, "col_2", 2L, "col_3", null), row.get("map")); + assertEquals(map("col_1", 2L,"col_2", 2L,"col_3", null), row.get("map")); row = r.next(); - assertEquals(map("col_1", 3L, "col_2", 3L, "col_3", 3L), row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(map("col_1", 3L,"col_2", 3L,"col_3", 3L), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); } - static void assertRow(Result r, long lineNo, Object... data) { + static void assertRow(Result r, long lineNo, Object...data) { Map row = r.next(); Map map = map(data); assertEquals(map, row.get("map")); Map stringMap = new LinkedHashMap<>(map.size()); - map.forEach((k, v) -> stringMap.put(k, v == null ? null : v.toString())); + map.forEach((k,v) -> stringMap.put(k,v == null ? null : v.toString())); assertEquals(new ArrayList<>(map.values()), row.get("list")); assertEquals(lineNo, row.get("lineNo")); } - static void assertRow(Result r, String name, Number age, long lineNo) { Map row = r.next(); - assertEquals(map("name", name, "age", age), row.get("map")); + assertEquals(map("name", name,"age", age), row.get("map")); assertEquals(asList(name, age), row.get("list")); assertEquals(lineNo, row.get("lineNo")); } - @Test - public void testLoadCsvSkip() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Kids',{skip:1,limit:1})", - map("url", loadTest), // 'file:load_test.xlsx' + @Test public void testLoadCsvSkip() throws Exception { + testResult(db, "CALL apoc.load.xls($url,'Kids',{skip:1,limit:1})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { assertRow(r, "Rana", 11L, 0L); - assertFalse("Should not have another row", r.hasNext()); + assertFalse("Should not have another row",r.hasNext()); }); } - - @Test - public void testLoadCsvIgnoreFields() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Kids',{ignore:['age']})", - map("url", loadTest), // 'file:load_test.xlsx' + @Test public void testLoadCsvIgnoreFields() throws Exception { + testResult(db, "CALL apoc.load.xls($url,'Kids',{ignore:['age']})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { - assertRow(r, 0L, "name", "Selma"); - assertRow(r, 1L, "name", "Rana"); - assertRow(r, 2L, "name", "Selina"); - assertFalse("Should not have another row", r.hasNext()); + assertRow(r,0L,"name","Selma"); + assertRow(r,1L,"name","Rana"); + assertRow(r,2L,"name","Selina"); + assertFalse("Should not have another row",r.hasNext()); }); } @Test @Ignore public void testLoadCsvByUrl() throws Exception { - URL url = new URL( - "https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/3.3/src/test/resources/load_test.xlsx"); - testResult(db, "CALL apoc.load.xls($url,'Kids')", map("url", url.toString()), (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); - assertFalse(r.hasNext()); - }); + URL url = new URL("https://raw.githubusercontent.com/neo4j-contrib/neo4j-apoc-procedures/3.3/src/test/resources/load_test.xlsx"); + testResult(db, "CALL apoc.load.xls($url,'Kids')", map("url", url.toString()), + (r) -> { + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); + assertFalse(r.hasNext()); + }); + } @Test @Ignore public void testLoadCsvByUrlRedirect() throws Exception { URL url = new URL("http://bit.ly/2nXgHA2"); - testResult(db, "CALL apoc.load.xls($url,'Kids')", map("url", url.toString()), (r) -> { - assertRow(r, 0L, "name", "Selma", "age", "8"); - assertRow(r, 1L, "name", "Rana", "age", "11"); - assertRow(r, 2L, "name", "Selina", "age", "18"); - assertFalse("Should not have another row", r.hasNext()); - }); + testResult(db, "CALL apoc.load.xls($url,'Kids')", map("url", url.toString()), + (r) -> { + assertRow(r,0L,"name","Selma","age","8"); + assertRow(r,1L,"name","Rana","age","11"); + assertRow(r,2L,"name","Selina","age","18"); + assertFalse("Should not have another row",r.hasNext()); + }); } @Test public void testLoadCsvNoFailOnError() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Kids',{failOnError:false})", - map("url", loadTest), // 'file:load_test.xlsx' + testResult(db, "CALL apoc.load.xls($url,'Kids',{failOnError:false})", map("url",loadTest), // 'file:load_test.xlsx' (r) -> { - assertRow(r, 0L, "name", "Selma", "age", 8L); - assertRow(r, 1L, "name", "Rana", "age", 11L); - assertRow(r, 2L, "name", "Selina", "age", 18L); - assertFalse("Should not have another row", r.hasNext()); + assertRow(r,0L,"name","Selma","age",8L); + assertRow(r,1L,"name","Rana","age",11L); + assertRow(r,2L,"name","Selina","age",18L); + assertFalse("Should not have another row",r.hasNext()); }); } @Test public void testLoadXlsDateWithMappingTypeString() throws Exception { - LocalDateTime date = LocalDateTime.of(2018, 10, 10, 0, 0, 0); - LocalDateTime time = LocalDateTime.of(1899, 12, 31, 12, 01, 10); + LocalDateTime date = LocalDateTime.of(2018,10,10, 0, 0, 0); + LocalDateTime time = LocalDateTime.of(1899,12,31, 12,01,10); - testResult( - db, - "CALL apoc.load.xls($url,'sheet',{mapping:{Date:{type:'String', dateFormat: 'iso_date'}}})", - map("url", testDate), + testResult(db, "CALL apoc.load.xls($url,'sheet',{mapping:{Date:{type:'String', dateFormat: 'iso_date'}}})", map("url", testDate), (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); assertEquals(asList("2018/05/10", "2018/10/05", "Alan"), row.get("list")); - assertEquals(Util.map("Date", "2018/05/10", "Data", "2018/10/05", "Name", "Alan"), row.get("map")); - assertTrue(r.hasNext()); + assertEquals(Util.map("Date", "2018/05/10", "Data", "2018/10/05","Name", "Alan"), row.get("map")); + assertTrue( r.hasNext()); row = r.next(); assertEquals(1L, row.get("lineNo")); assertEquals(asList("2018-09-10", date, "Jack"), row.get("list")); - assertEquals(Util.map("Date", "2018-09-10", "Data", date, "Name", "Jack"), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertEquals(Util.map("Date", "2018-09-10", "Data", date, "Name", "Jack"), row.get("map")); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(2L, row.get("lineNo")); assertEquals(asList("2018/05/10 12:10:10", date, date), row.get("list")); assertEquals(Util.map("Date", "2018/05/10 12:10:10", "Data", date, "Name", date), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(3L, row.get("lineNo")); assertEquals(asList(null, date, time), row.get("list")); assertEquals(Util.map("Date", null, "Data", date, "Name", time), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(4L, row.get("lineNo")); assertEquals(asList("2011-01-01T12:00:00.05381+01:00", null, null), row.get("list")); - assertEquals( - Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), - row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); } @Test public void testLoadXlsDateWithMappingArrayTypeString() throws Exception { - LocalDateTime date = LocalDateTime.of(2018, 10, 10, 0, 0, 0); - LocalDateTime time = LocalDateTime.of(1899, 12, 31, 12, 01, 10); + LocalDateTime date = LocalDateTime.of(2018,10,10, 0, 0, 0); + LocalDateTime time = LocalDateTime.of(1899,12,31, 12,01,10); String elementExpected = "2018-09-10T00:00:00"; - testResult( - db, - "CALL apoc.load.xls($url,'sheet',{mapping:{Date:{type:'String', dateFormat: '', dateParse: []}}})", - map("url", testDate), + testResult(db, "CALL apoc.load.xls($url,'sheet',{mapping:{Date:{type:'String', dateFormat: '', dateParse: []}}})", map("url",testDate), (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); @@ -414,8 +288,8 @@ public void testLoadXlsDateWithMappingArrayTypeString() throws Exception { row = r.next(); assertEquals(1L, row.get("lineNo")); assertEquals(asList(elementExpected, date, "Jack"), row.get("list")); - assertEquals(Util.map("Date", elementExpected, "Data", date, "Name", "Jack"), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertEquals(Util.map("Date", elementExpected, "Data", date, "Name", "Jack"), row.get("map")); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(2L, row.get("lineNo")); assertEquals(asList("2018/05/10 12:10:10", date, date), row.get("list")); @@ -425,33 +299,27 @@ public void testLoadXlsDateWithMappingArrayTypeString() throws Exception { assertEquals(3L, row.get("lineNo")); assertEquals(asList(null, date, time), row.get("list")); assertEquals(Util.map("Date", null, "Data", date, "Name", time), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(4L, row.get("lineNo")); assertEquals(asList("2011-01-01T12:00:00.05381+01:00", null, null), row.get("list")); - assertEquals( - Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), - row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); } @Test public void testLoadXlsDateWithMappingArrayTypeDate() throws Exception { - LocalDateTime time = LocalDateTime.of(1899, 12, 31, 12, 01, 10); - LocalDateTime localDateTimeValue = LocalDateTime.of(2018, 9, 10, 0, 0, 0); - LocalDateTime localDateTimeValue1 = LocalDateTime.of(2018, 10, 10, 0, 0, 0); + LocalDateTime time = LocalDateTime.of(1899,12,31, 12,01,10); + LocalDateTime localDateTimeValue = LocalDateTime.of(2018,9,10, 0,0,0); + LocalDateTime localDateTimeValue1 = LocalDateTime.of(2018,10,10, 0,0,0); - LocalDate localDate = LocalDate.of(2018, 10, 5); - LocalDate LocalDate1 = LocalDate.of(2018, 10, 10); + LocalDate localDate = LocalDate.of(2018,10,5); + LocalDate LocalDate1 = LocalDate.of(2018,10,10); - List pattern = - asList("wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd", "yyyy/dd/MM", "yyyy-dd-MM'T'hh:mm:ss"); + List pattern = asList("wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd", "yyyy/dd/MM", "yyyy-dd-MM'T'hh:mm:ss"); - testResult( - db, - "CALL apoc.load.xls($url,'sheet',{mapping:{Data:{type: 'Date', dateParse: $pattern}}})", - map("url", testDate, "pattern", pattern), + testResult(db, "CALL apoc.load.xls($url,'sheet',{mapping:{Data:{type: 'Date', dateParse: $pattern}}})", map("url",testDate, "pattern", pattern), (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); @@ -461,44 +329,35 @@ public void testLoadXlsDateWithMappingArrayTypeDate() throws Exception { row = r.next(); assertEquals(1L, row.get("lineNo")); assertEquals(asList(localDateTimeValue, LocalDate1, "Jack"), row.get("list")); - assertEquals( - Util.map("Date", localDateTimeValue, "Data", LocalDate1, "Name", "Jack"), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertEquals(Util.map("Date", localDateTimeValue, "Data", LocalDate1, "Name", "Jack"), row.get("map")); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(2L, row.get("lineNo")); assertEquals(asList("2018/05/10 12:10:10", LocalDate1, localDateTimeValue1), row.get("list")); - assertEquals( - Util.map("Date", "2018/05/10 12:10:10", "Data", LocalDate1, "Name", localDateTimeValue1), - row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertEquals(Util.map("Date", "2018/05/10 12:10:10", "Data", LocalDate1, "Name", localDateTimeValue1), row.get("map")); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(3L, row.get("lineNo")); assertEquals(asList(null, LocalDate1, time), row.get("list")); assertEquals(Util.map("Date", null, "Data", LocalDate1, "Name", time), row.get("map")); - assertTrue("Should have another row", r.hasNext()); + assertTrue("Should have another row",r.hasNext()); row = r.next(); assertEquals(4L, row.get("lineNo")); assertEquals(asList("2011-01-01T12:00:00.05381+01:00", null, null), row.get("list")); - assertEquals( - Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), - row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertEquals(Util.map("Date", "2011-01-01T12:00:00.05381+01:00", "Data", null, "Name", null), row.get("map")); + assertFalse("Should not have another row",r.hasNext()); }); } @Test public void testLoadXlsDateWithMappingArrayTypeDateTime() throws Exception { - LocalDateTime localDateTimeValue = LocalDateTime.of(2018, 5, 10, 12, 10, 10, 0); - LocalDateTime localDateTimeValue1 = LocalDateTime.of(2018, 10, 10, 0, 0, 0, 0); + LocalDateTime localDateTimeValue = LocalDateTime.of(2018,5,10, 12,10,10, 0); + LocalDateTime localDateTimeValue1 = LocalDateTime.of(2018,10,10, 0,0,0, 0); - List pattern = asList( - "wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_local_date_time"); + List pattern = asList("wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_local_date_time"); - testResult( - db, - "CALL apoc.load.xls($url,'dateTime',{mapping:{Date:{type: 'LOCAL_DATE_TIME', dateParse: $pattern}}})", - map("url", testDate, "pattern", pattern), + testResult(db, "CALL apoc.load.xls($url,'dateTime',{mapping:{Date:{type: 'LOCAL_DATE_TIME', dateParse: $pattern}}})", map("url",testDate, "pattern", pattern), (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); @@ -509,23 +368,18 @@ public void testLoadXlsDateWithMappingArrayTypeDateTime() throws Exception { assertEquals(1L, row.get("lineNo")); assertEquals(asList(localDateTimeValue1), row.get("list")); assertEquals(Util.map("Date", localDateTimeValue1), row.get("map")); - assertFalse("Should not have another row", r.hasNext()); + assertFalse("Should not have another row",r.hasNext()); }); } @Test public void testLoadXlsDateWithMappingArrayTypeZoneDateTime() throws Exception { - ZonedDateTime zonedDateTime = - ZonedDateTime.of(LocalDateTime.of(2011, 1, 1, 12, 0, 0, 53810000), ZoneOffset.of("+05:00")); + ZonedDateTime zonedDateTime = ZonedDateTime.of(LocalDateTime.of(2011,1,1,12,0,0, 53810000), ZoneOffset.of("+05:00")); - List pattern = asList( - "wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_zoned_date_time"); + List pattern = asList("wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_zoned_date_time"); - testResult( - db, - "CALL apoc.load.xls($url,'zonedDateTime',{mapping:{Date:{type: 'DATE_TIME', dateParse: $pattern}}})", - map("url", testDate, "pattern", pattern), + testResult(db, "CALL apoc.load.xls($url,'zonedDateTime',{mapping:{Date:{type: 'DATE_TIME', dateParse: $pattern}}})", map("url",testDate, "pattern", pattern), (r) -> { Map row = r.next(); assertEquals(0L, row.get("lineNo")); @@ -538,15 +392,10 @@ public void testLoadXlsDateWithMappingArrayTypeZoneDateTime() throws Exception { @Test(expected = RuntimeException.class) public void testLoadXlsDateWithMappingArrayTypeZoneDateTimeWithError() throws Exception { - List pattern = asList( - "wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_local_date_time"); + List pattern = asList("wrongPath", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy/MM/dd'T'HH:mm:ss", "yyyy/dd/MM", "iso_local_date_time"); try { - testCall( - db, - "CALL apoc.load.xls($url,'dateTime',{mapping:{Date:{type: 'DATE_TIME', dateParse: $pattern}}})", - map("url", testDate, "pattern", pattern), - (r) -> {}); + testCall(db, "CALL apoc.load.xls($url,'dateTime',{mapping:{Date:{type: 'DATE_TIME', dateParse: $pattern}}})", map("url",testDate, "pattern", pattern), (r) -> { }); } catch (Exception e) { Throwable except = ExceptionUtils.getRootCause(e); assertTrue(except instanceof RuntimeException); @@ -557,9 +406,8 @@ public void testLoadXlsDateWithMappingArrayTypeZoneDateTimeWithError() throws Ex @Test public void testLoadXlsColumnsAfterZ() { - testCall( - db, - "CALL apoc.load.xls($url, 'Sheet1!A1:AY10') yield list\n" + "return *", + testCall(db, "CALL apoc.load.xls($url, 'Sheet1!A1:AY10') yield list\n" + + "return *", map("url", testColumnsAfterZ), (row) -> { List list = (List) row.get("list"); @@ -577,50 +425,90 @@ public void testLoadXlsEmptyColumn() { var config = Map.of("skipNulls", true, "firstCellNum", 0, "lastCellNum", 5); Map params = map("url", url, "config", config); - testResult(db, "CALL apoc.load.xls($url,'test1', $config)", params, (r) -> { - Map firstMap = Map.of("b", 2L, "c", 3L, "d", 4L, "e", 5L); - Map secondMap = Map.of("b", 7L, "c", 8L, "d", 9L, "e", 10L); - assertIssue2403Excel(r, firstMap, secondMap); - }); - - testResult(db, "CALL apoc.load.xls($url,'test2', $config)", params, (r) -> { - Map firstMap = Map.of("c", 3L, "d", 4L, "e", 5L); - Map secondMap = Map.of("c", 8L, "d", 9L, "e", 10L); - assertIssue2403Excel(r, firstMap, secondMap); - }); - - testResult(db, "CALL apoc.load.xls($url,'test3', $config)", params, (r) -> { - Map firstMap = Map.of("a", 1L, "Empty__1", 2L, "c", 3L, "d", 4L, "e", 5L); - Map secondMap = Map.of("a", 6L, "Empty__1", 7L, "c", 8L, "d", 9L, "e", 10L); - assertIssue2403Excel(r, firstMap, secondMap); - }); - - testResult(db, "CALL apoc.load.xls($url,'test4', $config)", params, (r) -> { - Map firstMap = Map.of("a", 1L, "Empty__1", 2L, "c", 3L, "d", 4L, "e", 5L); - Map secondMap = Map.of("a", 6L, "Empty__1", 7L, "c", 8L, "d", 9L, "e", 10L); - assertIssue2403Excel(r, firstMap, secondMap); - }); - - testResult(db, "CALL apoc.load.xls($url,'test5', $config)", params, (r) -> { - Map row = r.next(); - assertEquals(Map.of("a", 1L, "Empty__1", 2L, "c", 3L, "d", 4L, "e", 5L), row.get("map")); - assertEquals(firstRow, row.get("list")); - - row = r.next(); - Map nullMap = map("a", null, "Empty__1", null, "c", null, "d", null, "e", null); - List nullList = asList(null, null, null, null, null); - assertEquals(nullMap, row.get("map")); - assertEquals(nullList, row.get("list")); - - row = r.next(); - assertEquals(Map.of("a", 6L, "Empty__1", 7L, "c", 8L, "d", 9L, "e", 10L), row.get("map")); - assertEquals(secondRow, row.get("list")); - - assertFalse(r.hasNext()); - }); + testResult(db, "CALL apoc.load.xls($url,'test1', $config)", + params, + (r) -> { + Map firstMap = Map.of("b", 2L, + "c", 3L, + "d", 4L, + "e", 5L); + Map secondMap = Map.of("b", 7L, + "c", 8L, + "d", 9L, + "e", 10L); + assertIssue2403Excel(r, firstMap, secondMap); + }); + + testResult(db, "CALL apoc.load.xls($url,'test2', $config)", + params, + (r) -> { + Map firstMap = Map.of("c", 3L, + "d", 4L, + "e", 5L); + Map secondMap = Map.of("c", 8L, + "d", 9L, + "e", 10L); + assertIssue2403Excel(r, firstMap, secondMap); + }); + + testResult(db, "CALL apoc.load.xls($url,'test3', $config)", + params, + (r) -> { + Map firstMap = Map.of("a", 1L, "Empty__1", 2L, + "c", 3L, + "d", 4L, + "e", 5L); + Map secondMap = Map.of("a", 6L, "Empty__1", 7L, + "c", 8L, + "d", 9L, + "e", 10L); + assertIssue2403Excel(r, firstMap, secondMap); + }); + + testResult(db, "CALL apoc.load.xls($url,'test4', $config)", + params, + (r) -> { + Map firstMap = Map.of("a", 1L, "Empty__1", 2L, + "c", 3L, + "d", 4L, + "e", 5L); + Map secondMap = Map.of("a", 6L, "Empty__1", 7L, + "c", 8L, + "d", 9L, + "e", 10L); + assertIssue2403Excel(r, firstMap, secondMap); + }); + + testResult(db, "CALL apoc.load.xls($url,'test5', $config)", + params, + (r) -> { + Map row = r.next(); + assertEquals(Map.of("a", 1L, "Empty__1", 2L, + "c", 3L, + "d", 4L, + "e", 5L), row.get("map")); + assertEquals(firstRow, row.get("list")); + + row = r.next(); + Map nullMap = map("a", null, "Empty__1", null, "c", null, "d", null, "e", null); + List nullList = asList(null, null, null, null, null); + assertEquals(nullMap, row.get("map")); + assertEquals(nullList, row.get("list")); + + row = r.next(); + assertEquals(Map.of("a", 6L, "Empty__1", 7L, + "c", 8L, + "d", 9L, + "e", 10L), row.get("map")); + assertEquals(secondRow, row.get("list")); + + assertFalse(r.hasNext()); + }); } - private void assertIssue2403Excel(Result r, Map firstMap, Map secondMap) { + private void assertIssue2403Excel(Result r, + Map firstMap, + Map secondMap) { Map row = r.next(); assertEquals(firstMap, row.get("map")); @@ -638,4 +526,14 @@ private void assertIssue2403Excel(Result r, Map firstMap, Map { + assertRow(r,0L,"String","Test","Boolean",true,"Integer",2L,"Float",1.5d,"Array",asList(1L,2L,3L)); + assertFalse("Should not have another row",r.hasNext()); + }); + } + } diff --git a/full/src/test/java/apoc/util/ExtendedTestUtil.java b/full/src/test/java/apoc/util/ExtendedTestUtil.java index 91f712677d..fd8926294d 100644 --- a/full/src/test/java/apoc/util/ExtendedTestUtil.java +++ b/full/src/test/java/apoc/util/ExtendedTestUtil.java @@ -2,6 +2,9 @@ import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallAssertions; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.neo4j.test.assertion.Assert.assertEventually; @@ -16,10 +19,41 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.ResultTransformer; +import org.neo4j.internal.helpers.collection.Iterators; import org.neo4j.test.assertion.Assert; public class ExtendedTestUtil { + + public static void assertMapEquals(Map expected, Map actual) { + assertMapEquals(null, expected, actual); + } + + public static void assertMapEquals(String errMsg, Map expected, Map actual) { + if (expected == null) { + assertNull(actual); + } else { + assertEquals(errMsg, expected.keySet(), actual.keySet()); + + actual.forEach((key, actualValue) -> { + Object expectedValue = expected.get(key); + boolean valuesAreArrays = key != null && actualValue != null + && actualValue.getClass().isArray() + && expectedValue.getClass().isArray(); + + if (actualValue instanceof Map) { + assertMapEquals(errMsg, (Map) expectedValue, (Map) actualValue); + } else if (valuesAreArrays) { + Object[] expectedArray = Iterators.array(expectedValue); + Object[] actualArray = Iterators.array(actualValue); + assertArrayEquals(expectedArray, actualArray); + } else { + assertEquals(errMsg, expectedValue, actualValue); + } + }); + } + } + /** * similar to @link {@link TestUtil#testCallEventually(GraphDatabaseService, String, Consumer, long)} * but re-execute the {@link GraphDatabaseService#executeTransactionally(String, Map, ResultTransformer)} in case of error diff --git a/full/src/test/java/apoc/util/GexfTestUtil.java b/full/src/test/java/apoc/util/GexfTestUtil.java new file mode 100644 index 0000000000..ac62c9625b --- /dev/null +++ b/full/src/test/java/apoc/util/GexfTestUtil.java @@ -0,0 +1,122 @@ +package apoc.util; + +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.test.rule.DbmsRule; + +import java.util.List; +import java.util.Map; + +import static apoc.util.ExtendedTestUtil.assertRelationship; +import static apoc.util.MapUtil.map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class GexfTestUtil { + public static void testImportGexfCommon(DbmsRule db, String file) { + TestUtil.testCall( + db, + "CALL apoc.import.gexf($file, {readLabels:true})", + map("file", file), + (r) -> { + assertEquals("gexf", r.get("format")); + assertEquals(5L, r.get("nodes")); + assertEquals(8L, r.get("relationships")); + }); + + TestUtil.testCallCount(db, "MATCH (n) RETURN n",5); + + TestUtil.testResult(db, "MATCH (n:Gephi) RETURN properties(n) as props", r -> { + ResourceIterator propsIterator = r.columnAs("props"); + Map props = propsIterator.next(); + assertEquals("http://gephi.org", props.get("0")); + assertEquals(1.0f, props.get("1")); + + props = propsIterator.next(); + assertEquals("http://test.gephi.org", props.get("0")); + }); + + TestUtil.testResult(db, "MATCH (n:BarabasiLab) RETURN properties(n) as props", r -> { + ResourceIterator propsIterator = r.columnAs("props"); + Map props = propsIterator.next(); + assertEquals("http://barabasilab.com", props.get("0")); + assertEquals(1.0f, props.get("1")); + }); + + Map multiDataTypeNodeProps = Map.of( + "0", "http://gephi.org", + "1", 1.0f, + "room", 10, + "price", Double.parseDouble("10.02"), + "projects", 300L, + "members", new String[] {"Altomare", "Sterpeto", "Lino"}, + "pins", new boolean[]{true, false, true, false} + ); + + TestUtil.testResult( + db, + "MATCH ()-[rel]->() RETURN rel ORDER BY rel.score", + r -> { + final ResourceIterator rels = r.columnAs("rel"); + + assertRelationship(rels.next(), "KNOWS", Map.of("score", 1.5f), + List.of("Gephi"), multiDataTypeNodeProps, + List.of("Webatlas"), Map.of("0", "http://webatlas.fr", "1", 2.0f) + ); + + assertRelationship(rels.next(), "BAZ", + Map.of("score", 2.0f, "foo", "bar"), + List.of("Gephi"), multiDataTypeNodeProps, + List.of("Gephi"), multiDataTypeNodeProps + ); + + assertRelationship(rels.next(), "HAS_TICKET", Map.of("score", 3f, "ajeje", "brazorf"), + List.of("Gephi"), + multiDataTypeNodeProps, + List.of("RTGI"), + Map.of("0", "http://rtgi.fr", "1", 1.0f) + ); + + assertRelationship(rels.next(), "KNOWS", + Map.of(), + List.of("Gephi"), + multiDataTypeNodeProps, + List.of("RTGI"), + Map.of("0", "http://rtgi.fr", "1", 1.0f) + ); + + assertRelationship(rels.next(), "KNOWS", + Map.of(), + List.of("Webatlas"), + Map.of("0", "http://webatlas.fr", "1", 2.0f), + List.of("Gephi"), + multiDataTypeNodeProps + ); + + assertRelationship(rels.next(), "KNOWS", + Map.of(), + List.of("RTGI"), Map.of("0", "http://rtgi.fr", "1", 1.0f), + List.of("Webatlas"), Map.of("0", "http://webatlas.fr", "1", 2.0f) + ); + + assertRelationship(rels.next(), "KNOWS", + Map.of(), + List.of("Gephi"), + multiDataTypeNodeProps, + List.of("Webatlas", "BarabasiLab"), + Map.of("0", "http://barabasilab.com", "1", 1.0f, "2", false) + ); + + assertRelationship(rels.next(), "KNOWS", + Map.of(), + List.of("Gephi"), + Map.of("0", "http://test.gephi.org", "1", 2.0f), + List.of("Webatlas", "BarabasiLab"), + Map.of("0", "http://barabasilab.com", "1", 1.0f, "2", false) + ); + + assertFalse(rels.hasNext()); + } + ); + } +} diff --git a/full/src/test/resources/test_1.csv b/full/src/test/resources/test_1.csv new file mode 100644 index 0000000000..06ae461058 --- /dev/null +++ b/full/src/test/resources/test_1.csv @@ -0,0 +1,4 @@ +name,age +Bobby,20 +Oronzo,45 +Bonzo,22 diff --git a/full/src/test/resources/test_all.arrow b/full/src/test/resources/test_all.arrow new file mode 100644 index 0000000000..85e269fc10 Binary files /dev/null and b/full/src/test/resources/test_all.arrow differ diff --git a/full/src/test/resources/test_all.parquet b/full/src/test/resources/test_all.parquet new file mode 100644 index 0000000000..430cfdf88b Binary files /dev/null and b/full/src/test/resources/test_all.parquet differ diff --git a/test-utils/src/main/java/apoc/util/GoogleCloudStorageContainerExtension.java b/test-utils/src/main/java/apoc/util/GoogleCloudStorageContainerExtension.java index d06caaf74d..1d81cd4db2 100644 --- a/test-utils/src/main/java/apoc/util/GoogleCloudStorageContainerExtension.java +++ b/test-utils/src/main/java/apoc/util/GoogleCloudStorageContainerExtension.java @@ -42,4 +42,9 @@ public GoogleCloudStorageContainerExtension withMountedResourceFile(String resou this.withClasspathResourceMapping(resourceFilePath, "/data" + gcsPath, BindMode.READ_ONLY); return this; } + + public static String gcsUrl(GoogleCloudStorageContainerExtension gcs, String file) { + String path = String.format("b/folder/o/%s?alt=media", file); + return String.format("http://%s:%d/storage/v1/%s", gcs.getContainerIpAddress(), gcs.getMappedPort(4443), path); + } }