diff --git a/core/src/main/java/apoc/load/util/LoadCsvConfig.java b/core/src/main/java/apoc/load/util/LoadCsvConfig.java index 911f25f310..a8a2531bb0 100644 --- a/core/src/main/java/apoc/load/util/LoadCsvConfig.java +++ b/core/src/main/java/apoc/load/util/LoadCsvConfig.java @@ -24,19 +24,17 @@ import apoc.load.Mapping; import apoc.util.CompressionConfig; -import apoc.util.Util; import java.util.*; public class LoadCsvConfig extends CompressionConfig { public static final char DEFAULT_ARRAY_SEP = ';'; - public static final char DEFAULT_SEP = ','; + public static final String DEFAULT_SEP = ","; public static final char DEFAULT_QUOTE_CHAR = '"'; // this is the same value as ICSVParser.DEFAULT_ESCAPE_CHARACTER public static final char DEFAULT_ESCAPE_CHAR = '\\'; - private final boolean ignoreErrors; - private char separator; + private String separator; private char arraySep; private char quoteChar; private char escapeChar; @@ -59,8 +57,7 @@ public LoadCsvConfig(Map config) { if (config == null) { config = Collections.emptyMap(); } - ignoreErrors = Util.toBoolean(config.getOrDefault("ignoreErrors", false)); - separator = parseCharFromConfig(config, "sep", DEFAULT_SEP); + separator = parseStringFromConfig(config, "sep", DEFAULT_SEP); arraySep = parseCharFromConfig(config, "arraySep", DEFAULT_ARRAY_SEP); quoteChar = parseCharFromConfig(config, "quoteChar", DEFAULT_QUOTE_CHAR); escapeChar = parseCharFromConfig(config, "escapeChar", DEFAULT_ESCAPE_CHAR); @@ -83,6 +80,17 @@ public LoadCsvConfig(Map config) { mappings = createMapping(mapping, arraySep, ignore); } + private static String parseStringFromConfig(Map config, String key, String defaultValue) { + String separator = (String) config.getOrDefault(key, defaultValue); + if ("TAB".equals(separator)) { + return "\t"; + } + if ("NONE".equals(separator)) { + return "\0"; + } + return separator; + } + private Map createMapping( Map> mapping, char arraySep, List ignore) { if (mapping.isEmpty()) return Collections.emptyMap(); @@ -94,7 +102,7 @@ private Map createMapping( return result; } - public char getSeparator() { + public String getSeparator() { return separator; } @@ -146,10 +154,6 @@ public char getEscapeChar() { return escapeChar; } - public boolean getIgnoreErrors() { - return ignoreErrors; - } - public boolean isIgnoreQuotations() { return ignoreQuotations; } diff --git a/core/src/test/java/apoc/export/arrow/ArrowTest.java b/core/src/test/java/apoc/export/arrow/ArrowTest.java index f3fb0bd293..933c3f6f78 100644 --- a/core/src/test/java/apoc/export/arrow/ArrowTest.java +++ b/core/src/test/java/apoc/export/arrow/ArrowTest.java @@ -21,20 +21,15 @@ import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; import static org.junit.Assert.assertEquals; import apoc.ApocSettings; -import apoc.graph.Graphs; -import apoc.load.LoadArrow; -import apoc.meta.Meta; -import apoc.util.JsonUtil; -import apoc.util.TestUtil; -import com.fasterxml.jackson.core.JsonProcessingException; import java.io.File; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -65,75 +60,9 @@ public class ArrowTest { directory.toPath().toAbsolutePath()) .withSetting(ApocSettings.apoc_export_file_enabled, true); - public static final List> EXPECTED = List.of( - new HashMap<>() { - { - put("name", "Adam"); - put("bffSince", null); - put("", null); - put("", 0L); - put("age", 42L); - put("labels", List.of("User")); - put("male", true); - put("", null); - put("kids", List.of("Sam", "Anna", "Grace")); - put( - "place", - Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D)); - put("", null); - put("since", null); - put( - "born", - LocalDateTime.parse("2015-05-18T19:32:24.000") - .atOffset(ZoneOffset.UTC) - .toZonedDateTime()); - } - }, - new HashMap<>() { - { - put("name", "Jim"); - put("bffSince", null); - put("", null); - put("", 1L); - put("age", 42L); - put("labels", List.of("User")); - put("male", null); - put("", null); - put("kids", null); - put("place", null); - put("", null); - put("since", null); - put("born", null); - } - }, - new HashMap<>() { - { - put("name", null); - put("bffSince", "P5M1DT12H"); - put("", 0L); - put("", 0L); - put("age", null); - put("labels", null); - put("male", null); - put("", "KNOWS"); - put("kids", null); - put("place", null); - put("", 1L); - put("since", 1993L); - put("born", null); - } - }); - @BeforeClass public static void beforeClass() { - db.executeTransactionally( - "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); - TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, Graphs.class, Meta.class); - } - - @AfterClass - public static void teardown() { - db.shutdown(); + initDbCommon(db); } @Before @@ -142,21 +71,13 @@ public void before() { apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); } - private byte[] extractByteArray(Result result) { - return result.columnAs("byteArray").next(); - } - - private String extractFileName(Result result) { - return result.columnAs("file").next(); + @AfterClass + public static void teardown() { + db.shutdown(); } - private T readValue(String json, Class clazz) { - if (json == null) return null; - try { - return JsonUtil.OBJECT_MAPPER.readValue(json, clazz); - } catch (JsonProcessingException e) { - return null; - } + private byte[] extractByteArray(Result result) { + return result.columnAs("byteArray").next(); } @Test @@ -215,7 +136,7 @@ public void testFileRoundtripArrowQuery() { String file = db.executeTransactionally( "CALL apoc.export.arrow.query('query_test.arrow', $query) YIELD file", Map.of("query", returnQuery), - this::extractFileName); + ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; @@ -251,22 +172,7 @@ public void testStreamRoundtripArrowGraph() { // then final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); - } - - private List> getActual(Result result) { - return result.stream() - .map(m -> (Map) m.get("value")) - .map(m -> { - final Map newMap = new HashMap(m); - newMap.put("place", readValue((String) m.get("place"), Map.class)); - return newMap; - }) - .collect(Collectors.toList()); + testLoadArrow(db, query, Map.of("byteArray", byteArray)); } @Test @@ -277,15 +183,11 @@ public void testFileRoundtripArrowGraph() { + "CALL apoc.export.arrow.graph('graph_test.arrow', graph) YIELD file " + "RETURN file", Map.of(), - this::extractFileName); + ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("file", file), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("file", file)); } @Test @@ -310,26 +212,18 @@ private void testStreamRoundtripAllCommon() { // then final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("byteArray", byteArray)); } @Test public void testFileRoundtripArrowAll() { // given - when String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), this::extractFileName); + "CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), ArrowTestUtil::extractFileName); // then final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value"; - db.executeTransactionally(query, Map.of("file", file), result -> { - final List> actual = getActual(result); - assertEquals(EXPECTED, actual); - return null; - }); + testLoadArrow(db, query, Map.of("file", file)); } @Test @@ -365,7 +259,7 @@ public void testFileVolumeArrowAll() { String file = db.executeTransactionally( "CALL apoc.export.arrow.query('volume_test.arrow', 'MATCH (n:ArrowNode) RETURN n.id AS id') YIELD file ", Map.of(), - this::extractFileName); + ArrowTestUtil::extractFileName); final List expected = LongStream.range(0, 10000).mapToObj(l -> l).collect(Collectors.toList()); diff --git a/core/src/test/java/apoc/export/arrow/ArrowTestUtil.java b/core/src/test/java/apoc/export/arrow/ArrowTestUtil.java new file mode 100644 index 0000000000..9d96c08fe3 --- /dev/null +++ b/core/src/test/java/apoc/export/arrow/ArrowTestUtil.java @@ -0,0 +1,210 @@ +package apoc.export.arrow; + +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import apoc.graph.Graphs; +import apoc.load.LoadArrow; +import apoc.meta.Meta; +import apoc.util.JsonUtil; +import apoc.util.TestUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; +import org.neo4j.kernel.impl.util.ValueUtils; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.values.AnyValue; +import org.neo4j.values.storable.DurationValue; +import org.neo4j.values.storable.LocalDateTimeValue; +import org.neo4j.values.storable.PointValue; +import org.neo4j.values.virtual.VirtualValues; + +public class ArrowTestUtil { + public static String ARROW_BASE_FOLDER = "target/arrowImport"; + + public static final List> EXPECTED = List.of( + new HashMap<>() { + { + put("name", "Adam"); + put("bffSince", null); + put("", null); + put("", 0L); + put("age", 42L); + put("labels", List.of("User")); + put("male", true); + put("", null); + put("kids", List.of("Sam", "Anna", "Grace")); + put( + "place", + Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D)); + put("", null); + put("since", null); + put( + "born", + LocalDateTime.parse("2015-05-18T19:32:24.000") + .atOffset(ZoneOffset.UTC) + .toZonedDateTime()); + } + }, + new HashMap<>() { + { + put("name", "Jim"); + put("bffSince", null); + put("", null); + put("", 1L); + put("age", 42L); + put("labels", List.of("User")); + put("male", null); + put("", null); + put("kids", null); + put("place", null); + put("", null); + put("since", null); + put("born", null); + } + }, + new HashMap<>() { + { + put("name", null); + put("bffSince", "P5M1DT12H"); + put("", 0L); + put("", 0L); + put("age", null); + put("labels", null); + put("male", null); + put("", "KNOWS"); + put("kids", null); + put("place", null); + put("", 1L); + put("since", 1993L); + put("born", null); + } + }); + + public static void initDbCommon(GraphDatabaseService db) { + db.executeTransactionally( + "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); + TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, Graphs.class, Meta.class); + } + + public static final Map MAPPING_ALL = Map.of( + "mapping", + Map.of("bffSince", "Duration", "place", "Point", "listInt", "LongArray", "born", "LocalDateTime")); + + public static void createNodesForImportTests(DbmsRule db) { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + } + + public static void testLoadArrow(GraphDatabaseService db, String query, Map params) { + db.executeTransactionally(query, params, result -> { + final List> actual = getActual(result); + assertEquals(EXPECTED, actual); + return null; + }); + } + + private static List> getActual(Result result) { + return result.stream() + .map(m -> (Map) m.get("value")) + .map(m -> { + final Map newMap = new HashMap(m); + newMap.put("place", readValue((String) m.get("place"), Map.class)); + return newMap; + }) + .collect(Collectors.toList()); + } + + private static T readValue(String json, Class clazz) { + if (json == null) return null; + try { + return JsonUtil.OBJECT_MAPPER.readValue(json, clazz); + } catch (JsonProcessingException e) { + return null; + } + } + + public static void testImportCommon(DbmsRule db, Object file, Map config) { + // then + Map params = Map.of("file", file, "config", config); + + // remove current data + db.executeTransactionally("MATCH (n) DETACH DELETE n"); + + final String query = "CALL apoc.import.arrow($file, $config)"; + testCall(db, query, params, r -> { + assertEquals(4L, r.get("nodes")); + assertEquals(1L, r.get("relationships")); + }); + + testCall(db, "MATCH (start:User)-[rel:KNOWS]->(end:User) RETURN start, rel, end", r -> { + Node start = (Node) r.get("start"); + assertFirstUserNodeProps(start.getAllProperties()); + Node end = (Node) r.get("end"); + assertSecondUserNodeProps(end.getAllProperties()); + Relationship rel = (Relationship) r.get("rel"); + assertRelationshipProps(rel.getAllProperties()); + }); + + testResult(db, "MATCH (m:Another) RETURN m", r -> { + ResourceIterator m = r.columnAs("m"); + Node node = m.next(); + assertFirstAnotherNodeProps(node.getAllProperties()); + node = m.next(); + assertSecondAnotherNodeProps(node.getAllProperties()); + assertFalse(r.hasNext()); + }); + } + + public static String extractFileName(Result result) { + return result.columnAs("file").next(); + } + + public static byte[] extractByteArray(Result result) { + return result.columnAs("value").next(); + } + + public static void assertFirstUserNodeProps(Map props) { + assertEquals("Adam", props.get("name")); + assertEquals(42L, props.get("age")); + assertEquals(true, props.get("male")); + assertArrayEquals(new String[] {"Sam", "Anna", "Grace"}, (String[]) props.get("kids")); + Map latitude = Map.of("latitude", 13.1D, "longitude", 33.46789D, "height", 100.0D); + assertEquals( + PointValue.fromMap(VirtualValues.map( + latitude.keySet().toArray(new String[0]), + latitude.values().stream().map(ValueUtils::of).toArray(AnyValue[]::new))), + props.get("place")); + assertEquals(LocalDateTimeValue.parse("2015-05-18T19:32:24.000").asObject(), props.get("born")); + } + + public static void assertSecondUserNodeProps(Map props) { + assertEquals("Jim", props.get("name")); + assertEquals(42L, props.get("age")); + } + + public static void assertFirstAnotherNodeProps(Map map) { + assertEquals(1L, map.get("foo")); + assertArrayEquals(new long[] {1L, 2L}, (long[]) map.get("listInt")); + } + + public static void assertSecondAnotherNodeProps(Map map) { + assertEquals("Sam", map.get("bar")); + } + + public static void assertRelationshipProps(Map props) { + assertEquals(DurationValue.parse("P5M1DT12H"), props.get("bffSince")); + assertEquals(1993L, props.get("since")); + } +} diff --git a/core/src/test/java/apoc/util/s3/S3TestUtil.java b/core/src/test/java/apoc/util/s3/S3TestUtil.java index 3fde37e95b..274f45c67d 100644 --- a/core/src/test/java/apoc/util/s3/S3TestUtil.java +++ b/core/src/test/java/apoc/util/s3/S3TestUtil.java @@ -83,4 +83,13 @@ public static void assertS3KeyEventually(Runnable runnable) { 30L, TimeUnit.SECONDS); } + + public static String removeRegionFromUrl(S3Container s3Container, String url) { + return url.replace(s3Container.getEndpointConfiguration().getSigningRegion() + ".", ""); + } + + public static String putToS3AndGetUrl(S3Container s3Container, String filename) { + String url = s3Container.putFile(filename); + return removeRegionFromUrl(s3Container, url); + } } diff --git a/docs/asciidoc/modules/ROOT/pages/import/load-csv.adoc b/docs/asciidoc/modules/ROOT/pages/import/load-csv.adoc index 1803b0904b..e09508c929 100644 --- a/docs/asciidoc/modules/ROOT/pages/import/load-csv.adoc +++ b/docs/asciidoc/modules/ROOT/pages/import/load-csv.adoc @@ -74,7 +74,7 @@ Besides the file you can pass in a config map: | skip | none | skip result rows | limit | none | limit result rows | header | true | indicates if file has a header -| sep | ',' | separator character or 'TAB' +| sep | ',' | separator string or 'TAB' | quoteChar | '"' | the char to use for quoted elements | arraySep | ';' | array separator | ignore | [] | which columns to ignore @@ -115,6 +115,38 @@ RETURN *; | 1 | [12,["Tea","Milk"]] | {"age":12,"drinks":["Tea","Milk"]} |=== + +Given the following csv: + +.testMultiCharSep.csv +[source,csv] +---- +nameSEPARage +SelmaSEPAR8 +RanaSEPAR11 +SelinaSEPAR18 + +---- + +we can execute: + +[source,cypher] +---- +CALL apoc.load.csv('testMultiCharSep.csv', {sep:'SEPAR'}) +YIELD lineNo, map, list +RETURN *; +---- + + +.Results +[opts="header",cols="1,2,2"] +|=== +| lineNo | list | map +| 0 | ["Selma", "9"] | {name: "Selma", age: "9"} +| 1 | ["Rana", "12"] | {name: "Rana", age: "12"} +| 2 | ["Selina", "19"] | {name: "Selina", age: "19"} +|=== + == Transaction Batching To handle large files, `USING PERIODIC COMMIT` can be prepended to `LOAD CSV`, you'll have to watch out though for *Eager* operations which might break that behavior. diff --git a/extended/src/test/java/apoc/load/GexfTest.java b/extended/src/test/java/apoc/load/GexfTest.java new file mode 100644 index 0000000000..bc7dd02e37 --- /dev/null +++ b/extended/src/test/java/apoc/load/GexfTest.java @@ -0,0 +1,111 @@ +package apoc.load; + +import apoc.util.TestUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.graphdb.Relationship; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +import java.util.List; +import java.util.Map; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.util.ExtendedTestUtil.assertRelationship; +import static apoc.util.GexfTestUtil.testImportGexfCommon; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class GexfTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void setup() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + TestUtil.registerProcedure(db, Gexf.class); + } + + @After + public void tearDown() { + db.shutdown(); + } + + @Test + public void testLoadGexf() { + final String file = ClassLoader.getSystemResource("gexf/single-node.gexf").toString(); + testCall( + db, + "CALL apoc.load.gexf($file)", + Map.of("file", file), + (row) -> { + Map value = (Map) row.get("value"); + String expected = "{_type=gexf, _children=[{_type=graph, defaultedgetype=directed, _children=[{_type=nodes, _children=[{_type=node, _children=[{_type=attvalues, _children=[{_type=attvalue, for=0, value=http://gephi.org}]}], id=0, label=bar}]}]}], version=1.2}"; + assertEquals(expected, value.toString()); + }); + } + + @Test + public void testImportGexf() { + final String file = ClassLoader.getSystemResource("gexf/data.gexf").toString(); + testImportGexfCommon(db, file); + } + + @Test + public void testImportGexfWithStoreNodeIds() { + final String file = ClassLoader.getSystemResource("gexf/single-node.gexf").toString(); + TestUtil.testCall( + db, + "CALL apoc.import.gexf($file, {storeNodeIds: true})", + map("file", file), + (r) -> { + assertEquals("gexf", r.get("format")); + assertEquals(1L, r.get("nodes")); + }); + + Map props = TestUtil.singleResultFirstColumn(db, "MATCH (n) RETURN properties(n) AS props"); + assertEquals("http://gephi.org", props.get("0")); + assertTrue( props.containsKey("id") ); + } + + @Test + public void testImportGexfWithDefaultRelationshipTypeSourceAndTargetConfigs() { + String defaultRelType = "TEST_DEFAULT"; + final String file = ClassLoader.getSystemResource("gexf/single-rel.gexf").toString(); + + db.executeTransactionally("CREATE (:Foo {startId: 'start'})"); + db.executeTransactionally("CREATE (:Bar {endId: 'end'})"); + + TestUtil.testCall( + db, + "CALL apoc.import.gexf($file, {defaultRelationshipType: $defaultRelType, source: $source, target: $target})", + map("file", file, + "defaultRelType", defaultRelType, + "source", map("label", "Foo", "id", "startId"), + "target", map("label", "Bar", "id", "endId") + ), + (r) -> { + assertEquals("gexf", r.get("format")); + assertEquals(1L, r.get("relationships")); + }); + + TestUtil.testCall(db, "MATCH ()-[rel]->() RETURN rel", r -> { + Relationship rel = (Relationship) r.get("rel"); + assertRelationship(rel, defaultRelType, + Map.of(), + List.of("Foo"), + Map.of("startId", "start"), + List.of("Bar"), + Map.of("endId", "end") + ); + }); + } +} diff --git a/full-it/build.gradle b/full-it/build.gradle index 7edc0248d2..1f2410c7a2 100644 --- a/full-it/build.gradle +++ b/full-it/build.gradle @@ -15,8 +15,15 @@ dependencies { testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.770' testImplementation group: 'org.xmlunit', name: 'xmlunit-core', version: '2.9.1' + testImplementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1' + testImplementation group: 'org.apache.poi', name: 'poi', version: '5.1.0' + testImplementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.1.0' + testImplementation 'com.azure:azure-storage-blob:12.22.0' testImplementation group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0' + testImplementation 'org.mock-server:mockserver-netty:5.15.0' + testImplementation 'org.mock-server:mockserver-client-java:5.15.0' + configurations.all { exclude group: 'org.slf4j', module: 'slf4j-nop' exclude group: 'ch.qos.logback', module: 'logback-classic' diff --git a/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java new file mode 100644 index 0000000000..03953ee64b --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/ArrowAzureStorageTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.full.it.azure; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +import apoc.export.arrow.ArrowTestUtil; +import java.util.Map; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +@Ignore( + "This test won't work until the Azure Storage files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = putToAzureStorageAndGetUrl("test_all.arrow"); + + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = putToAzureStorageAndGetUrl("test_all_import.arrow"); + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java b/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java new file mode 100644 index 0000000000..f8148dd877 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/AzureStorageBaseTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.full.it.azure; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.sas.BlobSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +public class AzureStorageBaseTest { + + public static GenericContainer azuriteContainer; + public static BlobContainerClient containerClient; + + @BeforeClass + public static void setUp() throws Exception { + DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite"); + azuriteContainer = new GenericContainer<>(azuriteImg).withExposedPorts(10000); + + azuriteContainer.start(); + + var accountName = "devstoreaccount1"; + var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + var blobEndpoint = String.format( + "http://%s:%d/%s", azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName); + var connectionString = String.format( + "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;", + accountName, accountKey, blobEndpoint); + + containerClient = new BlobContainerClientBuilder() + .connectionString(connectionString) + .containerName("test-container") + .buildClient(); + containerClient.create(); + } + + @AfterClass + public static void teardown() { + azuriteContainer.close(); + } + + public static String putToAzureStorageAndGetUrl(String url) { + try { + File file = new File(url); + byte[] content = FileUtils.readFileToByteArray(file); + + var blobClient = getBlobClient(content); + BlobSasPermission permission = new BlobSasPermission().setReadPermission(true); + OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1); + String sasToken = blobClient.generateSas( + new BlobServiceSasSignatureValues(expiryTime, permission), + new Context("Azure-Storage-Log-String-To-Sign", "true")); + return blobClient.getBlobUrl() + "?" + sasToken; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static BlobClient getBlobClient(byte[] content) { + var blobName = "blob-" + UUID.randomUUID(); + var blobClient = containerClient.getBlobClient(blobName); + blobClient.upload(new ByteArrayInputStream(content)); + return blobClient; + } +} diff --git a/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java new file mode 100644 index 0000000000..99420d4fd3 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/ImportAzureStorageTest.java @@ -0,0 +1,42 @@ +package apoc.full.it.azure; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; + +import apoc.export.arrow.ImportArrow; +import apoc.util.TestUtil; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class ImportAzureStorageTest extends AzureStorageBaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + TestUtil.registerProcedure(db, ImportArrow.class); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToAzureStorageAndGetUrl(fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java b/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java new file mode 100644 index 0000000000..b19c2b442a --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/azure/LoadAzureStorageTest.java @@ -0,0 +1,70 @@ +package apoc.full.it.azure; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.full.it.util.ExtendedITUtil.testLoadXmlCommon; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.LoadXlsTest.testLoadXlsCommon; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadXls; +import apoc.load.Xml; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class LoadAzureStorageTest extends AzureStorageBaseTest { + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + AzureStorageBaseTest.setUp(); + + TestUtil.registerProcedure( + db, LoadCsv.class, LoadDirectory.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + @Test + public void testLoadCsv() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToAzureStorageAndGetUrl(EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } +} diff --git a/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java new file mode 100644 index 0000000000..0a173a4cdf --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/ArrowGoogleCloudStorageTest.java @@ -0,0 +1,59 @@ +package apoc.full.it.gc; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.util.GoogleCloudStorageContainerExtension; +import java.util.Map; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +@Ignore( + "This test won't work until the Google Cloud files will be correctly handled via FileUtils, placed in APOC Core") +public class ArrowGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension(); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + initDbCommon(db); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = gcsUrl(gcs, "test_all.arrow"); + + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = gcsUrl(gcs, "test_all_import.arrow"); + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java new file mode 100644 index 0000000000..dadbe684f5 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/ImportGoogleCloudStorageTest.java @@ -0,0 +1,45 @@ +package apoc.full.it.gc; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; + +import apoc.export.arrow.ImportArrow; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class ImportGoogleCloudStorageTest { + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test_all.arrow", "/folder/test_all.arrow"); + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + + gcs.start(); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + TestUtil.registerProcedure(db, ImportArrow.class); + createNodesForImportTests(db); + + String url = gcsUrl(gcs, "test_all.arrow"); + testImportCommon(db, url, MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java b/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java new file mode 100644 index 0000000000..b7781efc86 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/gc/LoadGoogleCloudStorageTest.java @@ -0,0 +1,139 @@ +package apoc.full.it.gc; + +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.load.LoadCsvTest.assertRow; +import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl; +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import apoc.load.LoadCsv; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadXls; +import apoc.load.Xml; +import apoc.util.GoogleCloudStorageContainerExtension; +import apoc.util.TestUtil; +import apoc.util.Util; +import apoc.xml.XmlTestUtils; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.Result; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class LoadGoogleCloudStorageTest { + + public static GoogleCloudStorageContainerExtension gcs = new GoogleCloudStorageContainerExtension() + .withMountedResourceFile("test.csv", "/folder/test.csv") + .withMountedResourceFile("map.json", "/folder/map.json") + .withMountedResourceFile("xml/books.xml", "/folder/books.xml") + .withMountedResourceFile("load_test.xlsx", "/folder/load_test.xlsx") + .withMountedResourceFile("wikipedia.html", "/folder/wikipedia.html"); + + @ClassRule + public static DbmsRule db = new ImpermanentDbmsRule(); + + @BeforeClass + public static void setUp() throws Exception { + gcs.start(); + TestUtil.registerProcedure(db, LoadCsv.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class); + } + + @AfterClass + public static void tearDown() { + gcs.close(); + db.shutdown(); + } + + @Test + public void testLoadCsv() { + String url = gcsUrl(gcs, "test.csv"); + + testResult(db, "CALL apoc.load.csv($url)", map("url", url), (r) -> { + assertRow(r, "Selma", "8", 0L); + assertRow(r, "Rana", "11", 1L); + assertRow(r, "Selina", "18", 2L); + assertFalse("It should be the last record", r.hasNext()); + }); + } + + @Test + public void testLoadJSON() { + String url = gcsUrl(gcs, "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = gcsUrl(gcs, "books.xml"); + testCall( + db, + "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", + Util.map("url", url), + (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + @Test + public void testLoadXls() { + String url = gcsUrl(gcs, "load_test.xlsx"); + testResult( + db, + "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", + map("url", url), // 'file:load_test.xlsx' + (r) -> { + assertXlsRow( + r, + 0L, + "String", + "Test", + "Boolean", + true, + "Integer", + 2L, + "Float", + 1.5d, + "Array", + asList(1L, 2L, 3L)); + assertFalse("Should not have another row", r.hasNext()); + }); + } + + @Test + public void testLoadHtml() { + String url = gcsUrl(gcs, "wikipedia.html"); + + Map query = map("links", "a[href]"); + + testCall(db, "CALL apoc.load.html($url,$query)", map("url", url, "query", query), row -> { + final List> actual = (List) ((Map) row.get("value")).get("links"); + assertEquals(106, actual.size()); + assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); + }); + } + + static void assertXlsRow(Result r, long lineNo, Object... data) { + Map row = r.next(); + Map map = map(data); + assertEquals(map, row.get("map")); + Map stringMap = new LinkedHashMap<>(map.size()); + map.forEach((k, v) -> stringMap.put(k, v == null ? null : v.toString())); + assertEquals(new ArrayList<>(map.values()), row.get("list")); + assertEquals(lineNo, row.get("lineNo")); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java b/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java new file mode 100644 index 0000000000..50692bb71c --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/ArrowS3Test.java @@ -0,0 +1,57 @@ +package apoc.full.it.s3; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.export.arrow.ArrowTestUtil.testLoadArrow; + +import apoc.export.arrow.ArrowTestUtil; +import apoc.export.arrow.ImportArrow; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import java.util.Map; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class ArrowS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void beforeClass() { + initDbCommon(db); + TestUtil.registerProcedure(db, ImportArrow.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testFileRoundtripWithLoadArrow() { + String url = s3Container.getUrl("test_all.arrow"); + + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + final String query = "CALL apoc.load.arrow($file, {})"; + testLoadArrow(db, query, Map.of("file", file)); + } + + @Test + public void testFileRoundtripWithImportArrow() { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + + String url = s3Container.getUrl("test_all_import.arrow"); + String file = db.executeTransactionally( + "CALL apoc.export.arrow.all($url) YIELD file", Map.of("url", url), ArrowTestUtil::extractFileName); + + // check that the exported file is correct + testImportCommon(db, file, ArrowTestUtil.MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java b/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java new file mode 100644 index 0000000000..c0eb558e18 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/ImportS3Test.java @@ -0,0 +1,55 @@ +package apoc.full.it.s3; + +import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.export.arrow.ArrowTestUtil.ARROW_BASE_FOLDER; +import static apoc.export.arrow.ArrowTestUtil.MAPPING_ALL; +import static apoc.export.arrow.ArrowTestUtil.createNodesForImportTests; +import static apoc.export.arrow.ArrowTestUtil.initDbCommon; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.util.s3.S3TestUtil.putToS3AndGetUrl; + +import apoc.export.arrow.ImportArrow; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import java.io.File; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class ImportS3Test extends S3BaseTest { + private static File directory = new File(ARROW_BASE_FOLDER); + + static { //noinspection ResultOfMethodCallIgnored + directory.mkdirs(); + } + + @Rule + public DbmsRule db = new ImpermanentDbmsRule() + .withSetting( + GraphDatabaseSettings.load_csv_file_url_root, + directory.toPath().toAbsolutePath()); + + @BeforeClass + public static void beforeClass() { + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); + } + + @Test + public void testImportArrow() { + initDbCommon(db); + TestUtil.registerProcedure(db, ImportArrow.class); + createNodesForImportTests(db); + + String fileWithPath = EXTENDED_RESOURCES_PATH + "test_all.arrow"; + String url = putToS3AndGetUrl(s3Container, fileWithPath); + + testImportCommon(db, url, MAPPING_ALL); + } +} diff --git a/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java b/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java new file mode 100644 index 0000000000..9268f366e9 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/s3/LoadS3Test.java @@ -0,0 +1,70 @@ +package apoc.full.it.s3; + +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG; +import static apoc.ApocConfig.apocConfig; +import static apoc.full.it.util.ExtendedITUtil.EXTENDED_RESOURCES_PATH; +import static apoc.full.it.util.ExtendedITUtil.testLoadJsonCommon; +import static apoc.full.it.util.ExtendedITUtil.testLoadXmlCommon; +import static apoc.load.LoadCsvTest.commonTestLoadCsv; +import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon; +import static apoc.load.LoadXlsTest.testLoadXlsCommon; +import static apoc.util.s3.S3TestUtil.putToS3AndGetUrl; + +import apoc.load.LoadCsv; +import apoc.load.LoadDirectory; +import apoc.load.LoadHtml; +import apoc.load.LoadJson; +import apoc.load.LoadXls; +import apoc.load.Xml; +import apoc.util.TestUtil; +import apoc.util.s3.S3BaseTest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.test.rule.ImpermanentDbmsRule; + +public class LoadS3Test extends S3BaseTest { + + @Rule + public DbmsRule db = new ImpermanentDbmsRule(); + + @Before + public void setUp() throws Exception { + TestUtil.registerProcedure( + db, LoadCsv.class, LoadDirectory.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); + apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false); + } + + @Test + public void testLoadCsv() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "test.csv"); + commonTestLoadCsv(db, url); + } + + @Test + public void testLoadJson() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "map.json"); + testLoadJsonCommon(db, url); + } + + @Test + public void testLoadXml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "xml/books.xml"); + testLoadXmlCommon(db, url); + } + + @Test + public void testLoadXls() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "load_test.xlsx"); + testLoadXlsCommon(db, url); + } + + @Test + public void testLoadHtml() { + String url = putToS3AndGetUrl(s3Container, EXTENDED_RESOURCES_PATH + "wikipedia.html"); + testLoadHtmlWithGetLinksCommon(db, url); + } +} diff --git a/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java b/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java new file mode 100644 index 0000000000..68d29c98c1 --- /dev/null +++ b/full-it/src/test/java/apoc/full/it/util/ExtendedITUtil.java @@ -0,0 +1,35 @@ +package apoc.full.it.util; + +import static apoc.util.MapUtil.map; +import static apoc.util.TestUtil.testCall; +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + +import apoc.util.Util; +import apoc.xml.XmlTestUtils; +import org.junit.Assert; +import org.neo4j.driver.internal.util.Iterables; +import org.neo4j.graphdb.GraphDatabaseService; + +public class ExtendedITUtil { + public static String RESOURCES_PATH = "src/test/resources/"; + public static final String EXTENDED_PATH = "../full/"; + public static final String EXTENDED_RESOURCES_PATH = EXTENDED_PATH + RESOURCES_PATH; + + public static void testLoadXmlCommon(GraphDatabaseService db, String url) { + testCall( + db, + "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", + Util.map("url", url), + (r) -> { + Object value = Iterables.single(r.values()); + Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); + }); + } + + public static void testLoadJsonCommon(GraphDatabaseService db, String url) { + testCall(db, "CALL apoc.load.jsonParams($url, null, null)", map("url", url), (row) -> { + assertEquals(map("foo", asList(1L, 2L, 3L)), row.get("value")); + }); + } +} diff --git a/full/build.gradle b/full/build.gradle index 50f1ced756..94d3704210 100644 --- a/full/build.gradle +++ b/full/build.gradle @@ -151,7 +151,11 @@ dependencies { compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-comprehend', version: '1.12.770' , withoutJacksons testImplementation group: 'com.amazonaws', name: 'aws-java-sdk-comprehend', version: '1.12.770' , withoutJacksons - implementation group: 'com.opencsv', name: 'opencsv', version: '5.7.1' + implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4' + implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.10.0', { + exclude group: 'org.apache.commons', module: 'commons-io' + exclude group: 'org.apache.commons', module: 'commons-lang3' + } implementation group: 'commons-beanutils', name: 'commons-beanutils', version: '1.9.4' implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.17.0', withoutJacksons @@ -183,6 +187,8 @@ dependencies { testImplementation group: 'com.sun.mail', name: 'javax.mail', version: '1.6.0' testImplementation group: 'org.zapodot', name: 'embedded-ldap-junit', version: '0.9.0' testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.4.0' + testImplementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: '1.13.1', withoutServers + testImplementation group: 'com.opencsv', name: 'opencsv', version: '5.7.1' configurations.all { exclude group: 'org.slf4j', module: 'slf4j-nop' diff --git a/full/src/main/java/apoc/load/LoadCsv.java b/full/src/main/java/apoc/load/LoadCsv.java index 9fa19ee849..cb1fb00b6a 100644 --- a/full/src/main/java/apoc/load/LoadCsv.java +++ b/full/src/main/java/apoc/load/LoadCsv.java @@ -20,6 +20,7 @@ import static apoc.util.FileUtils.closeReaderSafely; import static apoc.util.Util.cleanUrl; +import static apoc.util.Util.isSumOutOfRange; import static java.util.Collections.emptyList; import apoc.Extended; @@ -28,15 +29,13 @@ import apoc.load.util.Results; import apoc.util.FileUtils; import apoc.util.Util; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; import java.io.IOException; import java.util.*; import java.util.function.Consumer; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -45,6 +44,8 @@ @Extended public class LoadCsv { + public static final String ERROR_WRONG_COL_SEPARATOR = + ". Please check whether you included a delimiter before a column separator or forgot a column separator."; @Context public GraphDatabaseService db; @@ -77,7 +78,7 @@ public Stream csvParams( } reader = FileUtils.readerFor(urlOrBinary, httpHeaders, payload, config.getCompressionAlgo()); return streamCsv(url, config, reader); - } catch (IOException | CsvValidationException e) { + } catch (Exception e) { closeReaderSafely(reader); if (!config.isFailOnError()) return Stream.of(new CSVResult( @@ -99,46 +100,50 @@ public Stream csvParams( } public Stream streamCsv(@Name("url") String url, LoadCsvConfig config, CountingReader reader) - throws IOException, CsvValidationException { - - CSVReader csv = new CSVReaderBuilder(reader) - .withCSVParser(new CSVParserBuilder() - .withEscapeChar(config.getEscapeChar()) - .withQuoteChar(config.getQuoteChar()) - .withIgnoreQuotations(config.isIgnoreQuotations()) - .withSeparator(config.getSeparator()) - .build()) + throws IOException { + + CSVFormat csvFormat = CSVFormat.DEFAULT + .builder() + .setEscape(config.getEscapeChar()) + .setQuote(config.isIgnoreQuotations() ? '\0' : config.getQuoteChar()) + .setDelimiter(config.getSeparator()) .build(); - String[] header = getHeader(csv, config); + Iterator csvIterator = csvFormat.parse(reader).iterator(); + + String[] header = getHeader(csvIterator, config); boolean checkIgnore = !config.getIgnore().isEmpty() || config.getMappings().values().stream().anyMatch(m -> m.ignore); return StreamSupport.stream( - new CSVSpliterator( - csv, - header, - url, - config.getSkip(), - config.getLimit(), - checkIgnore, - config.getMappings(), - config.getNullValues(), - config.getResults(), - config.getIgnoreErrors()), - false) - .onClose(() -> closeReaderSafely(reader)); + new CSVSpliterator( + csvIterator, + header, + url, + config.getSkip(), + config.getLimit(), + checkIgnore, + config.getMappings(), + config.getNullValues(), + config.getResults(), + config.isIgnoreQuotations(), + config.getQuoteChar(), + config.isFailOnError()), + false); } - private String[] getHeader(CSVReader csv, LoadCsvConfig config) throws IOException, CsvValidationException { + private static final Mapping EMPTY = + new Mapping("", Collections.emptyMap(), LoadCsvConfig.DEFAULT_ARRAY_SEP, false); + + private String[] getHeader(Iterator csv, LoadCsvConfig config) throws IOException { if (!config.isHasHeader()) return null; - String[] headers = csv.readNext(); + String[] headers = csv.next().values(); List ignore = config.getIgnore(); if (ignore.isEmpty()) return headers; Map mappings = config.getMappings(); for (int i = 0; i < headers.length; i++) { String header = headers[i]; - if (ignore.contains(header) || mappings.getOrDefault(header, Mapping.EMPTY).ignore) { + if (ignore.contains(header) || mappings.getOrDefault(header, EMPTY).ignore) { headers[i] = null; } } @@ -146,7 +151,7 @@ private String[] getHeader(CSVReader csv, LoadCsvConfig config) throws IOExcepti } private static class CSVSpliterator extends Spliterators.AbstractSpliterator { - private final CSVReader csv; + private final Iterator csv; private final String[] header; private final String url; private final long limit; @@ -154,11 +159,13 @@ private static class CSVSpliterator extends Spliterators.AbstractSpliterator mapping; private final List nullValues; private final EnumSet results; - private final boolean ignoreErrors; + private final boolean failOnError; + private final boolean ignoreQuotations; + private final String quoteChar; long lineNo; public CSVSpliterator( - CSVReader csv, + Iterator csv, String[] header, String url, long skip, @@ -167,8 +174,9 @@ public CSVSpliterator( Map mapping, List nullValues, EnumSet results, - boolean ignoreErrors) - throws IOException, CsvValidationException { + boolean ignoreQuotations, + char quoteChar, + boolean failOnError) { super(Long.MAX_VALUE, Spliterator.ORDERED); this.csv = csv; this.header = header; @@ -177,33 +185,50 @@ public CSVSpliterator( this.mapping = mapping; this.nullValues = nullValues; this.results = results; - this.ignoreErrors = ignoreErrors; - this.limit = Util.isSumOutOfRange(skip, limit) ? Long.MAX_VALUE : (skip + limit); + this.failOnError = failOnError; + this.limit = isSumOutOfRange(skip, limit) ? Long.MAX_VALUE : (skip + limit); lineNo = skip; + this.ignoreQuotations = ignoreQuotations; + this.quoteChar = String.valueOf(quoteChar); while (skip-- > 0) { - csv.readNext(); + csv.next(); } } @Override public boolean tryAdvance(Consumer action) { + final String message = + "Error reading CSV from " + (url == null ? "binary" : " URL " + cleanUrl(url)) + " at " + lineNo; try { - String[] row = csv.readNext(); - if (row != null && lineNo < limit) { + if (csv.hasNext() && lineNo < limit) { + String[] row = csv.next().values(); + removeQuotes(row, ignoreQuotations, quoteChar); action.accept(new CSVResult(header, row, lineNo, ignore, mapping, nullValues, results)); lineNo++; return true; } return false; - } catch (IOException | CsvValidationException e) { - throw new RuntimeException( - "Error reading CSV from " + (url == null ? "binary" : " URL " + cleanUrl(url)) + " at " - + lineNo, - e); } catch (ArrayIndexOutOfBoundsException e) { - throw new RuntimeException( - "Error reading CSV from " + (url == null ? "binary" : " URL " + cleanUrl(url)) + " at " + lineNo - + ". Please check whether you included a delimiter before a column separator or forgot a column separator."); + String messageIdxOfBound = message + ERROR_WRONG_COL_SEPARATOR; + RuntimeException exception = new RuntimeException(messageIdxOfBound); + return skipOrFail(exception); + } + } + + private boolean skipOrFail(RuntimeException exception) { + if (failOnError) { + throw exception; + } + lineNo++; + return true; + } + + private void removeQuotes(String[] row, boolean ignoreQuotations, String quoteChar) { + if (!ignoreQuotations) { + return; + } + for (int i = 0; i < row.length; i++) { + row[i] = row[i].replace(quoteChar, ""); } } } diff --git a/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java b/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java new file mode 100644 index 0000000000..d20f2c2cc8 --- /dev/null +++ b/full/src/test/java/apoc/export/arrow/ArrowTestUtil.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.export.arrow; + +import static apoc.util.TestUtil.testCall; +import static apoc.util.TestUtil.testResult; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import apoc.graph.Graphs; +import apoc.load.LoadArrow; +import apoc.meta.Meta; +import apoc.util.JsonUtil; +import apoc.util.TestUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; +import org.neo4j.kernel.impl.util.ValueUtils; +import org.neo4j.test.rule.DbmsRule; +import org.neo4j.values.AnyValue; +import org.neo4j.values.storable.DurationValue; +import org.neo4j.values.storable.LocalDateTimeValue; +import org.neo4j.values.storable.PointValue; +import org.neo4j.values.virtual.VirtualValues; + +public class ArrowTestUtil { + public static String ARROW_BASE_FOLDER = "target/arrowImport"; + + public static final List> EXPECTED = List.of( + new HashMap<>() { + { + put("name", "Adam"); + put("bffSince", null); + put("", null); + put("", 0L); + put("age", 42L); + put("labels", List.of("User")); + put("male", true); + put("", null); + put("kids", List.of("Sam", "Anna", "Grace")); + put( + "place", + Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D)); + put("", null); + put("since", null); + put( + "born", + LocalDateTime.parse("2015-05-18T19:32:24.000") + .atOffset(ZoneOffset.UTC) + .toZonedDateTime()); + } + }, + new HashMap<>() { + { + put("name", "Jim"); + put("bffSince", null); + put("", null); + put("", 1L); + put("age", 42L); + put("labels", List.of("User")); + put("male", null); + put("", null); + put("kids", null); + put("place", null); + put("", null); + put("since", null); + put("born", null); + } + }, + new HashMap<>() { + { + put("name", null); + put("bffSince", "P5M1DT12H"); + put("", 0L); + put("", 0L); + put("age", null); + put("labels", null); + put("male", null); + put("", "KNOWS"); + put("kids", null); + put("place", null); + put("", 1L); + put("since", 1993L); + put("born", null); + } + }); + + public static void initDbCommon(GraphDatabaseService db) { + db.executeTransactionally( + "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); + TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, ImportArrow.class, Graphs.class, Meta.class); + } + + public static final Map MAPPING_ALL = Map.of( + "mapping", + Map.of("bffSince", "Duration", "place", "Point", "listInt", "LongArray", "born", "LocalDateTime")); + + public static void createNodesForImportTests(DbmsRule db) { + db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); + } + + public static void testLoadArrow(GraphDatabaseService db, String query, Map params) { + db.executeTransactionally(query, params, result -> { + final List> actual = getActual(result); + assertEquals(EXPECTED, actual); + return null; + }); + } + + private static List> getActual(Result result) { + return result.stream() + .map(m -> (Map) m.get("value")) + .map(m -> { + final Map newMap = new HashMap(m); + newMap.put("place", readValue((String) m.get("place"), Map.class)); + return newMap; + }) + .collect(Collectors.toList()); + } + + private static T readValue(String json, Class clazz) { + if (json == null) return null; + try { + return JsonUtil.OBJECT_MAPPER.readValue(json, clazz); + } catch (JsonProcessingException e) { + return null; + } + } + + public static void testImportCommon(DbmsRule db, Object file, Map config) { + // then + Map params = Map.of("file", file, "config", config); + + // remove current data + db.executeTransactionally("MATCH (n) DETACH DELETE n"); + + final String query = "CALL apoc.import.arrow($file, $config)"; + testCall(db, query, params, r -> { + assertEquals(4L, r.get("nodes")); + assertEquals(1L, r.get("relationships")); + }); + + testCall(db, "MATCH (start:User)-[rel:KNOWS]->(end:User) RETURN start, rel, end", r -> { + Node start = (Node) r.get("start"); + assertFirstUserNodeProps(start.getAllProperties()); + Node end = (Node) r.get("end"); + assertSecondUserNodeProps(end.getAllProperties()); + Relationship rel = (Relationship) r.get("rel"); + assertRelationshipProps(rel.getAllProperties()); + }); + + testResult(db, "MATCH (m:Another) RETURN m", r -> { + ResourceIterator m = r.columnAs("m"); + Node node = m.next(); + assertFirstAnotherNodeProps(node.getAllProperties()); + node = m.next(); + assertSecondAnotherNodeProps(node.getAllProperties()); + assertFalse(r.hasNext()); + }); + } + + public static String extractFileName(Result result) { + return result.columnAs("file").next(); + } + + public static byte[] extractByteArray(Result result) { + return result.columnAs("value").next(); + } + + public static void assertFirstUserNodeProps(Map props) { + assertEquals("Adam", props.get("name")); + assertEquals(42L, props.get("age")); + assertEquals(true, props.get("male")); + assertArrayEquals(new String[] {"Sam", "Anna", "Grace"}, (String[]) props.get("kids")); + Map latitude = Map.of("latitude", 13.1D, "longitude", 33.46789D, "height", 100.0D); + assertEquals( + PointValue.fromMap(VirtualValues.map( + latitude.keySet().toArray(new String[0]), + latitude.values().stream().map(ValueUtils::of).toArray(AnyValue[]::new))), + props.get("place")); + assertEquals(LocalDateTimeValue.parse("2015-05-18T19:32:24.000").asObject(), props.get("born")); + } + + public static void assertSecondUserNodeProps(Map props) { + assertEquals("Jim", props.get("name")); + assertEquals(42L, props.get("age")); + } + + public static void assertFirstAnotherNodeProps(Map map) { + assertEquals(1L, map.get("foo")); + assertArrayEquals(new long[] {1L, 2L}, (long[]) map.get("listInt")); + } + + public static void assertSecondAnotherNodeProps(Map map) { + assertEquals("Sam", map.get("bar")); + } + + public static void assertRelationshipProps(Map props) { + assertEquals(DurationValue.parse("P5M1DT12H"), props.get("bffSince")); + assertEquals(1993L, props.get("since")); + } +} diff --git a/full/src/test/java/apoc/export/arrow/ImportArrowTest.java b/full/src/test/java/apoc/export/arrow/ImportArrowTest.java index 59206a37c4..7b55685871 100644 --- a/full/src/test/java/apoc/export/arrow/ImportArrowTest.java +++ b/full/src/test/java/apoc/export/arrow/ImportArrowTest.java @@ -1,13 +1,28 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package apoc.export.arrow; import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; -import static apoc.util.TestUtil.testCall; -import static apoc.util.TestUtil.testResult; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static apoc.export.arrow.ArrowTestUtil.ARROW_BASE_FOLDER; +import static apoc.export.arrow.ArrowTestUtil.testImportCommon; import apoc.meta.Meta; import apoc.util.TestUtil; @@ -19,21 +34,11 @@ import org.junit.ClassRule; import org.junit.Test; import org.neo4j.configuration.GraphDatabaseSettings; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.ResourceIterator; -import org.neo4j.graphdb.Result; -import org.neo4j.kernel.impl.util.ValueUtils; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; -import org.neo4j.values.AnyValue; -import org.neo4j.values.storable.DurationValue; -import org.neo4j.values.storable.LocalDateTimeValue; -import org.neo4j.values.storable.PointValue; -import org.neo4j.values.virtual.VirtualValues; public class ImportArrowTest { - private static File directory = new File("target/arrowImport"); + private static File directory = new File(ARROW_BASE_FOLDER); static { //noinspection ResultOfMethodCallIgnored directory.mkdirs(); @@ -59,7 +64,7 @@ public void before() { db.executeTransactionally("MATCH (n) DETACH DELETE n"); db.executeTransactionally( - "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace', 'Qwe'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); + "CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})"); db.executeTransactionally("CREATE (:Another {foo:1, listInt: [1,2]}), (:Another {bar:'Sam'})"); apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); @@ -68,100 +73,27 @@ public void before() { @Test public void testStreamRoundtripImportArrowAll() { - final byte[] bytes = - db.executeTransactionally("CALL apoc.export.arrow.stream.all", Map.of(), this::extractByteArray); + final byte[] bytes = db.executeTransactionally( + "CALL apoc.export.arrow.stream.all", Map.of(), ArrowTestUtil::extractByteArray); - testImportCommon(bytes, MAPPING_ALL); + testImportCommon(db, bytes, MAPPING_ALL); } @Test public void testFileRoundtripImportArrowAll() { String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), this::extractFileName); + "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), ArrowTestUtil::extractFileName); - testImportCommon(file, MAPPING_ALL); + testImportCommon(db, file, MAPPING_ALL); } @Test public void testFileRoundtripImportArrowAllWithSmallBatchSize() { String file = db.executeTransactionally( - "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), this::extractFileName); + "CALL apoc.export.arrow.all('test_all.arrow') YIELD file", Map.of(), ArrowTestUtil::extractFileName); Map config = new HashMap<>(MAPPING_ALL); config.put("batchSize", 1); - testImportCommon(file, config); - } - - private void testImportCommon(Object file, Map config) { - // then - Map params = Map.of("file", file, "config", config); - - // remove current data - db.executeTransactionally("MATCH (n) DETACH DELETE n"); - - final String query = "CALL apoc.import.arrow($file, $config)"; - testCall(db, query, params, r -> { - assertEquals(4L, r.get("nodes")); - assertEquals(1L, r.get("relationships")); - }); - - testCall(db, "MATCH (start:User)-[rel:KNOWS]->(end:User) RETURN start, rel, end", r -> { - Node start = (Node) r.get("start"); - assertFirstUserNodeProps(start.getAllProperties()); - Node end = (Node) r.get("end"); - assertSecondUserNodeProps(end.getAllProperties()); - Relationship rel = (Relationship) r.get("rel"); - assertRelationshipProps(rel.getAllProperties()); - }); - - testResult(db, "MATCH (m:Another) RETURN m", r -> { - ResourceIterator m = r.columnAs("m"); - Node node = m.next(); - assertFirstAnotherNodeProps(node.getAllProperties()); - node = m.next(); - assertSecondAnotherNodeProps(node.getAllProperties()); - assertFalse(r.hasNext()); - }); - } - - private String extractFileName(Result result) { - return result.columnAs("file").next(); - } - - private byte[] extractByteArray(Result result) { - return result.columnAs("value").next(); - } - - private static void assertFirstUserNodeProps(Map props) { - assertEquals("Adam", props.get("name")); - assertEquals(42L, props.get("age")); - assertEquals(true, props.get("male")); - assertArrayEquals(new String[] {"Sam", "Anna", "Grace", "Qwe"}, (String[]) props.get("kids")); - Map latitude = Map.of("latitude", 13.1D, "longitude", 33.46789D, "height", 100.0D); - assertEquals( - PointValue.fromMap(VirtualValues.map( - latitude.keySet().toArray(new String[0]), - latitude.values().stream().map(ValueUtils::of).toArray(AnyValue[]::new))), - props.get("place")); - assertEquals(LocalDateTimeValue.parse("2015-05-18T19:32:24.000").asObject(), props.get("born")); - } - - private static void assertSecondUserNodeProps(Map props) { - assertEquals("Jim", props.get("name")); - assertEquals(42L, props.get("age")); - } - - private static void assertFirstAnotherNodeProps(Map map) { - assertEquals(1L, map.get("foo")); - assertArrayEquals(new long[] {1L, 2L}, (long[]) map.get("listInt")); - } - - private static void assertSecondAnotherNodeProps(Map map) { - assertEquals("Sam", map.get("bar")); - } - - private static void assertRelationshipProps(Map props) { - assertEquals(DurationValue.parse("P5M1DT12H"), props.get("bffSince")); - assertEquals(1993L, props.get("since")); + testImportCommon(db, file, config); } } diff --git a/full/src/test/java/apoc/export/csv/ExportXlsTest.java b/full/src/test/java/apoc/export/csv/ExportXlsTest.java index 4c34091a81..02a5d7b39e 100644 --- a/full/src/test/java/apoc/export/csv/ExportXlsTest.java +++ b/full/src/test/java/apoc/export/csv/ExportXlsTest.java @@ -253,7 +253,7 @@ public void testExportQueryXlsWithJoinedLabels() { "MATCH p = (u:User{name: 'Andrea'})-[r:COMPANY]->(c:Company{name: 'Larus'}) DELETE p"); } - private void assertResults( + public static void assertResults( String fileName, Map r, final String source, @@ -270,7 +270,7 @@ private void assertResults( assertTrue("Should get time greater than 0", ((long) r.get("time")) >= 0); } - private void assertResults(String fileName, Map r, final String source) { + public static void assertResults(String fileName, Map r, final String source) { assertResults(fileName, r, source, 8L, 2L, 6); } diff --git a/full/src/test/java/apoc/load/LoadCsvTest.java b/full/src/test/java/apoc/load/LoadCsvTest.java index 152b91fa28..0aa5569989 100644 --- a/full/src/test/java/apoc/load/LoadCsvTest.java +++ b/full/src/test/java/apoc/load/LoadCsvTest.java @@ -18,6 +18,9 @@ */ package apoc.load; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; +import static apoc.load.LoadCsv.ERROR_WRONG_COL_SEPARATOR; import static apoc.util.BinaryTestUtil.fileToBinary; import static apoc.util.CompressionConfig.COMPRESSION; import static apoc.util.MapUtil.map; @@ -29,7 +32,6 @@ import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; -import apoc.ApocSettings; import apoc.util.CompressionAlgo; import apoc.util.TestUtil; import apoc.util.Util; @@ -48,10 +50,12 @@ import org.mockserver.integration.ClientAndServer; import org.mockserver.model.Header; import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.QueryExecutionException; import org.neo4j.graphdb.Result; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; +import org.neo4j.values.storable.*; import org.testcontainers.containers.GenericContainer; public class LoadCsvTest { @@ -73,14 +77,8 @@ public static void stopServer() { mockServer.stop(); } - @After - public void cleanup() { - db.shutdown(); - } - @Rule public DbmsRule db = new ImpermanentDbmsRule() - .withSetting(ApocSettings.apoc_import_file_enabled, true) .withSetting( GraphDatabaseSettings.load_csv_file_url_root, Paths.get(getUrlFileName("test.csv").toURI()).getParent()); @@ -92,16 +90,13 @@ public LoadCsvTest() throws URISyntaxException {} @Before public void setUp() throws Exception { TestUtil.registerProcedure(db, LoadCsv.class); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); } @Test public void testLoadCsv() throws Exception { String url = "test.csv"; - testResult( - db, - "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", - map("url", url), // 'file:test.csv' - this::commonAssertionsLoadCsv); + commonTestLoadCsv(db, url); } @Test @@ -118,16 +113,84 @@ public void testLoadCsvWithBinary() { CompressionAlgo.DEFLATE.name(), "results", List.of("map", "list", "stringMap", "strings"))), - this::commonAssertionsLoadCsv); + LoadCsvTest::commonAssertionsLoadCsv); + } + + @Test + public void testLoadCsvWithQuote() { + String url = "testQuote.csv"; + testResult( + db, + "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", + map("url", url), // 'file:test.csv' + r -> { + assertRow(r, 0L, "name", "Selma", "age", "8"); + assertRow(r, 1L, "name", "Rana", "age", "11"); + assertRow(r, 2L, "name", "Seli,na", "age", "18"); + assertFalse(r.hasNext()); + }); + } + + @Test + public void testLoadCsvWithQuoteAndIgnoreQuotations() { + String url = "testQuote.csv"; + testResult( + db, + "CALL apoc.load.csv($url,{ignoreQuotations: true, results:['list']})", + map("url", url), // 'file:test.csv' + r -> { + Map row = r.next(); + assertEquals(List.of("Selma", "8"), row.get("list")); + + row = r.next(); + assertEquals(List.of("Rana", "11"), row.get("list")); + + row = r.next(); + assertEquals(List.of("Seli", "na", "18"), row.get("list")); + + assertFalse(r.hasNext()); + }); + } + + @Test + public void testLoadCsvWithMultiCharSeparator() { + String url = "testMultiCharSep.csv"; + Map conf = map("results", List.of("map", "list", "stringMap", "strings"), "sep", "SEP"); + testResult( + db, + "CALL apoc.load.csv($url, $conf)", + map("url", url, "conf", conf), + LoadCsvTest::commonAssertionsLoadCsv); } - private void commonAssertionsLoadCsv(Result r) { + private static void commonAssertionsLoadCsv(Result r) { assertRow(r, 0L, "name", "Selma", "age", "8"); assertRow(r, 1L, "name", "Rana", "age", "11"); assertRow(r, 2L, "name", "Selina", "age", "18"); assertFalse(r.hasNext()); } + @Test + public void testLoadCsvWithNoneSeparator() { + String url = "test.csv"; + testResult( + db, + "CALL apoc.load.csv($url, {sep:'NONE'})", + map("url", url), // 'file:test.csv' + r -> { + Object actualList = r.next().get("list"); + assertEquals(List.of("Selma,8"), actualList); + + actualList = r.next().get("list"); + assertEquals(List.of("Rana,11"), actualList); + + actualList = r.next().get("list"); + assertEquals(List.of("Selina,18"), actualList); + + assertFalse(r.hasNext()); + }); + } + /* WITH 'file:///test.csv' AS url CALL apoc.load.csv(url,) YIELD map AS m @@ -189,7 +252,7 @@ static void assertRow(Result r, long lineNo, Object... data) { assertEquals(lineNo, row.get("lineNo")); } - static void assertRow(Result r, String name, String age, long lineNo) { + public static void assertRow(Result r, String name, String age, long lineNo) { Map row = r.next(); assertEquals(map("name", name, "age", age), row.get("map")); assertEquals(asList(name, age), row.get("list")); @@ -239,17 +302,12 @@ public void testLoadCsvTabSeparator() throws Exception { @Test public void testLoadCsvEscape() { - String url = "test-escape.csv"; + URL url = getUrlFileName("test-escape.csv"); final List results = List.of("map", "list", "stringMap", "strings"); - testResult(db, "CALL apoc.load.csv($url, $config)", map("url", url, "config", map("results", results)), (r) -> { - assertRow(r, 0L, "name", "Naruto", "surname", "Uzumaki"); - assertRow(r, 1L, "name", "Minato", "surname", "Namikaze"); - assertFalse(r.hasNext()); - }); testResult( db, - "CALL apoc.load.csv($url,$config)", - map("url", url, "config", map("results", results, "escapeChar", "NONE")), + "CALL apoc.load.csv($url, $config)", + map("url", url.toString(), "config", map("results", results)), (r) -> { assertRow(r, 0L, "name", "Narut\\o", "surname", "Uzu\\maki"); assertRow(r, 1L, "name", "Minat\\o", "surname", "Nami\\kaze"); @@ -358,7 +416,7 @@ public void testPipeArraySeparator() throws Exception { @Test public void testWithSpacesInFileName() throws Exception { - String url = "test pipe column with spaces in filename.csv"; + String url = "file:///test%20pipe%20column%20with%20spaces%20in%20filename.csv"; testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings'],mapping:{name:{type:'string'},beverage:{array:true,arraySep:'|',type:'string'}}})", @@ -513,7 +571,7 @@ public void testLoadCsvParamsWithBasicAuth() throws JsonProcessingException { @Test public void testLoadCsvByUrlRedirect() throws Exception { - URL url = new URL("http://bit.ly/2nXgHA2"); + URL url = new URL("https://bit.ly/2nXgHA2"); testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", @@ -552,6 +610,40 @@ public void testLoadCsvNoFailOnError() throws Exception { }); } + @Test + public void testIssue3156FailOnErrorFalse() { + String url = getUrlFileName("faulty.csv").getPath(); + testResult(db, "CALL apoc.load.csv($url, {failOnError: false})", map("url", url), (r) -> { + Map row = r.next(); + assertEquals(0L, row.get("lineNo")); + assertEquals(List.of("Galata Tower", "", "Turkey", "67"), row.get("list")); + row = r.next(); + assertEquals(1L, row.get("lineNo")); + assertEquals(List.of("Belem Tower", "Lisbon", "", "30"), row.get("list")); + row = r.next(); + assertEquals(3L, row.get("lineNo")); + assertEquals(List.of("", "London", "United Kingdom", "96"), row.get("list")); + row = r.next(); + assertEquals(4L, row.get("lineNo")); + assertEquals(List.of("Leaning tower", "Pisa", "Italia", "56"), row.get("list")); + row = r.next(); + assertEquals(5L, row.get("lineNo")); + assertEquals(List.of("Eiffel Tower", "Paris", "France", "300"), row.get("list")); + assertFalse(r.hasNext()); + }); + } + + @Test + public void testIssue3156FailOnErrorTrue() { + String url = getUrlFileName("faulty.csv").getPath(); + try { + testResult(db, "CALL apoc.load.csv($url, {failOnError: true})", map("url", url), Result::resultAsString); + } catch (RuntimeException e) { + String message = e.getMessage(); + assertTrue("Actual error message is: " + message, message.contains(ERROR_WRONG_COL_SEPARATOR)); + } + } + @Test public void testLoadCsvZip() throws Exception { String url = "testload.zip"; @@ -615,7 +707,7 @@ public void testLoadCsvTgz() throws Exception { @Test public void testLoadCsvZipByUrl() throws Exception { URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar?raw=true"); + "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/3.4/src/test/resources/testload.zip?raw=true"); testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", @@ -630,8 +722,7 @@ public void testLoadCsvZipByUrl() throws Exception { @Test public void testLoadCsvTarByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar?raw=true"); + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tar?raw=true"); testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", @@ -646,8 +737,7 @@ public void testLoadCsvTarByUrl() throws Exception { @Test public void testLoadCsvTarGzByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tar.gz?raw=true"); + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tar.gz?raw=true"); testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", @@ -662,8 +752,7 @@ public void testLoadCsvTarGzByUrl() throws Exception { @Test public void testLoadCsvTgzByUrl() throws Exception { - URL url = new URL( - "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/4.4/core/src/test/resources/testload.tgz?raw=true"); + URL url = new URL("https://github.com/neo4j/apoc/blob/dev/core/src/test/resources/testload.tgz?raw=true"); testResult( db, "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", @@ -719,4 +808,12 @@ private static String fromListOfMapToCsvString(List> mapList .withHeader()) .writeValueAsString(mapList); } + + public static void commonTestLoadCsv(GraphDatabaseService db, String url) { + testResult( + db, + "CALL apoc.load.csv($url,{results:['map','list','stringMap','strings']})", + map("url", url), // 'file:test.csv' + LoadCsvTest::commonAssertionsLoadCsv); + } } diff --git a/full/src/test/java/apoc/load/LoadHtmlTest.java b/full/src/test/java/apoc/load/LoadHtmlTest.java index 8850405c01..d751f65a7b 100644 --- a/full/src/test/java/apoc/load/LoadHtmlTest.java +++ b/full/src/test/java/apoc/load/LoadHtmlTest.java @@ -44,6 +44,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.QueryExecutionException; import org.neo4j.test.rule.DbmsRule; import org.neo4j.test.rule.ImpermanentDbmsRule; @@ -314,17 +315,8 @@ private void loadHtmlWithSelector(int expected, String selector) { @Test public void testQueryMetadataWithGetLinks() { - Map query = map("links", "a[href]"); - - testCall( - db, - "CALL apoc.load.html($url,$query)", - map("url", new File("src/test/resources/wikipedia.html").toURI().toString(), "query", query), - row -> { - final List> actual = (List) ((Map) row.get("value")).get("links"); - assertEquals(106, actual.size()); - assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); - }); + String url = new File("src/test/resources/wikipedia.html").toURI().toString(); + testLoadHtmlWithGetLinksCommon(db, url); } @Test @@ -649,6 +641,16 @@ private void testCallGeneratedJsWithBrowser(String browser) { }); } + public static void testLoadHtmlWithGetLinksCommon(GraphDatabaseService db, String url) { + Map query = map("links", "a[href]"); + + testCall(db, "CALL apoc.load.html($url,$query)", map("url", url, "query", query), row -> { + final List> actual = (List) ((Map) row.get("value")).get("links"); + assertEquals(106, actual.size()); + assertTrue(actual.stream().allMatch(i -> i.get("tagName").equals("a"))); + }); + } + public static void skipIfBrowserNotPresentOrCompatible(Runnable runnable) { try { runnable.run(); diff --git a/full/src/test/java/apoc/load/LoadXlsTest.java b/full/src/test/java/apoc/load/LoadXlsTest.java index d7a6cd49b5..ff19302f5f 100644 --- a/full/src/test/java/apoc/load/LoadXlsTest.java +++ b/full/src/test/java/apoc/load/LoadXlsTest.java @@ -18,6 +18,8 @@ */ package apoc.load; +import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; +import static apoc.ApocConfig.apocConfig; import static apoc.util.MapUtil.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; @@ -25,7 +27,6 @@ import static java.util.Collections.emptyList; import static org.junit.Assert.*; -import apoc.ApocSettings; import apoc.util.TestUtil; import apoc.util.Util; import java.net.URL; @@ -40,11 +41,11 @@ import java.util.Set; import java.util.function.BiFunction; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.QueryExecutionException; import org.neo4j.graphdb.Result; import org.neo4j.internal.helpers.collection.Iterators; @@ -71,40 +72,17 @@ public class LoadXlsTest { .getPath(); @Rule - public DbmsRule db = new ImpermanentDbmsRule().withSetting(ApocSettings.apoc_import_file_enabled, true); + public DbmsRule db = new ImpermanentDbmsRule(); @Before public void setUp() throws Exception { TestUtil.registerProcedure(db, LoadXls.class); - } - - @After - public void teardown() { - db.shutdown(); + apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true); } @Test public void testLoadXls() throws Exception { - testResult( - db, - "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", - map("url", loadTest), // 'file:load_test.xlsx' - (r) -> { - assertRow( - r, - 0L, - "String", - "Test", - "Boolean", - true, - "Integer", - 2L, - "Float", - 1.5d, - "Array", - asList(1L, 2L, 3L)); - assertFalse("Should not have another row", r.hasNext()); - }); + testLoadXlsCommon(db, loadTest); } @Test @@ -638,4 +616,27 @@ private void assertIssue2403Excel(Result r, Map firstMap, Map { + assertRow( + r, + 0L, + "String", + "Test", + "Boolean", + true, + "Integer", + 2L, + "Float", + 1.5d, + "Array", + asList(1L, 2L, 3L)); + assertFalse("Should not have another row", r.hasNext()); + }); + } } diff --git a/full/src/test/java/apoc/util/ExtendedTestUtil.java b/full/src/test/java/apoc/util/ExtendedTestUtil.java index 3d5a69c702..876882883a 100644 --- a/full/src/test/java/apoc/util/ExtendedTestUtil.java +++ b/full/src/test/java/apoc/util/ExtendedTestUtil.java @@ -32,30 +32,6 @@ public class ExtendedTestUtil { - public static void assertRelationship( - Relationship rel, - String expectedRelType, - Map expectedProps, - List expectedStartNodeLabels, - Map expectedStartNodeProps, - List expectedEndNodeLabels, - Map expectedEndNodeProps) { - - Node startNode = rel.getStartNode(); - Node endNode = rel.getEndNode(); - assertMapEquals(expectedProps, rel.getAllProperties()); - assertEquals(RelationshipType.withName(expectedRelType), rel.getType()); - Set