Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NOID] Fixes #4207: Integration Tests for Load Procedures with Cloud Object Storage (#4226) #4326

Closed
wants to merge 9 commits into from
Closed

[NOID] Fixes #4207: Integration Tests for Load Procedures with Cloud Object Storage (#4226) #4326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f8acd14
[NOID] Fixes #4207: Integration Tests for Load Procedures with Cloud …
vga91 Dec 11, 2024
2ef06d1
[NOID] fix arrow tests
vga91 Jan 15, 2025
211c678
[NOID] Fixes #3156: apoc.load.csv, failure despite failOnError:false …
vga91 Mar 14, 2024
b1e67bc
[NOID] fix compile errors
vga91 Jan 15, 2025
888d54a
[NOID] Fixes #1948: multi char separator in apoc.load.csv (#3973)
vga91 Mar 20, 2024
bbeed0f
[NOID] Fixes #3218: java.lang.NullPointerException when setting the t…
vga91 Mar 14, 2024
2eaa5ba
[NOID] changes after rebase
vga91 Jan 17, 2025
a41f77e
[NOID] changed LoadCsv as dev one
vga91 Jan 20, 2025
bb53992
[NOID] 4.4 changes
vga91 Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions core/src/main/java/apoc/load/util/LoadCsvConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,17 @@

import apoc.load.Mapping;
import apoc.util.CompressionConfig;
import apoc.util.Util;
import java.util.*;

public class LoadCsvConfig extends CompressionConfig {

public static final char DEFAULT_ARRAY_SEP = ';';
public static final char DEFAULT_SEP = ',';
public static final String DEFAULT_SEP = ",";
public static final char DEFAULT_QUOTE_CHAR = '"';
// this is the same value as ICSVParser.DEFAULT_ESCAPE_CHARACTER
public static final char DEFAULT_ESCAPE_CHAR = '\\';

private final boolean ignoreErrors;
private char separator;
private String separator;
private char arraySep;
private char quoteChar;
private char escapeChar;
Expand All @@ -59,8 +57,7 @@ public LoadCsvConfig(Map<String, Object> config) {
if (config == null) {
config = Collections.emptyMap();
}
ignoreErrors = Util.toBoolean(config.getOrDefault("ignoreErrors", false));
separator = parseCharFromConfig(config, "sep", DEFAULT_SEP);
separator = parseStringFromConfig(config, "sep", DEFAULT_SEP);
arraySep = parseCharFromConfig(config, "arraySep", DEFAULT_ARRAY_SEP);
quoteChar = parseCharFromConfig(config, "quoteChar", DEFAULT_QUOTE_CHAR);
escapeChar = parseCharFromConfig(config, "escapeChar", DEFAULT_ESCAPE_CHAR);
Expand All @@ -83,6 +80,17 @@ public LoadCsvConfig(Map<String, Object> config) {
mappings = createMapping(mapping, arraySep, ignore);
}

private static String parseStringFromConfig(Map<String, Object> config, String key, String defaultValue) {
String separator = (String) config.getOrDefault(key, defaultValue);
if ("TAB".equals(separator)) {
return "\t";
}
if ("NONE".equals(separator)) {
return "\0";
}
return separator;
}

private Map<String, Mapping> createMapping(
Map<String, Map<String, Object>> mapping, char arraySep, List<String> ignore) {
if (mapping.isEmpty()) return Collections.emptyMap();
Expand All @@ -94,7 +102,7 @@ private Map<String, Mapping> createMapping(
return result;
}

public char getSeparator() {
public String getSeparator() {
return separator;
}

Expand Down Expand Up @@ -146,10 +154,6 @@ public char getEscapeChar() {
return escapeChar;
}

public boolean getIgnoreErrors() {
return ignoreErrors;
}

public boolean isIgnoreQuotations() {
return ignoreQuotations;
}
Expand Down
138 changes: 16 additions & 122 deletions core/src/test/java/apoc/export/arrow/ArrowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,15 @@
import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
import static apoc.ApocConfig.apocConfig;
import static apoc.export.arrow.ArrowTestUtil.initDbCommon;
import static apoc.export.arrow.ArrowTestUtil.testLoadArrow;
import static org.junit.Assert.assertEquals;

import apoc.ApocSettings;
import apoc.graph.Graphs;
import apoc.load.LoadArrow;
import apoc.meta.Meta;
import apoc.util.JsonUtil;
import apoc.util.TestUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.File;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -65,75 +60,9 @@ public class ArrowTest {
directory.toPath().toAbsolutePath())
.withSetting(ApocSettings.apoc_export_file_enabled, true);

public static final List<Map<String, Object>> EXPECTED = List.of(
new HashMap<>() {
{
put("name", "Adam");
put("bffSince", null);
put("<source.id>", null);
put("<id>", 0L);
put("age", 42L);
put("labels", List.of("User"));
put("male", true);
put("<type>", null);
put("kids", List.of("Sam", "Anna", "Grace"));
put(
"place",
Map.of("crs", "wgs-84-3d", "longitude", 33.46789D, "latitude", 13.1D, "height", 100.0D));
put("<target.id>", null);
put("since", null);
put(
"born",
LocalDateTime.parse("2015-05-18T19:32:24.000")
.atOffset(ZoneOffset.UTC)
.toZonedDateTime());
}
},
new HashMap<>() {
{
put("name", "Jim");
put("bffSince", null);
put("<source.id>", null);
put("<id>", 1L);
put("age", 42L);
put("labels", List.of("User"));
put("male", null);
put("<type>", null);
put("kids", null);
put("place", null);
put("<target.id>", null);
put("since", null);
put("born", null);
}
},
new HashMap<>() {
{
put("name", null);
put("bffSince", "P5M1DT12H");
put("<source.id>", 0L);
put("<id>", 0L);
put("age", null);
put("labels", null);
put("male", null);
put("<type>", "KNOWS");
put("kids", null);
put("place", null);
put("<target.id>", 1L);
put("since", 1993L);
put("born", null);
}
});

@BeforeClass
public static void beforeClass() {
db.executeTransactionally(
"CREATE (f:User {name:'Adam',age:42,male:true,kids:['Sam','Anna','Grace'], born:localdatetime('2015-05-18T19:32:24.000'), place:point({latitude: 13.1, longitude: 33.46789, height: 100.0})})-[:KNOWS {since: 1993, bffSince: duration('P5M1.5D')}]->(b:User {name:'Jim',age:42})");
TestUtil.registerProcedure(db, ExportArrow.class, LoadArrow.class, Graphs.class, Meta.class);
}

@AfterClass
public static void teardown() {
db.shutdown();
initDbCommon(db);
}

@Before
Expand All @@ -142,21 +71,13 @@ public void before() {
apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
}

private byte[] extractByteArray(Result result) {
return result.<byte[]>columnAs("byteArray").next();
}

private String extractFileName(Result result) {
return result.<String>columnAs("file").next();
@AfterClass
public static void teardown() {
db.shutdown();
}

private <T> T readValue(String json, Class<T> clazz) {
if (json == null) return null;
try {
return JsonUtil.OBJECT_MAPPER.readValue(json, clazz);
} catch (JsonProcessingException e) {
return null;
}
private byte[] extractByteArray(Result result) {
return result.<byte[]>columnAs("byteArray").next();
}

@Test
Expand Down Expand Up @@ -215,7 +136,7 @@ public void testFileRoundtripArrowQuery() {
String file = db.executeTransactionally(
"CALL apoc.export.arrow.query('query_test.arrow', $query) YIELD file",
Map.of("query", returnQuery),
this::extractFileName);
ArrowTestUtil::extractFileName);

// then
final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value";
Expand Down Expand Up @@ -251,22 +172,7 @@ public void testStreamRoundtripArrowGraph() {

// then
final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value";
db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> {
final List<Map<String, Object>> actual = getActual(result);
assertEquals(EXPECTED, actual);
return null;
});
}

private List<Map<String, Object>> getActual(Result result) {
return result.stream()
.map(m -> (Map<String, Object>) m.get("value"))
.map(m -> {
final Map<String, Object> newMap = new HashMap(m);
newMap.put("place", readValue((String) m.get("place"), Map.class));
return newMap;
})
.collect(Collectors.toList());
testLoadArrow(db, query, Map.of("byteArray", byteArray));
}

@Test
Expand All @@ -277,15 +183,11 @@ public void testFileRoundtripArrowGraph() {
+ "CALL apoc.export.arrow.graph('graph_test.arrow', graph) YIELD file "
+ "RETURN file",
Map.of(),
this::extractFileName);
ArrowTestUtil::extractFileName);

// then
final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value";
db.executeTransactionally(query, Map.of("file", file), result -> {
final List<Map<String, Object>> actual = getActual(result);
assertEquals(EXPECTED, actual);
return null;
});
testLoadArrow(db, query, Map.of("file", file));
}

@Test
Expand All @@ -310,26 +212,18 @@ private void testStreamRoundtripAllCommon() {

// then
final String query = "CALL apoc.load.arrow.stream($byteArray) YIELD value " + "RETURN value";
db.executeTransactionally(query, Map.of("byteArray", byteArray), result -> {
final List<Map<String, Object>> actual = getActual(result);
assertEquals(EXPECTED, actual);
return null;
});
testLoadArrow(db, query, Map.of("byteArray", byteArray));
}

@Test
public void testFileRoundtripArrowAll() {
// given - when
String file = db.executeTransactionally(
"CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), this::extractFileName);
"CALL apoc.export.arrow.all('all_test.arrow') YIELD file", Map.of(), ArrowTestUtil::extractFileName);

// then
final String query = "CALL apoc.load.arrow($file) YIELD value " + "RETURN value";
db.executeTransactionally(query, Map.of("file", file), result -> {
final List<Map<String, Object>> actual = getActual(result);
assertEquals(EXPECTED, actual);
return null;
});
testLoadArrow(db, query, Map.of("file", file));
}

@Test
Expand Down Expand Up @@ -365,7 +259,7 @@ public void testFileVolumeArrowAll() {
String file = db.executeTransactionally(
"CALL apoc.export.arrow.query('volume_test.arrow', 'MATCH (n:ArrowNode) RETURN n.id AS id') YIELD file ",
Map.of(),
this::extractFileName);
ArrowTestUtil::extractFileName);

final List<Long> expected = LongStream.range(0, 10000).mapToObj(l -> l).collect(Collectors.toList());

Expand Down
Loading
Loading