diff --git a/src/main/java/com/codingchili/ApplicationLauncher.java b/src/main/java/com/codingchili/ApplicationLauncher.java index 257f263..bea93f2 100644 --- a/src/main/java/com/codingchili/ApplicationLauncher.java +++ b/src/main/java/com/codingchili/ApplicationLauncher.java @@ -30,8 +30,11 @@ public static void main(String[] args) { private ApplicationLauncher(String[] args) { VertxOptions options = new VertxOptions(); - options.setMaxEventLoopExecuteTime(options.getMaxEventLoopExecuteTime() * 10) + + options.setMaxWorkerExecuteTime(options.getMaxWorkerExecuteTime() * 20) // 20 minutes. + .setMaxEventLoopExecuteTime(options.getMaxEventLoopExecuteTime() * 10) // 10 seconds. .setBlockedThreadCheckInterval(8000); + vertx = Vertx.vertx(); ImportEventCodec.registerOn(vertx); diff --git a/src/main/java/com/codingchili/Model/CSVParser.java b/src/main/java/com/codingchili/Model/CSVParser.java index 1d91df9..e4ba1c7 100644 --- a/src/main/java/com/codingchili/Model/CSVParser.java +++ b/src/main/java/com/codingchili/Model/CSVParser.java @@ -110,7 +110,7 @@ private void readRowCount() { reset(); for (long i = 0; i < fileSize; i++) { - if (get() == '\r') { + if (get() == TOKEN_LF) { rows++; row = rows; } @@ -118,13 +118,15 @@ private void readRowCount() { } private void readHeaders() { + AtomicInteger fieldId = new AtomicInteger(0); reset(); for (long i = 0; i < fileSize; i++) { byte current = get(); - if (current == TOKEN_LF || current == TOKEN_CR) { + if (current == TOKEN_LF) { Arrays.stream(new String(buffer.array()).split(",")) .map(header -> header.replaceAll("\"", "")) + .map(header -> (header.isEmpty()) ? "header_" + fieldId.incrementAndGet() : header) .map(String::trim).forEach(header -> { headers.put(header, ""); }); @@ -183,17 +185,20 @@ private JsonObject readRow() { process(columnsRead, json); done = true; break; - case TOKEN_LF: - // skip LF characters. - break; case TOKEN_CR: - // final header is being read and EOL appears. - if (columnsRead.get() == headers.size() - 1) { - process(columnsRead, json); - done = true; - break; - } else { - throw new ColumnsHeadersMismatchException(columnsRead.get(), headers.size() - 1, row + 1); + // skip CR characters. + break; + case TOKEN_LF: + // ignore empty lines. + if (buffer.position() > 0) { + // final header is being read and EOL appears. + if (columnsRead.get() == headers.size() - 1) { + process(columnsRead, json); + done = true; + break; + } else { + throw new ColumnsHeadersMismatchException(columnsRead.get(), headers.size() - 1, row + 1); + } } case TOKEN_QUOTE: // toggle quoted to support commas within quotes. @@ -223,16 +228,19 @@ private JsonObject readRow() { return json; } + + private static final Predicate floatPattern = Pattern.compile("^[0-9]+\\.[0-9]+$").asPredicate(); private static final Predicate numberPattern = Pattern.compile("^[0-9]+$").asPredicate(); private static final Predicate boolPattern = Pattern.compile("^(true|false)$").asPredicate(); private Object parseDatatype(byte[] data) { String line = new String(data).trim(); - // skip regex parsing on dry-run. if (line.length() > 0) { if (numberPattern.test(line)) { return Long.parseLong(line); + } else if (floatPattern.test(line)) { + return Double.parseDouble(line); } else if (boolPattern.test(line)) { return Boolean.parseBoolean(line); } else { diff --git a/src/test/resources/test.csv b/src/test/resources/test.csv index a6c6f9c..6306f62 100644 --- a/src/test/resources/test.csv +++ b/src/test/resources/test.csv @@ -1 +1,3 @@ -Column 1, Column 2, Column 3 cell 6.1, cell 6.2, cell 6.3 cell 7.1, cell 7.2, cell 7.3 \ No newline at end of file +Column 1, Column 2, Column 3 +cell 6.1, cell 6.2, cell 6.3 +cell 7.1, cell 7.2, cell 7.3 \ No newline at end of file