From 57e7fd9e50a06e5ba4053c98f06cbfe99f3936bd Mon Sep 17 00:00:00 2001 From: Rafael Zubairov Date: Fri, 28 Jul 2017 23:30:42 -0700 Subject: [PATCH] Opt parser that allows re-using cache instead of building a set every new time. JMH suite integration. --- README_jmh.txt | 23 +++++ pom.xml | 73 +++++++++++++++- src/jmh/java/org/noggit/SimpleBenchmark.java | 47 +++++++++++ src/main/java/org/noggit/JSONParser.java | 16 ++-- src/main/java/org/noggit/JSONParserOpt.java | 37 +++++++++ .../java/org/noggit/JSONParserOptTest.java | 83 +++++++++++++++++++ 6 files changed, 269 insertions(+), 10 deletions(-) create mode 100644 README_jmh.txt create mode 100644 src/jmh/java/org/noggit/SimpleBenchmark.java create mode 100644 src/main/java/org/noggit/JSONParserOpt.java create mode 100644 src/test/java/org/noggit/JSONParserOptTest.java diff --git a/README_jmh.txt b/README_jmh.txt new file mode 100644 index 0000000..151cc34 --- /dev/null +++ b/README_jmh.txt @@ -0,0 +1,23 @@ +How to run JMH tests: +mvn install -Pjmh + +Latest results for comparison: +Benchmark Mode Cnt Score Error Units +SimpleBenchmark.simpleTestCore thrpt 200 4259193.317 ± 38393.175 ops/s +SimpleBenchmark.simpleTestCore:·gc.alloc.rate thrpt 200 2120.341 ± 19.104 MB/sec +SimpleBenchmark.simpleTestCore:·gc.alloc.rate.norm thrpt 200 784.000 ± 0.001 B/op +SimpleBenchmark.simpleTestCore:·gc.churn.PS_Eden_Space thrpt 200 2119.441 ± 21.079 MB/sec +SimpleBenchmark.simpleTestCore:·gc.churn.PS_Eden_Space.norm thrpt 200 783.710 ± 3.804 B/op +SimpleBenchmark.simpleTestCore:·gc.churn.PS_Survivor_Space thrpt 200 0.194 ± 0.010 MB/sec +SimpleBenchmark.simpleTestCore:·gc.churn.PS_Survivor_Space.norm thrpt 200 0.072 ± 0.004 B/op +SimpleBenchmark.simpleTestCore:·gc.count thrpt 200 4150.000 counts +SimpleBenchmark.simpleTestCore:·gc.time thrpt 200 1927.000 ms +SimpleBenchmark.simpleTestOps thrpt 200 6353975.392 ± 76349.733 ops/s +SimpleBenchmark.simpleTestOps:·gc.alloc.rate thrpt 200 258.212 ± 3.099 MB/sec +SimpleBenchmark.simpleTestOps:·gc.alloc.rate.norm thrpt 200 64.000 ± 0.001 B/op +SimpleBenchmark.simpleTestOps:·gc.churn.PS_Eden_Space thrpt 200 258.384 ± 3.458 MB/sec +SimpleBenchmark.simpleTestOps:·gc.churn.PS_Eden_Space.norm thrpt 200 64.056 ± 0.485 B/op +SimpleBenchmark.simpleTestOps:·gc.churn.PS_Survivor_Space thrpt 200 0.094 ± 0.007 MB/sec +SimpleBenchmark.simpleTestOps:·gc.churn.PS_Survivor_Space.norm thrpt 200 0.023 ± 0.002 B/op +SimpleBenchmark.simpleTestOps:·gc.count thrpt 200 2513.000 counts +SimpleBenchmark.simpleTestOps:·gc.time thrpt 200 1295.000 ms \ No newline at end of file diff --git a/pom.xml b/pom.xml index f43100d..e066dee 100755 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ Noggit http://github.com/yonik/noggit Noggit is the world's fastest streaming JSON parser for Java. - + org.sonatype.oss oss-parent @@ -40,7 +40,6 @@ UTF-8 - @@ -66,10 +65,78 @@ org.apache.maven.plugins maven-eclipse-plugin 2.9 - + + + + jmh + + + org.openjdk.jmh + jmh-core + 1.19 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.19 + test + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.9.1 + + + add-test-source + generate-test-sources + + add-test-source + + + + src/jmh/java + + + + + + + org.codehaus.mojo + exec-maven-plugin + + + run-benchmarks + integration-test + + exec + + + test + java + + -classpath + + org.openjdk.jmh.Main + -prof + gc + .* + + + + + + + + + + diff --git a/src/jmh/java/org/noggit/SimpleBenchmark.java b/src/jmh/java/org/noggit/SimpleBenchmark.java new file mode 100644 index 0000000..19599ad --- /dev/null +++ b/src/jmh/java/org/noggit/SimpleBenchmark.java @@ -0,0 +1,47 @@ +package org.noggit; + +import org.openjdk.jmh.annotations.*; + +import java.io.IOException; + +@State(Scope.Benchmark) +public class SimpleBenchmark { + + JSONParserOpt parser = new JSONParserOpt("{'count': 1600}"); + + @Benchmark + @BenchmarkMode(Mode.Throughput) + public long simpleTestCore() throws IOException { + JSONParser parser = new JSONParser("{'count': 1600}", 0, "{'count': 1600}".length()); + int i = parser.nextEvent(); + int j = parser.nextEvent(); + int k = parser.getString().hashCode(); + int sum = i + j + k + parser.nextEvent(); + + parser = new JSONParser("{'count': 1600}", 0, "{'count': 1600}".length()); + i = parser.nextEvent(); + j = parser.nextEvent(); + k = parser.getString().hashCode(); + sum = sum + i + j + k + parser.nextEvent(); + + return sum; + } + + @Benchmark + @BenchmarkMode(Mode.Throughput) + public long simpleTestOps() throws IOException { + parser.reUse("{'count': 1600}", 0, "{'count': 1600}".length()); + int i = parser.nextEvent(); + int j = parser.nextEvent(); + int k = parser.getString().hashCode(); + int sum = i + j + k + parser.nextEvent(); + + parser.reUse("{'count': 1600}", 0, "{'count': 1600}".length()); + i = parser.nextEvent(); + j = parser.nextEvent(); + k = parser.getString().hashCode(); + sum = sum + i + j + k + parser.nextEvent(); + + return sum; + } +} diff --git a/src/main/java/org/noggit/JSONParser.java b/src/main/java/org/noggit/JSONParser.java index 2a13bcd..1b10e3e 100755 --- a/src/main/java/org/noggit/JSONParser.java +++ b/src/main/java/org/noggit/JSONParser.java @@ -105,10 +105,10 @@ public static String getEventString( int e ) protected int flags = FLAGS_DEFAULT; - protected final char[] buf; // input buffer with JSON text in it + protected char[] buf; // input buffer with JSON text in it protected int start; // current position in the buffer protected int end; // end position in the buffer (one past last valid index) - protected final Reader in; // optional reader to obtain data from + protected Reader in; // optional reader to obtain data from protected boolean eof=false; // true if the end of the stream was reached. protected long gpos; // global position = gpos + start @@ -132,6 +132,8 @@ public JSONParser(Reader in, char[] buffer) { // idea - if someone passes us a CharArrayReader, we could // directly use that buffer as it's protected. + protected JSONParser() {} + public JSONParser(char[] data, int start, int end) { this.in = null; this.buf = data; @@ -162,13 +164,13 @@ public int setFlags(int flags) { } // temporary output buffer - private final CharArr out = new CharArr(64); + protected final CharArr out = new CharArr(64); // We need to keep some state in order to (at a minimum) know if // we should skip ',' or ':'. private byte[] stack = new byte[16]; - private int ptr=0; // pointer into the stack of parser states - private byte state=0; // current parser state + protected int ptr=0; // pointer into the stack of parser states + protected byte state=0; // current parser state // parser states stored in the stack private static final byte DID_OBJSTART =1; // '{' just read @@ -178,7 +180,7 @@ public int setFlags(int flags) { private static final byte DID_MEMVAL =5; // object member value (map val) just read // info about value that was just read (or is in the middle of being read) - private int valstate; + protected int valstate; // push current parser state (use at start of new container) private final void push() { @@ -439,7 +441,7 @@ private String errEscape(int a, int b) { private boolean bool; // boolean value read private long lval; // long value read - private int nstate; // current state while reading a number + protected int nstate; // current state while reading a number private static final int HAS_FRACTION = 0x01; // nstate flag, '.' already read private static final int HAS_EXPONENT = 0x02; // nstate flag, '[eE][+-]?[0-9]' already read diff --git a/src/main/java/org/noggit/JSONParserOpt.java b/src/main/java/org/noggit/JSONParserOpt.java new file mode 100644 index 0000000..d90d45d --- /dev/null +++ b/src/main/java/org/noggit/JSONParserOpt.java @@ -0,0 +1,37 @@ +package org.noggit; + +public class JSONParserOpt extends JSONParser { + + public JSONParserOpt(String data) { + this(data, 0, data.length()); + } + + public JSONParserOpt(String data, int start, int end) { + this.in = null; + this.start = start; + this.end = end; + int power = 32 - Integer.numberOfLeadingZeros(end - start); + this.buf = new char[1 << Math.max(power, 13)]; + this.valstate = 0; + + data.getChars(start,end,buf,0); + } + + public void reUse(String data, int start, int end) { + if (buf.length < end - start + 1) { + int power = 32 - Integer.numberOfLeadingZeros(end - start); + buf = new char[1 << power]; + } + + this.valstate = 0; + this.start = start; + this.end = end; + ptr = 0; + state = 0; + event = 0; + gpos = 0; + stringTerm = 0; + nstate = 0; + data.getChars(start, end, buf,0); + } +} diff --git a/src/test/java/org/noggit/JSONParserOptTest.java b/src/test/java/org/noggit/JSONParserOptTest.java new file mode 100644 index 0000000..02ab395 --- /dev/null +++ b/src/test/java/org/noggit/JSONParserOptTest.java @@ -0,0 +1,83 @@ +package org.noggit; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestResult; + +import java.io.IOException; + +public class JSONParserOptTest extends TestCase { + + public void testSingle() throws IOException { + JSONParserOpt parser = new JSONParserOpt("{'count': 1600}"); + + // { + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'count' + assertEquals(JSONParser.STRING, parser.nextEvent()); + assertEquals("count", parser.getString()); + + // 1600 + assertEquals(JSONParser.LONG, parser.nextEvent()); + assertEquals(1600, parser.getLong()); + + // } + assertEquals(JSONParser.OBJECT_END, parser.nextEvent()); + + String data = "{'quatro': {'gone': 128} }"; + parser.reUse(data, 0, data.length()); + + // { + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'quatro' + assertEquals(JSONParser.STRING, parser.nextEvent()); + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'gone' + assertEquals(JSONParser.STRING, parser.nextEvent()); + assertEquals("gone", parser.getString()); + + // 1600 + assertEquals(JSONParser.LONG, parser.nextEvent()); + assertEquals(128, parser.getLong()); + + // }} + assertEquals(JSONParser.OBJECT_END, parser.nextEvent()); + assertEquals(JSONParser.OBJECT_END, parser.nextEvent()); + } + + public void testFinishInMiddle() throws IOException { + JSONParserOpt parser = new JSONParserOpt("{'count': 1600}"); + + // { + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'count' + assertEquals(JSONParser.STRING, parser.nextEvent()); + + String data = "{'quatro': {'gone': 128} }"; + parser.reUse(data, 0, data.length()); + + // { + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'quatro' + assertEquals(JSONParser.STRING, parser.nextEvent()); + assertEquals(JSONParser.OBJECT_START, parser.nextEvent()); + + // 'gone' + assertEquals(JSONParser.STRING, parser.nextEvent()); + assertEquals("gone", parser.getString()); + + // 1600 + assertEquals(JSONParser.LONG, parser.nextEvent()); + assertEquals(128, parser.getLong()); + + // }} + assertEquals(JSONParser.OBJECT_END, parser.nextEvent()); + assertEquals(JSONParser.OBJECT_END, parser.nextEvent()); + } + +}