Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Opt parser that allows re-using cache instead of building a set every… #20

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Opt parser that allows re-using cache instead of building a set every… #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions README_jmh.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
How to run JMH tests:
mvn install -Pjmh

Latest results for comparison:
Benchmark Mode Cnt Score Error Units
SimpleBenchmark.simpleTestCore thrpt 200 4259193.317 ± 38393.175 ops/s
SimpleBenchmark.simpleTestCore:·gc.alloc.rate thrpt 200 2120.341 ± 19.104 MB/sec
SimpleBenchmark.simpleTestCore:·gc.alloc.rate.norm thrpt 200 784.000 ± 0.001 B/op
SimpleBenchmark.simpleTestCore:·gc.churn.PS_Eden_Space thrpt 200 2119.441 ± 21.079 MB/sec
SimpleBenchmark.simpleTestCore:·gc.churn.PS_Eden_Space.norm thrpt 200 783.710 ± 3.804 B/op
SimpleBenchmark.simpleTestCore:·gc.churn.PS_Survivor_Space thrpt 200 0.194 ± 0.010 MB/sec
SimpleBenchmark.simpleTestCore:·gc.churn.PS_Survivor_Space.norm thrpt 200 0.072 ± 0.004 B/op
SimpleBenchmark.simpleTestCore:·gc.count thrpt 200 4150.000 counts
SimpleBenchmark.simpleTestCore:·gc.time thrpt 200 1927.000 ms
SimpleBenchmark.simpleTestOps thrpt 200 6353975.392 ± 76349.733 ops/s
SimpleBenchmark.simpleTestOps:·gc.alloc.rate thrpt 200 258.212 ± 3.099 MB/sec
SimpleBenchmark.simpleTestOps:·gc.alloc.rate.norm thrpt 200 64.000 ± 0.001 B/op
SimpleBenchmark.simpleTestOps:·gc.churn.PS_Eden_Space thrpt 200 258.384 ± 3.458 MB/sec
SimpleBenchmark.simpleTestOps:·gc.churn.PS_Eden_Space.norm thrpt 200 64.056 ± 0.485 B/op
SimpleBenchmark.simpleTestOps:·gc.churn.PS_Survivor_Space thrpt 200 0.094 ± 0.007 MB/sec
SimpleBenchmark.simpleTestOps:·gc.churn.PS_Survivor_Space.norm thrpt 200 0.023 ± 0.002 B/op
SimpleBenchmark.simpleTestOps:·gc.count thrpt 200 2513.000 counts
SimpleBenchmark.simpleTestOps:·gc.time thrpt 200 1295.000 ms
73 changes: 70 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<name>Noggit</name>
<url>http://github.com/yonik/noggit</url>
<description>Noggit is the world's fastest streaming JSON parser for Java.</description>

<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
Expand Down Expand Up @@ -40,7 +40,6 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>


<dependencies>
<dependency>
Expand All @@ -66,10 +65,78 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.9</version>
</plugin>
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>jmh</id>
<dependencies>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
<version>1.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-generator-annprocess</artifactId>
<version>1.19</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.9.1</version>
<executions>
<execution>
<id>add-test-source</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>src/jmh/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
<id>run-benchmarks</id>
<phase>integration-test</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<classpathScope>test</classpathScope>
<executable>java</executable>
<arguments>
<argument>-classpath</argument>
<classpath/>
<argument>org.openjdk.jmh.Main</argument>
<argument>-prof</argument>
<argument>gc</argument>
<argument>.*</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

</project>


47 changes: 47 additions & 0 deletions src/jmh/java/org/noggit/SimpleBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package org.noggit;

import org.openjdk.jmh.annotations.*;

import java.io.IOException;

@State(Scope.Benchmark)
public class SimpleBenchmark {

JSONParserOpt parser = new JSONParserOpt("{'count': 1600}");

@Benchmark
@BenchmarkMode(Mode.Throughput)
public long simpleTestCore() throws IOException {
JSONParser parser = new JSONParser("{'count': 1600}", 0, "{'count': 1600}".length());
int i = parser.nextEvent();
int j = parser.nextEvent();
int k = parser.getString().hashCode();
int sum = i + j + k + parser.nextEvent();

parser = new JSONParser("{'count': 1600}", 0, "{'count': 1600}".length());
i = parser.nextEvent();
j = parser.nextEvent();
k = parser.getString().hashCode();
sum = sum + i + j + k + parser.nextEvent();

return sum;
}

@Benchmark
@BenchmarkMode(Mode.Throughput)
public long simpleTestOps() throws IOException {
parser.reUse("{'count': 1600}", 0, "{'count': 1600}".length());
int i = parser.nextEvent();
int j = parser.nextEvent();
int k = parser.getString().hashCode();
int sum = i + j + k + parser.nextEvent();

parser.reUse("{'count': 1600}", 0, "{'count': 1600}".length());
i = parser.nextEvent();
j = parser.nextEvent();
k = parser.getString().hashCode();
sum = sum + i + j + k + parser.nextEvent();

return sum;
}
}
16 changes: 9 additions & 7 deletions src/main/java/org/noggit/JSONParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,10 @@ public static String getEventString( int e )

protected int flags = FLAGS_DEFAULT;

protected final char[] buf; // input buffer with JSON text in it
protected char[] buf; // input buffer with JSON text in it
protected int start; // current position in the buffer
protected int end; // end position in the buffer (one past last valid index)
protected final Reader in; // optional reader to obtain data from
protected Reader in; // optional reader to obtain data from
protected boolean eof=false; // true if the end of the stream was reached.
protected long gpos; // global position = gpos + start

Expand All @@ -132,6 +132,8 @@ public JSONParser(Reader in, char[] buffer) {
// idea - if someone passes us a CharArrayReader, we could
// directly use that buffer as it's protected.

protected JSONParser() {}

public JSONParser(char[] data, int start, int end) {
this.in = null;
this.buf = data;
Expand Down Expand Up @@ -162,13 +164,13 @@ public int setFlags(int flags) {
}

// temporary output buffer
private final CharArr out = new CharArr(64);
protected final CharArr out = new CharArr(64);

// We need to keep some state in order to (at a minimum) know if
// we should skip ',' or ':'.
private byte[] stack = new byte[16];
private int ptr=0; // pointer into the stack of parser states
private byte state=0; // current parser state
protected int ptr=0; // pointer into the stack of parser states
protected byte state=0; // current parser state

// parser states stored in the stack
private static final byte DID_OBJSTART =1; // '{' just read
Expand All @@ -178,7 +180,7 @@ public int setFlags(int flags) {
private static final byte DID_MEMVAL =5; // object member value (map val) just read

// info about value that was just read (or is in the middle of being read)
private int valstate;
protected int valstate;

// push current parser state (use at start of new container)
private final void push() {
Expand Down Expand Up @@ -439,7 +441,7 @@ private String errEscape(int a, int b) {

private boolean bool; // boolean value read
private long lval; // long value read
private int nstate; // current state while reading a number
protected int nstate; // current state while reading a number
private static final int HAS_FRACTION = 0x01; // nstate flag, '.' already read
private static final int HAS_EXPONENT = 0x02; // nstate flag, '[eE][+-]?[0-9]' already read

Expand Down
37 changes: 37 additions & 0 deletions src/main/java/org/noggit/JSONParserOpt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.noggit;

public class JSONParserOpt extends JSONParser {

public JSONParserOpt(String data) {
this(data, 0, data.length());
}

public JSONParserOpt(String data, int start, int end) {
this.in = null;
this.start = start;
this.end = end;
int power = 32 - Integer.numberOfLeadingZeros(end - start);
this.buf = new char[1 << Math.max(power, 13)];
this.valstate = 0;

data.getChars(start,end,buf,0);
}

public void reUse(String data, int start, int end) {
if (buf.length < end - start + 1) {
int power = 32 - Integer.numberOfLeadingZeros(end - start);
buf = new char[1 << power];
}

this.valstate = 0;
this.start = start;
this.end = end;
ptr = 0;
state = 0;
event = 0;
gpos = 0;
stringTerm = 0;
nstate = 0;
data.getChars(start, end, buf,0);
}
}
83 changes: 83 additions & 0 deletions src/test/java/org/noggit/JSONParserOptTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package org.noggit;

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestResult;

import java.io.IOException;

public class JSONParserOptTest extends TestCase {

public void testSingle() throws IOException {
JSONParserOpt parser = new JSONParserOpt("{'count': 1600}");

// {
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'count'
assertEquals(JSONParser.STRING, parser.nextEvent());
assertEquals("count", parser.getString());

// 1600
assertEquals(JSONParser.LONG, parser.nextEvent());
assertEquals(1600, parser.getLong());

// }
assertEquals(JSONParser.OBJECT_END, parser.nextEvent());

String data = "{'quatro': {'gone': 128} }";
parser.reUse(data, 0, data.length());

// {
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'quatro'
assertEquals(JSONParser.STRING, parser.nextEvent());
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'gone'
assertEquals(JSONParser.STRING, parser.nextEvent());
assertEquals("gone", parser.getString());

// 1600
assertEquals(JSONParser.LONG, parser.nextEvent());
assertEquals(128, parser.getLong());

// }}
assertEquals(JSONParser.OBJECT_END, parser.nextEvent());
assertEquals(JSONParser.OBJECT_END, parser.nextEvent());
}

public void testFinishInMiddle() throws IOException {
JSONParserOpt parser = new JSONParserOpt("{'count': 1600}");

// {
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'count'
assertEquals(JSONParser.STRING, parser.nextEvent());

String data = "{'quatro': {'gone': 128} }";
parser.reUse(data, 0, data.length());

// {
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'quatro'
assertEquals(JSONParser.STRING, parser.nextEvent());
assertEquals(JSONParser.OBJECT_START, parser.nextEvent());

// 'gone'
assertEquals(JSONParser.STRING, parser.nextEvent());
assertEquals("gone", parser.getString());

// 1600
assertEquals(JSONParser.LONG, parser.nextEvent());
assertEquals(128, parser.getLong());

// }}
assertEquals(JSONParser.OBJECT_END, parser.nextEvent());
assertEquals(JSONParser.OBJECT_END, parser.nextEvent());
}

}