From edf26f01b81fde97c6f34677494bd21a98d95f64 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 26 Feb 2021 00:13:34 +0000 Subject: [PATCH 1/2] Improve performance of flag types ### package.json Update dependencies ### test.js Added tests for: * Empty String * BigInt > 64 bit * NaN * Infinity * Undefined * Function ### test-bench2.js Added benchmarks for measuring performance of different data types ### main.js Changes: * separate and clarify flag-only data types * added flags for: true, false, big bigint, nan, infinity, undefined, function * changed Buffer.alloc to Buffer.allocUnsafe (faster) * add support for big bigint (>64bit, stored as hex buffer) * serialize functions and retrieve actual functions instead of string representation (breaking change) * store undefined as flag and retrieve actual undefined instead of "undefined" (breaking change) ### hash.cc Added short-circuiting for flag-only types ### Performance changes: * Numbers -> 50% faster writes * Booleans -> 100% faster writes and 200% faster reads * Bigint -> 50% faster writes * Big Bigint -> no longer throws, performance is a bit slower than strings * Null -> 50% faster writes, 200% faster reads * NaN -> 100% faster writes, 200% faster reads * Infinity -> 100% faster writes, 200% faster reads * Undefined -> 50% faster writes, 200% faster reads * Function -> now returns an actual function, performance is a bit slower than objects --- hash.cc | 22 +++++-- main.js | 164 +++++++++++++++++++++++++++++++------------------ package.json | 2 +- test-bench2.js | 43 +++++++++++++ test/test.js | 49 +++++++++++++++ 5 files changed, 215 insertions(+), 65 deletions(-) create mode 100644 test-bench2.js diff --git a/hash.cc b/hash.cc index 2d44dbd..16823be 100755 --- a/hash.cc +++ b/hash.cc @@ -52,16 +52,22 @@ Napi::Value MegaHash::Set(const Napi::CallbackInfo& info) { Napi::Buffer keyBuf = info[0].As>(); unsigned char *key = keyBuf.Data(); MH_KLEN_T keyLength = (MH_KLEN_T)keyBuf.Length(); - - Napi::Buffer valueBuf = info[1].As>(); - unsigned char *value = valueBuf.Data(); - MH_LEN_T valueLength = (MH_LEN_T)valueBuf.Length(); - + unsigned char flags = 0; if (info.Length() > 2) { flags = (unsigned char)info[2].As().Uint32Value(); } + + // short circuit flag-only values + if(info[1].IsUndefined()) { + Response resp = this->hash->store( key, keyLength, 0, 0, flags ); + return Napi::Number::New(env, (double)resp.result); + } + Napi::Buffer valueBuf = info[1].As>(); + unsigned char *value = valueBuf.Data(); + MH_LEN_T valueLength = (MH_LEN_T)valueBuf.Length(); + Response resp = this->hash->store( key, keyLength, value, valueLength, flags ); return Napi::Number::New(env, (double)resp.result); } @@ -77,6 +83,12 @@ Napi::Value MegaHash::Get(const Napi::CallbackInfo& info) { Response resp = this->hash->fetch( key, keyLength ); if (resp.result == MH_OK) { + + // short circuit flag-only values + if(!resp.contentLength && resp.flags) { + return Napi::Number::New(env, (double)resp.flags); + } + Napi::Buffer valueBuf = Napi::Buffer::Copy( env, resp.content, resp.contentLength ); if (!valueBuf) return env.Undefined(); diff --git a/main.js b/main.js index 312b972..ff7e630 100755 --- a/main.js +++ b/main.js @@ -7,45 +7,93 @@ var MegaHash = require('bindings')('megahash').MegaHash; const MH_TYPE_BUFFER = 0; const MH_TYPE_STRING = 1; const MH_TYPE_NUMBER = 2; -const MH_TYPE_BOOLEAN = 3; -const MH_TYPE_OBJECT = 4; -const MH_TYPE_BIGINT = 5; -const MH_TYPE_NULL = 6; +const MH_TYPE_TRUE = 3; +const MH_TYPE_FALSE = 4; +const MH_TYPE_OBJECT = 5; +const MH_TYPE_BIGINT = 6; +const MH_TYPE_BIG_BIGINT_POS = 7; +const MH_TYPE_BIG_BIGINT_NEG = 8; +const MH_TYPE_NULL = 9; +const MH_TYPE_NAN = 10; +const MH_TYPE_INFINITY_POS = 11; +const MH_TYPE_INFINITY_NEG = 12; +const MH_TYPE_FUNCTION = 13; +const MH_TYPE_UNDEFINED = 14; MegaHash.prototype.set = function(key, value) { // store key/value in hash, auto-convert format to buffer var flags = MH_TYPE_BUFFER; var keyBuf = Buffer.isBuffer(key) ? key : Buffer.from(''+key, 'utf8'); if (!keyBuf.length) throw new Error("Key must have length"); - var valueBuf = value; - - if (!Buffer.isBuffer(valueBuf)) { - if (valueBuf === null) { - valueBuf = Buffer.alloc(0); - flags = MH_TYPE_NULL; - } - else if (typeof(valueBuf) == 'object') { - valueBuf = Buffer.from( JSON.stringify(value) ); - flags = MH_TYPE_OBJECT; - } - else if (typeof(valueBuf) == 'number') { - valueBuf = Buffer.alloc(8); - valueBuf.writeDoubleBE( value ); - flags = MH_TYPE_NUMBER; - } - else if (typeof(valueBuf) == 'bigint') { - valueBuf = Buffer.alloc(8); - valueBuf.writeBigInt64BE( value ); - flags = MH_TYPE_BIGINT; - } - else if (typeof(valueBuf) == 'boolean') { - valueBuf = Buffer.alloc(1); - valueBuf.writeUInt8( value ? 1 : 0 ); - flags = MH_TYPE_BOOLEAN; - } - else { - valueBuf = Buffer.from(''+value, 'utf8'); - flags = MH_TYPE_STRING; + var valueBuf; + + if(Buffer.isBuffer(value)) { + valueBuf = value; + } + else { + switch(typeof value) { + case "undefined": + flags = MH_TYPE_UNDEFINED; + break; + + case "number": + if(Number.isFinite(value)) { + valueBuf = Buffer.allocUnsafe(8); + valueBuf.writeDoubleBE(value); + flags = MH_TYPE_NUMBER; + } else { + if(Number.isNaN(value)) { + flags = MH_TYPE_NAN; + } else { + flags = value > 0 ? MH_TYPE_INFINITY_POS : MH_TYPE_INFINITY_NEG; + } + } + break; + + case "boolean": + flags = value ? MH_TYPE_TRUE : MH_TYPE_FALSE; + break; + + case "bigint": + if(value === BigInt.asIntN(64,value)) { + valueBuf = Buffer.allocUnsafe(8); + valueBuf.writeBigInt64BE(value); + flags = MH_TYPE_BIGINT; + } else { + flags = value > 0n ? MH_TYPE_BIG_BIGINT_POS : MH_TYPE_BIG_BIGINT_NEG; + value = value.toString(16); + if(flags === MH_TYPE_BIG_BIGINT_NEG) { value = value.slice(1); } + if(value.length % 2) { value = '0'+value; } + valueBuf = Buffer.from(value, 'hex'); + } + break; + + case "object": + if(value === null) { + flags = MH_TYPE_NULL; + } else { + valueBuf = Buffer.from(JSON.stringify(value)); + flags = MH_TYPE_OBJECT; + } + break; + + case "function": + try { + // check if function is serializable, otherwise it will throw when attempting to fetch it + new Function(`return ${value.toString()}`); + } catch(e) { + throw new Error(`This function is not serializable - ${e.message} in "${value}"`); + } + valueBuf = Buffer.from(''+value, 'utf8'); + flags = MH_TYPE_FUNCTION; + break; + + default: + if(value) { + valueBuf = Buffer.from(''+value, 'utf8'); + } + flags = MH_TYPE_STRING; + break; } } @@ -58,32 +106,30 @@ MegaHash.prototype.get = function(key) { if (!keyBuf.length) throw new Error("Key must have length"); var value = this._get( keyBuf ); - if (!value || !value.flags) return value; - - switch (value.flags) { - case MH_TYPE_NULL: - value = null; - break; - - case MH_TYPE_OBJECT: - value = JSON.parse( value.toString() ); - break; - - case MH_TYPE_NUMBER: - value = value.readDoubleBE(); - break; - - case MH_TYPE_BIGINT: - value = value.readBigInt64BE(); - break; - - case MH_TYPE_BOOLEAN: - value = (value.readUInt8() == 1) ? true : false; - break; - - case MH_TYPE_STRING: - value = value.toString(); - break; + + if(Buffer.isBuffer(value)) { + if(!value.flags) return value; + switch (value.flags) { + case MH_TYPE_OBJECT: value = JSON.parse(value.toString()); break; + case MH_TYPE_NUMBER: value = value.readDoubleBE(); break; + case MH_TYPE_BIGINT: value = value.readBigInt64BE(); break; + case MH_TYPE_BIG_BIGINT_POS: value = BigInt("0x"+value.hexSlice()); break; + case MH_TYPE_BIG_BIGINT_NEG: value = -BigInt("0x"+value.hexSlice()); break; + case MH_TYPE_FUNCTION: value = new Function(`return ${value.toString()}`)(); break; + case MH_TYPE_STRING: value = value.toString(); break; + } + } + else if(typeof value === "number") { + switch (value) { + case MH_TYPE_UNDEFINED: value = void 0; break; + case MH_TYPE_NULL: value = null; break; + case MH_TYPE_NAN: value = NaN; break; + case MH_TYPE_INFINITY_POS: value = Infinity; break; + case MH_TYPE_INFINITY_NEG: value = -Infinity; break; + case MH_TYPE_TRUE: value = true; break; + case MH_TYPE_FALSE: value = false; break; + case MH_TYPE_STRING: value = ""; break; // empty strings are short-circuited + } } return value; diff --git a/package.json b/package.json index 57d1500..22247a2 100755 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "node-addon-api": "^3.1.0" }, "devDependencies": { - "pixl-unit": "^1.0.0" + "pixl-unit": "^1.0.11" }, "scripts": { "test": "pixl-unit test/test.js" diff --git a/test-bench2.js b/test-bench2.js new file mode 100644 index 0000000..4980751 --- /dev/null +++ b/test-bench2.js @@ -0,0 +1,43 @@ +const { performance } = require("perf_hooks"); +const MegaHash = require('.'); +const hash = new MegaHash(); +const N = 5000000; + +console.log(`Benchmarking ${N} operations for each type`); + +run("buffer", Buffer.from("abcdfeghijklmnopqrstuvwxyz")); +run("string", "abcdfeghijklmnopqrstuvwxyz"); +run("number", 79483759); +run("true", true); +run("false", false); +run("object", {a:10, b:20, c:30, d:null, e:"abc"}); +run("bigint", 3948759398787n); +run("big bigint", 48579384759349273948273985798475928349726347263957298759387459834n); +run("null", null); +run("nan", NaN); +run("infinity", Infinity); +run("function", () => true); +run("undefined", undefined); + +function run(name, data) { + try { + let time = performance.now(); + for(let i = 0; i < N; i++) { + hash.set(i, data); + } + time = performance.now() - time; + console.log(`${name} - write: ${(N * 1000 / time).toFixed(3)} op/s (${time.toFixed(3)}ms)`); + + time = performance.now(); + for(let i = 0; i < N; i++) { + hash.get(i); + } + time = performance.now() - time; + console.log(`${name} - read: ${(N * 1000 / time).toFixed(3)} op/s (${time.toFixed(3)}ms)`); + //console.log(process.memoryUsage()) + //console.log(`${name} - integrity test`, data, hash.get(1)); + hash.clear(); + } catch(e) { + console.log(`${name} - Failed (${e})`); + } +} diff --git a/test/test.js b/test/test.js index c9f94e2..c70d6ad 100644 --- a/test/test.js +++ b/test/test.js @@ -27,6 +27,14 @@ module.exports = { test.ok(value === "there", 'Basic string set/get'); test.done(); }, + + function testEmptyString(test) { + var hash = new MegaHash(); + hash.set("hello", ""); + var value = hash.get("hello"); + test.ok(value === "", 'Empty string set/get'); + test.done(); + }, function testUnicodeValue(test) { var hash = new MegaHash(); @@ -149,7 +157,32 @@ module.exports = { hash.set( "big2", -9007199254740992n ); test.ok( hash.get("big2") === -9007199254740992n, "Negative BigInt came through unscathed" ); + + hash.set( "big3", 90071998759837958703870284254740993n ); + test.ok( hash.get("big3") === 90071998759837958703870284254740993n, "Very big positive BigInt came through unscathed" ); + hash.set( "big4", -90071998759837958703870284254740993n ); + test.ok( hash.get("big4") === -90071998759837958703870284254740993n, "Very big negative BigInt came through unscathed" ); + + test.done(); + }, + + function testNaN(test) { + var hash = new MegaHash(); + hash.set("nope", NaN); + test.ok( Number.isNaN(hash.get("nope")), "NaN" ); + test.done(); + }, + + function testInfinity(test) { + var hash = new MegaHash(); + + hash.set("infinity", Infinity); + test.ok( hash.get("infinity") === Infinity, "Positive infinity" ); + + hash.set("infinity2", -Infinity); + test.ok( hash.get("infinity2") === -Infinity, "Negative infinity" ); + test.done(); }, @@ -159,6 +192,22 @@ module.exports = { test.ok( hash.get("nope") === null, "Null is null" ); test.done(); }, + + function testUndefined(test) { + var hash = new MegaHash(); + hash.set("nope", undefined); + test.ok( hash.get("nope") === undefined, "Undefined" ); + test.done(); + }, + + function testFunction(test) { + var hash = new MegaHash(); + var f = function(a) { return 10; }; + hash.set("function", f); + var result = hash.get("function"); + test.ok( typeof result === "function" && result.toString() === f.toString(), "Function serialization" ); + test.done(); + }, function testBooleans(test) { var hash = new MegaHash(); From 5ac732f47051823e735c847647bac2499a5d52da Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:14:36 +0100 Subject: [PATCH 2/2] move and rename test file --- test-bench2.js => test/test-types.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test-bench2.js => test/test-types.js (97%) diff --git a/test-bench2.js b/test/test-types.js similarity index 97% rename from test-bench2.js rename to test/test-types.js index 4980751..3ae3f28 100644 --- a/test-bench2.js +++ b/test/test-types.js @@ -1,5 +1,5 @@ const { performance } = require("perf_hooks"); -const MegaHash = require('.'); +const MegaHash = require('../'); const hash = new MegaHash(); const N = 5000000;