From c9ea2ea82ff38242207dd3be63dc4ebcbf6cdfae Mon Sep 17 00:00:00 2001 From: molsonkiko <46202915+molsonkiko@users.noreply.github.com> Date: Tue, 24 Sep 2024 19:31:41 -0700 Subject: [PATCH] Add rand_schema RemesPath function Also improve performance of string generation in RandomJsonFromSchema --- CHANGELOG.md | 1 + .../JSONTools/RandomJsonFromSchema.cs | 24 ++--- .../JSONTools/RemesPathFunctions.cs | 11 +++ JsonToolsNppPlugin/Properties/AssemblyInfo.cs | 4 +- JsonToolsNppPlugin/Tests/RemesPathTests.cs | 54 +++++++++-- docs/README.md | 7 +- docs/RemesPath.md | 5 + most recent errors.txt | 96 +++++++++---------- 8 files changed, 124 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f39f4e5..f47e5f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added 1. When a file is renamed, the name of a [tree view](/docs/README.md#json-tools-overview) associated with that file also changes to match the new name. +2. Add [`rand_schema` RemesPath function](/docs/RemesPath.md#non-vectorized-functions) to generate random JSON from schema. ### Changed diff --git a/JsonToolsNppPlugin/JSONTools/RandomJsonFromSchema.cs b/JsonToolsNppPlugin/JSONTools/RandomJsonFromSchema.cs index b7deb58..f9ebee8 100644 --- a/JsonToolsNppPlugin/JSONTools/RandomJsonFromSchema.cs +++ b/JsonToolsNppPlugin/JSONTools/RandomJsonFromSchema.cs @@ -105,7 +105,7 @@ private JNode RandomNull(JNode schema, JObject refs, int recursionDepth) return new JNode(); } - private static readonly string PRINTABLE = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\r\n"; + public static readonly string PRINTABLE = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\r\n"; private static readonly string EXTENDED_ASCII = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"; @@ -139,24 +139,14 @@ private JNode RandomString(JNode schema, JObject refs, int recursionDepth) exclusiveMaxLength = (int)maxLengthVal + 1; } int length = random.Next(minLength, exclusiveMaxLength); - StringBuilder sb = new StringBuilder(length); - if (extendedAsciiStrings) - { - for (int ii = 0; ii < length; ii++) - { - char randChar = EXTENDED_ASCII[random.Next(1, 256)]; // not allowing \x00 because that will terminate string early in C - sb.Append(randChar); - } - } - else + char[] chars = new char[length]; + for (int ii = 0; ii < length; ii++) { - for (int ii = 0; ii < length; ii++) - { - char randChar = PRINTABLE[random.Next(PRINTABLE.Length)]; - sb.Append(randChar); - } + chars[ii] = extendedAsciiStrings + ? EXTENDED_ASCII[random.Next(1, 256)] // not allowing \x00 because that will terminate string early in C + : PRINTABLE[random.Next(PRINTABLE.Length)]; } - return new JNode(sb.ToString(), Dtype.STR, 0); + return new JNode(new string(chars), Dtype.STR, 0); } #endregion diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs index feff352..11af791 100644 --- a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs +++ b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs @@ -1630,6 +1630,16 @@ public static JNode RandomFrom0To1(List args) return new JNode(rand, Dtype.FLOAT, 0); } + public static JNode RandomFromSchema(List args) + { + JNode node = args[0]; + int minArrayLength = args.Count >= 2 && args[1].value is long l && l < int.MaxValue && l >= 0 ? (int)l : 0; + int maxArrayLength = args.Count >= 3 && args[2].value is long l2 && l2 < int.MaxValue && l2 >= 0 ? (int)l2 : 10; + bool extendedAsciiStrings = args.Count >= 4 && args[3].value is bool b && b; + bool usePatterns = args.Count >= 5 && args[4].value is bool b2 && b2; + return RandomJsonFromSchema.RandomJson(node, minArrayLength, maxArrayLength, extendedAsciiStrings, usePatterns); + } + /// /// randint(start: int, end: int=null) -> int

/// returns a random integer from start (inclusive) to end (exclusive)

@@ -3675,6 +3685,7 @@ public static JNode ObjectsToJNode(object obj) ["pivot"] = new ArgFunction(Pivot, "pivot", Dtype.OBJ, 3, int.MaxValue, false, new Dtype[] { Dtype.ARR, Dtype.STR | Dtype.INT, Dtype.STR | Dtype.INT, /* any # of args */ Dtype.STR | Dtype.INT }), ["quantile"] = new ArgFunction(Quantile, "quantile", Dtype.FLOAT, 2, 2, false, new Dtype[] {Dtype.ARR, Dtype.FLOAT}), ["rand"] = new ArgFunction(RandomFrom0To1, "rand", Dtype.FLOAT, 0, 0, false, new Dtype[] {}, false), + ["rand_schema"] = new ArgFunction(RandomFromSchema, "rand_schema", Dtype.ANYTHING, 1, 5, false, new Dtype[] { Dtype.OBJ, Dtype.INT, Dtype.INT, Dtype.BOOL, Dtype.BOOL }, isDeterministic: false), ["randint"] = new ArgFunction(RandomInteger, "randint", Dtype.INT, 1, 2, false, new Dtype[] {Dtype.INT, Dtype.INT}, false), ["range"] = new ArgFunction(Range, "range", Dtype.ARR, 1, 3, false, new Dtype[] {Dtype.INT, Dtype.INT, Dtype.INT}), ["s_cat"] = new ArgFunction(StrCat, "s_cat", Dtype.STR, 1, int.MaxValue, false, new Dtype[] {Dtype.ANYTHING, /* any # of args */ Dtype.ANYTHING}), diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs index e230ee1..54ee447 100644 --- a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs +++ b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs @@ -28,5 +28,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("8.1.0.9")] -[assembly: AssemblyFileVersion("8.1.0.9")] +[assembly: AssemblyVersion("8.1.0.10")] +[assembly: AssemblyFileVersion("8.1.0.10")] diff --git a/JsonToolsNppPlugin/Tests/RemesPathTests.cs b/JsonToolsNppPlugin/Tests/RemesPathTests.cs index bd95dff..c7115fa 100644 --- a/JsonToolsNppPlugin/Tests/RemesPathTests.cs +++ b/JsonToolsNppPlugin/Tests/RemesPathTests.cs @@ -779,7 +779,6 @@ public static bool Test() } // the rand() and randint() functions require a special test because their outputs are nondeterministic ii += 7; - bool testFailed = false; string randints1argQuery = "flatten(@.foo)[:]->randint(1000)"; string randints2argQuery = "range(9)[:]->randint(-700, 800)"; string randintsIfElseQuery = "var stuff = j`[\"foo\", \"bar\", \"baz\", \"quz\"]`; range(17)[:]->at(stuff, abs(randint(-20, 20) % 4))"; @@ -814,7 +813,8 @@ public static bool Test() testsFailed += 3; Npp.AddLine($"While testing randint, got error {RemesParser.PrettifyException(ex)}"); } - for (int randNum = 0; randNum < 28 && !testFailed; randNum++) + bool allRandTestsFailed = false; + for (int randNum = 0; randNum < 28 && !allRandTestsFailed; randNum++) { // rand() try @@ -822,14 +822,14 @@ public static bool Test() result = remesparser.Search("rand()", foo); if (!(result.value is double d && d >= 0d && d < 1d)) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(rand(), foo) to return a double between 0 and 1, but instead got {result.ToString()}"); } } catch (Exception ex) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(rand(), foo) to return a double between 0 and 1 but instead threw" + $" an exception:\n{RemesParser.PrettifyException(ex)}"); @@ -840,14 +840,14 @@ public static bool Test() result = remesparser.Search("ifelse(rand() < 0.5, a, b)", foo); if (!(result.value is string s && (s == "a" || s == "b"))) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(ifelse(rand(), a, b), foo) to return \"a\" or \"b\", but instead got {result.ToString()}"); } } catch (Exception ex) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(ifelse(rand(), a, b), foo) to return \"a\" or \"b\" but instead threw" + $" an exception:\n{RemesParser.PrettifyException(ex)}"); @@ -858,7 +858,7 @@ public static bool Test() result = remesparser.Search("j`[1,2,3]`[:]{rand()}[0]", foo); if (!(result is JArray arr) || arr.children.All(x => x == arr[0])) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(j`[1,2,3]`{{{{rand(@)}}}}[0], foo) to return array of doubles that aren't all equal" + $", but instead got {result.ToString()}"); @@ -866,7 +866,7 @@ public static bool Test() } catch (Exception ex) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(j`[1,2,3]`{{{{rand(@)}}}}[0] to return array of doubles that aren't equal, but instead threw" + $" an exception:\n{RemesParser.PrettifyException(ex)}"); @@ -879,14 +879,14 @@ public static bool Test() result = remesparser.Search(q, foo); if (!(result is JArray arr && arr[0].value is double d1 && d1 >= 0 && d1 <= 1 && arr.children.All(x => x.value is double xd && xd == d1))) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(\"{q}\", foo) to return an array where every value is the same double between 0 and 1, but instead got result {result.ToString()}"); } } catch (Exception ex) { - testFailed = true; + allRandTestsFailed = true; testsFailed++; Npp.AddLine($"Expected remesparser.Search(\"{q}\", foo) to return an array where every value is the same double between 0 and 1, but instead got exception {RemesParser.PrettifyException(ex)}"); } @@ -949,6 +949,40 @@ public static bool Test() $"but instead got {result.ToString()}."); } } + // test that rand_schema correctly uses all args + var randFromSchemaTestcases = new (string query, Func passIf)[] + { + ("rand_schema(j`{\"type\": \"boolean\"}`)", x => x.value is bool), // base case, simple schema and no args + ("rand_schema(j`{\"type\": \"array\", \"items\": {\"type\": [\"integer\", \"null\"]}}`, 11, 13)", x => x is JArray jarr && jarr.Length >= 11 && jarr.Length <= 13 && jarr.children.All(y => y.type == Dtype.INT || y.type == Dtype.NULL)), // make sure minArrayLength and maxArrayLength are followed + ("rand_schema(j`{\"type\": \"string\", \"pattern\": \"\\\\\\\\d{4,9}|foo\"}`,,,,true)", x => x.value is string s && (s == "foo" || (s.Length <= 9 && s.Length >= 4 && s.All(y => y.IsDigit())))), // make sure isPattern = true makes string follow regex + ("rand_schema(j`{\"type\": \"string\", \"pattern\": \"\\\\\\\\d{4,9}|foo\"}`)", x => x.value is string s && !(s == "foo" || (s.Length <= 9 && s.Length >= 4 && s.All(y => y.IsDigit())))), // make sure isPattern = false makes string ignore regex + ("rand_schema(j`{\"type\": \"array\", \"items\": {\"type\": \"boolean\"}}`)", x => x is JArray jarr && jarr.Length >= 0 && jarr.Length <= 10 && jarr.children.All(y => y.value is bool)), // make sure default array length range applies when no minArrayLength or maxArrayLength specified + ("rand_schema(j`{\"type\": \"string\", \"maxLength\": 17, \"minLength\": 14}`,,,true)", x => x.value is string s && s.Length >= 14 && s.Length <= 17 && s.All(y => y >= 1 && y <= 255) && s.Any(y => RandomJsonFromSchema.PRINTABLE.IndexOf(y) < 0)), // make sure extendedAsciiStrings = true makes string extended ASCII + ("rand_schema(j`{\"type\": \"string\", \"maxLength\": 12, \"minLength\": 9}`)", x => x.value is string s && s.Length >= 9 && s.Length <= 12 && s.All(y => RandomJsonFromSchema.PRINTABLE.IndexOf(y) >= 0)), // make sure extendedAsciiStrings = false makes string only printables + }; + var jnull = new JNode(); + foreach ((string query, Func passIf) in randFromSchemaTestcases) + { + ii++; + try + { + for (int jj = 0; jj < 6; jj++) + { + JNode node = remesparser.Search(query, jnull); + if (!passIf(node)) + { + testsFailed++; + Npp.AddLine($"Failed test {query} because output {node.ToString()} did not pass test"); + break; + } + } + } + catch (Exception ex) + { + testsFailed++; + Npp.AddLine($"Failed test {query} because of exception {ex}"); + } + } /** * Test s_csv and s_fa caching (only need to use s_csv, because s_fa uses the same caching system) **/ diff --git a/docs/README.md b/docs/README.md index c8a82b2..1911ad6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -911,7 +911,7 @@ This tool can only validate the following keywords: ## Generating random JSON from a schema ## -The plugin can also generate random JSON from a schema. The default minimum and maximum array lengths (for schemas where the `minItems` and `maxItems` keywords are omitted) are `0` and `10` respectively. +The plugin can also generate random JSON from a schema. The default minimum and maximum array lengths (for schemas where the `minItems` and `maxItems` keywords are omitted) are `0` and `10` respectively, and can be customized with the `minArrayLength` and `maxArrayLength` settings. *Added in version 4.8.1:* You can also use a non-schema file to generate random JSON. A schema will be generated on the fly, and that schema will be used to make the random JSON. @@ -948,7 +948,12 @@ The following keywords are supported for random JSON generation: ### Keywords for strings ### * [minLength](https://json-schema.org/draft/2020-12/json-schema-validation.html#name-minlength) and [maxLength](https://json-schema.org/draft/2020-12/json-schema-validation.html#name-maxlength) (*added in [7.1](/CHANGELOG.md#710---2024-02-28)*) + * if `minLength` is not specified, strings will have a minimum length of * [pattern](https://json-schema.org/draft/2020-12/json-schema-validation.html#name-pattern) (*see note below on [random strings from regex](#random-strings-from-regex-added-in-v81)*) + * If there is no `pattern` keyword, or if `generate_random_patterns` is `False`, the following are all true: + * All strings will have fewer than 11 characters. + * If `extended_ascii_strings` is `False` (the default), the characters in each string will be drawn from the set of printable ASCII characters, ``"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\r\n"`` + * If `extended_ascii_strings` is `True`, the characters in each string will have UTF-16 codes between 1 (`\x01`) and 255 (`ΓΏ`) ### Random strings from regex (*added in v8.1*) diff --git a/docs/RemesPath.md b/docs/RemesPath.md index 6d5fee5..7436f02 100644 --- a/docs/RemesPath.md +++ b/docs/RemesPath.md @@ -536,6 +536,11 @@ Then the returned value is `0.6*10 + 0.4*8`, or 9.2. Random number between 0 (inclusive) and 1 (exclusive). *Added in [v5.2](/CHANGELOG.md#520---2023-06-04)* +--- +`rand_schema(schema: object, minArrayLength: int = 0, maxArrayLength: int = 10, extendedAsciiStrings: bool = false, usePatterns: bool = false)` + +Creates [random JSON from `schema`](/docs/README.md#generating-random-json-from-a-schema), where the four optional arguments take the place of the global settings `minArrayLength`, `maxArrayLength`, `extended_ascii_strings`, and `generate_random_patterns`, respectively. + --- `randint(start: int, end: int=null) -> int` diff --git a/most recent errors.txt b/most recent errors.txt index f327129..0a36803 100644 --- a/most recent errors.txt +++ b/most recent errors.txt @@ -1,4 +1,4 @@ -Test results for JsonTools v8.1.0.7 on Notepad++ 8.7.0 64bit +Test results for JsonTools v8.1.0.10 on Notepad++ 8.7.0 64bit NOTE: Ctrl-F (regular expressions *on*) for "Failed [1-9]\d*" to find all failed tests Tests failed: YAML dumper ========================= @@ -107,7 +107,7 @@ Testing RemesPath parser and compiler The queried JSON in the RemesParser tests is named foo:{"foo": [[0, 1, 2], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]], "bar": {"a": false, "b": ["a`g", "bah"]}, "baz": "z", "quz": {}, "jub": [], "guzo": [[[1]], [[2], [3]]], "7": [{"foo": 2}, 1], "_": {"0": 0}} Failed 0 tests. -Passed 530 tests. +Passed 537 tests. ========================= Testing RemesPath throws errors on bad inputs ========================= @@ -206,33 +206,33 @@ Testing JsonParser performance Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 89556 into JNode took 2.906 +/- 1.981 ms over 32 trials -Load times (ms): 2, 2, 5, 1, 1, 6, 2, 1, 1, 4, 1, 2, 7, 1, 1, 1, 5, 1, 1, 6, 1, 1, 2, 7, 1, 1, 1, 4, 1, 1, 5, 1 +To convert JSON string of size 89556 into JNode took 3.288 +/- 2.503 ms over 32 trials +Load times (ms): 2, 11, 2, 2, 5, 1, 1, 1, 5, 2, 2, 9, 1, 1, 2, 4, 1, 1, 7, 2, 2, 2, 5, 1, 1, 6, 1, 1, 2, 6, 2, 2 ========================= Performance tests for RemesPath (float arithmetic) ========================= -Compiling query "@[@[:].a * @[:].t < @[:].e]" took 0.094 ms the first time, including approximately 0.111 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.028 +/- 0.015 ms over 40 trials -Query times (ms): 0.083, 0.037, 0.023, 0.041, 0.023, 0.022, 0.022, 0.031, 0.022, 0.022, 0.022, 0.022, 0.022, 0.026, 0.023, 0.023, 0.022, 0.022, 0.022, 0.027, 0.022, 0.023, 0.023, 0.029, 0.022, 0.026, 0.023, 0.022, 0.023, 0.022, 0.022, 0.025, 0.022, 0.022, 0.023, 0.022, 0.022, 0.092, 0.037, 0.045 +Compiling query "@[@[:].a * @[:].t < @[:].e]" took 0.084 ms the first time, including approximately 0.082 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.112 +/- 0.515 ms over 40 trials +Query times (ms): 0.12, 0.124, 0.023, 0.021, 0.023, 0.032, 0.022, 0.028, 0.049, 0.022, 0.022, 0.022, 0.022, 0.025, 3.324, 0.03, 0.023, 0.022, 0.022, 0.024, 0.022, 0.022, 0.022, 0.022, 0.022, 0.023, 0.022, 0.023, 0.022, 0.033, 0.024, 0.023, 0.022, 0.022, 0.022, 0.021, 0.021, 0.023, 0.029, 0.022 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (string operations) ========================= -Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" took 0.04 ms the first time, including approximately 0.042 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.14 +/- 0.513 ms over 40 trials -Query times (ms): 0.102, 0.058, 0.064, 0.067, 0.089, 0.054, 0.054, 0.055, 0.054, 0.054, 0.053, 0.054, 0.053, 0.054, 0.053, 0.054, 0.054, 0.054, 0.053, 0.06, 0.054, 0.06, 0.055, 0.053, 0.054, 0.054, 0.054, 0.054, 0.054, 0.053, 3.345, 0.057, 0.07, 0.066, 0.053, 0.052, 0.052, 0.052, 0.052, 0.052 +Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" took 0.04 ms the first time, including approximately 0.043 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.078 +/- 0.029 ms over 40 trials +Query times (ms): 0.121, 0.058, 0.076, 0.084, 0.054, 0.053, 0.054, 0.054, 0.054, 0.053, 0.054, 0.054, 0.053, 0.054, 0.053, 0.055, 0.056, 0.1, 0.159, 0.115, 0.11, 0.123, 0.098, 0.096, 0.094, 0.099, 0.124, 0.099, 0.104, 0.127, 0.069, 0.055, 0.052, 0.054, 0.055, 0.092, 0.052, 0.054, 0.056, 0.09 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "NK nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" took 0.207 ms the first time, including approximately 0.383 ms to tokenize the query. Subsequent executions are effectively free due to caching. +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" took 0.216 ms the first time, including approximately 0.217 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var qmask = @[:].q; var nmax_q = max(@[qmask].n); var nmax_notq = max(@[not qmask].n); -ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.017 +/- 0.01 ms over 40 trials -Query times (ms): 0.078, 0.019, 0.016, 0.016, 0.015, 0.016, 0.015, 0.015, 0.016, 0.015, 0.015, 0.015, 0.016, 0.016, 0.015, 0.015, 0.015, 0.015, 0.015, 0.016, 0.016, 0.015, 0.021, 0.015, 0.015, 0.015, 0.015, 0.015, 0.015, 0.015, 0.015, 0.016, 0.015, 0.015, 0.015, 0.016, 0.015, 0.015, 0.015, 0.015 +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.021 +/- 0.015 ms over 40 trials +Query times (ms): 0.11, 0.019, 0.017, 0.016, 0.017, 0.018, 0.021, 0.025, 0.019, 0.027, 0.016, 0.016, 0.043, 0.015, 0.015, 0.016, 0.016, 0.016, 0.016, 0.015, 0.015, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015, 0.015, 0.016, 0.015, 0.036, 0.025, 0.026, 0.025, 0.025, 0.024, 0.017, 0.015, 0.015, 0.016 Preview of result: "when q=false, nmax= 9830935647.0" ... ========================= @@ -271,11 +271,11 @@ Performance tests for RemesPath (references to compile-time constant variables) Compiling query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" took 0.123 ms the first time, including approximately 0.136 ms to tokenize the query. Subsequent executions are effectively free due to caching. +@[:]->at(@, X)->at(@, onetwo)" took 0.081 ms the first time, including approximately 0.101 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.015 +/- 0.008 ms over 40 trials -Query times (ms): 0.059, 0.013, 0.021, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.013, 0.014, 0.012, 0.012, 0.013, 0.018, 0.029, 0.012, 0.013, 0.012, 0.032, 0.019, 0.012, 0.012, 0.012, 0.012, 0.012, 0.013, 0.012, 0.012, 0.013, 0.012, 0.013, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012, 0.012 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.026 +/- 0.061 ms over 40 trials +Query times (ms): 0.055, 0.013, 0.013, 0.012, 0.013, 0.013, 0.012, 0.013, 0.013, 0.014, 0.013, 0.012, 0.013, 0.012, 0.013, 0.013, 0.012, 0.014, 0.013, 0.013, 0.012, 0.013, 0.013, 0.014, 0.012, 0.013, 0.013, 0.014, 0.013, 0.012, 0.012, 0.013, 0.021, 0.026, 0.022, 0.026, 0.034, 0.404, 0.046, 0.022 Preview of result: [[1695727848, 0.28756263873668497], [2126430375, 0.0076779412970817704], [5310550656, 0.38076977264568701], [2519183283, 0.15317622093055799], [6610062385, 0.66299622587066598], [987168256, 0.92441018999992797], [6615003609, 0.91711269122594696], [4465232046, 0.68431193185153605], [8654414565, 0.631 ... ========================= @@ -284,29 +284,29 @@ Performance tests for RemesPath (references to variables that are not compile-ti Compiling query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" took 0.084 ms the first time, including approximately 0.125 ms to tokenize the query. Subsequent executions are effectively free due to caching. +@[:]->at(@, X)->at(@, onetwo)" took 0.092 ms the first time, including approximately 0.088 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.026 +/- 0.054 ms over 40 trials -Query times (ms): 0.056, 0.016, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.015, 0.016, 0.022, 0.018, 0.02, 0.359, 0.025, 0.016, 0.015, 0.015, 0.016, 0.015, 0.023, 0.016, 0.016, 0.015, 0.016, 0.015, 0.015, 0.016, 0.015, 0.015, 0.016, 0.015, 0.015, 0.016, 0.016 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.019 +/- 0.008 ms over 40 trials +Query times (ms): 0.058, 0.017, 0.035, 0.016, 0.02, 0.017, 0.015, 0.016, 0.017, 0.017, 0.019, 0.017, 0.016, 0.023, 0.028, 0.028, 0.045, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016 Preview of result: [[1695727848, 0.28756263873668497], [2126430375, 0.0076779412970817704], [5310550656, 0.38076977264568701], [2519183283, 0.15317622093055799], [6610062385, 0.66299622587066598], [987168256, 0.92441018999992797], [6615003609, 0.91711269122594696], [4465232046, 0.68431193185153605], [8654414565, 0.631 ... ========================= Performance tests for RemesPath (simple string mutations) ========================= -Compiling query "@[:].z = s_sub(@, g, B)" took 0.048 ms the first time, including approximately 0.048 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.018 +/- 0.01 ms over 40 trials -Query times (ms): 0.076, 0.014, 0.011, 0.024, 0.014, 0.014, 0.014, 0.013, 0.017, 0.014, 0.013, 0.015, 0.014, 0.017, 0.016, 0.024, 0.026, 0.033, 0.02, 0.018, 0.016, 0.017, 0.017, 0.016, 0.017, 0.017, 0.018, 0.021, 0.016, 0.019, 0.016, 0.021, 0.012, 0.011, 0.013, 0.012, 0.011, 0.012, 0.011, 0.011 +Compiling query "@[:].z = s_sub(@, g, B)" took 0.064 ms the first time, including approximately 0.046 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.072 +/- 0.334 ms over 40 trials +Query times (ms): 0.047, 0.011, 0.02, 0.02, 0.02, 0.014, 0.015, 0.019, 0.032, 0.026, 0.075, 0.018, 0.014, 0.014, 0.013, 0.015, 0.015, 0.011, 0.01, 0.011, 0.01, 0.01, 0.011, 0.011, 0.013, 0.009, 0.01, 0.024, 0.018, 2.156, 0.026, 0.024, 0.021, 0.022, 0.022, 0.022, 0.019, 0.023, 0.012, 0.014 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (simple number mutations) ========================= -Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" took 0.075 ms the first time, including approximately 0.074 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.041 +/- 0.046 ms over 40 trials -Query times (ms): 0.039, 0.029, 0.026, 0.027, 0.019, 0.017, 0.02, 0.019, 0.017, 0.018, 0.038, 0.025, 0.02, 0.044, 0.036, 0.055, 0.056, 0.041, 0.022, 0.02, 0.037, 0.033, 0.038, 0.035, 0.031, 0.032, 0.035, 0.071, 0.057, 0.315, 0.034, 0.041, 0.036, 0.035, 0.027, 0.026, 0.028, 0.062, 0.037, 0.032 +Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" took 0.128 ms the first time, including approximately 0.128 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.077 +/- 0.309 ms over 40 trials +Query times (ms): 0.054, 2.005, 0.035, 0.033, 0.032, 0.031, 0.032, 0.034, 0.034, 0.031, 0.034, 0.056, 0.037, 0.047, 0.03, 0.033, 0.026, 0.027, 0.027, 0.025, 0.026, 0.027, 0.025, 0.018, 0.017, 0.018, 0.018, 0.025, 0.032, 0.02, 0.02, 0.017, 0.018, 0.02, 0.021, 0.019, 0.018, 0.016, 0.018, 0.018 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= @@ -316,12 +316,12 @@ Performance tests for RemesPath (mutations with a for loop) Compiling query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" took 0.276 ms the first time, including approximately 0.197 ms to tokenize the query. Subsequent executions are effectively free due to caching. +end for;" took 0.119 ms the first time, including approximately 0.135 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" on JNode from JSON of size 89556 into took 0.083 +/- 0.144 ms over 40 trials -Query times (ms): 0.082, 0.056, 0.04, 0.04, 0.038, 0.037, 0.038, 0.093, 0.071, 0.071, 0.069, 0.04, 0.039, 0.04, 0.064, 0.072, 0.064, 0.052, 0.039, 0.076, 0.076, 0.071, 0.072, 0.073, 0.066, 0.099, 0.087, 0.141, 0.09, 0.968, 0.048, 0.038, 0.038, 0.038, 0.056, 0.036, 0.04, 0.039, 0.038, 0.07 +end for;" on JNode from JSON of size 89556 into took 0.074 +/- 0.161 ms over 40 trials +Query times (ms): 0.066, 0.041, 0.068, 0.039, 0.038, 0.036, 1.076, 0.039, 0.038, 0.037, 0.062, 0.035, 0.037, 0.037, 0.037, 0.038, 0.038, 0.05, 0.073, 0.053, 0.039, 0.038, 0.038, 0.044, 0.038, 0.037, 0.038, 0.038, 0.045, 0.037, 0.036, 0.05, 0.044, 0.038, 0.066, 0.093, 0.081, 0.069, 0.071, 0.066 Preview of result: [["bar", false], ["bar", false], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["ba ... ========================= @@ -330,32 +330,32 @@ Testing performance of JSON compression and pretty-printing Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To compress JNode from JSON string of 89556 took 4.493 +/- 0.854 ms over 64 trials (minimal whitespace, sortKeys=TRUE) -To compress JNode from JSON string of 89556 took 2.225 +/- 0.457 ms over 64 trials (minimal whitespace, sortKeys=FALSE) -To Google-style pretty-print JNode from JSON string of 89556 took 4.619 +/- 0.856 ms over 64 trials (sortKeys=true, indent=4) -To Whitesmith-style pretty-print JNode from JSON string of 89556 took 5.027 +/- 1.06 ms over 64 trials (sortKeys=true, indent=4) -To PPrint-style pretty-print JNode from JSON string of 89556 took 7.612 +/- 1.818 ms over 64 trials (sortKeys=true, indent=4) +To compress JNode from JSON string of 89556 took 4.839 +/- 1.066 ms over 64 trials (minimal whitespace, sortKeys=TRUE) +To compress JNode from JSON string of 89556 took 2.556 +/- 0.773 ms over 64 trials (minimal whitespace, sortKeys=FALSE) +To Google-style pretty-print JNode from JSON string of 89556 took 5.078 +/- 1.18 ms over 64 trials (sortKeys=true, indent=4) +To Whitesmith-style pretty-print JNode from JSON string of 89556 took 4.715 +/- 1.04 ms over 64 trials (sortKeys=true, indent=4) +To PPrint-style pretty-print JNode from JSON string of 89556 took 7.098 +/- 1.252 ms over 64 trials (sortKeys=true, indent=4) ========================= Testing performance of JsonSchemaValidator and random JSON creation ========================= -To create a random set of JSON from file at path C:\Program Files\Notepad++\plugins\JsonTools\testfiles\tweet_schema.json of size 174722 (array of 15 items) based on the matching schema took 7.692 +/- 3.885 ms over 25 trials -To compile the schema to a validation function took 0.431 +/- 0.82 ms over 25 trials -To validate JSON of size 174722 (array of 15 items) based on the compiled schema took 1.21 +/- 0.285 ms over 25 trials +To create a random set of JSON from file at path C:\Program Files\Notepad++\plugins\JsonTools\testfiles\tweet_schema.json of size 182611 (array of 15 items) based on the matching schema took 6.853 +/- 3.316 ms over 25 trials +To compile the schema to a validation function took 0.395 +/- 0.768 ms over 25 trials +To validate JSON of size 182611 (array of 15 items) based on the compiled schema took 1.078 +/- 0.247 ms over 25 trials ========================= Testing performance of random JSON from schema with patterns and patternProperties ========================= -To create a random set of JSON from string (see TestRunner.cs) of size 30036 (array of 120 items) based on the matching schema took 1.796 +/- 0.904 ms over 25 trials -To compile the schema to a validation function took 0.376 +/- 0.39 ms over 25 trials -To validate JSON of size 30036 (array of 120 items) based on the compiled schema took 10.531 +/- 1.17 ms over 25 trials +To create a random set of JSON from string (see TestRunner.cs) of size 32400 (array of 120 items) based on the matching schema took 1.758 +/- 0.96 ms over 25 trials +To compile the schema to a validation function took 0.286 +/- 0.059 ms over 25 trials +To validate JSON of size 32400 (array of 120 items) based on the compiled schema took 10.02 +/- 0.909 ms over 25 trials ========================= Testing performance of random JSON from schema *ignoring* patterns and patternProperties ========================= -To create a random set of JSON from string (see TestRunner.cs) of size 11114 (array of 120 items) based on the matching schema took 1.125 +/- 0.945 ms over 25 trials -To compile the schema to a validation function took 0.34 +/- 0.408 ms over 25 trials -To validate JSON of size 11114 (array of 120 items) based on the compiled schema took 6.235 +/- 0.619 ms over 25 trials +To create a random set of JSON from string (see TestRunner.cs) of size 10017 (array of 120 items) based on the matching schema took 1.059 +/- 0.628 ms over 25 trials +To compile the schema to a validation function took 0.27 +/- 0.038 ms over 25 trials +To validate JSON of size 10017 (array of 120 items) based on the compiled schema took 6.485 +/- 0.655 ms over 25 trials ========================= Testing JSON grepper's API request tool =========================