From 1ad88178f49efd9eff9a6bb62e9f4409aa99918e Mon Sep 17 00:00:00 2001 From: Grzegorz Sawko Date: Sat, 18 Dec 2021 17:21:17 +0000 Subject: [PATCH 1/5] implemented streaming deserialization for json --- go.mod | 3 ++- go.sum | 2 ++ std/json.joke | 7 ++++++- std/json/a_json.go | 17 +++++++++++++++++ std/json/a_json_slow_init.go | 6 ++++++ std/json/json_native.go | 30 +++++++++++++++++++++++++++++- 6 files changed, 62 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index baf9753cc..ce2f93040 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,9 @@ require ( github.com/candid82/liner v1.4.0 github.com/jcburley/go-spew v1.3.0 github.com/pkg/profile v1.2.1 + github.com/yuin/goldmark v1.3.2 go.etcd.io/bbolt v1.3.3 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect gopkg.in/yaml.v2 v2.2.2 - github.com/yuin/goldmark v1.3.2 + olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect ) diff --git a/go.sum b/go.sum index a9b7e2394..6efaacd7d 100644 --- a/go.sum +++ b/go.sum @@ -16,3 +16,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= diff --git a/std/json.joke b/std/json.joke index a86006e09..053c36556 100644 --- a/std/json.joke +++ b/std/json.joke @@ -19,4 +19,9 @@ :go "writeString(v)"} [^Object v]) - +(defn json-seq + "Returns the json records from rdr as a lazy sequence. + rdr must be a string or implement io.Reader." + {:added "1.0" + :go "jsonSeqOpts(rdr)"} + [^Object rdr]) \ No newline at end of file diff --git a/std/json/a_json.go b/std/json/a_json.go index 3e7f6d073..6e77f164c 100644 --- a/std/json/a_json.go +++ b/std/json/a_json.go @@ -6,6 +6,23 @@ import ( . "github.com/candid82/joker/core" ) +var __json_seq__P ProcFn = __json_seq_ +var json_seq_ Proc = Proc{Fn: __json_seq__P, Name: "json_seq_", Package: "std/json"} + +func __json_seq_(_args []Object) Object { + _c := len(_args) + switch { + case _c == 1: + rdr := ExtractObject(_args, 0) + _res := jsonSeqOpts(rdr) + return _res + + default: + PanicArity(_c) + } + return NIL +} + var __read_string__P ProcFn = __read_string_ var read_string_ Proc = Proc{Fn: __read_string__P, Name: "read_string_", Package: "std/json"} diff --git a/std/json/a_json_slow_init.go b/std/json/a_json_slow_init.go index e62807803..7397a9952 100644 --- a/std/json/a_json_slow_init.go +++ b/std/json/a_json_slow_init.go @@ -14,6 +14,12 @@ func InternsOrThunks() { } jsonNamespace.ResetMeta(MakeMeta(nil, `Implements encoding and decoding of JSON as defined in RFC 4627.`, "1.0")) + jsonNamespace.InternVar("json-seq", json_seq_, + MakeMeta( + NewListFrom(NewVectorFrom(MakeSymbol("rdr"))), + `Returns the json records from rdr as a lazy sequence. + rdr must be a string or implement io.Reader.`, "1.0")) + jsonNamespace.InternVar("read-string", read_string_, MakeMeta( NewListFrom(NewVectorFrom(MakeSymbol("s")), NewVectorFrom(MakeSymbol("s"), MakeSymbol("opts"))), diff --git a/std/json/json_native.go b/std/json/json_native.go index bb9afe393..1f3fe2cf8 100644 --- a/std/json/json_native.go +++ b/std/json/json_native.go @@ -3,8 +3,9 @@ package json import ( "encoding/json" "fmt" - . "github.com/candid82/joker/core" + "io" + "strings" ) func fromObject(obj Object) interface{} { @@ -96,6 +97,33 @@ func readString(s string, opts Map) Object { return toObject(v, keywordize) } +func jsonLazySeq(dec *json.Decoder) *LazySeq { + var c = func(args []Object) Object { + var o interface{} + err := dec.Decode(&o) + if err == io.EOF { + return EmptyList + } + PanicOnErr(err) + obj := toObject(o, false) + return NewConsSeq(obj, jsonLazySeq(dec)) + } + return NewLazySeq(Proc{Fn: c}) +} + +func jsonSeqOpts(src Object) Object { + var dec *json.Decoder + switch src := src.(type) { + case String: + dec = json.NewDecoder(strings.NewReader(src.S)) + case io.Reader: + dec = json.NewDecoder(src) + default: + panic(RT.NewError("src must be a string or io.Reader")) + } + return jsonLazySeq(dec) +} + func writeString(obj Object) String { res, err := json.Marshal(fromObject(obj)) if err != nil { From 1f87c30da62c257054aa3e492cc49a7398cdedd3 Mon Sep 17 00:00:00 2001 From: Roman Bataev Date: Sun, 19 Dec 2021 09:35:02 -0800 Subject: [PATCH 2/5] v0.18.0 --- core/procs.go | 2 +- docs/joker.json.html | 14 ++++++++++++++ docs/main.js | 2 +- go.mod | 1 - go.sum | 2 -- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/procs.go b/core/procs.go index 76a43bee9..fa4b7c4c9 100644 --- a/core/procs.go +++ b/core/procs.go @@ -38,7 +38,7 @@ const ( PRINT_IF_NOT_NIL ) -const VERSION = "v0.17.3" +const VERSION = "v0.18.0" const ( CLJ Dialect = iota diff --git a/docs/joker.json.html b/docs/joker.json.html index fa4d2b524..5d9bb443a 100644 --- a/docs/joker.json.html +++ b/docs/joker.json.html @@ -29,6 +29,9 @@

Summary

Index