From 29a8548a428d5db5d48ca2fbfb2ab95155da2f23 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 14 Apr 2019 19:59:17 -0700 Subject: [PATCH 01/15] [log] Use go mod for benchmark - previously it was relying on GOPATH - with replace directive, we can even allow benchmark a in develop version - update test dependency for gommon --- go.mod | 6 ++--- go.sum | 13 +++++------ log/_benchmarks/Makefile | 13 ++++++----- log/_benchmarks/benchmark_test.go | 12 +++++----- log/_benchmarks/go.mod | 20 ++++++++++++++++ log/_benchmarks/go.sum | 38 +++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 23 deletions(-) create mode 100644 log/_benchmarks/go.mod create mode 100644 log/_benchmarks/go.sum diff --git a/go.mod b/go.mod index cb26c4b..15b3581 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,9 @@ module github.com/dyweb/gommon require ( github.com/davecgh/go-spew v1.1.1 - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.2 // indirect - github.com/stretchr/testify v1.2.2 - gopkg.in/yaml.v2 v2.2.1 + github.com/stretchr/testify v1.3.0 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index cf2333c..0d2b340 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -10,9 +9,9 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/log/_benchmarks/Makefile b/log/_benchmarks/Makefile index 95aab99..ab6e26c 100644 --- a/log/_benchmarks/Makefile +++ b/log/_benchmarks/Makefile @@ -1,12 +1,13 @@ -# TODO: the dependency is from global go path ... +GO = GO111MODULE=on go + bench: - go test -run none -bench . -benchtime 3s -benchmem -memprofile p.out + $(GO) test -run none -bench . -benchtime 3s -benchmem -memprofile p.out bench-gommon: - go test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out + $(GO) test -run none -bench=".*/gommon" -benchtime 3s -benchmem -memprofile p.out bench-gommon-no-fields: - go test -run none -bench="BenchmarkWithoutFieldsText/gommon.F" -benchtime 3s -benchmem -memprofile p.out + $(GO) test -run none -bench="BenchmarkWithoutFieldsText/gommon.F" -benchtime 3s -benchmem -memprofile p.out bench-gommon-no-context-with-fields: - go test -run none -bench="BenchmarkNoContextWithFieldsJSON/gommon.F" -benchtime 3s -benchmem -memprofile p.out + $(GO) test -run none -bench="BenchmarkNoContextWithFieldsJSON/gommon.F" -benchtime 3s -benchmem -memprofile p.out pprof-ui: # TODO: need to give it binary path otherwise it will throw error - go tool pprof -http=:8080 p.out \ No newline at end of file + $(GO) tool pprof -http=:8080 p.out \ No newline at end of file diff --git a/log/_benchmarks/benchmark_test.go b/log/_benchmarks/benchmark_test.go index ce6b411..4b2252e 100644 --- a/log/_benchmarks/benchmark_test.go +++ b/log/_benchmarks/benchmark_test.go @@ -112,7 +112,7 @@ func BenchmarkDisabledLevelNoFormat(b *testing.B) { msg := "If you support level you should not see me and should not cause allocation, I know I talk too much" b.Run("gommon", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.ErrorLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -124,7 +124,7 @@ func BenchmarkDisabledLevelNoFormat(b *testing.B) { }) b.Run("gommon.F", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.ErrorLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -134,7 +134,7 @@ func BenchmarkDisabledLevelNoFormat(b *testing.B) { }) b.Run("gommon.check", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.ErrorLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -220,7 +220,7 @@ func BenchmarkWithoutFieldsText(b *testing.B) { b.Run("gommon", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.InfoLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -230,7 +230,7 @@ func BenchmarkWithoutFieldsText(b *testing.B) { }) b.Run("gommon.F", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.InfoLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -313,7 +313,7 @@ func BenchmarkWithoutFieldsTextFormat(b *testing.B) { b.Run("gommon", func(b *testing.B) { logger := dlog.NewTestLogger(dlog.InfoLevel) - logger.SetHandler(dlog.NewIOHandler(ioutil.Discard)) + logger.SetHandler(dlog.NewTextHandler(ioutil.Discard)) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { diff --git a/log/_benchmarks/go.mod b/log/_benchmarks/go.mod new file mode 100644 index 0000000..d4cbed5 --- /dev/null +++ b/log/_benchmarks/go.mod @@ -0,0 +1,20 @@ +module github.com/dyweb/logbench + +require ( + github.com/apex/log v1.1.0 + github.com/davecgh/go-spew v1.1.1 + github.com/dyweb/gommon v0.0.10 + github.com/fatih/color v1.7.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/mattn/go-colorable v0.1.1 // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/rs/zerolog v1.13.0 + github.com/sirupsen/logrus v1.4.1 + github.com/spf13/cobra v0.0.3 + github.com/stretchr/testify v1.2.2 + go.uber.org/atomic v1.3.2 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.9.1 + gopkg.in/yaml.v2 v2.2.1 + k8s.io/klog v0.3.0 +) diff --git a/log/_benchmarks/go.sum b/log/_benchmarks/go.sum new file mode 100644 index 0000000..6da6018 --- /dev/null +++ b/log/_benchmarks/go.sum @@ -0,0 +1,38 @@ +github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= +github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dyweb/gommon v0.0.10 h1:CSqX2eneXCAhKxNB6/zx49mo5FN8Kr+clJJqEYL24+A= +github.com/dyweb/gommon v0.0.10/go.mod h1:5p9BED+x4tHzEh6EfM71QGe19XHUHkkhnRVpftNsdjw= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= From 9f901e34bb19f0d03cec1ec34e1a3e3e5f7e4354 Mon Sep 17 00:00:00 2001 From: at15 Date: Fri, 19 Apr 2019 11:00:49 -0700 Subject: [PATCH 02/15] [log][doc] Init design for simplfied log registry --- .../design/2019-04-14-simplify-registry.md | 20 +++++++++++++++++++ log/registry.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 log/doc/design/2019-04-14-simplify-registry.md diff --git a/log/doc/design/2019-04-14-simplify-registry.md b/log/doc/design/2019-04-14-simplify-registry.md new file mode 100644 index 0000000..38b03ba --- /dev/null +++ b/log/doc/design/2019-04-14-simplify-registry.md @@ -0,0 +1,20 @@ +# 2019-04-14 Simplify registry + +## Issues + +- [#110](https://github.com/dyweb/gommon/issues/110) Simplify logger registry + +## Background + +Although gommon has been keeping a tree of logger, but it is never put into use, there is no UI (commandline/web) that +shows the hierarchy or make use of the hierarchy to generate filtered output, further more forcing the hierarchy makes +the library hard to use. + +Currently registry is like a folder while loggers are like file, a registry has three types, application, library and +package, the original idea is library project can export a registry to allow projects importing this library to control +its behavior directly, however having registry actually make things complex because individual package is independent, +`foo` and `foo/bar` does need to have same hierarchy as their on disk one, further more go does not allow cycle import. + +The main problem for logger is fields is still a bit hard to use because unlike other loggers, we only have a `Add` +method to add fields in place and requires using `Copy` to create a copy, it most other loggers they make a copy +directly when adding fields (some like zap and zerolog serialize the fields when adding them) \ No newline at end of file diff --git a/log/registry.go b/log/registry.go index 1822501..9b3619f 100644 --- a/log/registry.go +++ b/log/registry.go @@ -153,7 +153,7 @@ func WalkLogger(root *Registry, cb func(l *Logger)) { walkLogger(root, nil, nil, cb) } -// walkLogger loops loggers in current registry first, then visist its children in DFS +// walkLogger loops loggers in current registry first, then visit its children in DFS // It will create the map if it is nil, so caller don't need to do the bootstrap, // However, caller can provide a map so they can use it to get all the loggers and registry func walkLogger(root *Registry, loggers map[*Logger]bool, registries map[*Registry]bool, cb func(l *Logger)) { From a7c672d3e19caf8c97f7674a38bb14fe5f9b5c57 Mon Sep 17 00:00:00 2001 From: at15 Date: Thu, 25 Apr 2019 09:54:10 -0700 Subject: [PATCH 03/15] [log][doc] Add survey for log4j2 - now it keeps hierarchy in logger config instead of logger like in log4j1, appenders is inherited and it can be stopped using special flag --- .../design/2019-04-14-simplify-registry.md | 7 ++- log/doc/survey/log4j.md | 43 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/log/doc/design/2019-04-14-simplify-registry.md b/log/doc/design/2019-04-14-simplify-registry.md index 38b03ba..a696513 100644 --- a/log/doc/design/2019-04-14-simplify-registry.md +++ b/log/doc/design/2019-04-14-simplify-registry.md @@ -17,4 +17,9 @@ its behavior directly, however having registry actually make things complex beca The main problem for logger is fields is still a bit hard to use because unlike other loggers, we only have a `Add` method to add fields in place and requires using `Copy` to create a copy, it most other loggers they make a copy -directly when adding fields (some like zap and zerolog serialize the fields when adding them) \ No newline at end of file +directly when adding fields (some like zap and zerolog serialize the fields when adding them) + +Go logging library rarely keep a tree hierarchy, it is not the case in Java. +In [Solr](../survey/solr.md) where log4j1 is used, relationship is kept inside loggers, +in [log4j 2](../survey/log4j.md) they introduced logger config with hierarchy, it is determined by logger name and dot. +(TODO: is this hierarchy applied every time when log or only once on configuration change) \ No newline at end of file diff --git a/log/doc/survey/log4j.md b/log/doc/survey/log4j.md index c737b42..9dc25ce 100644 --- a/log/doc/survey/log4j.md +++ b/log/doc/survey/log4j.md @@ -13,4 +13,45 @@ if (logger.isTraceEnabled()) { // Java-8 lamdba logger.trace("Some long-running operation returned {}", () -> expensiveOperation()); -```` \ No newline at end of file +```` + +When looking at old [Solr survey](solr.md) and looking for its `getCurrentLoggers` method, +found it is [no longer the case in log4j 2](https://stackoverflow.com/a/18653927) + + ## V2 architecture + +http://logging.apache.org/log4j/2.x/manual/architecture.html + +Tl;DR configuration has the parent child relationship and updates on parent will be used in children + +> In Log4j 1.x the Logger Hierarchy was maintained through a relationship between Loggers. In Log4j 2 this relationship no longer exists. +Instead, the hierarchy is maintained in the relationship between LoggerConfig objects + +Logger names are case-sensitive and they follow the hierarchical naming rule + +> A LoggerConfig is said to be an ancestor of another LoggerConfig if its name followed by a dot is a prefix of the descendant logger name. +A LoggerConfig is said to be a parent of a child LoggerConfig if there are no ancestors between itself and the descendant LoggerConfig + +- logger has name, and the best known strategy so far is using fully qualified class names (fqcn ...) +- logger config can reference its parent + +Filter + +- called before appender +- three results + - accept + - deny + - neutral, pass to other filter + +Appender + +- allow log to multiple location, console, file, remote socket .... + +> An Appender can be added to a Logger by calling the addLoggerAppender method of the current Configuration. +If a LoggerConfig matching the name of the Logger does not exist, one will be created, +the Appender will be attached to it and then all Loggers will be notified to update their LoggerConfig references + +- **all Loggers will be notified to update their LoggerConfig references** + +> Each enabled logging request for a given logger will be forwarded to all the appenders in that Logger's LoggerConfig as well as the Appenders of the LoggerConfig's parents. +In other words, Appenders are inherited additively from the LoggerConfig hierarchy. \ No newline at end of file From 57dfc50bc6a984b809ce75b89af3880b58d4bef8 Mon Sep 17 00:00:00 2001 From: at15 Date: Sat, 27 Apr 2019 10:57:56 -0700 Subject: [PATCH 04/15] [doc][log] Finish design simplify registry #110 --- .../design/2019-04-14-simplify-registry.md | 75 ++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/log/doc/design/2019-04-14-simplify-registry.md b/log/doc/design/2019-04-14-simplify-registry.md index a696513..67811d1 100644 --- a/log/doc/design/2019-04-14-simplify-registry.md +++ b/log/doc/design/2019-04-14-simplify-registry.md @@ -2,6 +2,8 @@ ## Issues +[All issues in 0.0.11](https://github.com/dyweb/gommon/issues?q=is%3Aopen+is%3Aissue+project%3Adyweb%2Fgommon%2F2+milestone%3A0.0.11) + - [#110](https://github.com/dyweb/gommon/issues/110) Simplify logger registry ## Background @@ -19,7 +21,74 @@ The main problem for logger is fields is still a bit hard to use because unlike method to add fields in place and requires using `Copy` to create a copy, it most other loggers they make a copy directly when adding fields (some like zap and zerolog serialize the fields when adding them) -Go logging library rarely keep a tree hierarchy, it is not the case in Java. -In [Solr](../survey/solr.md) where log4j1 is used, relationship is kept inside loggers, +Go logging library rarely keep a tree hierarchy, normally people use a global logger, i.e. `log`, `logrus`, `glog` +It is not the case in Java, in [Solr](../survey/solr.md) where log4j1 is used, relationship is kept inside loggers, in [log4j 2](../survey/log4j.md) they introduced logger config with hierarchy, it is determined by logger name and dot. -(TODO: is this hierarchy applied every time when log or only once on configuration change) \ No newline at end of file +(TODO: is this hierarchy applied every time when log or only once on configuration change) + +A main problem of having hierarchy is config becomes harder, things like using debug level becomes changing level to debug +for all loggers based on registry, for those didn't register in registry, they will never get updated +([#97](https://github.com/dyweb/gommon/issues/97) register to global logger registry by default) + +## Design + +Why we need to keep track of logger + +- allow individual logger to have and change to different behaviors (i.e. handler) at runtime, otherwise we can just point to a global config in each logger or simply have a global logger +- when update logger config, it should be propagate to all loggers + - logger config is read heavy, so we pay cost when update, there is very few config update, but config read happens every time when log + +What kind of logger should get tracked + +- long lived (span the entire application lifecycle), i.e. package level logger +- very few instances, i.e. singleton/limited struct loggers +- has value in customization (i.e. change level) + +What kind of logger should NOT get tracked + +- created as copy for adding context for current func, i.e. a method logger from struct logger, `st.logger.WithMethod()` +- too many of them, will blow up registry (unless deregister is provided, but this requires remove one element from slice) + +Registry + +- only one type of registry, package registry, no more application & library, main package is application, +your meta package (import and alias all your subpackages) is library (saw in controller runtime [alias.go](https://github.com/kubernetes-sigs/controller-runtime/blob/master/alias.go)) + +## Implementation + +````go +// registry +var globalRegistryGroup + +type registryGroup { + mu sync.Mutex + registris map[string]*Registry // key is package name +} + +func NewRegistry() { + reg := Registry{id: caller(), default: xxx, loggers:[]{xxx}} + globalRegistryGroup.add(id.Pkg, reg) + return reg +} + +func WalkRegistry(fn func(r *Registry)) { + // can first sort the keys to have a consistent walk order + // ... +} + +func WalkLogger() { + // walk into logger of individual registry +} + +// usage +// in pkg.go +var logReg = dlog.NewRegistry() +var log = logReg.Default() + +// in struct impl +func NewController() *Controller{ + c := &Controller{} + c.logger = logReg.NewLogger(?id) + return c +} +```` \ No newline at end of file From a58af960b57d75203f6a57765bf28eb31010d110 Mon Sep 17 00:00:00 2001 From: at15 Date: Sat, 27 Apr 2019 11:46:15 -0700 Subject: [PATCH 05/15] [log] Set default log level using env Fix #60 - previously the default is hard coded to info level, which is fine until you start adding debug level log in `init` func, normally when you change log level use command flag, the `SetLevel` is called in main package, at that time, all the `init` has being called - by using env var, the gommon/log package itself's `init` can check environment variable directly, since all the packages that use gommon log have their `init` called **after** gommon's, even if they use debug in `init` they can still see the message if they set the correct level in environment variable - it is not tested, might need to add e2e test --- log/level.go | 50 ++++++++++++++++++++++++++++++++++++++++++- log/logger_factory.go | 4 +++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/log/level.go b/log/level.go index 2de9d86..e904414 100644 --- a/log/level.go +++ b/log/level.go @@ -1,6 +1,17 @@ package log -import "github.com/dyweb/gommon/util/color" +import ( + "fmt" + "os" + "strings" + + "github.com/dyweb/gommon/util/color" +) + +// LevelEnvKey is the environment variable name for setting default log level +const LevelEnvKey = "GOMMON_LOG_LEVEL" + +var defaultLevel = InfoLevel // Level is log level // TODO: allow change default logging level at compile time @@ -29,6 +40,29 @@ const ( // PrintLevel is for library/application that requires a Printf based logger interface const PrintLevel = InfoLevel +// ParseLevel converts a level string to log level, it is case insensitive. +// For invalid input, it returns InfoLevel and a non nil error +func ParseLevel(s string) (Level, error) { + switch strings.ToLower(s) { + case "fatal": + return FatalLevel, nil + case "panic": + return PanicLevel, nil + case "error": + return ErrorLevel, nil + case "warn": + return WarnLevel, nil + case "info": + return InfoLevel, nil + case "debug": + return DebugLevel, nil + case "trace": + return TraceLevel, nil + default: + return InfoLevel, fmt.Errorf("not a valid gommon log level %s", s) + } +} + var levelColoredStrings = []string{ FatalLevel: color.RedStart + "fatal" + color.End, PanicLevel: color.RedStart + "panic" + color.End, @@ -104,3 +138,17 @@ func (level Level) ColoredString() string { func (level Level) ColoredAlignedUpperString() string { return levelColoredAlignedUpperStrings[level] } + +func init() { + // Set default log level based on env var so debug log in `init` would show + // https://github.com/dyweb/gommon/issues/60 + envLevel := os.Getenv(LevelEnvKey) + if envLevel == "" { + return + } + lvl, err := ParseLevel(envLevel) + if err != nil { + return + } + defaultLevel = lvl +} diff --git a/log/logger_factory.go b/log/logger_factory.go index 2eba1a0..34cfbc9 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -2,6 +2,8 @@ package log // logger_factory.go creates logger without register them to registry +// TODO: they should all become private ... + func NewPackageLogger() *Logger { return NewPackageLoggerWithSkip(1) } @@ -64,7 +66,7 @@ func newLogger(parent *Logger, child *Logger) *Logger { } } else { child.h = DefaultHandler() - child.level = InfoLevel + child.level = defaultLevel } return child } From c4b326291c57005b23336004ad271c241f5581fb Mon Sep 17 00:00:00 2001 From: at15 Date: Sat, 27 Apr 2019 12:54:01 -0700 Subject: [PATCH 06/15] [log] Use flat registry and registry group #110 - previously log registry is like file system, a registry (folder) can have loggers (file) and other registries (sub folders), maintaining this hierarchy requires extra work and package in go is not that like java where a child folder is a subpackage of parent (most time in practice it is, but it is not in language spec, every package is just package of its own) - also there is no implicit global registry, making updating log config hard (unless everyone add their registry into a specific place, someone's registry is going to be missed, which happended a lot) #97 - `logutil` is no longer needed, this requires update to all the library and application using gommon, since most of them are very outdated, this is not a big deal (have to update them anyway ...) - not tested ... even the gommon cli could break at this point --- cmd/gommon/main.go | 11 +- generator/pkg.go | 4 +- log/handlers/cli/example/main.go | 9 +- log/handlers/json/example/main.go | 7 +- log/logger_factory.go | 19 +-- log/logger_identity_test.go | 21 +-- log/logger_test.go | 8 +- log/pkg.go | 24 +++ log/registry.go | 242 +++++++++++------------------- log/registry_test.go | 13 +- noodle/pkg.go | 5 +- util/fsutil/pkg.go | 5 +- util/logutil/pkg.go | 22 --- util/logutil/pkg_test.go | 9 -- 14 files changed, 138 insertions(+), 261 deletions(-) delete mode 100644 util/logutil/pkg.go delete mode 100644 util/logutil/pkg_test.go diff --git a/cmd/gommon/main.go b/cmd/gommon/main.go index 7b14b9d..0a97732 100644 --- a/cmd/gommon/main.go +++ b/cmd/gommon/main.go @@ -18,10 +18,10 @@ import ( "github.com/dyweb/gommon/log/handlers/cli" "github.com/dyweb/gommon/noodle" "github.com/dyweb/gommon/util/fsutil" - "github.com/dyweb/gommon/util/logutil" ) -var log, logReg = dlog.NewApplicationLoggerAndRegistry("gommon") +var logReg = dlog.NewRegistry() +var log = logReg.Logger() var verbose = false var ( @@ -40,8 +40,8 @@ func main() { Long: "Generate go files for gommon", PersistentPreRun: func(cmd *cobra.Command, args []string) { if verbose { - dlog.SetLevel(logReg, dlog.DebugLevel) - dlog.EnableSource(logReg) + dlog.SetLevel(dlog.DebugLevel) + dlog.EnableSource() } }, Run: func(cmd *cobra.Command, args []string) { @@ -195,6 +195,5 @@ func addBuildIgnoreCmd() *cobra.Command { } func init() { - logReg.AddRegistry(logutil.Registry()) - dlog.SetHandler(logReg, cli.New(os.Stderr, true)) + dlog.SetHandler(cli.New(os.Stderr, true)) } diff --git a/generator/pkg.go b/generator/pkg.go index ee0f07e..fff77bc 100644 --- a/generator/pkg.go +++ b/generator/pkg.go @@ -4,7 +4,6 @@ package generator // import "github.com/dyweb/gommon/generator" import ( dlog "github.com/dyweb/gommon/log" "github.com/dyweb/gommon/noodle" - "github.com/dyweb/gommon/util/logutil" ) const ( @@ -15,7 +14,8 @@ const ( DefaultGeneratedFile = "gommon_generated.go" ) -var log, logReg = logutil.NewPackageLoggerAndRegistry() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() type ConfigFile struct { // Loggers is helper methods on struct for gommon/log to build a tree for logger, this is subject to change diff --git a/log/handlers/cli/example/main.go b/log/handlers/cli/example/main.go index 847b74d..38e6456 100644 --- a/log/handlers/cli/example/main.go +++ b/log/handlers/cli/example/main.go @@ -8,14 +8,15 @@ import ( "github.com/dyweb/gommon/log/handlers/cli" ) -var log, logReg = dlog.NewApplicationLoggerAndRegistry("example") +var logReg = dlog.NewRegistry() +var log = logReg.Logger() func main() { - dlog.SetHandler(logReg, cli.New(os.Stderr, true)) + dlog.SetHandler(cli.New(os.Stderr, true)) if len(os.Args) > 1 { if os.Args[1] == "nocolor" || os.Args[1] == "no" { - dlog.SetHandler(logReg, cli.NewNoColor(os.Stderr)) + dlog.SetHandler(cli.NewNoColor(os.Stderr)) } } @@ -34,7 +35,7 @@ func main() { }() log.Panic("I just want to panic") }() - dlog.SetLevel(logReg, dlog.DebugLevel) + dlog.SetLevel(dlog.DebugLevel) log.Debug("I will sleep for a while") time.Sleep(500 * time.Millisecond) log.Fatal("I am red") diff --git a/log/handlers/json/example/main.go b/log/handlers/json/example/main.go index cfd1453..750cea1 100644 --- a/log/handlers/json/example/main.go +++ b/log/handlers/json/example/main.go @@ -8,10 +8,11 @@ import ( "github.com/dyweb/gommon/log/handlers/json" ) -var log, logReg = dlog.NewApplicationLoggerAndRegistry("example") +var logReg = dlog.NewRegistry() +var log = logReg.Logger() func main() { - dlog.SetHandler(logReg, json.New(os.Stderr)) + dlog.SetHandler(json.New(os.Stderr)) log.Info("hi") log.Infof("open file %s", "foo.yml") log.InfoF("open", @@ -27,7 +28,7 @@ func main() { }() log.Panic("I just want to panic") }() - dlog.SetLevel(logReg, dlog.DebugLevel) + dlog.SetLevel(dlog.DebugLevel) log.Debug("I will sleep for a while") time.Sleep(500 * time.Millisecond) log.Fatal("I am red") diff --git a/log/logger_factory.go b/log/logger_factory.go index 34cfbc9..e23192e 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -16,15 +16,6 @@ func NewPackageLoggerWithSkip(skip int) *Logger { return newLogger(nil, l) } -// Deprecated: use Copy method on package logger -func NewFunctionLogger(packageLogger *Logger) *Logger { - id := NewIdentityFromCaller(1) - l := &Logger{ - id: &id, - } - return newLogger(packageLogger, l) -} - func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { id := loggable.LoggerIdentity(func() Identity { return NewIdentityFromCaller(1) @@ -37,15 +28,6 @@ func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { return l } -// Deprecated: use Copy method on struct logger -func NewMethodLogger(structLogger *Logger) *Logger { - id := NewIdentityFromCaller(1) - l := &Logger{ - id: &id, - } - return newLogger(structLogger, l) -} - // NewTestLogger does not have identity and handler, it is mainly used for benchmark test func NewTestLogger(level Level) *Logger { l := &Logger{ @@ -54,6 +36,7 @@ func NewTestLogger(level Level) *Logger { return l } +// TODO: change this func signature ... func newLogger(parent *Logger, child *Logger) *Logger { if parent != nil { child.h = parent.h diff --git a/log/logger_identity_test.go b/log/logger_identity_test.go index 8ece563..8e3ba68 100644 --- a/log/logger_identity_test.go +++ b/log/logger_identity_test.go @@ -10,11 +10,6 @@ import ( var lg = NewPackageLogger() -func foo() *Logger { - funcLog := NewFunctionLogger(lg) - return funcLog -} - func fooUseCopy() *Logger { return lg.Copy() } @@ -35,11 +30,6 @@ func (f *Foo) LoggerIdentity(justCallMe func() Identity) Identity { return justCallMe() } -func (f *Foo) method() *Logger { - mlog := NewMethodLogger(f.log) - return mlog -} - func (f *Foo) methodUseCopy() *Logger { return f.log.Copy() } @@ -61,7 +51,6 @@ func TestNewPackageLogger(t *testing.T) { } func TestNewFunctionLogger(t *testing.T) { - assert.Equal(t, FunctionLogger, foo().id.Type) assert.Equal(t, FunctionLogger, fooUseCopy().id.Type) } @@ -72,21 +61,13 @@ func TestNewStructLogger(t *testing.T) { assert.Equal(t, "struct", id.Type.String()) assert.Equal(t, "Foo", id.Struct) assert.Equal(t, MagicStructLoggerFunctionName, id.Function) - assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:35", + assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:30", fmt.Sprintf("%s:%d", id.File, id.Line)) } func TestNewMethodLogger(t *testing.T) { slog := NewStructLogger(lg, dummyFoo) dummyFoo.log = slog - mlog := dummyFoo.method() - id := mlog.id - assert.Equal(t, MethodLogger, id.Type) - assert.Equal(t, "method", id.Type.String()) - assert.Equal(t, "Foo", id.Struct) - assert.Equal(t, "method", id.Function) - assert.Equal(t, testutil.GOPATH()+"/src/github.com/dyweb/gommon/log/logger_identity_test.go:39", - fmt.Sprintf("%s:%d", id.File, id.Line)) assert.Equal(t, MethodLogger, dummyFoo.methodUseCopy().id.Type) diff --git a/log/logger_test.go b/log/logger_test.go index db4ba03..b59290d 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -6,24 +6,20 @@ import ( asst "github.com/stretchr/testify/assert" ) -// TODO: test NewIOHandler - -// TODO: might use setup and tear down (newer version of go has this built in?) func TestLogger_SetHandler(t *testing.T) { assert := asst.New(t) - logger := NewFunctionLogger(nil) + logger := NewTestLogger(defaultLevel) th := NewTestHandler() logger.SetHandler(th) logger.Info("hello logger") assert.True(th.HasLog(InfoLevel, "hello logger")) - // TODO: test time ... } func TestLogger_SetLevel(t *testing.T) { assert := asst.New(t) - logger := NewFunctionLogger(nil) + logger := NewTestLogger(defaultLevel) th := NewTestHandler() logger.SetHandler(th) logger.SetLevel(DebugLevel) diff --git a/log/pkg.go b/log/pkg.go index a607ced..8db8f9d 100644 --- a/log/pkg.go +++ b/log/pkg.go @@ -9,3 +9,27 @@ type LoggableStruct interface { SetLogger(logger *Logger) LoggerIdentity(justCallMe func() Identity) Identity } + +func SetLevel(level Level) { + WalkLogger(func(l *Logger) { + l.SetLevel(level) + }) +} + +func SetHandler(handler Handler) { + WalkLogger(func(l *Logger) { + l.SetHandler(handler) + }) +} + +func EnableSource() { + WalkLogger(func(l *Logger) { + l.EnableSource() + }) +} + +func DisableSource() { + WalkLogger(func(l *Logger) { + l.DisableSource() + }) +} diff --git a/log/registry.go b/log/registry.go index 9b3619f..6e783d9 100644 --- a/log/registry.go +++ b/log/registry.go @@ -1,6 +1,7 @@ package log import ( + "sort" "sync" "github.com/dyweb/gommon/util/runtimeutil" @@ -9,107 +10,91 @@ import ( // registry.go is used for maintain relationship between loggers across packages and projects // it also contains util func for traverse registry and logger -// Registry contains child registry and loggers -type Registry struct { - mu sync.Mutex - children []*Registry - loggers []*Logger +var globalRegistryGroup = newRegistryGroup() - // immutable - identity RegistryIdentity +type registryGroup struct { + // we didn't use RWMutex because when walking a group, the main purpose is to modify loggers + // inside registries, so two walking should not happens in parallel + mu sync.Mutex + registries map[string]*Registry } -type RegistryType uint8 - -const ( - UnknownRegistry RegistryType = iota - ApplicationRegistry - LibraryRegistry - PackageRegistry -) - -func (r RegistryType) String() string { - switch r { - case UnknownRegistry: - return "unk" - case ApplicationRegistry: - return "app" - case LibraryRegistry: - return "lib" - case PackageRegistry: - return "pkg" - default: - return "unk" +func newRegistryGroup() *registryGroup { + return ®istryGroup{ + registries: make(map[string]*Registry), } } -type RegistryIdentity struct { - // Project is specified by user, i.e. for all the packages under gommon, they would have github.com/dyweb/gommon - Project string - // Package is detected base on runtime, i.e. github.com/dyweb/gommon/noodle - Package string - // Type is specified by user when creating registry - Type RegistryType - // File is where create registry is called - File string - // Line is where create registry is called - Line int -} +func (rg *registryGroup) add(reg *Registry) { + rg.mu.Lock() + defer rg.mu.Unlock() -func NewLibraryRegistry(project string) Registry { - return Registry{ - identity: newRegistryId(project, LibraryRegistry, 0), + id := reg.identity + if id == "" { + panic("log registry identity is empty") + } + oReg, ok := rg.registries[id] + if ok { + if oReg == reg { + return + } else { + panic("log registry is already registered for " + id) + } } + rg.registries[id] = reg } -// TODO: validate skip -func NewApplicationLoggerAndRegistry(project string) (*Logger, *Registry) { - reg := Registry{ - identity: newRegistryId(project, ApplicationRegistry, 1), - } - logger := NewPackageLoggerWithSkip(1) - reg.AddLogger(logger) - return logger, ® +// Registry contains default and tracked loggers, it is per package +type Registry struct { + mu sync.Mutex + loggers []*Logger + + // identity is a string for package + identity string } -func NewPackageLoggerAndRegistryWithSkip(project string, skip int) (*Logger, *Registry) { +// NewRegistry create a log registry with a default logger for a package. +// It registers itself in globalRegistryGroup so it can be updated later using WalkRegistries +func NewRegistry() *Registry { + frame := runtimeutil.GetCallerFrame(1) + pkg, _ := runtimeutil.SplitPackageFunc(frame.Function) reg := Registry{ - identity: newRegistryId(project, PackageRegistry, skip+1), + identity: pkg, + loggers: []*Logger{NewPackageLoggerWithSkip(1)}, } - logger := NewPackageLoggerWithSkip(skip + 1) - reg.AddLogger(logger) - return logger, ® + globalRegistryGroup.add(®) + return ® } -func newRegistryId(proj string, tpe RegistryType, skip int) RegistryIdentity { - // TODO: check if the skip works .... we need another package for testing that - frame := runtimeutil.GetCallerFrame(skip + 1) - pkg, _ := runtimeutil.SplitPackageFunc(frame.Function) - return RegistryIdentity{ - Project: proj, - Package: pkg, - Type: tpe, - File: frame.File, - Line: frame.Line, - } +func (r *Registry) Identity() string { + return r.identity } -// AddRegistry is for adding a package level log registry to a library/application level log registry -// It skips add if child registry already there -func (r *Registry) AddRegistry(child *Registry) { - r.mu.Lock() - defer r.mu.Unlock() - for _, c := range r.children { - if c == child { - return - } +// Logger returns the default logger in registry +func (r *Registry) Logger() *Logger { + if len(r.loggers) < 1 { + panic("no default logger found in registry") + } + return r.loggers[0] +} + +// NewLogger creates a logger based on default logger and register it in registry. +// It should be used sparingly, if you need to add more fields as context for a func, +// you should make copy from default logger using methods like TODO: WithFields? +// to avoid register them in registry +func (r *Registry) NewLogger() *Logger { + // no lock is added because addLogger also acquire lock + id := NewIdentityFromCaller(1) + l := Logger{ + id: &id, } - r.children = append(r.children, child) + newLogger(r.Logger(), &l) + r.addLogger(&l) + return &l } -// AddLogger is used for registering a logger to package level log registry -// It skips add if the logger is already there -func (r *Registry) AddLogger(l *Logger) { +// addLogger registers a logger into registry. It's a nop if the logger is already there +func (r *Registry) addLogger(l *Logger) { r.mu.Lock() defer r.mu.Unlock() for _, ol := range r.loggers { @@ -120,87 +105,32 @@ func (r *Registry) AddLogger(l *Logger) { r.loggers = append(r.loggers, l) } -func (r *Registry) Identity() RegistryIdentity { - return r.identity -} - -func SetLevel(root *Registry, level Level) { - WalkLogger(root, func(l *Logger) { - l.SetLevel(level) - }) -} - -func SetHandler(root *Registry, handler Handler) { - WalkLogger(root, func(l *Logger) { - l.SetHandler(handler) - }) -} - -func EnableSource(root *Registry) { - WalkLogger(root, func(l *Logger) { - l.EnableSource() - }) -} - -func DisableSource(root *Registry) { - WalkLogger(root, func(l *Logger) { - l.DisableSource() - }) -} +// WalkRegistry walks registry in globalRegistryGroup in sorted order of id (package path) +func WalkRegistry(cb func(r *Registry)) { + group := globalRegistryGroup + group.mu.Lock() + defer group.mu.Unlock() -// WalkLogger is PreOrderDfs -func WalkLogger(root *Registry, cb func(l *Logger)) { - walkLogger(root, nil, nil, cb) -} - -// walkLogger loops loggers in current registry first, then visit its children in DFS -// It will create the map if it is nil, so caller don't need to do the bootstrap, -// However, caller can provide a map so they can use it to get all the loggers and registry -func walkLogger(root *Registry, loggers map[*Logger]bool, registries map[*Registry]bool, cb func(l *Logger)) { - // first call - if loggers == nil { - loggers = make(map[*Logger]bool) - } - if registries == nil { - registries = make(map[*Registry]bool) + // visit in the order of sorted id + var ids []string + for id := range group.registries { + ids = append(ids, id) } - // pre order - registries[root] = true - for _, l := range root.loggers { - // avoid dup - if loggers[l] { - continue - } - loggers[l] = true - cb(l) // visit - } - // dfs - for _, r := range root.children { - // avoid cycle - if registries[r] { - continue - } - walkLogger(r, loggers, registries, cb) + sort.Strings(ids) + + for _, id := range ids { + cb(group.registries[id]) } } -func WalkRegistry(root *Registry, cb func(r *Registry)) { - walkRegistry(root, nil, cb) -} +// WalkLogger calls WalkRegistry and within each registry, walk in insert order of loggers +func WalkLogger(cb func(l *Logger)) { + WalkRegistry(func(r *Registry) { + r.mu.Lock() + defer r.mu.Unlock() -func walkRegistry(root *Registry, registries map[*Registry]bool, cb func(r *Registry)) { - // first call - if registries == nil { - registries = make(map[*Registry]bool) - } - registries[root] = true - cb(root) // visit - // dfs - for _, r := range root.children { - // avoid dup - if registries[r] { - continue + for _, l := range r.loggers { + cb(l) } - walkRegistry(r, registries, cb) - } + }) } diff --git a/log/registry_test.go b/log/registry_test.go index 2b77ee7..cc847d3 100644 --- a/log/registry_test.go +++ b/log/registry_test.go @@ -6,20 +6,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRegistry_AddRegistry(t *testing.T) { - rTop := Registry{} - r1 := Registry{} - rTop.AddRegistry(&r1) - assert.Equal(t, 1, len(rTop.children)) - rTop.AddRegistry(&r1) - assert.Equal(t, 1, len(rTop.children), "don't add registry if it already exists") -} - func TestRegistry_AddLogger(t *testing.T) { rTop := Registry{} l1 := NewTestLogger(InfoLevel) - rTop.AddLogger(l1) + rTop.addLogger(l1) assert.Equal(t, 1, len(rTop.loggers)) - rTop.AddLogger(l1) + rTop.addLogger(l1) assert.Equal(t, 1, len(rTop.loggers), "don't add logger if it already exists") } diff --git a/noodle/pkg.go b/noodle/pkg.go index f56977a..b6d99b6 100644 --- a/noodle/pkg.go +++ b/noodle/pkg.go @@ -4,7 +4,7 @@ package noodle import ( "net/http" - "github.com/dyweb/gommon/util/logutil" + dlog "github.com/dyweb/gommon/log" ) const ( @@ -12,7 +12,8 @@ const ( DefaultName = "Bowel" ) -var log, _ = logutil.NewPackageLoggerAndRegistry() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() // Bowel is the container for different types of noodles type Bowel interface { diff --git a/util/fsutil/pkg.go b/util/fsutil/pkg.go index fc2f7fb..60cc250 100644 --- a/util/fsutil/pkg.go +++ b/util/fsutil/pkg.go @@ -2,7 +2,8 @@ package fsutil // import "github.com/dyweb/gommon/util/fsutil" import ( - "github.com/dyweb/gommon/util/logutil" + dlog "github.com/dyweb/gommon/log" ) -var log, _ = logutil.NewPackageLoggerAndRegistry() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() diff --git a/util/logutil/pkg.go b/util/logutil/pkg.go deleted file mode 100644 index 72f01bc..0000000 --- a/util/logutil/pkg.go +++ /dev/null @@ -1,22 +0,0 @@ -// Package logutil is a registry of loggers, it is required for all lib and app that use gommon/log. -// You should add the registry as child of your library/application's child if you want to control gommon libraries -// logging behavior -package logutil - -import ( - "github.com/dyweb/gommon/log" -) - -const Project = "github.com/dyweb/gommon" - -var registry = log.NewLibraryRegistry(Project) - -func Registry() *log.Registry { - return ®istry -} - -func NewPackageLoggerAndRegistry() (*log.Logger, *log.Registry) { - logger, child := log.NewPackageLoggerAndRegistryWithSkip(Project, 1) - registry.AddRegistry(child) - return logger, child -} diff --git a/util/logutil/pkg_test.go b/util/logutil/pkg_test.go deleted file mode 100644 index 212a405..0000000 --- a/util/logutil/pkg_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package logutil - -import "testing" - -func TestNewPackageLoggerAndRegistry(t *testing.T) { - l, reg := NewPackageLoggerAndRegistry() - t.Log(l.Identity()) - t.Log(reg.Identity()) -} From 99b6829361da1910ec8c117d77b185d274aa0b0e Mon Sep 17 00:00:00 2001 From: at15 Date: Sat, 27 Apr 2019 13:14:06 -0700 Subject: [PATCH 07/15] [hack] Move go-dev docker image from build -> hack --- {scripts => hack}/docker-compose.yml | 0 {build => hack}/go-dev/Dockerfile | 0 {build => hack}/go-dev/Makefile | 0 {build => hack}/go-dev/README.md | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {scripts => hack}/docker-compose.yml (100%) rename {build => hack}/go-dev/Dockerfile (100%) rename {build => hack}/go-dev/Makefile (100%) rename {build => hack}/go-dev/README.md (100%) diff --git a/scripts/docker-compose.yml b/hack/docker-compose.yml similarity index 100% rename from scripts/docker-compose.yml rename to hack/docker-compose.yml diff --git a/build/go-dev/Dockerfile b/hack/go-dev/Dockerfile similarity index 100% rename from build/go-dev/Dockerfile rename to hack/go-dev/Dockerfile diff --git a/build/go-dev/Makefile b/hack/go-dev/Makefile similarity index 100% rename from build/go-dev/Makefile rename to hack/go-dev/Makefile diff --git a/build/go-dev/README.md b/hack/go-dev/README.md similarity index 100% rename from build/go-dev/README.md rename to hack/go-dev/README.md From 808729ccc81de4e1a91ad1051dbca31c8bf96beb Mon Sep 17 00:00:00 2001 From: at15 Date: Sat, 27 Apr 2019 13:21:16 -0700 Subject: [PATCH 08/15] [hack] Add go version 1.11.9 & 1.12.4 for go-dev --- hack/go-dev/Dockerfile | 2 +- hack/go-dev/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/go-dev/Dockerfile b/hack/go-dev/Dockerfile index 3cfaaa4..f673f72 100644 --- a/hack/go-dev/Dockerfile +++ b/hack/go-dev/Dockerfile @@ -38,7 +38,7 @@ ARG BUILD_GO_VERSION=1.11.2 # glide no longer have release, just hard code it to latest version ENV GO_VERSION=$BUILD_GO_VERSION \ - GLIDE_VERSION=v0.13.1 + GLIDE_VERSION=v0.13.2 # TODO: might put glide under GOPATH/bin RUN \ diff --git a/hack/go-dev/Makefile b/hack/go-dev/Makefile index 0bf0785..5a1740c 100644 --- a/hack/go-dev/Makefile +++ b/hack/go-dev/Makefile @@ -1,5 +1,5 @@ DOCKER_REPO = dyweb/go-dev -GO_VERSIONS = 1.10.7 1.11.4 +GO_VERSIONS = 1.11.9 1.12.4 BUILDS = $(addprefix build-, $(GO_VERSIONS)) PUSHS = $(addprefix push-, $(GO_VERSIONS)) From 42eb65b848876ff645477f3c859e72480c4f968d Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 11:02:42 -0700 Subject: [PATCH 09/15] [log] Make CopyFields unexported - it is only used for testing in TestHandler --- log/doc.go | 3 +- log/field.go | 22 +++++++------- log/field_test.go | 2 +- log/handler.go | 1 + log/handlers/cli/README.md | 10 +++---- log/handlers/json/README.md | 7 +++-- log/logger_factory.go | 1 + log/logger_tree.go | 60 ------------------------------------- log/testing.go | 2 +- 9 files changed, 26 insertions(+), 82 deletions(-) delete mode 100644 log/logger_tree.go diff --git a/log/doc.go b/log/doc.go index de99984..357d8d3 100644 --- a/log/doc.go +++ b/log/doc.go @@ -1,5 +1,4 @@ -// Package log provides structured logging with fine grained control -// over libraries using a tree of logger registry +// Package log provides structured logging with fine grained control over packages using logger registry // // TODO: add convention and usage package log diff --git a/log/field.go b/log/field.go index d7ac5ad..361070e 100644 --- a/log/field.go +++ b/log/field.go @@ -5,7 +5,7 @@ import ( ) // FieldType avoids doing type assertion or calling reflection -// TODO: difference between the two methods above +// TODO: difference between type assertion and reflection type FieldType uint8 const ( @@ -17,17 +17,9 @@ const ( // Fields is a slice of Field type Fields []Field -// CopyFields make a copy of the slice so modifying one won't have effect on another, -func CopyFields(fields Fields) Fields { - copied := make([]Field, len(fields)) - for i := 0; i < len(fields); i++ { - copied[i] = fields[i] - } - return copied -} - // Field is based on uber-go/zap https://github.com/uber-go/zap/blob/master/zapcore/field.go // It can be treated as a Union, the value is stored in either Int, Str or Interface +// TODO: interface{} is actually not used ... type Field struct { Key string Type FieldType @@ -69,3 +61,13 @@ func Stringer(k string, v fmt.Stringer) Field { Str: v.String(), } } + +// copyFields make a copy of the slice so modifying one won't have effect on another, +// It is only used in test +func copyFields(fields Fields) Fields { + copied := make([]Field, len(fields)) + for i := 0; i < len(fields); i++ { + copied[i] = fields[i] + } + return copied +} diff --git a/log/field_test.go b/log/field_test.go index 9397211..3ddd522 100644 --- a/log/field_test.go +++ b/log/field_test.go @@ -11,7 +11,7 @@ func TestCopyFields(t *testing.T) { Str("k1", "v1"), Int("k2", 2), } - copied := CopyFields(fields) + copied := copyFields(fields) assert.Equal(t, 2, cap(copied), "capacity is same as length") fields[0].Key = "k1modified" copied[0].Str = "v1modified" diff --git a/log/handler.go b/log/handler.go index 3088d5f..23f5a79 100644 --- a/log/handler.go +++ b/log/handler.go @@ -51,6 +51,7 @@ type Syncer interface { var defaultHandler = NewTextHandler(os.Stderr) // DefaultHandler returns the singleton defaultHandler instance, which logs to stderr in text format +// TODO: should allow customize default handler like default level, i.e. use json as default handler func DefaultHandler() Handler { return defaultHandler } diff --git a/log/handlers/cli/README.md b/log/handlers/cli/README.md index 842e7d4..4069400 100644 --- a/log/handlers/cli/README.md +++ b/log/handlers/cli/README.md @@ -19,14 +19,15 @@ import ( "github.com/dyweb/gommon/log/handlers/cli" ) -var log, logReg = dlog.NewApplicationLoggerAndRegistry("example") +var logReg = dlog.NewRegistry() +var log = logReg.Logger() func main() { - dlog.SetHandler(logReg, cli.New(os.Stderr, true)) + dlog.SetHandler(cli.New(os.Stderr, true)) if len(os.Args) > 1 { if os.Args[1] == "nocolor" || os.Args[1] == "no" { - dlog.SetHandler(logReg, cli.NewNoColor(os.Stderr)) + dlog.SetHandler(cli.NewNoColor(os.Stderr)) } } @@ -45,12 +46,11 @@ func main() { }() log.Panic("I just want to panic") }() - dlog.SetLevel(logReg, dlog.DebugLevel) + dlog.SetLevel(dlog.DebugLevel) log.Debug("I will sleep for a while") time.Sleep(500 * time.Millisecond) log.Fatal("I am red") } - ```` Result in diff --git a/log/handlers/json/README.md b/log/handlers/json/README.md index 7bff336..3c1c8ba 100644 --- a/log/handlers/json/README.md +++ b/log/handlers/json/README.md @@ -23,10 +23,11 @@ import ( "github.com/dyweb/gommon/log/handlers/json" ) -var log, logReg = dlog.NewApplicationLoggerAndRegistry("example") +var logReg = dlog.NewRegistry() +var log = logReg.Logger() func main() { - dlog.SetHandler(logReg, json.New(os.Stderr)) + dlog.SetHandler(json.New(os.Stderr)) log.Info("hi") log.Infof("open file %s", "foo.yml") log.InfoF("open", @@ -42,7 +43,7 @@ func main() { }() log.Panic("I just want to panic") }() - dlog.SetLevel(logReg, dlog.DebugLevel) + dlog.SetLevel(dlog.DebugLevel) log.Debug("I will sleep for a while") time.Sleep(500 * time.Millisecond) log.Fatal("I am red") diff --git a/log/logger_factory.go b/log/logger_factory.go index e23192e..197d167 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -48,6 +48,7 @@ func newLogger(parent *Logger, child *Logger) *Logger { child.fields = fields } } else { + // TODO: allow customize DefaultHandler child.h = DefaultHandler() child.level = defaultLevel } diff --git a/log/logger_tree.go b/log/logger_tree.go deleted file mode 100644 index 423e8ef..0000000 --- a/log/logger_tree.go +++ /dev/null @@ -1,60 +0,0 @@ -// +build ignore - -// TODO: enable this file after refactor on tree of logger is finished -package log - -import ( - "io" - "os" - - "github.com/dyweb/gommon/structure" -) - -func ToStringTree(root *Logger) *structure.StringTreeNode { - visited := make(map[*Logger]bool) - return toStringTreeHelper(root, visited) -} - -func toStringTreeHelper(root *Logger, visited map[*Logger]bool) *structure.StringTreeNode { - if visited[root] { - return nil - } - // TODO: might add logger level as well - n := &structure.StringTreeNode{Val: root.id.String()} - visited[root] = true - for _, group := range root.children { - for _, l := range group { - p := toStringTreeHelper(l, visited) - if p != nil { - n.Append(*p) - } - } - } - return n -} - -// FIXME: it seem without return value and extra parameter, we can't clone a tree? -func toStringTree(root *Logger) *structure.StringTreeNode { - visited := make(map[*Logger]bool) - PreOrderDFS(root, visited, func(l *Logger) { - - }) - return nil -} - -func (l *Logger) PrintTree() { - l.PrintTreeTo(os.Stdout) -} - -// FIXME: print tree is still having problem .... -//⇒ icehubd log -// app logger /home/at15/workspace/src/github.com/at15/go.ice/_example/github/pkg/util/logutil/pkg.go:8 -// └── lib logger /home/at15/workspace/src/github.com/at15/go.ice/ice/util/logutil/pkg.go:8 -// └── lib logger /home/at15/workspace/src/github.com/dyweb/gommon/util/logutil/pkg.go:7 -// │ └── pkg logger /home/at15/workspace/src/github.com/dyweb/gommon/config/pkg.go:8 - -// PrintTreeTo prints logger as a tree, using current logger as root -func (l *Logger) PrintTreeTo(w io.Writer) { - st := ToStringTree(l) - st.PrintTo(w) -} diff --git a/log/testing.go b/log/testing.go index dfc7442..882f5c6 100644 --- a/log/testing.go +++ b/log/testing.go @@ -33,7 +33,7 @@ func NewTestHandler() *TestHandler { func (h *TestHandler) HandleLog(level Level, time time.Time, msg string, source Caller, context Fields, fields Fields) { h.mu.Lock() - h.entries = append(h.entries, entry{level: level, time: time, msg: msg, source: source, context: CopyFields(context), fields: CopyFields(fields)}) + h.entries = append(h.entries, entry{level: level, time: time, msg: msg, source: source, context: copyFields(context), fields: copyFields(fields)}) h.mu.Unlock() } From 5f569e9e16d60a3cebc2c2e1b60403cc63d4d718 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 11:24:42 -0700 Subject: [PATCH 10/15] [log] Remove outdated examples --- log/_examples/simple/.gitignore | 2 -- log/_examples/simple/main.go | 46 ------------------------- log/_examples/uselib/foo/foo.go | 5 --- log/_examples/uselib/main.go | 19 ---------- log/_examples/uselib/service/service.go | 29 ---------------- log/_examples/uselib/storage/mem/mem.go | 29 ---------------- log/_examples/uselib/storage/storage.go | 22 ------------ log/field.go | 5 +-- log/logger.go | 7 ++-- log/logger_factory.go | 34 ++++++++---------- log/logger_identity.go | 16 +++++---- log/registry.go | 13 +++---- 12 files changed, 33 insertions(+), 194 deletions(-) delete mode 100644 log/_examples/simple/.gitignore delete mode 100644 log/_examples/simple/main.go delete mode 100644 log/_examples/uselib/foo/foo.go delete mode 100644 log/_examples/uselib/main.go delete mode 100644 log/_examples/uselib/service/service.go delete mode 100644 log/_examples/uselib/storage/mem/mem.go delete mode 100644 log/_examples/uselib/storage/storage.go diff --git a/log/_examples/simple/.gitignore b/log/_examples/simple/.gitignore deleted file mode 100644 index bd8888a..0000000 --- a/log/_examples/simple/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# the binary generate when using go build w/ gcflags -simple \ No newline at end of file diff --git a/log/_examples/simple/main.go b/log/_examples/simple/main.go deleted file mode 100644 index bd0296b..0000000 --- a/log/_examples/simple/main.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "os" - "time" - - dlog "github.com/dyweb/gommon/log" - "github.com/dyweb/gommon/log/handlers/cli" - "github.com/dyweb/gommon/log/handlers/json" -) - -var log, logReg = dlog.NewApplicationLoggerAndRegistry("simple") - -// simply log to stderr -func main() { - if len(os.Args) > 1 { - if os.Args[1] == "json" { - dlog.SetHandler(logReg, json.New(os.Stderr)) - } - if os.Args[1] == "cli" { - dlog.SetHandler(logReg, cli.New(os.Stderr, false)) - } - if os.Args[1] == "cli-d" { - dlog.SetHandler(logReg, cli.New(os.Stderr, true)) - } - } - dlog.SetLevel(logReg, dlog.DebugLevel) - log.Debug("show me the meaning of being lonely") - log.Info("this is love!") - log.Print("print is info level") - log.Warnf("this is love %d", 2) - log.InfoF("this love", dlog.Int("num", 2), dlog.Str("foo", "bar")) - dlog.EnableSource(logReg) - // TODO: show skip caller using a util func - log.Info("show me the line") - log.Infof("show the line %d", 2) - log.InfoF("show the line", dlog.Int("num", 2), dlog.Str("foo", "bar")) - log.DisableSource() - log.WarnF("I will sleep", dlog.Int("duration", 1)) - time.Sleep(10 * time.Millisecond) - log.Info("no more line number") - - log.AddField(dlog.Str("f1", "v1")) - log.Info("should have some extra context") - // TODO: panic and fatal -} diff --git a/log/_examples/uselib/foo/foo.go b/log/_examples/uselib/foo/foo.go deleted file mode 100644 index c3a322b..0000000 --- a/log/_examples/uselib/foo/foo.go +++ /dev/null @@ -1,5 +0,0 @@ -package foo - -const ( - FOO = "foo" -) diff --git a/log/_examples/uselib/main.go b/log/_examples/uselib/main.go deleted file mode 100644 index a226a2b..0000000 --- a/log/_examples/uselib/main.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "fmt" - - // you can use package with _ prefix, but it is ignored by go build? - "github.com/dyweb/gommon/log/_examples/uselib/foo" - "github.com/dyweb/gommon/log/_examples/uselib/service" - "github.com/dyweb/gommon/log/_examples/uselib/storage" - - _ "github.com/dyweb/gommon/log/_examples/uselib/storage/mem" -) - -// TODO: make it a full example, like user auth with multiple mock backends -func main() { - fmt.Println(foo.FOO) - auth := service.NewAuth(storage.Get("mem")) - auth.Check("jack", "123") -} diff --git a/log/_examples/uselib/service/service.go b/log/_examples/uselib/service/service.go deleted file mode 100644 index aada9bb..0000000 --- a/log/_examples/uselib/service/service.go +++ /dev/null @@ -1,29 +0,0 @@ -package service - -import ( - "errors" - - "github.com/dyweb/gommon/log/_examples/uselib/storage" -) - -type Auth struct { - s storage.Driver -} - -func NewAuth(driver storage.Driver) *Auth { - return &Auth{ - s: driver, - } -} - -func (a *Auth) Check(user string, password string) error { - //log2.NewIdentityFromCallerOld(0) - p, err := a.s.Get(user) - if err != nil { - return err - } - if p == password { - return nil - } - return errors.New("invalid password") -} diff --git a/log/_examples/uselib/storage/mem/mem.go b/log/_examples/uselib/storage/mem/mem.go deleted file mode 100644 index b9a5eb1..0000000 --- a/log/_examples/uselib/storage/mem/mem.go +++ /dev/null @@ -1,29 +0,0 @@ -package mem - -import ( - "github.com/dyweb/gommon/log/_examples/uselib/storage" -) - -type Store struct { - d map[string]string -} - -func (s *Store) Get(k string) (string, error) { - return s.d[k], nil -} - -func (s *Store) Set(k string, v string) { - s.d[k] = v -} - -func NewMemStorage() *Store { - return &Store{ - d: make(map[string]string, 5), - } -} - -func init() { - storage.Register("mem", func() storage.Driver { - return NewMemStorage() - }) -} diff --git a/log/_examples/uselib/storage/storage.go b/log/_examples/uselib/storage/storage.go deleted file mode 100644 index ac32353..0000000 --- a/log/_examples/uselib/storage/storage.go +++ /dev/null @@ -1,22 +0,0 @@ -package storage - -var drivers map[string]DriverFactory - -type Driver interface { - Get(k string) (string, error) - Set(k string, v string) -} - -type DriverFactory func() Driver - -func Register(name string, factory DriverFactory) { - drivers[name] = factory -} - -func Get(name string) Driver { - return drivers[name]() -} - -func init() { - drivers = make(map[string]DriverFactory, 3) -} diff --git a/log/field.go b/log/field.go index 361070e..c4a4cd8 100644 --- a/log/field.go +++ b/log/field.go @@ -63,11 +63,8 @@ func Stringer(k string, v fmt.Stringer) Field { } // copyFields make a copy of the slice so modifying one won't have effect on another, -// It is only used in test func copyFields(fields Fields) Fields { copied := make([]Field, len(fields)) - for i := 0; i < len(fields); i++ { - copied[i] = fields[i] - } + copy(copied, fields) return copied } diff --git a/log/logger.go b/log/logger.go index da3f3dd..08596a8 100644 --- a/log/logger.go +++ b/log/logger.go @@ -40,11 +40,8 @@ type Logger struct { // Copy create a new logger with different identity, the identity is based on where Copy is called // Normally you should call Copy inside func or method on a package/strcut logger func (l *Logger) Copy() *Logger { - id := NewIdentityFromCaller(1) - c := &Logger{ - id: &id, - } - return newLogger(l, c) + id := newIdentityFromCaller(1) + return copyOrCreateLogger(l, &id) } // AddField add field to current logger in place, it does NOT create a copy of logger diff --git a/log/logger_factory.go b/log/logger_factory.go index 197d167..67b98fa 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -5,30 +5,24 @@ package log // TODO: they should all become private ... func NewPackageLogger() *Logger { - return NewPackageLoggerWithSkip(1) + return newPackageLoggerWithSkip(1) } -func NewPackageLoggerWithSkip(skip int) *Logger { - id := NewIdentityFromCaller(skip + 1) - l := &Logger{ - id: &id, - } - return newLogger(nil, l) +func newPackageLoggerWithSkip(skip int) *Logger { + id := newIdentityFromCaller(skip + 1) + return copyOrCreateLogger(nil, &id) } func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { id := loggable.LoggerIdentity(func() Identity { - return NewIdentityFromCaller(1) + return newIdentityFromCaller(1) }) - l := &Logger{ - id: &id, - } - l = newLogger(packageLogger, l) + l := copyOrCreateLogger(packageLogger, &id) loggable.SetLogger(l) return l } -// NewTestLogger does not have identity and handler, it is mainly used for benchmark test +// NewTestLogger does NOT have identity nor handler, it is mainly used for benchmark func NewTestLogger(level Level) *Logger { l := &Logger{ level: level, @@ -36,21 +30,23 @@ func NewTestLogger(level Level) *Logger { return l } -// TODO: change this func signature ... -func newLogger(parent *Logger, child *Logger) *Logger { +// copyOrCreateLogger inherit handler, level, make copy of fields from parent (if present) +// Or create a new one using default handler, level and no fields +func copyOrCreateLogger(parent *Logger, id *Identity) *Logger { + child := Logger{ + id: id, + } if parent != nil { child.h = parent.h child.level = parent.level child.source = parent.source if len(parent.fields) != 0 { - fields := make([]Field, len(parent.fields)) - copy(fields, parent.fields) - child.fields = fields + child.fields = copyFields(parent.fields) } } else { // TODO: allow customize DefaultHandler child.h = DefaultHandler() child.level = defaultLevel } - return child + return &child } diff --git a/log/logger_identity.go b/log/logger_identity.go index c45cf44..f4ef53a 100644 --- a/log/logger_identity.go +++ b/log/logger_identity.go @@ -34,7 +34,8 @@ func (tpe LoggerType) String() string { // Identity is set based on logger's initialization location, // it is close to, but NOT exactly same as location of actual log. -// It is used for applying filter rules and print logger hierarchy. +// It can be used to inspect logger when traverse and print package hierarchy. +// NOTE: logger hierarchy is flat instead of tree after https://github.com/dyweb/gommon/issues/110 type Identity struct { Package string Function string @@ -48,15 +49,17 @@ var UnknownIdentity = Identity{Package: "unk", Type: UnknownLogger} const ( MagicStructLoggerFunctionName = "LoggerIdentity" - MagicPackageLoggerFunctionName = "init" + magicPackageLoggerFunctionName = "init" // a hack for new init func name after 1.12 // See https://github.com/dyweb/gommon/issues/108 - MagicPackageLoggerFunctionNameGo112 = "init.ializers" + magicPackageLoggerFunctionNameGo112 = "init.ializers" ) +// newIdentityFromCaller get package, struct, func/method using runtime package to avoid user manually set identity +// of a logger to specific package, which is error prone and go out of sync quickly. // TODO: document all the black magic here ... // https://github.com/dyweb/gommon/issues/32 -func NewIdentityFromCaller(skip int) Identity { +func newIdentityFromCaller(skip int) Identity { frame := runtimeutil.GetCallerFrame(skip + 1) var ( pkg string @@ -64,10 +67,10 @@ func NewIdentityFromCaller(skip int) Identity { st string ) tpe := UnknownLogger - // TODO: does it handle vendor correctly, and what about vgo ... + // TODO: does it handle vendor correctly, and what about go mod ... pkg, function = runtimeutil.SplitPackageFunc(frame.Function) tpe = FunctionLogger - if function == MagicPackageLoggerFunctionNameGo112 || function == MagicPackageLoggerFunctionName { + if function == magicPackageLoggerFunctionNameGo112 || function == magicPackageLoggerFunctionName { // https://github.com/dyweb/gommon/issues/108 there are two names, init and init.ializers (after go1.12) tpe = PackageLogger } else if runtimeutil.IsMethod(function) { @@ -101,6 +104,7 @@ func (id *Identity) SourceLocation() string { } func (id *Identity) String() string { + // TODO: add struct and func? or switch based on type return fmt.Sprintf("%s logger %s:%d", id.Type, id.File, id.Line) } diff --git a/log/registry.go b/log/registry.go index 6e783d9..d474a78 100644 --- a/log/registry.go +++ b/log/registry.go @@ -60,7 +60,7 @@ func NewRegistry() *Registry { pkg, _ := runtimeutil.SplitPackageFunc(frame.Function) reg := Registry{ identity: pkg, - loggers: []*Logger{NewPackageLoggerWithSkip(1)}, + loggers: []*Logger{newPackageLoggerWithSkip(1)}, } globalRegistryGroup.add(®) return ® @@ -84,13 +84,10 @@ func (r *Registry) Logger() *Logger { // to avoid register them in registry func (r *Registry) NewLogger() *Logger { // no lock is added because addLogger also acquire lock - id := NewIdentityFromCaller(1) - l := Logger{ - id: &id, - } - newLogger(r.Logger(), &l) - r.addLogger(&l) - return &l + id := newIdentityFromCaller(1) + l := copyOrCreateLogger(r.Logger(), &id) + r.addLogger(l) + return l } // addLogger registers a logger into registry. It's a nop if the logger is already there From ce015a6ebc9f1d3bcbb0fa600b350bc53d612a2f Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 12:02:24 -0700 Subject: [PATCH 11/15] [log] Add WithField and WithFields --- log/logger.go | 32 ++++++++++++++++++++-- log/logger_factory.go | 12 ++++----- log/logger_test.go | 63 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/log/logger.go b/log/logger.go index 08596a8..0ec3737 100644 --- a/log/logger.go +++ b/log/logger.go @@ -37,6 +37,29 @@ type Logger struct { id *Identity // use nil so we can have logger without identity } +// WithField is Copy + AddField +// Use WithField when a common context is shared inside entire func/method +// Use *F methods if you just want adhoc fields, this won't create copy of entire logger +// i.e. logger.InfoF(dlog.Str("user", "cat007")) instead of logger.WithField(dlog.Str("user", "cat007")) +// TODO: there are some optimization here, i.e. the length is known to be + 1 and there is no need to lock +func (l *Logger) WithField(f Field) *Logger { + // can't call Copy here because it will result in wrong identity + id := newIdentityFromCaller(1) + c := copyOrCreateLogger(l, &id) + c.AddField(f) + return c +} + +// WithFields is Copy + AddFields, see WithField doc on when and when not to use it. +// It is faster than calling WithField multiple times because of reduced alloc & copy on existing fields +func (l *Logger) WithFields(fields ...Field) *Logger { + // can't call Copy here because it will result in wrong identity + id := newIdentityFromCaller(1) + c := copyOrCreateLogger(l, &id) + c.AddFields(fields...) + return c +} + // Copy create a new logger with different identity, the identity is based on where Copy is called // Normally you should call Copy inside func or method on a package/strcut logger func (l *Logger) Copy() *Logger { @@ -45,7 +68,7 @@ func (l *Logger) Copy() *Logger { } // AddField add field to current logger in place, it does NOT create a copy of logger -// Use Copy if you want a copy +// Use WithField if you want a copy // It does NOT check duplication func (l *Logger) AddField(f Field) *Logger { l.mu.Lock() @@ -56,7 +79,7 @@ func (l *Logger) AddField(f Field) *Logger { } // AddFields add fields to current logger in place, it does NOT create a copy of logger -// Use Copy if you want a copy +// Use WithFields if you want a copy // It does NOT check duplication func (l *Logger) AddFields(fields ...Field) *Logger { l.mu.Lock() @@ -71,10 +94,12 @@ func (l *Logger) Flush() { l.h.Flush() } +// Level returns current level of logger func (l *Logger) Level() Level { return l.level } +// SetLevel sets level of logger, it is thread safe func (l *Logger) SetLevel(level Level) *Logger { l.mu.Lock() l.level = level @@ -82,6 +107,7 @@ func (l *Logger) SetLevel(level Level) *Logger { return l } +// SetHandler sets handler of logger, it is thread safe func (l *Logger) SetHandler(h Handler) *Logger { l.mu.Lock() l.h = h @@ -89,6 +115,7 @@ func (l *Logger) SetHandler(h Handler) *Logger { return l } +// EnableSource turns on logging source file and line number func (l *Logger) EnableSource() *Logger { l.mu.Lock() l.source = true @@ -96,6 +123,7 @@ func (l *Logger) EnableSource() *Logger { return l } +// DisableSource turns off logging source file and line number func (l *Logger) DisableSource() *Logger { l.mu.Lock() l.source = false diff --git a/log/logger_factory.go b/log/logger_factory.go index 67b98fa..b145224 100644 --- a/log/logger_factory.go +++ b/log/logger_factory.go @@ -2,17 +2,10 @@ package log // logger_factory.go creates logger without register them to registry -// TODO: they should all become private ... - func NewPackageLogger() *Logger { return newPackageLoggerWithSkip(1) } -func newPackageLoggerWithSkip(skip int) *Logger { - id := newIdentityFromCaller(skip + 1) - return copyOrCreateLogger(nil, &id) -} - func NewStructLogger(packageLogger *Logger, loggable LoggableStruct) *Logger { id := loggable.LoggerIdentity(func() Identity { return newIdentityFromCaller(1) @@ -30,6 +23,11 @@ func NewTestLogger(level Level) *Logger { return l } +func newPackageLoggerWithSkip(skip int) *Logger { + id := newIdentityFromCaller(skip + 1) + return copyOrCreateLogger(nil, &id) +} + // copyOrCreateLogger inherit handler, level, make copy of fields from parent (if present) // Or create a new one using default handler, level and no fields func copyOrCreateLogger(parent *Logger, id *Identity) *Logger { diff --git a/log/logger_test.go b/log/logger_test.go index b59290d..9a2445d 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -6,6 +6,69 @@ import ( asst "github.com/stretchr/testify/assert" ) +func TestLogger_AddField(t *testing.T) { + assert := asst.New(t) + + logger := NewTestLogger(defaultLevel) + th := NewTestHandler() + logger.SetHandler(th) + + logger.AddField(Str("foo", "bar")) + logger.InfoF("original") + e, ok := th.getLogByMessage("original") + assert.True(ok) + assert.Equal(e.context[0].Key, "foo") + assert.Equal(e.context[0].Str, "bar") + + // TODO: now AddField is not handling duplication ... + t.Run("duplication is NOT handled", func(t *testing.T) { + logger.AddField(Str("foo", "bar2")) + logger.InfoF("bar2") + e, ok := th.getLogByMessage("bar2") + assert.True(ok) + assert.Equal(e.context[0].Key, "foo") + assert.Equal(e.context[0].Str, "bar") + assert.Equal(e.context[1].Key, "foo") + assert.Equal(e.context[1].Str, "bar2") + }) +} + +func TestLogger_WithField(t *testing.T) { + assert := asst.New(t) + + l1 := NewTestLogger(defaultLevel) + th := NewTestHandler() + l1.SetHandler(th) + + l2 := l1.WithField(Str("logger", "l2")) + l1.AddField(Str("logger", "l1")) + + l1.InfoF("hi1") + l2.InfoF("hi2") + + e, ok := th.getLogByMessage("hi1") + assert.True(ok) + assert.Equal(e.context[0].Str, "l1") + + e, ok = th.getLogByMessage("hi2") + assert.True(ok) + assert.Equal(e.context[0].Str, "l2") +} + +func TestLogger_WithFields(t *testing.T) { + assert := asst.New(t) + + logger := NewTestLogger(defaultLevel) + th := NewTestHandler() + logger.SetHandler(th) + + logger = logger.WithFields(Int("a", 1), Str("foo", "bar")) + logger.InfoF("hi") + e, ok := th.getLogByMessage("hi") + assert.True(ok) + assert.Equal(2, len(e.context)) +} + func TestLogger_SetHandler(t *testing.T) { assert := asst.New(t) From f80f802f56f3f02925f8acc9310fe565be24a2f7 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 12:30:13 -0700 Subject: [PATCH 12/15] [structure] Deprecate structure package #113 --- Makefile | 4 ++-- {structure => legacy/structure}/README.md | 0 {structure => legacy/structure}/heap.go | 0 {structure => legacy/structure}/heap_test.go | 0 {structure => legacy/structure}/pkg.go | 0 {structure => legacy/structure}/priority_queue.go | 0 {structure => legacy/structure}/set.go | 0 {structure => legacy/structure}/set_test.go | 0 {structure => legacy/structure}/tree.go | 0 {structure => legacy/structure}/tree_test.go | 0 10 files changed, 2 insertions(+), 2 deletions(-) rename {structure => legacy/structure}/README.md (100%) rename {structure => legacy/structure}/heap.go (100%) rename {structure => legacy/structure}/heap_test.go (100%) rename {structure => legacy/structure}/pkg.go (100%) rename {structure => legacy/structure}/priority_queue.go (100%) rename {structure => legacy/structure}/set.go (100%) rename {structure => legacy/structure}/set_test.go (100%) rename {structure => legacy/structure}/tree.go (100%) rename {structure => legacy/structure}/tree_test.go (100%) diff --git a/Makefile b/Makefile index 3011faa..e260852 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,8 @@ help: GO = GO111MODULE=on go # -- build vars --- -PKGS =./errors/... ./generator/... ./log/... ./noodle/... ./structure/... ./util/... -PKGST =./cmd ./errors ./generator ./log ./noodle ./structure ./util +PKGS =./errors/... ./generator/... ./log/... ./noodle/... ./util/... +PKGST =./cmd ./errors ./generator ./log ./noodle ./util VERSION = 0.0.10 BUILD_COMMIT := $(shell git rev-parse HEAD) BUILD_TIME := $(shell date +%Y-%m-%dT%H:%M:%S%z) diff --git a/structure/README.md b/legacy/structure/README.md similarity index 100% rename from structure/README.md rename to legacy/structure/README.md diff --git a/structure/heap.go b/legacy/structure/heap.go similarity index 100% rename from structure/heap.go rename to legacy/structure/heap.go diff --git a/structure/heap_test.go b/legacy/structure/heap_test.go similarity index 100% rename from structure/heap_test.go rename to legacy/structure/heap_test.go diff --git a/structure/pkg.go b/legacy/structure/pkg.go similarity index 100% rename from structure/pkg.go rename to legacy/structure/pkg.go diff --git a/structure/priority_queue.go b/legacy/structure/priority_queue.go similarity index 100% rename from structure/priority_queue.go rename to legacy/structure/priority_queue.go diff --git a/structure/set.go b/legacy/structure/set.go similarity index 100% rename from structure/set.go rename to legacy/structure/set.go diff --git a/structure/set_test.go b/legacy/structure/set_test.go similarity index 100% rename from structure/set_test.go rename to legacy/structure/set_test.go diff --git a/structure/tree.go b/legacy/structure/tree.go similarity index 100% rename from structure/tree.go rename to legacy/structure/tree.go diff --git a/structure/tree_test.go b/legacy/structure/tree_test.go similarity index 100% rename from structure/tree_test.go rename to legacy/structure/tree_test.go From e2949fcbbe599dd48f696743de300dc2513a4124 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 12:31:42 -0700 Subject: [PATCH 13/15] [structure] Add build ignore #113 --- legacy/structure/heap.go | 2 ++ legacy/structure/heap_test.go | 2 ++ legacy/structure/pkg.go | 2 ++ legacy/structure/priority_queue.go | 2 ++ legacy/structure/set.go | 2 ++ legacy/structure/set_test.go | 2 ++ legacy/structure/tree.go | 2 ++ legacy/structure/tree_test.go | 2 ++ 8 files changed, 16 insertions(+) diff --git a/legacy/structure/heap.go b/legacy/structure/heap.go index daef30a..69ffa4f 100644 --- a/legacy/structure/heap.go +++ b/legacy/structure/heap.go @@ -1,3 +1,5 @@ +// +build ignore + package structure // https://golang.org/src/container/heap/heap.go diff --git a/legacy/structure/heap_test.go b/legacy/structure/heap_test.go index ad2a30a..6dfad56 100644 --- a/legacy/structure/heap_test.go +++ b/legacy/structure/heap_test.go @@ -1,3 +1,5 @@ +// +build ignore + package structure_test import ( diff --git a/legacy/structure/pkg.go b/legacy/structure/pkg.go index 0e9d7ce..2d25c8b 100644 --- a/legacy/structure/pkg.go +++ b/legacy/structure/pkg.go @@ -1,3 +1,5 @@ +// +build ignore + /* Package structure add some missing common data structures to Golang */ diff --git a/legacy/structure/priority_queue.go b/legacy/structure/priority_queue.go index 8fa44fd..b647f29 100644 --- a/legacy/structure/priority_queue.go +++ b/legacy/structure/priority_queue.go @@ -1,3 +1,5 @@ +// +build ignore + package structure // TODO: interface & impl diff --git a/legacy/structure/set.go b/legacy/structure/set.go index 33d9987..40bddef 100644 --- a/legacy/structure/set.go +++ b/legacy/structure/set.go @@ -1,3 +1,5 @@ +// +build ignore + package structure // Set is a map with string key and bool value diff --git a/legacy/structure/set_test.go b/legacy/structure/set_test.go index d4dd48d..3427193 100644 --- a/legacy/structure/set_test.go +++ b/legacy/structure/set_test.go @@ -1,3 +1,5 @@ +// +build ignore + package structure import ( diff --git a/legacy/structure/tree.go b/legacy/structure/tree.go index 16bcd3f..3496ebd 100644 --- a/legacy/structure/tree.go +++ b/legacy/structure/tree.go @@ -1,3 +1,5 @@ +// +build ignore + package structure import ( diff --git a/legacy/structure/tree_test.go b/legacy/structure/tree_test.go index 2117d8f..6f446b8 100644 --- a/legacy/structure/tree_test.go +++ b/legacy/structure/tree_test.go @@ -1,3 +1,5 @@ +// +build ignore + package structure import ( From d6c4339a0ec7fe9fec8ec0ae368895f8f43872e4 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 13:15:58 -0700 Subject: [PATCH 14/15] [doc] Remove oudated guides in README --- Dockerfile | 5 ++++- Makefile | 4 ---- README.md | 37 ++++++++----------------------------- directory.md | 1 - 4 files changed, 12 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ea2491..e7bb986 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,10 @@ # This Dockerfile is a demo of using go-dev to build a go binary using multi stage build # It is based on # https://docs.docker.com/v17.09/engine/userguide/eng-image/dockerfile_best-practices/#use-multi-stage-builds -FROM dyweb/go-dev:1.11.4 as builder +# +# The builder-image go-dev can be found in hack/go-dev +# Versions can be found on https://hub.docker.com/r/dyweb/go-dev/tags +FROM dyweb/go-dev:1.12.4 as builder LABEL maintainer="contact@dongyue.io" diff --git a/Makefile b/Makefile index e260852..c692cdf 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,6 @@ test unit test generate generate code using gommon loc lines of code (cloc required, brew install cloc) -Dev first time: -dep-install install dependencies based on lock file -dep-update update dependency based on spec and code - Build: install install all binaries under ./cmd to $$GOPATH/bin build compile all binary to ./build for current platform diff --git a/README.md b/README.md index dd895bd..4a1eb46 100644 --- a/README.md +++ b/README.md @@ -17,42 +17,21 @@ Gommon is a collection of common util libraries written in Go. -It has the following components: - - [errors](errors) error wrapping, inspection, multi error (error list), common error types -- [log](log) fine grained level control and reasonable performance +- [log](log) per package logger with [reasonable performance](log/_benchmarks/README.md) - [noodle](noodle) embed static assets for web application with `.noodleignore` support - [generator](generator) render go template, generate methods for logger interface based on `gommon.yml` -- [structure](structure) data structure like Set etc. to go -- [util](util) small utils over standard libraries utils - -Legacy - -- [config v1](config) A YAML config reader with template support -- [log v1](legacy/log) A logrus like structured logger -- [runner](legacy/runner) A os/exec wrapper -- [requests](requests) A pythonic wrapper for `net/http`, HTTP for Gopher - -## Dependencies - -Currently we only have one non standard library dependencies (cmd and examples are not considered), see [Gopkg.lock](Gopkg.lock) - -- [go-yaml/yaml](https://github.com/go-yaml/yaml) for read config written in YAML - - we don't need most feature of YAML, and want to have access to the parser directly to report which line has incorrect semantic (after checking it in application). - - might write one in [ANTLR](https://github.com/antlr/antlr4) - - we also have a DSL work in progress [RCL: Reika Configuration Language](https://github.com/at15/reika/issues/49), which is like [HCL](https://github.com/hashicorp/hcl2) - -Removed +- [util](util) wrappers for standard libraries -- [pkg/errors](https://github.com/pkg/errors) for including context in error, removed in [#59](https://github.com/dyweb/gommon/pull/59) -replaced by `gommon/errors` +It has little third party dependencies, only [go-yaml/yaml](https://github.com/go-yaml/yaml) in [util/cast](util/cast), +[go-shellquote](github.com/kballard/go-shellquote) in [generator](generator), +other dependencies like cobra are only for cli, see [go.mod](go.mod). ## Development -- install go https://golang.org/ -- install dep https://github.com/golang/dep -- `make dep-install` -- `make test` +- requires go1.12+. go1.11.x should work as well, the Makefile set `GO111MODULE=on` so you can use in GOPATH +- `make help` +- [Directory layout](directory.md) ## License diff --git a/directory.md b/directory.md index e616af4..181c78a 100644 --- a/directory.md +++ b/directory.md @@ -6,7 +6,6 @@ - [doc](doc) style guide and developer log - [errors](errors) error wrapping, multi error and error inspection - [generator](generator) generating interface methods, render go template, execute shell command. -- [legacy](legacy) legacy code base - [noodle](noodle) embed static assets for go binary with .ignore file support - [playground](playground) test library and replay issues - [scripts](scripts) test scripts From 39b9fe5b640999c859f616ab4eee81538dc8a6e2 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 13:16:30 -0700 Subject: [PATCH 15/15] Fix #112 Remove legacy package - moved to https://github.com/dyweb/gommon-legacy --- legacy/config/README.md | 65 ---- legacy/config/gommon.yml | 4 - legacy/config/gommon_generated.go | 19 - legacy/config/pkg.go | 27 -- .../config/testdata/multi_doc_multi_vars.yml | 31 -- .../config/testdata/multi_doc_single_vars.yml | 14 - legacy/config/testdata/single_doc_no_vars.yml | 2 - legacy/config/testdata/single_doc_vars.yml | 3 - legacy/config/testdata/structured.yml | 8 - legacy/config/yaml.go | 324 ------------------ legacy/config/yaml_test.go | 221 ------------ legacy/glide.lock | 33 -- legacy/glide.yaml | 14 - legacy/log/README.md | 47 --- legacy/log/config.example.yml | 5 - legacy/log/config.go | 52 --- legacy/log/config_test.go | 32 -- legacy/log/doc.go | 8 - legacy/log/entry.go | 129 ------- legacy/log/entry_generated.go | 73 ---- legacy/log/entry_generated.go.tmpl | 19 - legacy/log/entry_test.go | 59 ---- legacy/log/filter.go | 44 --- legacy/log/filter_test.go | 39 --- legacy/log/formatter.go | 110 ------ legacy/log/formatter_test.go | 3 - legacy/log/level.go | 126 ------- legacy/log/level_test.go | 30 -- legacy/log/logger.go | 134 -------- legacy/log/logger_test.go | 53 --- legacy/log/v2.md | 112 ------ legacy/pkg.go | 4 - legacy/requests/README.md | 19 - legacy/requests/auth.go | 53 --- legacy/requests/auth_test.go | 22 -- legacy/requests/builder.go | 69 ---- legacy/requests/builder_test.go | 25 -- legacy/requests/default.go | 35 -- legacy/requests/doc/history.md | 0 legacy/requests/pkg.go | 4 - legacy/requests/requests.go | 128 ------- legacy/requests/requests_test.go | 101 ------ legacy/requests/response.go | 31 -- legacy/runner/README.md | 6 - legacy/runner/command.go | 62 ---- legacy/runner/command_test.go | 28 -- legacy/runner/context.go | 25 -- legacy/runner/doc.go | 6 - legacy/runner/example.config.yml | 47 --- legacy/runner/pkg.go | 3 - legacy/structure/README.md | 18 - legacy/structure/heap.go | 89 ----- legacy/structure/heap_test.go | 33 -- legacy/structure/pkg.go | 6 - legacy/structure/priority_queue.go | 5 - legacy/structure/set.go | 51 --- legacy/structure/set_test.go | 45 --- legacy/structure/tree.go | 96 ------ legacy/structure/tree_test.go | 55 --- 59 files changed, 2906 deletions(-) delete mode 100644 legacy/config/README.md delete mode 100644 legacy/config/gommon.yml delete mode 100644 legacy/config/gommon_generated.go delete mode 100644 legacy/config/pkg.go delete mode 100644 legacy/config/testdata/multi_doc_multi_vars.yml delete mode 100644 legacy/config/testdata/multi_doc_single_vars.yml delete mode 100644 legacy/config/testdata/single_doc_no_vars.yml delete mode 100644 legacy/config/testdata/single_doc_vars.yml delete mode 100644 legacy/config/testdata/structured.yml delete mode 100644 legacy/config/yaml.go delete mode 100644 legacy/config/yaml_test.go delete mode 100644 legacy/glide.lock delete mode 100644 legacy/glide.yaml delete mode 100644 legacy/log/README.md delete mode 100644 legacy/log/config.example.yml delete mode 100644 legacy/log/config.go delete mode 100644 legacy/log/config_test.go delete mode 100644 legacy/log/doc.go delete mode 100644 legacy/log/entry.go delete mode 100644 legacy/log/entry_generated.go delete mode 100644 legacy/log/entry_generated.go.tmpl delete mode 100644 legacy/log/entry_test.go delete mode 100644 legacy/log/filter.go delete mode 100644 legacy/log/filter_test.go delete mode 100644 legacy/log/formatter.go delete mode 100644 legacy/log/formatter_test.go delete mode 100644 legacy/log/level.go delete mode 100644 legacy/log/level_test.go delete mode 100644 legacy/log/logger.go delete mode 100644 legacy/log/logger_test.go delete mode 100644 legacy/log/v2.md delete mode 100644 legacy/pkg.go delete mode 100644 legacy/requests/README.md delete mode 100644 legacy/requests/auth.go delete mode 100644 legacy/requests/auth_test.go delete mode 100644 legacy/requests/builder.go delete mode 100644 legacy/requests/builder_test.go delete mode 100644 legacy/requests/default.go delete mode 100644 legacy/requests/doc/history.md delete mode 100644 legacy/requests/pkg.go delete mode 100644 legacy/requests/requests.go delete mode 100644 legacy/requests/requests_test.go delete mode 100644 legacy/requests/response.go delete mode 100644 legacy/runner/README.md delete mode 100644 legacy/runner/command.go delete mode 100644 legacy/runner/command_test.go delete mode 100644 legacy/runner/context.go delete mode 100644 legacy/runner/doc.go delete mode 100644 legacy/runner/example.config.yml delete mode 100644 legacy/runner/pkg.go delete mode 100644 legacy/structure/README.md delete mode 100644 legacy/structure/heap.go delete mode 100644 legacy/structure/heap_test.go delete mode 100644 legacy/structure/pkg.go delete mode 100644 legacy/structure/priority_queue.go delete mode 100644 legacy/structure/set.go delete mode 100644 legacy/structure/set_test.go delete mode 100644 legacy/structure/tree.go delete mode 100644 legacy/structure/tree_test.go diff --git a/legacy/config/README.md b/legacy/config/README.md deleted file mode 100644 index 648d0b7..0000000 --- a/legacy/config/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Config - -Yaml configuration with template, inspired by [Ansible](http://docs.ansible.com/ansible/playbooks.html) and [Viper](https://github.com/spf13/viper) - -## Usage - -- [ ] TODO - -## Specification - -- using [golang's template](https://golang.org/pkg/text/template/) syntax, not ansible, not pongo2 -- only support single file, but can have multiple documents separated by `---` -- environment variables - - use `env` function, i.e. `home: {{ env "HOME"}}` -- variables - - special top level key `vars` is used via `var` function - - `vars` in different documents are merged, while for other top level keys, their value is replaced by latter documents - - `vars` in current document can be used in current document, we actually render the template of each document twice, so in vars - section you can use previous `vars` and all the syntax supported by golang's template -- [ ] condition (not tested, but golang template should support it) -- loop - - use the `range` syntax - -Using variables - -- [ ] TODO: will `xkb --target={{ $name }} --port={{ $db.port }}` have undesired blank and/or new line? - -````yaml -vars: - influxdb_port: 8080 - databases: - - influxdb - - kairosdb -influxdb: - port: {{ var "influxdb_port" }} -kairosdb: - port: {{ env "KAIROSDB_PORT" }} -tests: -{{ range $name := var "databases" -}} -{{ $db := var $name -}} - - xkb --target={{ $name }} --port={{ $db.port }} -{{ end } -base: {{ env "HOME" }} -```` - -Using multiple document - - - -- [ ] TODO: example -- [ ] TODO: the example in runner actually does not require any template features - -## YAML parsing problem - -- multiple document support, the author said it is easy, but he never tried to solve them, - - the [encoder & decoder PR](https://github.com/go-yaml/yaml/pull/163/) seems to support multiple documents but I don't -understand why it does. - - Actually it would be interesting to look into how to unmarshal stream data since that is needed for Xephon-B, -though for most formats, go already have encoder and decoder - - (Adopted) A easier way is to split the whole file by `---` before put it into go-yaml - -## Acknowledgement - -- https://github.com/spf13/viper -- https://github.com/go-yaml/yaml diff --git a/legacy/config/gommon.yml b/legacy/config/gommon.yml deleted file mode 100644 index a8f6375..0000000 --- a/legacy/config/gommon.yml +++ /dev/null @@ -1,4 +0,0 @@ -# configuration for gommon -loggers: - - struct: "*YAMLConfig" - receiver: c \ No newline at end of file diff --git a/legacy/config/gommon_generated.go b/legacy/config/gommon_generated.go deleted file mode 100644 index 9324c91..0000000 --- a/legacy/config/gommon_generated.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build ignore - -// Code generated by gommon from config/gommon.yml DO NOT EDIT. - -package config - -import dlog "github.com/dyweb/gommon/log" - -func (c *YAMLConfig) SetLogger(logger *dlog.Logger) { - c.log = logger -} - -func (c *YAMLConfig) GetLogger() *dlog.Logger { - return c.log -} - -func (c *YAMLConfig) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { - return justCallMe() -} diff --git a/legacy/config/pkg.go b/legacy/config/pkg.go deleted file mode 100644 index bdac9bd..0000000 --- a/legacy/config/pkg.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build ignore - -// Package config(Deprecated) supports go text/template, environment and self defined variables -package config - -import ( - "github.com/dyweb/gommon/util/logutil" -) - -var log = logutil.NewPackageLogger() - -const ( - yamlDocumentSeparator = "---" - defaultTemplateName = "gommon yaml" - defaultKeyDelimiter = "." -) - -type Path string - -type Reader interface { - Path() Path - Content() string -} - -type StructuredConfig interface { - Validate() error -} diff --git a/legacy/config/testdata/multi_doc_multi_vars.yml b/legacy/config/testdata/multi_doc_multi_vars.yml deleted file mode 100644 index 7b5d7cb..0000000 --- a/legacy/config/testdata/multi_doc_multi_vars.yml +++ /dev/null @@ -1,31 +0,0 @@ -vars: - databases: - - influxdb - - kairosdb - # defined in another document - - xephonk - influxdb: - backend: tsm - language: golang - kairosdb: - backend: cassandra - language: java - foo1: bar1 -foo: 1 ---- -vars: -# NOTE: when merging vars, it's a shallow merge, we merge by top level key under vars, so we can't have -# databases: -# - xephonk - xephonk: - backend: disk - language: golang - foo2: bar2 -{{ range $name := var "databases" }} -{{ $name }}: -{{ $db := var $name }} - name: {{ $name }} - backend: {{ $db.backend }} - language: {{ $db.language }} -{{ end }} -foo: 2 \ No newline at end of file diff --git a/legacy/config/testdata/multi_doc_single_vars.yml b/legacy/config/testdata/multi_doc_single_vars.yml deleted file mode 100644 index 5c72b5a..0000000 --- a/legacy/config/testdata/multi_doc_single_vars.yml +++ /dev/null @@ -1,14 +0,0 @@ -vars: - influxdb_port: 8081 - databases: - - influxdb - - kairosdb - ssl: false -foo: 1 ---- -{{ range $name := var "databases" }} -{{ $name }}: - name: {{ $name }} - ssl: {{ var "ssl" }} -{{ end }} -foo: 2 \ No newline at end of file diff --git a/legacy/config/testdata/single_doc_no_vars.yml b/legacy/config/testdata/single_doc_no_vars.yml deleted file mode 100644 index 1d67c5f..0000000 --- a/legacy/config/testdata/single_doc_no_vars.yml +++ /dev/null @@ -1,2 +0,0 @@ -home: {{ env "HOME" }} -foo: bar \ No newline at end of file diff --git a/legacy/config/testdata/single_doc_vars.yml b/legacy/config/testdata/single_doc_vars.yml deleted file mode 100644 index e3e5708..0000000 --- a/legacy/config/testdata/single_doc_vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -vars: - foo: bar -foo: {{ var "foo" }} diff --git a/legacy/config/testdata/structured.yml b/legacy/config/testdata/structured.yml deleted file mode 100644 index c958caf..0000000 --- a/legacy/config/testdata/structured.yml +++ /dev/null @@ -1,8 +0,0 @@ -logging: - level: info - color: true - source: false -mode: local -base: {{ env "HOME" }} -base2: {{ envOr "NOBODY_HAVE_SET_ME" "base2" }} -base3: {{ envOr "HOME" "/home/gommon" }} \ No newline at end of file diff --git a/legacy/config/yaml.go b/legacy/config/yaml.go deleted file mode 100644 index 2893e6c..0000000 --- a/legacy/config/yaml.go +++ /dev/null @@ -1,324 +0,0 @@ -// +build ignore - -package config - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "reflect" - "strings" - "sync" - "text/template" - - "gopkg.in/yaml.v2" - - "github.com/dyweb/gommon/errors" - dlog "github.com/dyweb/gommon/log" - "github.com/dyweb/gommon/util" - "github.com/dyweb/gommon/util/cast" -) - -// YAMLConfig is a thread safe struct for parse YAML file and get value -type YAMLConfig struct { - vars map[string]interface{} - data map[string]interface{} - keyDelimiter string - mu sync.RWMutex - log *dlog.Logger -} - -func LoadYAMLDirect(file string, cfg interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - if err := yaml.Unmarshal(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml") - } - return nil -} - -func LoadYAMLDirectFrom(r io.Reader, cfg interface{}) error { - b, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "can't read reader") - } - if err := yaml.Unmarshal(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml") - } - return nil -} - -func LoadYAMLDirectStrict(file string, cfg interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - if err := yaml.UnmarshalStrict(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml in strict mode") - } - return nil -} - -func LoadYAMLDirectFromStrict(r io.Reader, cfg interface{}) error { - b, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "can't read reader") - } - if err := yaml.UnmarshalStrict(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml in strict mode") - } - return nil -} - -// LoadYAMLAsStruct is a convenient wrapper for loading a single YAML file into struct, you should pass a pointer to the -// struct as second argument. It will remove the vars section. -func LoadYAMLAsStruct(file string, structuredConfig interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - c := NewYAMLConfig() - if err := c.ParseSingleDocument(b); err != nil { - return errors.Wrap(err, "can't parse as single document") - } - if err := c.Unmarshal(structuredConfig, true); err != nil { - return errors.Wrapf(err, "can't turn file %s into structured config", file) - } - return nil -} - -// SplitMultiDocument splits a yaml file that contains multiple documents and (only) trim the first one if it is empty -func SplitMultiDocument(data []byte) [][]byte { - docs := bytes.Split(data, []byte(yamlDocumentSeparator)) - // check the first one, it could be empty - if len(docs[0]) != 0 { - return docs - } - return docs[1:] -} - -// NewYAMLConfig returns a config with internal map structure initialized -func NewYAMLConfig() *YAMLConfig { - c := &YAMLConfig{} - c.log = dlog.NewStructLogger(log, c) - c.clear() - c.keyDelimiter = defaultKeyDelimiter - return c -} - -// clear is used by test for using one config object for several tests -// and is also used by constructor -func (c *YAMLConfig) clear() { - c.vars = make(map[string]interface{}) - c.data = make(map[string]interface{}) -} - -func (c *YAMLConfig) Var(name string) interface{} { - return c.vars[name] -} - -func (c *YAMLConfig) Env(name string) string { - return os.Getenv(name) -} - -func (c *YAMLConfig) EnvOr(name string, defaultVal string) string { - val := os.Getenv(name) - if val == "" { - return defaultVal - } - return val -} - -func (c *YAMLConfig) FuncMaps() template.FuncMap { - return template.FuncMap{ - "env": c.Env, - "envOr": c.EnvOr, - "var": c.Var, - } -} - -func (c *YAMLConfig) ParseMultiDocument(data []byte) error { - // split the doc, parse by order, add result to context so the following parser can use it - docs := SplitMultiDocument(data) - for _, doc := range docs { - err := c.ParseSingleDocument(doc) - if err != nil { - return errors.Wrap(err, "error when parsing one of the documents") - } - } - return nil -} - -func (c *YAMLConfig) ParseSingleDocument(doc []byte) error { - c.mu.Lock() - defer c.mu.Unlock() - - // we render the template twice, first time we use vars from previous documents and environment variables - // second time, we use vars declared in this document, if any. - - // this is the first render - rendered, err := c.renderDocument(doc) - if err != nil { - return errors.Wrap(err, "can't render template with previous documents' vars") - } - - c.log.Tracef("01-before\n%s", doc) - c.log.Tracef("01-after\n%s", rendered) - - tmpData := make(map[string]interface{}) - err = yaml.Unmarshal(rendered, &tmpData) - if err != nil { - return errors.Wrap(err, "can't parse rendered template yaml to map[string]interface{} after first render") - } - - // preserve the vars - if varsInCurrentDocument, hasVars := tmpData["vars"]; hasVars { - // NOTE: it's map[interface{}]interface{} instead of map[string]interface{} - // because how go-yaml handle the decoding - varsI, ok := varsInCurrentDocument.(map[interface{}]interface{}) - if !ok { - // TODO: test this, it seems if ok == false, then go-yaml should return already, which means ok should always be true? - return errors.Errorf("unable to cast %s to map[interface{}]interface{}", reflect.TypeOf(varsInCurrentDocument)) - } - vars := cast.ToStringMap(varsI) - util.MergeStringMap(c.vars, vars) - - // render again using vars in current document - rendered, err = c.renderDocument(doc) - if err != nil { - return errors.Wrap(err, "can't render template with vars in current document") - } - - c.log.Tracef("02-before\n%s", doc) - c.log.Tracef("02-after\n%s", rendered) - - tmpData = make(map[string]interface{}) - err = yaml.Unmarshal(rendered, &tmpData) - if err != nil { - return errors.Wrap(err, "can't parse rendered template yaml to map[string]interface{} after second render") - } - } - - // put the data into c.data - for k, v := range tmpData { - c.data[k] = v - } - // NOTE: vars are merged instead of overwritten like other top level keys - c.data["vars"] = c.vars - return nil -} - -func (c *YAMLConfig) Get(key string) interface{} { - val, err := c.GetOrFail(key) - if err != nil { - // TODO: so Get does not do any error handling? what did viper do? - c.log.Debugf("can't get key %s due to %s", key, err.Error()) - } - return val -} - -func (c *YAMLConfig) GetOrDefault(key string, defaultVal interface{}) interface{} { - val, err := c.GetOrFail(key) - if err != nil { - c.log.Debugf("use default %v for key %s due to %s", defaultVal, key, err.Error()) - return defaultVal - } - return val -} - -func (c *YAMLConfig) GetOrFail(key string) (interface{}, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - path := strings.Split(key, c.keyDelimiter) - val, err := searchMap(c.data, path) - if err != nil { - return nil, err - } - return val, nil -} - -func (c *YAMLConfig) Unmarshal(structuredConfig interface{}, removeVars bool) error { - c.mu.Lock() - defer c.mu.Unlock() - - // TODO: maybe we no longer need to keep vars in data? - if removeVars { - delete(c.data, "vars") - defer func() { c.data["vars"] = c.vars }() - } - out, err := yaml.Marshal(c.data) - // FIXME: those Wrapf should be Wrap - if err != nil { - return errors.Wrapf(err, "can't marshal data back to yaml") - } - err = yaml.Unmarshal(out, structuredConfig) - if err != nil { - return errors.Wrapf(err, "can't unmarshal data to specified config struct") - } - return nil -} - -func (c *YAMLConfig) UnmarshalKey(key string, structuredConfig interface{}) error { - val, err := c.GetOrFail(key) - if err != nil { - return errors.Wrapf(err, "can't get the val for unmarshal") - } - out, err := yaml.Marshal(val) - if err != nil { - return errors.Wrapf(err, "can't marshal data back to yaml") - } - err = yaml.Unmarshal(out, structuredConfig) - if err != nil { - return errors.Wrapf(err, "can't unmarshal data to specified config struct") - } - return nil -} - -// TODO: what if the user use key that includes `.`, like {"github.com": 123} -func searchMap(src map[string]interface{}, path []string) (interface{}, error) { - var result interface{} - if len(path) == 0 { - return result, errors.New("path is empty, at least provide one segment") - } - result = src - previousPath := "" - for i := 0; i < len(path); i++ { - key := path[i] - //c.log.Debug(key) - //c.log.Debug(result) - if reflect.TypeOf(result).Kind() != reflect.Map { - return nil, errors.Errorf("%s is not a map but %s, %v", previousPath, reflect.TypeOf(result), result) - } - m, ok := result.(map[string]interface{}) - if !ok { - m = cast.ToStringMap(result.(map[interface{}]interface{})) - } - // FIXED: this is a tricky problem, if you use `:` here, you create a new local variable instead update the one - // outside the loop, that's all the Debug(result) for - //result, ok := m[key] - result, ok = m[key] - if !ok { - return result, errors.Errorf("key: %s does not exists in path: %s, val: %v", key, previousPath, m) - } - //c.log.Debug(result) - previousPath += key - } - return result, nil -} - -func (c *YAMLConfig) renderDocument(tplBytes []byte) ([]byte, error) { - tmpl, err := template.New(defaultTemplateName).Funcs(c.FuncMaps()).Parse(string(tplBytes[:])) - if err != nil { - return nil, errors.Wrapf(err, "can't parse template") - } - var b bytes.Buffer - err = tmpl.Execute(&b, "") - if err != nil { - return nil, errors.Wrapf(err, "can't render template") - } - return b.Bytes(), nil -} diff --git a/legacy/config/yaml_test.go b/legacy/config/yaml_test.go deleted file mode 100644 index 8d5b5df..0000000 --- a/legacy/config/yaml_test.go +++ /dev/null @@ -1,221 +0,0 @@ -// +build ignore - -package config - -import ( - "os" - "testing" - - dlog "github.com/dyweb/gommon/log" - "github.com/dyweb/gommon/util/testutil" - asst "github.com/stretchr/testify/assert" -) - -// Deprecated this only works for old logging package -type logConfig struct { - Level string `yaml:"level"` - Color bool `yaml:"color"` - Source bool `yaml:"source"` -} - -type structuredConfig struct { - Logging logConfig `yaml:"logging"` - Mode string `yaml:"mode"` - Base string `yaml:"base"` - Base2 string `yaml:"base2"` - Base3 string `yaml:"base3"` - XXX map[string]interface{} `yaml:",inline"` // NOTE: this is used to catch unmatched fields -} - -func TestNewYAMLConfig(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - assert.Equal(c.GetLogger().Level(), dlog.InfoLevel) - assert.Equal(c.GetLogger().Identity().Type, dlog.StructLogger) - assert.Equal(c.GetLogger().Identity().Struct, "YAMLConfig") - assert.Equal(c.GetLogger().Identity().Package, "github.com/dyweb/gommon/config") -} - -func TestYAMLConfig_ParseWithoutTemplate(t *testing.T) { - assert := asst.New(t) - var dat = ` -a: Easy! -b: - c: 2 - d: [3, 4] -` - c := NewYAMLConfig() - err := c.ParseMultiDocument([]byte(dat)) - assert.Nil(err) - - // NOTE： this is invalid yaml because when you use ` syntax to declare long string in Golang, - // the indent are also included in the string, so this yaml has indent without any parent, which is invalid - var invalidDat = ` - a: Easy! - b: - c: 2 - d: [3, 4] - ` - // the print should show you the string has indent you may not be expecting - //log.Print(invalidDat) - err = c.ParseMultiDocument([]byte(invalidDat)) - assert.NotNil(err) - //log.Print(err.Error()) -} - -func TestLoadYAMLAsStruct(t *testing.T) { - assert := asst.New(t) - var conf structuredConfig - assert.Nil(LoadYAMLAsStruct("testdata/structured.yml", &conf)) -} - -func TestSplitMultiDocument(t *testing.T) { - assert := asst.New(t) - var multi = `--- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -` - documents := SplitMultiDocument([]byte(multi)) - //for _, d := range documents { - // t.Log(string(d[:])) - //} - assert.Equal(2, len(documents)) - documents = SplitMultiDocument([]byte("---")) - assert.Equal(1, len(documents)) - // without the starting `---` - var multi2 = ` -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -` - documents = SplitMultiDocument([]byte(multi2)) - assert.Equal(2, len(documents)) -} - -func TestYAMLConfig_ParseSingleDocument(t *testing.T) { - cases := []struct { - file string - }{ - {"single_doc_no_vars"}, - {"single_doc_vars"}, - } - c := NewYAMLConfig() - //c.GetLogger().SetLevel(dlog.TraceLevel) - for _, tc := range cases { - t.Run(tc.file, func(t *testing.T) { - assert := asst.New(t) - c.clear() - doc := testutil.ReadFixture(t, "testdata/"+tc.file+".yml") - assert.Nil(c.ParseSingleDocument(doc)) - // TODO: expect value, not just log - t.Log(c.data) - }) - } -} - -func TestYAMLConfig_ParseMultiDocument(t *testing.T) { - cases := []struct { - file string - }{ - {"multi_doc_single_vars"}, - {"multi_doc_multi_vars"}, - } - c := NewYAMLConfig() - //util.UseVerboseLog() - for _, tc := range cases { - t.Run(tc.file, func(t *testing.T) { - assert := asst.New(t) - c.clear() - doc := testutil.ReadFixture(t, "testdata/"+tc.file+".yml") - assert.Nil(c.ParseMultiDocument(doc)) - t.Log(c.data) - }) - } - //util.DisableVerboseLog() -} - -func TestYAMLConfig_Get(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - //util.UseVerboseLog() - assert.Equal("bar1", c.Get("vars.foo1")) - assert.Equal(nil, c.Get("vars.that_does_not_exists")) - // NOTE: top level keys other than vars are overwritten instead of merged - assert.Equal(2, c.Get("foo")) - //util.DisableVerboseLog() -} - -func TestYAMLConfig_GetOrDefault(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - assert.Equal("lalala", c.GetOrDefault("vars.oh_lala", "lalala")) -} - -func TestYAMLConfig_GetOrFail(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - _, err = c.GetOrFail("vars.oh_lala") - assert.NotNil(err) -} - -func TestYAMLConfig_Unmarshal(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/structured.yml")) - assert.Nil(err) - var conf structuredConfig - // `vars` is always there even if it is not shown in config, sometimes we want user not to specify any fields we - // didn't use, and use XXX (see above) to capture them, the removeVars flag would remove vars before Unmarshal and put - // it back afterwards - err = c.Unmarshal(&conf, false) - assert.Nil(err) - assert.NotNil(conf.XXX) - assert.Equal("local", conf.Mode) - // test envOr - assert.Equal("base2", conf.Base2) - assert.Equal(os.Getenv("HOME"), conf.Base3) - err = c.Unmarshal(&conf, true) - assert.Nil(err) - assert.Nil(conf.XXX) -} - -func TestYAMLConfig_UnmarshalKey(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/structured.yml")) - assert.Nil(err) - var conf logConfig - err = c.UnmarshalKey("logging", &conf) - assert.Nil(err) - //t.Log(conf) - assert.Equal("info", conf.Level) -} - -func TestSearchMap(t *testing.T) { - assert := asst.New(t) - var m = make(map[string]interface{}) - var m2 = make(map[string]interface{}) - m["xephonk"] = m2 - m2["name"] = "xephonk" - m2["port"] = 8080 - val, err := searchMap(m, []string{"xephonk", "name"}) - assert.Nil(err) - assert.Equal("xephonk", val) - _, err = searchMap(m, []string{"xephonk", "bar"}) - assert.NotNil(err) -} diff --git a/legacy/glide.lock b/legacy/glide.lock deleted file mode 100644 index db8b38a..0000000 --- a/legacy/glide.lock +++ /dev/null @@ -1,33 +0,0 @@ -hash: ee2996db5b78313eadaeb56875b5ebefc2e6aeba666bc3080dc29ebfcc984876 -updated: 2017-09-16T15:58:45.391163291-07:00 -imports: -- name: github.com/kballard/go-shellquote - version: d8ec1a69a250a17bb0e419c386eac1f3711dc142 - vcs: git -- name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d - vcs: git -- name: golang.org/x/net - version: 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 - subpackages: - - proxy -- name: golang.org/x/tools - version: 496819729719f9d07692195e0a94d6edd2251389 - subpackages: - - benchmark/parse -- name: gopkg.in/yaml.v2 - version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b -testImports: -- name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib -- name: github.com/stretchr/testify - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 - vcs: git - subpackages: - - assert diff --git a/legacy/glide.yaml b/legacy/glide.yaml deleted file mode 100644 index aeb612c..0000000 --- a/legacy/glide.yaml +++ /dev/null @@ -1,14 +0,0 @@ -package: github.com/dyweb/gommon -import: -- package: gopkg.in/yaml.v2 -- package: github.com/pkg/errors - vcs: git - version: 0.8.0 -- package: github.com/kballard/go-shellquote - vcs: git - version: d8ec1a69a250a17bb0e419c386eac1f3711dc142 -- package: golang.org/x/tools/benchmark/parse -testImport: -- package: github.com/stretchr/testify - vcs: git - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 diff --git a/legacy/log/README.md b/legacy/log/README.md deleted file mode 100644 index 4ee576c..0000000 --- a/legacy/log/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Log that support filter by field - -This a simple re-implementation of [logrus](https://github.com/sirupsen/logrus) - -## Development - -Generated code - -- https://github.com/benbjohnson/tmpl saw it from InfluxDB, plan to put this in Ayi, or use pongo - - `go get github.com/benbjohnson/tmpl` - - `tmpl -data '["Trace", "Debug", "Info", "Warn", "Error"]' entry_generated.go.tmpl` - -## Added - -- `Filter` and `PkgFilter` -- `TraceLevel` - -## Fixed - -- Remove duplicate code in `Logger` and `Entry` by only allow using `Entry` to log -- Use elasped time in `TextFormatter`, see [issue](https://github.com/sirupsen/logrus/issues/457) -- `FatalLevel` should be more severe than `PanicLevel` - -## Removed - -- lock on logger when call log for `Entry` -- Support for blocking Hook -- Trim `\n` when using `*f` - -## TODO - -- [ ] read filters from command line or config files -- [ ] `WithFields` -- [ ] pool for `Entry` and `bytes.Writer` -- [ ] JSON Formatter -- [ ] Multiple output -- [ ] async Hook -- [ ] Batch write and flush like [zap](https://github.com/uber-go/zap) -- [ ] Shutdown handler - -## DONE - -- thread safe by not using pointer receiver https://github.com/at15/go-learning/issues/3 -- log to stdout, implemented `TextFormatter` -- leveled logging, add `Trace` level -- support field -- filter by field \ No newline at end of file diff --git a/legacy/log/config.example.yml b/legacy/log/config.example.yml deleted file mode 100644 index e21383f..0000000 --- a/legacy/log/config.example.yml +++ /dev/null @@ -1,5 +0,0 @@ -level: info -color: true -source: true -showElapsedTime: false -timeFormat: "2006-01-02T15:04:05Z07:00" \ No newline at end of file diff --git a/legacy/log/config.go b/legacy/log/config.go deleted file mode 100644 index 0ecf7dc..0000000 --- a/legacy/log/config.go +++ /dev/null @@ -1,52 +0,0 @@ -// +build ignore - -package log - -import ( - "time" - - "github.com/pkg/errors" -) - -type Config struct { - Level string `yaml:"level" json:"level"` - Color bool `yaml:"color" json:"color"` - Source bool `yaml:"source" json:"source"` - ShowElapsedTime bool `yaml:"showElapsedTime" json:"showElapsedTime"` - TimeFormat string `yaml:"timeFormat" json:"timeFormat"` - XXX map[string]interface{} `yaml:",inline"` -} - -func (c *Config) Validate() error { - if c.XXX != nil { - return errors.Errorf("undefined fields found %v", c.XXX) - } - if _, err := ParseLevel(c.Level, false); err != nil { - return errors.Wrap(err, "invalid log level in config") - } - // valid time format - t := time.Now() - s := t.Format(c.TimeFormat) - t2, err := time.Parse(c.TimeFormat, s) - if err != nil || t.Unix() != t2.Unix() { - return errors.Wrap(err, "invalid time format string in config") - } - return nil -} - -func (log *Logger) ApplyConfig(c *Config) error { - if err := c.Validate(); err != nil { - return err - } - if err := log.SetLevel(c.Level); err != nil { - return err - } - if c.Source { - log.EnableSourceLine() - } - log.Formatter.SetColor(c.Color) - log.Formatter.SetElapsedTime(c.ShowElapsedTime) - log.Formatter.SetTimeFormat(c.TimeFormat) - // TODO: pkg filter should also be considered - return nil -} diff --git a/legacy/log/config_test.go b/legacy/log/config_test.go deleted file mode 100644 index d885722..0000000 --- a/legacy/log/config_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" - "io/ioutil" -) - -func TestConfig_Validate(t *testing.T) { - assert := asst.New(t) - b := readFixture(t, "config.example.yml") - var c Config - assert.Nil(yaml.Unmarshal(b, &c)) - assert.Nil(c.Validate()) - l := NewLogger() - assert.Nil(l.ApplyConfig(&c)) - e := l.NewEntryWithPkg("pkg1") - e.Info("test config") -} - -// FIXME: this is copied from util library to avoid import cycle -func readFixture(t *testing.T, path string) []byte { - b, err := ioutil.ReadFile(path) - if err != nil { - t.Fatalf("can't read fixture %s: %v", path, err) - } - return b -} diff --git a/legacy/log/doc.go b/legacy/log/doc.go deleted file mode 100644 index 6486c65..0000000 --- a/legacy/log/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build ignore - -/* -Package log(Deprecated) can filter log by field and support multiple level -*/ -package log - -// TODO: https://github.com/fluhus/godoc-tricks diff --git a/legacy/log/entry.go b/legacy/log/entry.go deleted file mode 100644 index 1b15479..0000000 --- a/legacy/log/entry.go +++ /dev/null @@ -1,129 +0,0 @@ -// +build ignore - -package log - -import ( - "fmt" - "os" - "runtime" - "strings" - "time" - - "github.com/pkg/errors" -) - -// Entry is the real logger -type Entry struct { - Logger *Logger - Pkg string // TODO: we should make use of this and stop storing the it in the map field? - EntryLevel Level - Fields Fields - Time time.Time - Level Level - Message string -} - -// SetPkgAlias allows use shorter name for pkg when logging -func (entry *Entry) SetPkgAlias(alias string) { - entry.Fields["pkg"] = alias -} - -func (entry *Entry) SetEntryLevel(s string) error { - newLevel, err := ParseLevel(s, false) - if err != nil { - return errors.WithMessage(err, fmt.Sprintf("can't set logging level to %s", s)) - } - entry.EntryLevel = newLevel - return nil -} - -// AddField adds tag to entry -func (entry *Entry) AddField(key string, value string) { - entry.Fields[key] = value -} - -// AddFields adds multiple tags to entry -func (entry *Entry) AddFields(fields Fields) { - for k, v := range fields { - entry.Fields[k] = v - } -} - -// DeleteField remove a tag from entry, this was added for benchmark to remove the automatically added pkg tag when using RegisterPkg -func (entry *Entry) DeleteField(key string) { - delete(entry.Fields, key) -} - -// This function is not defined with a pointer receiver because we change -// the attribute of struct without using lock, if we use pointer, it would -// become race condition for multiple goroutines. -// see https://github.com/at15/go-learning/issues/3 -func (entry Entry) log(level Level, msg string) bool { - entry.Time = time.Now() - entry.Level = level - entry.Message = msg - // don't log if it can't pass the filter - for _, filter := range entry.Logger.Filters[level] { - if !filter.Accept(&entry) { - return false - } - } - // add source code line if required - if entry.Logger.showSourceLine { - // TODO: what if the user also have tag called source - _, file, line, ok := runtime.Caller(2) - if !ok { - file = "" - line = 1 - } else { - lastSlash := strings.LastIndex(file, "/") - file = file[lastSlash+1:] - } - entry.AddField("source", fmt.Sprintf("%s:%d", file, line)) - } - - serialized, err := entry.Logger.Formatter.Format(&entry) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to serialize, %v\n", err) - return false - } - _, err = entry.Logger.Out.Write(serialized) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to write, %v\n", err) - return false - } - return true -} - -func (entry *Entry) Panic(args ...interface{}) { - if entry.EntryLevel >= PanicLevel { - entry.log(PanicLevel, fmt.Sprint(args...)) - } - panic(fmt.Sprint(args...)) -} - -func (entry *Entry) Fatal(args ...interface{}) { - if entry.EntryLevel >= FatalLevel { - entry.log(FatalLevel, fmt.Sprint(args...)) - } - // TODO: allow register handlers like logrus - os.Exit(1) -} - -// Printf functions -// NOTE: the *f functions does NOT call * functions like logrus does, it just copy and paste - -func (entry *Entry) Panicf(format string, args ...interface{}) { - if entry.EntryLevel >= PanicLevel { - entry.log(PanicLevel, fmt.Sprintf(format, args...)) - } - panic(fmt.Sprint(args...)) -} - -func (entry *Entry) Fatalf(format string, args ...interface{}) { - if entry.EntryLevel >= FatalLevel { - entry.log(FatalLevel, fmt.Sprintf(format, args...)) - } - // TODO: allow register handlers like logrus - os.Exit(1) -} diff --git a/legacy/log/entry_generated.go b/legacy/log/entry_generated.go deleted file mode 100644 index d09832e..0000000 --- a/legacy/log/entry_generated.go +++ /dev/null @@ -1,73 +0,0 @@ -// +build ignore - -// Generated by tmpl -// https://github.com/benbjohnson/tmpl -// -// DO NOT EDIT! -// Source: entry_generated.go.tmpl - -package log - -import ( - "fmt" -) - -func (entry *Entry) Trace(args ...interface{}) { - if entry.EntryLevel >= TraceLevel { - entry.log(TraceLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Tracef(format string, args ...interface{}) { - if entry.EntryLevel >= TraceLevel { - entry.log(TraceLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Debug(args ...interface{}) { - if entry.EntryLevel >= DebugLevel { - entry.log(DebugLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Debugf(format string, args ...interface{}) { - if entry.EntryLevel >= DebugLevel { - entry.log(DebugLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Info(args ...interface{}) { - if entry.EntryLevel >= InfoLevel { - entry.log(InfoLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Infof(format string, args ...interface{}) { - if entry.EntryLevel >= InfoLevel { - entry.log(InfoLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Warn(args ...interface{}) { - if entry.EntryLevel >= WarnLevel { - entry.log(WarnLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Warnf(format string, args ...interface{}) { - if entry.EntryLevel >= WarnLevel { - entry.log(WarnLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Error(args ...interface{}) { - if entry.EntryLevel >= ErrorLevel { - entry.log(ErrorLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Errorf(format string, args ...interface{}) { - if entry.EntryLevel >= ErrorLevel { - entry.log(ErrorLevel, fmt.Sprintf(format, args...)) - } -} diff --git a/legacy/log/entry_generated.go.tmpl b/legacy/log/entry_generated.go.tmpl deleted file mode 100644 index c787bb7..0000000 --- a/legacy/log/entry_generated.go.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -package log - -import ( - "fmt" -) - -{{ range .}} -func (entry *Entry) {{.}}(args ...interface{}) { - if entry.EntryLevel >= {{.}}Level { - entry.log({{.}}Level, fmt.Sprint(args...)) - } -} - -func (entry *Entry) {{.}}f(format string, args ...interface{}) { - if entry.EntryLevel >= {{.}}Level { - entry.log({{.}}Level, fmt.Sprintf(format, args...)) - } -} -{{ end }} \ No newline at end of file diff --git a/legacy/log/entry_test.go b/legacy/log/entry_test.go deleted file mode 100644 index 99c463c..0000000 --- a/legacy/log/entry_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// +build ignore - -package log - -import "testing" - -func TestEntry_log(t *testing.T) { - logger := NewLogger() - entry := logger.NewEntry() - entry.AddField("pkg", "dummy.d") - entry.AddField("name", "jack") - entry.log(DebugLevel, "hahaha") - -} - -// TODO: split this test -func TestEntry_LeveledLog(t *testing.T) { - logger := NewLogger() - f := NewTextFormatter() - f.EnableColor = true - logger.Formatter = f - entry := logger.NewEntry() - // NOTE: when run it in IDEA, the terminal does not have color, run it in real terminal it will show - entry.Debug("You should not see me!") - entry.Infof("%s %d", "haha", 1) - entry.Warnf("%s %d", "haha", 1) - entry.Errorf("%s %d", "haha", 1) - - logger2 := NewLogger() - f2 := NewTextFormatter() - f2.EnableColor = true - f2.EnableElapsedTime = false - logger2.Formatter = f2 - entry2 := logger2.NewEntry() - entry2.Info("I should have full timestamp") - - logger3 := NewLogger() - logger3.Level = DebugLevel - f3 := NewTextFormatter() - f3.EnableColor = false - f3.EnableTimeStamp = false - logger3.Formatter = f3 - entry3 := logger3.NewEntry() - entry3.Info("I should have no timestamp") - entry3.Debug("You should see me!") - - // source code line - logger4 := NewLogger() - logger4.EnableSourceLine() - entry4 := logger4.NewEntry() - entry4.Info("show me source") -} - -func TestEntry_DeleteField(t *testing.T) { - logger := NewLogger() - entry := logger.RegisterPkg() - entry.DeleteField("pkg") - t.Log(len(entry.Fields)) -} diff --git a/legacy/log/filter.go b/legacy/log/filter.go deleted file mode 100644 index 63bd85b..0000000 --- a/legacy/log/filter.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build ignore - -package log - -import ( - st "github.com/dyweb/gommon/structure" -) - -// Filter determines if the entry should be logged -type Filter interface { - Accept(entry *Entry) bool - FilterName() string - FilterDescription() string -} - -var _ Filter = (*PkgFilter)(nil) - -// PkgFilter only allows entry without `pkg` field or `pkg` value in the allow set to pass -// TODO: we should support level -// TODO: a more efficient way might be trie tree and use `/` to divide package into segments instead of using character -type PkgFilter struct { - allow st.Set -} - -// Accept checks if the entry.Pkg (NOT entry.Fields["pkg"]) is in the white list -func (filter *PkgFilter) Accept(entry *Entry) bool { - return filter.allow.Contains(entry.Pkg) -} - -// FilterName implements Filter interface -func (filter *PkgFilter) FilterName() string { - return "PkgFilter" -} - -func (filter *PkgFilter) FilterDescription() string { - return "Filter log based on their pkg tag value, it is logged if it does not have pkg field or in whitelist" -} - -// NewPkgFilter returns a filter that allow log that contains `pkg` filed in the allow set -func NewPkgFilter(allow st.Set) *PkgFilter { - return &PkgFilter{ - allow: allow, - } -} diff --git a/legacy/log/filter_test.go b/legacy/log/filter_test.go deleted file mode 100644 index ce91638..0000000 --- a/legacy/log/filter_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestFilterInterface(t *testing.T) { - t.Parallel() - var _ Filter = (*PkgFilter)(nil) -} - -func TestPkgFilter_Filter(t *testing.T) { - t.Parallel() - assert := asst.New(t) - - allow := make(map[string]bool) - // NOTE: we don't use package name with dot because - // - when we get the package using reflection, we got / - // - when access fields in config, we use dot notation, viper.Get("logging.ayi/app/git") is different with viper.Get("logging.ayi.app.gi") - allow["ayi/app/git"] = true - f := NewPkgFilter(allow) - - entryWithoutPkg := &Entry{} - assert.False(f.Accept(entryWithoutPkg)) - entryWithAllowedPkg := &Entry{Pkg: "ayi/app/git"} - assert.True(f.Accept(entryWithAllowedPkg)) - entryWithDisallowedPkg := &Entry{Pkg: "ayi/app/web"} - assert.False(f.Accept(entryWithDisallowedPkg)) - - // NOTE: we are using entry.Pkg instead of entry.Fields["pkg"] - field := make(map[string]string, 1) - field["pkg"] = "ayi/app/git" - entryWithAllowedPkgInFields := &Entry{Fields: field} - assert.False(f.Accept(entryWithAllowedPkgInFields)) -} diff --git a/legacy/log/formatter.go b/legacy/log/formatter.go deleted file mode 100644 index b9a5e76..0000000 --- a/legacy/log/formatter.go +++ /dev/null @@ -1,110 +0,0 @@ -// +build ignore - -package log - -import ( - "bytes" - "fmt" - "time" -) - -type Formatter interface { - Format(*Entry) ([]byte, error) - SetColor(bool) - SetElapsedTime(bool) - SetTimeFormat(string) -} - -var ( - baseTimeStamp = time.Now() -) - -const ( - nocolor = 0 - red = 31 - green = 32 - yellow = 33 - blue = 34 - gray = 37 -) - -const ( - defaultTimeStampFormat = time.RFC3339 -) - -var _ Formatter = (*TextFormatter)(nil) - -type TextFormatter struct { - EnableColor bool - EnableTimeStamp bool - EnableElapsedTime bool - TimeStampFormat string -} - -func NewTextFormatter() *TextFormatter { - return &TextFormatter{ - EnableColor: false, - EnableTimeStamp: true, - EnableElapsedTime: true, - TimeStampFormat: defaultTimeStampFormat, - } -} - -func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { - var b *bytes.Buffer - // TODO: may use a pool - b = &bytes.Buffer{} - var levelColor = nocolor - if f.EnableColor { - switch entry.Level { - case InfoLevel: - levelColor = blue - case WarnLevel: - levelColor = yellow - case ErrorLevel, FatalLevel, PanicLevel: - levelColor = red - } - } - if levelColor != nocolor { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m", levelColor, entry.Level.ShortUpperString()) - } else { - b.WriteString(entry.Level.ShortUpperString()) - } - if f.EnableTimeStamp { - if f.EnableElapsedTime { - // NOTE: the elapsedTime copied from logrus is wrong - // https://github.com/sirupsen/logrus/pull/465 - fmt.Fprintf(b, "[%04d]", int(entry.Time.Sub(baseTimeStamp)/time.Second)) - } else { - // NOTE: config.Validate would check if timestamp format set by user is valid, and time.Format does not return error - fmt.Fprintf(b, "[%s]", entry.Time.Format(f.TimeStampFormat)) - } - } - b.WriteByte(' ') - b.WriteString(entry.Message) - b.WriteByte(' ') - for k, v := range entry.Fields { - b.WriteString(k) - b.WriteByte('=') - b.WriteString(v) - b.WriteByte(' ') - } - b.WriteByte('\n') - return b.Bytes(), nil -} - -func (f *TextFormatter) SetColor(b bool) { - f.EnableColor = b -} - -func (f *TextFormatter) SetElapsedTime(b bool) { - f.EnableElapsedTime = b -} - -func (f *TextFormatter) SetTimeFormat(tf string) { - if tf == "" { - f.TimeStampFormat = defaultTimeStampFormat - } else { - f.TimeStampFormat = tf - } -} diff --git a/legacy/log/formatter_test.go b/legacy/log/formatter_test.go deleted file mode 100644 index 60bcf63..0000000 --- a/legacy/log/formatter_test.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build ignore - -package log diff --git a/legacy/log/level.go b/legacy/log/level.go deleted file mode 100644 index ea83aa4..0000000 --- a/legacy/log/level.go +++ /dev/null @@ -1,126 +0,0 @@ -// +build ignore - -package log - -import ( - "strings" - - "github.com/pkg/errors" -) - -// Level is log level -type Level uint8 - -const ( - // FatalLevel log error and call `os.Exit(1)` - FatalLevel Level = iota - // PanicLevel log error and call `panic` - PanicLevel - // ErrorLevel log error - ErrorLevel - // WarnLevel log warning - WarnLevel - // InfoLevel log info - InfoLevel - // DebugLevel log debug message, user should enable DebugLevel logging when report bug - DebugLevel - // TraceLevel is very verbose, user should enable it only on packages they are currently investing instead of globally - TraceLevel -) - -// ShortUpperString returns the first 4 characters of a level in upper case -func (level Level) ShortUpperString() string { - switch level { - case FatalLevel: - return "FATA" - case PanicLevel: - return "PANI" - case ErrorLevel: - return "ERRO" - case WarnLevel: - return "WARN" - case InfoLevel: - return "INFO" - case DebugLevel: - return "DEBU" - case TraceLevel: - return "TRAC" - default: - return "UNKN" - } -} - -func (level Level) String() string { - switch level { - case FatalLevel: - return "fatal" - case PanicLevel: - return "panic" - case ErrorLevel: - return "error" - case WarnLevel: - return "warn" - case InfoLevel: - return "info" - case DebugLevel: - return "debug" - case TraceLevel: - return "trace" - default: - return "unknown" - } -} - -// ParseLevel match the level string with Level, it will use strings.HasPrefix in non strict mode -func ParseLevel(s string, strict bool) (Level, error) { - str := strings.ToLower(s) - if strict { - switch str { - case "fatal": - return FatalLevel, nil - case "panic": - return PanicLevel, nil - case "error": - return ErrorLevel, nil - case "warn": - return WarnLevel, nil - case "info": - return InfoLevel, nil - case "debug": - return DebugLevel, nil - case "trace": - return TraceLevel, nil - default: - return Level(250), errors.Errorf("unknown log level %s", str) - } - } - switch { - case strings.HasPrefix(str, "f"): - return FatalLevel, nil - case strings.HasPrefix(str, "p"): - return PanicLevel, nil - case strings.HasPrefix(str, "e"): - return ErrorLevel, nil - case strings.HasPrefix(str, "w"): - return WarnLevel, nil - case strings.HasPrefix(str, "i"): - return InfoLevel, nil - case strings.HasPrefix(str, "d"): - return DebugLevel, nil - case strings.HasPrefix(str, "t"): - return TraceLevel, nil - default: - return Level(250), errors.Errorf("unknown log level %s", str) - } -} - -// AllLevels includes all the logging level -var AllLevels = []Level{ - FatalLevel, - PanicLevel, - ErrorLevel, - WarnLevel, - InfoLevel, - DebugLevel, - TraceLevel, -} diff --git a/legacy/log/level_test.go b/legacy/log/level_test.go deleted file mode 100644 index 6c92167..0000000 --- a/legacy/log/level_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestParseLevel(t *testing.T) { - assert := asst.New(t) - // strict - for _, l := range AllLevels { - s := l.String() - l2, err := ParseLevel(s, true) - assert.Nil(err) - assert.Equal(l, l2) - } - // not strict - levels := []string{"FA", "Pa", "er", "Warn", "infooo", "debugggg", "Tracer"} - for i := 0; i < len(AllLevels); i++ { - l, err := ParseLevel(levels[i], false) - assert.Nil(err) - assert.Equal(AllLevels[i], l) - } - // invalid - _, err := ParseLevel("haha", false) - assert.NotNil(err) -} diff --git a/legacy/log/logger.go b/legacy/log/logger.go deleted file mode 100644 index 1e87697..0000000 --- a/legacy/log/logger.go +++ /dev/null @@ -1,134 +0,0 @@ -// +build ignore - -package log - -import ( - "fmt" - "io" - "os" - "sync" - - "github.com/dyweb/gommon/util/runtimeutil" - "github.com/pkg/errors" -) - -// Fields is key-value string pair to annotate the log and can be used by filter -type Fields map[string]string - -// Logger is used to set output, formatter and filters, the real log operation is in Entry -type Logger struct { - Out io.Writer - Formatter Formatter - Level Level - mu sync.Mutex // FIXME: this mutex is never used, I guess I was following logrus when I wrote this - showSourceLine bool - Filters map[Level]map[string]Filter - Entries map[string]*Entry -} - -// NewLogger returns a new logger using StdOut and InfoLevel -func NewLogger() *Logger { - f := make(map[Level]map[string]Filter, len(AllLevels)) - for _, level := range AllLevels { - f[level] = make(map[string]Filter, 1) - } - l := &Logger{ - Out: os.Stdout, - Formatter: NewTextFormatter(), - Level: InfoLevel, - Filters: f, - Entries: make(map[string]*Entry), - showSourceLine: false, - } - return l -} - -func (log *Logger) SetLevel(s string) error { - newLevel, err := ParseLevel(s, false) - if err != nil { - return errors.WithMessage(err, fmt.Sprintf("can't set logging level to %s", s)) - } - if log.Level == newLevel { - return nil - } - // update all the registered entries - for _, entry := range log.Entries { - // only update entry's level if the new global level is more verbose - // TODO: allow force set level to all registered entries - if newLevel > entry.EntryLevel { - entry.EntryLevel = newLevel - } - } - log.Level = newLevel - return nil -} - -// EnableSourceLine add `source` field when logging, it use runtime.Caller(), the overhead has not been measured -func (log *Logger) EnableSourceLine() { - log.showSourceLine = true -} - -// DisableSourceLine does not show `source` field -func (log *Logger) DisableSourceLine() { - log.showSourceLine = false -} - -// AddFilter add a filter to logger, the filter should be simple string check on fields, i.e. PkgFilter check pkg field -func (log *Logger) AddFilter(filter Filter, level Level) { - log.Filters[level][filter.FilterName()] = filter -} - -// NewEntry returns an Entry with empty Fields -// Deprecated: use RegisterPkg instead -func (log *Logger) NewEntry() *Entry { - // TODO: may use pool, but need benchmark to see if using pool provides improvement - return &Entry{ - Logger: log, - Pkg: "", - EntryLevel: log.Level, - Fields: make(map[string]string, 1), - } -} - -// NewEntryWithPkg returns an Entry with pkg Field set to pkgName, should be used with PkgFilter -// Deprecated: use RegisterPkg instead -func (log *Logger) NewEntryWithPkg(pkgName string) *Entry { - fields := make(map[string]string, 1) - fields["pkg"] = pkgName - e := &Entry{ - Logger: log, - Pkg: pkgName, - EntryLevel: log.Level, - Fields: fields, - } - log.Entries[pkgName] = e - return e -} - -// RegisterPkg creates a new entry with pkg field set to the caller's package and register this entry to logger -func (log *Logger) RegisterPkg() *Entry { - fields := make(map[string]string, 1) - pkg := runtimeutil.GetCallerPackage(2) - fields["pkg"] = pkg - e := &Entry{ - Logger: log, - Pkg: pkg, - EntryLevel: log.Level, - Fields: fields, - } - log.Entries[pkg] = e - return e -} - -func (log *Logger) RegisteredPkgs() map[string]*Entry { - return log.Entries -} - -func (log *Logger) PrintEntries() { - // TODO: sort and print it in a hierarchy - // github.com/dyweb/Ayi/web - // github.com/dyweb/Ayi/web/static - for pkg := range log.Entries { - fmt.Println(pkg) - } -} diff --git a/legacy/log/logger_test.go b/legacy/log/logger_test.go deleted file mode 100644 index d9800ce..0000000 --- a/legacy/log/logger_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - "bytes" - asst "github.com/stretchr/testify/assert" - "io" - "os" -) - -func TestLogger_AddFilter(t *testing.T) { - assert := asst.New(t) - logger := NewLogger() - allow := make(map[string]bool) - pkgFilter := NewPkgFilter(allow) - logger.AddFilter(pkgFilter, DebugLevel) - assert.Equal(1, len(logger.Filters[DebugLevel])) -} - -func TestLogger_NewEntryWithPkg(t *testing.T) { - assert := asst.New(t) - logger := NewLogger() - entry := logger.NewEntryWithPkg("x.dummy") - assert.Equal(1, len(entry.Fields)) - entry.Info("show me the pkg") -} - -func TestLogger_SetLevel(t *testing.T) { - assert := asst.New(t) - - var b bytes.Buffer - writer := io.MultiWriter(os.Stdout, &b) - logger := NewLogger() - logger.Out = writer - entry1 := logger.NewEntryWithPkg("pkg1") - assert.Equal(entry1.EntryLevel, InfoLevel) - entry2 := logger.NewEntryWithPkg("pkg2") - - entry1.Info("should see me") - entry1.Debug("should not see me") - assert.NotContains(b.String(), "not") - - assert.Nil(entry2.SetEntryLevel("trace")) - assert.Nil(logger.SetLevel("debug")) - - entry1.Debug("should see me") - entry1.Trace("should not see me") - entry2.Trace("should see me") - assert.NotContains(b.String(), "not") -} diff --git a/legacy/log/v2.md b/legacy/log/v2.md deleted file mode 100644 index d8bf31c..0000000 --- a/legacy/log/v2.md +++ /dev/null @@ -1,112 +0,0 @@ -# Log v2 - -Design Goals - -- easy to maintain, use standard library as much as possible -- acceptable and easy to use by default, increase performance if really needed -- testable, should not rely on human eyeball to see if the log is correct -- configurable, if a library use gommon/log, then the application using the library can config the logging of that library like the library itself -- out of box, default config struct, http handler (unix sock?), print log entry relations -- not using package level variable, especially pools - -Non Goals - -- sampling - -## Survey - -## log - -https://golang.org/src/log/log.go - -````go -func (l *Logger) Printf(format string, v ...interface{}) { - l.Output(2, fmt.Sprintf(format, v...)) -} - -// Output writes the output for a logging event. The string s contains -// the text to print after the prefix specified by the flags of the -// Logger. A newline is appended if the last character of s is not -// already a newline. Calldepth is used to recover the PC and is -// provided for generality, although at the moment on all pre-defined -// paths it will be 2. -func (l *Logger) Output(calldepth int, s string) error { - // Get time early if we need it. - var now time.Time - if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { - now = time.Now() - } - var file string - var line int - l.mu.Lock() - defer l.mu.Unlock() - if l.flag&(Lshortfile|Llongfile) != 0 { - // Release lock while getting caller info - it's expensive. - l.mu.Unlock() - var ok bool - _, file, line, ok = runtime.Caller(calldepth) - if !ok { - file = "???" - line = 0 - } - l.mu.Lock() - } - l.buf = l.buf[:0] - l.formatHeader(&l.buf, now, file, line) - l.buf = append(l.buf, s...) - if len(s) == 0 || s[len(s)-1] != '\n' { - l.buf = append(l.buf, '\n') - } - _, err := l.out.Write(l.buf) - return err -} -```` - -### Logrus - -https://github.com/sirupsen/logrus - -The first version is entirely modeled after logrus - -common - -- not use pointer receiver on Entry struct to avoid race condition -- using formatter + writer - - introduced extra copy from `fmt.Sprintf`, it is possible to put data into writer directly - - [ ] TODO: will there be race condition causing different log mixed with each other, like `[info][warn]msg1msg2` instead of `[info]msg1 [warn]msg2` - -differences - -- no pool for log entry -- no `*ln` function, just `*` and `*f` `*` means Debug, Info ... - -### apex/log - -- use handler instead of formatter + writer - -````go -// The HandlerFunc type is an adapter to allow the use of ordinary functions as -// log handlers. If f is a function with the appropriate signature, -// HandlerFunc(f) is a Handler object that calls f. -type HandlerFunc func(*Entry) error - -// HandleLog calls f(e). -func (f HandlerFunc) HandleLog(e *Entry) error { - return f(e) -} -```` - -I guess it's modeled after `net/http` - -````go -// The HandlerFunc type is an adapter to allow the use of -// ordinary functions as HTTP handlers. If f is a function -// with the appropriate signature, HandlerFunc(f) is a -// Handler that calls f. -type HandlerFunc func(ResponseWriter, *Request) - -// ServeHTTP calls f(w, r). -func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { -f(w, r) -} -```` \ No newline at end of file diff --git a/legacy/pkg.go b/legacy/pkg.go deleted file mode 100644 index d945b1c..0000000 --- a/legacy/pkg.go +++ /dev/null @@ -1,4 +0,0 @@ -// +build ignore - -// Package legacy contains deprecated gommon packages -package legacy diff --git a/legacy/requests/README.md b/legacy/requests/README.md deleted file mode 100644 index 6a8fa0a..0000000 --- a/legacy/requests/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Requests - -A wrapper around net/http with less public global variables - -Originally from [xephon-b](https://github.com/xephonhq/xephon-b) - -## Usage - -TODO: - -- it was mainly used for building config for http client to use socks5 proxy -- [ ] https://github.com/hashicorp/go-getter like features - -## Changelog - -- use same config as `DefaultTransport` in `net/http` -- add client struct and default client which wraps `http.Client` -- support socks5 proxy https://github.com/dyweb/gommon/issues/18 -- rename `GetJSON` to get `GetJSONStringMap`, this should break old Xephon-B code, which is now in tsdb-proxy \ No newline at end of file diff --git a/legacy/requests/auth.go b/legacy/requests/auth.go deleted file mode 100644 index d99116d..0000000 --- a/legacy/requests/auth.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build ignore - -package requests - -import ( - "encoding/base64" - "net/http" - "strings" - - "github.com/dyweb/gommon/errors" -) - -// NOTE: net/http already implemented it https://golang.org/pkg/net/http/#Request.BasicAuth -const AuthorizationHeader = "Authorization" - -func GenerateBasicAuth(username string, password string) string { - return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)) -} - -func ExtractBasicAuth(val string) (username string, password string, err error) { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization - s := strings.Split(val, " ") - if len(s) != 2 { - err = errors.New("invalid authorization header, must have type and base64 encoded value separated by space") - return - } - tpe := strings.ToLower(s[0]) - if tpe != "basic" { - err = errors.Errorf("got %s instead basic auth", tpe) - return - } - decoded, err := base64.StdEncoding.DecodeString(s[1]) - if err != nil { - err = errors.New("invalid base64 encoding") - return - } - ss := strings.Split(string(decoded), ":") - if len(ss) != 2 { - errors.Errorf("invalid username:password, got %d segments after split by ':'", len(ss)) - } - username = ss[0] - password = ss[1] - err = nil - return -} - -func ExtractBasicAuthFromRequest(r *http.Request) (string, string, error) { - val := r.Header.Get(AuthorizationHeader) - if val == "" { - return "", "", errors.New("Authorization header is not set") - } - return ExtractBasicAuth(val) -} diff --git a/legacy/requests/auth_test.go b/legacy/requests/auth_test.go deleted file mode 100644 index 6623cb4..0000000 --- a/legacy/requests/auth_test.go +++ /dev/null @@ -1,22 +0,0 @@ -// +build ignore - -package requests - -import ( - asst "github.com/stretchr/testify/assert" - - "testing" -) - -func TestExtractBasicAuth(t *testing.T) { - assert := asst.New(t) - username := "Aladdin" - password := "OpenSesame" - // get from Postman - header := "Basic QWxhZGRpbjpPcGVuU2VzYW1l" - assert.Equal(header, GenerateBasicAuth(username, password)) - u, p, err := ExtractBasicAuth(header) - assert.Nil(err) - assert.Equal(username, u) - assert.Equal(password, p) -} diff --git a/legacy/requests/builder.go b/legacy/requests/builder.go deleted file mode 100644 index 3f588f5..0000000 --- a/legacy/requests/builder.go +++ /dev/null @@ -1,69 +0,0 @@ -// +build ignore - -package requests - -import ( - "crypto/tls" - "net/http" - - "golang.org/x/net/proxy" - - "github.com/dyweb/gommon/errors" -) - -// TODO: might switch to functional options https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis -// TODO: http proxy? might just use environment variable? https://stackoverflow.com/questions/14661511/setting-up-proxy-for-http-client -// TransportBuilder is the initial builder, its method use value receiver and return a new copy for chaining and keep itself unchanged -var TransportBuilder transportBuilder - -type transportBuilder struct { - skipKeyVerify bool - socks5Host string - auth *proxy.Auth -} - -func (b transportBuilder) SkipKeyVerify() transportBuilder { - b.skipKeyVerify = true - return b -} - -// UseShadowSocks uses the default config for shadowsocks local -func (b transportBuilder) UseShadowSocks() transportBuilder { - return b.UseSocks5(ShadowSocksLocal, "", "") -} - -func (b transportBuilder) UseSocks5(host string, username string, password string) transportBuilder { - b.socks5Host = host - if username == "" && password == "" { - b.auth = nil - } else { - b.auth.User = username - b.auth.Password = password - } - return b -} - -func (b transportBuilder) Build() (*http.Transport, error) { - transport := NewDefaultTransport() - if b.skipKeyVerify { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - if b.socks5Host != "" { - // NOTE: actually the implementation does not connect at all, the err is always nil - dialer, err := proxy.SOCKS5("tcp", b.socks5Host, b.auth, proxy.Direct) - if err != nil { - return nil, errors.Wrapf(err, "can't create socks5 dialer to %s with %s:%s", b.socks5Host, b.auth.User, b.auth.Password) - } - // TODO: Dial is deprecated and we should use DialContext, maybe contribute to golang/x/net - transport.Dial = dialer.Dial - } - return transport, nil -} - -func init() { - TransportBuilder = transportBuilder{ - skipKeyVerify: false, - socks5Host: "", - auth: &proxy.Auth{}, - } -} diff --git a/legacy/requests/builder_test.go b/legacy/requests/builder_test.go deleted file mode 100644 index d03a6ec..0000000 --- a/legacy/requests/builder_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build ignore - -package requests - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestTransportBuilder_UseSocks5(t *testing.T) { - assert := asst.New(t) - b := TransportBuilder.UseSocks5("127.0.0.1:1080", "", "") - assert.NotNil(TransportBuilder.auth) - assert.Nil(b.auth) -} - -func TestTransportBuilder_Build(t *testing.T) { - assert := asst.New(t) - b := TransportBuilder.UseSocks5("127.0.0.1:1080", "", "") - // NOTE: it does not connect to the proxy, so the test should pass regardless of the running proxy - tr, err := b.Build() - assert.Nil(err) - assert.NotNil(tr) -} diff --git a/legacy/requests/default.go b/legacy/requests/default.go deleted file mode 100644 index 7e5c82e..0000000 --- a/legacy/requests/default.go +++ /dev/null @@ -1,35 +0,0 @@ -// +build ignore - -package requests - -import ( - "net" - "net/http" - "time" -) - -// Default Transport Client that is same as https://golang.org/src/net/http/transport.go -// It's similar to https://github.com/hashicorp/go-cleanhttp - -// NewDefaultTransport is copied from net/http/transport.go -func NewDefaultTransport() *http.Transport { - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } -} - -// NewDefaultClient returns a client using NewDefaultTransport -func NewDefaultClient() *http.Client { - return &http.Client{ - Transport: NewDefaultTransport(), - } -} diff --git a/legacy/requests/doc/history.md b/legacy/requests/doc/history.md deleted file mode 100644 index e69de29..0000000 diff --git a/legacy/requests/pkg.go b/legacy/requests/pkg.go deleted file mode 100644 index 5c5f254..0000000 --- a/legacy/requests/pkg.go +++ /dev/null @@ -1,4 +0,0 @@ -// +build ignore - -// Package requests(Deprecated) is a wrapper around net/http with less public global variables -package requests // import "github.com/dyweb/gommon/requests" diff --git a/legacy/requests/requests.go b/legacy/requests/requests.go deleted file mode 100644 index f678142..0000000 --- a/legacy/requests/requests.go +++ /dev/null @@ -1,128 +0,0 @@ -// +build ignore - -// Package requests wrap net/http like requests did for python -// it is easy to use, but not very efficient -package requests - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "strings" - - "github.com/dyweb/gommon/errors" -) - -const ( - ShadowSocksLocal = "127.0.0.1:1080" - ContentJSON = "application/json" -) - -var defaultClient = &Client{h: NewDefaultClient(), content: ContentJSON} - -type Client struct { - h *http.Client - content string -} - -func NewClient(options ...func(h *http.Client)) *Client { - c := &Client{h: NewDefaultClient(), content: ContentJSON} - for _, option := range options { - option(c.h) - } - return c -} - -func (c *Client) makeRequest(method string, url string, body io.Reader) (*Response, error) { - if c.h == nil { - return nil, errors.New("http client is not initialized") - } - var res *http.Response - var err error - switch method { - case http.MethodGet: - res, err = c.h.Get(url) - case http.MethodPost: - res, err = c.h.Post(url, c.content, body) - } - response := &Response{} - if err != nil { - return response, errors.Wrap(err, "error making request") - } - defer res.Body.Close() - resContent, err := ioutil.ReadAll(res.Body) - if err != nil { - return response, errors.Wrap(err, "error reading body") - } - response.Res = res - response.StatusCode = res.StatusCode - response.Data = resContent - return response, nil -} - -// TODO: this defaultish wrapping methods should be generated by gommon generator -func Post(url string, data io.Reader) (*Response, error) { - return defaultClient.Post(url, data) -} - -func (c *Client) Post(url string, data io.Reader) (*Response, error) { - return c.makeRequest(http.MethodPost, url, data) -} - -func PostString(url string, data string) (*Response, error) { - return defaultClient.PostString(url, data) -} - -func (c *Client) PostString(url string, data string) (*Response, error) { - return c.makeRequest(http.MethodPost, url, ioutil.NopCloser(strings.NewReader(data))) -} - -func PostBytes(url string, data []byte) (*Response, error) { - return defaultClient.PostBytes(url, data) -} - -func (c *Client) PostBytes(url string, data []byte) (*Response, error) { - return c.makeRequest(http.MethodPost, url, ioutil.NopCloser(bytes.NewReader(data))) -} - -func Get(url string) (*Response, error) { - return defaultClient.Get(url) -} - -func (c *Client) Get(url string) (*Response, error) { - return c.makeRequest(http.MethodGet, url, nil) -} - -func GetJSON(url string, data interface{}) error { - return defaultClient.GetJSON(url, data) -} - -func (c *Client) GetJSON(url string, data interface{}) error { - res, err := c.Get(url) - if err != nil { - return errors.Wrap(err, "error getting response") - } - err = res.JSON(data) - if err != nil { - return errors.Wrap(err, "error parsing json") - } - return nil -} - -func GetJSONStringMap(url string) (map[string]string, error) { - return defaultClient.GetJSONStringMap(url) -} - -func (c *Client) GetJSONStringMap(url string) (map[string]string, error) { - var data map[string]string - res, err := c.Get(url) - if err != nil { - return data, errors.Wrap(err, "error getting response") - } - data, err = res.JSONStringMap() - if err != nil { - return data, errors.Wrap(err, "error parsing json") - } - return data, nil -} diff --git a/legacy/requests/requests_test.go b/legacy/requests/requests_test.go deleted file mode 100644 index e69a883..0000000 --- a/legacy/requests/requests_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// +build ignore - -package requests - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestRequestsE2E(t *testing.T) { - if testing.Short() { - t.Skip("skip requests e2e test") - } - - // create an echo server - version := "0.0.1" - mux := http.NewServeMux() - mux.HandleFunc("/echo", func(w http.ResponseWriter, r *http.Request) { - data, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Log("can't read request body") - } - fmt.Fprintf(w, "%s", data) - // The Server will close the request body. The ServeHTTP - // Handler does not need to. - //r.Body.Close() - }) - mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "{\"version\": \"%s\"}", version) - }) - - testServer := httptest.NewServer(mux) - defer testServer.Close() - - versionURL := testServer.URL + "/version" - echoURL := testServer.URL + "/echo" - - c := NewClient() - t.Run("c.Get", func(t *testing.T) { - assert := asst.New(t) - res, err := c.Get(versionURL) - assert.Nil(err) - assert.Equal(http.StatusOK, res.Res.StatusCode) - }) - - t.Run("Get", func(t *testing.T) { - assert := asst.New(t) - res, err := Get(versionURL) - assert.Nil(err) - assert.Equal(http.StatusOK, res.Res.StatusCode) - }) - - t.Run("c.GetJSONStringMap", func(t *testing.T) { - assert := asst.New(t) - data, err := c.GetJSONStringMap(versionURL) - assert.Nil(err) - assert.Equal(version, data["version"]) - }) - - t.Run("GetJSONStringMap", func(t *testing.T) { - assert := asst.New(t) - data, err := GetJSONStringMap(versionURL) - assert.Nil(err) - assert.Equal(version, data["version"]) - }) - - t.Run("c.PostString", func(t *testing.T) { - assert := asst.New(t) - payload := fmt.Sprintf("{\"version\": \"%s\"}", version) - res, err := c.PostString(echoURL, payload) - assert.Nil(err) - assert.Equal(payload, string(res.Data)) - }) - - t.Run("c.PostString", func(t *testing.T) { - assert := asst.New(t) - payload := fmt.Sprintf("{\"version\": \"%s\"}", version) - res, err := PostString(echoURL, payload) - assert.Nil(err) - assert.Equal(payload, string(res.Data)) - }) -} - -func TestNewClient(t *testing.T) { - assert := asst.New(t) - tr, err := TransportBuilder.UseShadowSocks().Build() - assert.Nil(err) - c := NewClient(func(h *http.Client) { - h.Transport = tr - }) - assert.NotNil(c) - // uncomment the following if you have local socks 5 proxy running - //_, err = c.Get("https://google.com") - //t.Log(err) - //assert.Nil(err) -} diff --git a/legacy/requests/response.go b/legacy/requests/response.go deleted file mode 100644 index 171053d..0000000 --- a/legacy/requests/response.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build ignore - -package requests - -import ( - "encoding/json" - "net/http" - - "github.com/dyweb/gommon/errors" -) - -type Response struct { - Res *http.Response - StatusCode int - Data []byte -} - -func (res *Response) JSON(data interface{}) error { - if err := json.Unmarshal(res.Data, &data); err != nil { - return errors.Wrap(err, "error unmarshal json using map[string]string") - } - return nil -} - -func (res *Response) JSONStringMap() (map[string]string, error) { - var data map[string]string - if err := json.Unmarshal(res.Data, &data); err != nil { - return data, errors.Wrap(err, "error unmarshal json using map[string]string") - } - return data, nil -} diff --git a/legacy/runner/README.md b/legacy/runner/README.md deleted file mode 100644 index b13f100..0000000 --- a/legacy/runner/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Command runner - -TODO - -- github.com/kballard/go-shellquote saw it from https://github.com/github/hub -- [ ] https://github.com/mvdan/sh A shell parser, formatter and interpreter (POSIX/Bash/mksh) \ No newline at end of file diff --git a/legacy/runner/command.go b/legacy/runner/command.go deleted file mode 100644 index 00dec2c..0000000 --- a/legacy/runner/command.go +++ /dev/null @@ -1,62 +0,0 @@ -// +build ignore - -package runner - -import ( - "os" - "os/exec" - - st "github.com/dyweb/gommon/structure" - "github.com/kballard/go-shellquote" - "github.com/pkg/errors" -) - -// TODO: possible using https://github.com/mvdan/sh a shell parser, formatter in go -// commands that should be called using shell, -// because mostly they would be expecting shell expansion on parameters -var shellCommands = st.NewSet("echo", "rm", "cp", "mv", "mkdir", "tar") - -// NewCmd can properly split the executable with its arguments -// TODO: may need to add a context to handle things like using shell or not -func NewCmd(cmdStr string) (*exec.Cmd, error) { - segments, err := shellquote.Split(cmdStr) - if err != nil { - return nil, errors.Wrap(err, "can't parse command") - } - return exec.Command(segments[0], segments[1:]...), nil -} - -// NewCmdWithAutoShell automatically use `sh -c` syntax for a small list of executable -// because most people expect shell expansion i.e. wild chars when using them -// TODO: test if it really works, current unit test just test the number of arguments -func NewCmdWithAutoShell(cmdStr string) (*exec.Cmd, error) { - segments, err := shellquote.Split(cmdStr) - if err != nil { - return nil, errors.Wrap(err, "can't parse command") - } - name := segments[0] - useShell := shellCommands.Contains(name) - if useShell { - // TODO: may use shellquote join? - // NOTE: http://stackoverflow.com/questions/18946837/go-variadic-function-and-too-many-arguments - // the `append` here is a must "sh", "-c", segments[1:]... won't work - return exec.Command("sh", append([]string{"-c"}, segments[1:]...)...), nil - } - return exec.Command(segments[0], segments[1:]...), nil -} - -func RunCommand(cmdStr string) error { - cmd, err := NewCmd(cmdStr) - if err != nil { - return errors.Wrap(err, "can't create cmd from command string") - } - // TODO: dry run, maybe add a context - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - return errors.Wrap(err, "failure when executing command") - } - return nil -} diff --git a/legacy/runner/command_test.go b/legacy/runner/command_test.go deleted file mode 100644 index b9fbafa..0000000 --- a/legacy/runner/command_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build ignore - -package runner - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// https://github.com/dyweb/Ayi/issues/58 -// NOTE: it's not extra quote, it's os/exec can't expand * like shell does -// when run `rm *.aux` in shell, shell expands `*.aux` to real file names -func TestNewCommand_ExtraQuote(t *testing.T) { - assert := assert.New(t) - cmd, _ := NewCmd("rm *.aux") - assert.Equal(2, len(cmd.Args)) - assert.Equal("*.aux", cmd.Args[1]) -} - -func TestNewCmdWithAutoShell(t *testing.T) { - assert := assert.New(t) - cmd, _ := NewCmdWithAutoShell("rm *.aux") - assert.Equal(3, len(cmd.Args)) - // TODO: why this is not /bin/sh - assert.Equal("sh", cmd.Args[0]) - assert.Equal("-c", cmd.Args[1]) -} diff --git a/legacy/runner/context.go b/legacy/runner/context.go deleted file mode 100644 index 3f6e137..0000000 --- a/legacy/runner/context.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build ignore - -package runner - -// FIXME: this is never used, and it may not be a good idea to use context, since there is context package -// Context describes how the command should be run -// i.e. use shell instead of using os/exec, -// use self defined logic instead of lookup an executable -// run in background -type Context struct { - AutoShell bool - Foreground bool - Block bool - Dry bool -} - -// NewContext returns a context with convention -func NewContext() *Context { - return &Context{ - AutoShell: true, - Foreground: true, - Block: true, - Dry: false, - } -} diff --git a/legacy/runner/doc.go b/legacy/runner/doc.go deleted file mode 100644 index 98d80cc..0000000 --- a/legacy/runner/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build ignore - -/* -Package runner(Deprecated) run commands with some convention -*/ -package runner diff --git a/legacy/runner/example.config.yml b/legacy/runner/example.config.yml deleted file mode 100644 index 9986828..0000000 --- a/legacy/runner/example.config.yml +++ /dev/null @@ -1,47 +0,0 @@ -# an example of the new runner package config - -# global config for the runner -scripts-config: - # disable autoshell, no longer check for command like rm, mv and switch to sh -c for them - autoshell: false - -# definition of scripts -scripts: - # call gofmt using os/exec - fmt: gofmt . - # will be translated to sh -c "rm *.obj" if autoshell is enabled in global config - clean: rm *.obj - # a command with detail context - clean-plus: - cmd: rm *.obj - # force using sh -c regardless of global autoshell config - shell: true - # allow this command to have error and continue other commands, rm will exit when no file is found - fallible: true - test: - # use array to execute a list of commands one by one - - go install - # you can also use sh -c for force using shell, no extra sh -c will be added around it - - sh -c "go test -v -cover $(glide novendor)" - build: - # call the test command we previously defined - - test - - go install - - gox -output="build/Ayi_{{.OS}}_{{.Arch}}" - # you can mix object into the array and use the attributes you use for a top command - - cmd: rm *.obj - shell: true - fallible: true - serve: - - cmd: Ayi web static - desc: serve static web content - # run in background, all the background commands are run in parallel, since they won't end in a short time - background: true - # restart if the running process exit - restart: true - # TODO: maybe we should not support the number, need to have counter for cmd syntax - number: 1 - - cmd: Ayi web notify - desc: serve notification and display in browser page - background: true - restart: true \ No newline at end of file diff --git a/legacy/runner/pkg.go b/legacy/runner/pkg.go deleted file mode 100644 index 696b9a9..0000000 --- a/legacy/runner/pkg.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build ignore - -package runner diff --git a/legacy/structure/README.md b/legacy/structure/README.md deleted file mode 100644 index d5d8fc1..0000000 --- a/legacy/structure/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Data structure - -## Set - -A non thread safe implementation of basic Set operations - -- Add -- Cardinality -- Contains -- Equal - -## Tree - -StringTree is manly used for printing i.e. directory layout, log entry hierarchy - -### Acknowledgement - -- https://github.com/deckarep/golang-set/ \ No newline at end of file diff --git a/legacy/structure/heap.go b/legacy/structure/heap.go deleted file mode 100644 index 69ffa4f..0000000 --- a/legacy/structure/heap.go +++ /dev/null @@ -1,89 +0,0 @@ -// +build ignore - -package structure - -// https://golang.org/src/container/heap/heap.go -// TODO: it seems the type is called Interface .... heap.Interface and heap.Heap ... - -// IntHeap is simple min int heap -type IntHeap struct { - data []int -} - -func (h *IntHeap) Len() int { - return len(h.data) -} - -func (h *IntHeap) Insert(x int) { - h.data = append(h.data, x) - // bubble up - h.up(len(h.data) - 1) -} - -// Peak returns the root without of removing it -func (h *IntHeap) Peak() int { - // TODO: need ok to return invalid value? - if len(h.data) == 0 { - return -1 - } - return h.data[0] -} - -// Poll return and remove current root -func (h *IntHeap) Poll() int { - if len(h.data) == 0 { - return -1 - } - - t := h.data[0] - if len(h.data) == 1 { - h.data = h.data[:0] - return t - } - h.data[0] = h.data[len(h.data)-1] - h.data = h.data[:len(h.data)-1] // -1 - h.down(0) - return t -} - -func (h *IntHeap) up(i int) { - for { - p := (i - 1) / 2 - if h.data[p] <= h.data[i] { - break - } - // swap - h.data[p], h.data[i] = h.data[i], h.data[p] - // keep going up - i = p - } -} - -func (h *IntHeap) down(i0 int) { - i := i0 - for { - iv := h.data[i] - l := 2*i + 1 - if l >= len(h.data)-1 { - break - } - lv := h.data[l] - r := l + 1 - rv := h.data[r] - // need to pick between left and right, the smaller one should come up - if lv <= rv && iv > lv { - h.data[i] = lv - h.data[l] = iv - i = l - continue - } - if rv <= lv && iv > rv { - h.data[i] = rv - h.data[r] = iv - i = r - continue - } - // iv < rv && iv < lv - break - } -} diff --git a/legacy/structure/heap_test.go b/legacy/structure/heap_test.go deleted file mode 100644 index 6dfad56..0000000 --- a/legacy/structure/heap_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build ignore - -package structure_test - -import ( - "sort" - "testing" - - "github.com/dyweb/gommon/structure" - "github.com/stretchr/testify/assert" -) - -// TODO: assertion on edge cases ... and why GoLand is not having hint? due to dep mode enabled? -func TestIntHeap_Insert(t *testing.T) { - h := structure.IntHeap{} - h.Insert(2) - h.Insert(1) - t.Log(h) - t.Log(h.Poll()) - t.Log(h) - - h2 := structure.IntHeap{} - unsorted := []int{3, 1, 4, 5, 6, -1, 2, 8, 9, 1} - for i := 0; i < len(unsorted); i++ { - h2.Insert(unsorted[i]) - } - var sorted []int - for h2.Len() != 0 { - sorted = append(sorted, h2.Poll()) - } - sort.Ints(unsorted) - assert.Equal(t, unsorted, sorted) -} diff --git a/legacy/structure/pkg.go b/legacy/structure/pkg.go deleted file mode 100644 index 2d25c8b..0000000 --- a/legacy/structure/pkg.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build ignore - -/* -Package structure add some missing common data structures to Golang -*/ -package structure // import "github.com/dyweb/gommon/structure" diff --git a/legacy/structure/priority_queue.go b/legacy/structure/priority_queue.go deleted file mode 100644 index b647f29..0000000 --- a/legacy/structure/priority_queue.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build ignore - -package structure - -// TODO: interface & impl diff --git a/legacy/structure/set.go b/legacy/structure/set.go deleted file mode 100644 index 40bddef..0000000 --- a/legacy/structure/set.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build ignore - -package structure - -// Set is a map with string key and bool value -// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go -type Set map[string]bool - -// NewSet return a pointer of a set using arguments passed to the function -func NewSet(args ...string) *Set { - // TODO: would this length for the map? - m := make(Set, len(args)) - for _, v := range args { - m[v] = true - } - return &m -} - -// Cardinality return the size of the set -func (set *Set) Cardinality() int { - return len(*set) -} - -// Size is an alias for Cardinality -func (set *Set) Size() int { - return len(*set) -} - -// Contains check if a key is presented in the map, it does NOT check the bool value -func (set *Set) Contains(key string) bool { - _, ok := (*set)[key] - return ok -} - -// Add add an element to set regardless of it is already in the set -func (set *Set) Add(key string) { - (*set)[key] = true -} - -// Equal check if two sets have exactly same elements -func (set *Set) Equal(other *Set) bool { - if set.Cardinality() != other.Cardinality() { - return false - } - for key := range *set { - if !other.Contains(key) { - return false - } - } - return true -} diff --git a/legacy/structure/set_test.go b/legacy/structure/set_test.go deleted file mode 100644 index 3427193..0000000 --- a/legacy/structure/set_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build ignore - -package structure - -import ( - "fmt" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestSet_Contains(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.True(s.Contains("a")) - assert.False(s.Contains("d")) -} - -func TestSet_Cardinality(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.Equal(3, s.Cardinality()) -} - -func TestSet_Size(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.Equal(s.Size(), s.Cardinality()) -} - -func TestSet_Equal(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - s2 := NewSet("a") - s3 := NewSet("a") - assert.True(s.Equal(s)) - assert.False(s.Equal(s2)) - assert.True(s2.Equal(s3)) -} - -func ExampleNewSet() { - s := NewSet("a", "b", "c") - fmt.Println(s.Size()) - // Output: 3 -} diff --git a/legacy/structure/tree.go b/legacy/structure/tree.go deleted file mode 100644 index 3496ebd..0000000 --- a/legacy/structure/tree.go +++ /dev/null @@ -1,96 +0,0 @@ -// +build ignore - -package structure - -import ( - "io" - "os" -) - -var ( - vLineStart = []byte("├") // this is in extended ASCII http://www.theasciicode.com.ar/extended-ascii-code/box-drawings-single-line-vertical-right-character-ascii-code-195.html - vLine = []byte("│") // this is not |, it won't have space vertically - corner = []byte("└") - hLine = []byte("── ") // we got a space at the end of hLine - space = []byte(" ") - hSpace = []byte(" ") - nextLine = []byte("\n") -) - -type StringTreeNode struct { - Val string - Children []StringTreeNode -} - -func (tree *StringTreeNode) Append(child StringTreeNode) { - tree.Children = append(tree.Children, child) -} - -func (tree *StringTreeNode) Print() { - tree.PrintTo(os.Stdout) -} - -/* -tree command example output -. -├── benchgraph -│   ├── echart.go -│   ├── echart_test.go -│   ├── fixture -│   │   ├── zap-no-delete-field.txt -│   │   └── zap.txt -│   ├── main.go -│   ├── Makefile -│   ├── tsdb-bench.html -│   └── zap.html -├── gommon -│   └── main.go -└── README.md -*/ - -func (tree *StringTreeNode) PrintTo(w io.Writer) { - //treePrintHelper(tree, 0, true, false, w) - treePrintHelper(tree, 0, true, true, w) -} - -func treePrintHelper(tree *StringTreeNode, level int, lastOfParent bool, lastOfUs bool, w io.Writer) { - // print the prefix before me, both vertically and horizontally - for i := 0; i < level-2; i++ { - w.Write(vLine) - w.Write(hSpace) - } - // avoid the extra vertical line, we use level-2 and handle level-1 here - // main - // └── http - // │ └── auth - if level > 1 { - if lastOfParent { - w.Write(space) - w.Write(hSpace) - } else { - w.Write(vLine) - w.Write(hSpace) - } - } - // ├── or └── - if level != 0 { - if !lastOfUs { - w.Write(vLineStart) - } else { - w.Write(corner) - } - w.Write(hLine) - } - // my value - w.Write([]byte(tree.Val)) - w.Write(nextLine) - // children - level++ - n := len(tree.Children) - for i := 0; i < n-1; i++ { - treePrintHelper(&tree.Children[i], level, lastOfUs, false, w) - } - if n > 0 { - treePrintHelper(&tree.Children[n-1], level, lastOfUs, true, w) - } -} diff --git a/legacy/structure/tree_test.go b/legacy/structure/tree_test.go deleted file mode 100644 index 6f446b8..0000000 --- a/legacy/structure/tree_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build ignore - -package structure - -import ( - "bytes" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestStringTreeNode_PrintTo(t *testing.T) { - assert := asst.New(t) - expected := - `root -├── level1-A -│ └── level2-A -└── level1-B -` - root := StringTreeNode{Val: "root"} - root.Append(StringTreeNode{Val: "level1-A"}) - root.Children[0].Append(StringTreeNode{Val: "level2-A"}) - root.Append(StringTreeNode{Val: "level1-B"}) - var b bytes.Buffer - root.PrintTo(&b) - assert.Equal(expected, string(b.Bytes())) - - // FIXED: there were extra vertical lines - // main - // └── http - // │ └── auth - root.Children = root.Children[0:1] - expected = - `root -└── level1-A - └── level2-A -` - b.Reset() - root.PrintTo(&b) - assert.Equal(expected, string(b.Bytes())) - - // TODO: test more complex situation like this - //. - //├── benchgraph - //│   ├── echart.go - //│   ├── echart_test.go - //│ │ └── zap-no-delete-field.txt - //│ │ - //│   └── fixture - //│      ├── zap-no-delete-field.txt - //│      └── zap.txt - //├── gommon - //│   └── main.go - //└── README.md -}