From 39b9fe5b640999c859f616ab4eee81538dc8a6e2 Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 28 Apr 2019 13:16:30 -0700 Subject: [PATCH] Fix #112 Remove legacy package - moved to https://github.com/dyweb/gommon-legacy --- legacy/config/README.md | 65 ---- legacy/config/gommon.yml | 4 - legacy/config/gommon_generated.go | 19 - legacy/config/pkg.go | 27 -- .../config/testdata/multi_doc_multi_vars.yml | 31 -- .../config/testdata/multi_doc_single_vars.yml | 14 - legacy/config/testdata/single_doc_no_vars.yml | 2 - legacy/config/testdata/single_doc_vars.yml | 3 - legacy/config/testdata/structured.yml | 8 - legacy/config/yaml.go | 324 ------------------ legacy/config/yaml_test.go | 221 ------------ legacy/glide.lock | 33 -- legacy/glide.yaml | 14 - legacy/log/README.md | 47 --- legacy/log/config.example.yml | 5 - legacy/log/config.go | 52 --- legacy/log/config_test.go | 32 -- legacy/log/doc.go | 8 - legacy/log/entry.go | 129 ------- legacy/log/entry_generated.go | 73 ---- legacy/log/entry_generated.go.tmpl | 19 - legacy/log/entry_test.go | 59 ---- legacy/log/filter.go | 44 --- legacy/log/filter_test.go | 39 --- legacy/log/formatter.go | 110 ------ legacy/log/formatter_test.go | 3 - legacy/log/level.go | 126 ------- legacy/log/level_test.go | 30 -- legacy/log/logger.go | 134 -------- legacy/log/logger_test.go | 53 --- legacy/log/v2.md | 112 ------ legacy/pkg.go | 4 - legacy/requests/README.md | 19 - legacy/requests/auth.go | 53 --- legacy/requests/auth_test.go | 22 -- legacy/requests/builder.go | 69 ---- legacy/requests/builder_test.go | 25 -- legacy/requests/default.go | 35 -- legacy/requests/doc/history.md | 0 legacy/requests/pkg.go | 4 - legacy/requests/requests.go | 128 ------- legacy/requests/requests_test.go | 101 ------ legacy/requests/response.go | 31 -- legacy/runner/README.md | 6 - legacy/runner/command.go | 62 ---- legacy/runner/command_test.go | 28 -- legacy/runner/context.go | 25 -- legacy/runner/doc.go | 6 - legacy/runner/example.config.yml | 47 --- legacy/runner/pkg.go | 3 - legacy/structure/README.md | 18 - legacy/structure/heap.go | 89 ----- legacy/structure/heap_test.go | 33 -- legacy/structure/pkg.go | 6 - legacy/structure/priority_queue.go | 5 - legacy/structure/set.go | 51 --- legacy/structure/set_test.go | 45 --- legacy/structure/tree.go | 96 ------ legacy/structure/tree_test.go | 55 --- 59 files changed, 2906 deletions(-) delete mode 100644 legacy/config/README.md delete mode 100644 legacy/config/gommon.yml delete mode 100644 legacy/config/gommon_generated.go delete mode 100644 legacy/config/pkg.go delete mode 100644 legacy/config/testdata/multi_doc_multi_vars.yml delete mode 100644 legacy/config/testdata/multi_doc_single_vars.yml delete mode 100644 legacy/config/testdata/single_doc_no_vars.yml delete mode 100644 legacy/config/testdata/single_doc_vars.yml delete mode 100644 legacy/config/testdata/structured.yml delete mode 100644 legacy/config/yaml.go delete mode 100644 legacy/config/yaml_test.go delete mode 100644 legacy/glide.lock delete mode 100644 legacy/glide.yaml delete mode 100644 legacy/log/README.md delete mode 100644 legacy/log/config.example.yml delete mode 100644 legacy/log/config.go delete mode 100644 legacy/log/config_test.go delete mode 100644 legacy/log/doc.go delete mode 100644 legacy/log/entry.go delete mode 100644 legacy/log/entry_generated.go delete mode 100644 legacy/log/entry_generated.go.tmpl delete mode 100644 legacy/log/entry_test.go delete mode 100644 legacy/log/filter.go delete mode 100644 legacy/log/filter_test.go delete mode 100644 legacy/log/formatter.go delete mode 100644 legacy/log/formatter_test.go delete mode 100644 legacy/log/level.go delete mode 100644 legacy/log/level_test.go delete mode 100644 legacy/log/logger.go delete mode 100644 legacy/log/logger_test.go delete mode 100644 legacy/log/v2.md delete mode 100644 legacy/pkg.go delete mode 100644 legacy/requests/README.md delete mode 100644 legacy/requests/auth.go delete mode 100644 legacy/requests/auth_test.go delete mode 100644 legacy/requests/builder.go delete mode 100644 legacy/requests/builder_test.go delete mode 100644 legacy/requests/default.go delete mode 100644 legacy/requests/doc/history.md delete mode 100644 legacy/requests/pkg.go delete mode 100644 legacy/requests/requests.go delete mode 100644 legacy/requests/requests_test.go delete mode 100644 legacy/requests/response.go delete mode 100644 legacy/runner/README.md delete mode 100644 legacy/runner/command.go delete mode 100644 legacy/runner/command_test.go delete mode 100644 legacy/runner/context.go delete mode 100644 legacy/runner/doc.go delete mode 100644 legacy/runner/example.config.yml delete mode 100644 legacy/runner/pkg.go delete mode 100644 legacy/structure/README.md delete mode 100644 legacy/structure/heap.go delete mode 100644 legacy/structure/heap_test.go delete mode 100644 legacy/structure/pkg.go delete mode 100644 legacy/structure/priority_queue.go delete mode 100644 legacy/structure/set.go delete mode 100644 legacy/structure/set_test.go delete mode 100644 legacy/structure/tree.go delete mode 100644 legacy/structure/tree_test.go diff --git a/legacy/config/README.md b/legacy/config/README.md deleted file mode 100644 index 648d0b7..0000000 --- a/legacy/config/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Config - -Yaml configuration with template, inspired by [Ansible](http://docs.ansible.com/ansible/playbooks.html) and [Viper](https://github.com/spf13/viper) - -## Usage - -- [ ] TODO - -## Specification - -- using [golang's template](https://golang.org/pkg/text/template/) syntax, not ansible, not pongo2 -- only support single file, but can have multiple documents separated by `---` -- environment variables - - use `env` function, i.e. `home: {{ env "HOME"}}` -- variables - - special top level key `vars` is used via `var` function - - `vars` in different documents are merged, while for other top level keys, their value is replaced by latter documents - - `vars` in current document can be used in current document, we actually render the template of each document twice, so in vars - section you can use previous `vars` and all the syntax supported by golang's template -- [ ] condition (not tested, but golang template should support it) -- loop - - use the `range` syntax - -Using variables - -- [ ] TODO: will `xkb --target={{ $name }} --port={{ $db.port }}` have undesired blank and/or new line? - -````yaml -vars: - influxdb_port: 8080 - databases: - - influxdb - - kairosdb -influxdb: - port: {{ var "influxdb_port" }} -kairosdb: - port: {{ env "KAIROSDB_PORT" }} -tests: -{{ range $name := var "databases" -}} -{{ $db := var $name -}} - - xkb --target={{ $name }} --port={{ $db.port }} -{{ end } -base: {{ env "HOME" }} -```` - -Using multiple document - - - -- [ ] TODO: example -- [ ] TODO: the example in runner actually does not require any template features - -## YAML parsing problem - -- multiple document support, the author said it is easy, but he never tried to solve them, - - the [encoder & decoder PR](https://github.com/go-yaml/yaml/pull/163/) seems to support multiple documents but I don't -understand why it does. - - Actually it would be interesting to look into how to unmarshal stream data since that is needed for Xephon-B, -though for most formats, go already have encoder and decoder - - (Adopted) A easier way is to split the whole file by `---` before put it into go-yaml - -## Acknowledgement - -- https://github.com/spf13/viper -- https://github.com/go-yaml/yaml diff --git a/legacy/config/gommon.yml b/legacy/config/gommon.yml deleted file mode 100644 index a8f6375..0000000 --- a/legacy/config/gommon.yml +++ /dev/null @@ -1,4 +0,0 @@ -# configuration for gommon -loggers: - - struct: "*YAMLConfig" - receiver: c \ No newline at end of file diff --git a/legacy/config/gommon_generated.go b/legacy/config/gommon_generated.go deleted file mode 100644 index 9324c91..0000000 --- a/legacy/config/gommon_generated.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build ignore - -// Code generated by gommon from config/gommon.yml DO NOT EDIT. - -package config - -import dlog "github.com/dyweb/gommon/log" - -func (c *YAMLConfig) SetLogger(logger *dlog.Logger) { - c.log = logger -} - -func (c *YAMLConfig) GetLogger() *dlog.Logger { - return c.log -} - -func (c *YAMLConfig) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { - return justCallMe() -} diff --git a/legacy/config/pkg.go b/legacy/config/pkg.go deleted file mode 100644 index bdac9bd..0000000 --- a/legacy/config/pkg.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build ignore - -// Package config(Deprecated) supports go text/template, environment and self defined variables -package config - -import ( - "github.com/dyweb/gommon/util/logutil" -) - -var log = logutil.NewPackageLogger() - -const ( - yamlDocumentSeparator = "---" - defaultTemplateName = "gommon yaml" - defaultKeyDelimiter = "." -) - -type Path string - -type Reader interface { - Path() Path - Content() string -} - -type StructuredConfig interface { - Validate() error -} diff --git a/legacy/config/testdata/multi_doc_multi_vars.yml b/legacy/config/testdata/multi_doc_multi_vars.yml deleted file mode 100644 index 7b5d7cb..0000000 --- a/legacy/config/testdata/multi_doc_multi_vars.yml +++ /dev/null @@ -1,31 +0,0 @@ -vars: - databases: - - influxdb - - kairosdb - # defined in another document - - xephonk - influxdb: - backend: tsm - language: golang - kairosdb: - backend: cassandra - language: java - foo1: bar1 -foo: 1 ---- -vars: -# NOTE: when merging vars, it's a shallow merge, we merge by top level key under vars, so we can't have -# databases: -# - xephonk - xephonk: - backend: disk - language: golang - foo2: bar2 -{{ range $name := var "databases" }} -{{ $name }}: -{{ $db := var $name }} - name: {{ $name }} - backend: {{ $db.backend }} - language: {{ $db.language }} -{{ end }} -foo: 2 \ No newline at end of file diff --git a/legacy/config/testdata/multi_doc_single_vars.yml b/legacy/config/testdata/multi_doc_single_vars.yml deleted file mode 100644 index 5c72b5a..0000000 --- a/legacy/config/testdata/multi_doc_single_vars.yml +++ /dev/null @@ -1,14 +0,0 @@ -vars: - influxdb_port: 8081 - databases: - - influxdb - - kairosdb - ssl: false -foo: 1 ---- -{{ range $name := var "databases" }} -{{ $name }}: - name: {{ $name }} - ssl: {{ var "ssl" }} -{{ end }} -foo: 2 \ No newline at end of file diff --git a/legacy/config/testdata/single_doc_no_vars.yml b/legacy/config/testdata/single_doc_no_vars.yml deleted file mode 100644 index 1d67c5f..0000000 --- a/legacy/config/testdata/single_doc_no_vars.yml +++ /dev/null @@ -1,2 +0,0 @@ -home: {{ env "HOME" }} -foo: bar \ No newline at end of file diff --git a/legacy/config/testdata/single_doc_vars.yml b/legacy/config/testdata/single_doc_vars.yml deleted file mode 100644 index e3e5708..0000000 --- a/legacy/config/testdata/single_doc_vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -vars: - foo: bar -foo: {{ var "foo" }} diff --git a/legacy/config/testdata/structured.yml b/legacy/config/testdata/structured.yml deleted file mode 100644 index c958caf..0000000 --- a/legacy/config/testdata/structured.yml +++ /dev/null @@ -1,8 +0,0 @@ -logging: - level: info - color: true - source: false -mode: local -base: {{ env "HOME" }} -base2: {{ envOr "NOBODY_HAVE_SET_ME" "base2" }} -base3: {{ envOr "HOME" "/home/gommon" }} \ No newline at end of file diff --git a/legacy/config/yaml.go b/legacy/config/yaml.go deleted file mode 100644 index 2893e6c..0000000 --- a/legacy/config/yaml.go +++ /dev/null @@ -1,324 +0,0 @@ -// +build ignore - -package config - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "reflect" - "strings" - "sync" - "text/template" - - "gopkg.in/yaml.v2" - - "github.com/dyweb/gommon/errors" - dlog "github.com/dyweb/gommon/log" - "github.com/dyweb/gommon/util" - "github.com/dyweb/gommon/util/cast" -) - -// YAMLConfig is a thread safe struct for parse YAML file and get value -type YAMLConfig struct { - vars map[string]interface{} - data map[string]interface{} - keyDelimiter string - mu sync.RWMutex - log *dlog.Logger -} - -func LoadYAMLDirect(file string, cfg interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - if err := yaml.Unmarshal(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml") - } - return nil -} - -func LoadYAMLDirectFrom(r io.Reader, cfg interface{}) error { - b, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "can't read reader") - } - if err := yaml.Unmarshal(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml") - } - return nil -} - -func LoadYAMLDirectStrict(file string, cfg interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - if err := yaml.UnmarshalStrict(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml in strict mode") - } - return nil -} - -func LoadYAMLDirectFromStrict(r io.Reader, cfg interface{}) error { - b, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "can't read reader") - } - if err := yaml.UnmarshalStrict(b, cfg); err != nil { - return errors.Wrap(err, "can't parse yaml in strict mode") - } - return nil -} - -// LoadYAMLAsStruct is a convenient wrapper for loading a single YAML file into struct, you should pass a pointer to the -// struct as second argument. It will remove the vars section. -func LoadYAMLAsStruct(file string, structuredConfig interface{}) error { - b, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "can't read config file %s", file) - } - c := NewYAMLConfig() - if err := c.ParseSingleDocument(b); err != nil { - return errors.Wrap(err, "can't parse as single document") - } - if err := c.Unmarshal(structuredConfig, true); err != nil { - return errors.Wrapf(err, "can't turn file %s into structured config", file) - } - return nil -} - -// SplitMultiDocument splits a yaml file that contains multiple documents and (only) trim the first one if it is empty -func SplitMultiDocument(data []byte) [][]byte { - docs := bytes.Split(data, []byte(yamlDocumentSeparator)) - // check the first one, it could be empty - if len(docs[0]) != 0 { - return docs - } - return docs[1:] -} - -// NewYAMLConfig returns a config with internal map structure initialized -func NewYAMLConfig() *YAMLConfig { - c := &YAMLConfig{} - c.log = dlog.NewStructLogger(log, c) - c.clear() - c.keyDelimiter = defaultKeyDelimiter - return c -} - -// clear is used by test for using one config object for several tests -// and is also used by constructor -func (c *YAMLConfig) clear() { - c.vars = make(map[string]interface{}) - c.data = make(map[string]interface{}) -} - -func (c *YAMLConfig) Var(name string) interface{} { - return c.vars[name] -} - -func (c *YAMLConfig) Env(name string) string { - return os.Getenv(name) -} - -func (c *YAMLConfig) EnvOr(name string, defaultVal string) string { - val := os.Getenv(name) - if val == "" { - return defaultVal - } - return val -} - -func (c *YAMLConfig) FuncMaps() template.FuncMap { - return template.FuncMap{ - "env": c.Env, - "envOr": c.EnvOr, - "var": c.Var, - } -} - -func (c *YAMLConfig) ParseMultiDocument(data []byte) error { - // split the doc, parse by order, add result to context so the following parser can use it - docs := SplitMultiDocument(data) - for _, doc := range docs { - err := c.ParseSingleDocument(doc) - if err != nil { - return errors.Wrap(err, "error when parsing one of the documents") - } - } - return nil -} - -func (c *YAMLConfig) ParseSingleDocument(doc []byte) error { - c.mu.Lock() - defer c.mu.Unlock() - - // we render the template twice, first time we use vars from previous documents and environment variables - // second time, we use vars declared in this document, if any. - - // this is the first render - rendered, err := c.renderDocument(doc) - if err != nil { - return errors.Wrap(err, "can't render template with previous documents' vars") - } - - c.log.Tracef("01-before\n%s", doc) - c.log.Tracef("01-after\n%s", rendered) - - tmpData := make(map[string]interface{}) - err = yaml.Unmarshal(rendered, &tmpData) - if err != nil { - return errors.Wrap(err, "can't parse rendered template yaml to map[string]interface{} after first render") - } - - // preserve the vars - if varsInCurrentDocument, hasVars := tmpData["vars"]; hasVars { - // NOTE: it's map[interface{}]interface{} instead of map[string]interface{} - // because how go-yaml handle the decoding - varsI, ok := varsInCurrentDocument.(map[interface{}]interface{}) - if !ok { - // TODO: test this, it seems if ok == false, then go-yaml should return already, which means ok should always be true? - return errors.Errorf("unable to cast %s to map[interface{}]interface{}", reflect.TypeOf(varsInCurrentDocument)) - } - vars := cast.ToStringMap(varsI) - util.MergeStringMap(c.vars, vars) - - // render again using vars in current document - rendered, err = c.renderDocument(doc) - if err != nil { - return errors.Wrap(err, "can't render template with vars in current document") - } - - c.log.Tracef("02-before\n%s", doc) - c.log.Tracef("02-after\n%s", rendered) - - tmpData = make(map[string]interface{}) - err = yaml.Unmarshal(rendered, &tmpData) - if err != nil { - return errors.Wrap(err, "can't parse rendered template yaml to map[string]interface{} after second render") - } - } - - // put the data into c.data - for k, v := range tmpData { - c.data[k] = v - } - // NOTE: vars are merged instead of overwritten like other top level keys - c.data["vars"] = c.vars - return nil -} - -func (c *YAMLConfig) Get(key string) interface{} { - val, err := c.GetOrFail(key) - if err != nil { - // TODO: so Get does not do any error handling? what did viper do? - c.log.Debugf("can't get key %s due to %s", key, err.Error()) - } - return val -} - -func (c *YAMLConfig) GetOrDefault(key string, defaultVal interface{}) interface{} { - val, err := c.GetOrFail(key) - if err != nil { - c.log.Debugf("use default %v for key %s due to %s", defaultVal, key, err.Error()) - return defaultVal - } - return val -} - -func (c *YAMLConfig) GetOrFail(key string) (interface{}, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - path := strings.Split(key, c.keyDelimiter) - val, err := searchMap(c.data, path) - if err != nil { - return nil, err - } - return val, nil -} - -func (c *YAMLConfig) Unmarshal(structuredConfig interface{}, removeVars bool) error { - c.mu.Lock() - defer c.mu.Unlock() - - // TODO: maybe we no longer need to keep vars in data? - if removeVars { - delete(c.data, "vars") - defer func() { c.data["vars"] = c.vars }() - } - out, err := yaml.Marshal(c.data) - // FIXME: those Wrapf should be Wrap - if err != nil { - return errors.Wrapf(err, "can't marshal data back to yaml") - } - err = yaml.Unmarshal(out, structuredConfig) - if err != nil { - return errors.Wrapf(err, "can't unmarshal data to specified config struct") - } - return nil -} - -func (c *YAMLConfig) UnmarshalKey(key string, structuredConfig interface{}) error { - val, err := c.GetOrFail(key) - if err != nil { - return errors.Wrapf(err, "can't get the val for unmarshal") - } - out, err := yaml.Marshal(val) - if err != nil { - return errors.Wrapf(err, "can't marshal data back to yaml") - } - err = yaml.Unmarshal(out, structuredConfig) - if err != nil { - return errors.Wrapf(err, "can't unmarshal data to specified config struct") - } - return nil -} - -// TODO: what if the user use key that includes `.`, like {"github.com": 123} -func searchMap(src map[string]interface{}, path []string) (interface{}, error) { - var result interface{} - if len(path) == 0 { - return result, errors.New("path is empty, at least provide one segment") - } - result = src - previousPath := "" - for i := 0; i < len(path); i++ { - key := path[i] - //c.log.Debug(key) - //c.log.Debug(result) - if reflect.TypeOf(result).Kind() != reflect.Map { - return nil, errors.Errorf("%s is not a map but %s, %v", previousPath, reflect.TypeOf(result), result) - } - m, ok := result.(map[string]interface{}) - if !ok { - m = cast.ToStringMap(result.(map[interface{}]interface{})) - } - // FIXED: this is a tricky problem, if you use `:` here, you create a new local variable instead update the one - // outside the loop, that's all the Debug(result) for - //result, ok := m[key] - result, ok = m[key] - if !ok { - return result, errors.Errorf("key: %s does not exists in path: %s, val: %v", key, previousPath, m) - } - //c.log.Debug(result) - previousPath += key - } - return result, nil -} - -func (c *YAMLConfig) renderDocument(tplBytes []byte) ([]byte, error) { - tmpl, err := template.New(defaultTemplateName).Funcs(c.FuncMaps()).Parse(string(tplBytes[:])) - if err != nil { - return nil, errors.Wrapf(err, "can't parse template") - } - var b bytes.Buffer - err = tmpl.Execute(&b, "") - if err != nil { - return nil, errors.Wrapf(err, "can't render template") - } - return b.Bytes(), nil -} diff --git a/legacy/config/yaml_test.go b/legacy/config/yaml_test.go deleted file mode 100644 index 8d5b5df..0000000 --- a/legacy/config/yaml_test.go +++ /dev/null @@ -1,221 +0,0 @@ -// +build ignore - -package config - -import ( - "os" - "testing" - - dlog "github.com/dyweb/gommon/log" - "github.com/dyweb/gommon/util/testutil" - asst "github.com/stretchr/testify/assert" -) - -// Deprecated this only works for old logging package -type logConfig struct { - Level string `yaml:"level"` - Color bool `yaml:"color"` - Source bool `yaml:"source"` -} - -type structuredConfig struct { - Logging logConfig `yaml:"logging"` - Mode string `yaml:"mode"` - Base string `yaml:"base"` - Base2 string `yaml:"base2"` - Base3 string `yaml:"base3"` - XXX map[string]interface{} `yaml:",inline"` // NOTE: this is used to catch unmatched fields -} - -func TestNewYAMLConfig(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - assert.Equal(c.GetLogger().Level(), dlog.InfoLevel) - assert.Equal(c.GetLogger().Identity().Type, dlog.StructLogger) - assert.Equal(c.GetLogger().Identity().Struct, "YAMLConfig") - assert.Equal(c.GetLogger().Identity().Package, "github.com/dyweb/gommon/config") -} - -func TestYAMLConfig_ParseWithoutTemplate(t *testing.T) { - assert := asst.New(t) - var dat = ` -a: Easy! -b: - c: 2 - d: [3, 4] -` - c := NewYAMLConfig() - err := c.ParseMultiDocument([]byte(dat)) - assert.Nil(err) - - // NOTE： this is invalid yaml because when you use ` syntax to declare long string in Golang, - // the indent are also included in the string, so this yaml has indent without any parent, which is invalid - var invalidDat = ` - a: Easy! - b: - c: 2 - d: [3, 4] - ` - // the print should show you the string has indent you may not be expecting - //log.Print(invalidDat) - err = c.ParseMultiDocument([]byte(invalidDat)) - assert.NotNil(err) - //log.Print(err.Error()) -} - -func TestLoadYAMLAsStruct(t *testing.T) { - assert := asst.New(t) - var conf structuredConfig - assert.Nil(LoadYAMLAsStruct("testdata/structured.yml", &conf)) -} - -func TestSplitMultiDocument(t *testing.T) { - assert := asst.New(t) - var multi = `--- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -` - documents := SplitMultiDocument([]byte(multi)) - //for _, d := range documents { - // t.Log(string(d[:])) - //} - assert.Equal(2, len(documents)) - documents = SplitMultiDocument([]byte("---")) - assert.Equal(1, len(documents)) - // without the starting `---` - var multi2 = ` -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -` - documents = SplitMultiDocument([]byte(multi2)) - assert.Equal(2, len(documents)) -} - -func TestYAMLConfig_ParseSingleDocument(t *testing.T) { - cases := []struct { - file string - }{ - {"single_doc_no_vars"}, - {"single_doc_vars"}, - } - c := NewYAMLConfig() - //c.GetLogger().SetLevel(dlog.TraceLevel) - for _, tc := range cases { - t.Run(tc.file, func(t *testing.T) { - assert := asst.New(t) - c.clear() - doc := testutil.ReadFixture(t, "testdata/"+tc.file+".yml") - assert.Nil(c.ParseSingleDocument(doc)) - // TODO: expect value, not just log - t.Log(c.data) - }) - } -} - -func TestYAMLConfig_ParseMultiDocument(t *testing.T) { - cases := []struct { - file string - }{ - {"multi_doc_single_vars"}, - {"multi_doc_multi_vars"}, - } - c := NewYAMLConfig() - //util.UseVerboseLog() - for _, tc := range cases { - t.Run(tc.file, func(t *testing.T) { - assert := asst.New(t) - c.clear() - doc := testutil.ReadFixture(t, "testdata/"+tc.file+".yml") - assert.Nil(c.ParseMultiDocument(doc)) - t.Log(c.data) - }) - } - //util.DisableVerboseLog() -} - -func TestYAMLConfig_Get(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - //util.UseVerboseLog() - assert.Equal("bar1", c.Get("vars.foo1")) - assert.Equal(nil, c.Get("vars.that_does_not_exists")) - // NOTE: top level keys other than vars are overwritten instead of merged - assert.Equal(2, c.Get("foo")) - //util.DisableVerboseLog() -} - -func TestYAMLConfig_GetOrDefault(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - assert.Equal("lalala", c.GetOrDefault("vars.oh_lala", "lalala")) -} - -func TestYAMLConfig_GetOrFail(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/multi_doc_multi_vars.yml")) - assert.Nil(err) - _, err = c.GetOrFail("vars.oh_lala") - assert.NotNil(err) -} - -func TestYAMLConfig_Unmarshal(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/structured.yml")) - assert.Nil(err) - var conf structuredConfig - // `vars` is always there even if it is not shown in config, sometimes we want user not to specify any fields we - // didn't use, and use XXX (see above) to capture them, the removeVars flag would remove vars before Unmarshal and put - // it back afterwards - err = c.Unmarshal(&conf, false) - assert.Nil(err) - assert.NotNil(conf.XXX) - assert.Equal("local", conf.Mode) - // test envOr - assert.Equal("base2", conf.Base2) - assert.Equal(os.Getenv("HOME"), conf.Base3) - err = c.Unmarshal(&conf, true) - assert.Nil(err) - assert.Nil(conf.XXX) -} - -func TestYAMLConfig_UnmarshalKey(t *testing.T) { - assert := asst.New(t) - c := NewYAMLConfig() - err := c.ParseMultiDocument(testutil.ReadFixture(t, "testdata/structured.yml")) - assert.Nil(err) - var conf logConfig - err = c.UnmarshalKey("logging", &conf) - assert.Nil(err) - //t.Log(conf) - assert.Equal("info", conf.Level) -} - -func TestSearchMap(t *testing.T) { - assert := asst.New(t) - var m = make(map[string]interface{}) - var m2 = make(map[string]interface{}) - m["xephonk"] = m2 - m2["name"] = "xephonk" - m2["port"] = 8080 - val, err := searchMap(m, []string{"xephonk", "name"}) - assert.Nil(err) - assert.Equal("xephonk", val) - _, err = searchMap(m, []string{"xephonk", "bar"}) - assert.NotNil(err) -} diff --git a/legacy/glide.lock b/legacy/glide.lock deleted file mode 100644 index db8b38a..0000000 --- a/legacy/glide.lock +++ /dev/null @@ -1,33 +0,0 @@ -hash: ee2996db5b78313eadaeb56875b5ebefc2e6aeba666bc3080dc29ebfcc984876 -updated: 2017-09-16T15:58:45.391163291-07:00 -imports: -- name: github.com/kballard/go-shellquote - version: d8ec1a69a250a17bb0e419c386eac1f3711dc142 - vcs: git -- name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d - vcs: git -- name: golang.org/x/net - version: 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 - subpackages: - - proxy -- name: golang.org/x/tools - version: 496819729719f9d07692195e0a94d6edd2251389 - subpackages: - - benchmark/parse -- name: gopkg.in/yaml.v2 - version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b -testImports: -- name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib -- name: github.com/stretchr/testify - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 - vcs: git - subpackages: - - assert diff --git a/legacy/glide.yaml b/legacy/glide.yaml deleted file mode 100644 index aeb612c..0000000 --- a/legacy/glide.yaml +++ /dev/null @@ -1,14 +0,0 @@ -package: github.com/dyweb/gommon -import: -- package: gopkg.in/yaml.v2 -- package: github.com/pkg/errors - vcs: git - version: 0.8.0 -- package: github.com/kballard/go-shellquote - vcs: git - version: d8ec1a69a250a17bb0e419c386eac1f3711dc142 -- package: golang.org/x/tools/benchmark/parse -testImport: -- package: github.com/stretchr/testify - vcs: git - version: 2402e8e7a02fc811447d11f881aa9746cdc57983 diff --git a/legacy/log/README.md b/legacy/log/README.md deleted file mode 100644 index 4ee576c..0000000 --- a/legacy/log/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Log that support filter by field - -This a simple re-implementation of [logrus](https://github.com/sirupsen/logrus) - -## Development - -Generated code - -- https://github.com/benbjohnson/tmpl saw it from InfluxDB, plan to put this in Ayi, or use pongo - - `go get github.com/benbjohnson/tmpl` - - `tmpl -data '["Trace", "Debug", "Info", "Warn", "Error"]' entry_generated.go.tmpl` - -## Added - -- `Filter` and `PkgFilter` -- `TraceLevel` - -## Fixed - -- Remove duplicate code in `Logger` and `Entry` by only allow using `Entry` to log -- Use elasped time in `TextFormatter`, see [issue](https://github.com/sirupsen/logrus/issues/457) -- `FatalLevel` should be more severe than `PanicLevel` - -## Removed - -- lock on logger when call log for `Entry` -- Support for blocking Hook -- Trim `\n` when using `*f` - -## TODO - -- [ ] read filters from command line or config files -- [ ] `WithFields` -- [ ] pool for `Entry` and `bytes.Writer` -- [ ] JSON Formatter -- [ ] Multiple output -- [ ] async Hook -- [ ] Batch write and flush like [zap](https://github.com/uber-go/zap) -- [ ] Shutdown handler - -## DONE - -- thread safe by not using pointer receiver https://github.com/at15/go-learning/issues/3 -- log to stdout, implemented `TextFormatter` -- leveled logging, add `Trace` level -- support field -- filter by field \ No newline at end of file diff --git a/legacy/log/config.example.yml b/legacy/log/config.example.yml deleted file mode 100644 index e21383f..0000000 --- a/legacy/log/config.example.yml +++ /dev/null @@ -1,5 +0,0 @@ -level: info -color: true -source: true -showElapsedTime: false -timeFormat: "2006-01-02T15:04:05Z07:00" \ No newline at end of file diff --git a/legacy/log/config.go b/legacy/log/config.go deleted file mode 100644 index 0ecf7dc..0000000 --- a/legacy/log/config.go +++ /dev/null @@ -1,52 +0,0 @@ -// +build ignore - -package log - -import ( - "time" - - "github.com/pkg/errors" -) - -type Config struct { - Level string `yaml:"level" json:"level"` - Color bool `yaml:"color" json:"color"` - Source bool `yaml:"source" json:"source"` - ShowElapsedTime bool `yaml:"showElapsedTime" json:"showElapsedTime"` - TimeFormat string `yaml:"timeFormat" json:"timeFormat"` - XXX map[string]interface{} `yaml:",inline"` -} - -func (c *Config) Validate() error { - if c.XXX != nil { - return errors.Errorf("undefined fields found %v", c.XXX) - } - if _, err := ParseLevel(c.Level, false); err != nil { - return errors.Wrap(err, "invalid log level in config") - } - // valid time format - t := time.Now() - s := t.Format(c.TimeFormat) - t2, err := time.Parse(c.TimeFormat, s) - if err != nil || t.Unix() != t2.Unix() { - return errors.Wrap(err, "invalid time format string in config") - } - return nil -} - -func (log *Logger) ApplyConfig(c *Config) error { - if err := c.Validate(); err != nil { - return err - } - if err := log.SetLevel(c.Level); err != nil { - return err - } - if c.Source { - log.EnableSourceLine() - } - log.Formatter.SetColor(c.Color) - log.Formatter.SetElapsedTime(c.ShowElapsedTime) - log.Formatter.SetTimeFormat(c.TimeFormat) - // TODO: pkg filter should also be considered - return nil -} diff --git a/legacy/log/config_test.go b/legacy/log/config_test.go deleted file mode 100644 index d885722..0000000 --- a/legacy/log/config_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" - "io/ioutil" -) - -func TestConfig_Validate(t *testing.T) { - assert := asst.New(t) - b := readFixture(t, "config.example.yml") - var c Config - assert.Nil(yaml.Unmarshal(b, &c)) - assert.Nil(c.Validate()) - l := NewLogger() - assert.Nil(l.ApplyConfig(&c)) - e := l.NewEntryWithPkg("pkg1") - e.Info("test config") -} - -// FIXME: this is copied from util library to avoid import cycle -func readFixture(t *testing.T, path string) []byte { - b, err := ioutil.ReadFile(path) - if err != nil { - t.Fatalf("can't read fixture %s: %v", path, err) - } - return b -} diff --git a/legacy/log/doc.go b/legacy/log/doc.go deleted file mode 100644 index 6486c65..0000000 --- a/legacy/log/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build ignore - -/* -Package log(Deprecated) can filter log by field and support multiple level -*/ -package log - -// TODO: https://github.com/fluhus/godoc-tricks diff --git a/legacy/log/entry.go b/legacy/log/entry.go deleted file mode 100644 index 1b15479..0000000 --- a/legacy/log/entry.go +++ /dev/null @@ -1,129 +0,0 @@ -// +build ignore - -package log - -import ( - "fmt" - "os" - "runtime" - "strings" - "time" - - "github.com/pkg/errors" -) - -// Entry is the real logger -type Entry struct { - Logger *Logger - Pkg string // TODO: we should make use of this and stop storing the it in the map field? - EntryLevel Level - Fields Fields - Time time.Time - Level Level - Message string -} - -// SetPkgAlias allows use shorter name for pkg when logging -func (entry *Entry) SetPkgAlias(alias string) { - entry.Fields["pkg"] = alias -} - -func (entry *Entry) SetEntryLevel(s string) error { - newLevel, err := ParseLevel(s, false) - if err != nil { - return errors.WithMessage(err, fmt.Sprintf("can't set logging level to %s", s)) - } - entry.EntryLevel = newLevel - return nil -} - -// AddField adds tag to entry -func (entry *Entry) AddField(key string, value string) { - entry.Fields[key] = value -} - -// AddFields adds multiple tags to entry -func (entry *Entry) AddFields(fields Fields) { - for k, v := range fields { - entry.Fields[k] = v - } -} - -// DeleteField remove a tag from entry, this was added for benchmark to remove the automatically added pkg tag when using RegisterPkg -func (entry *Entry) DeleteField(key string) { - delete(entry.Fields, key) -} - -// This function is not defined with a pointer receiver because we change -// the attribute of struct without using lock, if we use pointer, it would -// become race condition for multiple goroutines. -// see https://github.com/at15/go-learning/issues/3 -func (entry Entry) log(level Level, msg string) bool { - entry.Time = time.Now() - entry.Level = level - entry.Message = msg - // don't log if it can't pass the filter - for _, filter := range entry.Logger.Filters[level] { - if !filter.Accept(&entry) { - return false - } - } - // add source code line if required - if entry.Logger.showSourceLine { - // TODO: what if the user also have tag called source - _, file, line, ok := runtime.Caller(2) - if !ok { - file = "" - line = 1 - } else { - lastSlash := strings.LastIndex(file, "/") - file = file[lastSlash+1:] - } - entry.AddField("source", fmt.Sprintf("%s:%d", file, line)) - } - - serialized, err := entry.Logger.Formatter.Format(&entry) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to serialize, %v\n", err) - return false - } - _, err = entry.Logger.Out.Write(serialized) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to write, %v\n", err) - return false - } - return true -} - -func (entry *Entry) Panic(args ...interface{}) { - if entry.EntryLevel >= PanicLevel { - entry.log(PanicLevel, fmt.Sprint(args...)) - } - panic(fmt.Sprint(args...)) -} - -func (entry *Entry) Fatal(args ...interface{}) { - if entry.EntryLevel >= FatalLevel { - entry.log(FatalLevel, fmt.Sprint(args...)) - } - // TODO: allow register handlers like logrus - os.Exit(1) -} - -// Printf functions -// NOTE: the *f functions does NOT call * functions like logrus does, it just copy and paste - -func (entry *Entry) Panicf(format string, args ...interface{}) { - if entry.EntryLevel >= PanicLevel { - entry.log(PanicLevel, fmt.Sprintf(format, args...)) - } - panic(fmt.Sprint(args...)) -} - -func (entry *Entry) Fatalf(format string, args ...interface{}) { - if entry.EntryLevel >= FatalLevel { - entry.log(FatalLevel, fmt.Sprintf(format, args...)) - } - // TODO: allow register handlers like logrus - os.Exit(1) -} diff --git a/legacy/log/entry_generated.go b/legacy/log/entry_generated.go deleted file mode 100644 index d09832e..0000000 --- a/legacy/log/entry_generated.go +++ /dev/null @@ -1,73 +0,0 @@ -// +build ignore - -// Generated by tmpl -// https://github.com/benbjohnson/tmpl -// -// DO NOT EDIT! -// Source: entry_generated.go.tmpl - -package log - -import ( - "fmt" -) - -func (entry *Entry) Trace(args ...interface{}) { - if entry.EntryLevel >= TraceLevel { - entry.log(TraceLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Tracef(format string, args ...interface{}) { - if entry.EntryLevel >= TraceLevel { - entry.log(TraceLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Debug(args ...interface{}) { - if entry.EntryLevel >= DebugLevel { - entry.log(DebugLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Debugf(format string, args ...interface{}) { - if entry.EntryLevel >= DebugLevel { - entry.log(DebugLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Info(args ...interface{}) { - if entry.EntryLevel >= InfoLevel { - entry.log(InfoLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Infof(format string, args ...interface{}) { - if entry.EntryLevel >= InfoLevel { - entry.log(InfoLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Warn(args ...interface{}) { - if entry.EntryLevel >= WarnLevel { - entry.log(WarnLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Warnf(format string, args ...interface{}) { - if entry.EntryLevel >= WarnLevel { - entry.log(WarnLevel, fmt.Sprintf(format, args...)) - } -} - -func (entry *Entry) Error(args ...interface{}) { - if entry.EntryLevel >= ErrorLevel { - entry.log(ErrorLevel, fmt.Sprint(args...)) - } -} - -func (entry *Entry) Errorf(format string, args ...interface{}) { - if entry.EntryLevel >= ErrorLevel { - entry.log(ErrorLevel, fmt.Sprintf(format, args...)) - } -} diff --git a/legacy/log/entry_generated.go.tmpl b/legacy/log/entry_generated.go.tmpl deleted file mode 100644 index c787bb7..0000000 --- a/legacy/log/entry_generated.go.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -package log - -import ( - "fmt" -) - -{{ range .}} -func (entry *Entry) {{.}}(args ...interface{}) { - if entry.EntryLevel >= {{.}}Level { - entry.log({{.}}Level, fmt.Sprint(args...)) - } -} - -func (entry *Entry) {{.}}f(format string, args ...interface{}) { - if entry.EntryLevel >= {{.}}Level { - entry.log({{.}}Level, fmt.Sprintf(format, args...)) - } -} -{{ end }} \ No newline at end of file diff --git a/legacy/log/entry_test.go b/legacy/log/entry_test.go deleted file mode 100644 index 99c463c..0000000 --- a/legacy/log/entry_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// +build ignore - -package log - -import "testing" - -func TestEntry_log(t *testing.T) { - logger := NewLogger() - entry := logger.NewEntry() - entry.AddField("pkg", "dummy.d") - entry.AddField("name", "jack") - entry.log(DebugLevel, "hahaha") - -} - -// TODO: split this test -func TestEntry_LeveledLog(t *testing.T) { - logger := NewLogger() - f := NewTextFormatter() - f.EnableColor = true - logger.Formatter = f - entry := logger.NewEntry() - // NOTE: when run it in IDEA, the terminal does not have color, run it in real terminal it will show - entry.Debug("You should not see me!") - entry.Infof("%s %d", "haha", 1) - entry.Warnf("%s %d", "haha", 1) - entry.Errorf("%s %d", "haha", 1) - - logger2 := NewLogger() - f2 := NewTextFormatter() - f2.EnableColor = true - f2.EnableElapsedTime = false - logger2.Formatter = f2 - entry2 := logger2.NewEntry() - entry2.Info("I should have full timestamp") - - logger3 := NewLogger() - logger3.Level = DebugLevel - f3 := NewTextFormatter() - f3.EnableColor = false - f3.EnableTimeStamp = false - logger3.Formatter = f3 - entry3 := logger3.NewEntry() - entry3.Info("I should have no timestamp") - entry3.Debug("You should see me!") - - // source code line - logger4 := NewLogger() - logger4.EnableSourceLine() - entry4 := logger4.NewEntry() - entry4.Info("show me source") -} - -func TestEntry_DeleteField(t *testing.T) { - logger := NewLogger() - entry := logger.RegisterPkg() - entry.DeleteField("pkg") - t.Log(len(entry.Fields)) -} diff --git a/legacy/log/filter.go b/legacy/log/filter.go deleted file mode 100644 index 63bd85b..0000000 --- a/legacy/log/filter.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build ignore - -package log - -import ( - st "github.com/dyweb/gommon/structure" -) - -// Filter determines if the entry should be logged -type Filter interface { - Accept(entry *Entry) bool - FilterName() string - FilterDescription() string -} - -var _ Filter = (*PkgFilter)(nil) - -// PkgFilter only allows entry without `pkg` field or `pkg` value in the allow set to pass -// TODO: we should support level -// TODO: a more efficient way might be trie tree and use `/` to divide package into segments instead of using character -type PkgFilter struct { - allow st.Set -} - -// Accept checks if the entry.Pkg (NOT entry.Fields["pkg"]) is in the white list -func (filter *PkgFilter) Accept(entry *Entry) bool { - return filter.allow.Contains(entry.Pkg) -} - -// FilterName implements Filter interface -func (filter *PkgFilter) FilterName() string { - return "PkgFilter" -} - -func (filter *PkgFilter) FilterDescription() string { - return "Filter log based on their pkg tag value, it is logged if it does not have pkg field or in whitelist" -} - -// NewPkgFilter returns a filter that allow log that contains `pkg` filed in the allow set -func NewPkgFilter(allow st.Set) *PkgFilter { - return &PkgFilter{ - allow: allow, - } -} diff --git a/legacy/log/filter_test.go b/legacy/log/filter_test.go deleted file mode 100644 index ce91638..0000000 --- a/legacy/log/filter_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestFilterInterface(t *testing.T) { - t.Parallel() - var _ Filter = (*PkgFilter)(nil) -} - -func TestPkgFilter_Filter(t *testing.T) { - t.Parallel() - assert := asst.New(t) - - allow := make(map[string]bool) - // NOTE: we don't use package name with dot because - // - when we get the package using reflection, we got / - // - when access fields in config, we use dot notation, viper.Get("logging.ayi/app/git") is different with viper.Get("logging.ayi.app.gi") - allow["ayi/app/git"] = true - f := NewPkgFilter(allow) - - entryWithoutPkg := &Entry{} - assert.False(f.Accept(entryWithoutPkg)) - entryWithAllowedPkg := &Entry{Pkg: "ayi/app/git"} - assert.True(f.Accept(entryWithAllowedPkg)) - entryWithDisallowedPkg := &Entry{Pkg: "ayi/app/web"} - assert.False(f.Accept(entryWithDisallowedPkg)) - - // NOTE: we are using entry.Pkg instead of entry.Fields["pkg"] - field := make(map[string]string, 1) - field["pkg"] = "ayi/app/git" - entryWithAllowedPkgInFields := &Entry{Fields: field} - assert.False(f.Accept(entryWithAllowedPkgInFields)) -} diff --git a/legacy/log/formatter.go b/legacy/log/formatter.go deleted file mode 100644 index b9a5e76..0000000 --- a/legacy/log/formatter.go +++ /dev/null @@ -1,110 +0,0 @@ -// +build ignore - -package log - -import ( - "bytes" - "fmt" - "time" -) - -type Formatter interface { - Format(*Entry) ([]byte, error) - SetColor(bool) - SetElapsedTime(bool) - SetTimeFormat(string) -} - -var ( - baseTimeStamp = time.Now() -) - -const ( - nocolor = 0 - red = 31 - green = 32 - yellow = 33 - blue = 34 - gray = 37 -) - -const ( - defaultTimeStampFormat = time.RFC3339 -) - -var _ Formatter = (*TextFormatter)(nil) - -type TextFormatter struct { - EnableColor bool - EnableTimeStamp bool - EnableElapsedTime bool - TimeStampFormat string -} - -func NewTextFormatter() *TextFormatter { - return &TextFormatter{ - EnableColor: false, - EnableTimeStamp: true, - EnableElapsedTime: true, - TimeStampFormat: defaultTimeStampFormat, - } -} - -func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { - var b *bytes.Buffer - // TODO: may use a pool - b = &bytes.Buffer{} - var levelColor = nocolor - if f.EnableColor { - switch entry.Level { - case InfoLevel: - levelColor = blue - case WarnLevel: - levelColor = yellow - case ErrorLevel, FatalLevel, PanicLevel: - levelColor = red - } - } - if levelColor != nocolor { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m", levelColor, entry.Level.ShortUpperString()) - } else { - b.WriteString(entry.Level.ShortUpperString()) - } - if f.EnableTimeStamp { - if f.EnableElapsedTime { - // NOTE: the elapsedTime copied from logrus is wrong - // https://github.com/sirupsen/logrus/pull/465 - fmt.Fprintf(b, "[%04d]", int(entry.Time.Sub(baseTimeStamp)/time.Second)) - } else { - // NOTE: config.Validate would check if timestamp format set by user is valid, and time.Format does not return error - fmt.Fprintf(b, "[%s]", entry.Time.Format(f.TimeStampFormat)) - } - } - b.WriteByte(' ') - b.WriteString(entry.Message) - b.WriteByte(' ') - for k, v := range entry.Fields { - b.WriteString(k) - b.WriteByte('=') - b.WriteString(v) - b.WriteByte(' ') - } - b.WriteByte('\n') - return b.Bytes(), nil -} - -func (f *TextFormatter) SetColor(b bool) { - f.EnableColor = b -} - -func (f *TextFormatter) SetElapsedTime(b bool) { - f.EnableElapsedTime = b -} - -func (f *TextFormatter) SetTimeFormat(tf string) { - if tf == "" { - f.TimeStampFormat = defaultTimeStampFormat - } else { - f.TimeStampFormat = tf - } -} diff --git a/legacy/log/formatter_test.go b/legacy/log/formatter_test.go deleted file mode 100644 index 60bcf63..0000000 --- a/legacy/log/formatter_test.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build ignore - -package log diff --git a/legacy/log/level.go b/legacy/log/level.go deleted file mode 100644 index ea83aa4..0000000 --- a/legacy/log/level.go +++ /dev/null @@ -1,126 +0,0 @@ -// +build ignore - -package log - -import ( - "strings" - - "github.com/pkg/errors" -) - -// Level is log level -type Level uint8 - -const ( - // FatalLevel log error and call `os.Exit(1)` - FatalLevel Level = iota - // PanicLevel log error and call `panic` - PanicLevel - // ErrorLevel log error - ErrorLevel - // WarnLevel log warning - WarnLevel - // InfoLevel log info - InfoLevel - // DebugLevel log debug message, user should enable DebugLevel logging when report bug - DebugLevel - // TraceLevel is very verbose, user should enable it only on packages they are currently investing instead of globally - TraceLevel -) - -// ShortUpperString returns the first 4 characters of a level in upper case -func (level Level) ShortUpperString() string { - switch level { - case FatalLevel: - return "FATA" - case PanicLevel: - return "PANI" - case ErrorLevel: - return "ERRO" - case WarnLevel: - return "WARN" - case InfoLevel: - return "INFO" - case DebugLevel: - return "DEBU" - case TraceLevel: - return "TRAC" - default: - return "UNKN" - } -} - -func (level Level) String() string { - switch level { - case FatalLevel: - return "fatal" - case PanicLevel: - return "panic" - case ErrorLevel: - return "error" - case WarnLevel: - return "warn" - case InfoLevel: - return "info" - case DebugLevel: - return "debug" - case TraceLevel: - return "trace" - default: - return "unknown" - } -} - -// ParseLevel match the level string with Level, it will use strings.HasPrefix in non strict mode -func ParseLevel(s string, strict bool) (Level, error) { - str := strings.ToLower(s) - if strict { - switch str { - case "fatal": - return FatalLevel, nil - case "panic": - return PanicLevel, nil - case "error": - return ErrorLevel, nil - case "warn": - return WarnLevel, nil - case "info": - return InfoLevel, nil - case "debug": - return DebugLevel, nil - case "trace": - return TraceLevel, nil - default: - return Level(250), errors.Errorf("unknown log level %s", str) - } - } - switch { - case strings.HasPrefix(str, "f"): - return FatalLevel, nil - case strings.HasPrefix(str, "p"): - return PanicLevel, nil - case strings.HasPrefix(str, "e"): - return ErrorLevel, nil - case strings.HasPrefix(str, "w"): - return WarnLevel, nil - case strings.HasPrefix(str, "i"): - return InfoLevel, nil - case strings.HasPrefix(str, "d"): - return DebugLevel, nil - case strings.HasPrefix(str, "t"): - return TraceLevel, nil - default: - return Level(250), errors.Errorf("unknown log level %s", str) - } -} - -// AllLevels includes all the logging level -var AllLevels = []Level{ - FatalLevel, - PanicLevel, - ErrorLevel, - WarnLevel, - InfoLevel, - DebugLevel, - TraceLevel, -} diff --git a/legacy/log/level_test.go b/legacy/log/level_test.go deleted file mode 100644 index 6c92167..0000000 --- a/legacy/log/level_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestParseLevel(t *testing.T) { - assert := asst.New(t) - // strict - for _, l := range AllLevels { - s := l.String() - l2, err := ParseLevel(s, true) - assert.Nil(err) - assert.Equal(l, l2) - } - // not strict - levels := []string{"FA", "Pa", "er", "Warn", "infooo", "debugggg", "Tracer"} - for i := 0; i < len(AllLevels); i++ { - l, err := ParseLevel(levels[i], false) - assert.Nil(err) - assert.Equal(AllLevels[i], l) - } - // invalid - _, err := ParseLevel("haha", false) - assert.NotNil(err) -} diff --git a/legacy/log/logger.go b/legacy/log/logger.go deleted file mode 100644 index 1e87697..0000000 --- a/legacy/log/logger.go +++ /dev/null @@ -1,134 +0,0 @@ -// +build ignore - -package log - -import ( - "fmt" - "io" - "os" - "sync" - - "github.com/dyweb/gommon/util/runtimeutil" - "github.com/pkg/errors" -) - -// Fields is key-value string pair to annotate the log and can be used by filter -type Fields map[string]string - -// Logger is used to set output, formatter and filters, the real log operation is in Entry -type Logger struct { - Out io.Writer - Formatter Formatter - Level Level - mu sync.Mutex // FIXME: this mutex is never used, I guess I was following logrus when I wrote this - showSourceLine bool - Filters map[Level]map[string]Filter - Entries map[string]*Entry -} - -// NewLogger returns a new logger using StdOut and InfoLevel -func NewLogger() *Logger { - f := make(map[Level]map[string]Filter, len(AllLevels)) - for _, level := range AllLevels { - f[level] = make(map[string]Filter, 1) - } - l := &Logger{ - Out: os.Stdout, - Formatter: NewTextFormatter(), - Level: InfoLevel, - Filters: f, - Entries: make(map[string]*Entry), - showSourceLine: false, - } - return l -} - -func (log *Logger) SetLevel(s string) error { - newLevel, err := ParseLevel(s, false) - if err != nil { - return errors.WithMessage(err, fmt.Sprintf("can't set logging level to %s", s)) - } - if log.Level == newLevel { - return nil - } - // update all the registered entries - for _, entry := range log.Entries { - // only update entry's level if the new global level is more verbose - // TODO: allow force set level to all registered entries - if newLevel > entry.EntryLevel { - entry.EntryLevel = newLevel - } - } - log.Level = newLevel - return nil -} - -// EnableSourceLine add `source` field when logging, it use runtime.Caller(), the overhead has not been measured -func (log *Logger) EnableSourceLine() { - log.showSourceLine = true -} - -// DisableSourceLine does not show `source` field -func (log *Logger) DisableSourceLine() { - log.showSourceLine = false -} - -// AddFilter add a filter to logger, the filter should be simple string check on fields, i.e. PkgFilter check pkg field -func (log *Logger) AddFilter(filter Filter, level Level) { - log.Filters[level][filter.FilterName()] = filter -} - -// NewEntry returns an Entry with empty Fields -// Deprecated: use RegisterPkg instead -func (log *Logger) NewEntry() *Entry { - // TODO: may use pool, but need benchmark to see if using pool provides improvement - return &Entry{ - Logger: log, - Pkg: "", - EntryLevel: log.Level, - Fields: make(map[string]string, 1), - } -} - -// NewEntryWithPkg returns an Entry with pkg Field set to pkgName, should be used with PkgFilter -// Deprecated: use RegisterPkg instead -func (log *Logger) NewEntryWithPkg(pkgName string) *Entry { - fields := make(map[string]string, 1) - fields["pkg"] = pkgName - e := &Entry{ - Logger: log, - Pkg: pkgName, - EntryLevel: log.Level, - Fields: fields, - } - log.Entries[pkgName] = e - return e -} - -// RegisterPkg creates a new entry with pkg field set to the caller's package and register this entry to logger -func (log *Logger) RegisterPkg() *Entry { - fields := make(map[string]string, 1) - pkg := runtimeutil.GetCallerPackage(2) - fields["pkg"] = pkg - e := &Entry{ - Logger: log, - Pkg: pkg, - EntryLevel: log.Level, - Fields: fields, - } - log.Entries[pkg] = e - return e -} - -func (log *Logger) RegisteredPkgs() map[string]*Entry { - return log.Entries -} - -func (log *Logger) PrintEntries() { - // TODO: sort and print it in a hierarchy - // github.com/dyweb/Ayi/web - // github.com/dyweb/Ayi/web/static - for pkg := range log.Entries { - fmt.Println(pkg) - } -} diff --git a/legacy/log/logger_test.go b/legacy/log/logger_test.go deleted file mode 100644 index d9800ce..0000000 --- a/legacy/log/logger_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build ignore - -package log - -import ( - "testing" - - "bytes" - asst "github.com/stretchr/testify/assert" - "io" - "os" -) - -func TestLogger_AddFilter(t *testing.T) { - assert := asst.New(t) - logger := NewLogger() - allow := make(map[string]bool) - pkgFilter := NewPkgFilter(allow) - logger.AddFilter(pkgFilter, DebugLevel) - assert.Equal(1, len(logger.Filters[DebugLevel])) -} - -func TestLogger_NewEntryWithPkg(t *testing.T) { - assert := asst.New(t) - logger := NewLogger() - entry := logger.NewEntryWithPkg("x.dummy") - assert.Equal(1, len(entry.Fields)) - entry.Info("show me the pkg") -} - -func TestLogger_SetLevel(t *testing.T) { - assert := asst.New(t) - - var b bytes.Buffer - writer := io.MultiWriter(os.Stdout, &b) - logger := NewLogger() - logger.Out = writer - entry1 := logger.NewEntryWithPkg("pkg1") - assert.Equal(entry1.EntryLevel, InfoLevel) - entry2 := logger.NewEntryWithPkg("pkg2") - - entry1.Info("should see me") - entry1.Debug("should not see me") - assert.NotContains(b.String(), "not") - - assert.Nil(entry2.SetEntryLevel("trace")) - assert.Nil(logger.SetLevel("debug")) - - entry1.Debug("should see me") - entry1.Trace("should not see me") - entry2.Trace("should see me") - assert.NotContains(b.String(), "not") -} diff --git a/legacy/log/v2.md b/legacy/log/v2.md deleted file mode 100644 index d8bf31c..0000000 --- a/legacy/log/v2.md +++ /dev/null @@ -1,112 +0,0 @@ -# Log v2 - -Design Goals - -- easy to maintain, use standard library as much as possible -- acceptable and easy to use by default, increase performance if really needed -- testable, should not rely on human eyeball to see if the log is correct -- configurable, if a library use gommon/log, then the application using the library can config the logging of that library like the library itself -- out of box, default config struct, http handler (unix sock?), print log entry relations -- not using package level variable, especially pools - -Non Goals - -- sampling - -## Survey - -## log - -https://golang.org/src/log/log.go - -````go -func (l *Logger) Printf(format string, v ...interface{}) { - l.Output(2, fmt.Sprintf(format, v...)) -} - -// Output writes the output for a logging event. The string s contains -// the text to print after the prefix specified by the flags of the -// Logger. A newline is appended if the last character of s is not -// already a newline. Calldepth is used to recover the PC and is -// provided for generality, although at the moment on all pre-defined -// paths it will be 2. -func (l *Logger) Output(calldepth int, s string) error { - // Get time early if we need it. - var now time.Time - if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { - now = time.Now() - } - var file string - var line int - l.mu.Lock() - defer l.mu.Unlock() - if l.flag&(Lshortfile|Llongfile) != 0 { - // Release lock while getting caller info - it's expensive. - l.mu.Unlock() - var ok bool - _, file, line, ok = runtime.Caller(calldepth) - if !ok { - file = "???" - line = 0 - } - l.mu.Lock() - } - l.buf = l.buf[:0] - l.formatHeader(&l.buf, now, file, line) - l.buf = append(l.buf, s...) - if len(s) == 0 || s[len(s)-1] != '\n' { - l.buf = append(l.buf, '\n') - } - _, err := l.out.Write(l.buf) - return err -} -```` - -### Logrus - -https://github.com/sirupsen/logrus - -The first version is entirely modeled after logrus - -common - -- not use pointer receiver on Entry struct to avoid race condition -- using formatter + writer - - introduced extra copy from `fmt.Sprintf`, it is possible to put data into writer directly - - [ ] TODO: will there be race condition causing different log mixed with each other, like `[info][warn]msg1msg2` instead of `[info]msg1 [warn]msg2` - -differences - -- no pool for log entry -- no `*ln` function, just `*` and `*f` `*` means Debug, Info ... - -### apex/log - -- use handler instead of formatter + writer - -````go -// The HandlerFunc type is an adapter to allow the use of ordinary functions as -// log handlers. If f is a function with the appropriate signature, -// HandlerFunc(f) is a Handler object that calls f. -type HandlerFunc func(*Entry) error - -// HandleLog calls f(e). -func (f HandlerFunc) HandleLog(e *Entry) error { - return f(e) -} -```` - -I guess it's modeled after `net/http` - -````go -// The HandlerFunc type is an adapter to allow the use of -// ordinary functions as HTTP handlers. If f is a function -// with the appropriate signature, HandlerFunc(f) is a -// Handler that calls f. -type HandlerFunc func(ResponseWriter, *Request) - -// ServeHTTP calls f(w, r). -func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { -f(w, r) -} -```` \ No newline at end of file diff --git a/legacy/pkg.go b/legacy/pkg.go deleted file mode 100644 index d945b1c..0000000 --- a/legacy/pkg.go +++ /dev/null @@ -1,4 +0,0 @@ -// +build ignore - -// Package legacy contains deprecated gommon packages -package legacy diff --git a/legacy/requests/README.md b/legacy/requests/README.md deleted file mode 100644 index 6a8fa0a..0000000 --- a/legacy/requests/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Requests - -A wrapper around net/http with less public global variables - -Originally from [xephon-b](https://github.com/xephonhq/xephon-b) - -## Usage - -TODO: - -- it was mainly used for building config for http client to use socks5 proxy -- [ ] https://github.com/hashicorp/go-getter like features - -## Changelog - -- use same config as `DefaultTransport` in `net/http` -- add client struct and default client which wraps `http.Client` -- support socks5 proxy https://github.com/dyweb/gommon/issues/18 -- rename `GetJSON` to get `GetJSONStringMap`, this should break old Xephon-B code, which is now in tsdb-proxy \ No newline at end of file diff --git a/legacy/requests/auth.go b/legacy/requests/auth.go deleted file mode 100644 index d99116d..0000000 --- a/legacy/requests/auth.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build ignore - -package requests - -import ( - "encoding/base64" - "net/http" - "strings" - - "github.com/dyweb/gommon/errors" -) - -// NOTE: net/http already implemented it https://golang.org/pkg/net/http/#Request.BasicAuth -const AuthorizationHeader = "Authorization" - -func GenerateBasicAuth(username string, password string) string { - return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)) -} - -func ExtractBasicAuth(val string) (username string, password string, err error) { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization - s := strings.Split(val, " ") - if len(s) != 2 { - err = errors.New("invalid authorization header, must have type and base64 encoded value separated by space") - return - } - tpe := strings.ToLower(s[0]) - if tpe != "basic" { - err = errors.Errorf("got %s instead basic auth", tpe) - return - } - decoded, err := base64.StdEncoding.DecodeString(s[1]) - if err != nil { - err = errors.New("invalid base64 encoding") - return - } - ss := strings.Split(string(decoded), ":") - if len(ss) != 2 { - errors.Errorf("invalid username:password, got %d segments after split by ':'", len(ss)) - } - username = ss[0] - password = ss[1] - err = nil - return -} - -func ExtractBasicAuthFromRequest(r *http.Request) (string, string, error) { - val := r.Header.Get(AuthorizationHeader) - if val == "" { - return "", "", errors.New("Authorization header is not set") - } - return ExtractBasicAuth(val) -} diff --git a/legacy/requests/auth_test.go b/legacy/requests/auth_test.go deleted file mode 100644 index 6623cb4..0000000 --- a/legacy/requests/auth_test.go +++ /dev/null @@ -1,22 +0,0 @@ -// +build ignore - -package requests - -import ( - asst "github.com/stretchr/testify/assert" - - "testing" -) - -func TestExtractBasicAuth(t *testing.T) { - assert := asst.New(t) - username := "Aladdin" - password := "OpenSesame" - // get from Postman - header := "Basic QWxhZGRpbjpPcGVuU2VzYW1l" - assert.Equal(header, GenerateBasicAuth(username, password)) - u, p, err := ExtractBasicAuth(header) - assert.Nil(err) - assert.Equal(username, u) - assert.Equal(password, p) -} diff --git a/legacy/requests/builder.go b/legacy/requests/builder.go deleted file mode 100644 index 3f588f5..0000000 --- a/legacy/requests/builder.go +++ /dev/null @@ -1,69 +0,0 @@ -// +build ignore - -package requests - -import ( - "crypto/tls" - "net/http" - - "golang.org/x/net/proxy" - - "github.com/dyweb/gommon/errors" -) - -// TODO: might switch to functional options https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis -// TODO: http proxy? might just use environment variable? https://stackoverflow.com/questions/14661511/setting-up-proxy-for-http-client -// TransportBuilder is the initial builder, its method use value receiver and return a new copy for chaining and keep itself unchanged -var TransportBuilder transportBuilder - -type transportBuilder struct { - skipKeyVerify bool - socks5Host string - auth *proxy.Auth -} - -func (b transportBuilder) SkipKeyVerify() transportBuilder { - b.skipKeyVerify = true - return b -} - -// UseShadowSocks uses the default config for shadowsocks local -func (b transportBuilder) UseShadowSocks() transportBuilder { - return b.UseSocks5(ShadowSocksLocal, "", "") -} - -func (b transportBuilder) UseSocks5(host string, username string, password string) transportBuilder { - b.socks5Host = host - if username == "" && password == "" { - b.auth = nil - } else { - b.auth.User = username - b.auth.Password = password - } - return b -} - -func (b transportBuilder) Build() (*http.Transport, error) { - transport := NewDefaultTransport() - if b.skipKeyVerify { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - if b.socks5Host != "" { - // NOTE: actually the implementation does not connect at all, the err is always nil - dialer, err := proxy.SOCKS5("tcp", b.socks5Host, b.auth, proxy.Direct) - if err != nil { - return nil, errors.Wrapf(err, "can't create socks5 dialer to %s with %s:%s", b.socks5Host, b.auth.User, b.auth.Password) - } - // TODO: Dial is deprecated and we should use DialContext, maybe contribute to golang/x/net - transport.Dial = dialer.Dial - } - return transport, nil -} - -func init() { - TransportBuilder = transportBuilder{ - skipKeyVerify: false, - socks5Host: "", - auth: &proxy.Auth{}, - } -} diff --git a/legacy/requests/builder_test.go b/legacy/requests/builder_test.go deleted file mode 100644 index d03a6ec..0000000 --- a/legacy/requests/builder_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build ignore - -package requests - -import ( - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestTransportBuilder_UseSocks5(t *testing.T) { - assert := asst.New(t) - b := TransportBuilder.UseSocks5("127.0.0.1:1080", "", "") - assert.NotNil(TransportBuilder.auth) - assert.Nil(b.auth) -} - -func TestTransportBuilder_Build(t *testing.T) { - assert := asst.New(t) - b := TransportBuilder.UseSocks5("127.0.0.1:1080", "", "") - // NOTE: it does not connect to the proxy, so the test should pass regardless of the running proxy - tr, err := b.Build() - assert.Nil(err) - assert.NotNil(tr) -} diff --git a/legacy/requests/default.go b/legacy/requests/default.go deleted file mode 100644 index 7e5c82e..0000000 --- a/legacy/requests/default.go +++ /dev/null @@ -1,35 +0,0 @@ -// +build ignore - -package requests - -import ( - "net" - "net/http" - "time" -) - -// Default Transport Client that is same as https://golang.org/src/net/http/transport.go -// It's similar to https://github.com/hashicorp/go-cleanhttp - -// NewDefaultTransport is copied from net/http/transport.go -func NewDefaultTransport() *http.Transport { - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } -} - -// NewDefaultClient returns a client using NewDefaultTransport -func NewDefaultClient() *http.Client { - return &http.Client{ - Transport: NewDefaultTransport(), - } -} diff --git a/legacy/requests/doc/history.md b/legacy/requests/doc/history.md deleted file mode 100644 index e69de29..0000000 diff --git a/legacy/requests/pkg.go b/legacy/requests/pkg.go deleted file mode 100644 index 5c5f254..0000000 --- a/legacy/requests/pkg.go +++ /dev/null @@ -1,4 +0,0 @@ -// +build ignore - -// Package requests(Deprecated) is a wrapper around net/http with less public global variables -package requests // import "github.com/dyweb/gommon/requests" diff --git a/legacy/requests/requests.go b/legacy/requests/requests.go deleted file mode 100644 index f678142..0000000 --- a/legacy/requests/requests.go +++ /dev/null @@ -1,128 +0,0 @@ -// +build ignore - -// Package requests wrap net/http like requests did for python -// it is easy to use, but not very efficient -package requests - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "strings" - - "github.com/dyweb/gommon/errors" -) - -const ( - ShadowSocksLocal = "127.0.0.1:1080" - ContentJSON = "application/json" -) - -var defaultClient = &Client{h: NewDefaultClient(), content: ContentJSON} - -type Client struct { - h *http.Client - content string -} - -func NewClient(options ...func(h *http.Client)) *Client { - c := &Client{h: NewDefaultClient(), content: ContentJSON} - for _, option := range options { - option(c.h) - } - return c -} - -func (c *Client) makeRequest(method string, url string, body io.Reader) (*Response, error) { - if c.h == nil { - return nil, errors.New("http client is not initialized") - } - var res *http.Response - var err error - switch method { - case http.MethodGet: - res, err = c.h.Get(url) - case http.MethodPost: - res, err = c.h.Post(url, c.content, body) - } - response := &Response{} - if err != nil { - return response, errors.Wrap(err, "error making request") - } - defer res.Body.Close() - resContent, err := ioutil.ReadAll(res.Body) - if err != nil { - return response, errors.Wrap(err, "error reading body") - } - response.Res = res - response.StatusCode = res.StatusCode - response.Data = resContent - return response, nil -} - -// TODO: this defaultish wrapping methods should be generated by gommon generator -func Post(url string, data io.Reader) (*Response, error) { - return defaultClient.Post(url, data) -} - -func (c *Client) Post(url string, data io.Reader) (*Response, error) { - return c.makeRequest(http.MethodPost, url, data) -} - -func PostString(url string, data string) (*Response, error) { - return defaultClient.PostString(url, data) -} - -func (c *Client) PostString(url string, data string) (*Response, error) { - return c.makeRequest(http.MethodPost, url, ioutil.NopCloser(strings.NewReader(data))) -} - -func PostBytes(url string, data []byte) (*Response, error) { - return defaultClient.PostBytes(url, data) -} - -func (c *Client) PostBytes(url string, data []byte) (*Response, error) { - return c.makeRequest(http.MethodPost, url, ioutil.NopCloser(bytes.NewReader(data))) -} - -func Get(url string) (*Response, error) { - return defaultClient.Get(url) -} - -func (c *Client) Get(url string) (*Response, error) { - return c.makeRequest(http.MethodGet, url, nil) -} - -func GetJSON(url string, data interface{}) error { - return defaultClient.GetJSON(url, data) -} - -func (c *Client) GetJSON(url string, data interface{}) error { - res, err := c.Get(url) - if err != nil { - return errors.Wrap(err, "error getting response") - } - err = res.JSON(data) - if err != nil { - return errors.Wrap(err, "error parsing json") - } - return nil -} - -func GetJSONStringMap(url string) (map[string]string, error) { - return defaultClient.GetJSONStringMap(url) -} - -func (c *Client) GetJSONStringMap(url string) (map[string]string, error) { - var data map[string]string - res, err := c.Get(url) - if err != nil { - return data, errors.Wrap(err, "error getting response") - } - data, err = res.JSONStringMap() - if err != nil { - return data, errors.Wrap(err, "error parsing json") - } - return data, nil -} diff --git a/legacy/requests/requests_test.go b/legacy/requests/requests_test.go deleted file mode 100644 index e69a883..0000000 --- a/legacy/requests/requests_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// +build ignore - -package requests - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestRequestsE2E(t *testing.T) { - if testing.Short() { - t.Skip("skip requests e2e test") - } - - // create an echo server - version := "0.0.1" - mux := http.NewServeMux() - mux.HandleFunc("/echo", func(w http.ResponseWriter, r *http.Request) { - data, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Log("can't read request body") - } - fmt.Fprintf(w, "%s", data) - // The Server will close the request body. The ServeHTTP - // Handler does not need to. - //r.Body.Close() - }) - mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "{\"version\": \"%s\"}", version) - }) - - testServer := httptest.NewServer(mux) - defer testServer.Close() - - versionURL := testServer.URL + "/version" - echoURL := testServer.URL + "/echo" - - c := NewClient() - t.Run("c.Get", func(t *testing.T) { - assert := asst.New(t) - res, err := c.Get(versionURL) - assert.Nil(err) - assert.Equal(http.StatusOK, res.Res.StatusCode) - }) - - t.Run("Get", func(t *testing.T) { - assert := asst.New(t) - res, err := Get(versionURL) - assert.Nil(err) - assert.Equal(http.StatusOK, res.Res.StatusCode) - }) - - t.Run("c.GetJSONStringMap", func(t *testing.T) { - assert := asst.New(t) - data, err := c.GetJSONStringMap(versionURL) - assert.Nil(err) - assert.Equal(version, data["version"]) - }) - - t.Run("GetJSONStringMap", func(t *testing.T) { - assert := asst.New(t) - data, err := GetJSONStringMap(versionURL) - assert.Nil(err) - assert.Equal(version, data["version"]) - }) - - t.Run("c.PostString", func(t *testing.T) { - assert := asst.New(t) - payload := fmt.Sprintf("{\"version\": \"%s\"}", version) - res, err := c.PostString(echoURL, payload) - assert.Nil(err) - assert.Equal(payload, string(res.Data)) - }) - - t.Run("c.PostString", func(t *testing.T) { - assert := asst.New(t) - payload := fmt.Sprintf("{\"version\": \"%s\"}", version) - res, err := PostString(echoURL, payload) - assert.Nil(err) - assert.Equal(payload, string(res.Data)) - }) -} - -func TestNewClient(t *testing.T) { - assert := asst.New(t) - tr, err := TransportBuilder.UseShadowSocks().Build() - assert.Nil(err) - c := NewClient(func(h *http.Client) { - h.Transport = tr - }) - assert.NotNil(c) - // uncomment the following if you have local socks 5 proxy running - //_, err = c.Get("https://google.com") - //t.Log(err) - //assert.Nil(err) -} diff --git a/legacy/requests/response.go b/legacy/requests/response.go deleted file mode 100644 index 171053d..0000000 --- a/legacy/requests/response.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build ignore - -package requests - -import ( - "encoding/json" - "net/http" - - "github.com/dyweb/gommon/errors" -) - -type Response struct { - Res *http.Response - StatusCode int - Data []byte -} - -func (res *Response) JSON(data interface{}) error { - if err := json.Unmarshal(res.Data, &data); err != nil { - return errors.Wrap(err, "error unmarshal json using map[string]string") - } - return nil -} - -func (res *Response) JSONStringMap() (map[string]string, error) { - var data map[string]string - if err := json.Unmarshal(res.Data, &data); err != nil { - return data, errors.Wrap(err, "error unmarshal json using map[string]string") - } - return data, nil -} diff --git a/legacy/runner/README.md b/legacy/runner/README.md deleted file mode 100644 index b13f100..0000000 --- a/legacy/runner/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Command runner - -TODO - -- github.com/kballard/go-shellquote saw it from https://github.com/github/hub -- [ ] https://github.com/mvdan/sh A shell parser, formatter and interpreter (POSIX/Bash/mksh) \ No newline at end of file diff --git a/legacy/runner/command.go b/legacy/runner/command.go deleted file mode 100644 index 00dec2c..0000000 --- a/legacy/runner/command.go +++ /dev/null @@ -1,62 +0,0 @@ -// +build ignore - -package runner - -import ( - "os" - "os/exec" - - st "github.com/dyweb/gommon/structure" - "github.com/kballard/go-shellquote" - "github.com/pkg/errors" -) - -// TODO: possible using https://github.com/mvdan/sh a shell parser, formatter in go -// commands that should be called using shell, -// because mostly they would be expecting shell expansion on parameters -var shellCommands = st.NewSet("echo", "rm", "cp", "mv", "mkdir", "tar") - -// NewCmd can properly split the executable with its arguments -// TODO: may need to add a context to handle things like using shell or not -func NewCmd(cmdStr string) (*exec.Cmd, error) { - segments, err := shellquote.Split(cmdStr) - if err != nil { - return nil, errors.Wrap(err, "can't parse command") - } - return exec.Command(segments[0], segments[1:]...), nil -} - -// NewCmdWithAutoShell automatically use `sh -c` syntax for a small list of executable -// because most people expect shell expansion i.e. wild chars when using them -// TODO: test if it really works, current unit test just test the number of arguments -func NewCmdWithAutoShell(cmdStr string) (*exec.Cmd, error) { - segments, err := shellquote.Split(cmdStr) - if err != nil { - return nil, errors.Wrap(err, "can't parse command") - } - name := segments[0] - useShell := shellCommands.Contains(name) - if useShell { - // TODO: may use shellquote join? - // NOTE: http://stackoverflow.com/questions/18946837/go-variadic-function-and-too-many-arguments - // the `append` here is a must "sh", "-c", segments[1:]... won't work - return exec.Command("sh", append([]string{"-c"}, segments[1:]...)...), nil - } - return exec.Command(segments[0], segments[1:]...), nil -} - -func RunCommand(cmdStr string) error { - cmd, err := NewCmd(cmdStr) - if err != nil { - return errors.Wrap(err, "can't create cmd from command string") - } - // TODO: dry run, maybe add a context - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - return errors.Wrap(err, "failure when executing command") - } - return nil -} diff --git a/legacy/runner/command_test.go b/legacy/runner/command_test.go deleted file mode 100644 index b9fbafa..0000000 --- a/legacy/runner/command_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build ignore - -package runner - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// https://github.com/dyweb/Ayi/issues/58 -// NOTE: it's not extra quote, it's os/exec can't expand * like shell does -// when run `rm *.aux` in shell, shell expands `*.aux` to real file names -func TestNewCommand_ExtraQuote(t *testing.T) { - assert := assert.New(t) - cmd, _ := NewCmd("rm *.aux") - assert.Equal(2, len(cmd.Args)) - assert.Equal("*.aux", cmd.Args[1]) -} - -func TestNewCmdWithAutoShell(t *testing.T) { - assert := assert.New(t) - cmd, _ := NewCmdWithAutoShell("rm *.aux") - assert.Equal(3, len(cmd.Args)) - // TODO: why this is not /bin/sh - assert.Equal("sh", cmd.Args[0]) - assert.Equal("-c", cmd.Args[1]) -} diff --git a/legacy/runner/context.go b/legacy/runner/context.go deleted file mode 100644 index 3f6e137..0000000 --- a/legacy/runner/context.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build ignore - -package runner - -// FIXME: this is never used, and it may not be a good idea to use context, since there is context package -// Context describes how the command should be run -// i.e. use shell instead of using os/exec, -// use self defined logic instead of lookup an executable -// run in background -type Context struct { - AutoShell bool - Foreground bool - Block bool - Dry bool -} - -// NewContext returns a context with convention -func NewContext() *Context { - return &Context{ - AutoShell: true, - Foreground: true, - Block: true, - Dry: false, - } -} diff --git a/legacy/runner/doc.go b/legacy/runner/doc.go deleted file mode 100644 index 98d80cc..0000000 --- a/legacy/runner/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build ignore - -/* -Package runner(Deprecated) run commands with some convention -*/ -package runner diff --git a/legacy/runner/example.config.yml b/legacy/runner/example.config.yml deleted file mode 100644 index 9986828..0000000 --- a/legacy/runner/example.config.yml +++ /dev/null @@ -1,47 +0,0 @@ -# an example of the new runner package config - -# global config for the runner -scripts-config: - # disable autoshell, no longer check for command like rm, mv and switch to sh -c for them - autoshell: false - -# definition of scripts -scripts: - # call gofmt using os/exec - fmt: gofmt . - # will be translated to sh -c "rm *.obj" if autoshell is enabled in global config - clean: rm *.obj - # a command with detail context - clean-plus: - cmd: rm *.obj - # force using sh -c regardless of global autoshell config - shell: true - # allow this command to have error and continue other commands, rm will exit when no file is found - fallible: true - test: - # use array to execute a list of commands one by one - - go install - # you can also use sh -c for force using shell, no extra sh -c will be added around it - - sh -c "go test -v -cover $(glide novendor)" - build: - # call the test command we previously defined - - test - - go install - - gox -output="build/Ayi_{{.OS}}_{{.Arch}}" - # you can mix object into the array and use the attributes you use for a top command - - cmd: rm *.obj - shell: true - fallible: true - serve: - - cmd: Ayi web static - desc: serve static web content - # run in background, all the background commands are run in parallel, since they won't end in a short time - background: true - # restart if the running process exit - restart: true - # TODO: maybe we should not support the number, need to have counter for cmd syntax - number: 1 - - cmd: Ayi web notify - desc: serve notification and display in browser page - background: true - restart: true \ No newline at end of file diff --git a/legacy/runner/pkg.go b/legacy/runner/pkg.go deleted file mode 100644 index 696b9a9..0000000 --- a/legacy/runner/pkg.go +++ /dev/null @@ -1,3 +0,0 @@ -// +build ignore - -package runner diff --git a/legacy/structure/README.md b/legacy/structure/README.md deleted file mode 100644 index d5d8fc1..0000000 --- a/legacy/structure/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Data structure - -## Set - -A non thread safe implementation of basic Set operations - -- Add -- Cardinality -- Contains -- Equal - -## Tree - -StringTree is manly used for printing i.e. directory layout, log entry hierarchy - -### Acknowledgement - -- https://github.com/deckarep/golang-set/ \ No newline at end of file diff --git a/legacy/structure/heap.go b/legacy/structure/heap.go deleted file mode 100644 index 69ffa4f..0000000 --- a/legacy/structure/heap.go +++ /dev/null @@ -1,89 +0,0 @@ -// +build ignore - -package structure - -// https://golang.org/src/container/heap/heap.go -// TODO: it seems the type is called Interface .... heap.Interface and heap.Heap ... - -// IntHeap is simple min int heap -type IntHeap struct { - data []int -} - -func (h *IntHeap) Len() int { - return len(h.data) -} - -func (h *IntHeap) Insert(x int) { - h.data = append(h.data, x) - // bubble up - h.up(len(h.data) - 1) -} - -// Peak returns the root without of removing it -func (h *IntHeap) Peak() int { - // TODO: need ok to return invalid value? - if len(h.data) == 0 { - return -1 - } - return h.data[0] -} - -// Poll return and remove current root -func (h *IntHeap) Poll() int { - if len(h.data) == 0 { - return -1 - } - - t := h.data[0] - if len(h.data) == 1 { - h.data = h.data[:0] - return t - } - h.data[0] = h.data[len(h.data)-1] - h.data = h.data[:len(h.data)-1] // -1 - h.down(0) - return t -} - -func (h *IntHeap) up(i int) { - for { - p := (i - 1) / 2 - if h.data[p] <= h.data[i] { - break - } - // swap - h.data[p], h.data[i] = h.data[i], h.data[p] - // keep going up - i = p - } -} - -func (h *IntHeap) down(i0 int) { - i := i0 - for { - iv := h.data[i] - l := 2*i + 1 - if l >= len(h.data)-1 { - break - } - lv := h.data[l] - r := l + 1 - rv := h.data[r] - // need to pick between left and right, the smaller one should come up - if lv <= rv && iv > lv { - h.data[i] = lv - h.data[l] = iv - i = l - continue - } - if rv <= lv && iv > rv { - h.data[i] = rv - h.data[r] = iv - i = r - continue - } - // iv < rv && iv < lv - break - } -} diff --git a/legacy/structure/heap_test.go b/legacy/structure/heap_test.go deleted file mode 100644 index 6dfad56..0000000 --- a/legacy/structure/heap_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build ignore - -package structure_test - -import ( - "sort" - "testing" - - "github.com/dyweb/gommon/structure" - "github.com/stretchr/testify/assert" -) - -// TODO: assertion on edge cases ... and why GoLand is not having hint? due to dep mode enabled? -func TestIntHeap_Insert(t *testing.T) { - h := structure.IntHeap{} - h.Insert(2) - h.Insert(1) - t.Log(h) - t.Log(h.Poll()) - t.Log(h) - - h2 := structure.IntHeap{} - unsorted := []int{3, 1, 4, 5, 6, -1, 2, 8, 9, 1} - for i := 0; i < len(unsorted); i++ { - h2.Insert(unsorted[i]) - } - var sorted []int - for h2.Len() != 0 { - sorted = append(sorted, h2.Poll()) - } - sort.Ints(unsorted) - assert.Equal(t, unsorted, sorted) -} diff --git a/legacy/structure/pkg.go b/legacy/structure/pkg.go deleted file mode 100644 index 2d25c8b..0000000 --- a/legacy/structure/pkg.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build ignore - -/* -Package structure add some missing common data structures to Golang -*/ -package structure // import "github.com/dyweb/gommon/structure" diff --git a/legacy/structure/priority_queue.go b/legacy/structure/priority_queue.go deleted file mode 100644 index b647f29..0000000 --- a/legacy/structure/priority_queue.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build ignore - -package structure - -// TODO: interface & impl diff --git a/legacy/structure/set.go b/legacy/structure/set.go deleted file mode 100644 index 40bddef..0000000 --- a/legacy/structure/set.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build ignore - -package structure - -// Set is a map with string key and bool value -// It is not thread safe and modeled after https://github.com/deckarep/golang-set/blob/master/threadunsafe.go -type Set map[string]bool - -// NewSet return a pointer of a set using arguments passed to the function -func NewSet(args ...string) *Set { - // TODO: would this length for the map? - m := make(Set, len(args)) - for _, v := range args { - m[v] = true - } - return &m -} - -// Cardinality return the size of the set -func (set *Set) Cardinality() int { - return len(*set) -} - -// Size is an alias for Cardinality -func (set *Set) Size() int { - return len(*set) -} - -// Contains check if a key is presented in the map, it does NOT check the bool value -func (set *Set) Contains(key string) bool { - _, ok := (*set)[key] - return ok -} - -// Add add an element to set regardless of it is already in the set -func (set *Set) Add(key string) { - (*set)[key] = true -} - -// Equal check if two sets have exactly same elements -func (set *Set) Equal(other *Set) bool { - if set.Cardinality() != other.Cardinality() { - return false - } - for key := range *set { - if !other.Contains(key) { - return false - } - } - return true -} diff --git a/legacy/structure/set_test.go b/legacy/structure/set_test.go deleted file mode 100644 index 3427193..0000000 --- a/legacy/structure/set_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build ignore - -package structure - -import ( - "fmt" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestSet_Contains(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.True(s.Contains("a")) - assert.False(s.Contains("d")) -} - -func TestSet_Cardinality(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.Equal(3, s.Cardinality()) -} - -func TestSet_Size(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - assert.Equal(s.Size(), s.Cardinality()) -} - -func TestSet_Equal(t *testing.T) { - assert := asst.New(t) - s := NewSet("a", "b", "c") - s2 := NewSet("a") - s3 := NewSet("a") - assert.True(s.Equal(s)) - assert.False(s.Equal(s2)) - assert.True(s2.Equal(s3)) -} - -func ExampleNewSet() { - s := NewSet("a", "b", "c") - fmt.Println(s.Size()) - // Output: 3 -} diff --git a/legacy/structure/tree.go b/legacy/structure/tree.go deleted file mode 100644 index 3496ebd..0000000 --- a/legacy/structure/tree.go +++ /dev/null @@ -1,96 +0,0 @@ -// +build ignore - -package structure - -import ( - "io" - "os" -) - -var ( - vLineStart = []byte("├") // this is in extended ASCII http://www.theasciicode.com.ar/extended-ascii-code/box-drawings-single-line-vertical-right-character-ascii-code-195.html - vLine = []byte("│") // this is not |, it won't have space vertically - corner = []byte("└") - hLine = []byte("── ") // we got a space at the end of hLine - space = []byte(" ") - hSpace = []byte(" ") - nextLine = []byte("\n") -) - -type StringTreeNode struct { - Val string - Children []StringTreeNode -} - -func (tree *StringTreeNode) Append(child StringTreeNode) { - tree.Children = append(tree.Children, child) -} - -func (tree *StringTreeNode) Print() { - tree.PrintTo(os.Stdout) -} - -/* -tree command example output -. -├── benchgraph -│   ├── echart.go -│   ├── echart_test.go -│   ├── fixture -│   │   ├── zap-no-delete-field.txt -│   │   └── zap.txt -│   ├── main.go -│   ├── Makefile -│   ├── tsdb-bench.html -│   └── zap.html -├── gommon -│   └── main.go -└── README.md -*/ - -func (tree *StringTreeNode) PrintTo(w io.Writer) { - //treePrintHelper(tree, 0, true, false, w) - treePrintHelper(tree, 0, true, true, w) -} - -func treePrintHelper(tree *StringTreeNode, level int, lastOfParent bool, lastOfUs bool, w io.Writer) { - // print the prefix before me, both vertically and horizontally - for i := 0; i < level-2; i++ { - w.Write(vLine) - w.Write(hSpace) - } - // avoid the extra vertical line, we use level-2 and handle level-1 here - // main - // └── http - // │ └── auth - if level > 1 { - if lastOfParent { - w.Write(space) - w.Write(hSpace) - } else { - w.Write(vLine) - w.Write(hSpace) - } - } - // ├── or └── - if level != 0 { - if !lastOfUs { - w.Write(vLineStart) - } else { - w.Write(corner) - } - w.Write(hLine) - } - // my value - w.Write([]byte(tree.Val)) - w.Write(nextLine) - // children - level++ - n := len(tree.Children) - for i := 0; i < n-1; i++ { - treePrintHelper(&tree.Children[i], level, lastOfUs, false, w) - } - if n > 0 { - treePrintHelper(&tree.Children[n-1], level, lastOfUs, true, w) - } -} diff --git a/legacy/structure/tree_test.go b/legacy/structure/tree_test.go deleted file mode 100644 index 6f446b8..0000000 --- a/legacy/structure/tree_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build ignore - -package structure - -import ( - "bytes" - "testing" - - asst "github.com/stretchr/testify/assert" -) - -func TestStringTreeNode_PrintTo(t *testing.T) { - assert := asst.New(t) - expected := - `root -├── level1-A -│ └── level2-A -└── level1-B -` - root := StringTreeNode{Val: "root"} - root.Append(StringTreeNode{Val: "level1-A"}) - root.Children[0].Append(StringTreeNode{Val: "level2-A"}) - root.Append(StringTreeNode{Val: "level1-B"}) - var b bytes.Buffer - root.PrintTo(&b) - assert.Equal(expected, string(b.Bytes())) - - // FIXED: there were extra vertical lines - // main - // └── http - // │ └── auth - root.Children = root.Children[0:1] - expected = - `root -└── level1-A - └── level2-A -` - b.Reset() - root.PrintTo(&b) - assert.Equal(expected, string(b.Bytes())) - - // TODO: test more complex situation like this - //. - //├── benchgraph - //│   ├── echart.go - //│   ├── echart_test.go - //│ │ └── zap-no-delete-field.txt - //│ │ - //│   └── fixture - //│      ├── zap-no-delete-field.txt - //│      └── zap.txt - //├── gommon - //│   └── main.go - //└── README.md -}