From 6eb9b04f70bb16a7c451a0a270b70c4536675371 Mon Sep 17 00:00:00 2001 From: Dmitry Mozzherin Date: Sun, 6 Feb 2022 10:03:24 -0600 Subject: [PATCH] add optional log aggregation to NSQ-messaging service (close #77) --- LICENSE | 2 +- README.md | 105 +++++++++++++++++++++++---------- config/config.go | 66 +++++++++++++++------ gnverifier.go | 28 +++++---- gnverifier/LICENSE | 2 +- gnverifier/cmd/gnverifier.yaml | 15 +++++ gnverifier/cmd/root.go | 37 +++++++++++- go.mod | 3 + go.sum | 8 +-- interface.go | 16 +++++ io/web/server.go | 38 ++++++++++-- 11 files changed, 248 insertions(+), 72 deletions(-) diff --git a/LICENSE b/LICENSE index 64d2eb1..4f370e2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020-2021 gnames +Copyright (c) 2020-2022 gnames Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index da6dacb..5c5d40b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ search feature. * [Installation](#installation) * [Using Homebrew on Mac OS X, Linux, and Linux on Windows ([WSL2])](#using-homebrew-on-mac-os-x-linux-and-linux-on-windows-wsl2) * [MS Windows](#ms-windows) - * [Linux and Mac](#linux-and-mac) + * [Linux and Mac (without Homebrew)](#linux-and-mac-without-homebrew) * [Compile from source](#compile-from-source) * [Usage](#usage) * [As a web service](#as-a-web-service) @@ -33,6 +33,8 @@ search feature. * [only_preferred](#only_preferred) * [quiet](#quiet) * [sources](#sources) + * [web-logs](#web-logs) + * [nsqd-tcp](#nsqd-tcp) * [Configuration file](#configuration-file) * [Advanced Search Query Language](#advanced-search-query-language) * [Examples of searches](#examples-of-searches) @@ -49,15 +51,15 @@ If you want to cite GNverifier, use [DOI generated by Zenodo][Zenodo DOI]: * Small and fast app to verify scientific names against many biodiversity databases. The app is a client to a [verifier API]. * It provides 6 different match levels: - * Exact: complete match with a canonical form or a full name-string from a - data source. - * Fuzzy: if exact match did not happen, it tries to match name-strings - assuming spelling errors. - * Partial: strips middle or last epithets from bi- or multi-nomial names - and tries to match what is left. - * PartialFuzzy: the same as Partial but assuming spelling mistakes. - * Virus: verification of virus names. - * FacetedSearch: marks [advanced-search](#advanced-search) queries. + * **Exact**: complete match with a canonical form or a full name-string + from a data source. + * **Fuzzy**: if exact match did not happen, it tries to match name-strings + assuming spelling errors. + * **Partial**: strips middle or last epithets from bi- or multi-nomial names + and tries to match what is left. + * **PartialFuzzy**: the same as Partial but assuming spelling mistakes. + * **Virus**: verification of virus names. + * **FacetedSearch**: marks [advanced-search](#advanced-search) queries. * Taxonomic resolution. If a database contains taxonomic information, it returns the currently accepted name for the provided name-string. * Best match is returned according to the match score. Data sources with some @@ -76,7 +78,7 @@ If you want to cite GNverifier, use [DOI generated by Zenodo][Zenodo DOI]: to find abbreviated names, search by author, year etc. * Supports feeding data via pipes of an operating system. This feature allows to chain the program together with other tools. -* `GNverifier` includes a web-based graphical user interface identical to its +* [GNverifier] includes a web-based graphical user interface identical to its "official" [web-service]. ## Installation @@ -88,7 +90,7 @@ developed for Mac OS X. Now it is also available on Linux, and can easily be used on Windows 10, if Windows Subsystem for Linux (WSL) is [installed][WSL install]. -To use `GNverifier` with Homebrew: +To use [GNverifier] with Homebrew: 1. Install [Homebrew] @@ -101,7 +103,7 @@ brew install gnverifier ### MS Windows -Download the latest release from [github], unzip. +Download the [latest release] from GitHub, unzip. One possible way would be to create a default folder for executables and place ``GNverifier`` there. @@ -118,17 +120,17 @@ copy path_to\gnverifier.exe C:\Users\your_username\bin environment variable. Another, simpler way, would be to use ``cd C:\Users\your_username\bin`` command -in ``cmd`` terminal window. The ``GNverifier`` program then will be automatically +in ``cmd`` terminal window. The [GNverifier] program then will be automatically found by Windows operating system when you run its commands from that directory. You can also read a more detailed guide for Windows users in [a PDF document][win-pdf]. -### Linux and Mac +### Linux and Mac (without Homebrew) -Download the latest release from [github], untar, and install binary somewhere -in your path. +If [Homebrew] is not installed, download the [latest release] from GitHub, +untar, and install binary somewhere in your path. ```bash tar xvf gnverifier-linux-0.1.0.tar.xz @@ -146,8 +148,8 @@ go get github.com/gnames/gnverifier/gnverifier ## Usage -``GNverifier`` takes one name-string or a text file with one name-string per -line as an argument, sends a query with these data to [remote ``gnames`` +[GNverifier] takes one name-string or a text file with one name-string per +line as an argument, sends a query with these data to a [remote GNames server][gnames] to match the name-strigs against many different biodiversity databases and returns results to STDOUT either in JSON, CSV or TSV format. @@ -278,7 +280,7 @@ significantly speeds up parsin of the JSON on the user side. #### jobs -If the list of names if very large, it is possible to tell GNverifier to +If the list of names if very large, it is possible to tell [GNverifier] to run requests in parallel. In this example GNverifier will run 8 processes simultaneously. The order of returned names will be somewhat randomized. @@ -317,7 +319,7 @@ Removes log messages from the output. Note that results of verification go to STDOUT, while log messages go to STDERR. So instead of using `-q` flag STDERR can be redirected to `/dev/null`: -``` +```bash gnverifier "Puma concolor" -q >verif-results.csv #or @@ -327,11 +329,11 @@ gnverifier "Puma concolor 2>/dev/null >verif-results.csv #### sources -By default ``GNverifier`` returns only one "best" result of a match. If a user +By default [GNverifier] returns only one "best" result of a match. If a user has a particular interest in a data set, s/he can set it with this option, and all matches that exist for this source will be returned as well. You need to provide a data source id for a dataset. Ids can be found at the following -[URL][data_source_ids]. Some of them are provided in the ``GNverifier`` help +[URL][data_source_ids]. Some of them are provided in the GNverifier help output as well. Data from such sources will be returned in preferred_results section of JSON @@ -354,19 +356,57 @@ gnverifier "Bubo bubo" -s 0 # potentially even more results get returned by adding --all_matches flag gnverifier "Bubo bubo" -s 0 -M ``` + The `sources` option would overwrite `ds:` settings in case of advanced search. +### web-logs + +Requires `--port`. Enables output of logs for web-services. + +```bash +gnverifier -p 8777 --web-logs +``` + +### nsqd-tcp + +Rrequires `--port`. Allows to redirect web-service log output to [NSQ] +messaging server's TCP-based endpoint. It is handy for aggregations of logs +from [GNverifier] web-services running inside of Docker containers or in +Kubernetes pods. + +```bash +gnverifier -p 8777 --nsqd-tcp=localhost:4150 +# with logs printed out +gnverifier -p 8777 --nsqd-tcp=localhost:4150 --with-logs +``` + ### Configuration file If you find yourself using the same flags over and over again, it makes sense to edit configuration file instead. It is located at `$HOME/.config/gnverifier.yaml`. After that you do not need to use command line -options and flags. +options and flags. Configuration file is self-documented, the [default +gnverifier.yaml] is located on GitHub ```bash gnverifier file.txt ``` +In case if [GNverifier] runs as a web-based user interface, it is also +possible to use environment variables for configuration. + +| Env. Var. | Configuration | +| :---------------------- | :----------------- | +| GNV_FORMAT | Format | +| GNV_PREFERRED_ONLY | PreferredOnly | +| GNV_DATA_SOURCES | DataSources | +| GNV_WITH_ALL_MATCHES | WithAllMatches | +| GNV_WITH_CAPITALIZATION | WithCapitalization | +| GNV_VERIFIER_URL | VerifierURL | +| GNV_JOBS | Jobs | +| GNV_WEB_LOGS_NSQD_TCP | WebLogsNsqdTCP | +| GNV_WITH_WEB_LOGS | WithWebLogs | + ### Advanced Search Query Language Example: `g:M. sp:gallop. au:Oliv. y:1750-1799` or `n:M. gallop. Oliv. 1750-1799` @@ -409,7 +449,7 @@ It includes following operators: `tx:Magnoliopsida`). `all:` -: If true, [gnverifier] will show all results, not only the best ones. +: If true, [GNverifier] will show all results, not only the best ones. The setting can be `true` or `false` (`all:t`, `all:f`). This setting will become true if `sources` command line option is set to `0`. @@ -444,25 +484,28 @@ gnverifier "g:Cara. isp:daurica ds:1,12" Authors: [Dmitry Mozzherin][dimus] -Copyright © 2020-2021 Dmitry Mozzherin. See [LICENSE] for further +Copyright © 2020-2022 Dmitry Mozzherin. See [LICENSE] for further details. -[WSL2]: https://docs.microsoft.com/en-us/windows/wsl/install -[verifier API]: https://apidoc.globalnames.org/gnames-beta [Catalogue of Life]: https://catalogueoflife.org/ [GBIF]: https://www.gbif.org/ +[GNverifier]: https://github.com/gnames/gnverifier [Homebrew]: https://brew.sh/ +[LICENSE]: https://github.com/gnames/gnverifier/blob/master/LICENSE +[NSQ]: https://nsq.io/overview/quick_start.html [WSL install]: https://docs.microsoft.com/en-us/windows/wsl/install-win10 +[WSL2]: https://docs.microsoft.com/en-us/windows/wsl/install [WoRMS]: https://marinespecies.org/ [Zenodo DOI]: https://zenodo.org/badge/latestdoi/297323648 [data_source_ids]: https://verifier.globalnames.org/data_sources +[default gnverifier.yaml]: https://github.com/gnames/gnverifier/blob/master/gnverifier/cmd/gnverifier.yaml [dimus]: https://github.com/dimus -[github]: https://github.com/gnames/gnverifier/releases/latest -[gnames]: https://hub.apitree.com/dimus/gnames/ +[latest release]: https://github.com/gnames/gnverifier/releases/latest +[gnames]: https://apidoc.globalnames.org/gnames-beta [go-install]: https://golang.org/doc/install -[LICENSE]: https://github.com/gnames/gnverifier/blob/master/LICENSE [test directory]: https://github.com/gnames/gnverifier/tree/master/testdata [uBio]: https://ubio.org/ +[verifier API]: https://apidoc.globalnames.org/gnames-beta [web-service]: https://verifier.globalnames.org [win-pdf]: https://github.com/gnames/gnverifier/blob/master/use-gnverifier-windows.pdf [winpath]: https://www.computerhope.com/issues/ch000549.htm diff --git a/config/config.go b/config/config.go index 7d091a1..96af3d0 100644 --- a/config/config.go +++ b/config/config.go @@ -6,18 +6,43 @@ import ( // Config collects and stores external configuration data. type Config struct { + // Batch is the size of the string slices fed into input channel for + // verification. + Batch int + + // DataSources are IDs of DataSources that are important for + // user. Normally only one "the best" reusult returns. If user gives + // preferred sources, then matches from these sources are also + // returned. + DataSources []int + // Format determins the output. It can be either JSON or CSV. Format gnfmt.Format + // Jobs is the number of verification jobs to run in parallel. + Jobs int + + // NamesNumThreshold the number of names after which POST gets redirected + // to GET. + NamesNumThreshold int + + // VerifierURL URL for gnames verification service. It only needs to + // be changed if user sets local version of gnames. + VerifierURL string + // PreferredOnly hides BestResult if the user wants to see only // preferred results. PreferredOnly bool - // DataSources are IDs of DataSources that are important for - // user. Normally only one "the best" reusult returns. If user gives - // preferred sources, then matches from these sources are also - // returned. - DataSources []int + // WebLogsNsqdTCP provides an address to the NSQ messenger TCP service. If + // this value is set and valid, the web logs will be published to the NSQ. + // The option is ignored if `Port` is not set. + // + // If WithWebLogs option is set to `false`, but `WebLogsNsqdTCP` is set to a + // valid URL, the logs will be sent to the NSQ messanging service, but they + // wil not appear as STRERR output. + // Example: `127.0.0.1:4150` + WebLogsNsqdTCP string // WithAllMatches flag; if true, results include all matches per source, // not only the best match. @@ -27,20 +52,9 @@ type Config struct { // will be capitalized when appropriate. WithCapitalization bool - // VerifierURL URL for gnames verification service. It only needs to - // be changed if user sets local version of gnames. - VerifierURL string - - // Jobs is the number of verification jobs to run in parallel. - Jobs int - - // Batch is the size of the string slices fed into input channel for - // verification. - Batch int - - // NamesNumThreshold the number of names after which POST gets redirected - // to GET. - NamesNumThreshold int + // WithWebLogs flag enables logs when running web-service. This flag is + // ignored if `Port` value is not set. + WithWebLogs bool } // Option is a type of all options for Config. @@ -104,6 +118,20 @@ func OptNamesNumThreshold(i int) Option { } } +// OptWebLogsNsqdTCP provides a URL to NSQ messanging service. +func OptWebLogsNsqdTCP(s string) Option { + return func(cfg *Config) { + cfg.WebLogsNsqdTCP = s + } +} + +// OptWithWebLogs sets the WithWebLogs field. +func OptWithWebLogs(b bool) Option { + return func(cfg *Config) { + cfg.WithWebLogs = b + } +} + // New is a Config constructor that takes external options to // update default values to external ones. func New(opts ...Option) Config { diff --git a/gnverifier.go b/gnverifier.go index 0c9a4fa..7e0a1c7 100644 --- a/gnverifier.go +++ b/gnverifier.go @@ -15,18 +15,18 @@ import ( ) type gnverifier struct { - config config.Config + cfg config.Config verifier verifier.Verifier } // New constructs an object that implements GNVerifier interface // and can be used for matching strings to scientfic names. func New( - cnf config.Config, + cfg config.Config, vfr verifier.Verifier, ) GNverifier { return gnverifier{ - config: cnf, + cfg: cfg, verifier: vfr, } } @@ -49,14 +49,14 @@ func (gnv gnverifier) DataSource(id int) (vlib.DataSource, error) { // ChangeConfig modifies configuration. func (gnv gnverifier) ChangeConfig(opts ...config.Option) GNverifier { for i := range opts { - opts[i](&gnv.config) + opts[i](&gnv.cfg) } return gnv } // Config returns configuration data. func (gnv gnverifier) Config() config.Config { - return gnv.config + return gnv.cfg } // VerifyOne verifies one input string and returns results @@ -88,14 +88,14 @@ func (gnv gnverifier) VerifyStream( out chan []vlib.Name, ) { var wg sync.WaitGroup - wg.Add(gnv.config.Jobs) + wg.Add(gnv.cfg.Jobs) ctx, cancel := context.WithCancel(context.Background()) defer cancel() vwChan := gnv.loadNames(ctx, in) - for i := 0; i < gnv.config.Jobs; i++ { + for i := 0; i < gnv.cfg.Jobs; i++ { go gnv.VerifyWorker(ctx, vwChan, out, &wg) } @@ -155,9 +155,17 @@ func (gnv gnverifier) loadNames( func (gnv gnverifier) setParams(names []string) vlib.Input { res := vlib.Input{ NameStrings: names, - DataSources: gnv.config.DataSources, - WithCapitalization: gnv.config.WithCapitalization, - WithAllMatches: gnv.config.WithAllMatches, + DataSources: gnv.cfg.DataSources, + WithCapitalization: gnv.cfg.WithCapitalization, + WithAllMatches: gnv.cfg.WithAllMatches, } return res } + +func (gnv gnverifier) WithWebLogs() bool { + return gnv.cfg.WithWebLogs +} + +func (gnv gnverifier) WebLogsNsqdTCP() string { + return gnv.cfg.WebLogsNsqdTCP +} diff --git a/gnverifier/LICENSE b/gnverifier/LICENSE index 8d6c2bf..3e60c56 100644 --- a/gnverifier/LICENSE +++ b/gnverifier/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright © 2020-2021 Dmitry Mozzherin +Copyright © 2020-2022 Dmitry Mozzherin Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/gnverifier/cmd/gnverifier.yaml b/gnverifier/cmd/gnverifier.yaml index 10eefdc..ef244ef 100644 --- a/gnverifier/cmd/gnverifier.yaml +++ b/gnverifier/cmd/gnverifier.yaml @@ -37,3 +37,18 @@ # # Jobs: 4 +# WebLogsNsqdTCP provides an address to the NSQ messenger TCP service. If +# this value is set and valid, the web logs will be published to the NSQ. +# The option is ignored if `Port` is not set. +# +# If WithWebLogs option is set to `false`, but `WebLogsNsqdTCP` is set to a +# valid URL, the logs will be sent to the NSQ messanging service, but they +# wil not appear as STRERR output. +# Example: `127.0.0.1:4150` +# +# WebLogsNsqdTCP: + +# WithWebLogs flag enables logs when running web-service. This flag is +# ignored if `Port` value is not set. +# +# WithWebLogs: false diff --git a/gnverifier/cmd/root.go b/gnverifier/cmd/root.go index 59d1114..dd24828 100644 --- a/gnverifier/cmd/root.go +++ b/gnverifier/cmd/root.go @@ -42,12 +42,14 @@ var ( // configuration file, if it exists. type cfgData struct { Format string + Jobs int PreferredOnly bool PreferredSources []int + VerifierURL string + WebLogsNsqdTCP string WithAllMatches bool WithCapitalization bool - VerifierURL string - Jobs int + WithWebLogs bool } // rootCmd represents the base command when called without any subcommands @@ -121,6 +123,15 @@ https://github.com/gnames/gnverifier port, _ := cmd.Flags().GetInt("port") if port > 0 { + weblogs, _ := cmd.Flags().GetBool("web-logs") + if weblogs { + webOpts = append(webOpts, config.OptWithWebLogs(true)) + } + nsqAddr, _ := cmd.Flags().GetString("nsqd-tcp") + if nsqAddr != "" { + webOpts = append(webOpts, config.OptWebLogsNsqdTCP(nsqAddr)) + } + log.SetFormatter(&log.JSONFormatter{}) cnf := config.New(webOpts...) vfr := verifrest.New(cnf.VerifierURL) @@ -193,6 +204,8 @@ func init() { rootCmd.Flags().StringP("verifier_url", "v", "", `URL for verification service. Default: https://verifier.globalnames.org/api/v0`) + rootCmd.Flags().BoolP("web-logs", "", false, "enable logs for the web service") + rootCmd.Flags().StringP("nsqd-tcp", "", "", "an addresss pointing to NSQ TCP service for logs redirection (e.g. 127.0.0.1:4150)") } // initConfig reads in config file and ENV variables if set. @@ -211,6 +224,20 @@ func initConfig() { viper.AddConfigPath(configDir) viper.SetConfigName(configFile) + // Set environment variables to override + // config file settings + _ = viper.BindEnv("Format", "GNV_FORMAT") + _ = viper.BindEnv("PreferredOnly", "GNV_PREFERRED_ONLY") + _ = viper.BindEnv("DataSources", "GNV_DATA_SOURCES") + _ = viper.BindEnv("WithAllMatches", "GNV_WITH_ALL_MATCHES") + _ = viper.BindEnv("WithCapitalization", "GNV_WITH_CAPITALIZATION") + _ = viper.BindEnv("VerifierURL", "GNV_VERIFIER_URL") + _ = viper.BindEnv("Jobs", "GNV_JOBS") + _ = viper.BindEnv("WebLogsNsqdTCP", "GNV_WEB_LOGS_NSQD_TCP") + _ = viper.BindEnv("WithWebLogs", "GNV_WITH_WEB_LOGS") + + viper.AutomaticEnv() // read in environment variables that match + configPath := filepath.Join(configDir, fmt.Sprintf("%s.yaml", configFile)) touchConfigFile(configPath) @@ -255,6 +282,12 @@ func getOpts() { if cfg.Jobs > 0 { opts = append(opts, config.OptJobs(cfg.Jobs)) } + if cfg.WebLogsNsqdTCP != "" { + opts = append(opts, config.OptWebLogsNsqdTCP(cfg.WebLogsNsqdTCP)) + } + if cfg.WithWebLogs { + opts = append(opts, config.OptWithWebLogs(true)) + } } // showVersionFlag provides version and the build timestamp. If it returns diff --git a/go.mod b/go.mod index 8f57093..03cdf5a 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/labstack/echo/v4 v4.6.3 github.com/labstack/gommon v0.3.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.4.1 + github.com/sfgrp/lognsq v0.1.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.10.1 @@ -24,6 +25,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/snappy v0.0.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -34,6 +36,7 @@ require ( github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nsqio/go-nsq v1.1.0 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pointlander/compress v1.1.1-0.20190518213731-ff44bd196cc3 // indirect diff --git a/go.sum b/go.sum index fa84721..1ca7c5d 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,6 @@ github.com/gnames/gnfmt v0.2.0 h1:CjE1HxdqyTwufua5wMCdILWnCsCfRiHe5G4TgxR8aAI= github.com/gnames/gnfmt v0.2.0/go.mod h1:0Aog37s1ZNpmUwVQOf+lnx0SQq8r2EvfE/pLYGiJlJQ= github.com/gnames/gnlib v0.3.2/go.mod h1:IIo4lQ8hmW/pCmudLgwJ4KdoKWF6B4jFCdVcowG/evw= github.com/gnames/gnlib v0.6.6/go.mod h1:DgK8NcrG2YhOS1Nx0cxt8+Gdwu1cw/abSZdhkQKjIH8= -github.com/gnames/gnlib v0.8.0 h1:ZLkpBn7/mkU9VYQ3CiZozZoEPYghzHrkHAo0qW4sILI= -github.com/gnames/gnlib v0.8.0/go.mod h1:19DRcg4P/oP623qYZjG1yIBUpdqIh0UXeq2GXdz6/TU= github.com/gnames/gnlib v0.8.1 h1:mvEXvDa1ZNqvuc+lNBwZDCbgbDYHeZj61rBGuiN2hwk= github.com/gnames/gnlib v0.8.1/go.mod h1:19DRcg4P/oP623qYZjG1yIBUpdqIh0UXeq2GXdz6/TU= github.com/gnames/gnparser v1.5.5/go.mod h1:prvrAYGrtT/LxN8ovG3LFojgjR7VDcao/cNWYjUyYw0= @@ -191,6 +189,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= @@ -370,6 +369,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -421,6 +421,8 @@ github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDN github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sfgrp/lognsq v0.1.1 h1:Cg6J5TO0AlDg7OpmQSdM2liY0lhQEFf3fWKSW6V9xaY= +github.com/sfgrp/lognsq v0.1.1/go.mod h1:aNbHWh6z8OIKU/3jwQIYnd3hDh0f4Nmdj0kyAiHKvh0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -431,8 +433,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/afero v1.8.1 h1:izYHOT71f9iZ7iq37Uqjael60/vYC6vMtzedudZ0zEk= github.com/spf13/afero v1.8.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= diff --git a/interface.go b/interface.go index 59de537..3b98e86 100644 --- a/interface.go +++ b/interface.go @@ -11,6 +11,11 @@ import ( // methods needed to verify (reconcile/resolve) strings to scientific // names. type GNverifier interface { + NameVerifier + WebLogger +} + +type NameVerifier interface { // VerifyOne takes a name-string and returns the result of verification. VerifyOne(name string) (vlib.Name, error) @@ -41,3 +46,14 @@ type GNverifier interface { // GetVersion returns version of the gnverifier GetVersion() gnvers.Version } + +// WebLogger contains methods for enabling and aggregating logs from the +// GNverifier web-service. +type WebLogger interface { + // WithWebLogs returns true if web logs are enabled. + WithWebLogs() bool + + // WebLogsNsqdTCP returns an address to a NSQ messaging TCP service or + // an empty string. + WebLogsNsqdTCP() string +} diff --git a/io/web/server.go b/io/web/server.go index 8c5b32e..fbf00b7 100644 --- a/io/web/server.go +++ b/io/web/server.go @@ -20,10 +20,11 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/labstack/gommon/log" + nsqcfg "github.com/sfgrp/lognsq/config" + "github.com/sfgrp/lognsq/ent/nsq" + "github.com/sfgrp/lognsq/io/nsqio" ) -const withLogs = true - type formInput struct { Names string `query:"names" form:"names"` Format string `query:"format" form:"format"` @@ -44,8 +45,10 @@ func Run(gnv gnverifier.GNverifier, port int) { e := echo.New() e.Use(middleware.Gzip()) - if withLogs { - e.Use(middleware.Logger()) + + loggerNSQ := setLogger(e, gnv) + if loggerNSQ != nil { + defer loggerNSQ.Stop() } e.Renderer, err = NewTemplate() @@ -308,3 +311,30 @@ func formatRows(data Data, prefOnly bool, f gnfmt.Format) []string { } return res } + +func setLogger(e *echo.Echo, m gnverifier.GNverifier) nsq.NSQ { + nsqAddr := m.WebLogsNsqdTCP() + withLogs := m.WithWebLogs() + + if nsqAddr != "" { + cfg := nsqcfg.Config{ + StderrLogs: withLogs, + Topic: "gnverifier", + Address: nsqAddr, + } + remote, err := nsqio.New(cfg) + logCfg := middleware.DefaultLoggerConfig + if err == nil { + logCfg.Output = remote + } + e.Use(middleware.LoggerWithConfig(logCfg)) + if err != nil { + log.Warn(err) + } + return remote + } else if withLogs { + e.Use(middleware.Logger()) + return nil + } + return nil +}