From 6f168387f909bb7cbd8d0d280bebe9bd44a880c2 Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Mon, 4 Apr 2022 21:02:32 +0300 Subject: [PATCH 1/6] Small fmt --- cmd/api-firewall/internal/handlers/check.go | 1 - cmd/api-firewall/internal/handlers/openapi.go | 1 - cmd/api-firewall/internal/handlers/routes.go | 3 +-- cmd/api-firewall/tests/main_test.go | 8 ++++---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/api-firewall/internal/handlers/check.go b/cmd/api-firewall/internal/handlers/check.go index 001759c..82efda5 100644 --- a/cmd/api-firewall/internal/handlers/check.go +++ b/cmd/api-firewall/internal/handlers/check.go @@ -5,7 +5,6 @@ import ( "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" - "github.com/wallarm/api-firewall/internal/platform/proxy" "github.com/wallarm/api-firewall/internal/platform/web" ) diff --git a/cmd/api-firewall/internal/handlers/openapi.go b/cmd/api-firewall/internal/handlers/openapi.go index d3f8d13..4c3cabe 100644 --- a/cmd/api-firewall/internal/handlers/openapi.go +++ b/cmd/api-firewall/internal/handlers/openapi.go @@ -13,7 +13,6 @@ import ( "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" "github.com/valyala/fastjson" - "github.com/wallarm/api-firewall/internal/config" "github.com/wallarm/api-firewall/internal/platform/oauth2" "github.com/wallarm/api-firewall/internal/platform/openapi3" diff --git a/cmd/api-firewall/internal/handlers/routes.go b/cmd/api-firewall/internal/handlers/routes.go index 6dfe493..f984705 100644 --- a/cmd/api-firewall/internal/handlers/routes.go +++ b/cmd/api-firewall/internal/handlers/routes.go @@ -2,7 +2,6 @@ package handlers import ( "crypto/rsa" - "github.com/wallarm/api-firewall/internal/platform/denylist" "io/ioutil" "net/url" "os" @@ -14,9 +13,9 @@ import ( "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" "github.com/valyala/fastjson" - "github.com/wallarm/api-firewall/internal/config" "github.com/wallarm/api-firewall/internal/mid" + "github.com/wallarm/api-firewall/internal/platform/denylist" woauth2 "github.com/wallarm/api-firewall/internal/platform/oauth2" "github.com/wallarm/api-firewall/internal/platform/openapi3" "github.com/wallarm/api-firewall/internal/platform/proxy" diff --git a/cmd/api-firewall/tests/main_test.go b/cmd/api-firewall/tests/main_test.go index 0288d32..7812a5a 100644 --- a/cmd/api-firewall/tests/main_test.go +++ b/cmd/api-firewall/tests/main_test.go @@ -4,10 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/golang/mock/gomock" - "github.com/sirupsen/logrus" - "github.com/valyala/fasthttp" - "github.com/wallarm/api-firewall/internal/platform/denylist" "io" "net" "net/url" @@ -17,8 +13,12 @@ import ( "testing" "time" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + "github.com/valyala/fasthttp" "github.com/wallarm/api-firewall/cmd/api-firewall/internal/handlers" "github.com/wallarm/api-firewall/internal/config" + "github.com/wallarm/api-firewall/internal/platform/denylist" "github.com/wallarm/api-firewall/internal/platform/openapi3" "github.com/wallarm/api-firewall/internal/platform/router" "github.com/wallarm/api-firewall/internal/platform/tests" From a8f3d65776dd1a2570a84189d756ab3001b6878b Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Sat, 9 Apr 2022 00:06:53 +0300 Subject: [PATCH 2/6] Update cache params --- cmd/api-firewall/tests/main_test.go | 4 +--- internal/config/config.go | 4 +--- internal/platform/denylist/denylist.go | 8 +++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/api-firewall/tests/main_test.go b/cmd/api-firewall/tests/main_test.go index 7812a5a..54c8823 100644 --- a/cmd/api-firewall/tests/main_test.go +++ b/cmd/api-firewall/tests/main_test.go @@ -325,9 +325,7 @@ func (s *ServiceTests) testBlockMode(t *testing.T) { func (s *ServiceTests) testDenylist(t *testing.T) { cacheCfg := config.Cache{ - NumCounters: 100000000, - MaxCost: 2147483648, - BufferItems: 64, + MaxElements: 100_000_000, } tokensCfg := config.Token{ diff --git a/internal/config/config.go b/internal/config/config.go index 18d05a9..322b67a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,9 +31,7 @@ type JWT struct { } type Cache struct { - NumCounters int64 `conf:"default:100000000"` - MaxCost int64 `conf:"default:2147483648"` - BufferItems int64 `conf:"default:64"` + MaxElements int64 `conf:"default:1_000_000"` } type Token struct { diff --git a/internal/platform/denylist/denylist.go b/internal/platform/denylist/denylist.go index 22162af..6ca264b 100644 --- a/internal/platform/denylist/denylist.go +++ b/internal/platform/denylist/denylist.go @@ -10,6 +10,8 @@ import ( "github.com/wallarm/api-firewall/internal/config" ) +const BufferItems = 64 + type DeniedTokens struct { Cache *ristretto.Cache ElementsNum int @@ -18,9 +20,9 @@ type DeniedTokens struct { func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*DeniedTokens, error) { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: cfg.Denylist.Cache.NumCounters, - MaxCost: cfg.Denylist.Cache.MaxCost, - BufferItems: cfg.Denylist.Cache.BufferItems, + NumCounters: cfg.Denylist.Cache.MaxElements, + MaxCost: cfg.Denylist.Cache.MaxElements * 10, + BufferItems: BufferItems, }) if err != nil { return nil, err From afd1d254bdc66a84ee2bb4467847a484e888aee6 Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Sat, 9 Apr 2022 10:36:47 +0300 Subject: [PATCH 3/6] Fix MaxCost values --- internal/platform/denylist/denylist.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/platform/denylist/denylist.go b/internal/platform/denylist/denylist.go index 6ca264b..4fcfabb 100644 --- a/internal/platform/denylist/denylist.go +++ b/internal/platform/denylist/denylist.go @@ -20,8 +20,8 @@ type DeniedTokens struct { func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*DeniedTokens, error) { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: cfg.Denylist.Cache.MaxElements, - MaxCost: cfg.Denylist.Cache.MaxElements * 10, + NumCounters: cfg.Denylist.Cache.MaxElements * 10, + MaxCost: cfg.Denylist.Cache.MaxElements, BufferItems: BufferItems, }) if err != nil { From d09914e64bf03fb4d3d69747fe3641826f0f2f05 Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Sat, 9 Apr 2022 13:36:11 +0300 Subject: [PATCH 4/6] Delete cache configuration env variables --- cmd/api-firewall/tests/main_test.go | 7 +- internal/config/config.go | 5 -- internal/platform/denylist/denylist.go | 119 ++++++++++++++----------- 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/cmd/api-firewall/tests/main_test.go b/cmd/api-firewall/tests/main_test.go index 54c8823..a1f615c 100644 --- a/cmd/api-firewall/tests/main_test.go +++ b/cmd/api-firewall/tests/main_test.go @@ -324,10 +324,6 @@ func (s *ServiceTests) testBlockMode(t *testing.T) { func (s *ServiceTests) testDenylist(t *testing.T) { - cacheCfg := config.Cache{ - MaxElements: 100_000_000, - } - tokensCfg := config.Token{ CookieName: testDeniedCookieName, HeaderName: "", @@ -344,8 +340,7 @@ func (s *ServiceTests) testDenylist(t *testing.T) { }, Denylist: struct { Tokens config.Token - Cache config.Cache - }{Tokens: tokensCfg, Cache: cacheCfg}, + }{Tokens: tokensCfg}, } logger := logrus.New() diff --git a/internal/config/config.go b/internal/config/config.go index 322b67a..47929ec 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,10 +30,6 @@ type JWT struct { SecretKey string `conf:""` } -type Cache struct { - MaxElements int64 `conf:"default:1_000_000"` -} - type Token struct { CookieName string `conf:""` HeaderName string `conf:""` @@ -43,7 +39,6 @@ type Token struct { type Denylist struct { Tokens Token - Cache Cache } type Introspection struct { diff --git a/internal/platform/denylist/denylist.go b/internal/platform/denylist/denylist.go index 4fcfabb..99ee6cb 100644 --- a/internal/platform/denylist/denylist.go +++ b/internal/platform/denylist/denylist.go @@ -4,84 +4,99 @@ import ( "bufio" "io" "os" + "strings" "github.com/dgraph-io/ristretto" "github.com/sirupsen/logrus" "github.com/wallarm/api-firewall/internal/config" ) -const BufferItems = 64 +const ( + BufferItems = 64 + ElementCost = 1 +) type DeniedTokens struct { Cache *ristretto.Cache - ElementsNum int + ElementsNum int64 } func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*DeniedTokens, error) { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: cfg.Denylist.Cache.MaxElements * 10, - MaxCost: cfg.Denylist.Cache.MaxElements, - BufferItems: BufferItems, - }) + if cfg.Denylist.Tokens.File == "" { + return nil, nil + } + + var totalEntries int64 + var totalCacheCapacity int64 + + // open tokens storage + f, err := os.Open(cfg.Denylist.Tokens.File) if err != nil { return nil, err } - totalEntries := 0 + // count non-empty entries and total cache capacity in bytes + c := bufio.NewScanner(f) + for c.Scan() { + if c.Text() != "" { + totalCacheCapacity += int64(len(c.Text())) + totalEntries += 1 + } + } + err = c.Err() + if err != nil { + return nil, err + } - // Loading tokens to the cache - if cfg.Denylist.Tokens.File != "" { + // go to the beginning of the storage file + if _, err = f.Seek(0, io.SeekStart); err != nil { + return nil, err + } - f, err := os.Open(cfg.Denylist.Tokens.File) - if err != nil { - return nil, err - } + logger.Debugf("Denylist: total entries (lines) found in the file: %d", totalEntries) - // count non-empty entries - c := bufio.NewScanner(f) - for c.Scan() { - if c.Text() != "" { - totalEntries += 1 - } - } - err = c.Err() - if err != nil { - return nil, err - } + // max cost = total bytes found in the storage + 5% + maxCost := totalCacheCapacity + totalCacheCapacity/20 - if _, err = f.Seek(0, io.SeekStart); err != nil { - return nil, err - } + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: maxCost * 10, // recommended value + MaxCost: maxCost, + BufferItems: BufferItems, + }) + if err != nil { + return nil, err + } + + var numOfElements int64 + totalEntries10P := totalEntries / 10 - logger.Debugf("Denylist: total entries (lines) found in the file: %d", totalEntries) - - totalEntries10P := totalEntries / 10 - numOfElements := 0 - current10P := 0 - s := bufio.NewScanner(f) - for s.Scan() { - if s.Text() != "" { - if ok := cache.Set(s.Text(), nil, 1); ok { - numOfElements += 1 - if numOfElements%totalEntries10P == 0 { - current10P += 10 - logger.Debugf("Denylist: loaded %d perecents of tokens. Total elements in the cache: %d", current10P, numOfElements) - } - } else { - logger.Errorf("Denylist: can't add the token to the cache: %s", s.Text()) + // 10% counter + counter10P := 0 + + // tokens loading to the cache + s := bufio.NewScanner(f) + for s.Scan() { + if s.Text() != "" { + if ok := cache.Set(strings.TrimSpace(s.Text()), nil, ElementCost); ok { + numOfElements += 1 + if numOfElements%totalEntries10P == 0 { + counter10P += 10 + logger.Debugf("Denylist: loaded %d perecents of tokens. Total elements in the cache: %d", counter10P, numOfElements) } - cache.Wait() + } else { + logger.Errorf("Denylist: can't add the token to the cache: %s", s.Text()) } + cache.Wait() } - err = s.Err() - if err != nil { - return nil, err - } + } + err = s.Err() + if err != nil { + return nil, err + } - if err := f.Close(); err != nil { - return nil, err - } + if err := f.Close(); err != nil { + return nil, err } return &DeniedTokens{Cache: cache, ElementsNum: totalEntries}, nil From b4341f14af60d92b6af8cfd49a51e2d0d2cc1525 Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Mon, 11 Apr 2022 15:26:47 +0300 Subject: [PATCH 5/6] Add debug message of cache capacity size --- internal/platform/denylist/denylist.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/platform/denylist/denylist.go b/internal/platform/denylist/denylist.go index 99ee6cb..d0546bd 100644 --- a/internal/platform/denylist/denylist.go +++ b/internal/platform/denylist/denylist.go @@ -59,6 +59,8 @@ func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*DeniedTokens, // max cost = total bytes found in the storage + 5% maxCost := totalCacheCapacity + totalCacheCapacity/20 + logger.Debugf("Denylist: cache capacity: %d bytes", maxCost) + cache, err := ristretto.NewCache(&ristretto.Config{ NumCounters: maxCost * 10, // recommended value MaxCost: maxCost, From 43ddad316e285f52ab886d982d2486b3b78e397a Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Mon, 11 Apr 2022 18:19:03 +0300 Subject: [PATCH 6/6] Update demo example --- demo/docker-compose/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/demo/docker-compose/docker-compose.yml b/demo/docker-compose/docker-compose.yml index 4d07328..b373bff 100644 --- a/demo/docker-compose/docker-compose.yml +++ b/demo/docker-compose/docker-compose.yml @@ -20,10 +20,6 @@ services: APIFW_DENYLIST_TOKENS_COOKIE_NAME: "test" APIFW_DENYLIST_TOKENS_HEADER_NAME: "" APIFW_DENYLIST_TOKENS_TRIM_BEARER_PREFIX: "true" - # Denylist: Cache - APIFW_DENYLIST_CACHE_NUM_COUNTERS: "10000000" - APIFW_DENYLIST_CACHE_MAX_COST: "2147483648" - APIFW_DENYLIST_CACHE_BUFFER_ITEMS: "64" volumes: - ./volumes/api-firewall:/opt/resources:ro ports: