From 973f49e69e13116d8358aa16e6ec92bf90009059 Mon Sep 17 00:00:00 2001 From: Nikolay Tkachenko Date: Mon, 4 Apr 2022 20:53:52 +0300 Subject: [PATCH] Change the feature name from blacklist to denylist. Fix empty string in the db file issue. --- README.md | 2 +- cmd/api-firewall/internal/handlers/routes.go | 6 +-- cmd/api-firewall/main.go | 11 +++-- cmd/api-firewall/tests/main_test.go | 22 +++++----- demo/docker-compose/docker-compose.yml | 18 ++++---- ...tokens.blacklist.db => tokens.denylist.db} | 0 internal/config/config.go | 4 +- internal/mid/{blacklist.go => denylist.go} | 22 +++++----- .../blacklist.go => denylist/denylist.go} | 44 ++++++++++--------- 9 files changed, 66 insertions(+), 63 deletions(-) rename demo/docker-compose/volumes/api-firewall/{tokens.blacklist.db => tokens.denylist.db} (100%) rename internal/mid/{blacklist.go => denylist.go} (55%) rename internal/platform/{blacklist/blacklist.go => denylist/denylist.go} (53%) diff --git a/README.md b/README.md index a336dac..b76369d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The **key features** of API Firewall are: * Stop API data breaches by blocking malformed API responses * Discover Shadow API endpoints * Validate JWT access tokens for OAuth 2.0 protocol-based authentication -* (NEW) Blacklist compromised API tokens, keys and Cookies +* (NEW) Denylist compromised API tokens, keys, and Cookies The product is **open source**, available at DockerHub and already got 1 billion (!!!) pulls. To support this project, you can star the [repository](https://hub.docker.com/r/wallarm/api-firewall). diff --git a/cmd/api-firewall/internal/handlers/routes.go b/cmd/api-firewall/internal/handlers/routes.go index cc72fa2..6dfe493 100644 --- a/cmd/api-firewall/internal/handlers/routes.go +++ b/cmd/api-firewall/internal/handlers/routes.go @@ -2,7 +2,7 @@ package handlers import ( "crypto/rsa" - "github.com/wallarm/api-firewall/internal/platform/blacklist" + "github.com/wallarm/api-firewall/internal/platform/denylist" "io/ioutil" "net/url" "os" @@ -24,7 +24,7 @@ import ( "github.com/wallarm/api-firewall/internal/platform/web" ) -func OpenapiProxy(cfg *config.APIFWConfiguration, serverUrl *url.URL, shutdown chan os.Signal, logger *logrus.Logger, proxy proxy.Pool, swagRouter *router.Router, blacklistedTokens *blacklist.BlacklistedTokens) fasthttp.RequestHandler { +func OpenapiProxy(cfg *config.APIFWConfiguration, serverUrl *url.URL, shutdown chan os.Signal, logger *logrus.Logger, proxy proxy.Pool, swagRouter *router.Router, deniedTokens *denylist.DeniedTokens) fasthttp.RequestHandler { var parserPool fastjson.ParserPool @@ -64,7 +64,7 @@ func OpenapiProxy(cfg *config.APIFWConfiguration, serverUrl *url.URL, shutdown c } } // Construct the web.App which holds all routes as well as common Middleware. - app := web.NewApp(shutdown, cfg, logger, mid.Logger(logger), mid.Errors(logger), mid.Panics(logger), mid.Proxy(cfg, serverUrl), mid.Blacklist(cfg, blacklistedTokens, logger)) + app := web.NewApp(shutdown, cfg, logger, mid.Logger(logger), mid.Errors(logger), mid.Panics(logger), mid.Proxy(cfg, serverUrl), mid.Denylist(cfg, deniedTokens, logger)) for _, route := range swagRouter.Routes { pathParamLength := 0 diff --git a/cmd/api-firewall/main.go b/cmd/api-firewall/main.go index a3d550c..3f7add1 100644 --- a/cmd/api-firewall/main.go +++ b/cmd/api-firewall/main.go @@ -3,7 +3,6 @@ package main import ( "expvar" // Register the expvar handlers "fmt" - "github.com/wallarm/api-firewall/internal/platform/blacklist" "mime" "net/url" "os" @@ -17,9 +16,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" - "github.com/wallarm/api-firewall/cmd/api-firewall/internal/handlers" "github.com/wallarm/api-firewall/internal/config" + "github.com/wallarm/api-firewall/internal/platform/denylist" "github.com/wallarm/api-firewall/internal/platform/openapi3" "github.com/wallarm/api-firewall/internal/platform/proxy" "github.com/wallarm/api-firewall/internal/platform/router" @@ -203,12 +202,12 @@ func run(logger *logrus.Logger) error { logger.Infof("%s: Initializing Cache", logPrefix) - blacklistedTokens, err := blacklist.New(&cfg, logger) + deniedTokens, err := denylist.New(&cfg, logger) if err != nil { - return errors.Wrap(err, "blacklist init error") + return errors.Wrap(err, "denylist init error") } - logger.Infof("%s: Loaded %d tokens to the cache", logPrefix, blacklistedTokens.ElementsNum) + logger.Infof("%s: Loaded %d tokens to the cache", logPrefix, deniedTokens.ElementsNum) // ========================================================================= // Start API Service @@ -235,7 +234,7 @@ func run(logger *logrus.Logger) error { signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM) api := fasthttp.Server{ - Handler: handlers.OpenapiProxy(&cfg, serverUrl, shutdown, logger, pool, swagRouter, blacklistedTokens), + Handler: handlers.OpenapiProxy(&cfg, serverUrl, shutdown, logger, pool, swagRouter, deniedTokens), ReadTimeout: cfg.ReadTimeout, WriteTimeout: cfg.WriteTimeout, Logger: logger, diff --git a/cmd/api-firewall/tests/main_test.go b/cmd/api-firewall/tests/main_test.go index 417a52a..0288d32 100644 --- a/cmd/api-firewall/tests/main_test.go +++ b/cmd/api-firewall/tests/main_test.go @@ -7,7 +7,7 @@ import ( "github.com/golang/mock/gomock" "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" - "github.com/wallarm/api-firewall/internal/platform/blacklist" + "github.com/wallarm/api-firewall/internal/platform/denylist" "io" "net" "net/url" @@ -168,8 +168,8 @@ const ( testOauthJWTKeyHS = "qwertyuiopasdfghjklzxcvbnm123456" testContentType = "test" - testBlacklistedCookieName = "testCookieName" - testBlacklistedToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzb21lIjoicGF5bG9hZDk5OTk5ODUifQ.S9P-DEiWg7dlI81rLjnJWCA6h9Q4ewTizxrsxOPGmNA" + testDeniedCookieName = "testCookieName" + testDeniedToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzb21lIjoicGF5bG9hZDk5OTk5ODUifQ.S9P-DEiWg7dlI81rLjnJWCA6h9Q4ewTizxrsxOPGmNA" ) type ServiceTests struct { @@ -227,7 +227,7 @@ func TestBasic(t *testing.T) { t.Run("basicDisableMode", apifwTests.testDisableMode) t.Run("commonParamters", apifwTests.testCommonParameters) - t.Run("basicBlacklist", apifwTests.testBlacklist) + t.Run("basicDenylist", apifwTests.testDenylist) t.Run("oauthIntrospectionReadSuccess", apifwTests.testOauthIntrospectionReadSuccess) t.Run("oauthIntrospectionReadUnsuccessful", apifwTests.testOauthIntrospectionReadUnsuccessful) @@ -322,7 +322,7 @@ func (s *ServiceTests) testBlockMode(t *testing.T) { } -func (s *ServiceTests) testBlacklist(t *testing.T) { +func (s *ServiceTests) testDenylist(t *testing.T) { cacheCfg := config.Cache{ NumCounters: 100000000, @@ -331,7 +331,7 @@ func (s *ServiceTests) testBlacklist(t *testing.T) { } tokensCfg := config.Token{ - CookieName: testBlacklistedCookieName, + CookieName: testDeniedCookieName, HeaderName: "", File: "../../../resources/test/tokens/test.db", } @@ -344,7 +344,7 @@ func (s *ServiceTests) testBlacklist(t *testing.T) { ShadowAPI: config.ShadowAPI{ ExcludeList: []int{404, 401}, }, - Blacklist: struct { + Denylist: struct { Tokens config.Token Cache config.Cache }{Tokens: tokensCfg, Cache: cacheCfg}, @@ -352,12 +352,12 @@ func (s *ServiceTests) testBlacklist(t *testing.T) { logger := logrus.New() - blacklistedTokens, err := blacklist.New(&cfg, logger) + deniedTokens, err := denylist.New(&cfg, logger) if err != nil { t.Fatal(err) } - handler := handlers.OpenapiProxy(&cfg, s.serverUrl, s.shutdown, s.logger, s.proxy, s.swagRouter, blacklistedTokens) + handler := handlers.OpenapiProxy(&cfg, s.serverUrl, s.shutdown, s.logger, s.proxy, s.swagRouter, deniedTokens) p, err := json.Marshal(map[string]interface{}{ "firstname": "test", @@ -397,8 +397,8 @@ func (s *ServiceTests) testBlacklist(t *testing.T) { reqCtx.Response.StatusCode()) } - // add blacklisted token to the Cookie header of the successful HTTP request (200) - req.Header.SetCookie(testBlacklistedCookieName, testBlacklistedToken) + // add denied token to the Cookie header of the successful HTTP request (200) + req.Header.SetCookie(testDeniedCookieName, testDeniedToken) reqCtx = fasthttp.RequestCtx{ Request: *req, diff --git a/demo/docker-compose/docker-compose.yml b/demo/docker-compose/docker-compose.yml index ce1263a..4d07328 100644 --- a/demo/docker-compose/docker-compose.yml +++ b/demo/docker-compose/docker-compose.yml @@ -15,15 +15,15 @@ services: APIFW_SERVER_DIAL_TIMEOUT: "200ms" APIFW_REQUEST_VALIDATION: "BLOCK" APIFW_RESPONSE_VALIDATION: "BLOCK" - # Blacklist: Token - APIFW_BLACKLIST_TOKENS_FILE: "/opt/resources/tokens.blacklist.db" - APIFW_BLACKLIST_TOKENS_COOKIE_NAME: "test" - APIFW_BLACKLIST_TOKENS_HEADER_NAME: "" - APIFW_BLACKLIST_TOKENS_TRIM_BEARER_PREFIX: "true" - # Blacklist: Cache - APIFW_BLACKLIST_CACHE_NUM_COUNTERS: "10000000" - APIFW_BLACKLIST_CACHE_MAX_COST: "2147483648" - APIFW_BLACKLIST_CACHE_BUFFER_ITEMS: "64" + # Denylist: Token + APIFW_DENYLIST_TOKENS_FILE: "/opt/resources/tokens.denylist.db" + APIFW_DENYLIST_TOKENS_COOKIE_NAME: "test" + APIFW_DENYLIST_TOKENS_HEADER_NAME: "" + APIFW_DENYLIST_TOKENS_TRIM_BEARER_PREFIX: "true" + # Denylist: Cache + APIFW_DENYLIST_CACHE_NUM_COUNTERS: "10000000" + APIFW_DENYLIST_CACHE_MAX_COST: "2147483648" + APIFW_DENYLIST_CACHE_BUFFER_ITEMS: "64" volumes: - ./volumes/api-firewall:/opt/resources:ro ports: diff --git a/demo/docker-compose/volumes/api-firewall/tokens.blacklist.db b/demo/docker-compose/volumes/api-firewall/tokens.denylist.db similarity index 100% rename from demo/docker-compose/volumes/api-firewall/tokens.blacklist.db rename to demo/docker-compose/volumes/api-firewall/tokens.denylist.db diff --git a/internal/config/config.go b/internal/config/config.go index cc3c4ec..18d05a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -43,7 +43,7 @@ type Token struct { File string `conf:""` } -type Blacklist struct { +type Denylist struct { Tokens Token Cache Cache } @@ -85,5 +85,5 @@ type APIFWConfiguration struct { AddValidationStatusHeader bool `conf:"default:false"` APISpecs string `conf:"default:swagger.json,env:API_SPECS"` ShadowAPI ShadowAPI - Blacklist Blacklist + Denylist Denylist } diff --git a/internal/mid/blacklist.go b/internal/mid/denylist.go similarity index 55% rename from internal/mid/blacklist.go rename to internal/mid/denylist.go index 04cbc74..b31db27 100644 --- a/internal/mid/blacklist.go +++ b/internal/mid/denylist.go @@ -6,12 +6,12 @@ import ( "github.com/sirupsen/logrus" "github.com/valyala/fasthttp" "github.com/wallarm/api-firewall/internal/config" - "github.com/wallarm/api-firewall/internal/platform/blacklist" + "github.com/wallarm/api-firewall/internal/platform/denylist" "github.com/wallarm/api-firewall/internal/platform/web" ) -// Blacklist forbidden requests with tokens in the blacklist -func Blacklist(cfg *config.APIFWConfiguration, blacklistedTokens *blacklist.BlacklistedTokens, logger *logrus.Logger) web.Middleware { +// Denylist forbidden requests with tokens in the blacklist +func Denylist(cfg *config.APIFWConfiguration, deniedTokens *denylist.DeniedTokens, logger *logrus.Logger) web.Middleware { // This is the actual middleware function to be executed. m := func(before web.Handler) web.Handler { @@ -20,21 +20,21 @@ func Blacklist(cfg *config.APIFWConfiguration, blacklistedTokens *blacklist.Blac h := func(ctx *fasthttp.RequestCtx) error { // check existence and emptiness of the cache - if blacklistedTokens != nil && blacklistedTokens.ElementsNum > 0 { + if deniedTokens != nil && deniedTokens.ElementsNum > 0 { //TODO: update getting token - if cfg.Blacklist.Tokens.CookieName != "" { - token := string(ctx.Request.Header.Cookie(cfg.Blacklist.Tokens.CookieName)) - _, found := blacklistedTokens.Cache.Get(token) + if cfg.Denylist.Tokens.CookieName != "" { + token := string(ctx.Request.Header.Cookie(cfg.Denylist.Tokens.CookieName)) + _, found := deniedTokens.Cache.Get(token) if found { return web.RespondError(ctx, cfg.CustomBlockStatusCode, nil) } } - if cfg.Blacklist.Tokens.HeaderName != "" { - token := string(ctx.Request.Header.Peek(cfg.Blacklist.Tokens.HeaderName)) - if cfg.Blacklist.Tokens.TrimBearerPrefix { + if cfg.Denylist.Tokens.HeaderName != "" { + token := string(ctx.Request.Header.Peek(cfg.Denylist.Tokens.HeaderName)) + if cfg.Denylist.Tokens.TrimBearerPrefix { token = strings.TrimPrefix(token, "Bearer ") } - _, found := blacklistedTokens.Cache.Get(token) + _, found := deniedTokens.Cache.Get(token) if found { return web.RespondError(ctx, cfg.CustomBlockStatusCode, nil) } diff --git a/internal/platform/blacklist/blacklist.go b/internal/platform/denylist/denylist.go similarity index 53% rename from internal/platform/blacklist/blacklist.go rename to internal/platform/denylist/denylist.go index 2e6a26c..22162af 100644 --- a/internal/platform/blacklist/blacklist.go +++ b/internal/platform/denylist/denylist.go @@ -1,4 +1,4 @@ -package blacklist +package denylist import ( "bufio" @@ -10,17 +10,17 @@ import ( "github.com/wallarm/api-firewall/internal/config" ) -type BlacklistedTokens struct { +type DeniedTokens struct { Cache *ristretto.Cache ElementsNum int } -func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*BlacklistedTokens, error) { +func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*DeniedTokens, error) { cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: cfg.Blacklist.Cache.NumCounters, - MaxCost: cfg.Blacklist.Cache.MaxCost, - BufferItems: cfg.Blacklist.Cache.BufferItems, + NumCounters: cfg.Denylist.Cache.NumCounters, + MaxCost: cfg.Denylist.Cache.MaxCost, + BufferItems: cfg.Denylist.Cache.BufferItems, }) if err != nil { return nil, err @@ -29,17 +29,19 @@ func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*BlacklistedTok totalEntries := 0 // Loading tokens to the cache - if cfg.Blacklist.Tokens.File != "" { + if cfg.Denylist.Tokens.File != "" { - f, err := os.Open(cfg.Blacklist.Tokens.File) + f, err := os.Open(cfg.Denylist.Tokens.File) if err != nil { return nil, err } - // count entries + // count non-empty entries c := bufio.NewScanner(f) for c.Scan() { - totalEntries += 1 + if c.Text() != "" { + totalEntries += 1 + } } err = c.Err() if err != nil { @@ -50,23 +52,25 @@ func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*BlacklistedTok return nil, err } - logger.Debugf("Blacklist: total entries (lines) found in the file: %d", totalEntries) + logger.Debugf("Denylist: total entries (lines) found in the file: %d", totalEntries) totalEntries10P := totalEntries / 10 numOfElements := 0 current10P := 0 s := bufio.NewScanner(f) for s.Scan() { - if ok := cache.Set(s.Text(), nil, 1); ok { - numOfElements += 1 - if numOfElements%totalEntries10P == 0 { - current10P += 10 - logger.Debugf("Blacklist: loaded %d perecents of tokens. Total elements in the cache: %d", current10P, numOfElements) + if s.Text() != "" { + if ok := cache.Set(s.Text(), nil, 1); ok { + numOfElements += 1 + if numOfElements%totalEntries10P == 0 { + current10P += 10 + logger.Debugf("Denylist: loaded %d perecents of tokens. Total elements in the cache: %d", current10P, numOfElements) + } + } else { + logger.Errorf("Denylist: can't add the token to the cache: %s", s.Text()) } - } else { - logger.Errorf("Blacklist: can't add the token to the cache: %s", s.Text()) + cache.Wait() } - cache.Wait() } err = s.Err() if err != nil { @@ -78,5 +82,5 @@ func New(cfg *config.APIFWConfiguration, logger *logrus.Logger) (*BlacklistedTok } } - return &BlacklistedTokens{Cache: cache, ElementsNum: totalEntries}, nil + return &DeniedTokens{Cache: cache, ElementsNum: totalEntries}, nil }