diff --git a/middleware.go b/middleware.go
index 4fe71a4..7aabc7c 100644
--- a/middleware.go
+++ b/middleware.go
@@ -99,14 +99,15 @@ func (m *Middleware) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 			fmt.Println("IP is not authorize and basic auth is not valid")
 			return
 		}
-	}
-	req.URL.User = url.User(user)
-	if m.headerField != "" {
-		req.Header[m.headerField] = []string{user}
-	}
 
-	if m.removeHeader {
-		req.Header.Del(authorizationHeader)
+		req.URL.User = url.User(user)
+		if m.headerField != "" {
+			req.Header[m.headerField] = []string{user}
+		}
+
+		if m.removeHeader {
+			req.Header.Del(authorizationHeader)
+		}
 	}
 
 	fmt.Println("Request authorized")