From 8856677d673250ffe8b7f6ea00617211a4e9f067 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 2 May 2024 13:07:34 -0400 Subject: [PATCH 01/32] Testing in place --- Gemfile | 1 - lib/lua/.luarc.json | 6 - lib/lua/dist/wafris_core.lua | 305 ------------------- lib/wafris.rb | 501 +++++++++++++++++++++++++++++-- lib/wafris/configuration.rb | 161 +++++++--- lib/wafris/log_suppressor.rb | 3 +- lib/wafris/middleware.rb | 17 +- lib/wafris/version.rb | 2 +- stylua.toml | 3 - test/configuration_test.rb | 81 +++-- test/middleware_test.rb | 22 +- test/test_helper.rb | 18 ++ test/wafris_test.rb | 189 +++++++++++- test/wafris_test_custom_rules.db | Bin 0 -> 94208 bytes wafris.gemspec | 8 +- wafris_test.db | 0 wafris_test_custom_rules.db | 0 17 files changed, 869 insertions(+), 448 deletions(-) delete mode 100644 lib/lua/.luarc.json delete mode 100644 lib/lua/dist/wafris_core.lua delete mode 100644 stylua.toml create mode 100644 test/wafris_test_custom_rules.db create mode 100644 wafris_test.db create mode 100644 wafris_test_custom_rules.db diff --git a/Gemfile b/Gemfile index 61f1b84..7f4f5e9 100644 --- a/Gemfile +++ b/Gemfile @@ -3,4 +3,3 @@ source 'https://rubygems.org' gemspec - diff --git a/lib/lua/.luarc.json b/lib/lua/.luarc.json deleted file mode 100644 index 57ecf17..0000000 --- a/lib/lua/.luarc.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "diagnostics.disable": [ - "lowercase-global", - "undefined-global" - ] -} \ No newline at end of file diff --git a/lib/lua/dist/wafris_core.lua b/lib/lua/dist/wafris_core.lua deleted file mode 100644 index ed92d37..0000000 --- a/lib/lua/dist/wafris_core.lua +++ /dev/null @@ -1,305 +0,0 @@ - - -local USE_TIMESTAMPS_AS_REQUEST_IDS = false -local EXPIRATION_IN_SECONDS = tonumber(redis.call("HGET", "waf-settings", "expiration-time")) or 86400 -local EXPIRATION_OFFSET_IN_SECONDS = 3600 - - -local function get_timebucket(timestamp_in_seconds) - local startOfHourTimestamp = math.floor(timestamp_in_seconds / 3600) * 3600 - return tostring(startOfHourTimestamp) -end - -local function set_property_value_id_lookups(property_abbreviation, property_value) - - local value_key = property_abbreviation .. "V" .. property_value - local property_id = redis.call("GET", value_key) - - if property_id == false then - property_id = redis.call("INCR", property_abbreviation .. "-id-counter") - redis.call("SET", value_key, property_id) - redis.call("SET", property_abbreviation .. "I" .. property_id, property_value) - end - - redis.call("EXPIRE", value_key, EXPIRATION_IN_SECONDS + EXPIRATION_OFFSET_IN_SECONDS) - redis.call("EXPIRE", property_abbreviation .. "I" .. property_id, EXPIRATION_IN_SECONDS + EXPIRATION_OFFSET_IN_SECONDS) - - return property_id -end - -local function increment_leaderboard_for(property_abbreviation, property_id, timebucket) - - local key = property_abbreviation .. "L" .. timebucket - redis.call("ZINCRBY", key, 1, property_id) - redis.call("EXPIRE", key, EXPIRATION_IN_SECONDS) -end - -local function set_property_to_requests_list(property_abbreviation, property_id, request_id, timebucket) - - local key = property_abbreviation .. "R" .. property_id .. "-" .. timebucket - redis.call("LPUSH", key, request_id) - - redis.call("EXPIRE", key, EXPIRATION_IN_SECONDS + EXPIRATION_OFFSET_IN_SECONDS) -end - - -local function ip_in_hash(hash_name, ip_address) - local found_ip = redis.call('HEXISTS', hash_name, ip_address) - - if found_ip == 1 then - return ip_address - else - return false - end -end - -local function ip_in_cidr_range(cidr_set, ip_decimal_lexical) - - local higher_value = redis.call('ZRANGEBYLEX', cidr_set, '['..ip_decimal_lexical, '+', 'LIMIT', 0, 1)[1] - - local lower_value = redis.call('ZREVRANGEBYLEX', cidr_set, '['..ip_decimal_lexical, '-', 'LIMIT', 0, 1)[1] - - if not (higher_value and lower_value) then - return false - end - - local higher_compare = higher_value:match('([^%-]+)$') - local lower_compare = lower_value:match('([^%-]+)$') - - if higher_compare == lower_compare then - return lower_compare - else - return false - end -end - -local function escapePattern(s) - local patternSpecials = "[%^%$%(%)%%%.%[%]%*%+%-%?]" - return s:gsub(patternSpecials, "%%%1") -end - -local function match_by_pattern(property_abbreviation, property_value) - local hash_name = "rules-blocked-" .. property_abbreviation - - local patterns = redis.call('HKEYS', hash_name) - - for _, pattern in ipairs(patterns) do - if string.find(string.lower(property_value), string.lower(escapePattern(pattern))) then - return pattern - end - end - - return false -end - -local function blocked_by_rate_limit(request_properties) - - local rate_limiting_rules_values = redis.call('HKEYS', 'rules-blocked-rate-limits') - - for i, rule_name in ipairs(rate_limiting_rules_values) do - - local conditions_hash = redis.call('HGETALL', rule_name .. "-conditions") - - local all_conditions_match = true - - for j = 1, #conditions_hash, 2 do - local condition_key = conditions_hash[j] - local condition_value = conditions_hash[j + 1] - - if request_properties[condition_key] ~= condition_value then - all_conditions_match = false - break - end - end - - if all_conditions_match then - - local rule_settings_key = rule_name .. "-settings" - - local limit, time_period, limited_by, rule_id = unpack(redis.call('HMGET', rule_settings_key, 'limit', 'time-period', 'limited-by', 'rule-id')) - - local throttle_key = rule_name .. ":" .. limit .. "V" .. request_properties.ip - - local new_value = redis.call('INCR', throttle_key) - - if new_value == 1 then - redis.call('EXPIRE', throttle_key, tonumber(time_period)) - end - - if tonumber(new_value) >= tonumber(limit) then - return rule_id - else - return false - end - end - end -end - -local function check_rules(functions_to_check) - for _, check in ipairs(functions_to_check) do - - local rule = check.func(unpack(check.args)) - local category = check.category - - if type(rule) == "string" then - return rule, category - end - end - - return false, false -end - -local function check_blocks(request) - local rule_categories = { - { category = "bi", func = ip_in_hash, args = { "rules-blocked-i", request.ip } }, - { category = "bc", func = ip_in_cidr_range, args = { "rules-blocked-cidrs-set", request.ip_decimal_lexical } }, - { category = "bs", func = ip_in_cidr_range, args = { "rules-blocked-cidrs-subscriptions-set", request.ip_decimal_lexical } }, - { category = "bu", func = match_by_pattern, args = { "u", request.user_agent } }, - { category = "bp", func = match_by_pattern, args = { "p", request.path } }, - { category = "ba", func = match_by_pattern, args = { "a", request.parameters } }, - { category = "bh", func = match_by_pattern, args = { "h", request.host } }, - { category = "bm", func = match_by_pattern, args = { "m", request.method } }, - { category = "bd", func = match_by_pattern, args = { "rh", request.headers } }, - { category = "bpb", func = match_by_pattern, args = { "pb", request.post_body } }, - { category = "brl", func = blocked_by_rate_limit, args = { request } } - } - - return check_rules(rule_categories) -end - -local function check_allowed(request) - local rule_categories = { - { category = "ai", func = ip_in_hash, args = { "rules-allowed-i", request.ip } }, - { category = "ac", func = ip_in_cidr_range, args = { "rules-allowed-cidrs-set", request.ip_decimal_lexical } } - } - - return check_rules(rule_categories) -end - -local request = { - ["ip"] = ARGV[1], - ["ip_decimal_lexical"] = string.rep("0", 39 - #ARGV[2]) .. ARGV[2], - ["ts_in_milliseconds"] = ARGV[3], - ["ts_in_seconds"] = ARGV[3] / 1000, - ["user_agent"] = ARGV[4], - ["path"] = ARGV[5], - ["parameters"] = ARGV[6], - ["host"] = ARGV[7], - ["method"] = ARGV[8], - ["headers"] = ARGV[9], - ["post_body"] = ARGV[10], - ["ip_id"] = set_property_value_id_lookups("i", ARGV[1]), - ["user_agent_id"] = set_property_value_id_lookups("u", ARGV[4]), - ["path_id"] = set_property_value_id_lookups("p", ARGV[5]), - ["parameters_id"] = set_property_value_id_lookups("a", ARGV[6]), - ["host_id"] = set_property_value_id_lookups("h", ARGV[7]), - ["method_id"] = set_property_value_id_lookups("m", ARGV[8]) -} - - - -local current_timebucket = get_timebucket(request.ts_in_seconds) - - local blocked_rule = false - local blocked_category = nil - local treatment = "p" - - local stream_id - - if USE_TIMESTAMPS_AS_REQUEST_IDS == true then - stream_id = request.ts_in_milliseconds - else - stream_id = "*" - end - - local stream_args = { - "XADD", - "rStream", - "MINID", - tostring((current_timebucket - EXPIRATION_IN_SECONDS) * 1000 ), - stream_id, - "i", request.ip_id, - "u", request.user_agent_id, - "p", request.path_id, - "h", request.host_id, - "m", request.method_id, - "a", request.parameters_id, - } - - local allowed_rule, allowed_category = check_allowed(request) - - if allowed_rule then - table.insert(stream_args, "t") - table.insert(stream_args, "a") - - treatment = "a" - - table.insert(stream_args, "ac") - table.insert(stream_args, allowed_category) - - table.insert(stream_args, "ar") - table.insert(stream_args, allowed_rule) - - else - blocked_rule, blocked_category = check_blocks(request) - end - - if blocked_rule then - table.insert(stream_args, "t") - table.insert(stream_args, "b") - - treatment = "b" - - table.insert(stream_args, "bc") - table.insert(stream_args, blocked_category) - - table.insert(stream_args, "br") - table.insert(stream_args, blocked_rule) - end - - if blocked_rule == false and allowed_rule == false then - table.insert(stream_args, "t") - table.insert(stream_args, "p") - end - - local request_id = redis.call(unpack(stream_args)) - - increment_leaderboard_for("i", request.ip_id, current_timebucket) - increment_leaderboard_for("u", request.user_agent_id, current_timebucket) - increment_leaderboard_for("p", request.path_id, current_timebucket) - increment_leaderboard_for("a", request.parameters_id, current_timebucket) - increment_leaderboard_for("h", request.host_id, current_timebucket) - increment_leaderboard_for("m", request.method_id, current_timebucket) - increment_leaderboard_for("t", treatment, current_timebucket) - - set_property_to_requests_list("i", request.ip_id, request_id, current_timebucket) - set_property_to_requests_list("u", request.user_agent_id, request_id, current_timebucket) - set_property_to_requests_list("p", request.path_id, request_id, current_timebucket) - set_property_to_requests_list("a", request.parameters_id, request_id, current_timebucket) - set_property_to_requests_list("h", request.host_id, request_id, current_timebucket) - set_property_to_requests_list("m", request.method_id, request_id, current_timebucket) - set_property_to_requests_list("t", treatment, request_id, current_timebucket) - - if blocked_rule ~= false then - increment_leaderboard_for("bc", blocked_category, current_timebucket) - set_property_to_requests_list("bc", blocked_category, request_id, current_timebucket) - - increment_leaderboard_for("br", blocked_rule, current_timebucket) - set_property_to_requests_list("br", blocked_rule, request_id, current_timebucket) - end - - if allowed_rule ~= false then - increment_leaderboard_for("ac", allowed_category, current_timebucket) - set_property_to_requests_list("ac", allowed_category, request_id, current_timebucket) - - increment_leaderboard_for("ar", allowed_rule, current_timebucket) - set_property_to_requests_list("ar", allowed_rule, request_id, current_timebucket) - end - -if blocked_rule ~= false then - return "Blocked" -elseif allowed_rule ~= false then - return "Allowed" -else - return "Passed" -end diff --git a/lib/wafris.rb b/lib/wafris.rb index 6e94045..91e771e 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -1,14 +1,18 @@ + # frozen_string_literal: true -require 'connection_pool' + require 'rails' -require 'redis' +require 'sqlite3' +require 'ipaddr' +require 'httparty' require 'wafris/configuration' require 'wafris/middleware' require 'wafris/log_suppressor' require 'wafris/railtie' if defined?(Rails::Railtie) +ActiveSupport::Deprecation.behavior = :silence module Wafris class << self @@ -18,44 +22,485 @@ def configure raise ArgumentError unless block_given? self.configuration ||= Wafris::Configuration.new + yield(configuration) - LogSuppressor.puts_log( - "[Wafris] attempting firewall connection via Wafris.configure initializer." - ) unless configuration.quiet_mode + + LogSuppressor.puts_log("[Wafris] Configuration settings created.") + configuration.create_settings + rescue ArgumentError - LogSuppressor.puts_log( - "[Wafris] block is required to configure Wafris. More info can be found at: https://github.com/Wafris/wafris-rb" - ) - rescue StandardError => e - LogSuppressor.puts_log( - "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" - ) + puts "[Wafris] block is required to configure Wafris. More info can be found at: https://github.com/Wafris/wafris-rb" + + #rescue StandardError => e + # puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" + end - def allow_request?(request) - configuration.connection_pool.with do |conn| - time = Time.now.utc.to_i * 1000 - status = conn.evalsha( - configuration.core_sha, - argv: [ - request.ip, - IPAddr.new(request.ip).to_i, - time, - request.user_agent, - request.path, - request.query_string, - request.host, - request.request_method - ] + + def zero_pad(number, length) + number.to_s.rjust(length, "0") + end + + def ip_to_decimal_lexical_string(ip) + num = 0 + + if ip.include?(":") + ip = IPAddr.new(ip).to_string + hex = ip.delete(":") + (0...hex.length).step(4) do |i| + chunk = hex[i, 4].to_i(16) + num = num * (2**16) + chunk + end + elsif ip.include?(".") + ip.split(".").each do |chunk| + num = num * 256 + chunk.to_i + end + end + + str = num.to_s + zero_pad(str, 39) + end + + def ip_in_cidr_range(ip_address, table_name, db_connection) + lexical_address = ip_to_decimal_lexical_string(ip_address) + higher_value = db_connection.get_first_value("SELECT * FROM #{table_name} WHERE member > ? ORDER BY member ASC", [lexical_address]) + lower_value = db_connection.get_first_value("SELECT * FROM #{table_name} WHERE member < ? ORDER BY member DESC", [lexical_address]) + + if higher_value.nil? || lower_value.nil? + return nil + else + higher_compare = higher_value.split("-").last + lower_compare = lower_value.split("-").last + + if higher_compare == lower_compare + return lower_compare + else + return nil + end + end + end + + def get_country_code(ip, db_connection) + country_code = ip_in_cidr_range(ip, 'country_ip_ranges', db_connection) + + if country_code + country_code = country_code.split("_").first.split("G").last + return country_code + else + return "ZZ" + end + end + + def substring_match(request_property, table_name, db_connection) + result = db_connection.execute("SELECT entries FROM #{table_name}") + result.flatten.each do |entry| + if request_property.include?(entry) + return entry + end + end + return false + end + + def exact_match(request_property, table_name, db_connection) + result = db_connection.execute("SELECT entries FROM #{table_name} WHERE entries = ?", [request_property]) + return result.any? + end + + def check_rate_limit(ip, path, method, db_connection) + + # Correctly format the SQL query with placeholders + limiters = db_connection.execute("SELECT * FROM blocked_rate_limits WHERE path LIKE ? AND method = ?", ["%#{path}%", method]) + + # If no rate limiters are matched + if limiters.empty? + return false + end + + current_timestamp = Time.now.to_i + + # If any rate limiters are matched + # This implementation will block the request on any of the rate limiters + limiters.each do |limiter| + + # Limiter array mapping + # 0: id + # 1: path + # 2: method + # 3: interval + # 4: max_count + # 5: rule_id + + interval = limiter[3] + max_count = limiter[4] + rule_id = limiter[5] + + # Expire old timestamps + @rate_limiters.each do |ip, timestamps| + # Removes timestamps older than the interval + + + + @rate_limiters[ip] = timestamps.select { |timestamp| timestamp > current_timestamp - interval } + + # Remove the IP if there are no more timestamps for the IP + @rate_limiters.delete(ip) if @rate_limiters[ip].empty? + end + + # Check if the IP+Method is rate limited + + if @rate_limiters[ip] && @rate_limiters[ip].length >= max_count + # Request is rate limited + + + return rule_id + + else + # Request is not rate limited, so add the current timestamp + if @rate_limiters[ip] + @rate_limiters[ip] << current_timestamp + else + @rate_limiters[ip] = [current_timestamp] + end + + return false + end + + end + + end + + def send_upsync_requests(requests_array) + + begin + + headers = {'Content-Type' => 'application/json'} + body = {batch: requests_array}.to_json + response = HTTParty.post(@upsync_url, + :body => body, + :headers => headers, + :timeout => 300) + if response.code == 200 + puts "Upsync successful" + @upsync_status = 'Complete' + else + puts "Upsync Error. HTTP Response: #{response.code}" + end + rescue HTTParty::Error => e + puts "Upsync Response: #{response.code}" + puts "Failed to send upsync requests: #{e.message}" + end + return true + end + + # This method is used to queue upsync requests. It takes in several parameters including + # ip, user_agent, path, parameters, host, method, treatment, category, and rule. + # + # The 'treatment' parameter represents the action taken on the request, which can be + # 'Allowed', 'Blocked', or 'Passed'. + # + # The 'category' parameter represents the category of the rule that was matched, such as + # 'blocked_ip', 'allowed_cidr', etc. + # + # The 'rule' parameter represents the specific rule that was matched within the category + # ex: '192.23.5.4', 'SemRush', etc. + def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatment, category, rule) + + if @upsync_status != 'Disabled' + + # Add request to the queue + request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] + @upsync_queue << request + + # If the queue is full, send the requests to the upsync server + if @upsync_queue.length >= @upsync_queue_limit || (Time.now.to_i - @last_upsync_timestamp) >= @upsync_interval + requests_array = @upsync_queue + @upsync_queue = [] + @last_upsync_timestamp = Time.now.to_i + + @upsync_status = 'Uploading' + send_upsync_requests(requests_array) + end + + # Return the treatment - used to return 403 or 200 + return treatment + else + return "Passed" + end + + end + + # Pulls the latest rules from the server + def downsync_db(db_rule_category, current_filename = nil) + + lockfile_path = "#{@db_file_path}/#{db_rule_category}.lockfile" + + puts lockfile_path + + # Attempt to create a lockfile with exclusive access; skip if it exists + begin + lockfile = File.open(lockfile_path, File::RDWR|File::CREAT|File::EXCL) + rescue Errno::EEXIST + puts "Lockfile already exists, skipping downsync." + return + end + + begin + # Actual Downsync operations + filename = "" + + # Check server for new rules + #puts "Downloading from #{@downsync_url}/#{db_rule_category}/#{@api_key}?current_version=#{current_filename}" + uri = "#{@downsync_url}/#{db_rule_category}/#{@api_key}?current_version=#{current_filename}" + + response = HTTParty.get( + uri, + follow_redirects: true, # Enable following redirects + max_redirects: 2 # Maximum number of redirects to follow ) + + if response.code == 401 + @upsync_status = 'Disabled' + puts "Unauthorized: Bad or missing API key" + + filename = current_filename + + elsif response.code == 304 + @upsync_status = 'Enabled' + puts "No new rules to download" + + filename = current_filename + + elsif response.code == 200 + @upsync_status = 'Disabled' + + if current_filename + old_file_name = current_filename + end + + # Extract the filename from the response + content_disposition = response.headers['content-disposition'] + filename = content_disposition.match(/filename="(.+)"/)[1] + + # Save the body of the response to a new SQLite file + File.open(@db_file_path + "/" + filename, 'wb') { |file| file.write(response.body) } + + # Write the filename into the db_category.modfile + File.open("#{@db_file_path}/#{db_rule_category}.modfile", 'w') { |file| file.write(filename) } + + # Remove the old database file + if old_file_name + #puts "Removing old file: #{@db_file_path}/#{old_file_name}" + if File.exist?(@db_file_path + "/" + old_file_name) + File.delete(@db_file_path + "/" + old_file_name) + end + end + + end + + rescue Exception => e + puts "EXCEPTION: Error downloading rules: #{e.message}" + + # This gets set even if the API key is bad or other issues + # to prevent hammering the distribution server on every request + ensure + + # Reset the modified time of the modfile + unless File.exist?("#{@db_file_path}/#{db_rule_category}.modfile") + File.new("#{@db_file_path}/#{db_rule_category}.modfile", 'w') + end + + # Set the modified time of the modfile to the current time + File.utime(Time.now, Time.now, "#{@db_file_path}/#{db_rule_category}.modfile") + + # Ensure the lockfile is removed after operations + lockfile.close + File.delete(lockfile_path) + end + + return filename + + end + + # Returns the current database file, + # if the file is older than the interval, it will download the latest db + # if the file doesn't exist, it will download the latest db + # if the lockfile exists, it will return the current db + def current_db(db_rule_category) + + if db_rule_category == 'custom_rules' + interval = @downsync_custom_rules_interval + else + interval = @downsync_data_subscriptions_interval + end + + # Checks for existing current modfile, which contains the current db filename + if File.exist?("#{@db_file_path}/#{db_rule_category}.modfile") + + #puts "Modfile exists: #{@db_file_path}/#{db_rule_category}.modfile" + + # Get last Modified Time and current database file name + last_db_synctime = File.mtime("#{@db_file_path}/#{db_rule_category}.modfile").to_i + returned_db = File.read("#{@db_file_path}/#{db_rule_category}.modfile").strip + + # Check if the db file is older than the interval + if (Time.now.to_i - last_db_synctime) > interval + #puts "DB file is older than the interval" + + # Make sure that another process isn't already downloading the rules + if !File.exist?("#{@db_file_path}/#{db_rule_category}.lockfile") + returned_db = downsync_db(db_rule_category, returned_db) + end + + return returned_db + + # Current db is up to date + else + + returned_db = File.read("#{@db_file_path}/#{db_rule_category}.modfile").strip + + # If the modfile is empty (no db file name), return nil + # this can happen if the the api key is bad + if returned_db == "" + return nil + else + return returned_db + end + + end + + # No modfile exists, so download the latest db + else + + # Make sure that another process isn't already downloading the rules + if File.exist?("#{@db_file_path}/#{db_rule_category}.lockfile") + # Lockfile exists, but not modfile with a db filename + return nil + else + + # No modfile exists, so download the latest db + returned_db = downsync_db(db_rule_category, nil) + + if returned_db.nil? + return nil + else + return returned_db + end + + end + + end + + + + end + + # This is the main loop that evaluates the request + # as well as sorts out when downsync and upsync should be called + def evaluate(ip, user_agent, path, parameters, host, method) + + rules_db_filename = current_db('custom_rules') + data_subscriptions_db_filename = current_db('data_subscriptions') + + ap "Rules path: #{rules_db_filename}" + ap "Rules path nil: #{rules_db_filename.nil?}" + ap "Rules path empty: #{rules_db_filename.to_s.strip == ''}" + + ap "Data Subscriptions path: #{data_subscriptions_db_filename}" + ap "Data Subscriptions path nil: #{data_subscriptions_db_filename.nil?}" + ap "Data Subscriptions path empty: #{data_subscriptions_db_filename.to_s.strip == ''}" + + + if rules_db_filename.to_s.strip != '' && data_subscriptions_db_filename.strip.to_s.strip != '' + + rules_db = SQLite3::Database.new "#{@db_file_path}/#{rules_db_filename}" + data_subscriptions_db = SQLite3::Database.new "#{@db_file_path}/#{data_subscriptions_db_filename}" + + # Allowed IPs + if exact_match(ip, 'allowed_ips', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ai', ip) + end + + # Allowed CIDR Ranges + if ip_in_cidr_range(ip, 'allowed_cidr_ranges', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ac', ip) + end + + # Blocked IPs + if exact_match(ip, 'blocked_ips', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bi', ip) + end + + # Blocked CIDR Ranges + if ip_in_cidr_range(ip, 'blocked_cidr_ranges', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bc', ip) + end + + # Blocked Country Codes + country_code = get_country_code(ip, data_subscriptions_db) + if exact_match(country_code, 'blocked_country_codes', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "G_#{country_code}") + end + + # Blocked Reputation IP Ranges + if ip_in_cidr_range(ip, 'reputation_ip_ranges', data_subscriptions_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "R") + end + + # Blocked User Agents + user_agent_match = substring_match(user_agent, 'blocked_user_agents', rules_db) + if user_agent_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bu', user_agent_match) + end + + # Blocked Paths + path_match = substring_match(path, 'blocked_paths', rules_db) + if path_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bp', path_match) + end + + # Blocked Parameters + parameters_match = substring_match(parameters, 'blocked_parameters', rules_db) + if parameters_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'ba', parameters_match) + end + + # Blocked Hosts + if exact_match(host, 'blocked_hosts', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bh', host) + end + + # Blocked Methods + if exact_match(method, 'blocked_methods', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bm', method) + end + + # Rate Limiting + rule_id = check_rate_limit(ip, path, method, rules_db) + if rule_id + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'brl', rule_id) + end + + end + + # Passed if no allow or block rules matched + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Passed', 'passed', '-') + + end # end evaluate + + + + def allow_request?(request) + + status = "foo" if status.eql? 'Blocked' return false else return true end - end + end + end end diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index c604e55..84f7843 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -1,56 +1,143 @@ + +# Wafris setup and logs + +# - No startup messages in dev or test or CI environments +# - Way to disable WAF in v2 (disabled?) + +# API Key + + # - Local only mode "local_only" (TBD) + # - No upsync + # - Bad API key (checked on initial downsync) + # - No upsync + + # - No API key + # - Honeybadger says no api key in dev + # - Quiet mode on startup -> show no messages at startup + +# Verbose mode? +# - 1st time setup +# - Startup success +# - Downsync success +# - Upsync success + + # frozen_string_literal: true require_relative 'version' module Wafris class Configuration - attr_accessor :redis - attr_accessor :redis_pool_size - attr_accessor :maxmemory - attr_accessor :quiet_mode + + attr_accessor :api_key + attr_accessor :db_file_path + attr_accessor :db_file_name + attr_accessor :downsync_custom_rules_interval + attr_accessor :downsync_data_subscriptions_interval + attr_accessor :downsync_url + attr_accessor :upsync_url + attr_accessor :upsync_interval + attr_accessor :upsync_queue_limit + attr_accessor :local_only def initialize - @redis_pool_size = 20 - @maxmemory = 25 - @quiet_mode = false - end - def connection_pool - @connection_pool ||= - ConnectionPool.new(size: redis_pool_size) { redis } - end + # API Key - Required + if ENV['WAFRIS_API_KEY'] + @api_key = ENV['WAFRIS_API_KEY'] + else + @api_key = nil + LogSuppressor.puts_log("Firewall disabled as API key not set") + end - def create_settings - redis.hset('waf-settings', - 'version', Wafris::VERSION, - 'client', 'ruby', - 'maxmemory', @maxmemory) - LogSuppressor.puts_log( - "[Wafris] firewall enabled. Connected to Redis on #{redis.connection[:host]}. Ready to process requests. Set rules at: https://wafris.org/hub" - ) unless @quiet_mode - end + # DB FILE PATH LOCATION - Optional + if ENV['WAFRIS_DB_FILE_PATH'] + @db_file_path = ENV['WAFRIS_DB_FILE_PATH'] + else + #@db_file_path = Rails.root.join('tmp', 'wafris').to_s + @db_file_path = 'tmp/wafris' + end - def core_sha - @core_sha ||= redis.script(:load, wafris_core) - end + # Verify that the db_file_path exists + unless File.directory?(@db_file_path) + LogSuppressor.puts_log("DB File Path does not exist - creating it now.") + Dir.mkdir(@db_file_path) unless File.exists?(@db_file_path) + end - def wafris_core - read_lua_dist("wafris_core") + # DB FILE NAME - For local + if ENV['WAFRIS_DB_FILE_NAME'] + @db_file_name = ENV['WAFRIS_DB_FILE_NAME'] + else + @db_file_name = 'wafris.db' + end + + # DOWNSYNC + # Custom Rules are checked often (default 1 minute) - Optional + if ENV['WAFRIS_DOWNSYNC_CUSTOM_RULES_INTERVAL'] + @downsync_custom_rules_interval = ENV['WAFRIS_DOWNSYNC_CUSTOM_RULES_INTERVAL'].to_i + else + @downsync_custom_rules_interval = 60 + end + + # Data Subscriptions are checked rarely (default 1 day) - Optional + if ENV['WAFRIS_DOWNSYNC_DATA_SUBSCRIPTIONS_INTERVAL'] + @downsync_data_subscriptions_interval = ENV['WAFRIS_DOWNSYNC_DATA_SUBSCRIPTIONS_INTERVAL'].to_i + else + @downsync_data_subscriptions_interval = 86400 + end + + # Set Downsync URL - Optional + # Used for both DataSubscription and CustomRules + if ENV['WAFRIS_DOWNSYNC_URL'] + @downsync_url = ENV['WAFRIS_DOWNSYNC_URL'] + else + @downsync_url = 'https://distributor.wafris.org/v2/downsync' + end + + # UPSYNC - Optional + # Set Upsync URL + if ENV['WAFRIS_UPSYNC_URL'] + @upsync_url = ENV['WAFRIS_UPSYNC_URL'] + '/' + @api_key + else + @upsync_url = 'https://collector.wafris.org/v2/upsync/' + @api_key.to_s + end + + # Set Upsync Interval - Optional + if ENV['WAFRIS_UPSYNC_INTERVAL'] + @upsync_interval = ENV['WAFRIS_UPSYNC_INTERVAL'].to_i + else + @upsync_interval = 60 + end + + # Set Upsync Queued Request Limit - Optional + if ENV['WAFRIS_UPSYNC_QUEUE_LIMIT'] + @upsync_queue_limit = ENV['WAFRIS_UPSYNC_QUEUE_LIMIT'].to_i + else + @upsync_queue_limit = 1000 + end + + # Upsync Queue + @upsync_queue = [] + @last_upsync_timestamp = Time.now.to_i + + # Memory structure for rate limiting + @rate_limiters = {} + + # Disable Upsync if Downsync API Key is invalid + # This prevents the client from sending upsync requests + # if the API key is known bad + @upsync_status = 'Disabled' + end - private - def read_lua_dist(filename) - File.read( - file_path(filename) - ) - end + def create_settings - def file_path(filename) - File.join( - File.dirname(__FILE__), - "../lua/dist/#{filename}.lua" - ) + @version = Wafris::VERSION + + LogSuppressor.puts_log("[Wafris] Firewall launched successfully. Ready to process requests. Set rules at: https://hub.wafris.org/") + end + end end diff --git a/lib/wafris/log_suppressor.rb b/lib/wafris/log_suppressor.rb index 95acc02..5170b33 100644 --- a/lib/wafris/log_suppressor.rb +++ b/lib/wafris/log_suppressor.rb @@ -7,7 +7,8 @@ def self.puts_log(message) end def self.suppress_logs? - suppressed_environments.include?(current_environment) + suppressed_environments.include?(current_environment) || + (ENV['WAFRIS_LOG_LEVEL'] && ENV['WAFRIS_LOG_LEVEL'] == 'silent') end def self.suppressed_environments diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 31a3055..3f7a093 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -1,3 +1,6 @@ + +# "x-forwarded-for" -> split -> each last, check if it's int he list + # frozen_string_literal: true module Wafris @@ -29,20 +32,12 @@ def call(env) if Wafris.allow_request?(request) @app.call(env) else - LogSuppressor.puts_log( - "[Wafris] Blocked: #{request.ip} #{request.request_method} #{request.host} #{request.url}}" - ) + puts "[Wafris] Blocked: #{request.ip} #{request.request_method} #{request.host} #{request.url}}" [403, {}, ['Blocked']] end - rescue Redis::TimeoutError - LogSuppressor.puts_log( - "[Wafris] Wafris timed out during processing. Request passed without rules check." - ) - @app.call(env) rescue StandardError => e - LogSuppressor.puts_log( - "[Wafris] Redis connection error: #{e.message}. Request passed without rules check." - ) + puts "[Wafris] Redis connection error: #{e.message}. Request passed without rules check." + @app.call(env) end end diff --git a/lib/wafris/version.rb b/lib/wafris/version.rb index 3f78e84..2b58878 100644 --- a/lib/wafris/version.rb +++ b/lib/wafris/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Wafris - VERSION = "1.1.10" + VERSION = "2.0.0" end diff --git a/stylua.toml b/stylua.toml deleted file mode 100644 index 5d6c50d..0000000 --- a/stylua.toml +++ /dev/null @@ -1,3 +0,0 @@ -indent_type = "Spaces" -indent_width = 2 -column_width = 120 \ No newline at end of file diff --git a/test/configuration_test.rb b/test/configuration_test.rb index ead50d3..0b0dcce 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -1,49 +1,78 @@ # frozen_string_literal: true require 'test_helper' +require 'awesome_print' module Wafris describe Configuration do + before do + # Reset environment variables before each test + reset_environment_variables @config = Configuration.new end - describe "#initialize" do - it "allows setting attributes with a block" do - @config.redis = "some_redis_value" - @config.redis_pool_size = 30 - @config.quiet_mode = true - - _(@config.redis).must_equal "some_redis_value" - _(@config.redis_pool_size).must_equal 30 - _(@config.quiet_mode).must_equal true - end + after do + # Clean up or reset environment variables after each test + reset_environment_variables end - describe "#connection_pool" do - it "uses the set pool size" do - @config.redis_pool_size = 10 + describe "#initialize" do - _(@config.connection_pool.size).must_equal 10 + it "allows setting attributes with a block" do + @config.api_key = 'some_api_key' + @config.db_file_path = "/some/path" + @config.db_file_name = "wafris.db" + @config.downsync_custom_rules_interval = 600 + @config.downsync_data_subscriptions_interval = 864 + @config.downsync_url = 'https://example.com/v2/downsync' + @config.upsync_url = 'https://example.com/v2/upsync/' + @config.api_key + @config.upsync_interval = 600 + @config.upsync_queue_limit = 10 + + # Custom values are set + _(@config.api_key).must_equal "some_api_key" + _(@config.db_file_path).must_equal "/some/path" + _(@config.db_file_name).must_equal "wafris.db" + _(@config.downsync_custom_rules_interval).must_equal 600 + _(@config.downsync_data_subscriptions_interval).must_equal 864 + _(@config.downsync_url).must_equal 'https://example.com/v2/downsync' + _(@config.upsync_url).must_equal 'https://example.com/v2/upsync/' + @config.api_key + _(@config.upsync_interval).must_equal 600 + _(@config.upsync_queue_limit).must_equal 10 end - it "should default connection pool size" do - _(@config.connection_pool.size).must_equal 20 + it "sets default values if api key set" do + + _(@config.api_key).must_be_nil + _(@config.db_file_path).must_equal 'tmp/wafris' + _(@config.db_file_name).must_equal 'wafris.db' + _(@config.downsync_custom_rules_interval).must_equal 60 + _(@config.downsync_data_subscriptions_interval).must_equal 86400 + _(@config.downsync_url).must_equal 'https://distributor.wafris.org/v2/downsync' + _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync/' + _(@config.upsync_interval).must_equal 60 + _(@config.upsync_queue_limit).must_equal 1000 + end - end - describe "#create_settings" do - # This test assumes that a Redis server is running and accessible. - it "sets the waf settings in Redis" do - redis_mock = Minitest::Mock.new - redis_mock.expect(:hset, true, ['waf-settings', 'version', Wafris::VERSION, 'client', 'ruby', 'maxmemory', 25]) - redis_mock.expect(:connection, { host: 'localhost' }) + it "config setting takes precedence over env var setting" do - @config.redis = redis_mock - @config.create_settings - redis_mock.verify + # Set API Key via ENV + ENV['WAFRIS_API_KEY'] = '1234' + @env_config = Configuration.new + _(@env_config.api_key).must_equal '1234' + + # Override with config setting + @env_config.api_key = '5678' + _(@env_config.api_key).must_equal '5678' + end + end + + + end end diff --git a/test/middleware_test.rb b/test/middleware_test.rb index c33a3a6..18b82ae 100644 --- a/test/middleware_test.rb +++ b/test/middleware_test.rb @@ -3,25 +3,9 @@ require 'test_helper' module Wafris - describe Middleware do - it "should allow ok requests" do - Wafris.configure do |config| - config.redis = Redis.new - end + - get '/' + # Works with no api key (passes ) - _(last_response.status).must_equal 200 - end - - it "should rescue from a standard error with a message" do - Wafris.configure do |config| - config.redis = Redis.new(url: 'redis://foobar') - end - - get '/' - - _(last_response.status).must_equal 200 - end - end + end diff --git a/test/test_helper.rb b/test/test_helper.rb index 07abd8e..d686f56 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -9,6 +9,24 @@ class Minitest::Spec include Rack::Test::Methods + def reset_environment_variables + env_vars = [ + 'WAFRIS_API_KEY', + 'WAFRIS_DB_FILE_PATH', + 'WAFRIS_DB_FILE_NAME', + 'WAFRIS_DOWNSYNC_CUSTOM_RULES_INTERVAL', + 'WAFRIS_DOWNSYNC_DATA_SUBSCRIPTIONS_INTERVAL', + 'WAFRIS_DOWNSYNC_URL', + 'WAFRIS_UPSYNC_URL', + 'WAFRIS_UPSYNC_INTERVAL', + 'WAFRIS_UPSYNC_QUEUE_LIMIT' + ] + + env_vars.each do |var| + ENV[var] = nil + end + end + def app Rack::Builder.new do use Wafris::Middleware diff --git a/test/wafris_test.rb b/test/wafris_test.rb index e4f2d22..6dfc521 100644 --- a/test/wafris_test.rb +++ b/test/wafris_test.rb @@ -2,15 +2,190 @@ require 'test_helper' +if !ENV['WAFRIS_LOG_LEVEL'] + puts "\n\nSet WAFRIS_LOG_LEVEL to 'silent' to suppress log output in test.\n\n" +end + describe Wafris do - describe '.configure' do - it 'creates a connection pool with a 60 size' do - Wafris.configure do |config| - config.redis = Redis.new - config.redis_pool_size = 60 - end - _(Wafris.configuration.connection_pool.size).must_equal 60 + before do + # Reset environment variables before each test + reset_environment_variables + + Wafris.configure do |config| + config.api_key = 'some' + end + + @rules_db = SQLite3::Database.new "test/wafris_test_custom_rules.db" + + # The following variables are used to test the WAF rules + # and are in the test/wafris_test.rb file - which is generated + # from the Wafris Client Tests Ruleset + + # IPs + @non_blocked_ipv4 = '2.2.2.2' + @blocked_ipv4 = '1.1.1.1' + @blocked_ipv4_allow_ips_test = '7.7.7.7' + @blocked_ipv6_allow_cidrs_test = '2900:8805:2723:3d00:c83c:ba63:3e65:0001' + @non_blocked_ipv6 = '2600:8805:2723:3d00:c83c:ba63:3e65:0000' + @blocked_ipv6 = '2600:8805:2723:3d00:c83c:ba63:3e65:7a68' + + # Hosts + @blocked_host = 'blocked.com' + @non_blocked_host = 'example.com' + + # Paths + @blocked_path = '/blocked' + @non_blocked_path = '/nonblocked' + + # User Agents + @blocked_user_agent = 'blocked' + @non_blocked_user_agent = 'example' + + # Parameters + @blocked_parameters = 'blocked' + @non_blocked_parameters = 'example' + + # Methods + @blocked_method = 'PUT' + @non_blocked_method = 'GET' + + # Block CIDRs + @blocked_ipv4_cidr = '3.3.3.0/8' + @blocked_ipv6_cidr = '2600:8805:2723:3d00::/64' + @non_blocked_ipv6_cidr = '2700:8805:2723:3d00::/64' + + @ipv4_in_blocked_cidr = '3.3.3.1' + @ipv4_not_in_blocked_cidr = '4.4.4.4' + + @ipv6_in_blocked_cidr = '2600:8805:2723:3d00::1' + @ipv6_not_in_blocked_cidr = '2600:8805:2723:0000::2' + + # Allow IPs + @allowed_ipv4 = '3.3.3.9' + @allowed_ipv6 = '2600:8805:2723:3D00:0000:0000:0000:0001' + + # Allow CIDRs + @allowed_ipv4_cidr = '7.7.7.0/24' + @allowed_ipv6_cidr = '2900:8805:2723:3d00::/64' + + # Allow IPs in CIDRs + @ipv4_in_allowed_cidr = '7.7.7.1' + @ipv6_in_allowed_cidr = '2900:8805:2723:3d00:0000:0000:0000:0001' + end + + after do + reset_environment_variables + @rules_db.close + end + + + describe "CIDR lookups should work" do + + it "should return true if the blocked IPv6 is in the allowed_cidrs list" do + assert_equal true, !Wafris.ip_in_cidr_range(@blocked_ipv6_allow_cidrs_test, 'allowed_cidr_ranges', @rules_db).nil? + end + + it "should return true if the blocked IPv4 is in the blocked_cidrs list" do + assert_equal true, !Wafris.ip_in_cidr_range(@ipv4_in_blocked_cidr, 'blocked_cidr_ranges', @rules_db).nil? + end + + it "should return true if the blocked IPv6 is in the blocked_cidrs list" do + assert_equal true, !Wafris.ip_in_cidr_range(@ipv6_in_blocked_cidr, 'blocked_cidr_ranges', @rules_db).nil? + end + + it "should return false if the non-blocked IPv6 is not in the blocked_cidrs list" do + assert_nil Wafris.ip_in_cidr_range(@non_blocked_ipv6_cidr, 'blocked_cidr_ranges', @rules_db) + end + + it "should return true if the allowed IPv4 is in the allowed_cidrs list" do + assert_nil Wafris.ip_in_cidr_range(@allowed_ipv4_cidr, 'allowed_cidr_ranges', @rules_db) + end + + it "should return true if the allowed IPv6 is in the allowed_cidrs list" do + assert_equal true, !Wafris.ip_in_cidr_range(@ipv6_in_allowed_cidr, 'allowed_cidr_ranges', @rules_db).nil? + end + + end + + describe "Exact Matches should work" do + + it "should return true if the blocked IPv4 is in the allowed_ips list" do + assert_equal false, Wafris.exact_match(@blocked_ipv4_allow_ips_test, 'allowed_ips', @rules_db) + end + + it "should return false if the non-blocked IPv6 is not in the blocked_ips list" do + assert_equal false, Wafris.exact_match(@non_blocked_ipv6, 'blocked_ips', @rules_db) + end + + it "should return true if the blocked IPv6 is in the blocked_ips list" do + assert_equal true, Wafris.exact_match(@blocked_ipv6, 'blocked_ips', @rules_db) + end + + it "should return true if the allowed IPv6 is in the allowed_ips list" do + assert_equal true, Wafris.exact_match(@allowed_ipv6, 'allowed_ips', @rules_db) + end + + it "should return false if the IP is not in the allowed_ips list" do + assert_equal true, Wafris.exact_match(@allowed_ipv4, 'allowed_ips', @rules_db) + end + + it "should return false if the blocked IP is not in the allowed_ips list" do + assert_equal false, Wafris.exact_match(@blocked_ipv4, 'allowed_ips', @rules_db) + end + + it "should return true if the blocked IP is in the blocked_ips list" do + assert_equal true, Wafris.exact_match(@blocked_ipv4, 'blocked_ips', @rules_db) + end + + it "should return false if the non-blocked IP is not in the blocked_ips list" do + assert_equal false, Wafris.exact_match(@non_blocked_ipv4, 'blocked_ips', @rules_db) + end + + it "should return true if the blocked Host is in the blocked_hosts list" do + assert_equal true, Wafris.exact_match(@blocked_host, 'blocked_hosts', @rules_db) + end + + it "should return false if the non-blocked Host is not in the blocked_hosts list" do + assert_equal false, Wafris.exact_match(@non_blocked_host, 'blocked_hosts', @rules_db) + end + + it "should return true if the blocked Method is in the blocked_methods list" do + assert_equal true, Wafris.exact_match(@blocked_method, 'blocked_methods', @rules_db) + end + + it "should return false if the non-blocked Method is not in the blocked_methods list" do + assert_equal false, Wafris.exact_match(@non_blocked_method, 'blocked_methods', @rules_db) + end + + end + + describe "Substring Matches should work" do + it "should return true if the blocked Path is in the blocked_paths list" do + assert_equal true, !Wafris.substring_match(@blocked_path, 'blocked_paths', @rules_db).nil? + end + + it "should return false if the non-blocked Path is not in the blocked_paths list" do + assert_nil nil, Wafris.substring_match(@non_blocked_path, 'blocked_paths', @rules_db) + end + + it "should return true if the blocked User Agent is in the blocked_user_agents list" do + assert_equal true, !Wafris.substring_match(@blocked_user_agent, 'blocked_user_agents', @rules_db).nil? end + + it "should return false if the non-blocked User Agent is not in the blocked_user_agents list" do + assert_nil nil, Wafris.substring_match(@non_blocked_user_agent, 'blocked_user_agents', @rules_db) + end + + it "should return true if the blocked Parameters are in the blocked_parameters list" do + assert_equal true, !Wafris.substring_match(@blocked_parameters, 'blocked_parameters', @rules_db).nil? + end + + it "should return false if the non-blocked Parameters are not in the blocked_parameters list" do + assert_nil nil, Wafris.substring_match(@non_blocked_parameters, 'blocked_parameters', @rules_db) + end + end + + end diff --git a/test/wafris_test_custom_rules.db b/test/wafris_test_custom_rules.db new file mode 100644 index 0000000000000000000000000000000000000000..d85bf9dd13856d08b207c8c6b23761c61428ce89 GIT binary patch literal 94208 zcmeI*+i&Ak9>8%sPV1&=>w&^VtAeo92WF;2vE%rX0%B(<I@RLy^QYg;&iG?NGSQbm$A>)zmHE^ zot)b9<{PiHd!hM$r+d)qnT~QnQB~#lrl}~(74cOOU&HO9*eMPV#J_50f70$1W%HxY zmqqTyxkJVH!nm^Xx0UVX|15Ww?k#;@{<(Z-@!Q3t(s!j#7yh|$UH?}9QZLQ_dF}^M zm<<605SS`~{@g{q(s0!N6Yay@@Z9OFx+kKw0FDBZtL)a@Hi)_^1_YA zwe5!a@@p?P-ZoFGY`*@QnbUxI=^#9~8FnxC=N`>HkJkQv=eB$vIY}4u&m*T~bRK!h zC+D&7h+cWYSNrR6G$*Irxpmm<-f8Z1cEjW5a`p1n15`%bJa|G zH@6#aZ(n}z;rWW=sBQTKP8|}f`JJbAhL=3);h{_qITt1;>0zmdoC0zWhZpq9#)jHo zm4fK)b;DNgWK1WA4@pfAag$8*(wWTqTZSH?eYPfb*lqQ~=6?I2opzc@x{%v(roiOR zk6OLG*z&4*5cc*uyU9WOuoreeYV8j%v&e3JEC#R`$Z1md)_&Mb!@7L5B8Hs6DoaCr z(C)~%PZEvf%m+ULhGbHW>{w4S(oz1G<-z4~C9R~`NOLQ>N+(UGaZDc4SIyCo|Foo6 zp0m|{MGE<-)oqD}!|q7bY5H<*;A#Hgr$jdPG%pRmzfjgIw&?yDDgLzRkRVN2%2i17 zCl8hEYkM4`uPo}7=f$srHA&*uaoBCPJ`iJQPGDY20^oz4D~!$(F*7>kb2s z4;FGmjkC@jEBGeOSf1sN9_1{ojB=LVTHcD~nmvp8RQ zdR_fQx{&Rok;^C&=5pgZDVdE7h5JllcLIGq%X@%VkRmz8&#%AX15?kmB~2ofQ;tm1>;|e@xA!Qh5!NxAbaSY!u^F{JOk~zbg)R0%vFIX3G-qMEmbc`eQ`m2LL0SBLz?$N3#vg8Gz|DIyp!v+Ba5I_I{1Q0*~0R#|0009K1NuaDfUK-B-2kZaei|7AOQ-s8V00Iag zfB*srAb(UYp0tg_000IagfB*srAbW3}R009IL zKmY**5I_I{1Q0-AHU#Fi)smS15AOfJs~C4@BW$8X009ILKmY**5I_I{1Q0-AHUySb zU0GgL)zV9i?Ps-U{;yn^jmU`-0R#|0009ILKmY**5I_KdSrv%p|3%|5#rU`JSK}?Q z!-fC?2q1s}0tg_000IagfWV9i_{G9)%dNSVZCjpKbL^T`_uattE#LDj#|nbL@!gso z1pZT2X0v@Qx1tkx)q3F8E!(xLe!Xs0{pv`?f~a`vX4}4~@7T6f(el-^RkU1BuE_j< z#s)RvB7gt_2q1s}0tg_000Ib%2@K}{mTNn;YSnXHU(DaDLDdNy&$R=)ZizX(=URd7 z)}={xGXEc|XxHm@P;>l1v=?}F*K=#Nk%|wT|687Ki#HxTN2+*+`M>2_j$?bZK(5H& z|BqdE5)eQD0R#|0009ILKmY**5STiF==uNL|37t?m4FaH009ILKmY**5I_I{1Q3t} zqWOQ(_+51W|5wJFVuuX@1Q0*~0R#|0009ILKmdVR5~ypp)pOnaUz||$!FT^_an1AG z{$B{R+oe-0if0%^4-Oc+|DXB)EDdU6MF0T=5I_I{1Q0*~0R#{j6Bx|m$L`^O@Xi0p Z{C}+GgYW)7!~B23?f++(|4+F8{}+ot`0oG! literal 0 HcmV?d00001 diff --git a/wafris.gemspec b/wafris.gemspec index 2139b6e..8ceb0cb 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -16,15 +16,17 @@ Gem::Specification.new do |s| If you haven't already, please sign up for Wafris Hub at: - https://github.com/Wafris/wafris-rb + https://hub.wafris.org TEXT s.required_ruby_version = '>= 2.5' - s.add_runtime_dependency 'connection_pool', '>= 2.3' s.add_runtime_dependency 'rack', '>= 2.0' - s.add_runtime_dependency 'redis', '>= 4.8.0' + + s.add_dependency 'sqlite3' + s.add_dependency 'ipaddr' + s.add_dependency 'httparty' s.add_development_dependency 'minitest', '~> 5.1' s.add_development_dependency 'pry', '~> 0.14', '>= 0.14.1' diff --git a/wafris_test.db b/wafris_test.db new file mode 100644 index 0000000..e69de29 diff --git a/wafris_test_custom_rules.db b/wafris_test_custom_rules.db new file mode 100644 index 0000000..e69de29 From 713823603e987f0d707b2d1ec273bc3506ffde47 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Fri, 3 May 2024 11:35:51 -0400 Subject: [PATCH 02/32] progress --- Gemfile | 1 + config.ru | 12 ++++ lib/wafris.rb | 118 ++++++++++++++++-------------------- lib/wafris/configuration.rb | 14 +++++ lib/wafris/middleware.rb | 32 +++++++--- test/middleware_test.rb | 13 ++-- test/test_helper.rb | 3 + test/wafris_test.rb | 2 + wafris.gemspec | 6 +- 9 files changed, 119 insertions(+), 82 deletions(-) create mode 100644 config.ru diff --git a/Gemfile b/Gemfile index 7f4f5e9..61f1b84 100644 --- a/Gemfile +++ b/Gemfile @@ -3,3 +3,4 @@ source 'https://rubygems.org' gemspec + diff --git a/config.ru b/config.ru new file mode 100644 index 0000000..f2b0c02 --- /dev/null +++ b/config.ru @@ -0,0 +1,12 @@ +require 'rack' +require 'rack/reloader' + +use Rack::Reloader, 0 # 0 means reload on every request + +# Encoding defaults +Encoding.default_external = Encoding::UTF_8 +Encoding.default_internal = Encoding::UTF_8 + +require './lib/wafris/middleware' + +run Wafris::Middleware.new diff --git a/lib/wafris.rb b/lib/wafris.rb index 91e771e..36dd50b 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -18,22 +18,15 @@ module Wafris class << self attr_accessor :configuration - def configure - raise ArgumentError unless block_given? - - self.configuration ||= Wafris::Configuration.new - + def configure + self.configuration ||= Wafris::Configuration.new yield(configuration) LogSuppressor.puts_log("[Wafris] Configuration settings created.") - configuration.create_settings - rescue ArgumentError - puts "[Wafris] block is required to configure Wafris. More info can be found at: https://github.com/Wafris/wafris-rb" - - #rescue StandardError => e - # puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" + rescue StandardError => e + puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" end @@ -136,20 +129,20 @@ def check_rate_limit(ip, path, method, db_connection) rule_id = limiter[5] # Expire old timestamps - @rate_limiters.each do |ip, timestamps| + @configuration.rate_limiters.each do |ip, timestamps| # Removes timestamps older than the interval - @rate_limiters[ip] = timestamps.select { |timestamp| timestamp > current_timestamp - interval } + @configuration.rate_limiters[ip] = timestamps.select { |timestamp| timestamp > current_timestamp - interval } # Remove the IP if there are no more timestamps for the IP - @rate_limiters.delete(ip) if @rate_limiters[ip].empty? + @configuration.rate_limiters.delete(ip) if @configuration.rate_limiters[ip].empty? end # Check if the IP+Method is rate limited - if @rate_limiters[ip] && @rate_limiters[ip].length >= max_count + if @configuration.rate_limiters[ip] && @configuration.rate_limiters[ip].length >= max_count # Request is rate limited @@ -157,10 +150,10 @@ def check_rate_limit(ip, path, method, db_connection) else # Request is not rate limited, so add the current timestamp - if @rate_limiters[ip] - @rate_limiters[ip] << current_timestamp + if @configuration.rate_limiters[ip] + @configuration.rate_limiters[ip] << current_timestamp else - @rate_limiters[ip] = [current_timestamp] + @configuration.rate_limiters[ip] = [current_timestamp] end return false @@ -176,13 +169,13 @@ def send_upsync_requests(requests_array) headers = {'Content-Type' => 'application/json'} body = {batch: requests_array}.to_json - response = HTTParty.post(@upsync_url, + response = HTTParty.post(@configuration.upsync_url, :body => body, :headers => headers, :timeout => 300) if response.code == 200 puts "Upsync successful" - @upsync_status = 'Complete' + @configuration.upsync_status = 'Complete' else puts "Upsync Error. HTTP Response: #{response.code}" end @@ -206,19 +199,19 @@ def send_upsync_requests(requests_array) # ex: '192.23.5.4', 'SemRush', etc. def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatment, category, rule) - if @upsync_status != 'Disabled' + if @configuration.upsync_status != 'Disabled' # Add request to the queue request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] - @upsync_queue << request + @configuration.upsync_queue << request # If the queue is full, send the requests to the upsync server - if @upsync_queue.length >= @upsync_queue_limit || (Time.now.to_i - @last_upsync_timestamp) >= @upsync_interval - requests_array = @upsync_queue - @upsync_queue = [] - @last_upsync_timestamp = Time.now.to_i + if @configuration.upsync_queue.length >= @configuration.upsync_queue_limit || (Time.now.to_i - @configuration.last_upsync_timestamp) >= @configuration.upsync_interval + requests_array = @configuration.upsync_queue + @configuration.upsync_queue = [] + @configuration.last_upsync_timestamp = Time.now.to_i - @upsync_status = 'Uploading' + @configuration.upsync_status = 'Uploading' send_upsync_requests(requests_array) end @@ -233,7 +226,7 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme # Pulls the latest rules from the server def downsync_db(db_rule_category, current_filename = nil) - lockfile_path = "#{@db_file_path}/#{db_rule_category}.lockfile" + lockfile_path = "#{@configuration.db_file_path}/#{db_rule_category}.lockfile" puts lockfile_path @@ -250,8 +243,8 @@ def downsync_db(db_rule_category, current_filename = nil) filename = "" # Check server for new rules - #puts "Downloading from #{@downsync_url}/#{db_rule_category}/#{@api_key}?current_version=#{current_filename}" - uri = "#{@downsync_url}/#{db_rule_category}/#{@api_key}?current_version=#{current_filename}" + #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}" + uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}" response = HTTParty.get( uri, @@ -260,19 +253,19 @@ def downsync_db(db_rule_category, current_filename = nil) ) if response.code == 401 - @upsync_status = 'Disabled' + @configuration.upsync_status = 'Disabled' puts "Unauthorized: Bad or missing API key" filename = current_filename elsif response.code == 304 - @upsync_status = 'Enabled' + @configuration.upsync_status = 'Enabled' puts "No new rules to download" filename = current_filename elsif response.code == 200 - @upsync_status = 'Disabled' + @configuration.upsync_status = 'Disabled' if current_filename old_file_name = current_filename @@ -283,16 +276,16 @@ def downsync_db(db_rule_category, current_filename = nil) filename = content_disposition.match(/filename="(.+)"/)[1] # Save the body of the response to a new SQLite file - File.open(@db_file_path + "/" + filename, 'wb') { |file| file.write(response.body) } + File.open(@configuration.db_file_path + "/" + filename, 'wb') { |file| file.write(response.body) } # Write the filename into the db_category.modfile - File.open("#{@db_file_path}/#{db_rule_category}.modfile", 'w') { |file| file.write(filename) } + File.open("#{@configuration.db_file_path}/#{db_rule_category}.modfile", 'w') { |file| file.write(filename) } # Remove the old database file if old_file_name - #puts "Removing old file: #{@db_file_path}/#{old_file_name}" - if File.exist?(@db_file_path + "/" + old_file_name) - File.delete(@db_file_path + "/" + old_file_name) + #puts "Removing old file: #{@configuration.db_file_path}/#{old_file_name}" + if File.exist?(@configuration.db_file_path + "/" + old_file_name) + File.delete(@configuration.db_file_path + "/" + old_file_name) end end @@ -306,12 +299,12 @@ def downsync_db(db_rule_category, current_filename = nil) ensure # Reset the modified time of the modfile - unless File.exist?("#{@db_file_path}/#{db_rule_category}.modfile") - File.new("#{@db_file_path}/#{db_rule_category}.modfile", 'w') + unless File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.modfile") + File.new("#{@configuration.db_file_path}/#{db_rule_category}.modfile", 'w') end # Set the modified time of the modfile to the current time - File.utime(Time.now, Time.now, "#{@db_file_path}/#{db_rule_category}.modfile") + File.utime(Time.now, Time.now, "#{@configuration.db_file_path}/#{db_rule_category}.modfile") # Ensure the lockfile is removed after operations lockfile.close @@ -329,26 +322,26 @@ def downsync_db(db_rule_category, current_filename = nil) def current_db(db_rule_category) if db_rule_category == 'custom_rules' - interval = @downsync_custom_rules_interval + interval = @configuration.downsync_custom_rules_interval else - interval = @downsync_data_subscriptions_interval + interval = @configuration.downsync_data_subscriptions_interval end # Checks for existing current modfile, which contains the current db filename - if File.exist?("#{@db_file_path}/#{db_rule_category}.modfile") + if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.modfile") - #puts "Modfile exists: #{@db_file_path}/#{db_rule_category}.modfile" + puts "Modfile exists: #{@configuration.db_file_path}/#{db_rule_category}.modfile" # Get last Modified Time and current database file name - last_db_synctime = File.mtime("#{@db_file_path}/#{db_rule_category}.modfile").to_i - returned_db = File.read("#{@db_file_path}/#{db_rule_category}.modfile").strip + last_db_synctime = File.mtime("#{@configuration.db_file_path}/#{db_rule_category}.modfile").to_i + returned_db = File.read("#{@configuration.db_file_path}/#{db_rule_category}.modfile").strip # Check if the db file is older than the interval if (Time.now.to_i - last_db_synctime) > interval - #puts "DB file is older than the interval" + puts "DB file is older than the interval" # Make sure that another process isn't already downloading the rules - if !File.exist?("#{@db_file_path}/#{db_rule_category}.lockfile") + if !File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") returned_db = downsync_db(db_rule_category, returned_db) end @@ -357,7 +350,7 @@ def current_db(db_rule_category) # Current db is up to date else - returned_db = File.read("#{@db_file_path}/#{db_rule_category}.modfile").strip + returned_db = File.read("#{@configuration.db_file_path}/#{db_rule_category}.modfile").strip # If the modfile is empty (no db file name), return nil # this can happen if the the api key is bad @@ -372,9 +365,11 @@ def current_db(db_rule_category) # No modfile exists, so download the latest db else + puts "Modfile does not exist: #{@configuration.db_file_path}/#{db_rule_category}.modfile" + # Make sure that another process isn't already downloading the rules - if File.exist?("#{@db_file_path}/#{db_rule_category}.lockfile") - # Lockfile exists, but not modfile with a db filename + if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") + # Lockfile exists, but no modfile with a db filename return nil else @@ -398,7 +393,11 @@ def current_db(db_rule_category) # This is the main loop that evaluates the request # as well as sorts out when downsync and upsync should be called def evaluate(ip, user_agent, path, parameters, host, method) - + @configuration ||= Wafris::Configuration.new + + ap @configuration + + rules_db_filename = current_db('custom_rules') data_subscriptions_db_filename = current_db('data_subscriptions') @@ -488,19 +487,6 @@ def evaluate(ip, user_agent, path, parameters, host, method) end # end evaluate - - - def allow_request?(request) - - status = "foo" - - if status.eql? 'Blocked' - return false - else - return true - end - - end end end diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 84f7843..62e71b2 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -38,6 +38,7 @@ class Configuration attr_accessor :upsync_url attr_accessor :upsync_interval attr_accessor :upsync_queue_limit + attr_accessor :upsync_status attr_accessor :local_only def initialize @@ -127,9 +128,22 @@ def initialize # This prevents the client from sending upsync requests # if the API key is known bad @upsync_status = 'Disabled' + + return true end + def current_config + + output = "" + + instance_variables.each do |var| + output += "#{var} = #{instance_variable_get(var)}\n" + end + + return output + + end def create_settings diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 3f7a093..2e5e4e9 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -1,8 +1,8 @@ -# "x-forwarded-for" -> split -> each last, check if it's int he list - # frozen_string_literal: true +require 'wafris' + module Wafris class Middleware def initialize(app) @@ -29,16 +29,32 @@ def call(env) request = Rack::Request.new(env) - if Wafris.allow_request?(request) - @app.call(env) + # Forcing UTF-8 encoding on all strings for Sqlite3 compatibility + ip = request.ip.force_encoding('UTF-8') + user_agent = request.user_agent.force_encoding('UTF-8') + path = request.path.force_encoding('UTF-8') + parameters = Rack::Utils.build_query(request.params).force_encoding('UTF-8') + host = request.host.to_s.force_encoding('UTF-8') + request_method = String.new(request.request_method).force_encoding('UTF-8') + + # Submitted for evaluation + treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method) + + # These values match what the client tests expect (200, 404, 403, 500 + if treatment == 'Allowed' || treatment == 'Passed' + @app.call(env) + elsif treatment == 'Blocked' + [403, { 'content-type' => 'text/plain' }, ['Blocked']] else - puts "[Wafris] Blocked: #{request.ip} #{request.request_method} #{request.host} #{request.url}}" - [403, {}, ['Blocked']] + #ap request + [500, { 'content-type' => 'text/plain' }, ['Error']] end + rescue StandardError => e - puts "[Wafris] Redis connection error: #{e.message}. Request passed without rules check." - + + LogSuppressor.puts_log "[Wafris] Error in Middleware: #{e.message}" @app.call(env) + end end end diff --git a/test/middleware_test.rb b/test/middleware_test.rb index 18b82ae..1acf70a 100644 --- a/test/middleware_test.rb +++ b/test/middleware_test.rb @@ -3,9 +3,10 @@ require 'test_helper' module Wafris - - - # Works with no api key (passes ) - - -end + describe Middleware do + it "should pass requests if no API key" do + get '/' + _(last_response.status).must_equal 200 + end + end +end \ No newline at end of file diff --git a/test/test_helper.rb b/test/test_helper.rb index d686f56..16f4441 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -5,6 +5,9 @@ require "minitest/autorun" require 'rack' require 'rack/test' +require 'webmock' +require 'webmock/minitest' +require 'fakefs/safe' class Minitest::Spec include Rack::Test::Methods diff --git a/test/wafris_test.rb b/test/wafris_test.rb index 6dfc521..13226e2 100644 --- a/test/wafris_test.rb +++ b/test/wafris_test.rb @@ -2,6 +2,7 @@ require 'test_helper' + if !ENV['WAFRIS_LOG_LEVEL'] puts "\n\nSet WAFRIS_LOG_LEVEL to 'silent' to suppress log output in test.\n\n" end @@ -188,4 +189,5 @@ end + end diff --git a/wafris.gemspec b/wafris.gemspec index 8ceb0cb..81d76f0 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -8,7 +8,7 @@ Gem::Specification.new do |s| s.name = 'wafris' s.version = Wafris::VERSION s.summary = 'Web Application Firewall for Rack apps' - s.authors = ['Micahel Buckbee', 'Ryan Castillo'] + s.authors = ['Michael Buckbee', 'Ryan Castillo'] s.files = Dir.glob('{bin,lib}/**/*') s.license = 'Elastic-2.0' s.post_install_message = <<-TEXT @@ -31,7 +31,9 @@ Gem::Specification.new do |s| s.add_development_dependency 'minitest', '~> 5.1' s.add_development_dependency 'pry', '~> 0.14', '>= 0.14.1' s.add_development_dependency 'rack-test', '>= 0.6' - s.add_development_dependency 'rails', '>= 5.0' + s.add_development_dependency 'rails', '>= 6.0' s.add_development_dependency 'railties', '>= 5.0' s.add_development_dependency 'rake', '>= 12.0' + s.add_development_dependency 'webmock', '>= 0.49' + s.add_development_dependency 'fakefs', '>= 0.9' end From b1a102747d340a7204dad19b65b2197386831651 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 4 May 2024 18:02:31 -0400 Subject: [PATCH 03/32] Cleanup of deps --- lib/wafris.rb | 32 +++++++++++++-------------- lib/wafris/configuration.rb | 12 +++++----- lib/wafris/log_suppressor.rb | 2 +- lib/wafris/middleware.rb | 1 - test/ReadMe.md | 15 +++++++++++++ test/configuration_test.rb | 2 +- test/downsync_test.rb | 43 ++++++++++++++++++++++++++++++++++++ test/wafris_test.rb | 1 - wafris.gemspec | 4 +++- 9 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 test/ReadMe.md create mode 100644 test/downsync_test.rb diff --git a/lib/wafris.rb b/lib/wafris.rb index 36dd50b..1a2698a 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -6,6 +6,7 @@ require 'sqlite3' require 'ipaddr' require 'httparty' +require 'awesome_print' require 'wafris/configuration' require 'wafris/middleware' @@ -18,16 +19,18 @@ module Wafris class << self attr_accessor :configuration - def configure - self.configuration ||= Wafris::Configuration.new - yield(configuration) - - LogSuppressor.puts_log("[Wafris] Configuration settings created.") - configuration.create_settings + def configure + begin + self.configuration ||= Wafris::Configuration.new + yield(configuration) + + LogSuppressor.puts_log("[Wafris] Configuration settings created.") + configuration.create_settings rescue StandardError => e puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" - + end + end @@ -213,11 +216,15 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme @configuration.upsync_status = 'Uploading' send_upsync_requests(requests_array) + else + puts "Request queued: #{ip} #{treatment} #{category} #{rule}" + puts "Queue length: " + @configuration.upsync_queue.length.to_s end # Return the treatment - used to return 403 or 200 return treatment else + puts "Upsync is disabled. Returning 'Passed'" return "Passed" end @@ -395,20 +402,11 @@ def current_db(db_rule_category) def evaluate(ip, user_agent, path, parameters, host, method) @configuration ||= Wafris::Configuration.new - ap @configuration - + ap @configuration.current_config rules_db_filename = current_db('custom_rules') data_subscriptions_db_filename = current_db('data_subscriptions') - ap "Rules path: #{rules_db_filename}" - ap "Rules path nil: #{rules_db_filename.nil?}" - ap "Rules path empty: #{rules_db_filename.to_s.strip == ''}" - - ap "Data Subscriptions path: #{data_subscriptions_db_filename}" - ap "Data Subscriptions path nil: #{data_subscriptions_db_filename.nil?}" - ap "Data Subscriptions path empty: #{data_subscriptions_db_filename.to_s.strip == ''}" - if rules_db_filename.to_s.strip != '' && data_subscriptions_db_filename.strip.to_s.strip != '' diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 62e71b2..e93beaa 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -48,7 +48,7 @@ def initialize @api_key = ENV['WAFRIS_API_KEY'] else @api_key = nil - LogSuppressor.puts_log("Firewall disabled as API key not set") + LogSuppressor.puts_log("Firewall disabled as neither local only or API key set") end # DB FILE PATH LOCATION - Optional @@ -56,13 +56,13 @@ def initialize @db_file_path = ENV['WAFRIS_DB_FILE_PATH'] else #@db_file_path = Rails.root.join('tmp', 'wafris').to_s - @db_file_path = 'tmp/wafris' + @db_file_path = './tmp/wafris' end - # Verify that the db_file_path exists + # Ensure that the db_file_path exists unless File.directory?(@db_file_path) LogSuppressor.puts_log("DB File Path does not exist - creating it now.") - Dir.mkdir(@db_file_path) unless File.exists?(@db_file_path) + FileUtils.mkdir_p(@db_file_path) unless File.exist?(@db_file_path) end # DB FILE NAME - For local @@ -135,10 +135,10 @@ def initialize def current_config - output = "" + output = {} instance_variables.each do |var| - output += "#{var} = #{instance_variable_get(var)}\n" + output[var.to_s] = instance_variable_get(var) end return output diff --git a/lib/wafris/log_suppressor.rb b/lib/wafris/log_suppressor.rb index 5170b33..d1f218a 100644 --- a/lib/wafris/log_suppressor.rb +++ b/lib/wafris/log_suppressor.rb @@ -3,7 +3,7 @@ module Wafris class LogSuppressor def self.puts_log(message) - puts(message) unless suppress_logs? + puts("[Wafris] " + message) unless suppress_logs? end def self.suppress_logs? diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 2e5e4e9..1ba2ffd 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true -require 'wafris' module Wafris class Middleware diff --git a/test/ReadMe.md b/test/ReadMe.md new file mode 100644 index 0000000..e330f56 --- /dev/null +++ b/test/ReadMe.md @@ -0,0 +1,15 @@ + +# This Readme + +This readme covers the steps to testing and making changes to the Wafris RB client itself and not the installation and use of Wafris in Rails. + +## For local development + +1. Install rerun: `gem install rerun` + +2. Remove any Wafris environment variables: `bash ./remove-env-vars.sh` + +3. Set target environment variables: `source ./set-dev-env-vars.sh` + +3. From `test/dummy` run `rerun -d ../../ 'rails server -p 3333'` - this will relaunch the Rails app whenever a file changes in the wafris-rb gem. + diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 0b0dcce..0d4d0d7 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -45,7 +45,7 @@ module Wafris it "sets default values if api key set" do _(@config.api_key).must_be_nil - _(@config.db_file_path).must_equal 'tmp/wafris' + _(@config.db_file_path).must_equal './tmp/wafris' _(@config.db_file_name).must_equal 'wafris.db' _(@config.downsync_custom_rules_interval).must_equal 60 _(@config.downsync_data_subscriptions_interval).must_equal 86400 diff --git a/test/downsync_test.rb b/test/downsync_test.rb new file mode 100644 index 0000000..754c20c --- /dev/null +++ b/test/downsync_test.rb @@ -0,0 +1,43 @@ + + +require 'test_helper' + +if !ENV['WAFRIS_LOG_LEVEL'] + puts "\n\nSet WAFRIS_LOG_LEVEL to 'silent' to suppress log output in test.\n\n" +end + +describe Wafris do + + before do + # Reset environment variables before each test + reset_environment_variables + @current_custom_rule_db_file = nil + @current_data_subscription_db_file = nil + + end + + describe "Custom data should work from a cold start" do + + it "should confirm Modfiles exist" do + assert(File.exist?("tmp/custom_rules.modfile")) + assert(File.exist?("tmp/data_subscriptions.modfile")) + end + + it "should confirm Modfiles contain correct db filenames" do + assert(File.read("tmp/custom_rules.modfile").include?(".db")) + assert(File.read("tmp/data_subscriptions.modfile").include?(".db")) + end + + it "should confirm Custom Rules Lockfile cleanup" do + refute(File.exist?("tmp/custom_rules.lockfile")) + end + + it "should confirm Data Subscription Lockfile cleanup" do + refute(File.exist?("tmp/data_subscriptions.lockfile")) + end + + + end + + +end \ No newline at end of file diff --git a/test/wafris_test.rb b/test/wafris_test.rb index 13226e2..3cd4add 100644 --- a/test/wafris_test.rb +++ b/test/wafris_test.rb @@ -2,7 +2,6 @@ require 'test_helper' - if !ENV['WAFRIS_LOG_LEVEL'] puts "\n\nSet WAFRIS_LOG_LEVEL to 'silent' to suppress log output in test.\n\n" end diff --git a/wafris.gemspec b/wafris.gemspec index 81d76f0..581ae9b 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -27,6 +27,7 @@ Gem::Specification.new do |s| s.add_dependency 'sqlite3' s.add_dependency 'ipaddr' s.add_dependency 'httparty' + s.add_dependency 'awesome_print' s.add_development_dependency 'minitest', '~> 5.1' s.add_development_dependency 'pry', '~> 0.14', '>= 0.14.1' @@ -35,5 +36,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'railties', '>= 5.0' s.add_development_dependency 'rake', '>= 12.0' s.add_development_dependency 'webmock', '>= 0.49' - s.add_development_dependency 'fakefs', '>= 0.9' + + end From 7b4cfbc1a3b20c019b92e7d5d23045d20a83d75d Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 08:39:52 -0400 Subject: [PATCH 04/32] Updates --- lib/wafris.rb | 187 +++++++++++++++++++----------------- lib/wafris/configuration.rb | 26 ++++- lib/wafris/middleware.rb | 3 +- test/ReadMe-Development.md | 28 ++++++ test/ReadMe.md | 15 --- test/configuration_test.rb | 8 +- test/downsync_test.rb | 87 +++++++++++++---- test/test_helper.rb | 8 +- wafris.gemspec | 2 +- 9 files changed, 228 insertions(+), 136 deletions(-) create mode 100644 test/ReadMe-Development.md delete mode 100644 test/ReadMe.md diff --git a/lib/wafris.rb b/lib/wafris.rb index 1a2698a..3100f5e 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -1,4 +1,3 @@ - # frozen_string_literal: true @@ -27,6 +26,8 @@ def configure LogSuppressor.puts_log("[Wafris] Configuration settings created.") configuration.create_settings + @upsync_queue = [] + rescue StandardError => e puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" end @@ -172,10 +173,14 @@ def send_upsync_requests(requests_array) headers = {'Content-Type' => 'application/json'} body = {batch: requests_array}.to_json - response = HTTParty.post(@configuration.upsync_url, + + url_and_api_key = @configuration.upsync_url + '/' + @configuration.api_key + + response = HTTParty.post(url_and_api_key, :body => body, :headers => headers, :timeout => 300) + if response.code == 200 puts "Upsync successful" @configuration.upsync_status = 'Complete' @@ -206,19 +211,19 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme # Add request to the queue request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] - @configuration.upsync_queue << request + @upsync_queue << request # If the queue is full, send the requests to the upsync server - if @configuration.upsync_queue.length >= @configuration.upsync_queue_limit || (Time.now.to_i - @configuration.last_upsync_timestamp) >= @configuration.upsync_interval - requests_array = @configuration.upsync_queue - @configuration.upsync_queue = [] + if @upsync_queue.length >= @configuration.upsync_queue_limit || (Time.now.to_i - @configuration.last_upsync_timestamp) >= @configuration.upsync_interval + requests_array = @upsync_queue + @upsync_queue = [] @configuration.last_upsync_timestamp = Time.now.to_i @configuration.upsync_status = 'Uploading' send_upsync_requests(requests_array) else puts "Request queued: #{ip} #{treatment} #{category} #{rule}" - puts "Queue length: " + @configuration.upsync_queue.length.to_s + puts "Queue length: " + @upsync_queue.length.to_s end # Return the treatment - used to return 403 or 200 @@ -235,14 +240,17 @@ def downsync_db(db_rule_category, current_filename = nil) lockfile_path = "#{@configuration.db_file_path}/#{db_rule_category}.lockfile" - puts lockfile_path - + # Ensure the directory exists before attempting to open the lockfile + FileUtils.mkdir_p(@configuration.db_file_path) unless Dir.exist?(@configuration.db_file_path) + # Attempt to create a lockfile with exclusive access; skip if it exists begin lockfile = File.open(lockfile_path, File::RDWR|File::CREAT|File::EXCL) rescue Errno::EEXIST puts "Lockfile already exists, skipping downsync." return + rescue Exception => e + puts "EXCEPTION: Error creating lockfile: #{e.message}" end begin @@ -404,85 +412,90 @@ def evaluate(ip, user_agent, path, parameters, host, method) ap @configuration.current_config - rules_db_filename = current_db('custom_rules') - data_subscriptions_db_filename = current_db('data_subscriptions') - - - if rules_db_filename.to_s.strip != '' && data_subscriptions_db_filename.strip.to_s.strip != '' - - rules_db = SQLite3::Database.new "#{@db_file_path}/#{rules_db_filename}" - data_subscriptions_db = SQLite3::Database.new "#{@db_file_path}/#{data_subscriptions_db_filename}" - - # Allowed IPs - if exact_match(ip, 'allowed_ips', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ai', ip) - end - - # Allowed CIDR Ranges - if ip_in_cidr_range(ip, 'allowed_cidr_ranges', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ac', ip) - end - - # Blocked IPs - if exact_match(ip, 'blocked_ips', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bi', ip) - end - - # Blocked CIDR Ranges - if ip_in_cidr_range(ip, 'blocked_cidr_ranges', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bc', ip) - end - - # Blocked Country Codes - country_code = get_country_code(ip, data_subscriptions_db) - if exact_match(country_code, 'blocked_country_codes', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "G_#{country_code}") - end - - # Blocked Reputation IP Ranges - if ip_in_cidr_range(ip, 'reputation_ip_ranges', data_subscriptions_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "R") - end - - # Blocked User Agents - user_agent_match = substring_match(user_agent, 'blocked_user_agents', rules_db) - if user_agent_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bu', user_agent_match) - end - - # Blocked Paths - path_match = substring_match(path, 'blocked_paths', rules_db) - if path_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bp', path_match) - end - - # Blocked Parameters - parameters_match = substring_match(parameters, 'blocked_parameters', rules_db) - if parameters_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'ba', parameters_match) - end - - # Blocked Hosts - if exact_match(host, 'blocked_hosts', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bh', host) - end - - # Blocked Methods - if exact_match(method, 'blocked_methods', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bm', method) - end - - # Rate Limiting - rule_id = check_rate_limit(ip, path, method, rules_db) - if rule_id - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'brl', rule_id) + if @configuration.api_key.nil? + return "Passed" + else + + rules_db_filename = current_db('custom_rules') + data_subscriptions_db_filename = current_db('data_subscriptions') + + if rules_db_filename.to_s.strip != '' && data_subscriptions_db_filename.strip.to_s.strip != '' + + rules_db = SQLite3::Database.new "#{@configuration.db_file_path}/#{rules_db_filename}" + data_subscriptions_db = SQLite3::Database.new "#{@configuration.db_file_path}/#{data_subscriptions_db_filename}" + + + # Allowed IPs + if exact_match(ip, 'allowed_ips', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ai', ip) + end + + # Allowed CIDR Ranges + if ip_in_cidr_range(ip, 'allowed_cidr_ranges', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ac', ip) + end + + # Blocked IPs + if exact_match(ip, 'blocked_ips', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bi', ip) + end + + # Blocked CIDR Ranges + if ip_in_cidr_range(ip, 'blocked_cidr_ranges', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bc', ip) + end + + # Blocked Country Codes + country_code = get_country_code(ip, data_subscriptions_db) + if exact_match(country_code, 'blocked_country_codes', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "G_#{country_code}") + end + + # Blocked Reputation IP Ranges + if ip_in_cidr_range(ip, 'reputation_ip_ranges', data_subscriptions_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "R") + end + + # Blocked User Agents + user_agent_match = substring_match(user_agent, 'blocked_user_agents', rules_db) + if user_agent_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bu', user_agent_match) + end + + # Blocked Paths + path_match = substring_match(path, 'blocked_paths', rules_db) + if path_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bp', path_match) + end + + # Blocked Parameters + parameters_match = substring_match(parameters, 'blocked_parameters', rules_db) + if parameters_match + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'ba', parameters_match) + end + + # Blocked Hosts + if exact_match(host, 'blocked_hosts', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bh', host) + end + + # Blocked Methods + if exact_match(method, 'blocked_methods', rules_db) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bm', method) + end + + # Rate Limiting + rule_id = check_rate_limit(ip, path, method, rules_db) + if rule_id + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'brl', rule_id) + end + end - - end - - # Passed if no allow or block rules matched - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Passed', 'passed', '-') - + + # Passed if no allow or block rules matched + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Passed', 'passed', '-') + + end # end api_key.nil? end # end evaluate diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index e93beaa..6fe3ceb 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -1,8 +1,21 @@ # Wafris setup and logs -# - No startup messages in dev or test or CI environments -# - Way to disable WAF in v2 (disabled?) + # - No startup messages in dev or test or CI environments + # - Way to disable WAF in v2 (disabled?) + +# Proper Behavior + + # No API Key + # - No lockfile + # - No modfile + # - No db file + + + + + + # API Key @@ -15,6 +28,8 @@ # - Honeybadger says no api key in dev # - Quiet mode on startup -> show no messages at startup + + # Verbose mode? # - 1st time setup # - Startup success @@ -40,6 +55,7 @@ class Configuration attr_accessor :upsync_queue_limit attr_accessor :upsync_status attr_accessor :local_only + attr_accessor :last_upsync_timestamp def initialize @@ -94,13 +110,13 @@ def initialize else @downsync_url = 'https://distributor.wafris.org/v2/downsync' end - + # UPSYNC - Optional # Set Upsync URL if ENV['WAFRIS_UPSYNC_URL'] - @upsync_url = ENV['WAFRIS_UPSYNC_URL'] + '/' + @api_key + @upsync_url = ENV['WAFRIS_UPSYNC_URL'] else - @upsync_url = 'https://collector.wafris.org/v2/upsync/' + @api_key.to_s + @upsync_url = 'https://collector.wafris.org/v2/upsync' end # Set Upsync Interval - Optional diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 1ba2ffd..231bbc5 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -51,7 +51,8 @@ def call(env) rescue StandardError => e - LogSuppressor.puts_log "[Wafris] Error in Middleware: #{e.message}" + LogSuppressor.puts_log "[Wafris] Detailed Error: #{e.class} - #{e.message}" + LogSuppressor.puts_log "[Wafris] Backtrace: #{e.backtrace.join("\n")}" @app.call(env) end diff --git a/test/ReadMe-Development.md b/test/ReadMe-Development.md new file mode 100644 index 0000000..e7bda50 --- /dev/null +++ b/test/ReadMe-Development.md @@ -0,0 +1,28 @@ + +# This Readme + +This readme covers the steps to testing and making changes to the Wafris RB client itself and not the installation and use of Wafris in Rails. + +## For local development + +1. Install rerun: `gem install rerun` + +2. Navigate to the `test/dummy` directory + +3. Remove any Wafris environment variables: `bash ./remove-env-vars.sh` + +4. Set target environment variables: `source ./set-dev-env-vars.sh` + +5. From `test/dummy` run `rerun -d ../../ 'rails server -p 3333'` - this will relaunch the Rails app whenever a file changes in the wafris-rb gem. + +# Testing API Key + +As this is a client for the Wafris API, it can be tricky to develop against. Our recommendation is to use the test API key `wafris-client-test-api-key` for development. This key is loaded with the "Wafris Client Test" ruleset on the Wafris Hub. + +This will let you use the production (default) Downsync and Upsync endpoints without needing to set up a local development version of the Wafris Hub application. + +# Testing Endpoints + + + + diff --git a/test/ReadMe.md b/test/ReadMe.md deleted file mode 100644 index e330f56..0000000 --- a/test/ReadMe.md +++ /dev/null @@ -1,15 +0,0 @@ - -# This Readme - -This readme covers the steps to testing and making changes to the Wafris RB client itself and not the installation and use of Wafris in Rails. - -## For local development - -1. Install rerun: `gem install rerun` - -2. Remove any Wafris environment variables: `bash ./remove-env-vars.sh` - -3. Set target environment variables: `source ./set-dev-env-vars.sh` - -3. From `test/dummy` run `rerun -d ../../ 'rails server -p 3333'` - this will relaunch the Rails app whenever a file changes in the wafris-rb gem. - diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 0d4d0d7..5802f67 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -26,7 +26,7 @@ module Wafris @config.downsync_custom_rules_interval = 600 @config.downsync_data_subscriptions_interval = 864 @config.downsync_url = 'https://example.com/v2/downsync' - @config.upsync_url = 'https://example.com/v2/upsync/' + @config.api_key + @config.upsync_url = 'https://example.com/v2/upsync' @config.upsync_interval = 600 @config.upsync_queue_limit = 10 @@ -37,12 +37,12 @@ module Wafris _(@config.downsync_custom_rules_interval).must_equal 600 _(@config.downsync_data_subscriptions_interval).must_equal 864 _(@config.downsync_url).must_equal 'https://example.com/v2/downsync' - _(@config.upsync_url).must_equal 'https://example.com/v2/upsync/' + @config.api_key + _(@config.upsync_url).must_equal 'https://example.com/v2/upsync' _(@config.upsync_interval).must_equal 600 _(@config.upsync_queue_limit).must_equal 10 end - it "sets default values if api key set" do + it "sets default values" do _(@config.api_key).must_be_nil _(@config.db_file_path).must_equal './tmp/wafris' @@ -50,7 +50,7 @@ module Wafris _(@config.downsync_custom_rules_interval).must_equal 60 _(@config.downsync_data_subscriptions_interval).must_equal 86400 _(@config.downsync_url).must_equal 'https://distributor.wafris.org/v2/downsync' - _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync/' + _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync' _(@config.upsync_interval).must_equal 60 _(@config.upsync_queue_limit).must_equal 1000 diff --git a/test/downsync_test.rb b/test/downsync_test.rb index 754c20c..8d8c503 100644 --- a/test/downsync_test.rb +++ b/test/downsync_test.rb @@ -1,5 +1,3 @@ - - require 'test_helper' if !ENV['WAFRIS_LOG_LEVEL'] @@ -10,34 +8,83 @@ before do # Reset environment variables before each test - reset_environment_variables - @current_custom_rule_db_file = nil - @current_data_subscription_db_file = nil + reset_environment_variables + # Remove all cached files + remove_cache_directory + + # Using Localhost Hub for testing, should be changed to production + @downsync_url = 'http://localhost:3000/v2/downsync' + @upsync_url = 'http://localhost:3000/v2/upsync/' + end describe "Custom data should work from a cold start" do - it "should confirm Modfiles exist" do - assert(File.exist?("tmp/custom_rules.modfile")) - assert(File.exist?("tmp/data_subscriptions.modfile")) + it "should confirm no files at cold start" do + refute(Dir.glob("tmp/wafris/*.db").any?) + refute(File.exist?("tmp/wafris/custom_rules.modfile")) + refute(File.exist?("tmp/wafris/data_subscriptions.modfile")) end - - it "should confirm Modfiles contain correct db filenames" do - assert(File.read("tmp/custom_rules.modfile").include?(".db")) - assert(File.read("tmp/data_subscriptions.modfile").include?(".db")) - end - - it "should confirm Custom Rules Lockfile cleanup" do - refute(File.exist?("tmp/custom_rules.lockfile")) + + it "shouldn't raise exceptions if no API Key" do + Wafris.configure do |config| + config.api_key = nil + config.downsync_url = @downsync_url + config.upsync_url = @upsync_url + end + + Wafris.downsync_db('custom_rules', nil) end - - it "should confirm Data Subscription Lockfile cleanup" do - refute(File.exist?("tmp/data_subscriptions.lockfile")) + + it "should successfully downsync data subscription with a good API key" do + Wafris.configure do |config| + config.api_key = 'wafris-client-test-api-key' + config.downsync_url = @downsync_url + config.upsync_url = @upsync_url + end + + # Simulate a successful downsync operation + db_rule_category = 'data_subscriptions' + current_filename = nil + lockfile_path = "#{Wafris.configuration.db_file_path}/#{db_rule_category}.lockfile" + modfile_path = "#{Wafris.configuration.db_file_path}/#{db_rule_category}.modfile" + + # Perform the downsync + Wafris.downsync_db(db_rule_category, current_filename) + + # Check if the lockfile, modfile, and db file are created successfully + assert File.exist?(modfile_path), "Modfile was not created" + + # Read the value in the modfile + db_file_name = File.read(modfile_path) + #ap Wafris.configuration.db_file_path + #ap db_file_name + assert File.exist?(File.join(Wafris.configuration.db_file_path, db_file_name)), "DB file was not created" + + refute File.exist?(lockfile_path), "Lockfile should not exist" + end + + # Custom Rules + # Should create a lockfile + # Should create a modfile + # Should do something if api key is bad or error + # Should create a db file if success + # Should remove lockfile if success + + # Data Subscriptions + # Should create a lockfile + # Should create a modfile + # Should do something if api key is bad or error + # Should create a db file if success + # Should remove lockfile if success + + # Current DB + # Should refresh at interval end -end \ No newline at end of file +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 16f4441..2d736db 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -5,9 +5,6 @@ require "minitest/autorun" require 'rack' require 'rack/test' -require 'webmock' -require 'webmock/minitest' -require 'fakefs/safe' class Minitest::Spec include Rack::Test::Methods @@ -30,6 +27,11 @@ def reset_environment_variables end end + # Removes sync'd databases, lockfiles and modfiles + def remove_cache_directory + FileUtils.rm_rf('tmp') + end + def app Rack::Builder.new do use Wafris::Middleware diff --git a/wafris.gemspec b/wafris.gemspec index 581ae9b..b00e3bd 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -35,7 +35,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rails', '>= 6.0' s.add_development_dependency 'railties', '>= 5.0' s.add_development_dependency 'rake', '>= 12.0' - s.add_development_dependency 'webmock', '>= 0.49' + end From 37a4663e2a4c921f6cfdc5f0bc8f3f2e7fd74ebf Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 08:48:56 -0400 Subject: [PATCH 05/32] Adding meta fields to the upsync --- lib/wafris.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 3100f5e..a6d831f 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -172,7 +172,21 @@ def send_upsync_requests(requests_array) begin headers = {'Content-Type' => 'application/json'} - body = {batch: requests_array}.to_json + + if Rails && Rails.application + framework = 'rails' + else + framework = 'rack' + end + + body = { + meta: { + version: Wafris::VERSION, + client: 'wafris-rb', + framework: framework + }, + batch: requests_array + }.to_json url_and_api_key = @configuration.upsync_url + '/' + @configuration.api_key @@ -180,7 +194,7 @@ def send_upsync_requests(requests_array) :body => body, :headers => headers, :timeout => 300) - + if response.code == 200 puts "Upsync successful" @configuration.upsync_status = 'Complete' From 55e8df9fe0cb58768086061f9789a8a7f541c663 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 09:16:05 -0400 Subject: [PATCH 06/32] Update wafris.gemspec --- wafris.gemspec | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wafris.gemspec b/wafris.gemspec index b00e3bd..88ec031 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -12,11 +12,9 @@ Gem::Specification.new do |s| s.files = Dir.glob('{bin,lib}/**/*') s.license = 'Elastic-2.0' s.post_install_message = <<-TEXT - Thank you for installing the wafris gem. + Thank you for installing the Wafris gem. - If you haven't already, please sign up for Wafris Hub at: - - https://hub.wafris.org + Get your API key and set rules at https://hub.wafris.org TEXT From e1e84edcd7aaaff5889ab8cbecdc193902d14df2 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 09:16:17 -0400 Subject: [PATCH 07/32] Update wafris.gemspec --- wafris.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wafris.gemspec b/wafris.gemspec index 88ec031..767f690 100644 --- a/wafris.gemspec +++ b/wafris.gemspec @@ -14,7 +14,7 @@ Gem::Specification.new do |s| s.post_install_message = <<-TEXT Thank you for installing the Wafris gem. - Get your API key and set rules at https://hub.wafris.org + Get your API key and set firewall rules at https://hub.wafris.org TEXT From bda89bde14c9749d63f1e4b2f794e2aedf88b09c Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 09:59:30 -0400 Subject: [PATCH 08/32] Upsync queue fix --- lib/wafris.rb | 9 +++++---- lib/wafris/configuration.rb | 3 ++- test/configuration_test.rb | 3 +++ test/downsync_test.rb | 8 -------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index a6d831f..8f584f1 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -221,8 +221,9 @@ def send_upsync_requests(requests_array) # ex: '192.23.5.4', 'SemRush', etc. def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatment, category, rule) - if @configuration.upsync_status != 'Disabled' - + if @configuration.upsync_status != 'Disabled' || @configuration.upsync_status != 'Uploading' + @configuration.upsync_status = 'Uploading' + # Add request to the queue request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] @upsync_queue << request @@ -233,7 +234,7 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme @upsync_queue = [] @configuration.last_upsync_timestamp = Time.now.to_i - @configuration.upsync_status = 'Uploading' + send_upsync_requests(requests_array) else puts "Request queued: #{ip} #{treatment} #{category} #{rule}" @@ -294,7 +295,7 @@ def downsync_db(db_rule_category, current_filename = nil) filename = current_filename elsif response.code == 200 - @configuration.upsync_status = 'Disabled' + @configuration.upsync_status = 'Enabled' if current_filename old_file_name = current_filename diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 6fe3ceb..d7c394c 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -54,6 +54,7 @@ class Configuration attr_accessor :upsync_interval attr_accessor :upsync_queue_limit attr_accessor :upsync_status + attr_accessor :upsync_queue attr_accessor :local_only attr_accessor :last_upsync_timestamp @@ -133,7 +134,7 @@ def initialize @upsync_queue_limit = 1000 end - # Upsync Queue + # Upsync Queue Defaults @upsync_queue = [] @last_upsync_timestamp = Time.now.to_i diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 5802f67..87bc1d6 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -29,6 +29,7 @@ module Wafris @config.upsync_url = 'https://example.com/v2/upsync' @config.upsync_interval = 600 @config.upsync_queue_limit = 10 + @config.upsync_queue = ['foo'] # Custom values are set _(@config.api_key).must_equal "some_api_key" @@ -40,6 +41,7 @@ module Wafris _(@config.upsync_url).must_equal 'https://example.com/v2/upsync' _(@config.upsync_interval).must_equal 600 _(@config.upsync_queue_limit).must_equal 10 + _(@config.upsync_queue).must_equal ['foo'] end it "sets default values" do @@ -53,6 +55,7 @@ module Wafris _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync' _(@config.upsync_interval).must_equal 60 _(@config.upsync_queue_limit).must_equal 1000 + _(@config.upsync_queue).must_equal [] end diff --git a/test/downsync_test.rb b/test/downsync_test.rb index 8d8c503..977606b 100644 --- a/test/downsync_test.rb +++ b/test/downsync_test.rb @@ -13,10 +13,6 @@ # Remove all cached files remove_cache_directory - # Using Localhost Hub for testing, should be changed to production - @downsync_url = 'http://localhost:3000/v2/downsync' - @upsync_url = 'http://localhost:3000/v2/upsync/' - end describe "Custom data should work from a cold start" do @@ -30,8 +26,6 @@ it "shouldn't raise exceptions if no API Key" do Wafris.configure do |config| config.api_key = nil - config.downsync_url = @downsync_url - config.upsync_url = @upsync_url end Wafris.downsync_db('custom_rules', nil) @@ -40,8 +34,6 @@ it "should successfully downsync data subscription with a good API key" do Wafris.configure do |config| config.api_key = 'wafris-client-test-api-key' - config.downsync_url = @downsync_url - config.upsync_url = @upsync_url end # Simulate a successful downsync operation From 4b55e812f04c493408cb4bef3edd4a26e1e939a2 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Thu, 9 May 2024 10:26:45 -0400 Subject: [PATCH 09/32] Upating for sync --- lib/wafris.rb | 14 +++++++------- lib/wafris/configuration.rb | 4 ---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 8f584f1..8ac87bd 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -26,8 +26,6 @@ def configure LogSuppressor.puts_log("[Wafris] Configuration settings created.") configuration.create_settings - @upsync_queue = [] - rescue StandardError => e puts "[Wafris] firewall disabled due to: #{e.message}. Cannot connect via Wafris.configure. Please check your configuration settings. More info can be found at: https://github.com/Wafris/wafris-rb" end @@ -226,24 +224,26 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme # Add request to the queue request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] - @upsync_queue << request + @configuration.upsync_queue << request # If the queue is full, send the requests to the upsync server - if @upsync_queue.length >= @configuration.upsync_queue_limit || (Time.now.to_i - @configuration.last_upsync_timestamp) >= @configuration.upsync_interval - requests_array = @upsync_queue - @upsync_queue = [] + if @configuration.upsync_queue.length >= @configuration.upsync_queue_limit || (Time.now.to_i - @configuration.last_upsync_timestamp) >= @configuration.upsync_interval + requests_array = @configuration.upsync_queue + @configuration.upsync_queue = [] @configuration.last_upsync_timestamp = Time.now.to_i send_upsync_requests(requests_array) else puts "Request queued: #{ip} #{treatment} #{category} #{rule}" - puts "Queue length: " + @upsync_queue.length.to_s + puts "Queue length: " + @configuration.upsync_queue.length.to_s end + @configuration.upsync_status = 'Enabled' # Return the treatment - used to return 403 or 200 return treatment else + @configuration.upsync_status = 'Enabled' puts "Upsync is disabled. Returning 'Passed'" return "Passed" end diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index d7c394c..4e54f53 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -163,11 +163,7 @@ def current_config end def create_settings - @version = Wafris::VERSION - - LogSuppressor.puts_log("[Wafris] Firewall launched successfully. Ready to process requests. Set rules at: https://hub.wafris.org/") - end end From 4350b7037ef7bd997153fb03aa0ebd6606b672ff Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 25 May 2024 07:45:45 -0400 Subject: [PATCH 10/32] Fixing logging --- lib/wafris.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 8ac87bd..9240d47 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -284,8 +284,8 @@ def downsync_db(db_rule_category, current_filename = nil) if response.code == 401 @configuration.upsync_status = 'Disabled' - puts "Unauthorized: Bad or missing API key" - + LogSuppressor.puts_log("Unauthorized: Bad or missing API key") + LogSuppressor.puts_log("API Key: #{@configuration.api_key}") filename = current_filename elsif response.code == 304 @@ -422,11 +422,13 @@ def current_db(db_rule_category) # This is the main loop that evaluates the request # as well as sorts out when downsync and upsync should be called - def evaluate(ip, user_agent, path, parameters, host, method) + def evaluate(ip, user_agent, path, parameters, host, method, headers, body) @configuration ||= Wafris::Configuration.new ap @configuration.current_config + + if @configuration.api_key.nil? return "Passed" else From b1df86bd77d0e0431e32088218994ec5401fe5ec Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 28 May 2024 07:27:35 -0400 Subject: [PATCH 11/32] Fixing tests --- lib/wafris.rb | 5 ----- lib/wafris/configuration.rb | 8 ++++++++ lib/wafris/middleware.rb | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 9240d47..c823fc9 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -416,8 +416,6 @@ def current_db(db_rule_category) end - - end # This is the main loop that evaluates the request @@ -427,8 +425,6 @@ def evaluate(ip, user_agent, path, parameters, host, method, headers, body) ap @configuration.current_config - - if @configuration.api_key.nil? return "Passed" else @@ -440,7 +436,6 @@ def evaluate(ip, user_agent, path, parameters, host, method, headers, body) rules_db = SQLite3::Database.new "#{@configuration.db_file_path}/#{rules_db_filename}" data_subscriptions_db = SQLite3::Database.new "#{@configuration.db_file_path}/#{data_subscriptions_db_filename}" - # Allowed IPs if exact_match(ip, 'allowed_ips', rules_db) diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 4e54f53..c570e7a 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -57,6 +57,7 @@ class Configuration attr_accessor :upsync_queue attr_accessor :local_only attr_accessor :last_upsync_timestamp + attr_accessor :max_body_size_mb def initialize @@ -134,6 +135,13 @@ def initialize @upsync_queue_limit = 1000 end + # Set Maximium Body Size for Requests - Optional (in Megabytes) + if ENV['WAFRIS_MAX_BODY_SIZE_MB'] && ENV['WAFRIS_MAX_BODY_SIZE_MB'].to_i > 0 + @max_body_size_mb = ENV['WAFRIS_MAX_BODY_SIZE_MB'].to_i + else + @max_body_size_mb = 10 + end + # Upsync Queue Defaults @upsync_queue = [] @last_upsync_timestamp = Time.now.to_i diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 231bbc5..ed58094 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -37,7 +37,10 @@ def call(env) request_method = String.new(request.request_method).force_encoding('UTF-8') # Submitted for evaluation - treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method) + headers = request.headers.each_with_object({}) { |(k, v), h| h[k] = v.force_encoding('UTF-8') } + body = request.body.read.force_encoding('UTF-8') + + treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method, headers, body) # These values match what the client tests expect (200, 404, 403, 500 if treatment == 'Allowed' || treatment == 'Passed' From fa8bbbd77ef14ffdab5c423a6421f51b730d9dd0 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Mon, 3 Jun 2024 15:38:53 -0400 Subject: [PATCH 12/32] Fix limiters --- lib/wafris.rb | 40 ++++++++++++++---------------------- lib/wafris/configuration.rb | 41 +------------------------------------ lib/wafris/middleware.rb | 2 +- test/ReadMe-Development.md | 11 +++++++--- 4 files changed, 25 insertions(+), 69 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index c823fc9..d8e3eac 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -105,10 +105,10 @@ def exact_match(request_property, table_name, db_connection) def check_rate_limit(ip, path, method, db_connection) # Correctly format the SQL query with placeholders - limiters = db_connection.execute("SELECT * FROM blocked_rate_limits WHERE path LIKE ? AND method = ?", ["%#{path}%", method]) + limiters = db_connection.execute("SELECT * FROM blocked_rate_limits WHERE path = ? AND method = ?", [path, method]) # If no rate limiters are matched - if limiters.empty? + if limiters.empty? return false end @@ -134,8 +134,6 @@ def check_rate_limit(ip, path, method, db_connection) @configuration.rate_limiters.each do |ip, timestamps| # Removes timestamps older than the interval - - @configuration.rate_limiters[ip] = timestamps.select { |timestamp| timestamp > current_timestamp - interval } # Remove the IP if there are no more timestamps for the IP @@ -193,15 +191,13 @@ def send_upsync_requests(requests_array) :headers => headers, :timeout => 300) - if response.code == 200 - puts "Upsync successful" + if response.code == 200 @configuration.upsync_status = 'Complete' else - puts "Upsync Error. HTTP Response: #{response.code}" + LogSuppressor.puts_log("Upsync Error. HTTP Response: #{response.code}") end rescue HTTParty::Error => e - puts "Upsync Response: #{response.code}" - puts "Failed to send upsync requests: #{e.message}" + LogSuppressor.puts_log("Upsync Error. Failed to send upsync requests: #{e.message}") end return true end @@ -231,20 +227,21 @@ def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatme requests_array = @configuration.upsync_queue @configuration.upsync_queue = [] @configuration.last_upsync_timestamp = Time.now.to_i - send_upsync_requests(requests_array) - else - puts "Request queued: #{ip} #{treatment} #{category} #{rule}" - puts "Queue length: " + @configuration.upsync_queue.length.to_s end @configuration.upsync_status = 'Enabled' # Return the treatment - used to return 403 or 200 + + message = "Request #{treatment}" + message += " | Category: #{category}" unless category.blank? + message += " | Rule: #{rule}" unless rule.blank? + LogSuppressor.puts_log(message) + return treatment else - @configuration.upsync_status = 'Enabled' - puts "Upsync is disabled. Returning 'Passed'" + @configuration.upsync_status = 'Enabled' return "Passed" end @@ -262,10 +259,10 @@ def downsync_db(db_rule_category, current_filename = nil) begin lockfile = File.open(lockfile_path, File::RDWR|File::CREAT|File::EXCL) rescue Errno::EEXIST - puts "Lockfile already exists, skipping downsync." + LogSuppressor.puts_log("Lockfile already exists, skipping downsync.") return rescue Exception => e - puts "EXCEPTION: Error creating lockfile: #{e.message}" + LogSuppressor.puts_log("EXCEPTION: Error creating lockfile: #{e.message}") end begin @@ -290,7 +287,7 @@ def downsync_db(db_rule_category, current_filename = nil) elsif response.code == 304 @configuration.upsync_status = 'Enabled' - puts "No new rules to download" + LogSuppressor.puts_log("No new rules to download") filename = current_filename @@ -360,15 +357,12 @@ def current_db(db_rule_category) # Checks for existing current modfile, which contains the current db filename if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.modfile") - puts "Modfile exists: #{@configuration.db_file_path}/#{db_rule_category}.modfile" - # Get last Modified Time and current database file name last_db_synctime = File.mtime("#{@configuration.db_file_path}/#{db_rule_category}.modfile").to_i returned_db = File.read("#{@configuration.db_file_path}/#{db_rule_category}.modfile").strip # Check if the db file is older than the interval if (Time.now.to_i - last_db_synctime) > interval - puts "DB file is older than the interval" # Make sure that another process isn't already downloading the rules if !File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") @@ -395,8 +389,6 @@ def current_db(db_rule_category) # No modfile exists, so download the latest db else - puts "Modfile does not exist: #{@configuration.db_file_path}/#{db_rule_category}.modfile" - # Make sure that another process isn't already downloading the rules if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") # Lockfile exists, but no modfile with a db filename @@ -423,8 +415,6 @@ def current_db(db_rule_category) def evaluate(ip, user_agent, path, parameters, host, method, headers, body) @configuration ||= Wafris::Configuration.new - ap @configuration.current_config - if @configuration.api_key.nil? return "Passed" else diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index c570e7a..6411956 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -1,44 +1,4 @@ -# Wafris setup and logs - - # - No startup messages in dev or test or CI environments - # - Way to disable WAF in v2 (disabled?) - -# Proper Behavior - - # No API Key - # - No lockfile - # - No modfile - # - No db file - - - - - - - -# API Key - - # - Local only mode "local_only" (TBD) - # - No upsync - # - Bad API key (checked on initial downsync) - # - No upsync - - # - No API key - # - Honeybadger says no api key in dev - # - Quiet mode on startup -> show no messages at startup - - - -# Verbose mode? -# - 1st time setup -# - Startup success -# - Downsync success -# - Upsync success - - -# frozen_string_literal: true - require_relative 'version' module Wafris @@ -58,6 +18,7 @@ class Configuration attr_accessor :local_only attr_accessor :last_upsync_timestamp attr_accessor :max_body_size_mb + attr_accessor :rate_limiters def initialize diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index ed58094..bebade6 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -37,7 +37,7 @@ def call(env) request_method = String.new(request.request_method).force_encoding('UTF-8') # Submitted for evaluation - headers = request.headers.each_with_object({}) { |(k, v), h| h[k] = v.force_encoding('UTF-8') } + headers = env.each_with_object({}) { |(k, v), h| h[k] = v.force_encoding('UTF-8') if k.start_with?('HTTP_') } body = request.body.read.force_encoding('UTF-8') treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method, headers, body) diff --git a/test/ReadMe-Development.md b/test/ReadMe-Development.md index e7bda50..b3d802f 100644 --- a/test/ReadMe-Development.md +++ b/test/ReadMe-Development.md @@ -9,11 +9,11 @@ This readme covers the steps to testing and making changes to the Wafris RB clie 2. Navigate to the `test/dummy` directory -3. Remove any Wafris environment variables: `bash ./remove-env-vars.sh` +3. From `test/dummy` run -4. Set target environment variables: `source ./set-dev-env-vars.sh` +`WAFRIS_API_KEY='wafris-client-test-api-key' rerun -d ../../ 'rails server -p 3333'` -5. From `test/dummy` run `rerun -d ../../ 'rails server -p 3333'` - this will relaunch the Rails app whenever a file changes in the wafris-rb gem. +This will relaunch the Rails app whenever a file changes in the wafris-rb gem. # Testing API Key @@ -23,6 +23,11 @@ This will let you use the production (default) Downsync and Upsync endpoints wit # Testing Endpoints +Use the Wafris Client Tests - https://github.com/Wafris/wafris-client-tests to test the endpoints. +# Dummy application should be configured to: + +- Allow `GET` and `POST` requests `/` +- Allow `example.com` and `blocked.com` hosts in development From 4f05af631776a8a5e9f490155511470b85c4cd93 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Mon, 3 Jun 2024 15:48:40 -0400 Subject: [PATCH 13/32] Adding dummy test app --- test/dummy/.dockerignore | 37 ++++++ test/dummy/.gitattributes | 7 ++ test/dummy/.gitignore | 35 ++++++ test/dummy/.keep | 0 test/dummy/.ruby-version | 1 + test/dummy/Dockerfile | 58 ++++++++++ test/dummy/Gemfile | 45 ++++++++ test/dummy/Rakefile | 6 + test/dummy/app/assets/config/manifest.js | 2 + test/dummy/app/assets/images/.keep | 0 .../app/assets/stylesheets/application.css | 15 +++ .../app/controllers/application_controller.rb | 2 + test/dummy/app/controllers/concerns/.keep | 0 test/dummy/app/helpers/application_helper.rb | 2 + test/dummy/app/jobs/application_job.rb | 7 ++ test/dummy/app/models/concerns/.keep | 0 .../app/views/layouts/application.html.erb | 15 +++ test/dummy/bin/bundle | 109 ++++++++++++++++++ test/dummy/bin/docker-entrypoint | 3 + test/dummy/bin/rails | 4 + test/dummy/bin/rake | 4 + test/dummy/bin/setup | 25 ++++ test/dummy/config.ru | 6 + test/dummy/config/application.rb | 42 +++++++ test/dummy/config/boot.rb | 3 + test/dummy/config/credentials.yml.enc | 1 + test/dummy/config/environment.rb | 5 + test/dummy/config/environments/development.rb | 63 ++++++++++ test/dummy/config/environments/production.rb | 80 +++++++++++++ test/dummy/config/environments/test.rb | 54 +++++++++ test/dummy/config/initializers/assets.rb | 12 ++ .../initializers/content_security_policy.rb | 25 ++++ .../initializers/filter_parameter_logging.rb | 8 ++ test/dummy/config/initializers/inflections.rb | 16 +++ .../config/initializers/permissions_policy.rb | 13 +++ test/dummy/config/locales/en.yml | 31 +++++ test/dummy/config/puma.rb | 35 ++++++ test/dummy/config/routes.rb | 12 ++ test/dummy/lib/assets/.keep | 0 test/dummy/lib/tasks/.keep | 0 test/dummy/log/.keep | 0 test/dummy/public/404.html | 67 +++++++++++ test/dummy/public/422.html | 67 +++++++++++ test/dummy/public/500.html | 66 +++++++++++ .../public/apple-touch-icon-precomposed.png | 0 test/dummy/public/apple-touch-icon.png | 0 test/dummy/public/favicon.ico | 0 test/dummy/public/robots.txt | 1 + test/dummy/storage/.keep | 0 test/dummy/test/controllers/.keep | 0 test/dummy/test/fixtures/files/.keep | 0 test/dummy/test/helpers/.keep | 0 test/dummy/test/integration/.keep | 0 test/dummy/test/models/.keep | 0 test/dummy/test/test_helper.rb | 12 ++ test/dummy/tmp/.keep | 0 test/dummy/tmp/pids/.keep | 0 test/dummy/tmp/storage/.keep | 0 test/dummy/vendor/.keep | 0 59 files changed, 996 insertions(+) create mode 100644 test/dummy/.dockerignore create mode 100644 test/dummy/.gitattributes create mode 100644 test/dummy/.gitignore create mode 100644 test/dummy/.keep create mode 100644 test/dummy/.ruby-version create mode 100644 test/dummy/Dockerfile create mode 100644 test/dummy/Gemfile create mode 100644 test/dummy/Rakefile create mode 100644 test/dummy/app/assets/config/manifest.js create mode 100644 test/dummy/app/assets/images/.keep create mode 100644 test/dummy/app/assets/stylesheets/application.css create mode 100644 test/dummy/app/controllers/application_controller.rb create mode 100644 test/dummy/app/controllers/concerns/.keep create mode 100644 test/dummy/app/helpers/application_helper.rb create mode 100644 test/dummy/app/jobs/application_job.rb create mode 100644 test/dummy/app/models/concerns/.keep create mode 100644 test/dummy/app/views/layouts/application.html.erb create mode 100755 test/dummy/bin/bundle create mode 100755 test/dummy/bin/docker-entrypoint create mode 100755 test/dummy/bin/rails create mode 100755 test/dummy/bin/rake create mode 100755 test/dummy/bin/setup create mode 100644 test/dummy/config.ru create mode 100644 test/dummy/config/application.rb create mode 100644 test/dummy/config/boot.rb create mode 100644 test/dummy/config/credentials.yml.enc create mode 100644 test/dummy/config/environment.rb create mode 100644 test/dummy/config/environments/development.rb create mode 100644 test/dummy/config/environments/production.rb create mode 100644 test/dummy/config/environments/test.rb create mode 100644 test/dummy/config/initializers/assets.rb create mode 100644 test/dummy/config/initializers/content_security_policy.rb create mode 100644 test/dummy/config/initializers/filter_parameter_logging.rb create mode 100644 test/dummy/config/initializers/inflections.rb create mode 100644 test/dummy/config/initializers/permissions_policy.rb create mode 100644 test/dummy/config/locales/en.yml create mode 100644 test/dummy/config/puma.rb create mode 100644 test/dummy/config/routes.rb create mode 100644 test/dummy/lib/assets/.keep create mode 100644 test/dummy/lib/tasks/.keep create mode 100644 test/dummy/log/.keep create mode 100644 test/dummy/public/404.html create mode 100644 test/dummy/public/422.html create mode 100644 test/dummy/public/500.html create mode 100644 test/dummy/public/apple-touch-icon-precomposed.png create mode 100644 test/dummy/public/apple-touch-icon.png create mode 100644 test/dummy/public/favicon.ico create mode 100644 test/dummy/public/robots.txt create mode 100644 test/dummy/storage/.keep create mode 100644 test/dummy/test/controllers/.keep create mode 100644 test/dummy/test/fixtures/files/.keep create mode 100644 test/dummy/test/helpers/.keep create mode 100644 test/dummy/test/integration/.keep create mode 100644 test/dummy/test/models/.keep create mode 100644 test/dummy/test/test_helper.rb create mode 100644 test/dummy/tmp/.keep create mode 100644 test/dummy/tmp/pids/.keep create mode 100644 test/dummy/tmp/storage/.keep create mode 100644 test/dummy/vendor/.keep diff --git a/test/dummy/.dockerignore b/test/dummy/.dockerignore new file mode 100644 index 0000000..9612375 --- /dev/null +++ b/test/dummy/.dockerignore @@ -0,0 +1,37 @@ +# See https://docs.docker.com/engine/reference/builder/#dockerignore-file for more about ignoring files. + +# Ignore git directory. +/.git/ + +# Ignore bundler config. +/.bundle + +# Ignore all environment files (except templates). +/.env* +!/.env*.erb + +# Ignore all default key files. +/config/master.key +/config/credentials/*.key + +# Ignore all logfiles and tempfiles. +/log/* +/tmp/* +!/log/.keep +!/tmp/.keep + +# Ignore pidfiles, but keep the directory. +/tmp/pids/* +!/tmp/pids/.keep + +# Ignore storage (uploaded files in development and any SQLite databases). +/storage/* +!/storage/.keep +/tmp/storage/* +!/tmp/storage/.keep + +# Ignore assets. +/node_modules/ +/app/assets/builds/* +!/app/assets/builds/.keep +/public/assets diff --git a/test/dummy/.gitattributes b/test/dummy/.gitattributes new file mode 100644 index 0000000..4a66706 --- /dev/null +++ b/test/dummy/.gitattributes @@ -0,0 +1,7 @@ +# See https://git-scm.com/docs/gitattributes for more about git attribute files. + + +# Mark any vendored files as having been vendored. +vendor/* linguist-vendored +config/credentials/*.yml.enc diff=rails_credentials +config/credentials.yml.enc diff=rails_credentials diff --git a/test/dummy/.gitignore b/test/dummy/.gitignore new file mode 100644 index 0000000..5fb66c9 --- /dev/null +++ b/test/dummy/.gitignore @@ -0,0 +1,35 @@ +# See https://help.github.com/articles/ignoring-files for more about ignoring files. +# +# If you find yourself ignoring temporary files generated by your text editor +# or operating system, you probably want to add a global ignore instead: +# git config --global core.excludesfile '~/.gitignore_global' + +# Ignore bundler config. +/.bundle + +# Ignore all environment files (except templates). +/.env* +!/.env*.erb + +# Ignore all logfiles and tempfiles. +/log/* +/tmp/* +!/log/.keep +!/tmp/.keep + +# Ignore pidfiles, but keep the directory. +/tmp/pids/* +!/tmp/pids/ +!/tmp/pids/.keep + +# Ignore storage (uploaded files in development and any SQLite databases). +/storage/* +!/storage/.keep +/tmp/storage/* +!/tmp/storage/ +!/tmp/storage/.keep + +/public/assets + +# Ignore master key for decrypting credentials and more. +/config/master.key diff --git a/test/dummy/.keep b/test/dummy/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/.ruby-version b/test/dummy/.ruby-version new file mode 100644 index 0000000..c877459 --- /dev/null +++ b/test/dummy/.ruby-version @@ -0,0 +1 @@ +ruby-3.1.3 diff --git a/test/dummy/Dockerfile b/test/dummy/Dockerfile new file mode 100644 index 0000000..b73122d --- /dev/null +++ b/test/dummy/Dockerfile @@ -0,0 +1,58 @@ +# syntax = docker/dockerfile:1 + +# Make sure RUBY_VERSION matches the Ruby version in .ruby-version and Gemfile +ARG RUBY_VERSION=3.1.3 +FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base + +# Rails app lives here +WORKDIR /rails + +# Set production environment +ENV RAILS_ENV="production" \ + BUNDLE_DEPLOYMENT="1" \ + BUNDLE_PATH="/usr/local/bundle" \ + BUNDLE_WITHOUT="development" + + +# Throw-away build stage to reduce size of final image +FROM base as build + +# Install packages needed to build gems +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -y build-essential git pkg-config + +# Install application gems +COPY Gemfile Gemfile.lock ./ +RUN bundle install && \ + rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git + +# Copy application code +COPY . . + +# Precompiling assets for production without requiring secret RAILS_MASTER_KEY +RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile + + +# Final stage for app image +FROM base + +# Install packages needed for deployment +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -y curl && \ + rm -rf /var/lib/apt/lists /var/cache/apt/archives + +# Copy built artifacts: gems, application +COPY --from=build /usr/local/bundle /usr/local/bundle +COPY --from=build /rails /rails + +# Run and own only the runtime files as a non-root user for security +RUN useradd rails --create-home --shell /bin/bash && \ + chown -R rails:rails log tmp +USER rails:rails + +# Entrypoint prepares the database. +ENTRYPOINT ["/rails/bin/docker-entrypoint"] + +# Start the server by default, this can be overwritten at runtime +EXPOSE 3000 +CMD ["./bin/rails", "server"] diff --git a/test/dummy/Gemfile b/test/dummy/Gemfile new file mode 100644 index 0000000..ef9517c --- /dev/null +++ b/test/dummy/Gemfile @@ -0,0 +1,45 @@ +source "https://rubygems.org" + +ruby "3.1.3" + +gem 'wafris', path: '../..' + +# Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" +gem "rails", "~> 7.1.3", ">= 7.1.3.2" + +# The original asset pipeline for Rails [https://github.com/rails/sprockets-rails] +gem "sprockets-rails" + +# Use the Puma web server [https://github.com/puma/puma] +gem "puma", ">= 5.0" + +# Build JSON APIs with ease [https://github.com/rails/jbuilder] +#gem "jbuilder" + +# Use Kredis to get higher-level data types in Redis [https://github.com/rails/kredis] +# gem "kredis" + +# Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword] +# gem "bcrypt", "~> 3.1.7" + +# Windows does not include zoneinfo files, so bundle the tzinfo-data gem +gem "tzinfo-data", platforms: %i[ windows jruby ] + +group :development, :test do + # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem + #gem "debug", platforms: %i[ mri windows ] +end + +group :development do + # Use console on exceptions pages [https://github.com/rails/web-console] + #gem "web-console" + + # Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler] + # gem "rack-mini-profiler" + + # Speed up commands on slow machines / big apps [https://github.com/rails/spring] + # gem "spring" + + #gem "error_highlight", ">= 0.4.0", platforms: [:ruby] +end + diff --git a/test/dummy/Rakefile b/test/dummy/Rakefile new file mode 100644 index 0000000..9a5ea73 --- /dev/null +++ b/test/dummy/Rakefile @@ -0,0 +1,6 @@ +# Add your own tasks in files placed in lib/tasks ending in .rake, +# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. + +require_relative "config/application" + +Rails.application.load_tasks diff --git a/test/dummy/app/assets/config/manifest.js b/test/dummy/app/assets/config/manifest.js new file mode 100644 index 0000000..5918193 --- /dev/null +++ b/test/dummy/app/assets/config/manifest.js @@ -0,0 +1,2 @@ +//= link_tree ../images +//= link_directory ../stylesheets .css diff --git a/test/dummy/app/assets/images/.keep b/test/dummy/app/assets/images/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/app/assets/stylesheets/application.css b/test/dummy/app/assets/stylesheets/application.css new file mode 100644 index 0000000..288b9ab --- /dev/null +++ b/test/dummy/app/assets/stylesheets/application.css @@ -0,0 +1,15 @@ +/* + * This is a manifest file that'll be compiled into application.css, which will include all the files + * listed below. + * + * Any CSS (and SCSS, if configured) file within this directory, lib/assets/stylesheets, or any plugin's + * vendor/assets/stylesheets directory can be referenced here using a relative path. + * + * You're free to add application-wide styles to this file and they'll appear at the bottom of the + * compiled file so the styles you add here take precedence over styles defined in any other CSS + * files in this directory. Styles in this file should be added after the last require_* statement. + * It is generally better to create a new file per style scope. + * + *= require_tree . + *= require_self + */ diff --git a/test/dummy/app/controllers/application_controller.rb b/test/dummy/app/controllers/application_controller.rb new file mode 100644 index 0000000..09705d1 --- /dev/null +++ b/test/dummy/app/controllers/application_controller.rb @@ -0,0 +1,2 @@ +class ApplicationController < ActionController::Base +end diff --git a/test/dummy/app/controllers/concerns/.keep b/test/dummy/app/controllers/concerns/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/app/helpers/application_helper.rb b/test/dummy/app/helpers/application_helper.rb new file mode 100644 index 0000000..de6be79 --- /dev/null +++ b/test/dummy/app/helpers/application_helper.rb @@ -0,0 +1,2 @@ +module ApplicationHelper +end diff --git a/test/dummy/app/jobs/application_job.rb b/test/dummy/app/jobs/application_job.rb new file mode 100644 index 0000000..d394c3d --- /dev/null +++ b/test/dummy/app/jobs/application_job.rb @@ -0,0 +1,7 @@ +class ApplicationJob < ActiveJob::Base + # Automatically retry jobs that encountered a deadlock + # retry_on ActiveRecord::Deadlocked + + # Most jobs are safe to ignore if the underlying records are no longer available + # discard_on ActiveJob::DeserializationError +end diff --git a/test/dummy/app/models/concerns/.keep b/test/dummy/app/models/concerns/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/app/views/layouts/application.html.erb b/test/dummy/app/views/layouts/application.html.erb new file mode 100644 index 0000000..f72b4ef --- /dev/null +++ b/test/dummy/app/views/layouts/application.html.erb @@ -0,0 +1,15 @@ + + + + Dummy + + <%= csrf_meta_tags %> + <%= csp_meta_tag %> + + <%= stylesheet_link_tag "application" %> + + + + <%= yield %> + + diff --git a/test/dummy/bin/bundle b/test/dummy/bin/bundle new file mode 100755 index 0000000..42c7fd7 --- /dev/null +++ b/test/dummy/bin/bundle @@ -0,0 +1,109 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'bundle' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +require "rubygems" + +m = Module.new do + module_function + + def invoked_as_script? + File.expand_path($0) == File.expand_path(__FILE__) + end + + def env_var_version + ENV["BUNDLER_VERSION"] + end + + def cli_arg_version + return unless invoked_as_script? # don't want to hijack other binstubs + return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update` + bundler_version = nil + update_index = nil + ARGV.each_with_index do |a, i| + if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN + bundler_version = a + end + next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ + bundler_version = $1 + update_index = i + end + bundler_version + end + + def gemfile + gemfile = ENV["BUNDLE_GEMFILE"] + return gemfile if gemfile && !gemfile.empty? + + File.expand_path("../Gemfile", __dir__) + end + + def lockfile + lockfile = + case File.basename(gemfile) + when "gems.rb" then gemfile.sub(/\.rb$/, ".locked") + else "#{gemfile}.lock" + end + File.expand_path(lockfile) + end + + def lockfile_version + return unless File.file?(lockfile) + lockfile_contents = File.read(lockfile) + return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/ + Regexp.last_match(1) + end + + def bundler_requirement + @bundler_requirement ||= + env_var_version || + cli_arg_version || + bundler_requirement_for(lockfile_version) + end + + def bundler_requirement_for(version) + return "#{Gem::Requirement.default}.a" unless version + + bundler_gem_version = Gem::Version.new(version) + + bundler_gem_version.approximate_recommendation + end + + def load_bundler! + ENV["BUNDLE_GEMFILE"] ||= gemfile + + activate_bundler + end + + def activate_bundler + gem_error = activation_error_handling do + gem "bundler", bundler_requirement + end + return if gem_error.nil? + require_error = activation_error_handling do + require "bundler/version" + end + return if require_error.nil? && Gem::Requirement.new(bundler_requirement).satisfied_by?(Gem::Version.new(Bundler::VERSION)) + warn "Activating bundler (#{bundler_requirement}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_requirement}'`" + exit 42 + end + + def activation_error_handling + yield + nil + rescue StandardError, LoadError => e + e + end +end + +m.load_bundler! + +if m.invoked_as_script? + load Gem.bin_path("bundler", "bundle") +end diff --git a/test/dummy/bin/docker-entrypoint b/test/dummy/bin/docker-entrypoint new file mode 100755 index 0000000..de0b30b --- /dev/null +++ b/test/dummy/bin/docker-entrypoint @@ -0,0 +1,3 @@ +#!/bin/bash -e + +exec "${@}" diff --git a/test/dummy/bin/rails b/test/dummy/bin/rails new file mode 100755 index 0000000..efc0377 --- /dev/null +++ b/test/dummy/bin/rails @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path("../config/application", __dir__) +require_relative "../config/boot" +require "rails/commands" diff --git a/test/dummy/bin/rake b/test/dummy/bin/rake new file mode 100755 index 0000000..4fbf10b --- /dev/null +++ b/test/dummy/bin/rake @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +require_relative "../config/boot" +require "rake" +Rake.application.run diff --git a/test/dummy/bin/setup b/test/dummy/bin/setup new file mode 100755 index 0000000..cf2acd1 --- /dev/null +++ b/test/dummy/bin/setup @@ -0,0 +1,25 @@ +#!/usr/bin/env ruby +require "fileutils" + +# path to your application root. +APP_ROOT = File.expand_path("..", __dir__) + +def system!(*args) + system(*args, exception: true) +end + +FileUtils.chdir APP_ROOT do + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. + # Add necessary setup steps to this file. + + puts "== Installing dependencies ==" + system! "gem install bundler --conservative" + system("bundle check") || system!("bundle install") + + puts "\n== Removing old logs and tempfiles ==" + system! "bin/rails log:clear tmp:clear" + + puts "\n== Restarting application server ==" + system! "bin/rails restart" +end diff --git a/test/dummy/config.ru b/test/dummy/config.ru new file mode 100644 index 0000000..4a3c09a --- /dev/null +++ b/test/dummy/config.ru @@ -0,0 +1,6 @@ +# This file is used by Rack-based servers to start the application. + +require_relative "config/environment" + +run Rails.application +Rails.application.load_server diff --git a/test/dummy/config/application.rb b/test/dummy/config/application.rb new file mode 100644 index 0000000..b708d82 --- /dev/null +++ b/test/dummy/config/application.rb @@ -0,0 +1,42 @@ +require_relative "boot" + +require "rails" +# Pick the frameworks you want: +require "active_model/railtie" +require "active_job/railtie" +# require "active_record/railtie" +# require "active_storage/engine" +require "action_controller/railtie" +# require "action_mailer/railtie" +# require "action_mailbox/engine" +# require "action_text/engine" +require "action_view/railtie" +# require "action_cable/engine" +require "rails/test_unit/railtie" + +# Require the gems listed in Gemfile, including any gems +# you've limited to :test, :development, or :production. +Bundler.require(*Rails.groups) + +module Dummy + class Application < Rails::Application + # Initialize configuration defaults for originally generated Rails version. + config.load_defaults 7.1 + + # Please, add to the `ignore` list any other `lib` subdirectories that do + # not contain `.rb` files, or that should not be reloaded or eager loaded. + # Common ones are `templates`, `generators`, or `middleware`, for example. + config.autoload_lib(ignore: %w(assets tasks)) + + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") + + # Don't generate system test files. + config.generators.system_tests = nil + end +end diff --git a/test/dummy/config/boot.rb b/test/dummy/config/boot.rb new file mode 100644 index 0000000..2820116 --- /dev/null +++ b/test/dummy/config/boot.rb @@ -0,0 +1,3 @@ +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) + +require "bundler/setup" # Set up gems listed in the Gemfile. diff --git a/test/dummy/config/credentials.yml.enc b/test/dummy/config/credentials.yml.enc new file mode 100644 index 0000000..0e219a7 --- /dev/null +++ b/test/dummy/config/credentials.yml.enc @@ -0,0 +1 @@ +5ZSkqnSNdu1Hfn4bjOy8Y6PMJfHJeBLkfrsAMaggMv/p8Et1VtExq0H1qHp5R7gkuVB+f88tCZ+u9qDNHc/+Dy7Nr4ADYUAjaVtA+nnkDGU4PQqR1ertZYSJW5V3MotSkihxsQsEk1HuMq/kcFO7bZoiQhe+xhEGjC2mz+zbqUIHvbJt4OL0sOuPTiPW0qpIo4mQojzk6efBBoyru5kl0AIxnKxlWHADk20AWPuCUtyxY/R7qHGLnuf6jMNxLf0uKYk+mCjk2X9L+pfQE+DHyb7dqAwXFB19ot29sPhK6uUpuFK4viIho60XgdgJ/+IuQlmTBzPGYGiQZbZ4+3CKwiq3ral1MHAvumws1xLLJNoUepuGHV6f6wI7DQNjfZOoCeJjJNgtOSVRiiUyW4kl4qkpqrxD--5BkfPUhnSCbJU56L--LzmwpulY8DEquIzgyEsE/A== \ No newline at end of file diff --git a/test/dummy/config/environment.rb b/test/dummy/config/environment.rb new file mode 100644 index 0000000..cac5315 --- /dev/null +++ b/test/dummy/config/environment.rb @@ -0,0 +1,5 @@ +# Load the Rails application. +require_relative "application" + +# Initialize the Rails application. +Rails.application.initialize! diff --git a/test/dummy/config/environments/development.rb b/test/dummy/config/environments/development.rb new file mode 100644 index 0000000..d3b9978 --- /dev/null +++ b/test/dummy/config/environments/development.rb @@ -0,0 +1,63 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development + # since you don't have to restart the web server when you make code changes. + config.enable_reloading = true + + # To allow host testing + config.hosts << "example.com" + config.hosts << "blocked.com" + + # Do not eager load code on boot. + config.eager_load = false + + # Show full error reports. + config.consider_all_requests_local = true + + # Enable server timing + config.server_timing = true + + # Enable/disable caching. By default caching is disabled. + # Run rails dev:cache to toggle caching. + if Rails.root.join("tmp/caching-dev.txt").exist? + config.action_controller.perform_caching = true + config.action_controller.enable_fragment_cache_logging = true + + config.cache_store = :memory_store + config.public_file_server.headers = { + "Cache-Control" => "public, max-age=#{2.days.to_i}" + } + else + config.action_controller.perform_caching = false + + config.cache_store = :null_store + end + + # Print deprecation notices to the Rails logger. + config.active_support.deprecation = :log + + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Highlight code that enqueued background job in logs. + config.active_job.verbose_enqueue_logs = true + + # Suppress logger output for asset requests. + config.assets.quiet = true + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true + + # Raise error when a before_action's only/except options reference missing actions + config.action_controller.raise_on_missing_callback_actions = true +end diff --git a/test/dummy/config/environments/production.rb b/test/dummy/config/environments/production.rb new file mode 100644 index 0000000..86307ec --- /dev/null +++ b/test/dummy/config/environments/production.rb @@ -0,0 +1,80 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.enable_reloading = false + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Ensures that a master key has been made available in ENV["RAILS_MASTER_KEY"], config/master.key, or an environment + # key such as config/credentials/production.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + + # Disable serving static files from `public/`, relying on NGINX/Apache to do so instead. + # config.public_file_server.enabled = false + + # Compress CSS using a preprocessor. + # config.assets.css_compressor = :sass + + # Do not fall back to assets pipeline if a precompiled asset is missed. + config.assets.compile = false + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.asset_host = "http://assets.example.com" + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache + # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX + + # Assume all access to the app is happening through a SSL-terminating reverse proxy. + # Can be used together with config.force_ssl for Strict-Transport-Security and secure cookies. + # config.assume_ssl = true + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + config.force_ssl = true + + # Log to STDOUT by default + config.logger = ActiveSupport::Logger.new(STDOUT) + .tap { |logger| logger.formatter = ::Logger::Formatter.new } + .then { |logger| ActiveSupport::TaggedLogging.new(logger) } + + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] + + # "info" includes generic and useful information about system operation, but avoids logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). If you + # want to log everything, set the level to "debug". + config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info") + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment). + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "dummy_production" + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Don't log any deprecations. + config.active_support.report_deprecations = false + + # Enable DNS rebinding protection and other `Host` header attacks. + # config.hosts = [ + # "example.com", # Allow requests from example.com + # /.*\.example\.com/ # Allow requests from subdomains like `www.example.com` + # ] + # Skip DNS rebinding protection for the default health check endpoint. + # config.host_authorization = { exclude: ->(request) { request.path == "/up" } } +end diff --git a/test/dummy/config/environments/test.rb b/test/dummy/config/environments/test.rb new file mode 100644 index 0000000..d349c35 --- /dev/null +++ b/test/dummy/config/environments/test.rb @@ -0,0 +1,54 @@ +require "active_support/core_ext/integer/time" + +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # While tests run files are not watched, reloading is not necessary. + config.enable_reloading = false + + # Eager loading loads your entire application. When running a single test locally, + # this is usually not necessary, and can slow down your test suite. However, it's + # recommended that you enable it in continuous integration systems to ensure eager + # loading is working properly before deploying your code. + config.eager_load = ENV["CI"].present? + + # Configure public file server for tests with Cache-Control for performance. + config.public_file_server.enabled = true + config.public_file_server.headers = { + "Cache-Control" => "public, max-age=#{1.hour.to_i}" + } + + # Show full error reports and disable caching. + config.consider_all_requests_local = true + config.action_controller.perform_caching = false + config.cache_store = :null_store + + # Render exception templates for rescuable exceptions and raise for other exceptions. + config.action_dispatch.show_exceptions = :rescuable + + # Disable request forgery protection in test environment. + config.action_controller.allow_forgery_protection = false + + # Print deprecation notices to the stderr. + config.active_support.deprecation = :stderr + + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true + + # Raise error when a before_action's only/except options reference missing actions + config.action_controller.raise_on_missing_callback_actions = true +end diff --git a/test/dummy/config/initializers/assets.rb b/test/dummy/config/initializers/assets.rb new file mode 100644 index 0000000..2eeef96 --- /dev/null +++ b/test/dummy/config/initializers/assets.rb @@ -0,0 +1,12 @@ +# Be sure to restart your server when you modify this file. + +# Version of your assets, change this if you want to expire all your assets. +Rails.application.config.assets.version = "1.0" + +# Add additional assets to the asset load path. +# Rails.application.config.assets.paths << Emoji.images_path + +# Precompile additional assets. +# application.js, application.css, and all non-JS/CSS in the app/assets +# folder are already added. +# Rails.application.config.assets.precompile += %w( admin.js admin.css ) diff --git a/test/dummy/config/initializers/content_security_policy.rb b/test/dummy/config/initializers/content_security_policy.rb new file mode 100644 index 0000000..b3076b3 --- /dev/null +++ b/test/dummy/config/initializers/content_security_policy.rb @@ -0,0 +1,25 @@ +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy. +# See the Securing Rails Applications Guide for more information: +# https://guides.rubyonrails.org/security.html#content-security-policy-header + +# Rails.application.configure do +# config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end +# +# # Generate session nonces for permitted importmap, inline scripts, and inline styles. +# config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s } +# config.content_security_policy_nonce_directives = %w(script-src style-src) +# +# # Report violations without enforcing the policy. +# # config.content_security_policy_report_only = true +# end diff --git a/test/dummy/config/initializers/filter_parameter_logging.rb b/test/dummy/config/initializers/filter_parameter_logging.rb new file mode 100644 index 0000000..c2d89e2 --- /dev/null +++ b/test/dummy/config/initializers/filter_parameter_logging.rb @@ -0,0 +1,8 @@ +# Be sure to restart your server when you modify this file. + +# Configure parameters to be partially matched (e.g. passw matches password) and filtered from the log file. +# Use this to limit dissemination of sensitive information. +# See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors. +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/test/dummy/config/initializers/inflections.rb b/test/dummy/config/initializers/inflections.rb new file mode 100644 index 0000000..3860f65 --- /dev/null +++ b/test/dummy/config/initializers/inflections.rb @@ -0,0 +1,16 @@ +# Be sure to restart your server when you modify this file. + +# Add new inflection rules using the following format. Inflections +# are locale specific, and you may define rules for as many different +# locales as you wish. All of these examples are active by default: +# ActiveSupport::Inflector.inflections(:en) do |inflect| +# inflect.plural /^(ox)$/i, "\\1en" +# inflect.singular /^(ox)en/i, "\\1" +# inflect.irregular "person", "people" +# inflect.uncountable %w( fish sheep ) +# end + +# These inflection rules are supported but not enabled by default: +# ActiveSupport::Inflector.inflections(:en) do |inflect| +# inflect.acronym "RESTful" +# end diff --git a/test/dummy/config/initializers/permissions_policy.rb b/test/dummy/config/initializers/permissions_policy.rb new file mode 100644 index 0000000..7db3b95 --- /dev/null +++ b/test/dummy/config/initializers/permissions_policy.rb @@ -0,0 +1,13 @@ +# Be sure to restart your server when you modify this file. + +# Define an application-wide HTTP permissions policy. For further +# information see: https://developers.google.com/web/updates/2018/06/feature-policy + +# Rails.application.config.permissions_policy do |policy| +# policy.camera :none +# policy.gyroscope :none +# policy.microphone :none +# policy.usb :none +# policy.fullscreen :self +# policy.payment :self, "https://secure.example.com" +# end diff --git a/test/dummy/config/locales/en.yml b/test/dummy/config/locales/en.yml new file mode 100644 index 0000000..6c349ae --- /dev/null +++ b/test/dummy/config/locales/en.yml @@ -0,0 +1,31 @@ +# Files in the config/locales directory are used for internationalization and +# are automatically loaded by Rails. If you want to use locales other than +# English, add the necessary files in this directory. +# +# To use the locales, use `I18n.t`: +# +# I18n.t "hello" +# +# In views, this is aliased to just `t`: +# +# <%= t("hello") %> +# +# To use a different locale, set it with `I18n.locale`: +# +# I18n.locale = :es +# +# This would use the information in config/locales/es.yml. +# +# To learn more about the API, please read the Rails Internationalization guide +# at https://guides.rubyonrails.org/i18n.html. +# +# Be aware that YAML interprets the following case-insensitive strings as +# booleans: `true`, `false`, `on`, `off`, `yes`, `no`. Therefore, these strings +# must be quoted to be interpreted as strings. For example: +# +# en: +# "yes": yup +# enabled: "ON" + +en: + hello: "Hello world" diff --git a/test/dummy/config/puma.rb b/test/dummy/config/puma.rb new file mode 100644 index 0000000..afa809b --- /dev/null +++ b/test/dummy/config/puma.rb @@ -0,0 +1,35 @@ +# This configuration file will be evaluated by Puma. The top-level methods that +# are invoked here are part of Puma's configuration DSL. For more information +# about methods provided by the DSL, see https://puma.io/puma/Puma/DSL.html. + +# Puma can serve each request in a thread from an internal thread pool. +# The `threads` method setting takes two numbers: a minimum and maximum. +# Any libraries that use thread pools should be configured to match +# the maximum value specified for Puma. Default is set to 5 threads for minimum +# and maximum; this matches the default thread size of Active Record. +max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } +min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } +threads min_threads_count, max_threads_count + +# Specifies that the worker count should equal the number of processors in production. +if ENV["RAILS_ENV"] == "production" + require "concurrent-ruby" + worker_count = Integer(ENV.fetch("WEB_CONCURRENCY") { Concurrent.physical_processor_count }) + workers worker_count if worker_count > 1 +end + +# Specifies the `worker_timeout` threshold that Puma will use to wait before +# terminating a worker in development environments. +worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" + +# Specifies the `port` that Puma will listen on to receive requests; default is 3000. +port ENV.fetch("PORT") { 3000 } + +# Specifies the `environment` that Puma will run in. +environment ENV.fetch("RAILS_ENV") { "development" } + +# Specifies the `pidfile` that Puma will use. +pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } + +# Allow puma to be restarted by `bin/rails restart` command. +plugin :tmp_restart diff --git a/test/dummy/config/routes.rb b/test/dummy/config/routes.rb new file mode 100644 index 0000000..0ada1cb --- /dev/null +++ b/test/dummy/config/routes.rb @@ -0,0 +1,12 @@ +Rails.application.routes.draw do + # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html + + # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500. + # Can be used by load balancers and uptime monitors to verify that the app is live. + get "up" => "rails/health#show", as: :rails_health_check + + post '/' => Rails::WelcomeController.action(:index) + + # Defines the root path route ("/") + # root "posts#index" +end diff --git a/test/dummy/lib/assets/.keep b/test/dummy/lib/assets/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/lib/tasks/.keep b/test/dummy/lib/tasks/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/log/.keep b/test/dummy/log/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/public/404.html b/test/dummy/public/404.html new file mode 100644 index 0000000..2be3af2 --- /dev/null +++ b/test/dummy/public/404.html @@ -0,0 +1,67 @@ + + + + The page you were looking for doesn't exist (404) + + + + + + +
+
+

The page you were looking for doesn't exist.

+

You may have mistyped the address or the page may have moved.

+
+

If you are the application owner check the logs for more information.

+
+ + diff --git a/test/dummy/public/422.html b/test/dummy/public/422.html new file mode 100644 index 0000000..c08eac0 --- /dev/null +++ b/test/dummy/public/422.html @@ -0,0 +1,67 @@ + + + + The change you wanted was rejected (422) + + + + + + +
+
+

The change you wanted was rejected.

+

Maybe you tried to change something you didn't have access to.

+
+

If you are the application owner check the logs for more information.

+
+ + diff --git a/test/dummy/public/500.html b/test/dummy/public/500.html new file mode 100644 index 0000000..78a030a --- /dev/null +++ b/test/dummy/public/500.html @@ -0,0 +1,66 @@ + + + + We're sorry, but something went wrong (500) + + + + + + +
+
+

We're sorry, but something went wrong.

+
+

If you are the application owner check the logs for more information.

+
+ + diff --git a/test/dummy/public/apple-touch-icon-precomposed.png b/test/dummy/public/apple-touch-icon-precomposed.png new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/public/apple-touch-icon.png b/test/dummy/public/apple-touch-icon.png new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/public/favicon.ico b/test/dummy/public/favicon.ico new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/public/robots.txt b/test/dummy/public/robots.txt new file mode 100644 index 0000000..c19f78a --- /dev/null +++ b/test/dummy/public/robots.txt @@ -0,0 +1 @@ +# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file diff --git a/test/dummy/storage/.keep b/test/dummy/storage/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/controllers/.keep b/test/dummy/test/controllers/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/fixtures/files/.keep b/test/dummy/test/fixtures/files/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/helpers/.keep b/test/dummy/test/helpers/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/integration/.keep b/test/dummy/test/integration/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/models/.keep b/test/dummy/test/models/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/test/test_helper.rb b/test/dummy/test/test_helper.rb new file mode 100644 index 0000000..c3cf780 --- /dev/null +++ b/test/dummy/test/test_helper.rb @@ -0,0 +1,12 @@ +ENV["RAILS_ENV"] ||= "test" +require_relative "../config/environment" +require "rails/test_help" + +module ActiveSupport + class TestCase + # Run tests in parallel with specified workers + parallelize(workers: :number_of_processors) + + # Add more helper methods to be used by all tests here... + end +end diff --git a/test/dummy/tmp/.keep b/test/dummy/tmp/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/tmp/pids/.keep b/test/dummy/tmp/pids/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/tmp/storage/.keep b/test/dummy/tmp/storage/.keep new file mode 100644 index 0000000..e69de29 diff --git a/test/dummy/vendor/.keep b/test/dummy/vendor/.keep new file mode 100644 index 0000000..e69de29 From a6b46d304f37e9977061b45f1a4712240728e474 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 4 Jun 2024 14:29:49 -0400 Subject: [PATCH 14/32] Updates for request id and timestamp --- lib/wafris.rb | 32 ++++++++++++++++---------------- lib/wafris/middleware.rb | 10 ++++++++-- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index d8e3eac..82aa0bc 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -213,13 +213,13 @@ def send_upsync_requests(requests_array) # # The 'rule' parameter represents the specific rule that was matched within the category # ex: '192.23.5.4', 'SemRush', etc. - def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatment, category, rule) + def queue_upsync_request(ip, user_agent, path, parameters, host, method, treatment, category, rule, request_id, request_timestamp) if @configuration.upsync_status != 'Disabled' || @configuration.upsync_status != 'Uploading' @configuration.upsync_status = 'Uploading' # Add request to the queue - request = [ip, user_agent, path, parameters, host, method, treatment, category, rule] + request = [ip, user_agent, path, parameters, host, method, treatment, category, rule, request_id, request_timestamp] @configuration.upsync_queue << request # If the queue is full, send the requests to the upsync server @@ -412,7 +412,7 @@ def current_db(db_rule_category) # This is the main loop that evaluates the request # as well as sorts out when downsync and upsync should be called - def evaluate(ip, user_agent, path, parameters, host, method, headers, body) + def evaluate(ip, user_agent, path, parameters, host, method, headers, body, request_id, request_timestamp) @configuration ||= Wafris::Configuration.new if @configuration.api_key.nil? @@ -429,73 +429,73 @@ def evaluate(ip, user_agent, path, parameters, host, method, headers, body) # Allowed IPs if exact_match(ip, 'allowed_ips', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ai', ip) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ai', ip, request_id, request_timestamp) end # Allowed CIDR Ranges if ip_in_cidr_range(ip, 'allowed_cidr_ranges', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ac', ip) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Allowed', 'ac', ip, request_id, request_timestamp) end # Blocked IPs if exact_match(ip, 'blocked_ips', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bi', ip) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bi', ip, request_id, request_timestamp) end # Blocked CIDR Ranges if ip_in_cidr_range(ip, 'blocked_cidr_ranges', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bc', ip) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bc', ip, request_id, request_timestamp) end # Blocked Country Codes country_code = get_country_code(ip, data_subscriptions_db) if exact_match(country_code, 'blocked_country_codes', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "G_#{country_code}") + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "G_#{country_code}", request_id, request_timestamp) end # Blocked Reputation IP Ranges if ip_in_cidr_range(ip, 'reputation_ip_ranges', data_subscriptions_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "R") + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bs', "R", request_id, request_timestamp) end # Blocked User Agents user_agent_match = substring_match(user_agent, 'blocked_user_agents', rules_db) if user_agent_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bu', user_agent_match) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bu', user_agent_match, request_id, request_timestamp) end # Blocked Paths path_match = substring_match(path, 'blocked_paths', rules_db) if path_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bp', path_match) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bp', path_match, request_id, request_timestamp) end # Blocked Parameters parameters_match = substring_match(parameters, 'blocked_parameters', rules_db) if parameters_match - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'ba', parameters_match) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'ba', parameters_match, request_id, request_timestamp) end # Blocked Hosts if exact_match(host, 'blocked_hosts', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bh', host) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bh', host, request_id, request_timestamp) end # Blocked Methods if exact_match(method, 'blocked_methods', rules_db) - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bm', method) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'bm', method, request_id, request_timestamp) end # Rate Limiting rule_id = check_rate_limit(ip, path, method, rules_db) if rule_id - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'brl', rule_id) + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Blocked', 'brl', rule_id, request_id, request_timestamp) end end # Passed if no allow or block rules matched - return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Passed', 'passed', '-') + return queue_upsync_request(ip, user_agent, path, parameters, host, method, 'Passed', 'passed', '-', request_id, request_timestamp) end # end api_key.nil? end # end evaluate diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index bebade6..7a76994 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -35,12 +35,18 @@ def call(env) parameters = Rack::Utils.build_query(request.params).force_encoding('UTF-8') host = request.host.to_s.force_encoding('UTF-8') request_method = String.new(request.request_method).force_encoding('UTF-8') - + # Submitted for evaluation headers = env.each_with_object({}) { |(k, v), h| h[k] = v.force_encoding('UTF-8') if k.start_with?('HTTP_') } body = request.body.read.force_encoding('UTF-8') - treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method, headers, body) + request_id = env.fetch('action_dispatch.request_id', SecureRandom.uuid.to_s) + request_timestamp = Time.now.utc.to_i + + ap "Request id: #{request_id}" + ap "Request timestamp: #{request_timestamp}" + + treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method, headers, body, request_id, request_timestamp) # These values match what the client tests expect (200, 404, 403, 500 if treatment == 'Allowed' || treatment == 'Passed' From 68dc285da9469593ab333c9df5d6d66aebd31a29 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Fri, 7 Jun 2024 13:51:10 -0400 Subject: [PATCH 15/32] Update middleware.rb --- lib/wafris/middleware.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 7a76994..edcdb33 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -38,7 +38,7 @@ def call(env) # Submitted for evaluation headers = env.each_with_object({}) { |(k, v), h| h[k] = v.force_encoding('UTF-8') if k.start_with?('HTTP_') } - body = request.body.read.force_encoding('UTF-8') + body = request.body.read request_id = env.fetch('action_dispatch.request_id', SecureRandom.uuid.to_s) request_timestamp = Time.now.utc.to_i From 671c4c19b12bfba049010d63d928cf8225ce3edf Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sun, 16 Jun 2024 17:11:15 -0400 Subject: [PATCH 16/32] Lowering sync timings --- lib/wafris/configuration.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 6411956..8deeb3c 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -63,7 +63,7 @@ def initialize if ENV['WAFRIS_DOWNSYNC_DATA_SUBSCRIPTIONS_INTERVAL'] @downsync_data_subscriptions_interval = ENV['WAFRIS_DOWNSYNC_DATA_SUBSCRIPTIONS_INTERVAL'].to_i else - @downsync_data_subscriptions_interval = 86400 + @downsync_data_subscriptions_interval = 60 end # Set Downsync URL - Optional @@ -86,14 +86,14 @@ def initialize if ENV['WAFRIS_UPSYNC_INTERVAL'] @upsync_interval = ENV['WAFRIS_UPSYNC_INTERVAL'].to_i else - @upsync_interval = 60 + @upsync_interval = 10 end # Set Upsync Queued Request Limit - Optional if ENV['WAFRIS_UPSYNC_QUEUE_LIMIT'] @upsync_queue_limit = ENV['WAFRIS_UPSYNC_QUEUE_LIMIT'].to_i else - @upsync_queue_limit = 1000 + @upsync_queue_limit = 250 end # Set Maximium Body Size for Requests - Optional (in Megabytes) From 93b182fe59d7bf2ac8c8565d63f2dbee7aa3d219 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 18 Jun 2024 09:30:36 -0400 Subject: [PATCH 17/32] Updating tests --- lib/wafris/middleware.rb | 22 ++++++++++++++++++---- test/configuration_test.rb | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index edcdb33..65d663b 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -1,4 +1,3 @@ - # frozen_string_literal: true @@ -8,7 +7,9 @@ def initialize(app) @app = app end + def call(env) + user_defined_proxies = ENV['TRUSTED_PROXY_RANGES'].split(',') if ENV['TRUSTED_PROXY_RANGES'] valid_ipv4_octet = /\.(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])/ @@ -27,10 +28,23 @@ def call(env) Rack::Request.ip_filter = lambda { |ip| trusted_proxies.match?(ip) } request = Rack::Request.new(env) - # Forcing UTF-8 encoding on all strings for Sqlite3 compatibility - ip = request.ip.force_encoding('UTF-8') - user_agent = request.user_agent.force_encoding('UTF-8') + + # List of possible IP headers in order of priority + ip_headers = [ + 'HTTP_X_REAL_IP', + 'HTTP_X_TRUE_CLIENT_IP', + 'HTTP_FLY_CLIENT_IP', + 'HTTP_CF_CONNECTING_IP' + ] + + # Find the first header that is present in the environment + ip_header = ip_headers.find { |header| env[header] } + + # Use the found header or fallback to remote_ip if none of the headers are present + ip = (ip_header ? env[ip_header] : request.ip).force_encoding('UTF-8') + + user_agent = request.user_agent ? request.user_agent.force_encoding('UTF-8') : nil path = request.path.force_encoding('UTF-8') parameters = Rack::Utils.build_query(request.params).force_encoding('UTF-8') host = request.host.to_s.force_encoding('UTF-8') diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 87bc1d6..b1af0b1 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -50,7 +50,7 @@ module Wafris _(@config.db_file_path).must_equal './tmp/wafris' _(@config.db_file_name).must_equal 'wafris.db' _(@config.downsync_custom_rules_interval).must_equal 60 - _(@config.downsync_data_subscriptions_interval).must_equal 86400 + _(@config.downsync_data_subscriptions_interval).must_equal 60 _(@config.downsync_url).must_equal 'https://distributor.wafris.org/v2/downsync' _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync' _(@config.upsync_interval).must_equal 60 From dec27921bddff39d7d29e22e90bf5c3f20a16ce6 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 18 Jun 2024 15:53:51 -0400 Subject: [PATCH 18/32] Add sanity check for db tables --- lib/wafris.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 82aa0bc..8064570 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -308,13 +308,23 @@ def downsync_db(db_rule_category, current_filename = nil) # Write the filename into the db_category.modfile File.open("#{@configuration.db_file_path}/#{db_rule_category}.modfile", 'w') { |file| file.write(filename) } - # Remove the old database file - if old_file_name - #puts "Removing old file: #{@configuration.db_file_path}/#{old_file_name}" - if File.exist?(@configuration.db_file_path + "/" + old_file_name) - File.delete(@configuration.db_file_path + "/" + old_file_name) - end + # Sanity check that the downloaded db file has tables + # not empty or corrupted + db = SQLite3::Database.new @configuration.db_file_path + "/" + filename + if db.execute("SELECT name FROM sqlite_master WHERE type='table';").any? + # Remove the old database file + if old_file_name + if File.exist?(@configuration.db_file_path + "/" + old_file_name) + File.delete(@configuration.db_file_path + "/" + old_file_name) + end + end + + # DB file is bad or empty + else + filename = old_file_name + LogSuppressor.puts_log("DB Error - No tables exist in the db file #{@configuration.db_file_path}/#{filename}") end + end From d17d862bb03765a131000047d5dacc4f27423a01 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 18 Jun 2024 15:55:42 -0400 Subject: [PATCH 19/32] Add db sanity check --- lib/wafris.rb | 2 +- lib/wafris/middleware.rb | 3 --- test/configuration_test.rb | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 8064570..703b572 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -319,7 +319,7 @@ def downsync_db(db_rule_category, current_filename = nil) end end - # DB file is bad or empty + # DB file is bad or empty so keep using whatever we have now else filename = old_file_name LogSuppressor.puts_log("DB Error - No tables exist in the db file #{@configuration.db_file_path}/#{filename}") diff --git a/lib/wafris/middleware.rb b/lib/wafris/middleware.rb index 65d663b..91621ee 100644 --- a/lib/wafris/middleware.rb +++ b/lib/wafris/middleware.rb @@ -57,9 +57,6 @@ def call(env) request_id = env.fetch('action_dispatch.request_id', SecureRandom.uuid.to_s) request_timestamp = Time.now.utc.to_i - ap "Request id: #{request_id}" - ap "Request timestamp: #{request_timestamp}" - treatment = Wafris.evaluate(ip, user_agent, path, parameters, host, request_method, headers, body, request_id, request_timestamp) # These values match what the client tests expect (200, 404, 403, 500 diff --git a/test/configuration_test.rb b/test/configuration_test.rb index b1af0b1..3c6051f 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -53,7 +53,7 @@ module Wafris _(@config.downsync_data_subscriptions_interval).must_equal 60 _(@config.downsync_url).must_equal 'https://distributor.wafris.org/v2/downsync' _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync' - _(@config.upsync_interval).must_equal 60 + _(@config.upsync_interval).must_equal 10 _(@config.upsync_queue_limit).must_equal 1000 _(@config.upsync_queue).must_equal [] From aef10e44a87df6b06b36a930565342c6ace8b1d2 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 18 Jun 2024 16:41:13 -0400 Subject: [PATCH 20/32] Update for debug function --- lib/wafris.rb | 61 ++++++++++++++++++++++++++++++++++++- lib/wafris/configuration.rb | 5 +-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 703b572..0f1f277 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -32,7 +32,6 @@ def configure end - def zero_pad(number, length) number.to_s.rjust(length, "0") end @@ -510,6 +509,66 @@ def evaluate(ip, user_agent, path, parameters, host, method, headers, body, requ end # end api_key.nil? end # end evaluate + def debug(api_key) + + if ENV['WAFRIS_API_KEY'] + puts "Wafris API Key environment variable is set." + puts " - API Key: #{ENV['WAFRIS_API_KEY']}" + else + puts "Wafris API Key environment variable is not set." + end + + puts "\n" + puts "Wafris Configuration:" + + Wafris.configure do |config| + config.api_key = api_key + end + settings = Wafris.configuration + + settings.instance_variables.each do |ivar| + puts " - #{ivar}: #{Wafris.configuration.instance_variable_get(ivar)}" + end + + puts "\n" + if File.exist?(settings.db_file_path + "/" + "custom_rules.lockfile") + puts "Custom Rules Lockfile: #{settings.db_file_path}/custom_rules.lockfile exists" + puts " - Last Modified Time: #{File.mtime(settings.db_file_path + "/" + "custom_rules.lockfile")}" + else + puts "Custom Rules Lockfile: #{settings.db_file_path}/custom_rules.lockfile does not exist." + end + + puts "\n" + if File.exist?(settings.db_file_path + "/" + "custom_rules.modfile") + puts "Custom Rules Modfile: #{settings.db_file_path}/custom_rules.modfile exists" + puts " - Last Modified Time: #{File.mtime(settings.db_file_path + "/" + "custom_rules.modfile")}" + puts " - Contents: #{File.open(settings.db_file_path + "/" + "custom_rules.modfile", 'r').read}" + else + puts "Custom Rules Modfile: #{settings.db_file_path}/custom_rules.modfile does not exist." + end + + puts "\n" + if File.exist?(settings.db_file_path + "/" + "data_subscriptions.lockfile") + puts "Data Subscriptions Lockfile: #{settings.db_file_path}/data_subscriptions.lockfile exists" + puts " - Last Modified Time: #{File.mtime(settings.db_file_path + "/" + "data_subscriptions.lockfile")}" + else + puts "Data Subscriptions Lockfile: #{settings.db_file_path}/data_subscriptions.lockfile does not exist." + end + + puts "\n" + if File.exist?(settings.db_file_path + "/" + "data_subscriptions.modfile") + puts "Data Subscriptions Modfile: #{settings.db_file_path}/data_subscriptions.modfile exists" + puts " - Last Modified Time: #{File.mtime(settings.db_file_path + "/" + "data_subscriptions.modfile")}" + puts " - Contents: #{File.open(settings.db_file_path + "/" + "data_subscriptions.modfile", 'r').read}" + else + puts "Data Subscriptions Modfile: #{settings.db_file_path}/data_subscriptions.modfile does not exist." + end + + + + return true + end + end end diff --git a/lib/wafris/configuration.rb b/lib/wafris/configuration.rb index 8deeb3c..bad313d 100644 --- a/lib/wafris/configuration.rb +++ b/lib/wafris/configuration.rb @@ -26,8 +26,9 @@ def initialize if ENV['WAFRIS_API_KEY'] @api_key = ENV['WAFRIS_API_KEY'] else - @api_key = nil - LogSuppressor.puts_log("Firewall disabled as neither local only or API key set") + unless @api_key + LogSuppressor.puts_log("Firewall disabled as neither local only or API key set") + end end # DB FILE PATH LOCATION - Optional From 841799428b30cee346851235edc80a2838104d72 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 22 Jun 2024 08:13:38 -0400 Subject: [PATCH 21/32] Adding process id capture --- lib/wafris.rb | 9 ++++++--- test/configuration_test.rb | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 0f1f277..39befb4 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -268,9 +268,9 @@ def downsync_db(db_rule_category, current_filename = nil) # Actual Downsync operations filename = "" - # Check server for new rules - #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}" - uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}" + # Check server for new rules including process id + #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" + uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" response = HTTParty.get( uri, @@ -278,6 +278,9 @@ def downsync_db(db_rule_category, current_filename = nil) max_redirects: 2 # Maximum number of redirects to follow ) + # TODO: What to do if timeout + # TODO: What to do if error + if response.code == 401 @configuration.upsync_status = 'Disabled' LogSuppressor.puts_log("Unauthorized: Bad or missing API key") diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 3c6051f..4225064 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -54,7 +54,7 @@ module Wafris _(@config.downsync_url).must_equal 'https://distributor.wafris.org/v2/downsync' _(@config.upsync_url).must_equal 'https://collector.wafris.org/v2/upsync' _(@config.upsync_interval).must_equal 10 - _(@config.upsync_queue_limit).must_equal 1000 + _(@config.upsync_queue_limit).must_equal 250 _(@config.upsync_queue).must_equal [] end From fed1440dbe27f85664b68ccf8378e94986f884b9 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 22 Jun 2024 08:50:11 -0400 Subject: [PATCH 22/32] Update wafris.rb --- lib/wafris.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 39befb4..ee3e4ca 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -270,7 +270,7 @@ def downsync_db(db_rule_category, current_filename = nil) # Check server for new rules including process id #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" - uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" + uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Socket.gethostname}-#{Process.pid}" response = HTTParty.get( uri, From e399d072147754be960a2901250a2a977a6d9f07 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 22 Jun 2024 17:51:58 -0400 Subject: [PATCH 23/32] Tagging all downsync db logs --- lib/wafris.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index ee3e4ca..90d248b 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -258,10 +258,10 @@ def downsync_db(db_rule_category, current_filename = nil) begin lockfile = File.open(lockfile_path, File::RDWR|File::CREAT|File::EXCL) rescue Errno::EEXIST - LogSuppressor.puts_log("Lockfile already exists, skipping downsync.") + LogSuppressor.puts_log("[Wafris][Downsync] Lockfile already exists, skipping downsync.") return rescue Exception => e - LogSuppressor.puts_log("EXCEPTION: Error creating lockfile: #{e.message}") + LogSuppressor.puts_log("[Wafris][Downsync] Error creating lockfile: #{e.message}") end begin @@ -283,13 +283,13 @@ def downsync_db(db_rule_category, current_filename = nil) if response.code == 401 @configuration.upsync_status = 'Disabled' - LogSuppressor.puts_log("Unauthorized: Bad or missing API key") - LogSuppressor.puts_log("API Key: #{@configuration.api_key}") + LogSuppressor.puts_log("[Wafris][Downsync] Unauthorized: Bad or missing API key") + LogSuppressor.puts_log("[Wafris][Downsync] API Key: #{@configuration.api_key}") filename = current_filename elsif response.code == 304 @configuration.upsync_status = 'Enabled' - LogSuppressor.puts_log("No new rules to download") + LogSuppressor.puts_log("[Wafris][Downsync] No new rules to download") filename = current_filename @@ -324,14 +324,14 @@ def downsync_db(db_rule_category, current_filename = nil) # DB file is bad or empty so keep using whatever we have now else filename = old_file_name - LogSuppressor.puts_log("DB Error - No tables exist in the db file #{@configuration.db_file_path}/#{filename}") + LogSuppressor.puts_log("[Wafris][Downsync] DB Error - No tables exist in the db file #{@configuration.db_file_path}/#{filename}") end end rescue Exception => e - puts "EXCEPTION: Error downloading rules: #{e.message}" + LogSuppressor.puts_log("[Wafris][Downsync] Error downloading rules: #{e.message}") # This gets set even if the API key is bad or other issues # to prevent hammering the distribution server on every request From c2c05bb89d0b36a930d0a3f532478fef9cf0df0f Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 22 Jun 2024 17:57:42 -0400 Subject: [PATCH 24/32] Update wafris.rb --- lib/wafris.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 90d248b..7238d18 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -390,8 +390,8 @@ def current_db(db_rule_category) # If the modfile is empty (no db file name), return nil # this can happen if the the api key is bad - if returned_db == "" - return nil + if returned_db == '' + return '' else return returned_db end From 5b4577b23a5dd1e13b78bf8da382356680e1b8ca Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Sat, 22 Jun 2024 18:55:16 -0400 Subject: [PATCH 25/32] Update wafris.rb --- lib/wafris.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/wafris.rb b/lib/wafris.rb index 7238d18..1b1d73d 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -369,13 +369,20 @@ def current_db(db_rule_category) # Checks for existing current modfile, which contains the current db filename if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.modfile") + LogSuppressor.puts_log("[Wafris][Downsync] Modfile exists, skipping downsync") + # Get last Modified Time and current database file name last_db_synctime = File.mtime("#{@configuration.db_file_path}/#{db_rule_category}.modfile").to_i returned_db = File.read("#{@configuration.db_file_path}/#{db_rule_category}.modfile").strip + LogSuppressor.puts_log("[Wafris][Downsync] Modfile Last Modified Time: #{last_db_synctime}") + LogSuppressor.puts_log("[Wafris][Downsync] DB in Modfile: #{returned_db}") + # Check if the db file is older than the interval if (Time.now.to_i - last_db_synctime) > interval + LogSuppressor.puts_log("[Wafris][Downsync] DB is older than the interval") + # Make sure that another process isn't already downloading the rules if !File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") returned_db = downsync_db(db_rule_category, returned_db) @@ -386,6 +393,8 @@ def current_db(db_rule_category) # Current db is up to date else + LogSuppressor.puts_log("[Wafris][Downsync] DB is up to date") + returned_db = File.read("#{@configuration.db_file_path}/#{db_rule_category}.modfile").strip # If the modfile is empty (no db file name), return nil @@ -401,12 +410,16 @@ def current_db(db_rule_category) # No modfile exists, so download the latest db else + LogSuppressor.puts_log("[Wafris][Downsync] No modfile exists, downloading latest db") + # Make sure that another process isn't already downloading the rules if File.exist?("#{@configuration.db_file_path}/#{db_rule_category}.lockfile") + LogSuppressor.puts_log("[Wafris][Downsync] Lockfile exists, skipping downsync") # Lockfile exists, but no modfile with a db filename return nil else + LogSuppressor.puts_log("[Wafris][Downsync] No modfile exists, downloading latest db") # No modfile exists, so download the latest db returned_db = downsync_db(db_rule_category, nil) From 9f8f18beb1e160d3820891ea112de44078425403 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Mon, 24 Jun 2024 15:06:34 -0400 Subject: [PATCH 26/32] Adding hostname --- lib/wafris.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 1b1d73d..109cd98 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -270,7 +270,7 @@ def downsync_db(db_rule_category, current_filename = nil) # Check server for new rules including process id #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" - uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Socket.gethostname}-#{Process.pid}" + uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}&hostname=#{Socket.gethostname}" response = HTTParty.get( uri, From 3c418c22ecdec43dbed9ca1ee0be9bcfde2d2643 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Tue, 25 Jun 2024 16:50:56 -0400 Subject: [PATCH 27/32] Dummy test config changes to work closer to production --- test/dummy/Procfile.dev | 1 + test/dummy/config/environments/production.rb | 2 +- test/dummy/config/puma.rb | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 test/dummy/Procfile.dev diff --git a/test/dummy/Procfile.dev b/test/dummy/Procfile.dev new file mode 100644 index 0000000..c2c566e --- /dev/null +++ b/test/dummy/Procfile.dev @@ -0,0 +1 @@ +web: bundle exec puma -C config/puma.rb diff --git a/test/dummy/config/environments/production.rb b/test/dummy/config/environments/production.rb index 86307ec..4f15c74 100644 --- a/test/dummy/config/environments/production.rb +++ b/test/dummy/config/environments/production.rb @@ -41,7 +41,7 @@ # config.assume_ssl = true # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + config.force_ssl = false # Log to STDOUT by default config.logger = ActiveSupport::Logger.new(STDOUT) diff --git a/test/dummy/config/puma.rb b/test/dummy/config/puma.rb index afa809b..4bbac1e 100644 --- a/test/dummy/config/puma.rb +++ b/test/dummy/config/puma.rb @@ -12,11 +12,10 @@ threads min_threads_count, max_threads_count # Specifies that the worker count should equal the number of processors in production. -if ENV["RAILS_ENV"] == "production" + require "concurrent-ruby" worker_count = Integer(ENV.fetch("WEB_CONCURRENCY") { Concurrent.physical_processor_count }) workers worker_count if worker_count > 1 -end # Specifies the `worker_timeout` threshold that Puma will use to wait before # terminating a worker in development environments. From 987b1969acc9c715c14b3f9af68e6e2f31721a87 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Wed, 26 Jun 2024 14:29:28 -0400 Subject: [PATCH 28/32] Update wafris.rb --- lib/wafris.rb | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 109cd98..3de9281 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -169,9 +169,9 @@ def send_upsync_requests(requests_array) headers = {'Content-Type' => 'application/json'} if Rails && Rails.application - framework = 'rails' + framework = "Rails v#{Rails::VERSION::STRING}" else - framework = 'rack' + framework = "Rack v#{Rack::VERSION::STRING}" end body = { @@ -268,9 +268,24 @@ def downsync_db(db_rule_category, current_filename = nil) # Actual Downsync operations filename = "" + if Rails && Rails.application + framework = "Rails v#{Rails::VERSION::STRING}" + else + framework = "Rack v#{Rack::VERSION::STRING}" + end + + data = { + client_db: current_filename, + process_id: Process.pid, + hostname: Socket.gethostname, + version: Wafris::VERSION, + client: 'wafris-rb', + framework: framework + } + # Check server for new rules including process id #puts "Downloading from #{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}" - uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?current_version=#{current_filename}&process_id=#{Process.pid}&hostname=#{Socket.gethostname}" + uri = "#{@configuration.downsync_url}/#{db_rule_category}/#{@configuration.api_key}?#{data.to_query}" response = HTTParty.get( uri, From 96996326e21726440b4c0735536b151339de947f Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Wed, 3 Jul 2024 17:39:12 -0400 Subject: [PATCH 29/32] Updating content disposition --- lib/wafris.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/wafris.rb b/lib/wafris.rb index 3de9281..4a46a90 100644 --- a/lib/wafris.rb +++ b/lib/wafris.rb @@ -315,9 +315,9 @@ def downsync_db(db_rule_category, current_filename = nil) old_file_name = current_filename end - # Extract the filename from the response - content_disposition = response.headers['content-disposition'] - filename = content_disposition.match(/filename="(.+)"/)[1] + # Extract the filename from the response + content_disposition = response.headers['content-disposition'] + filename = content_disposition.split('filename=')[1].strip # Save the body of the response to a new SQLite file File.open(@configuration.db_file_path + "/" + filename, 'wb') { |file| file.write(response.body) } From e91a5c8bc4387390db1db8bc6d3fb9c7fca87354 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Wed, 17 Jul 2024 07:18:10 -0400 Subject: [PATCH 30/32] v2 Readme Update --- README.md | 40 ++++------------------------------------ 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 5409896..8079270 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Wafris for Ruby/Rails Wafris is an open-source Web Application Firewall (WAF) that runs within Rails (and other frameworks) powered by Redis. -Paired with [Wafris Hub](https://wafris.org/hub), you can create rules to block malicious traffic from hitting your application. +Paired with [Wafris Hub](https://hub.wafris.org), you can view your site traffic in real time and and create rules to block malicious traffic from hitting your application. ![Rules and Graph](docs/rules-and-graph.png) @@ -16,10 +16,9 @@ Need a better explanation? Read the overview at: [wafris.org](https://wafris.org ## Installation and Configuration -The Wafris Ruby client is a gem that installs a Rack middleware into your Rails/Sinatra/Rack application that communicates with a Redis instance. +The Wafris Ruby client is a gem that installs a Rack middleware into your Rails/Sinatra/Rack application filtering requests based on your created rules. ### Requirements -- Redis-rb 4.8+ - Rails 5+ - Ruby 2.5+ @@ -42,40 +41,9 @@ Update your Gemfile to include the Wafris gem and run gem 'wafris' ``` -### 3. Set your Redis Connection - -Specify your redis with the following initializer. We recommend storing the Redis URL as an environment variable or in a secret management system of your choosing rather than hard coding the string in the initializer. - -```ruby -# Create this file and add the following: -# config/initializers/wafris.rb - -if ENV["WAFRIS_REDIS_URL"] - Wafris.configure do |c| - c.redis = Redis.new( - url: ENV["WAFRIS_REDIS_URL"], - timeout: 0.25, - ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE }, - ) - c.redis_pool_size = 25 - c.quiet_mode = false - end -end -``` - -For more details and troubleshooting on the initializer, please read our [Wafris Initializer Guide](docs/wafris-initalizer.md). - -Not sure what Redis provider to use? Please read our [Wafris Redis Providers Guide](https://wafris.org/guides/redis-provisioning) - +### 3. Set your API Key -### 4. Deploy your application - -When deploying your application, you should see the following in your logs: - -``` -[Wafris] attempting firewall connection via Wafris.configure initializer. -[Wafris] firewall enabled. Connected to Redis on . Ready to process requests. Set rules at: https://wafris.org/hub -``` +In your production environment, you'll need to set the `WAFRIS_API_KEY` environment variable to your API key. When you sign up on [Wafris Hub](https://hub.wafris.org), you'll receive your API key along with per-platform instructions. ## Trusted Proxies From 27c3fc53a6724965c8ee8446166e792c4dbaf349 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Wed, 17 Jul 2024 07:21:54 -0400 Subject: [PATCH 31/32] Update version.rb --- lib/wafris/version.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wafris/version.rb b/lib/wafris/version.rb index 2b58878..897a9b6 100644 --- a/lib/wafris/version.rb +++ b/lib/wafris/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Wafris - VERSION = "2.0.0" -end + VERSION = "1.1.11" +end \ No newline at end of file From c08dcbcb52abf16477ec81a5a1d9d9348f8e0d13 Mon Sep 17 00:00:00 2001 From: Michael Buckbee Date: Wed, 17 Jul 2024 11:12:35 -0400 Subject: [PATCH 32/32] feat!: SQLite Release and migration instructions --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 8079270..d018502 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,19 @@ gem 'wafris' In your production environment, you'll need to set the `WAFRIS_API_KEY` environment variable to your API key. When you sign up on [Wafris Hub](https://hub.wafris.org), you'll receive your API key along with per-platform instructions. +## v1 Migration + +Version 1 of the Wafris Rails client gem is deprecated. While it will continue to work you will experience signifiant performance improvements moving to v2. + +The v2 Client does not depend on a Redis instance and instead uses locally sync'd SQLite databases. If you are currently using your own Redis instance, it will continue to work, but we would recommend creating a new WAF instance on Hub and migrating your existing rules. + +Update by running `bundle update wafris` and then updating your configuration. + +We recommend removing your existing `config/initializers/wafris.rb` file and instead setting the `WAFRIS_API_KEY` environment variable in your production environment. + +Your Wafris API key and platform specific instructions are available in the Setup section of your [Wafris Hub](https://hub.wafris.org) dashboard. + + ## Trusted Proxies If you have Cloudflare, Expedited WAF, or another service in front of your application that modifies the `x-forwarded-for` HTTP Request header, please review how to configure [Trusted Proxy Ranges](docs/trusted-proxies.md)