From 6ded2c673b1b018f977cc8a2d4c957eb414b51d1 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Wed, 19 Jun 2024 22:47:22 +0100 Subject: [PATCH 01/20] Endpoint fingerprint processor --- cmake/objects.cmake | 1 + src/builder/processor_builder.cpp | 11 ++++ src/parser/processor_parser.cpp | 40 +++++++++----- src/processor/http_fingerprint.cpp | 86 ++++++++++++++++++++++++++++++ src/processor/http_fingerprint.hpp | 34 ++++++++++++ 5 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 src/processor/http_fingerprint.cpp create mode 100644 src/processor/http_fingerprint.hpp diff --git a/cmake/objects.cmake b/cmake/objects.cmake index a10ee021c..359844d56 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -45,6 +45,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp + ${libddwaf_SOURCE_DIR}/src/processor/http_fingerprint.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp diff --git a/src/builder/processor_builder.cpp b/src/builder/processor_builder.cpp index b81de6337..fb18fd964 100644 --- a/src/builder/processor_builder.cpp +++ b/src/builder/processor_builder.cpp @@ -8,6 +8,7 @@ #include "builder/processor_builder.hpp" #include "processor/extract_schema.hpp" +#include "processor/http_fingerprint.hpp" namespace ddwaf { @@ -42,6 +43,14 @@ template <> struct typed_processor_builder { } }; +template <> struct typed_processor_builder { + std::shared_ptr build(const auto &spec) + { + return std::make_shared( + spec.id, spec.expr, spec.mappings, spec.evaluate, spec.output); + } +}; + template concept has_build_with_scanners = requires(typed_processor_builder b, Spec spec, Scanners scanners) { @@ -70,6 +79,8 @@ template switch (type) { case processor_type::extract_schema: return build_with_type(*this, scanners); + case processor_type::http_fingerprint: + return build_with_type(*this, scanners); default: break; } diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp index abac7e45d..433819288 100644 --- a/src/parser/processor_parser.cpp +++ b/src/parser/processor_parser.cpp @@ -8,11 +8,14 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "processor/base.hpp" +#include "processor/extract_schema.hpp" +#include "processor/http_fingerprint.hpp" #include namespace ddwaf::parser::v2 { namespace { +template std::vector parse_processor_mappings( const parameter::vector &root, address_container &addresses) { @@ -24,22 +27,26 @@ std::vector parse_processor_mappings( for (const auto &node : root) { auto mapping = static_cast(node); - // TODO support n:1 mappings and key paths - auto inputs = at(mapping, "inputs"); - if (inputs.empty()) { - throw ddwaf::parsing_error("empty processor input mapping"); - } + std::vector parameters; + for (const auto ¶m : T::param_names) { + // TODO support n:1 mappings and key paths + auto inputs = at(mapping, param); + if (inputs.empty()) { + throw ddwaf::parsing_error("empty processor input mapping"); + } - auto input = static_cast(inputs[0]); - auto input_address = at(input, "address"); - auto output = at(mapping, "output"); + auto input = static_cast(inputs[0]); + auto input_address = at(input, "address"); + + addresses.optional.emplace(input_address); - addresses.optional.emplace(input_address); + parameters.emplace_back(processor_parameter{ + {processor_target{get_target_index(input_address), std::move(input_address), {}}}}); + } + auto output = at(mapping, "output"); mappings.emplace_back(processor_mapping{ - {processor_parameter{ - {processor_target{get_target_index(input_address), std::move(input_address), {}}}}}, - {get_target_index(output), std::move(output), {}}}); + std::move(parameters), {get_target_index(output), std::move(output), {}}}); } return mappings; @@ -71,6 +78,8 @@ processor_container parse_processors( auto generator_id = at(node, "generator"); if (generator_id == "extract_schema") { type = processor_type::extract_schema; + } else if (generator_id == "http_fingerprint") { + type = processor_type::http_fingerprint; } else { DDWAF_WARN("Unknown generator: {}", generator_id); info.add_failed(id, "unknown generator '" + generator_id + "'"); @@ -82,7 +91,12 @@ processor_container parse_processors( auto params = at(node, "parameters"); auto mappings_vec = at(params, "mappings"); - auto mappings = parse_processor_mappings(mappings_vec, addresses); + std::vector mappings; + if (type == processor_type::extract_schema) { + mappings = parse_processor_mappings(mappings_vec, addresses); + } else { + mappings = parse_processor_mappings(mappings_vec, addresses); + } std::vector scanners; auto scanners_ref_array = at(params, "scanners", {}); diff --git a/src/processor/http_fingerprint.cpp b/src/processor/http_fingerprint.cpp new file mode 100644 index 000000000..bfe1bce7d --- /dev/null +++ b/src/processor/http_fingerprint.cpp @@ -0,0 +1,86 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "processor/http_fingerprint.hpp" + +#include "sha256.hpp" + +namespace ddwaf { + +namespace { + +struct string_buffer { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc, + // cppcoreguidelines-pro-type-reinterpret-cast) + explicit string_buffer(std::size_t length) + : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) + { + if (buffer == nullptr) { + throw std::bad_alloc{}; + } + } + + char &operator[](std::size_t idx) const { return buffer[idx]; } + + void append(std::string_view str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + void append_lowercase(std::string_view str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + void append(char c) { buffer[index++] = c; } + + char *move() + { + auto *ptr = buffer; + buffer = nullptr; + return ptr; + } + + char *buffer{nullptr}; + std::size_t index{0}; + std::size_t length; +}; + +// TODO write this directly into string buffer +std::string get_truncated_hash(std::string_view str) +{ + sha256_hash hasher; + hasher << str; + return hasher.digest(); +} + +} // namespace + +std::pair http_fingerprint::eval_impl( + const unary_argument &method, const unary_argument &uri_raw, + const unary_argument & /*body*/, + const unary_argument & /*query*/, ddwaf::timer & /*deadline*/) const +{ + // http---- + string_buffer buffer{4 + 1 + method.value.size() + 1 + 8 + 1 + 8 + 1 + 8}; + buffer.append("http-"); + buffer.append_lowercase(method.value); + buffer.append('-'); + + auto uri_hash = get_truncated_hash(uri_raw.value); + buffer.append(uri_hash.substr(0, 8)); + buffer.append('-'); + buffer.append("e3b0c442"); + buffer.append('-'); + buffer.append("e3b0c442"); + + ddwaf_object res; + ddwaf_object_stringl_nc(&res, buffer.move(), buffer.length); + return {res, object_store::attribute::none}; +} + +} // namespace ddwaf diff --git a/src/processor/http_fingerprint.hpp b/src/processor/http_fingerprint.hpp new file mode 100644 index 000000000..aa39a65ec --- /dev/null +++ b/src/processor/http_fingerprint.hpp @@ -0,0 +1,34 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include + +#include "processor/base.hpp" +#include "scanner.hpp" + +namespace ddwaf { + +class http_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{ + "method", "uri_raw", "query", "body"}; + + http_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &method, + const unary_argument &uri_raw, + const unary_argument &query, + const unary_argument &body, ddwaf::timer &deadline) const; +}; + +} // namespace ddwaf From d76eb99fde422cb90af8a8c22af2d8ad0d824571 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:43:25 +0100 Subject: [PATCH 02/20] wip --- src/processor/http_fingerprint.cpp | 32 +++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/processor/http_fingerprint.cpp b/src/processor/http_fingerprint.cpp index bfe1bce7d..4210bbb28 100644 --- a/src/processor/http_fingerprint.cpp +++ b/src/processor/http_fingerprint.cpp @@ -55,28 +55,46 @@ std::string get_truncated_hash(std::string_view str) { sha256_hash hasher; hasher << str; - return hasher.digest(); + return hasher.digest().substr(0, 8); } +std::string get_truncated_hash(const ddwaf_object &body) +{ + if (body.type != DDWAF_OBJ_MAP or body.nbEntries == 0) { + return ""; + } + + sha256_hash hasher; + for (unsigned i = 0; i < body.nbEntries; ++i) { + const auto &child = body.array[i]; + hasher << std::string_view{child.parameterName, static_cast(child.parameterNameLength)}; + }; + return hasher.digest().substr(0, 8); +} + + } // namespace std::pair http_fingerprint::eval_impl( const unary_argument &method, const unary_argument &uri_raw, - const unary_argument & /*body*/, - const unary_argument & /*query*/, ddwaf::timer & /*deadline*/) const + const unary_argument &body, + const unary_argument &query, ddwaf::timer &deadline) const { + if (deadline.expired()) { + throw ddwaf::timeout_exception(); + } + // http---- string_buffer buffer{4 + 1 + method.value.size() + 1 + 8 + 1 + 8 + 1 + 8}; buffer.append("http-"); buffer.append_lowercase(method.value); buffer.append('-'); - auto uri_hash = get_truncated_hash(uri_raw.value); - buffer.append(uri_hash.substr(0, 8)); + buffer.append(get_truncated_hash(uri_raw.value)); buffer.append('-'); - buffer.append("e3b0c442"); + buffer.append(get_truncated_hash(*body.value)); buffer.append('-'); - buffer.append("e3b0c442"); + buffer.append(get_truncated_hash(*query.value)); ddwaf_object res; ddwaf_object_stringl_nc(&res, buffer.move(), buffer.length); From 55180c14386220b40d1f6cb8227ab557f2563ab8 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Wed, 3 Jul 2024 22:14:51 +0100 Subject: [PATCH 03/20] Improvements? --- src/processor/http_fingerprint.cpp | 58 +++++++++++++++++++++++++----- src/sha256.cpp | 38 ++++++++++++-------- src/sha256.hpp | 9 +++-- 3 files changed, 80 insertions(+), 25 deletions(-) diff --git a/src/processor/http_fingerprint.cpp b/src/processor/http_fingerprint.cpp index 4210bbb28..099ea6cac 100644 --- a/src/processor/http_fingerprint.cpp +++ b/src/processor/http_fingerprint.cpp @@ -7,6 +7,7 @@ #include "processor/http_fingerprint.hpp" #include "sha256.hpp" +#include "transformer/lowercase.hpp" namespace ddwaf { @@ -25,6 +26,8 @@ struct string_buffer { char &operator[](std::size_t idx) const { return buffer[idx]; } + [[nodiscard]] std::span subspan(std::size_t len) const { return {&buffer[index], len}; } + void append(std::string_view str) { memcpy(&buffer[index], str.data(), str.size()); @@ -36,13 +39,24 @@ struct string_buffer { for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } } + template void append(std::array str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + template void append_lowercase(std::array str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + void append(char c) { buffer[index++] = c; } - char *move() + std::pair move() { auto *ptr = buffer; buffer = nullptr; - return ptr; + return {ptr, index}; } char *buffer{nullptr}; @@ -64,14 +78,41 @@ std::string get_truncated_hash(const ddwaf_object &body) return ""; } - sha256_hash hasher; + std::vector lc_key_buffer; + lc_key_buffer.reserve(body.nbEntries); + + scope_exit free_ptrs_at_exit([&lc_key_buffer]() { + for (auto *ptr : lc_key_buffer) { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc) + free(ptr); + } + }); + std::vector keys; + keys.reserve(body.nbEntries); + for (unsigned i = 0; i < body.nbEntries; ++i) { const auto &child = body.array[i]; - hasher << std::string_view{child.parameterName, static_cast(child.parameterNameLength)}; - }; - return hasher.digest().substr(0, 8); -} + std::string_view key{ + child.parameterName, static_cast(child.parameterNameLength)}; + cow_string lc_key{key}; + + if (transformer::lowercase::transform(lc_key)) { + auto [ptr, size] = lc_key.move(); + lc_key_buffer.emplace_back(ptr); + keys.emplace_back(ptr, size); + } else { + keys.emplace_back(key); + } + } + + std::sort(keys.begin(), keys.end()); + + sha256_hash hasher; + for (auto key : keys) { hasher << key; } + + return hasher.digest<8>(); +} } // namespace @@ -96,8 +137,9 @@ std::pair http_fingerprint::eval_impl( buffer.append('-'); buffer.append(get_truncated_hash(*query.value)); + auto [ptr, size] = buffer.move(); ddwaf_object res; - ddwaf_object_stringl_nc(&res, buffer.move(), buffer.length); + ddwaf_object_stringl_nc(&res, ptr, size); return {res, object_store::attribute::none}; } diff --git a/src/sha256.cpp b/src/sha256.cpp index 7ef19a7ee..5e6182ab0 100644 --- a/src/sha256.cpp +++ b/src/sha256.cpp @@ -84,9 +84,6 @@ constexpr std::array K256 = {0x428a2f98UL, 0x71374491UL, 0xb5c0fbc 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL}; -// Length of the string representation of the digest -constexpr std::size_t sha_digest_length = 64; - } // namespace sha256_hash &sha256_hash::operator<<(std::string_view str) @@ -145,7 +142,21 @@ sha256_hash &sha256_hash::operator<<(std::string_view str) return *this; } +template std::string sha256_hash::digest() + requires(N % 8 == 0 && N <= 64) +{ + std::array final_digest{0}; + write_digest(final_digest); + return std::string{final_digest.data(), N}; +} + +template std::string sha256_hash::digest<64>(); +template std::string sha256_hash::digest<8>(); + +template +void sha256_hash::write_digest(std::array &output) + requires(N % 8 == 0 && N <= 64) { auto *p = buffer.data(); size_t n = num; @@ -178,23 +189,20 @@ std::string sha256_hash::digest() num = 0; memset(p, 0, block_size); - std::array final_digest{0}; - for (unsigned int nn = 0; nn < sha_digest_length; nn += 8) { + for (unsigned int nn = 0; nn < N; nn += 8) { uint32_t ll = hash[nn >> 3]; - final_digest[nn + 0] = UINT8_TO_HEX_CHAR(static_cast((ll >> 28) & 0x0f)); - final_digest[nn + 1] = UINT8_TO_HEX_CHAR(static_cast((ll >> 24) & 0x0f)); - final_digest[nn + 2] = UINT8_TO_HEX_CHAR(static_cast((ll >> 20) & 0x0f)); - final_digest[nn + 3] = UINT8_TO_HEX_CHAR(static_cast((ll >> 16) & 0x0f)); - final_digest[nn + 4] = UINT8_TO_HEX_CHAR(static_cast((ll >> 12) & 0x0f)); - final_digest[nn + 5] = UINT8_TO_HEX_CHAR(static_cast((ll >> 8) & 0x0f)); - final_digest[nn + 6] = UINT8_TO_HEX_CHAR(static_cast((ll >> 4) & 0x0f)); - final_digest[nn + 7] = UINT8_TO_HEX_CHAR(static_cast(ll & 0x0f)); + output[nn + 0] = UINT8_TO_HEX_CHAR(static_cast((ll >> 28) & 0x0f)); + output[nn + 1] = UINT8_TO_HEX_CHAR(static_cast((ll >> 24) & 0x0f)); + output[nn + 2] = UINT8_TO_HEX_CHAR(static_cast((ll >> 20) & 0x0f)); + output[nn + 3] = UINT8_TO_HEX_CHAR(static_cast((ll >> 16) & 0x0f)); + output[nn + 4] = UINT8_TO_HEX_CHAR(static_cast((ll >> 12) & 0x0f)); + output[nn + 5] = UINT8_TO_HEX_CHAR(static_cast((ll >> 8) & 0x0f)); + output[nn + 6] = UINT8_TO_HEX_CHAR(static_cast((ll >> 4) & 0x0f)); + output[nn + 7] = UINT8_TO_HEX_CHAR(static_cast(ll & 0x0f)); } // Reset the hasher and return reset(); - - return std::string{final_digest.data(), 64}; } void sha256_hash::sha_block_data_order(const uint8_t *data, size_t len) diff --git a/src/sha256.hpp b/src/sha256.hpp index fe03e7c36..75357d6ac 100644 --- a/src/sha256.hpp +++ b/src/sha256.hpp @@ -21,7 +21,9 @@ class sha256_hash { sha256_hash &operator=(sha256_hash &&) noexcept = delete; sha256_hash &operator<<(std::string_view str); - [[nodiscard]] std::string digest(); + template + [[nodiscard]] std::string digest() + requires(N % 8 == 0 && N <= 64); void reset() { @@ -37,8 +39,11 @@ class sha256_hash { static constexpr std::array initial_hash_values{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; - void sha_block_data_order(const uint8_t *data, size_t len); + template + void write_digest(std::array &output) + requires(N % 8 == 0 && N <= 64); + void sha_block_data_order(const uint8_t *data, size_t len); std::array hash{initial_hash_values}; uint32_t length_low{0}; uint32_t length_high{0}; From 7295763861bf045bcef05f19e29f13a453d67bb2 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Thu, 4 Jul 2024 22:08:54 +0100 Subject: [PATCH 04/20] Add common functionality --- src/processor/fingerprint_common.hpp | 136 +++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 src/processor/fingerprint_common.hpp diff --git a/src/processor/fingerprint_common.hpp b/src/processor/fingerprint_common.hpp new file mode 100644 index 000000000..3d8d2ae06 --- /dev/null +++ b/src/processor/fingerprint_common.hpp @@ -0,0 +1,136 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include + +#include "sha256.hpp" +#include "transformer/lowercase.hpp" + +namespace ddwaf::fingerprint { + +struct string_buffer { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc, + // cppcoreguidelines-pro-type-reinterpret-cast) + explicit string_buffer(std::size_t length) + : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) + { + if (buffer == nullptr) { + throw std::bad_alloc{}; + } + } + + char &operator[](std::size_t idx) const { return buffer[idx]; } + + [[nodiscard]] std::span subspan(std::size_t len) const { return {&buffer[index], len}; } + + void append(std::string_view str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + void append_lowercase(std::string_view str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + template void append(std::array str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + template void append_lowercase(std::array str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + void append(char c) { buffer[index++] = c; } + + std::pair move() + { + auto *ptr = buffer; + buffer = nullptr; + return {ptr, index}; + } + + char *buffer{nullptr}; + std::size_t index{0}; + std::size_t length; +}; + +struct field_generator { + field_generator() = default; + virtual ~field_generator() = default; + field_generator(const field_generator&) = default; + field_generator(field_generator&&) = default; + field_generator& operator=(const field_generator&) = default; + field_generator& operator=(field_generator&&) = default; + + virtual std::size_t length() = 0; + virtual void operator()(string_buffer &output) = 0; +}; + +struct string_field : field_generator { + string_field() = default; + ~string_field() override = default; + string_field(const string_field&) = default; + string_field(string_field&&) = default; + string_field& operator=(const string_field&) = default; + string_field& operator=(string_field&&) = default; + + std::size_t length() override; + void operator()(string_buffer &output) override; +}; + +struct key_hash_field : field_generator { + key_hash_field() = default; + ~key_hash_field() override = default; + key_hash_field(const key_hash_field&) = default; + key_hash_field(key_hash_field&&) = default; + key_hash_field& operator=(const key_hash_field&) = default; + key_hash_field& operator=(key_hash_field&&) = default; + + std::size_t length() override; + void operator()(string_buffer &output) override; +}; + +struct value_hash_field : field_generator { + value_hash_field() = default; + ~value_hash_field() override = default; + value_hash_field(const value_hash_field&) = default; + value_hash_field(value_hash_field&&) = default; + value_hash_field& operator=(const value_hash_field&) = default; + value_hash_field& operator=(value_hash_field&&) = default; + + std::size_t length() override; + void operator()(string_buffer &output) override; +}; + +template +ddwaf_object generate_fragment(std::string_view header, std::array generators) +{ + std::size_t total_length = header.size() + 1; + for (auto & generator : generators) { + total_length += generator.length(); + } + + string_buffer buffer{total_length}; + buffer.append_lowercase(header); + buffer.append('-'); + + for (auto & generator : generators) { + generator(buffer); + } + + ddwaf_object res; + auto [ptr, size] = buffer.move(); + ddwaf_object_stringl_nc(&res, ptr, size); + + return res; +} + +} // namespace ddwaf::fingerprint From 8ee168b02d866dffb0c52f363eb21fe2d0d81a65 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:56:15 +0100 Subject: [PATCH 05/20] Fingerprint fragment generators --- cmake/objects.cmake | 1 + src/obfuscator.hpp | 3 +- src/processor/fingerprint_common.cpp | 90 +++++++++++++++++++ src/processor/fingerprint_common.hpp | 84 ++++++++++++++---- src/processor/http_fingerprint.cpp | 127 ++------------------------- src/sha256.cpp | 15 +++- src/sha256.hpp | 10 ++- 7 files changed, 182 insertions(+), 148 deletions(-) create mode 100644 src/processor/fingerprint_common.cpp diff --git a/cmake/objects.cmake b/cmake/objects.cmake index 359844d56..e9f432d11 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -46,6 +46,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp ${libddwaf_SOURCE_DIR}/src/processor/http_fingerprint.cpp + ${libddwaf_SOURCE_DIR}/src/processor/fingerprint_common.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp diff --git a/src/obfuscator.hpp b/src/obfuscator.hpp index 3dd01ad1d..26476428f 100644 --- a/src/obfuscator.hpp +++ b/src/obfuscator.hpp @@ -29,8 +29,9 @@ class obfuscator { static constexpr std::string_view redaction_msg{""}; static constexpr std::string_view default_key_regex_str{ - R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; + //R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; + R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; protected: std::unique_ptr key_regex{nullptr}; std::unique_ptr value_regex{nullptr}; diff --git a/src/processor/fingerprint_common.cpp b/src/processor/fingerprint_common.cpp new file mode 100644 index 000000000..317e4e563 --- /dev/null +++ b/src/processor/fingerprint_common.cpp @@ -0,0 +1,90 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "sha256.hpp" +#include "processor/fingerprint_common.hpp" +#include "transformer/lowercase.hpp" + +namespace ddwaf::fingerprint { + +// Retur true if the first argument is less than (i.e. is ordered before) the second +bool str_casei_cmp(std::string_view left, std::string_view right) +{ + auto n = std::min(left.size(), right.size()); + for (std::size_t i = 0; i < n; ++i) { + auto lc = ddwaf::tolower(left[i]); + auto rc = ddwaf::tolower(right[i]); + if (lc != rc ) { + return lc < rc; + } + } + return left.size() <= right.size(); +} + +void normalize_string(std::string_view key, std::string &buffer, bool trailing_separator) +{ + // Clear should not deallocate... + // TODO: verify + buffer.clear(); + + if (buffer.capacity() < key.size()) { + // Add space for the extra comma, just in case + buffer.reserve(key.size() + 1); + } + + for (auto c : key) { + if (c == ',') { + buffer.append(R"(\,)"); + } else { + buffer.append(1, ddwaf::tolower(c)); + } + } + + if (trailing_separator) { + buffer.append(1, ','); + } +} + +void string_hash_field::operator()(string_buffer &output) +{ + cow_string value_lc{value}; + transformer::lowercase::transform(value_lc); + + sha256_hash hasher; + hasher << static_cast(value_lc); + + hasher.write_digest(output.subspan<8>()); +} + +void key_hash_field::operator()(string_buffer &output) +{ + if (value.type != DDWAF_OBJ_MAP or value.nbEntries == 0) { return; } + + std::vector keys; + keys.reserve(value.nbEntries); + + for (unsigned i = 0; i < value.nbEntries; ++i) { + const auto &child = value.array[i]; + + std::string_view key{ + child.parameterName, static_cast(child.parameterNameLength)}; + + keys.emplace_back(key); + } + + std::sort(keys.begin(), keys.end(), str_casei_cmp); + + sha256_hash hasher; + std::string normalized; + for (unsigned i = 0; i < keys.size(); ++i) { + normalize_string(keys[i], normalized, (i + 1) < keys.size()); + hasher << normalized; + } + + hasher.write_digest(output.subspan<8>()); +} + +} // namespace ddwaf::fingerprint diff --git a/src/processor/fingerprint_common.hpp b/src/processor/fingerprint_common.hpp index 3d8d2ae06..540eef18a 100644 --- a/src/processor/fingerprint_common.hpp +++ b/src/processor/fingerprint_common.hpp @@ -5,16 +5,17 @@ // Copyright 2021 Datadog, Inc. #include +#include +#include +#include -#include "sha256.hpp" -#include "transformer/lowercase.hpp" +#include "utils.hpp" namespace ddwaf::fingerprint { struct string_buffer { - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc, - // cppcoreguidelines-pro-type-reinterpret-cast) explicit string_buffer(std::size_t length) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) { if (buffer == nullptr) { @@ -24,7 +25,10 @@ struct string_buffer { char &operator[](std::size_t idx) const { return buffer[idx]; } - [[nodiscard]] std::span subspan(std::size_t len) const { return {&buffer[index], len}; } + template + [[nodiscard]] std::span subspan() { + return std::span{&buffer[index], N}; + } void append(std::string_view str) { @@ -75,56 +79,98 @@ struct field_generator { }; struct string_field : field_generator { - string_field() = default; + explicit string_field(std::string_view input): value(input) {} ~string_field() override = default; string_field(const string_field&) = default; string_field(string_field&&) = default; string_field& operator=(const string_field&) = default; string_field& operator=(string_field&&) = default; - std::size_t length() override; + std::size_t length() override { return value.size(); } + void operator()(string_buffer &output) override { + output.append_lowercase(value); + } + + std::string_view value; +}; + +struct string_hash_field : field_generator { + explicit string_hash_field(std::string_view input): value(input) {} + ~string_hash_field() override = default; + string_hash_field(const string_hash_field&) = default; + string_hash_field(string_hash_field&&) = default; + string_hash_field& operator=(const string_hash_field&) = default; + string_hash_field& operator=(string_hash_field&&) = default; + + std::size_t length() override { return value.size(); } void operator()(string_buffer &output) override; + + std::string_view value; }; + struct key_hash_field : field_generator { - key_hash_field() = default; + explicit key_hash_field(const ddwaf_object &input): value(input) {} ~key_hash_field() override = default; key_hash_field(const key_hash_field&) = default; key_hash_field(key_hash_field&&) = default; key_hash_field& operator=(const key_hash_field&) = default; key_hash_field& operator=(key_hash_field&&) = default; - std::size_t length() override; + std::size_t length() override { + return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; + } void operator()(string_buffer &output) override; + + ddwaf_object value; }; struct value_hash_field : field_generator { - value_hash_field() = default; + explicit value_hash_field(const ddwaf_object &input): value(input) {} ~value_hash_field() override = default; value_hash_field(const value_hash_field&) = default; value_hash_field(value_hash_field&&) = default; value_hash_field& operator=(const value_hash_field&) = default; value_hash_field& operator=(value_hash_field&&) = default; - std::size_t length() override; + std::size_t length() override { + return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; + } void operator()(string_buffer &output) override; + + ddwaf_object value; }; -template -ddwaf_object generate_fragment(std::string_view header, std::array generators) +template +std::size_t generate_fragment_length(T &generator, Rest... rest) +{ + if constexpr (sizeof...(rest) > 0) { + return generator.length() + generate_fragment_length(rest...); + } else { + return generator.length(); + } +} + +template +void generate_fragment_field(string_buffer &buffer, T &generator, Rest... rest) { - std::size_t total_length = header.size() + 1; - for (auto & generator : generators) { - total_length += generator.length(); + generator(buffer); + if constexpr (sizeof...(rest) > 0) { + buffer.append('-'); + generate_fragment_field(buffer, rest...); } +} + +template +ddwaf_object generate_fragment(std::string_view header, Args... generators) +{ + std::size_t total_length = header.size() + 1 + generate_fragment_length(generators...); string_buffer buffer{total_length}; buffer.append_lowercase(header); buffer.append('-'); - for (auto & generator : generators) { - generator(buffer); - } + generate_fragment_field(buffer, generators...); ddwaf_object res; auto [ptr, size] = buffer.move(); diff --git a/src/processor/http_fingerprint.cpp b/src/processor/http_fingerprint.cpp index 099ea6cac..74dafa129 100644 --- a/src/processor/http_fingerprint.cpp +++ b/src/processor/http_fingerprint.cpp @@ -4,118 +4,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. -#include "processor/http_fingerprint.hpp" #include "sha256.hpp" +#include "processor/fingerprint_common.hpp" +#include "processor/http_fingerprint.hpp" #include "transformer/lowercase.hpp" namespace ddwaf { -namespace { - -struct string_buffer { - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc, - // cppcoreguidelines-pro-type-reinterpret-cast) - explicit string_buffer(std::size_t length) - : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) - { - if (buffer == nullptr) { - throw std::bad_alloc{}; - } - } - - char &operator[](std::size_t idx) const { return buffer[idx]; } - - [[nodiscard]] std::span subspan(std::size_t len) const { return {&buffer[index], len}; } - - void append(std::string_view str) - { - memcpy(&buffer[index], str.data(), str.size()); - index += str.size(); - } - - void append_lowercase(std::string_view str) - { - for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } - } - - template void append(std::array str) - { - memcpy(&buffer[index], str.data(), str.size()); - index += str.size(); - } - - template void append_lowercase(std::array str) - { - for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } - } - - void append(char c) { buffer[index++] = c; } - - std::pair move() - { - auto *ptr = buffer; - buffer = nullptr; - return {ptr, index}; - } - - char *buffer{nullptr}; - std::size_t index{0}; - std::size_t length; -}; - -// TODO write this directly into string buffer -std::string get_truncated_hash(std::string_view str) -{ - sha256_hash hasher; - hasher << str; - return hasher.digest().substr(0, 8); -} - -std::string get_truncated_hash(const ddwaf_object &body) -{ - if (body.type != DDWAF_OBJ_MAP or body.nbEntries == 0) { - return ""; - } - - std::vector lc_key_buffer; - lc_key_buffer.reserve(body.nbEntries); - - scope_exit free_ptrs_at_exit([&lc_key_buffer]() { - for (auto *ptr : lc_key_buffer) { - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc) - free(ptr); - } - }); - std::vector keys; - keys.reserve(body.nbEntries); - - for (unsigned i = 0; i < body.nbEntries; ++i) { - const auto &child = body.array[i]; - - std::string_view key{ - child.parameterName, static_cast(child.parameterNameLength)}; - cow_string lc_key{key}; - - if (transformer::lowercase::transform(lc_key)) { - auto [ptr, size] = lc_key.move(); - lc_key_buffer.emplace_back(ptr); - keys.emplace_back(ptr, size); - } else { - keys.emplace_back(key); - } - } - - std::sort(keys.begin(), keys.end()); - - sha256_hash hasher; - for (auto key : keys) { hasher << key; } - - return hasher.digest<8>(); -} - -} // namespace - std::pair http_fingerprint::eval_impl( const unary_argument &method, const unary_argument &uri_raw, const unary_argument &body, @@ -125,21 +21,12 @@ std::pair http_fingerprint::eval_impl( throw ddwaf::timeout_exception(); } - // http---- - string_buffer buffer{4 + 1 + method.value.size() + 1 + 8 + 1 + 8 + 1 + 8}; - buffer.append("http-"); - buffer.append_lowercase(method.value); - buffer.append('-'); - - buffer.append(get_truncated_hash(uri_raw.value)); - buffer.append('-'); - buffer.append(get_truncated_hash(*body.value)); - buffer.append('-'); - buffer.append(get_truncated_hash(*query.value)); + auto res = fingerprint::generate_fragment("http", + fingerprint::string_field{method.value}, + fingerprint::string_hash_field{uri_raw.value}, + fingerprint::key_hash_field{*body.value}, + fingerprint::key_hash_field{*query.value}); - auto [ptr, size] = buffer.move(); - ddwaf_object res; - ddwaf_object_stringl_nc(&res, ptr, size); return {res, object_store::attribute::none}; } diff --git a/src/sha256.cpp b/src/sha256.cpp index 5e6182ab0..afc02e0e2 100644 --- a/src/sha256.cpp +++ b/src/sha256.cpp @@ -15,6 +15,9 @@ // #include "sha256.hpp" +#include +#include +#include namespace ddwaf { namespace { @@ -146,16 +149,17 @@ template std::string sha256_hash::digest() requires(N % 8 == 0 && N <= 64) { - std::array final_digest{0}; - write_digest(final_digest); - return std::string{final_digest.data(), N}; + std::string final_digest; + final_digest.resize(N, 0); + write_digest(std::span{final_digest}); + return final_digest; } template std::string sha256_hash::digest<64>(); template std::string sha256_hash::digest<8>(); template -void sha256_hash::write_digest(std::array &output) +void sha256_hash::write_digest(std::span output) requires(N % 8 == 0 && N <= 64) { auto *p = buffer.data(); @@ -205,6 +209,9 @@ void sha256_hash::write_digest(std::array &output) reset(); } +template void sha256_hash::write_digest<8>(std::span output); +template void sha256_hash::write_digest<64>(std::span output); + void sha256_hash::sha_block_data_order(const uint8_t *data, size_t len) { unsigned int a; diff --git a/src/sha256.hpp b/src/sha256.hpp index 75357d6ac..cad18a97c 100644 --- a/src/sha256.hpp +++ b/src/sha256.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -25,6 +26,11 @@ class sha256_hash { [[nodiscard]] std::string digest() requires(N % 8 == 0 && N <= 64); + template + void write_digest(std::span output) + requires(N % 8 == 0 && N <= 64); + + void reset() { hash = initial_hash_values; @@ -39,10 +45,6 @@ class sha256_hash { static constexpr std::array initial_hash_values{0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; - template - void write_digest(std::array &output) - requires(N % 8 == 0 && N <= 64); - void sha_block_data_order(const uint8_t *data, size_t len); std::array hash{initial_hash_values}; uint32_t length_low{0}; From 0bf9fbdfea89dbe70fefb392c4a13d6e4bf352e6 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 5 Jul 2024 17:14:33 +0100 Subject: [PATCH 06/20] Rename some files --- cmake/objects.cmake | 3 +- src/builder/processor_builder.cpp | 2 +- src/obfuscator.hpp | 5 +- src/parser/processor_parser.cpp | 2 +- ...fingerprint_common.cpp => fingerprint.cpp} | 32 +++++- ...fingerprint_common.hpp => fingerprint.hpp} | 99 +++++++++++-------- src/processor/http_fingerprint.cpp | 33 ------- src/processor/http_fingerprint.hpp | 34 ------- src/sha256.hpp | 1 - 9 files changed, 91 insertions(+), 120 deletions(-) rename src/processor/{fingerprint_common.cpp => fingerprint.cpp} (69%) rename src/processor/{fingerprint_common.hpp => fingerprint.hpp} (55%) delete mode 100644 src/processor/http_fingerprint.cpp delete mode 100644 src/processor/http_fingerprint.hpp diff --git a/cmake/objects.cmake b/cmake/objects.cmake index e9f432d11..9a4974dac 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -45,8 +45,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp - ${libddwaf_SOURCE_DIR}/src/processor/http_fingerprint.cpp - ${libddwaf_SOURCE_DIR}/src/processor/fingerprint_common.cpp + ${libddwaf_SOURCE_DIR}/src/processor/fingerprint.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/sqli_detector.cpp ${libddwaf_SOURCE_DIR}/src/condition/ssrf_detector.cpp diff --git a/src/builder/processor_builder.cpp b/src/builder/processor_builder.cpp index fb18fd964..5a2625137 100644 --- a/src/builder/processor_builder.cpp +++ b/src/builder/processor_builder.cpp @@ -8,7 +8,7 @@ #include "builder/processor_builder.hpp" #include "processor/extract_schema.hpp" -#include "processor/http_fingerprint.hpp" +#include "processor/fingerprint.hpp" namespace ddwaf { diff --git a/src/obfuscator.hpp b/src/obfuscator.hpp index 26476428f..54c629810 100644 --- a/src/obfuscator.hpp +++ b/src/obfuscator.hpp @@ -29,9 +29,10 @@ class obfuscator { static constexpr std::string_view redaction_msg{""}; static constexpr std::string_view default_key_regex_str{ - //R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; + // R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; + + R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; - R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; protected: std::unique_ptr key_regex{nullptr}; std::unique_ptr value_regex{nullptr}; diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp index 433819288..3a3b0d44a 100644 --- a/src/parser/processor_parser.cpp +++ b/src/parser/processor_parser.cpp @@ -9,7 +9,7 @@ #include "parser/parser.hpp" #include "processor/base.hpp" #include "processor/extract_schema.hpp" -#include "processor/http_fingerprint.hpp" +#include "processor/fingerprint.hpp" #include namespace ddwaf::parser::v2 { diff --git a/src/processor/fingerprint_common.cpp b/src/processor/fingerprint.cpp similarity index 69% rename from src/processor/fingerprint_common.cpp rename to src/processor/fingerprint.cpp index 317e4e563..9b76168ea 100644 --- a/src/processor/fingerprint_common.cpp +++ b/src/processor/fingerprint.cpp @@ -4,11 +4,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. +#include "processor/fingerprint.hpp" #include "sha256.hpp" -#include "processor/fingerprint_common.hpp" #include "transformer/lowercase.hpp" -namespace ddwaf::fingerprint { +namespace ddwaf { +namespace fingerprint { // Retur true if the first argument is less than (i.e. is ordered before) the second bool str_casei_cmp(std::string_view left, std::string_view right) @@ -17,7 +18,7 @@ bool str_casei_cmp(std::string_view left, std::string_view right) for (std::size_t i = 0; i < n; ++i) { auto lc = ddwaf::tolower(left[i]); auto rc = ddwaf::tolower(right[i]); - if (lc != rc ) { + if (lc != rc) { return lc < rc; } } @@ -61,7 +62,9 @@ void string_hash_field::operator()(string_buffer &output) void key_hash_field::operator()(string_buffer &output) { - if (value.type != DDWAF_OBJ_MAP or value.nbEntries == 0) { return; } + if (value.type != DDWAF_OBJ_MAP or value.nbEntries == 0) { + return; + } std::vector keys; keys.reserve(value.nbEntries); @@ -87,4 +90,23 @@ void key_hash_field::operator()(string_buffer &output) hasher.write_digest(output.subspan<8>()); } -} // namespace ddwaf::fingerprint +} // namespace fingerprint + +// NOLINTNEXTLINE(readability-convert-member-functions-to-static) +std::pair http_fingerprint::eval_impl( + const unary_argument &method, const unary_argument &uri_raw, + const unary_argument &body, + const unary_argument &query, ddwaf::timer &deadline) const +{ + if (deadline.expired()) { + throw ddwaf::timeout_exception(); + } + + auto res = fingerprint::generate_fragment("http", fingerprint::string_field{method.value}, + fingerprint::string_hash_field{uri_raw.value}, fingerprint::key_hash_field{*body.value}, + fingerprint::key_hash_field{*query.value}); + + return {res, object_store::attribute::none}; +} + +} // namespace ddwaf diff --git a/src/processor/fingerprint_common.hpp b/src/processor/fingerprint.hpp similarity index 55% rename from src/processor/fingerprint_common.hpp rename to src/processor/fingerprint.hpp index 540eef18a..e0b744e16 100644 --- a/src/processor/fingerprint_common.hpp +++ b/src/processor/fingerprint.hpp @@ -4,18 +4,22 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. +#include +#include #include #include -#include -#include +#include "processor/base.hpp" +#include "scanner.hpp" #include "utils.hpp" -namespace ddwaf::fingerprint { +namespace ddwaf { +namespace fingerprint { struct string_buffer { explicit string_buffer(std::size_t length) - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, + // hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) { if (buffer == nullptr) { @@ -25,8 +29,8 @@ struct string_buffer { char &operator[](std::size_t idx) const { return buffer[idx]; } - template - [[nodiscard]] std::span subspan() { + template [[nodiscard]] std::span subspan() + { return std::span{&buffer[index], N}; } @@ -69,38 +73,36 @@ struct string_buffer { struct field_generator { field_generator() = default; virtual ~field_generator() = default; - field_generator(const field_generator&) = default; - field_generator(field_generator&&) = default; - field_generator& operator=(const field_generator&) = default; - field_generator& operator=(field_generator&&) = default; + field_generator(const field_generator &) = default; + field_generator(field_generator &&) = default; + field_generator &operator=(const field_generator &) = default; + field_generator &operator=(field_generator &&) = default; virtual std::size_t length() = 0; virtual void operator()(string_buffer &output) = 0; }; struct string_field : field_generator { - explicit string_field(std::string_view input): value(input) {} + explicit string_field(std::string_view input) : value(input) {} ~string_field() override = default; - string_field(const string_field&) = default; - string_field(string_field&&) = default; - string_field& operator=(const string_field&) = default; - string_field& operator=(string_field&&) = default; + string_field(const string_field &) = default; + string_field(string_field &&) = default; + string_field &operator=(const string_field &) = default; + string_field &operator=(string_field &&) = default; std::size_t length() override { return value.size(); } - void operator()(string_buffer &output) override { - output.append_lowercase(value); - } + void operator()(string_buffer &output) override { output.append_lowercase(value); } std::string_view value; }; struct string_hash_field : field_generator { - explicit string_hash_field(std::string_view input): value(input) {} + explicit string_hash_field(std::string_view input) : value(input) {} ~string_hash_field() override = default; - string_hash_field(const string_hash_field&) = default; - string_hash_field(string_hash_field&&) = default; - string_hash_field& operator=(const string_hash_field&) = default; - string_hash_field& operator=(string_hash_field&&) = default; + string_hash_field(const string_hash_field &) = default; + string_hash_field(string_hash_field &&) = default; + string_hash_field &operator=(const string_hash_field &) = default; + string_hash_field &operator=(string_hash_field &&) = default; std::size_t length() override { return value.size(); } void operator()(string_buffer &output) override; @@ -108,34 +110,29 @@ struct string_hash_field : field_generator { std::string_view value; }; - struct key_hash_field : field_generator { - explicit key_hash_field(const ddwaf_object &input): value(input) {} + explicit key_hash_field(const ddwaf_object &input) : value(input) {} ~key_hash_field() override = default; - key_hash_field(const key_hash_field&) = default; - key_hash_field(key_hash_field&&) = default; - key_hash_field& operator=(const key_hash_field&) = default; - key_hash_field& operator=(key_hash_field&&) = default; + key_hash_field(const key_hash_field &) = default; + key_hash_field(key_hash_field &&) = default; + key_hash_field &operator=(const key_hash_field &) = default; + key_hash_field &operator=(key_hash_field &&) = default; - std::size_t length() override { - return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; - } + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; } void operator()(string_buffer &output) override; ddwaf_object value; }; struct value_hash_field : field_generator { - explicit value_hash_field(const ddwaf_object &input): value(input) {} + explicit value_hash_field(const ddwaf_object &input) : value(input) {} ~value_hash_field() override = default; - value_hash_field(const value_hash_field&) = default; - value_hash_field(value_hash_field&&) = default; - value_hash_field& operator=(const value_hash_field&) = default; - value_hash_field& operator=(value_hash_field&&) = default; + value_hash_field(const value_hash_field &) = default; + value_hash_field(value_hash_field &&) = default; + value_hash_field &operator=(const value_hash_field &) = default; + value_hash_field &operator=(value_hash_field &&) = default; - std::size_t length() override { - return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; - } + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; } void operator()(string_buffer &output) override; ddwaf_object value; @@ -179,4 +176,24 @@ ddwaf_object generate_fragment(std::string_view header, Args... generators) return res; } -} // namespace ddwaf::fingerprint +} // namespace fingerprint + +class http_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{ + "method", "uri_raw", "query", "body"}; + + http_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &method, + const unary_argument &uri_raw, + const unary_argument &query, + const unary_argument &body, ddwaf::timer &deadline) const; +}; + +} // namespace ddwaf diff --git a/src/processor/http_fingerprint.cpp b/src/processor/http_fingerprint.cpp deleted file mode 100644 index 74dafa129..000000000 --- a/src/processor/http_fingerprint.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - - -#include "sha256.hpp" -#include "processor/fingerprint_common.hpp" -#include "processor/http_fingerprint.hpp" -#include "transformer/lowercase.hpp" - -namespace ddwaf { - -std::pair http_fingerprint::eval_impl( - const unary_argument &method, const unary_argument &uri_raw, - const unary_argument &body, - const unary_argument &query, ddwaf::timer &deadline) const -{ - if (deadline.expired()) { - throw ddwaf::timeout_exception(); - } - - auto res = fingerprint::generate_fragment("http", - fingerprint::string_field{method.value}, - fingerprint::string_hash_field{uri_raw.value}, - fingerprint::key_hash_field{*body.value}, - fingerprint::key_hash_field{*query.value}); - - return {res, object_store::attribute::none}; -} - -} // namespace ddwaf diff --git a/src/processor/http_fingerprint.hpp b/src/processor/http_fingerprint.hpp deleted file mode 100644 index aa39a65ec..000000000 --- a/src/processor/http_fingerprint.hpp +++ /dev/null @@ -1,34 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include - -#include "processor/base.hpp" -#include "scanner.hpp" - -namespace ddwaf { - -class http_fingerprint : public structured_processor { -public: - static constexpr std::array param_names{ - "method", "uri_raw", "query", "body"}; - - http_fingerprint(std::string id, std::shared_ptr expr, - std::vector mappings, bool evaluate, bool output) - : structured_processor( - std::move(id), std::move(expr), std::move(mappings), evaluate, output) - {} - - std::pair eval_impl( - const unary_argument &method, - const unary_argument &uri_raw, - const unary_argument &query, - const unary_argument &body, ddwaf::timer &deadline) const; -}; - -} // namespace ddwaf diff --git a/src/sha256.hpp b/src/sha256.hpp index cad18a97c..59bdcd69a 100644 --- a/src/sha256.hpp +++ b/src/sha256.hpp @@ -30,7 +30,6 @@ class sha256_hash { void write_digest(std::span output) requires(N % 8 == 0 && N <= 64); - void reset() { hash = initial_hash_values; From 11b96b9780a1b13f2c819ea69f7b9729324475e8 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 5 Jul 2024 17:29:50 +0100 Subject: [PATCH 07/20] Add other processors --- src/builder/processor_builder.cpp | 8 ++--- src/builder/processor_builder.hpp | 6 ++-- src/parser/processor_parser.cpp | 19 ++++++++++-- src/processor/fingerprint.cpp | 6 ++-- src/processor/fingerprint.hpp | 49 +++++++++++++++++++++++++++++-- 5 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/builder/processor_builder.cpp b/src/builder/processor_builder.cpp index 5a2625137..7ac0a572c 100644 --- a/src/builder/processor_builder.cpp +++ b/src/builder/processor_builder.cpp @@ -43,10 +43,10 @@ template <> struct typed_processor_builder { } }; -template <> struct typed_processor_builder { +template <> struct typed_processor_builder { std::shared_ptr build(const auto &spec) { - return std::make_shared( + return std::make_shared( spec.id, spec.expr, spec.mappings, spec.evaluate, spec.output); } }; @@ -79,8 +79,8 @@ template switch (type) { case processor_type::extract_schema: return build_with_type(*this, scanners); - case processor_type::http_fingerprint: - return build_with_type(*this, scanners); + case processor_type::http_endpoint_fingerprint: + return build_with_type(*this, scanners); default: break; } diff --git a/src/builder/processor_builder.hpp b/src/builder/processor_builder.hpp index f2d220e1b..0ab3be0f0 100644 --- a/src/builder/processor_builder.hpp +++ b/src/builder/processor_builder.hpp @@ -19,10 +19,10 @@ namespace ddwaf { enum class processor_type : unsigned { extract_schema, // Reserved - http_fingerprint, + http_endpoint_fingerprint, + http_network_fingerprint, + http_header_fingerprint, session_fingerprint, - network_fingerprint, - header_fingerprint, }; struct processor_builder { diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp index 3a3b0d44a..e95776734 100644 --- a/src/parser/processor_parser.cpp +++ b/src/parser/processor_parser.cpp @@ -78,8 +78,20 @@ processor_container parse_processors( auto generator_id = at(node, "generator"); if (generator_id == "extract_schema") { type = processor_type::extract_schema; - } else if (generator_id == "http_fingerprint") { - type = processor_type::http_fingerprint; + } else if (generator_id == "http_endpoint_fingerprint") { + type = processor_type::http_endpoint_fingerprint; + } else if (generator_id == "http_network_fingerprint") { + type = processor_type::http_network_fingerprint; + // Skip for now + continue; + } else if (generator_id == "http_header_fingerprint") { + type = processor_type::http_header_fingerprint; + // Skip for now + continue; + } else if (generator_id == "session_fingerprint") { + type = processor_type::session_fingerprint; + // Skip for now + continue; } else { DDWAF_WARN("Unknown generator: {}", generator_id); info.add_failed(id, "unknown generator '" + generator_id + "'"); @@ -95,7 +107,8 @@ processor_container parse_processors( if (type == processor_type::extract_schema) { mappings = parse_processor_mappings(mappings_vec, addresses); } else { - mappings = parse_processor_mappings(mappings_vec, addresses); + mappings = + parse_processor_mappings(mappings_vec, addresses); } std::vector scanners; diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 9b76168ea..c13756c26 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -93,10 +93,10 @@ void key_hash_field::operator()(string_buffer &output) } // namespace fingerprint // NOLINTNEXTLINE(readability-convert-member-functions-to-static) -std::pair http_fingerprint::eval_impl( +std::pair http_endpoint_fingerprint::eval_impl( const unary_argument &method, const unary_argument &uri_raw, - const unary_argument &body, - const unary_argument &query, ddwaf::timer &deadline) const + const unary_argument &query, + const unary_argument &body, ddwaf::timer &deadline) const { if (deadline.expired()) { throw ddwaf::timeout_exception(); diff --git a/src/processor/fingerprint.hpp b/src/processor/fingerprint.hpp index e0b744e16..721ce7bab 100644 --- a/src/processor/fingerprint.hpp +++ b/src/processor/fingerprint.hpp @@ -178,12 +178,12 @@ ddwaf_object generate_fragment(std::string_view header, Args... generators) } // namespace fingerprint -class http_fingerprint : public structured_processor { +class http_endpoint_fingerprint : public structured_processor { public: static constexpr std::array param_names{ "method", "uri_raw", "query", "body"}; - http_fingerprint(std::string id, std::shared_ptr expr, + http_endpoint_fingerprint(std::string id, std::shared_ptr expr, std::vector mappings, bool evaluate, bool output) : structured_processor( std::move(id), std::move(expr), std::move(mappings), evaluate, output) @@ -196,4 +196,49 @@ class http_fingerprint : public structured_processor { const unary_argument &body, ddwaf::timer &deadline) const; }; +class http_header_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{"headers"}; + + http_header_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &headers, ddwaf::timer &deadline) const; +}; + +class http_network_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{"headers"}; + + http_network_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &headers, ddwaf::timer &deadline) const; +}; + +class session_fingerprint : public structured_processor { +public: + static constexpr std::array param_names{ + "cookies", "session_id", "user_id"}; + + session_fingerprint(std::string id, std::shared_ptr expr, + std::vector mappings, bool evaluate, bool output) + : structured_processor( + std::move(id), std::move(expr), std::move(mappings), evaluate, output) + {} + + std::pair eval_impl( + const unary_argument &cookies, + const unary_argument &session_id, + const unary_argument &user_id, ddwaf::timer &deadline) const; +}; + } // namespace ddwaf From 0a998aa031064393a36c63f4f0b9d6573d19cac8 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 5 Jul 2024 19:34:43 +0100 Subject: [PATCH 08/20] Fixes --- src/processor/fingerprint.cpp | 4 ++-- src/processor/fingerprint.hpp | 13 +++++++------ src/rule.hpp | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index c13756c26..7b2fb123c 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -103,8 +103,8 @@ std::pair http_endpoint_fingerprint::eval } auto res = fingerprint::generate_fragment("http", fingerprint::string_field{method.value}, - fingerprint::string_hash_field{uri_raw.value}, fingerprint::key_hash_field{*body.value}, - fingerprint::key_hash_field{*query.value}); + fingerprint::string_hash_field{uri_raw.value}, fingerprint::key_hash_field{*query.value}, + fingerprint::key_hash_field{*body.value}); return {res, object_store::attribute::none}; } diff --git a/src/processor/fingerprint.hpp b/src/processor/fingerprint.hpp index 721ce7bab..946ea8de7 100644 --- a/src/processor/fingerprint.hpp +++ b/src/processor/fingerprint.hpp @@ -18,8 +18,7 @@ namespace fingerprint { struct string_buffer { explicit string_buffer(std::size_t length) - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, - // hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) { if (buffer == nullptr) { @@ -31,7 +30,9 @@ struct string_buffer { template [[nodiscard]] std::span subspan() { - return std::span{&buffer[index], N}; + std::span res{&buffer[index], N}; + index += N; + return res; } void append(std::string_view str) @@ -104,7 +105,7 @@ struct string_hash_field : field_generator { string_hash_field &operator=(const string_hash_field &) = default; string_hash_field &operator=(string_hash_field &&) = default; - std::size_t length() override { return value.size(); } + std::size_t length() override { return 8; } void operator()(string_buffer &output) override; std::string_view value; @@ -118,7 +119,7 @@ struct key_hash_field : field_generator { key_hash_field &operator=(const key_hash_field &) = default; key_hash_field &operator=(key_hash_field &&) = default; - std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; } + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } void operator()(string_buffer &output) override; ddwaf_object value; @@ -132,7 +133,7 @@ struct value_hash_field : field_generator { value_hash_field &operator=(const value_hash_field &) = default; value_hash_field &operator=(value_hash_field &&) = default; - std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? value.nbEntries : 0; } + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } void operator()(string_buffer &output) override; ddwaf_object value; diff --git a/src/rule.hpp b/src/rule.hpp index 50d1cdfdb..52d38934c 100644 --- a/src/rule.hpp +++ b/src/rule.hpp @@ -77,7 +77,7 @@ class rule { return std::nullopt; } - return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral}}; + return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; } [[nodiscard]] bool is_enabled() const { return enabled_; } From c6f15ac5bd032ac6a52fddbfb10714c2b9d17a7b Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 09:56:09 +0100 Subject: [PATCH 09/20] Small fix and test --- src/obfuscator.hpp | 2 -- src/processor/fingerprint.hpp | 2 +- tests/processor/fingerprint_test.cpp | 51 ++++++++++++++++++++++++++++ tools/waf_runner.cpp | 2 +- 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 tests/processor/fingerprint_test.cpp diff --git a/src/obfuscator.hpp b/src/obfuscator.hpp index 54c629810..dfa404a60 100644 --- a/src/obfuscator.hpp +++ b/src/obfuscator.hpp @@ -29,8 +29,6 @@ class obfuscator { static constexpr std::string_view redaction_msg{""}; static constexpr std::string_view default_key_regex_str{ - // R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; - R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"}; protected: diff --git a/src/processor/fingerprint.hpp b/src/processor/fingerprint.hpp index 946ea8de7..04e8a92b4 100644 --- a/src/processor/fingerprint.hpp +++ b/src/processor/fingerprint.hpp @@ -143,7 +143,7 @@ template std::size_t generate_fragment_length(T &generator, Rest... rest) { if constexpr (sizeof...(rest) > 0) { - return generator.length() + generate_fragment_length(rest...); + return generator.length() + 1 + generate_fragment_length(rest...); } else { return generator.length(); } diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp new file mode 100644 index 000000000..18f338c3c --- /dev/null +++ b/tests/processor/fingerprint_test.cpp @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/). Copyright 2023 Datadog, Inc. + +#include "../test_utils.hpp" +#include "ddwaf.h" +#include "matcher/regex_match.hpp" +#include "processor/fingerprint.hpp" + +using namespace ddwaf; +using namespace std::literals; + +namespace { + +TEST(TestHttpEndpointFingerprint, Basic) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, output.nbEntries}; + EXPECT_STRV(output_sv, "http-get-b8dfda97-0ac3796a-9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +} // namespace diff --git a/tools/waf_runner.cpp b/tools/waf_runner.cpp index bd073d5ae..07686bdfd 100644 --- a/tools/waf_runner.cpp +++ b/tools/waf_runner.cpp @@ -47,7 +47,7 @@ auto parse_args(int argc, char *argv[]) } return args; } -const char *key_regex = R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key)|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"; +const char *key_regex = R"((?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\.net[_-]sessionid|sid|jwt)"; const char *value_regex = R"((?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\.net(?:[_-]|-)sessionid|sid|jwt)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,})"; From 018697b1f28b23dae3f52efcf156eeaac7096f6d Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:15:07 +0100 Subject: [PATCH 10/20] Another test --- src/processor/fingerprint.cpp | 4 + tests/processor/fingerprint_test.cpp | 119 +++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 7b2fb123c..5487643b6 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -51,6 +51,10 @@ void normalize_string(std::string_view key, std::string &buffer, bool trailing_s void string_hash_field::operator()(string_buffer &output) { + if (value.empty()) { + return; + } + cow_string value_lc{value}; transformer::lowercase::transform(value_lc); diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index 18f338c3c..1c4be8399 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -48,4 +48,123 @@ TEST(TestHttpEndpointFingerprint, Basic) ddwaf_object_free(&output); } +TEST(TestHttpEndpointFingerprint, EmptyQuery) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, output.nbEntries}; + EXPECT_STRV(output_sv, "http-get-b8dfda97--9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, EmptyBody) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, output.nbEntries}; + EXPECT_STRV(output_sv, "http-get-b8dfda97-0ac3796a-"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, EmptyEverything) +{ + ddwaf_object query; + ddwaf_object_map(&query); + + ddwaf_object body; + ddwaf_object_map(&body); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = gen.eval_impl({{}, {}, false, ""}, {{}, {}, false, ""}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, output.nbEntries}; + EXPECT_STRV(output_sv, "http----"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, KeyConsistency) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key3,Key4", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KeY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "kEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY3", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KeY4", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, output.nbEntries}; + EXPECT_STRV(output_sv, "http-get-b8dfda97-ced401fa-ff07216e"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + } // namespace From 6bdf01efa8ed29fd72dc6bf08c30341dc3a0e88f Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:42:11 +0100 Subject: [PATCH 11/20] Integration tests for fingerprint processor --- src/processor/fingerprint.cpp | 9 +- .../ruleset/postprocessor.json | 0 .../ruleset/preprocessor.json | 0 .../ruleset/processor.json | 0 .../ruleset/processor_with_scanner_by_id.json | 0 .../processor_with_scanner_by_tags.json | 0 .../processors/{ => extract_schema}/test.cpp | 30 +-- .../fingerprint/ruleset/postprocessor.json | 82 +++++++ .../fingerprint/ruleset/preprocessor.json | 81 +++++++ .../fingerprint/ruleset/processor.json | 81 +++++++ .../processors/fingerprint/test.cpp | 202 ++++++++++++++++++ tests/processor/fingerprint_test.cpp | 8 +- 12 files changed, 473 insertions(+), 20 deletions(-) rename tests/integration/processors/{ => extract_schema}/ruleset/postprocessor.json (100%) rename tests/integration/processors/{ => extract_schema}/ruleset/preprocessor.json (100%) rename tests/integration/processors/{ => extract_schema}/ruleset/processor.json (100%) rename tests/integration/processors/{ => extract_schema}/ruleset/processor_with_scanner_by_id.json (100%) rename tests/integration/processors/{ => extract_schema}/ruleset/processor_with_scanner_by_tags.json (100%) rename tests/integration/processors/{ => extract_schema}/test.cpp (97%) create mode 100644 tests/integration/processors/fingerprint/ruleset/postprocessor.json create mode 100644 tests/integration/processors/fingerprint/ruleset/preprocessor.json create mode 100644 tests/integration/processors/fingerprint/ruleset/processor.json create mode 100644 tests/integration/processors/fingerprint/test.cpp diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 5487643b6..e65b44a61 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -106,8 +106,15 @@ std::pair http_endpoint_fingerprint::eval throw ddwaf::timeout_exception(); } + // Strip query parameter from raw URI + auto stripped_uri = uri_raw.value; + auto query_idx = stripped_uri.find_first_of('?'); + if (query_idx != std::string_view::npos) { + stripped_uri = stripped_uri.substr(0, query_idx); + } + auto res = fingerprint::generate_fragment("http", fingerprint::string_field{method.value}, - fingerprint::string_hash_field{uri_raw.value}, fingerprint::key_hash_field{*query.value}, + fingerprint::string_hash_field{stripped_uri}, fingerprint::key_hash_field{*query.value}, fingerprint::key_hash_field{*body.value}); return {res, object_store::attribute::none}; diff --git a/tests/integration/processors/ruleset/postprocessor.json b/tests/integration/processors/extract_schema/ruleset/postprocessor.json similarity index 100% rename from tests/integration/processors/ruleset/postprocessor.json rename to tests/integration/processors/extract_schema/ruleset/postprocessor.json diff --git a/tests/integration/processors/ruleset/preprocessor.json b/tests/integration/processors/extract_schema/ruleset/preprocessor.json similarity index 100% rename from tests/integration/processors/ruleset/preprocessor.json rename to tests/integration/processors/extract_schema/ruleset/preprocessor.json diff --git a/tests/integration/processors/ruleset/processor.json b/tests/integration/processors/extract_schema/ruleset/processor.json similarity index 100% rename from tests/integration/processors/ruleset/processor.json rename to tests/integration/processors/extract_schema/ruleset/processor.json diff --git a/tests/integration/processors/ruleset/processor_with_scanner_by_id.json b/tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_id.json similarity index 100% rename from tests/integration/processors/ruleset/processor_with_scanner_by_id.json rename to tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_id.json diff --git a/tests/integration/processors/ruleset/processor_with_scanner_by_tags.json b/tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_tags.json similarity index 100% rename from tests/integration/processors/ruleset/processor_with_scanner_by_tags.json rename to tests/integration/processors/extract_schema/ruleset/processor_with_scanner_by_tags.json diff --git a/tests/integration/processors/test.cpp b/tests/integration/processors/extract_schema/test.cpp similarity index 97% rename from tests/integration/processors/test.cpp rename to tests/integration/processors/extract_schema/test.cpp index 7d6fefd12..fdaed7461 100644 --- a/tests/integration/processors/test.cpp +++ b/tests/integration/processors/extract_schema/test.cpp @@ -4,14 +4,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. -#include "../../test_utils.hpp" +#include "../../../test_utils.hpp" using namespace ddwaf; namespace { -constexpr std::string_view base_dir = "integration/processors/"; +constexpr std::string_view base_dir = "integration/processors/extract_schema"; -TEST(TestProcessors, Postprocessor) +TEST(TestExtractSchemaIntegration, Postprocessor) { auto rule = read_json_file("postprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -55,7 +55,7 @@ TEST(TestProcessors, Postprocessor) ddwaf_destroy(handle); } -TEST(TestProcessors, Preprocessor) +TEST(TestExtractSchemaIntegration, Preprocessor) { auto rule = read_json_file("preprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -108,7 +108,7 @@ TEST(TestProcessors, Preprocessor) ddwaf_destroy(handle); } -TEST(TestProcessors, Processor) +TEST(TestExtractSchemaIntegration, Processor) { auto rule = read_json_file("processor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -166,7 +166,7 @@ TEST(TestProcessors, Processor) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorWithScannerByTags) +TEST(TestExtractSchemaIntegration, ProcessorWithScannerByTags) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -203,7 +203,7 @@ TEST(TestProcessors, ProcessorWithScannerByTags) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorWithScannerByID) +TEST(TestExtractSchemaIntegration, ProcessorWithScannerByID) { auto rule = read_json_file("processor_with_scanner_by_id.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -241,7 +241,7 @@ TEST(TestProcessors, ProcessorWithScannerByID) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorUpdate) +TEST(TestExtractSchemaIntegration, ProcessorUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -318,7 +318,7 @@ TEST(TestProcessors, ProcessorUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, ScannerUpdate) +TEST(TestExtractSchemaIntegration, ScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -397,7 +397,7 @@ TEST(TestProcessors, ScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorAndScannerUpdate) +TEST(TestExtractSchemaIntegration, ProcessorAndScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -475,7 +475,7 @@ TEST(TestProcessors, ProcessorAndScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, EmptyScannerUpdate) +TEST(TestExtractSchemaIntegration, EmptyScannerUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -552,7 +552,7 @@ TEST(TestProcessors, EmptyScannerUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, EmptyProcessorUpdate) +TEST(TestExtractSchemaIntegration, EmptyProcessorUpdate) { auto rule = read_json_file("processor_with_scanner_by_tags.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -627,7 +627,7 @@ TEST(TestProcessors, EmptyProcessorUpdate) ddwaf_destroy(handle); } -TEST(TestProcessors, PostprocessorWithEphemeralMapping) +TEST(TestExtractSchemaIntegration, PostprocessorWithEphemeralMapping) { auto rule = read_json_file("postprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -694,7 +694,7 @@ TEST(TestProcessors, PostprocessorWithEphemeralMapping) ddwaf_destroy(handle); } -TEST(TestProcessors, PreprocessorWithEphemeralMapping) +TEST(TestExtractSchemaIntegration, PreprocessorWithEphemeralMapping) { auto rule = read_json_file("preprocessor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); @@ -777,7 +777,7 @@ TEST(TestProcessors, PreprocessorWithEphemeralMapping) ddwaf_destroy(handle); } -TEST(TestProcessors, ProcessorEphemeralExpression) +TEST(TestExtractSchemaIntegration, ProcessorEphemeralExpression) { auto rule = read_json_file("processor.json", base_dir); ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); diff --git a/tests/integration/processors/fingerprint/ruleset/postprocessor.json b/tests/integration/processors/fingerprint/ruleset/postprocessor.json new file mode 100644 index 000000000..4d210d4e3 --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/postprocessor.json @@ -0,0 +1,82 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.body" + } + ], + "value": 8, + "type": "unsigned" + }, + "operator": "equals" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": false, + "output": true + } + ] +} diff --git a/tests/integration/processors/fingerprint/ruleset/preprocessor.json b/tests/integration/processors/fingerprint/ruleset/preprocessor.json new file mode 100644 index 000000000..d3d4ce0df --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/preprocessor.json @@ -0,0 +1,81 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "_dd.appsec.fp.http.endpoint" + } + ], + "regex": ".*" + }, + "operator": "match_regex" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": true, + "output": false + } + ] +} diff --git a/tests/integration/processors/fingerprint/ruleset/processor.json b/tests/integration/processors/fingerprint/ruleset/processor.json new file mode 100644 index 000000000..91c86875e --- /dev/null +++ b/tests/integration/processors/fingerprint/ruleset/processor.json @@ -0,0 +1,81 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.8.0" + }, + "rules": [ + { + "id": "rule1", + "name": "rule1", + "tags": { + "type": "flow1", + "category": "category1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "_dd.appsec.fp.http.endpoint" + } + ], + "regex": ".*" + }, + "operator": "match_regex" + } + ] + } + ], + "processors": [ + { + "id": "processor-001", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "equals", + "parameters": { + "inputs": [ + { + "address": "waf.context.processor", + "key_path": [ + "fingerprint" + ] + } + ], + "value": true, + "type": "boolean" + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": true, + "output": true + } + ] +} diff --git a/tests/integration/processors/fingerprint/test.cpp b/tests/integration/processors/fingerprint/test.cpp new file mode 100644 index 000000000..5ec090b55 --- /dev/null +++ b/tests/integration/processors/fingerprint/test.cpp @@ -0,0 +1,202 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "../../../test_utils.hpp" + +using namespace ddwaf; + +namespace { +constexpr std::string_view base_dir = "integration/processors/fingerprint"; + +TEST(TestFingerprintIntegration, Postprocessor) +{ + auto rule = read_json_file("postprocessor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 5); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_OK); + EXPECT_FALSE(out.timeout); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 1); + + auto json = test::object_to_json(out.derivatives); + EXPECT_STR(json, R"({"_dd.appsec.fp.http.endpoint":"http-put-729d56c3-2c70e12b-2c70e12b"})"); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +TEST(TestFingerprintIntegration, Preprocessor) +{ + auto rule = read_json_file("preprocessor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 6); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + EXPECT_TRUE(address_set.contains("_dd.appsec.fp.http.endpoint")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_MATCH); + EXPECT_FALSE(out.timeout); + + EXPECT_EVENTS(out, {.id = "rule1", + .name = "rule1", + .tags = {{"type", "flow1"}, {"category", "category1"}}, + .matches = {{.op = "match_regex", + .op_value = ".*", + .highlight = "http-put-729d56c3-2c70e12b-2c70e12b", + .args = {{ + .value = "http-put-729d56c3-2c70e12b-2c70e12b", + .address = "_dd.appsec.fp.http.endpoint", + .path = {}, + }}}}}); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 0); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +TEST(TestFingerprintIntegration, Processor) +{ + auto rule = read_json_file("processor.json", base_dir); + ASSERT_TRUE(rule.type != DDWAF_OBJ_INVALID); + ddwaf_handle handle = ddwaf_init(&rule, nullptr, nullptr); + ASSERT_NE(handle, nullptr); + ddwaf_object_free(&rule); + + uint32_t size; + const char *const *addresses = ddwaf_known_addresses(handle, &size); + EXPECT_EQ(size, 6); + std::unordered_set address_set(addresses, addresses + size); + EXPECT_TRUE(address_set.contains("server.request.body")); + EXPECT_TRUE(address_set.contains("server.request.uri.raw")); + EXPECT_TRUE(address_set.contains("server.request.method")); + EXPECT_TRUE(address_set.contains("server.request.query")); + EXPECT_TRUE(address_set.contains("waf.context.processor")); + EXPECT_TRUE(address_set.contains("_dd.appsec.fp.http.endpoint")); + + ddwaf_context context = ddwaf_context_init(handle); + ASSERT_NE(context, nullptr); + + ddwaf_object tmp; + + ddwaf_object map = DDWAF_OBJECT_MAP; + ddwaf_object settings = DDWAF_OBJECT_MAP; + + ddwaf_object body = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&body, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.body", &body); + + ddwaf_object query = DDWAF_OBJECT_MAP; + ddwaf_object_map_add(&query, "key", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&map, "server.request.query", &query); + + ddwaf_object_map_add( + &map, "server.request.uri.raw", ddwaf_object_string(&tmp, "/path/to/resource/?key=")); + ddwaf_object_map_add(&map, "server.request.method", ddwaf_object_string(&tmp, "PuT")); + + ddwaf_object_map_add(&settings, "fingerprint", ddwaf_object_bool(&tmp, true)); + ddwaf_object_map_add(&map, "waf.context.processor", &settings); + + ddwaf_result out; + ASSERT_EQ(ddwaf_run(context, &map, nullptr, &out, LONG_TIME), DDWAF_MATCH); + EXPECT_FALSE(out.timeout); + + EXPECT_EVENTS(out, {.id = "rule1", + .name = "rule1", + .tags = {{"type", "flow1"}, {"category", "category1"}}, + .matches = {{.op = "match_regex", + .op_value = ".*", + .highlight = "http-put-729d56c3-2c70e12b-2c70e12b", + .args = {{ + .value = "http-put-729d56c3-2c70e12b-2c70e12b", + .address = "_dd.appsec.fp.http.endpoint", + .path = {}, + }}}}}); + + EXPECT_EQ(ddwaf_object_size(&out.derivatives), 1); + + auto json = test::object_to_json(out.derivatives); + EXPECT_STR(json, R"({"_dd.appsec.fp.http.endpoint":"http-put-729d56c3-2c70e12b-2c70e12b"})"); + + ddwaf_result_free(&out); + ddwaf_context_destroy(context); + ddwaf_destroy(handle); +} + +} // namespace diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index 1c4be8399..26ad7d20e 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -41,7 +41,7 @@ TEST(TestHttpEndpointFingerprint, Basic) EXPECT_EQ(attr, object_store::attribute::none); std::string_view output_sv{output.stringValue, output.nbEntries}; - EXPECT_STRV(output_sv, "http-get-b8dfda97-0ac3796a-9798c0e4"); + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); ddwaf_object_free(&query); ddwaf_object_free(&body); @@ -72,7 +72,7 @@ TEST(TestHttpEndpointFingerprint, EmptyQuery) EXPECT_EQ(attr, object_store::attribute::none); std::string_view output_sv{output.stringValue, output.nbEntries}; - EXPECT_STRV(output_sv, "http-get-b8dfda97--9798c0e4"); + EXPECT_STRV(output_sv, "http-get-0ede9e60--9798c0e4"); ddwaf_object_free(&query); ddwaf_object_free(&body); @@ -102,7 +102,7 @@ TEST(TestHttpEndpointFingerprint, EmptyBody) EXPECT_EQ(attr, object_store::attribute::none); std::string_view output_sv{output.stringValue, output.nbEntries}; - EXPECT_STRV(output_sv, "http-get-b8dfda97-0ac3796a-"); + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-"); ddwaf_object_free(&query); ddwaf_object_free(&body); @@ -160,7 +160,7 @@ TEST(TestHttpEndpointFingerprint, KeyConsistency) EXPECT_EQ(attr, object_store::attribute::none); std::string_view output_sv{output.stringValue, output.nbEntries}; - EXPECT_STRV(output_sv, "http-get-b8dfda97-ced401fa-ff07216e"); + EXPECT_STRV(output_sv, "http-get-0ede9e60-ced401fa-ff07216e"); ddwaf_object_free(&query); ddwaf_object_free(&body); From 89e2c97dfb19501fe5bd70e1fa08577ff41ab597 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:56:44 +0100 Subject: [PATCH 12/20] Fix test for 32-bit --- tests/processor/fingerprint_test.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index 26ad7d20e..cabbd9b3f 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -40,7 +40,7 @@ TEST(TestHttpEndpointFingerprint, Basic) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, output.nbEntries}; + std::string_view output_sv{output.stringValue, static_cast(static_cast(output.nbEntries))}; EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); ddwaf_object_free(&query); @@ -71,7 +71,7 @@ TEST(TestHttpEndpointFingerprint, EmptyQuery) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, output.nbEntries}; + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; EXPECT_STRV(output_sv, "http-get-0ede9e60--9798c0e4"); ddwaf_object_free(&query); @@ -101,7 +101,7 @@ TEST(TestHttpEndpointFingerprint, EmptyBody) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, output.nbEntries}; + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-"); ddwaf_object_free(&query); @@ -125,7 +125,7 @@ TEST(TestHttpEndpointFingerprint, EmptyEverything) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, output.nbEntries}; + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; EXPECT_STRV(output_sv, "http----"); ddwaf_object_free(&query); @@ -159,7 +159,7 @@ TEST(TestHttpEndpointFingerprint, KeyConsistency) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, output.nbEntries}; + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; EXPECT_STRV(output_sv, "http-get-0ede9e60-ced401fa-ff07216e"); ddwaf_object_free(&query); From 1db715ff619b6d6f33593de2cd31997093912466 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:00:12 +0100 Subject: [PATCH 13/20] Lint --- tests/processor/fingerprint_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index cabbd9b3f..3ada35b41 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -40,7 +40,8 @@ TEST(TestHttpEndpointFingerprint, Basic) EXPECT_EQ(output.type, DDWAF_OBJ_STRING); EXPECT_EQ(attr, object_store::attribute::none); - std::string_view output_sv{output.stringValue, static_cast(static_cast(output.nbEntries))}; + std::string_view output_sv{ + output.stringValue, static_cast(static_cast(output.nbEntries))}; EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); ddwaf_object_free(&query); From f2554ce92f0c2d5717814cc0aac5f81a194c27ce Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:15:05 +0100 Subject: [PATCH 14/20] More tests --- tests/processor/fingerprint_test.cpp | 166 +++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index 3ada35b41..42aec2adf 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -168,4 +168,170 @@ TEST(TestHttpEndpointFingerprint, KeyConsistency) ddwaf_object_free(&output); } +TEST(TestHttpEndpointFingerprint, InvalidQueryType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_array(&query); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "Key1")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "key,3")); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60--9798c0e4"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, InvalidBodyType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_array(&body); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY1")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "3")); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, InvalidQueryAndBodyType) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_array(&query); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "Key1")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&query, ddwaf_object_string(&tmp, "key,3")); + + ddwaf_object body; + ddwaf_object_array(&body); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY1")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY2")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "KEY")); + ddwaf_object_array_add(&body, ddwaf_object_string(&tmp, "3")); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, static_cast(output.nbEntries)}; + EXPECT_STRV(output_sv, "http-get-0ede9e60--"); + + ddwaf_object_free(&query); + ddwaf_object_free(&body); + ddwaf_object_free(&output); +} + +TEST(TestHttpEndpointFingerprint, UriRawConsistency) +{ + ddwaf_object tmp; + + ddwaf_object query; + ddwaf_object_map(&query); + ddwaf_object_map_add(&query, "Key1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&query, "key,3", ddwaf_object_invalid(&tmp)); + + ddwaf_object body; + ddwaf_object_map(&body); + ddwaf_object_map_add(&body, "KEY1", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY2", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "KEY", ddwaf_object_invalid(&tmp)); + ddwaf_object_map_add(&body, "3", ddwaf_object_invalid(&tmp)); + + http_endpoint_fingerprint gen{"id", {}, {}, false, true}; + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever?param=hello"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/PaTh/To/WhAtEVER"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + ddwaf_object_free(&query); + ddwaf_object_free(&body); +} + } // namespace From 31d3e7ac574efbdd9dcde919a06349aba4caf63d Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:58:32 +0100 Subject: [PATCH 15/20] Reorganize code --- src/processor/fingerprint.cpp | 170 +++++++++++++++++++++++++++++++++- src/processor/fingerprint.hpp | 164 -------------------------------- 2 files changed, 165 insertions(+), 169 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index e65b44a61..09a175fda 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -9,7 +9,168 @@ #include "transformer/lowercase.hpp" namespace ddwaf { -namespace fingerprint { +namespace { + +struct string_buffer { + explicit string_buffer(std::size_t length) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) + { + if (buffer == nullptr) { + throw std::bad_alloc{}; + } + } + + char &operator[](std::size_t idx) const { return buffer[idx]; } + + template [[nodiscard]] std::span subspan() + { + std::span res{&buffer[index], N}; + index += N; + return res; + } + + void append(std::string_view str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + void append_lowercase(std::string_view str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + template void append(std::array str) + { + memcpy(&buffer[index], str.data(), str.size()); + index += str.size(); + } + + template void append_lowercase(std::array str) + { + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + } + + void append(char c) { buffer[index++] = c; } + + std::pair move() + { + auto *ptr = buffer; + buffer = nullptr; + return {ptr, index}; + } + + char *buffer{nullptr}; + std::size_t index{0}; + std::size_t length; +}; + +struct field_generator { + field_generator() = default; + virtual ~field_generator() = default; + field_generator(const field_generator &) = default; + field_generator(field_generator &&) = default; + field_generator &operator=(const field_generator &) = default; + field_generator &operator=(field_generator &&) = default; + + virtual std::size_t length() = 0; + virtual void operator()(string_buffer &output) = 0; +}; + +struct string_field : field_generator { + explicit string_field(std::string_view input) : value(input) {} + ~string_field() override = default; + string_field(const string_field &) = default; + string_field(string_field &&) = default; + string_field &operator=(const string_field &) = default; + string_field &operator=(string_field &&) = default; + + std::size_t length() override { return value.size(); } + void operator()(string_buffer &output) override { output.append_lowercase(value); } + + std::string_view value; +}; + +struct string_hash_field : field_generator { + explicit string_hash_field(std::string_view input) : value(input) {} + ~string_hash_field() override = default; + string_hash_field(const string_hash_field &) = default; + string_hash_field(string_hash_field &&) = default; + string_hash_field &operator=(const string_hash_field &) = default; + string_hash_field &operator=(string_hash_field &&) = default; + + std::size_t length() override { return 8; } + void operator()(string_buffer &output) override; + + std::string_view value; +}; + +struct key_hash_field : field_generator { + explicit key_hash_field(const ddwaf_object &input) : value(input) {} + ~key_hash_field() override = default; + key_hash_field(const key_hash_field &) = default; + key_hash_field(key_hash_field &&) = default; + key_hash_field &operator=(const key_hash_field &) = default; + key_hash_field &operator=(key_hash_field &&) = default; + + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } + void operator()(string_buffer &output) override; + + ddwaf_object value; +}; + +struct value_hash_field : field_generator { + explicit value_hash_field(const ddwaf_object &input) : value(input) {} + ~value_hash_field() override = default; + value_hash_field(const value_hash_field &) = default; + value_hash_field(value_hash_field &&) = default; + value_hash_field &operator=(const value_hash_field &) = default; + value_hash_field &operator=(value_hash_field &&) = default; + + std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } + void operator()(string_buffer &output) override; + + ddwaf_object value; +}; + +template +std::size_t generate_fragment_length(T &generator, Rest... rest) +{ + if constexpr (sizeof...(rest) > 0) { + return generator.length() + 1 + generate_fragment_length(rest...); + } else { + return generator.length(); + } +} + +template +void generate_fragment_field(string_buffer &buffer, T &generator, Rest... rest) +{ + generator(buffer); + if constexpr (sizeof...(rest) > 0) { + buffer.append('-'); + generate_fragment_field(buffer, rest...); + } +} + +template +ddwaf_object generate_fragment(std::string_view header, Args... generators) +{ + std::size_t total_length = header.size() + 1 + generate_fragment_length(generators...); + + string_buffer buffer{total_length}; + buffer.append_lowercase(header); + buffer.append('-'); + + generate_fragment_field(buffer, generators...); + + ddwaf_object res; + auto [ptr, size] = buffer.move(); + ddwaf_object_stringl_nc(&res, ptr, size); + + return res; +} // Retur true if the first argument is less than (i.e. is ordered before) the second bool str_casei_cmp(std::string_view left, std::string_view right) @@ -94,7 +255,7 @@ void key_hash_field::operator()(string_buffer &output) hasher.write_digest(output.subspan<8>()); } -} // namespace fingerprint +} // namespace // NOLINTNEXTLINE(readability-convert-member-functions-to-static) std::pair http_endpoint_fingerprint::eval_impl( @@ -113,9 +274,8 @@ std::pair http_endpoint_fingerprint::eval stripped_uri = stripped_uri.substr(0, query_idx); } - auto res = fingerprint::generate_fragment("http", fingerprint::string_field{method.value}, - fingerprint::string_hash_field{stripped_uri}, fingerprint::key_hash_field{*query.value}, - fingerprint::key_hash_field{*body.value}); + auto res = generate_fragment("http", string_field{method.value}, + string_hash_field{stripped_uri}, key_hash_field{*query.value}, key_hash_field{*body.value}); return {res, object_store::attribute::none}; } diff --git a/src/processor/fingerprint.hpp b/src/processor/fingerprint.hpp index 04e8a92b4..c6fb823bc 100644 --- a/src/processor/fingerprint.hpp +++ b/src/processor/fingerprint.hpp @@ -14,170 +14,6 @@ #include "utils.hpp" namespace ddwaf { -namespace fingerprint { - -struct string_buffer { - explicit string_buffer(std::size_t length) - // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) - : buffer(reinterpret_cast(malloc(sizeof(char) * length))), length(length) - { - if (buffer == nullptr) { - throw std::bad_alloc{}; - } - } - - char &operator[](std::size_t idx) const { return buffer[idx]; } - - template [[nodiscard]] std::span subspan() - { - std::span res{&buffer[index], N}; - index += N; - return res; - } - - void append(std::string_view str) - { - memcpy(&buffer[index], str.data(), str.size()); - index += str.size(); - } - - void append_lowercase(std::string_view str) - { - for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } - } - - template void append(std::array str) - { - memcpy(&buffer[index], str.data(), str.size()); - index += str.size(); - } - - template void append_lowercase(std::array str) - { - for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } - } - - void append(char c) { buffer[index++] = c; } - - std::pair move() - { - auto *ptr = buffer; - buffer = nullptr; - return {ptr, index}; - } - - char *buffer{nullptr}; - std::size_t index{0}; - std::size_t length; -}; - -struct field_generator { - field_generator() = default; - virtual ~field_generator() = default; - field_generator(const field_generator &) = default; - field_generator(field_generator &&) = default; - field_generator &operator=(const field_generator &) = default; - field_generator &operator=(field_generator &&) = default; - - virtual std::size_t length() = 0; - virtual void operator()(string_buffer &output) = 0; -}; - -struct string_field : field_generator { - explicit string_field(std::string_view input) : value(input) {} - ~string_field() override = default; - string_field(const string_field &) = default; - string_field(string_field &&) = default; - string_field &operator=(const string_field &) = default; - string_field &operator=(string_field &&) = default; - - std::size_t length() override { return value.size(); } - void operator()(string_buffer &output) override { output.append_lowercase(value); } - - std::string_view value; -}; - -struct string_hash_field : field_generator { - explicit string_hash_field(std::string_view input) : value(input) {} - ~string_hash_field() override = default; - string_hash_field(const string_hash_field &) = default; - string_hash_field(string_hash_field &&) = default; - string_hash_field &operator=(const string_hash_field &) = default; - string_hash_field &operator=(string_hash_field &&) = default; - - std::size_t length() override { return 8; } - void operator()(string_buffer &output) override; - - std::string_view value; -}; - -struct key_hash_field : field_generator { - explicit key_hash_field(const ddwaf_object &input) : value(input) {} - ~key_hash_field() override = default; - key_hash_field(const key_hash_field &) = default; - key_hash_field(key_hash_field &&) = default; - key_hash_field &operator=(const key_hash_field &) = default; - key_hash_field &operator=(key_hash_field &&) = default; - - std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } - void operator()(string_buffer &output) override; - - ddwaf_object value; -}; - -struct value_hash_field : field_generator { - explicit value_hash_field(const ddwaf_object &input) : value(input) {} - ~value_hash_field() override = default; - value_hash_field(const value_hash_field &) = default; - value_hash_field(value_hash_field &&) = default; - value_hash_field &operator=(const value_hash_field &) = default; - value_hash_field &operator=(value_hash_field &&) = default; - - std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } - void operator()(string_buffer &output) override; - - ddwaf_object value; -}; - -template -std::size_t generate_fragment_length(T &generator, Rest... rest) -{ - if constexpr (sizeof...(rest) > 0) { - return generator.length() + 1 + generate_fragment_length(rest...); - } else { - return generator.length(); - } -} - -template -void generate_fragment_field(string_buffer &buffer, T &generator, Rest... rest) -{ - generator(buffer); - if constexpr (sizeof...(rest) > 0) { - buffer.append('-'); - generate_fragment_field(buffer, rest...); - } -} - -template -ddwaf_object generate_fragment(std::string_view header, Args... generators) -{ - std::size_t total_length = header.size() + 1 + generate_fragment_length(generators...); - - string_buffer buffer{total_length}; - buffer.append_lowercase(header); - buffer.append('-'); - - generate_fragment_field(buffer, generators...); - - ddwaf_object res; - auto [ptr, size] = buffer.move(); - ddwaf_object_stringl_nc(&res, ptr, size); - - return res; -} - -} // namespace fingerprint class http_endpoint_fingerprint : public structured_processor { public: From 21afc1c784dc25928ac0621ce060b5cb6315c023 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:33:59 +0100 Subject: [PATCH 16/20] Add destructor to string_buffer --- src/processor/fingerprint.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 09a175fda..19c7b1935 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -21,6 +21,15 @@ struct string_buffer { } } + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,hicpp-no-malloc,cppcoreguidelines-pro-type-reinterpret-cast) + ~string_buffer() { free(buffer); } + + string_buffer(const string_buffer &) = delete; + string_buffer(string_buffer &&) = delete; + + string_buffer &operator=(const string_buffer &) = delete; + string_buffer &operator=(string_buffer &&) = delete; + char &operator[](std::size_t idx) const { return buffer[idx]; } template [[nodiscard]] std::span subspan() @@ -120,20 +129,6 @@ struct key_hash_field : field_generator { ddwaf_object value; }; -struct value_hash_field : field_generator { - explicit value_hash_field(const ddwaf_object &input) : value(input) {} - ~value_hash_field() override = default; - value_hash_field(const value_hash_field &) = default; - value_hash_field(value_hash_field &&) = default; - value_hash_field &operator=(const value_hash_field &) = default; - value_hash_field &operator=(value_hash_field &&) = default; - - std::size_t length() override { return value.type == DDWAF_OBJ_MAP ? 8 : 0; } - void operator()(string_buffer &output) override; - - ddwaf_object value; -}; - template std::size_t generate_fragment_length(T &generator, Rest... rest) { From cd315a45013c9b9d5b518d04d15208e8f5b9e12f Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:02:33 +0100 Subject: [PATCH 17/20] Address review comments --- src/processor/fingerprint.cpp | 4 +--- src/sha256.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 19c7b1935..36d2adf9d 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -167,7 +167,7 @@ ddwaf_object generate_fragment(std::string_view header, Args... generators) return res; } -// Retur true if the first argument is less than (i.e. is ordered before) the second +// Return true if the first argument is less than (i.e. is ordered before) the second bool str_casei_cmp(std::string_view left, std::string_view right) { auto n = std::min(left.size(), right.size()); @@ -183,8 +183,6 @@ bool str_casei_cmp(std::string_view left, std::string_view right) void normalize_string(std::string_view key, std::string &buffer, bool trailing_separator) { - // Clear should not deallocate... - // TODO: verify buffer.clear(); if (buffer.capacity() < key.size()) { diff --git a/src/sha256.cpp b/src/sha256.cpp index afc02e0e2..cabc8bb51 100644 --- a/src/sha256.cpp +++ b/src/sha256.cpp @@ -145,22 +145,22 @@ sha256_hash &sha256_hash::operator<<(std::string_view str) return *this; } -template +template std::string sha256_hash::digest() - requires(N % 8 == 0 && N <= 64) + requires(DigestLength % 8 == 0 && DigestLength <= 64) { std::string final_digest; - final_digest.resize(N, 0); - write_digest(std::span{final_digest}); + final_digest.resize(DigestLength, 0); + write_digest(std::span{final_digest}); return final_digest; } template std::string sha256_hash::digest<64>(); template std::string sha256_hash::digest<8>(); -template -void sha256_hash::write_digest(std::span output) - requires(N % 8 == 0 && N <= 64) +template +void sha256_hash::write_digest(std::span output) + requires(DigestLength % 8 == 0 && DigestLength <= 64) { auto *p = buffer.data(); size_t n = num; @@ -193,7 +193,7 @@ void sha256_hash::write_digest(std::span output) num = 0; memset(p, 0, block_size); - for (unsigned int nn = 0; nn < N; nn += 8) { + for (unsigned int nn = 0; nn < DigestLength; nn += 8) { uint32_t ll = hash[nn >> 3]; output[nn + 0] = UINT8_TO_HEX_CHAR(static_cast((ll >> 28) & 0x0f)); output[nn + 1] = UINT8_TO_HEX_CHAR(static_cast((ll >> 24) & 0x0f)); From 6dc09f5f65f46d8d357d5934e48bc883b049a7d0 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:23:57 +0100 Subject: [PATCH 18/20] Strip fragment as well --- src/processor/fingerprint.cpp | 6 +++--- tests/processor/fingerprint_test.cpp | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 36d2adf9d..81e02f1a9 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -262,9 +262,9 @@ std::pair http_endpoint_fingerprint::eval // Strip query parameter from raw URI auto stripped_uri = uri_raw.value; - auto query_idx = stripped_uri.find_first_of('?'); - if (query_idx != std::string_view::npos) { - stripped_uri = stripped_uri.substr(0, query_idx); + auto query_or_frag_idx = stripped_uri.find_first_of("?#"); + if (query_or_frag_idx != std::string_view::npos) { + stripped_uri = stripped_uri.substr(0, query_or_frag_idx); } auto res = generate_fragment("http", string_field{method.value}, diff --git a/tests/processor/fingerprint_test.cpp b/tests/processor/fingerprint_test.cpp index 42aec2adf..a22d20c28 100644 --- a/tests/processor/fingerprint_test.cpp +++ b/tests/processor/fingerprint_test.cpp @@ -302,6 +302,34 @@ TEST(TestHttpEndpointFingerprint, UriRawConsistency) ddwaf_object_free(&output); } + { + ddwaf::timer deadline{2s}; + auto [output, attr] = + gen.eval_impl({{}, {}, false, "GET"}, {{}, {}, false, "/path/to/whatever#fragment"}, + {{}, {}, false, &query}, {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + + { + ddwaf::timer deadline{2s}; + auto [output, attr] = gen.eval_impl({{}, {}, false, "GET"}, + {{}, {}, false, "/path/to/whatever?param=hello#fragment"}, {{}, {}, false, &query}, + {{}, {}, false, &body}, deadline); + EXPECT_EQ(output.type, DDWAF_OBJ_STRING); + EXPECT_EQ(attr, object_store::attribute::none); + + std::string_view output_sv{output.stringValue, + static_cast(static_cast(output.nbEntries))}; + EXPECT_STRV(output_sv, "http-get-0ede9e60-0ac3796a-9798c0e4"); + ddwaf_object_free(&output); + } + { ddwaf::timer deadline{2s}; auto [output, attr] = From 189cb8c6c5563837cb8a53d1d31e9fcbd087f809 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 15 Jul 2024 20:15:03 +0100 Subject: [PATCH 19/20] Bounds checks on string buffer --- src/processor/fingerprint.cpp | 49 ++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 81e02f1a9..9395afc3f 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -7,6 +7,7 @@ #include "processor/fingerprint.hpp" #include "sha256.hpp" #include "transformer/lowercase.hpp" +#include namespace ddwaf { namespace { @@ -32,8 +33,14 @@ struct string_buffer { char &operator[](std::size_t idx) const { return buffer[idx]; } - template [[nodiscard]] std::span subspan() + template + [[nodiscard]] std::span subspan() + requires(N > 0) { + if ((index + N - 1) >= length) { + throw std::out_of_range("span[index, N) beyond buffer limit"); + } + std::span res{&buffer[index], N}; index += N; return res; @@ -41,27 +48,45 @@ struct string_buffer { void append(std::string_view str) { + if (str.empty()) { + return; + } + + if ((index + str.length() - 1) >= length) { + throw std::out_of_range("appending string beyond buffer limit"); + } memcpy(&buffer[index], str.data(), str.size()); index += str.size(); } void append_lowercase(std::string_view str) { + if (str.empty()) { + return; + } + + if ((index + str.length() - 1) >= length) { + throw std::out_of_range("appending string beyond buffer limit"); + } + for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } } - template void append(std::array str) + template + void append(std::array str) + requires(N > 0) { - memcpy(&buffer[index], str.data(), str.size()); - index += str.size(); + append(std::string_view{str.data(), N}); } - template void append_lowercase(std::array str) + template + void append_lowercase(std::array str) + requires(N > 0) { - for (auto c : str) { buffer[index++] = ddwaf::tolower(c); } + append_lowercase(std::string_view{str.data(), N}); } - void append(char c) { buffer[index++] = c; } + void append(char c) { append(std::string_view{&c, 1}); } std::pair move() { @@ -267,8 +292,14 @@ std::pair http_endpoint_fingerprint::eval stripped_uri = stripped_uri.substr(0, query_or_frag_idx); } - auto res = generate_fragment("http", string_field{method.value}, - string_hash_field{stripped_uri}, key_hash_field{*query.value}, key_hash_field{*body.value}); + ddwaf_object res; + ddwaf_object_invalid(&res); + try { + res = generate_fragment("http", string_field{method.value}, string_hash_field{stripped_uri}, + key_hash_field{*query.value}, key_hash_field{*body.value}); + } catch (const std::out_of_range &e) { + DDWAF_WARN("Failed to generate http endpoint fingerprint: {}", e.what()); + } return {res, object_store::attribute::none}; } From 82078612ef99d78c7157f412676d94301d93d143 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:08:53 +0100 Subject: [PATCH 20/20] Address review comments --- src/parser/processor_parser.cpp | 12 ++++++------ src/processor/fingerprint.cpp | 16 +++++----------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/parser/processor_parser.cpp b/src/parser/processor_parser.cpp index e95776734..8b9a9a45e 100644 --- a/src/parser/processor_parser.cpp +++ b/src/parser/processor_parser.cpp @@ -15,9 +15,8 @@ namespace ddwaf::parser::v2 { namespace { -template std::vector parse_processor_mappings( - const parameter::vector &root, address_container &addresses) + const parameter::vector &root, address_container &addresses, const auto ¶m_names) { if (root.empty()) { throw ddwaf::parsing_error("empty mappings"); @@ -28,7 +27,7 @@ std::vector parse_processor_mappings( auto mapping = static_cast(node); std::vector parameters; - for (const auto ¶m : T::param_names) { + for (const auto ¶m : param_names) { // TODO support n:1 mappings and key paths auto inputs = at(mapping, param); if (inputs.empty()) { @@ -105,10 +104,11 @@ processor_container parse_processors( auto mappings_vec = at(params, "mappings"); std::vector mappings; if (type == processor_type::extract_schema) { - mappings = parse_processor_mappings(mappings_vec, addresses); - } else { mappings = - parse_processor_mappings(mappings_vec, addresses); + parse_processor_mappings(mappings_vec, addresses, extract_schema::param_names); + } else { + mappings = parse_processor_mappings( + mappings_vec, addresses, http_endpoint_fingerprint::param_names); } std::vector scanners; diff --git a/src/processor/fingerprint.cpp b/src/processor/fingerprint.cpp index 9395afc3f..774fc728c 100644 --- a/src/processor/fingerprint.cpp +++ b/src/processor/fingerprint.cpp @@ -31,8 +31,6 @@ struct string_buffer { string_buffer &operator=(const string_buffer &) = delete; string_buffer &operator=(string_buffer &&) = delete; - char &operator[](std::size_t idx) const { return buffer[idx]; } - template [[nodiscard]] std::span subspan() requires(N > 0) @@ -154,14 +152,10 @@ struct key_hash_field : field_generator { ddwaf_object value; }; -template -std::size_t generate_fragment_length(T &generator, Rest... rest) +template std::size_t generate_fragment_length(Generators &...generators) { - if constexpr (sizeof...(rest) > 0) { - return generator.length() + 1 + generate_fragment_length(rest...); - } else { - return generator.length(); - } + static_assert(sizeof...(generators) > 0, "At least one generator is required"); + return (generators.length() + ...) + sizeof...(generators) - 1; } template @@ -174,8 +168,8 @@ void generate_fragment_field(string_buffer &buffer, T &generator, Rest... rest) } } -template -ddwaf_object generate_fragment(std::string_view header, Args... generators) +template +ddwaf_object generate_fragment(std::string_view header, Generators... generators) { std::size_t total_length = header.size() + 1 + generate_fragment_length(generators...);