Generate all objects once

benchmark/main.cpp

@@ -140,52 +140,61 @@ int main(int argc, char *argv[])
 
     auto s = generate_settings(args);
 
-    std::map<std::string, std::vector<test_result>> all_results;
-    for (unsigned i = 0; i < s.runs; ++i) {
-        for (const auto &scenario : s.scenarios) {
-            benchmark::random::seed(s.seed);
-            YAML::Node spec = YAML::Load(utils::read_file(scenario));
+    std::unordered_map<std::string, ddwaf_handle> handles;
+    std::unordered_set<std::string> all_addresses;
+    unsigned max_cycles = 20;
+
+    // Preprocess scenarios
+    for (const auto &scenario : s.scenarios) {
+        benchmark::random::seed(s.seed);
+        YAML::Node spec = YAML::Load(utils::read_file(scenario));
+
+        auto name = spec["scenario"].as<std::string>();
+        auto ruleset = spec["ruleset"].as<ddwaf_object>();
+
+        ddwaf_config cfg{{0, 0, 0}, {nullptr, nullptr}, nullptr};
+        ddwaf_handle handle = ddwaf_init(&ruleset, &cfg, nullptr);
+        ddwaf_object_free(&ruleset);
+        if (handle == nullptr) {
+            std::cerr << "Invalid ruleset file" << std::endl;
+            utils::exit_failure();
+        }
 
-            auto name = spec["scenario"].as<std::string>();
-            auto ruleset = spec["ruleset"].as<ddwaf_object>();
+        handles.emplace(scenario, handle);
 
-            ddwaf_config cfg{{0, 0, 0}, {nullptr, nullptr}, nullptr};
-            ddwaf_handle handle = ddwaf_init(&ruleset, &cfg, nullptr);
-            ddwaf_object_free(&ruleset);
-            if (handle == nullptr) {
-                std::cerr << "Invalid ruleset file" << std::endl;
-                utils::exit_failure();
+        uint32_t addrs_len;
+        const auto *const addrs = ddwaf_known_addresses(handle, &addrs_len);
+        std::vector<std::string_view> addresses{addrs, addrs + static_cast<size_t>(addrs_len)};
+
+        for (auto a : addresses) { all_addresses.emplace(a); }
+
+        if (std::regex_search("random", s.fixtures)) {
+            auto cycle_spec = spec["max_cycles"];
+            if (cycle_spec.IsDefined()) {
+                max_cycles = std::max(max_cycles, cycle_spec.as<unsigned>());
             }
+        }
+    }
 
-            uint32_t addrs_len;
-            const auto *const addrs = ddwaf_known_addresses(handle, &addrs_len);
-            std::vector<std::string_view> addresses{addrs, addrs + static_cast<size_t>(addrs_len)};
+    benchmark::random::seed(s.seed);
+    benchmark::object_generator generator(std::move(all_addresses));
+    auto objects = generator(max_cycles);
 
-            benchmark::runner runner(std::move(name), s);
+    std::map<std::string, std::vector<test_result>> all_results;
+    for (unsigned i = 0; i < s.runs; ++i) {
+        for (const auto &[name, handle] : handles) {
+            benchmark::random::seed(s.seed);
+            benchmark::runner runner(name, s);
 
+            YAML::Node spec = YAML::Load(utils::read_file(name));
             if (std::regex_search("random", s.fixtures)) {
-                unsigned max_cycles = 20;
+                unsigned cycles = 20;
                 auto cycle_spec = spec["max_cycles"];
                 if (cycle_spec.IsDefined()) {
-                    max_cycles = cycle_spec.as<unsigned>();
+                    cycles = cycle_spec.as<unsigned>();
                 }
 
-                benchmark::object_generator generator(addresses);
-                runner.register_fixture<benchmark::run_fixture>(
-                    "random", handle, generator(max_cycles));
-            }
-
-            auto fixtures_spec = spec["fixtures"];
-            if (fixtures_spec.IsDefined()) {
-                for (auto it = fixtures_spec.begin(); it != fixtures_spec.end(); ++it) {
-                    auto custom_name = it->first.as<std::string>();
-                    if (!std::regex_search(custom_name, s.fixtures)) {
-                        continue;
-                    }
-                    std::vector objects{it->second.as<ddwaf_object>()};
-                    runner.register_fixture<benchmark::run_fixture>(
-                        custom_name, handle, std::move(objects));
-                }
+                runner.register_fixture<benchmark::run_fixture>("random", handle, objects, cycles);
             }
 
             auto results = runner.run();
@@ -198,12 +207,13 @@ int main(int argc, char *argv[])
                     it->second.emplace_back(std::move(value));
                 }
             }
-
-            ddwaf_destroy(handle);
         }
     }
 
     benchmark::output_results(s, all_results);
 
+    for (auto &o : objects) { ddwaf_object_free(&o); }
+    for (auto &[name, handle] : handles) { ddwaf_destroy(handle); }
+
     return EXIT_SUCCESS;
 }

benchmark/object_generator.cpp

@@ -156,7 +156,7 @@ std::vector<ddwaf_object> object_generator::operator()(unsigned n, object_specif
         ddwaf_object root;
         ddwaf_object_map(&root);
 
-        for (const auto addr : addresses_) {
+        for (const auto &addr : addresses_) {
             ddwaf_object value;
             if (spec.depth == 0) {
                 generate_string_object(value, spec.string_length);

benchmark/object_generator.hpp

@@ -6,10 +6,7 @@
 
 #pragma once
 
-#include <filesystem>
 #include <string>
-#include <string_view>
-#include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
@@ -35,7 +32,7 @@ struct object_specification {
 
 class object_generator {
 public:
-    explicit object_generator(std::vector<std::string_view> addresses)
+    explicit object_generator(std::unordered_set<std::string> addresses)
         : addresses_(std::move(addresses))
     {}
 
@@ -50,7 +47,7 @@ class object_generator {
     std::vector<ddwaf_object> operator()(unsigned n, object_specification spec = {}) const;
 
 protected:
-    std::vector<std::string_view> addresses_;
+    std::unordered_set<std::string> addresses_;
 };
 
 } // namespace ddwaf::benchmark

benchmark/run_fixture.cpp

@@ -14,8 +14,8 @@
 
 namespace ddwaf::benchmark {
 
-run_fixture::run_fixture(ddwaf_handle handle, std::vector<ddwaf_object> &&objects)
-    : objects_(std::move(objects)), handle_(handle)
+run_fixture::run_fixture(ddwaf_handle handle, std::vector<ddwaf_object> &objects, unsigned cycles)
+    : objects_(objects), cycles_(cycles), handle_(handle)
 {}
 
 bool run_fixture::set_up()
@@ -42,7 +42,7 @@ void run_fixture::warmup()
             if (i == current->nbEntries) {
                 object_stack.pop();
             } else {
-                object_stack.push({&current->array[i++], 0});
+                object_stack.emplace(&current->array[i++], 0);
             }
         }
     }
@@ -52,9 +52,10 @@ uint64_t run_fixture::test_main()
 {
     uint64_t total_runtime = 0;
 
-    for (auto &object : objects_) {
+    auto it = objects_.begin();
+    for (unsigned i = 0; i < cycles_ && it != objects_.end(); ++i, ++it) {
         ddwaf_result res;
-        auto code = ddwaf_run(ctx_, nullptr, &object, &res, std::numeric_limits<uint32_t>::max());
+        auto code = ddwaf_run(ctx_, nullptr, &*it, &res, std::numeric_limits<uint32_t>::max());
         if (code < 0) {
             throw std::runtime_error("WAF returned " + std::to_string(code));
         }

benchmark/run_fixture.hpp

@@ -15,11 +15,8 @@ namespace ddwaf::benchmark {
 
 class run_fixture : public fixture_base {
 public:
-    run_fixture(ddwaf_handle handle, std::vector<ddwaf_object> &&objects);
-    ~run_fixture() override
-    {
-        for (auto &o : objects_) { ddwaf_object_free(&o); }
-    }
+    run_fixture(ddwaf_handle handle, std::vector<ddwaf_object> &objects, unsigned cycles);
+    ~run_fixture() override = default;
 
     run_fixture(const run_fixture &) = delete;
     run_fixture &operator=(const run_fixture &) = delete;
@@ -35,7 +32,8 @@ class run_fixture : public fixture_base {
     void tear_down() override;
 
 protected:
-    std::vector<ddwaf_object> objects_;
+    std::vector<ddwaf_object> &objects_;
+    unsigned cycles_;
     ddwaf_handle handle_{nullptr};
     ddwaf_context ctx_{nullptr};
 };