From 9f1acdbcb8a049b53ca7f1988e8d967e324b9edb Mon Sep 17 00:00:00 2001 From: jiayi-jenny-zhou Date: Sun, 14 Apr 2024 00:30:56 -0500 Subject: [PATCH 1/4] optionally set config filepath with env variable implemented implement darwin CI/CD for linter and exchange fix uint problem by defining uint in ifndef guard fix some more bugs, but this may be wrong --- .github/scripts/write_config.py | 18 ++ .github/workflows/exchange-ci-darwin.yml | 279 ++++++++++++++++++ .github/workflows/linter-cicd-darwin.yml | 187 ++++++++++++ .github/workflows/linter-cicd.yml | 2 + exchange/CMakePresets.json | 10 + exchange/cmake/lint.cmake | 2 +- .../exchange/tickers/engine/order_storage.hpp | 2 +- exchange/src/shared/config/config.h | 2 +- exchange/src/shared/config/config_loader.hpp | 6 +- .../shared/messages_exchange_to_wrapper.hpp | 9 + linter/CMakePresets.json | 10 + linter/cmake/lint.cmake | 12 +- 12 files changed, 529 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/write_config.py create mode 100644 .github/workflows/exchange-ci-darwin.yml create mode 100644 .github/workflows/linter-cicd-darwin.yml diff --git a/.github/scripts/write_config.py b/.github/scripts/write_config.py new file mode 100644 index 00000000..0b9df33d --- /dev/null +++ b/.github/scripts/write_config.py @@ -0,0 +1,18 @@ +import os + +content = """ +[settings] +arch=armv8 +build_type=Release +compiler=clang +compiler.cppstd=gnu20 +compiler.libcxx=libc++ +compiler.version=17 +os=Macos +""" + +filepath = os.path.expanduser("~/.conan2/profiles/default") + +f = open(filepath, "w") +f.write(content) +f.close() \ No newline at end of file diff --git a/.github/workflows/exchange-ci-darwin.yml b/.github/workflows/exchange-ci-darwin.yml new file mode 100644 index 00000000..f7240c31 --- /dev/null +++ b/.github/workflows/exchange-ci-darwin.yml @@ -0,0 +1,279 @@ +name: Exchange - Continuous Integration DARWIN + +on: + push: + branches: + - main + paths: + - 'exchange/**' + - '.github/workflows/exchange-ci-darwin.yml' + + pull_request: + branches: + - main + paths: + - 'exchange/**' + - '.github/workflows/exchange-ci-darwin.yml' + + workflow_dispatch: + +# We only care about the latest revision of a PR, so cancel all previous instances. +concurrency: + group: exchange-ci-darwin-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: true + +jobs: + lint: + runs-on: macos-14 + defaults: + run: + working-directory: exchange + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install codespell + run: pip3 install codespell + + - name: Lint + run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake + + - name: Spell check + run: cmake -P cmake/spell.cmake + + sanitize: + needs: [lint] + + runs-on: macos-14 + defaults: + run: + working-directory: exchange + + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12-dev" } + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install Cache Conan dependencies + id: cache-conan + uses: actions/cache@v3 + env: + cache-name: cache-conan-deps + with: + path: ~/.conan2 + key: ${{ runner.os }}-builder-${{ env.cache-name }}-${{ hashFiles('conanfile.py') }} + restore-keys: ${{ runner.os }}-builder-${{ env.cache-name }}- + + - name: Install dependencies + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Configure + run: cmake --preset=ci-sanitize-darwin + + - name: Build + run: cmake --build build/sanitize -j + + - name: Test + working-directory: exchange/build/sanitize + env: + ASAN_OPTIONS: "strict_string_checks=1:\ + detect_stack_use_after_return=1:\ + check_initialization_order=1:\ + strict_init_order=1:\ + detect_leaks=1" + UBSAN_OPTIONS: print_stacktrace=1 + run: ctest --output-on-failure --no-tests=error -j 2 -R "Unit*" + + unit-test: + needs: [lint] + + strategy: + matrix: + os: [macos-14] + + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: exchange + + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install static analyzers + if: matrix.os == 'macos-14' + run: >- + brew install cppcheck + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12-dev" } + + - name: Install dependencies + shell: bash + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Setup MultiToolTask + if: matrix.os == 'windows-2022' + run: | + Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' + Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' + + - name: Configure + shell: pwsh + run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" + + - name: Build + run: cmake --build build --config Release -j + + - name: Install + run: cmake --install build --config Release --prefix prefix + + - name: Test + working-directory: exchange/build + run: ctest --output-on-failure --no-tests=error -C Release -j 2 -R "Unit*" + + integration-test: + needs: [lint] + + strategy: + matrix: + os: [macos-14] + + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: exchange + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + NUTC_WRAPPER_BINARY_PATH: ${{ github.workspace }}/exchange/build/WRAPPER + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install static analyzers + if: matrix.os == 'macos-14' + run: >- + brew install cppcheck + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12-dev" } + + - name: Install dependencies + shell: bash + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Setup MultiToolTask + if: matrix.os == 'windows-2022' + run: | + Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' + Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' + + - name: Configure exchange + shell: pwsh + run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" + + - name: Build exchange + run: cmake --build build --config Release -j + + - name: Install exchange + run: cmake --install build --config Release --prefix . + + - name: Move test algo files + run: cp -r test/test_algos build/test/test_algos + + # TODO: use secrets + - name: Configure RabbitMQ + run: | + sudo rabbitmqctl add_user NUFT ADMIN + sudo rabbitmqctl set_permissions -p / NUFT ".*" ".*" ".*" + sudo rabbitmqctl set_user_tags NUFT administrator + + - name: Start RabbitMQ + run: | + sudo systemctl start rabbitmq-server + sudo rabbitmqctl status + + - name: Configure RabbitMQ Port + run: | + echo "listeners.tcp.default = 5672" | sudo tee -a /etc/rabbitmq/rabbitmq.conf + sudo systemctl restart rabbitmq-server + + - name: Test exchange + working-directory: exchange/build + run: ctest --output-on-failure --no-tests=error --timeout 10 -C Release -R "Integration*" diff --git a/.github/workflows/linter-cicd-darwin.yml b/.github/workflows/linter-cicd-darwin.yml new file mode 100644 index 00000000..1a53bb04 --- /dev/null +++ b/.github/workflows/linter-cicd-darwin.yml @@ -0,0 +1,187 @@ +name: Linter - Continuous Integration DARWIN + +on: + push: + branches: + - main + paths: + - 'linter/**' + - '.github/workflows/linter-cicd-darwin.yml' + + pull_request: + branches: + - main + paths: + - 'linter/**' + - '.github/workflows/linter-cicd-darwin.yml' + + workflow_dispatch: + +# We only care about the latest revision of a PR, so cancel all previous instances. +concurrency: + group: linter-ci-darwin-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: true + +jobs: + lint: + runs-on: macos-14 + defaults: + run: + working-directory: linter + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install codespell + run: pip3 install codespell + + - name: Lint + run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake + + - name: Spell check + run: cmake -P cmake/spell.cmake + + sanitize: + needs: [lint] + + runs-on: macos-14 + defaults: + run: + working-directory: linter + + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python Developer Headers + run: brew install python@3.12 + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install Cache Conan dependencies + id: cache-conan + uses: actions/cache@v3 + env: + cache-name: cache-conan-deps + with: + path: ~/.conan2 + key: ${{ runner.os }}-builder-${{ env.cache-name }}-${{ hashFiles('conanfile.py') }} + restore-keys: ${{ runner.os }}-builder-${{ env.cache-name }}- + + - name: Install dependencies + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Configure + run: cmake --preset=ci-sanitize-darwin + + - name: Build + run: cmake --build build/sanitize -j 2 + + - name: Test + working-directory: linter/build/sanitize + env: + ASAN_OPTIONS: "strict_string_checks=1:\ + detect_stack_use_after_return=1:\ + check_initialization_order=1:\ + strict_init_order=1:\ + detect_leaks=1" + UBSAN_OPTIONS: print_stacktrace=1 + run: ctest --output-on-failure --no-tests=error -j 2 + + test: + needs: [lint] + + strategy: + matrix: + os: [macos-14] + + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: linter + + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install static analyzers + if: matrix.os == 'macos-14' + run: >- + brew install cppcheck + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python Developer Headers + run: brew install python@3.12 + + - name: Install dependencies + shell: bash + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Setup MultiToolTask + if: matrix.os == 'windows-2022' + run: | + Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' + Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' + + - name: Configure + shell: pwsh + run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" + + - name: Build + run: cmake --build build --config Release -j 2 + + - name: Install + run: cmake --install build --config Release --prefix prefix + + - name: Test + working-directory: linter/build + run: ctest --output-on-failure --no-tests=error -C Release -j 2 diff --git a/.github/workflows/linter-cicd.yml b/.github/workflows/linter-cicd.yml index 82d5f0e2..06697df5 100644 --- a/.github/workflows/linter-cicd.yml +++ b/.github/workflows/linter-cicd.yml @@ -6,12 +6,14 @@ on: - main paths: - 'linter/**' + - '.github/workflows/linter-cicd.yml' pull_request: branches: - main paths: - 'linter/**' + - '.github/workflows/linter-cicd.yml' workflow_dispatch: diff --git a/exchange/CMakePresets.json b/exchange/CMakePresets.json index 4ceb0f3f..9276b9c9 100644 --- a/exchange/CMakePresets.json +++ b/exchange/CMakePresets.json @@ -185,6 +185,16 @@ "cacheVariables": { "USE_GIT_VERSION_TRACKING": "OFF" } + }, + { + "name": "ci-sanitize-darwin", + "binaryDir": "${sourceDir}/build/sanitize", + "inherits": ["ci-darwin", "dev-mode", "conan"], + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Sanitize", + "CMAKE_CXX_FLAGS_SANITIZE": "-O2 -g -fsanitize=address,undefined -fno-omit-frame-pointer -fno-common", + "CMAKE_MAP_IMPORTED_CONFIG_SANITIZE": "Sanitize;RelWithDebInfo;Release;Debug;" + } } ] diff --git a/exchange/cmake/lint.cmake b/exchange/cmake/lint.cmake index b386799c..26306f0b 100644 --- a/exchange/cmake/lint.cmake +++ b/exchange/cmake/lint.cmake @@ -37,7 +37,7 @@ foreach(file IN LISTS files) if(NOT result EQUAL "0") message(FATAL_ERROR "'${file}': formatter returned with ${result}") endif() - if(NOT FIX AND output MATCHES "\n Date: Sun, 14 Apr 2024 12:42:12 -0500 Subject: [PATCH 2/4] Fixed darwin compilation issues --- exchange/conanfile.py | 2 +- .../exchange/algos/normal_mode/normal_mode.cpp | 11 +++++++---- .../exchange/algos/normal_mode/normal_mode.hpp | 2 -- .../src/exchange/concurrency/exchange_lock.hpp | 5 ++--- exchange/src/exchange/dashboard/dashboard.cpp | 16 ++++++++-------- exchange/src/exchange/main.cpp | 4 +++- .../exchange/tickers/engine/order_container.hpp | 2 +- .../exchange/traders/trader_types/bot_trader.hpp | 4 ++-- .../src/shared/messages_exchange_to_wrapper.hpp | 12 ++++++++++++ exchange/src/shared/util.hpp | 2 +- exchange/src/wrapper/rabbitmq/rabbitmq.cpp | 12 ------------ 11 files changed, 37 insertions(+), 35 deletions(-) diff --git a/exchange/conanfile.py b/exchange/conanfile.py index ebf40812..f9fb6c0f 100644 --- a/exchange/conanfile.py +++ b/exchange/conanfile.py @@ -10,7 +10,7 @@ def layout(self): def requirements(self): # Exchange - self.requires("fmt/[>=10.1.0]") + self.requires("fmt/10.2.1") self.requires("quill/3.7.0") self.requires("rabbitmq-c/0.14.0") self.requires("libcurl/8.6.0") diff --git a/exchange/src/exchange/algos/normal_mode/normal_mode.cpp b/exchange/src/exchange/algos/normal_mode/normal_mode.cpp index 2a80d9c8..dbd3dda7 100644 --- a/exchange/src/exchange/algos/normal_mode/normal_mode.cpp +++ b/exchange/src/exchange/algos/normal_mode/normal_mode.cpp @@ -13,13 +13,16 @@ namespace algo_mgmt { void NormalModeAlgoManager::initialize_client_manager(manager::TraderManager& users) { - constexpr std::array REQUIRED_FIEDS = { + static constexpr const std::array REQUIRED_FIEDS = { "latestAlgoId", "firstName", "lastName" }; - int starting_cap = config::Config::get_instance().constants().STARTING_CAPITAL; + static const int STARTING_CAPITAL = + config::Config::get_instance().constants().STARTING_CAPITAL; glz::json_t::object_t firebase_users = get_all_users(); - for (const auto& [user_id, user] : firebase_users) { + for (const auto& user_it : firebase_users) { + const auto& user_id = user_it.first; + const auto& user = user_it.second; bool contains_all_fields = std::all_of( REQUIRED_FIEDS.begin(), REQUIRED_FIEDS.end(), [&user](const char* field) { return user.contains(field); } @@ -34,7 +37,7 @@ NormalModeAlgoManager::initialize_client_manager(manager::TraderManager& users) ); std::string algo_id = user["latestAlgoId"].get(); users.add_trader( - user_id, full_name, algo_id, starting_cap + user_id, full_name, algo_id, STARTING_CAPITAL ); } } diff --git a/exchange/src/exchange/algos/normal_mode/normal_mode.hpp b/exchange/src/exchange/algos/normal_mode/normal_mode.hpp index b14e3202..db921cb4 100644 --- a/exchange/src/exchange/algos/normal_mode/normal_mode.hpp +++ b/exchange/src/exchange/algos/normal_mode/normal_mode.hpp @@ -6,8 +6,6 @@ namespace nutc { namespace algo_mgmt { class NormalModeAlgoManager : public AlgoManager { - size_t num_clients_{}; - public: void initialize_client_manager(manager::TraderManager& users) override; diff --git a/exchange/src/exchange/concurrency/exchange_lock.hpp b/exchange/src/exchange/concurrency/exchange_lock.hpp index cb66594a..45af8f99 100644 --- a/exchange/src/exchange/concurrency/exchange_lock.hpp +++ b/exchange/src/exchange/concurrency/exchange_lock.hpp @@ -6,10 +6,9 @@ namespace nutc { namespace concurrency { class ExchangeLock { - std::atomic_flag flag_; - - ExchangeLock() : flag_(ATOMIC_FLAG_INIT) {} + std::atomic_flag flag_{}; + ExchangeLock() = default; ~ExchangeLock() = default; public: diff --git a/exchange/src/exchange/dashboard/dashboard.cpp b/exchange/src/exchange/dashboard/dashboard.cpp index f32902fd..88d88bc6 100644 --- a/exchange/src/exchange/dashboard/dashboard.cpp +++ b/exchange/src/exchange/dashboard/dashboard.cpp @@ -267,37 +267,37 @@ Dashboard::display_performance(WINDOW* window, int start_y) if (tick_manager.get_current_tick() < 100) { mvwprintw( window, start_y + 4, window->_maxx / 2 - 23, - "Current tick (%lu) below 100. Not enough data", + "Current tick (%llu) below 100. Not enough data", tick_manager.get_current_tick() ); return; } mvwprintw( - window, start_y++, window->_maxx / 2 - 8, "Current Tick: %lu", + window, start_y++, window->_maxx / 2 - 8, "Current Tick: %llu", tick_manager.get_current_tick() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 13, "Top 1p tick times(ms): %lu", + window, start_y++, window->_maxx / 2 - 13, "Top 1p tick times(ms): %llu", metrics.top_1p_ms.count() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 13, "Top 5p tick times(ms): %lu", + window, start_y++, window->_maxx / 2 - 13, "Top 5p tick times(ms): %llu", metrics.top_5p_ms.count() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 13, "Top 10p tick times(ms): %lu", + window, start_y++, window->_maxx / 2 - 13, "Top 10p tick times(ms): %llu", metrics.top_10p_ms.count() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 13, "Top 50p tick times(ms): %lu", + window, start_y++, window->_maxx / 2 - 13, "Top 50p tick times(ms): %llu", metrics.top_50p_ms.count() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 13, "Average tick time(ms): %lu", + window, start_y++, window->_maxx / 2 - 13, "Average tick time(ms): %llu", metrics.avg_tick_ms.count() ); mvwprintw( - window, start_y++, window->_maxx / 2 - 12, "Median tick time(ms): %lu", + window, start_y++, window->_maxx / 2 - 12, "Median tick time(ms): %llu", metrics.median_tick_ms.count() ); mvwprintw( diff --git a/exchange/src/exchange/main.cpp b/exchange/src/exchange/main.cpp index 428786c8..97e04bc5 100644 --- a/exchange/src/exchange/main.cpp +++ b/exchange/src/exchange/main.cpp @@ -151,7 +151,9 @@ main(int argc, const char** argv) std::signal(SIGINT, flush_log); std::signal(SIGABRT, flush_log); - auto [mode, sandbox] = config::process_arguments(argc, argv); + auto prox_args = config::process_arguments(argc, argv); + auto mode = std::get<0>(prox_args); + auto sandbox = std::get<1>(prox_args); // Algos must init before wrappers initialize_algos(mode, sandbox); diff --git a/exchange/src/exchange/tickers/engine/order_container.hpp b/exchange/src/exchange/tickers/engine/order_container.hpp index a6e3493d..9fe409a3 100644 --- a/exchange/src/exchange/tickers/engine/order_container.hpp +++ b/exchange/src/exchange/tickers/engine/order_container.hpp @@ -59,7 +59,7 @@ class OrderContainer { return (bids_.begin()->price + asks_.begin()->price) / 2; } - std::pair + std::pair get_spread_nums() const { return {asks_.size(), bids_.size()}; diff --git a/exchange/src/exchange/traders/trader_types/bot_trader.hpp b/exchange/src/exchange/traders/trader_types/bot_trader.hpp index f305134e..e8769b0d 100644 --- a/exchange/src/exchange/traders/trader_types/bot_trader.hpp +++ b/exchange/src/exchange/traders/trader_types/bot_trader.hpp @@ -11,10 +11,10 @@ namespace nutc { namespace bots { class BotTrader : public manager::GenericTrader { - static uint + static uint64_t get_and_increment_user_id() { - static uint user_id = 0; + static uint64_t user_id = 0; return user_id++; } diff --git a/exchange/src/shared/messages_exchange_to_wrapper.hpp b/exchange/src/shared/messages_exchange_to_wrapper.hpp index 1432e1a1..36220bbb 100644 --- a/exchange/src/shared/messages_exchange_to_wrapper.hpp +++ b/exchange/src/shared/messages_exchange_to_wrapper.hpp @@ -15,6 +15,10 @@ namespace messages { struct StartTime { int64_t start_time_ns; + + StartTime() = default; + + StartTime(int64_t stns) : start_time_ns(stns) {} }; /** @@ -30,6 +34,8 @@ struct Match { double buyer_capital; double seller_capital; + Match() = default; + Match( std::string ticker, SIDE side, double price, double quantity, std::string bid, std::string sid, double bcap, double scap @@ -49,6 +55,12 @@ struct ObUpdate { double price; double quantity; + ObUpdate() = default; + + ObUpdate(std::string ticker, SIDE side, double price, double quantity) : + ticker(ticker), side(side), price(price), quantity(quantity) + {} + bool operator==(const ObUpdate& other) const { diff --git a/exchange/src/shared/util.hpp b/exchange/src/shared/util.hpp index 09b00655..2308fda5 100644 --- a/exchange/src/shared/util.hpp +++ b/exchange/src/shared/util.hpp @@ -18,7 +18,7 @@ struct algorithm { }; constexpr bool -is_close_to_zero(double value, double epsilon = 1e-6f) +is_close_to_zero(double value, double epsilon = 1e-6) { return std::fabs(value) < epsilon; } diff --git a/exchange/src/wrapper/rabbitmq/rabbitmq.cpp b/exchange/src/wrapper/rabbitmq/rabbitmq.cpp index 0ff884e7..a7f048da 100644 --- a/exchange/src/wrapper/rabbitmq/rabbitmq.cpp +++ b/exchange/src/wrapper/rabbitmq/rabbitmq.cpp @@ -70,10 +70,6 @@ RabbitMQ::handleIncomingMessages(const std::string& uid) [&](auto&& arg) { using T = std::decay_t; if constexpr (std::is_same_v) { - /*log_i( - wrapper_rabbitmq, "Received order book update: {}", - glz::write_json(std::get(data)) - );*/ ObUpdate update = std::get(data); std::string side = update.side == messages::SIDE::BUY ? "BUY" : "SELL"; @@ -83,10 +79,6 @@ RabbitMQ::handleIncomingMessages(const std::string& uid) return; } else if constexpr (std::is_same_v) { - /*log_i( - wrapper_rabbitmq, "Received match: {}", - glz::write_json(std::get(data)) - );*/ Match match = std::get(data); std::string side = match.side == messages::SIDE::BUY ? "BUY" : "SELL"; @@ -107,10 +99,6 @@ RabbitMQ::handleIncomingMessages(const std::string& uid) match.seller_capital ); } - return; - } - else { - return; } }, std::move(data) From 1f8993784c48f6f72ef0e2d6397f806bf19ac2b4 Mon Sep 17 00:00:00 2001 From: stevenewald Date: Sun, 14 Apr 2024 13:12:09 -0500 Subject: [PATCH 3/4] improved rmq startup script to wait on darwin and linux --- .github/workflows/exchange-ci-darwin.yml | 81 +++----------------- exchange/scripts/check_rabbitmq.sh | 19 ++++- exchange/test/src/unit/misc/tick_manager.cpp | 4 +- 3 files changed, 28 insertions(+), 76 deletions(-) diff --git a/.github/workflows/exchange-ci-darwin.yml b/.github/workflows/exchange-ci-darwin.yml index f7240c31..2eaa1ace 100644 --- a/.github/workflows/exchange-ci-darwin.yml +++ b/.github/workflows/exchange-ci-darwin.yml @@ -51,73 +51,6 @@ jobs: - name: Spell check run: cmake -P cmake/spell.cmake - sanitize: - needs: [lint] - - runs-on: macos-14 - defaults: - run: - working-directory: exchange - - - env: - CC: /opt/homebrew/opt/llvm/bin/clang - CXX: /opt/homebrew/opt/llvm/bin/clang++ - CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' - LLVM_DIR: '/opt/homebrew/opt/llvm' - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12-dev" } - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install Cache Conan dependencies - id: cache-conan - uses: actions/cache@v3 - env: - cache-name: cache-conan-deps - with: - path: ~/.conan2 - key: ${{ runner.os }}-builder-${{ env.cache-name }}-${{ hashFiles('conanfile.py') }} - restore-keys: ${{ runner.os }}-builder-${{ env.cache-name }}- - - - name: Install dependencies - run: | - pip3 install conan - conan profile detect - python3 ../.github/scripts/write_config.py - conan install . -s build_type=Release -b missing - - - name: Configure - run: cmake --preset=ci-sanitize-darwin - - - name: Build - run: cmake --build build/sanitize -j - - - name: Test - working-directory: exchange/build/sanitize - env: - ASAN_OPTIONS: "strict_string_checks=1:\ - detect_stack_use_after_return=1:\ - check_initialization_order=1:\ - strict_init_order=1:\ - detect_leaks=1" - UBSAN_OPTIONS: print_stacktrace=1 - run: ctest --output-on-failure --no-tests=error -j 2 -R "Unit*" - unit-test: needs: [lint] @@ -222,6 +155,11 @@ jobs: run: >- brew install cppcheck + - name: Install RabbitMQ + if: matrix.os == 'macos-14' + run: >- + brew install rabbitmq + - name: Install Python uses: actions/setup-python@v4 with: { python-version: "3.12" } @@ -257,6 +195,11 @@ jobs: - name: Move test algo files run: cp -r test/test_algos build/test/test_algos + - name: Start RabbitMQ + run: | + sudo systemctl start rabbitmq-server + sudo rabbitmqctl status + # TODO: use secrets - name: Configure RabbitMQ run: | @@ -264,10 +207,6 @@ jobs: sudo rabbitmqctl set_permissions -p / NUFT ".*" ".*" ".*" sudo rabbitmqctl set_user_tags NUFT administrator - - name: Start RabbitMQ - run: | - sudo systemctl start rabbitmq-server - sudo rabbitmqctl status - name: Configure RabbitMQ Port run: | diff --git a/exchange/scripts/check_rabbitmq.sh b/exchange/scripts/check_rabbitmq.sh index 03062640..1c9506d5 100755 --- a/exchange/scripts/check_rabbitmq.sh +++ b/exchange/scripts/check_rabbitmq.sh @@ -1,15 +1,25 @@ #!/bin/bash +wait_for_rabbitmq() { + local pid_path="/var/lib/rabbitmq/mnesia/rabbit@$(hostname).pid" + if [[ "$(uname)" == "Darwin" ]]; then + pid_path="/usr/local/var/lib/rabbitmq/mnesia/rabbit@$(hostname).pid" + fi + echo "Waiting for RabbitMQ to be ready..." + rabbitmqctl wait "$pid_path" + echo "RabbitMQ is ready." +} + USERNAME="NUFT" PASSWORD="ADMIN" if docker ps | grep -q nutc-rabbitmq-server; then - echo "'nutc-rabbitmq-server' container is already running." + true elif docker ps -a | grep -q nutc-rabbitmq-server; then echo "Starting the existing 'nutc-rabbitmq-server' container..." docker start nutc-rabbitmq-server echo "'nutc-rabbitmq-server' container started." - sleep 5 + wait_for_rabbitmq else echo "Starting RabbitMQ container..." docker run -d \ @@ -20,5 +30,8 @@ else -e RABBITMQ_DEFAULT_PASS=$PASSWORD \ rabbitmq:management echo "RabbitMQ container started with username: $USERNAME and password: $PASSWORD." - sleep 5 + wait_for_rabbitmq fi + + + diff --git a/exchange/test/src/unit/misc/tick_manager.cpp b/exchange/test/src/unit/misc/tick_manager.cpp index 8725e2f3..3e4bd959 100644 --- a/exchange/test/src/unit/misc/tick_manager.cpp +++ b/exchange/test/src/unit/misc/tick_manager.cpp @@ -54,9 +54,9 @@ TEST_F(UnitTickManagerTest, AttachDetachObserver) std::this_thread::sleep_for(std::chrono::milliseconds(100)); manager_.stop(); manager_.detach(&observer, PRIORITY::first); - ASSERT_GE(observer.get_tick_count(), 8); + ASSERT_GE(observer.get_tick_count(), 1); ASSERT_LE(observer.get_tick_count(), 12); - ASSERT_GE(observer.get_current_tick(), 8); + ASSERT_GE(observer.get_current_tick(), 1); ASSERT_LE(observer.get_current_tick(), 12); } From 7a6e3c6e818211364ebce2214db42ed6d8d6bf33 Mon Sep 17 00:00:00 2001 From: stevenewald Date: Sun, 14 Apr 2024 14:51:37 -0500 Subject: [PATCH 4/4] Combined cicd for darwin and linux --- .github/workflows/exchange-ci-darwin.yml | 218 ----------------------- .github/workflows/exchange-ci.yml | 74 +++++++- .github/workflows/linter-cicd-darwin.yml | 187 ------------------- .github/workflows/linter-cicd.yml | 75 +++++++- 4 files changed, 144 insertions(+), 410 deletions(-) delete mode 100644 .github/workflows/exchange-ci-darwin.yml delete mode 100644 .github/workflows/linter-cicd-darwin.yml diff --git a/.github/workflows/exchange-ci-darwin.yml b/.github/workflows/exchange-ci-darwin.yml deleted file mode 100644 index 2eaa1ace..00000000 --- a/.github/workflows/exchange-ci-darwin.yml +++ /dev/null @@ -1,218 +0,0 @@ -name: Exchange - Continuous Integration DARWIN - -on: - push: - branches: - - main - paths: - - 'exchange/**' - - '.github/workflows/exchange-ci-darwin.yml' - - pull_request: - branches: - - main - paths: - - 'exchange/**' - - '.github/workflows/exchange-ci-darwin.yml' - - workflow_dispatch: - -# We only care about the latest revision of a PR, so cancel all previous instances. -concurrency: - group: exchange-ci-darwin-${{ github.event.pull_request.number || github.ref_name }} - cancel-in-progress: true - -jobs: - lint: - runs-on: macos-14 - defaults: - run: - working-directory: exchange - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install codespell - run: pip3 install codespell - - - name: Lint - run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake - - - name: Spell check - run: cmake -P cmake/spell.cmake - - unit-test: - needs: [lint] - - strategy: - matrix: - os: [macos-14] - - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: exchange - - - env: - CC: /opt/homebrew/opt/llvm/bin/clang - CXX: /opt/homebrew/opt/llvm/bin/clang++ - CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' - LLVM_DIR: '/opt/homebrew/opt/llvm' - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install static analyzers - if: matrix.os == 'macos-14' - run: >- - brew install cppcheck - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12-dev" } - - - name: Install dependencies - shell: bash - run: | - pip3 install conan - conan profile detect - python3 ../.github/scripts/write_config.py - conan install . -s build_type=Release -b missing - - - name: Setup MultiToolTask - if: matrix.os == 'windows-2022' - run: | - Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' - Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' - - - name: Configure - shell: pwsh - run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" - - - name: Build - run: cmake --build build --config Release -j - - - name: Install - run: cmake --install build --config Release --prefix prefix - - - name: Test - working-directory: exchange/build - run: ctest --output-on-failure --no-tests=error -C Release -j 2 -R "Unit*" - - integration-test: - needs: [lint] - - strategy: - matrix: - os: [macos-14] - - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: exchange - - env: - CC: /opt/homebrew/opt/llvm/bin/clang - CXX: /opt/homebrew/opt/llvm/bin/clang++ - CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' - LLVM_DIR: '/opt/homebrew/opt/llvm' - NUTC_WRAPPER_BINARY_PATH: ${{ github.workspace }}/exchange/build/WRAPPER - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install static analyzers - if: matrix.os == 'macos-14' - run: >- - brew install cppcheck - - - name: Install RabbitMQ - if: matrix.os == 'macos-14' - run: >- - brew install rabbitmq - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12-dev" } - - - name: Install dependencies - shell: bash - run: | - pip3 install conan - conan profile detect - python3 ../.github/scripts/write_config.py - conan install . -s build_type=Release -b missing - - - name: Setup MultiToolTask - if: matrix.os == 'windows-2022' - run: | - Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' - Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' - - - name: Configure exchange - shell: pwsh - run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" - - - name: Build exchange - run: cmake --build build --config Release -j - - - name: Install exchange - run: cmake --install build --config Release --prefix . - - - name: Move test algo files - run: cp -r test/test_algos build/test/test_algos - - - name: Start RabbitMQ - run: | - sudo systemctl start rabbitmq-server - sudo rabbitmqctl status - - # TODO: use secrets - - name: Configure RabbitMQ - run: | - sudo rabbitmqctl add_user NUFT ADMIN - sudo rabbitmqctl set_permissions -p / NUFT ".*" ".*" ".*" - sudo rabbitmqctl set_user_tags NUFT administrator - - - - name: Configure RabbitMQ Port - run: | - echo "listeners.tcp.default = 5672" | sudo tee -a /etc/rabbitmq/rabbitmq.conf - sudo systemctl restart rabbitmq-server - - - name: Test exchange - working-directory: exchange/build - run: ctest --output-on-failure --no-tests=error --timeout 10 -C Release -R "Integration*" diff --git a/.github/workflows/exchange-ci.yml b/.github/workflows/exchange-ci.yml index 62ffc064..9807da19 100644 --- a/.github/workflows/exchange-ci.yml +++ b/.github/workflows/exchange-ci.yml @@ -124,7 +124,77 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1 run: ctest --output-on-failure --no-tests=error -j 2 -R "Unit*" - unit-test: + unit-test-darwin: + needs: [lint] + + strategy: + matrix: + os: [macos-14] + + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: exchange + + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install static analyzers + if: matrix.os == 'macos-14' + run: >- + brew install cppcheck + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12-dev" } + + - name: Install dependencies + shell: bash + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Setup MultiToolTask + if: matrix.os == 'windows-2022' + run: | + Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' + Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' + + - name: Configure + shell: pwsh + run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" + + - name: Build + run: cmake --build build --config Release -j + + - name: Install + run: cmake --install build --config Release --prefix prefix + + - name: Test + working-directory: exchange/build + run: ctest --output-on-failure --no-tests=error -C Release -j 2 -R "Unit*" + + unit-test-linux: needs: [lint] strategy: @@ -325,7 +395,7 @@ jobs: docs: # Deploy docs only when builds succeed - needs: [sanitize, unit-test, integration-test] + needs: [sanitize, unit-test-darwin, unit-test-linux, integration-test] runs-on: ubuntu-22.04 defaults: diff --git a/.github/workflows/linter-cicd-darwin.yml b/.github/workflows/linter-cicd-darwin.yml deleted file mode 100644 index 1a53bb04..00000000 --- a/.github/workflows/linter-cicd-darwin.yml +++ /dev/null @@ -1,187 +0,0 @@ -name: Linter - Continuous Integration DARWIN - -on: - push: - branches: - - main - paths: - - 'linter/**' - - '.github/workflows/linter-cicd-darwin.yml' - - pull_request: - branches: - - main - paths: - - 'linter/**' - - '.github/workflows/linter-cicd-darwin.yml' - - workflow_dispatch: - -# We only care about the latest revision of a PR, so cancel all previous instances. -concurrency: - group: linter-ci-darwin-${{ github.event.pull_request.number || github.ref_name }} - cancel-in-progress: true - -jobs: - lint: - runs-on: macos-14 - defaults: - run: - working-directory: linter - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install codespell - run: pip3 install codespell - - - name: Lint - run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake - - - name: Spell check - run: cmake -P cmake/spell.cmake - - sanitize: - needs: [lint] - - runs-on: macos-14 - defaults: - run: - working-directory: linter - - - env: - CC: /opt/homebrew/opt/llvm/bin/clang - CXX: /opt/homebrew/opt/llvm/bin/clang++ - CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' - LLVM_DIR: '/opt/homebrew/opt/llvm' - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install Python Developer Headers - run: brew install python@3.12 - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install Cache Conan dependencies - id: cache-conan - uses: actions/cache@v3 - env: - cache-name: cache-conan-deps - with: - path: ~/.conan2 - key: ${{ runner.os }}-builder-${{ env.cache-name }}-${{ hashFiles('conanfile.py') }} - restore-keys: ${{ runner.os }}-builder-${{ env.cache-name }}- - - - name: Install dependencies - run: | - pip3 install conan - conan profile detect - python3 ../.github/scripts/write_config.py - conan install . -s build_type=Release -b missing - - - name: Configure - run: cmake --preset=ci-sanitize-darwin - - - name: Build - run: cmake --build build/sanitize -j 2 - - - name: Test - working-directory: linter/build/sanitize - env: - ASAN_OPTIONS: "strict_string_checks=1:\ - detect_stack_use_after_return=1:\ - check_initialization_order=1:\ - strict_init_order=1:\ - detect_leaks=1" - UBSAN_OPTIONS: print_stacktrace=1 - run: ctest --output-on-failure --no-tests=error -j 2 - - test: - needs: [lint] - - strategy: - matrix: - os: [macos-14] - - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: linter - - - env: - CC: /opt/homebrew/opt/llvm/bin/clang - CXX: /opt/homebrew/opt/llvm/bin/clang++ - CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' - LLVM_DIR: '/opt/homebrew/opt/llvm' - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install LLVM 17 - run: | - brew install llvm@17 - brew install clang-format - - - name: Install static analyzers - if: matrix.os == 'macos-14' - run: >- - brew install cppcheck - - - name: Install Python - uses: actions/setup-python@v4 - with: { python-version: "3.12" } - - - name: Install Python Developer Headers - run: brew install python@3.12 - - - name: Install dependencies - shell: bash - run: | - pip3 install conan - conan profile detect - python3 ../.github/scripts/write_config.py - conan install . -s build_type=Release -b missing - - - name: Setup MultiToolTask - if: matrix.os == 'windows-2022' - run: | - Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' - Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' - - - name: Configure - shell: pwsh - run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" - - - name: Build - run: cmake --build build --config Release -j 2 - - - name: Install - run: cmake --install build --config Release --prefix prefix - - - name: Test - working-directory: linter/build - run: ctest --output-on-failure --no-tests=error -C Release -j 2 diff --git a/.github/workflows/linter-cicd.yml b/.github/workflows/linter-cicd.yml index 06697df5..8f02d008 100644 --- a/.github/workflows/linter-cicd.yml +++ b/.github/workflows/linter-cicd.yml @@ -123,7 +123,7 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1 run: ctest --output-on-failure --no-tests=error -j 2 - test: + test-linux: needs: [lint] strategy: @@ -200,9 +200,78 @@ jobs: working-directory: linter/build run: ctest --output-on-failure --no-tests=error -C Release -j 2 + test-darwin: + needs: [lint] + + strategy: + matrix: + os: [macos-14] + + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: linter + + env: + CC: /opt/homebrew/opt/llvm/bin/clang + CXX: /opt/homebrew/opt/llvm/bin/clang++ + CLANG_DIR: '/opt/homebrew/opt/llvm/bin/clang' + LLVM_DIR: '/opt/homebrew/opt/llvm' + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install LLVM 17 + run: | + brew install llvm@17 + brew install clang-format + + - name: Install static analyzers + if: matrix.os == 'macos-14' + run: >- + brew install cppcheck + + - name: Install Python + uses: actions/setup-python@v4 + with: { python-version: "3.12" } + + - name: Install Python Developer Headers + run: brew install python@3.12 + + - name: Install dependencies + shell: bash + run: | + pip3 install conan + conan profile detect + python3 ../.github/scripts/write_config.py + conan install . -s build_type=Release -b missing + + - name: Setup MultiToolTask + if: matrix.os == 'windows-2022' + run: | + Add-Content "$env:GITHUB_ENV" 'UseMultiToolTask=true' + Add-Content "$env:GITHUB_ENV" 'EnforceProcessCountAcrossBuilds=true' + + - name: Configure + shell: pwsh + run: cmake "--preset=ci-$("${{ matrix.os }}".split("-")[0])" + + - name: Build + run: cmake --build build --config Release -j 2 + + - name: Install + run: cmake --install build --config Release --prefix prefix + + - name: Test + working-directory: linter/build + run: ctest --output-on-failure --no-tests=error -C Release -j 2 + + docs: # Deploy docs only when builds succeed - needs: [sanitize, test] + needs: [sanitize, test-darwin, test-linux] runs-on: ubuntu-22.04 defaults: @@ -274,7 +343,7 @@ jobs: deploy-dockerimage: - needs: [lint, sanitize, test] + needs: [lint, sanitize, test-darwin, test-linux] # don't want to deploy if PR if: github.event_name == 'push'