diff --git a/.github/scripts/conan-profile.sh b/.github/scripts/conan-profile.sh index e8b4be85..1df327dd 100644 --- a/.github/scripts/conan-profile.sh +++ b/.github/scripts/conan-profile.sh @@ -12,6 +12,5 @@ profile="$(conan profile path default)" mv "$profile" "${profile}.bak" sed -e 's/^\(compiler\.cppstd=\).\{1,\}$/\1'"$std/" \ - -e 's/^\(compiler\.version=\).\{1,\}$/\1'"$version/" \ "${profile}.bak" > "$profile" rm "${profile}.bak" diff --git a/.github/workflows/exchange-ci.yml b/.github/workflows/exchange-ci.yml index 91b0edb5..6f4384fa 100644 --- a/.github/workflows/exchange-ci.yml +++ b/.github/workflows/exchange-ci.yml @@ -288,7 +288,7 @@ jobs: LLVM_DIR: '/usr/lib/llvm-18/lib/cmake/llvm' NUTC_WRAPPER_BINARY_PATH: ${{ github.workspace }}/exchange/build/WRAPPER NUTC_CPP_TEMPLATE_PATH: ${{ github.workspace }}/exchange/template.cpp - NUTC_SPAWNER_BINARY_PATH: ${{ github.workspace }}/exchange/build/LINTER-spawner + NUTC_LINTER_SPAWNER_BINARY_PATH: ${{ github.workspace }}/exchange/build/LINTER-spawner steps: - uses: actions/checkout@v3 diff --git a/exchange/CMakeLists.txt b/exchange/CMakeLists.txt index 8f8f50e3..2de366f9 100644 --- a/exchange/CMakeLists.txt +++ b/exchange/CMakeLists.txt @@ -242,7 +242,7 @@ target_include_directories( "$" ) -target_compile_features(LINTER_lib PUBLIC cxx_std_20) +target_compile_features(LINTER_lib PUBLIC cxx_std_23) target_link_libraries(LINTER_lib PUBLIC fmt::fmt) target_link_libraries(LINTER_lib PUBLIC quill::quill) @@ -258,7 +258,6 @@ target_include_directories( LINTER_spawner_lib ${warning_guard} PUBLIC "$" - "$" ) target_link_libraries(LINTER_spawner_lib PRIVATE quill::quill) @@ -275,7 +274,7 @@ add_executable(LINTER::exe ALIAS LINTER_exe) set_property(TARGET LINTER_exe PROPERTY OUTPUT_NAME LINTER) -target_compile_features(LINTER_exe PRIVATE cxx_std_20) +target_compile_features(LINTER_exe PRIVATE cxx_std_23) target_link_libraries(LINTER_exe PRIVATE LINTER_lib) target_link_libraries(LINTER_exe PRIVATE fmt::fmt) @@ -295,7 +294,7 @@ add_executable(LINTER_spawner::exe ALIAS LINTER_spawner_exe) set_property(TARGET LINTER_spawner_exe PROPERTY OUTPUT_NAME LINTER_spawner) -target_compile_features(LINTER_spawner_exe PRIVATE cxx_std_20) +target_compile_features(LINTER_spawner_exe PRIVATE cxx_std_23) target_link_libraries(LINTER_spawner_exe PRIVATE LINTER_spawner_lib) target_link_libraries(LINTER_spawner_exe PRIVATE CURL::libcurl) @@ -307,8 +306,8 @@ target_link_libraries(LINTER_spawner_exe PRIVATE Python3::Python) # ---- COMMON ---------- add_library(COMMON_lib OBJECT - src/common/file_operations/file_operations.cpp src/common/firebase/firebase.cpp + src/common/file_operations/file_operations.cpp src/common/util.cpp src/common/types/decimal.cpp src/common/types/algorithm/local_algorithm.cpp @@ -336,11 +335,11 @@ target_link_libraries(COMMON_lib PRIVATE boost::boost) target_link_libraries(EXCHANGE_lib PUBLIC COMMON_lib) target_link_libraries(EXCHANGE_exe PRIVATE COMMON_lib) -target_link_libraries(LINTER_lib PRIVATE COMMON_lib) -target_link_libraries(LINTER_exe PUBLIC COMMON_lib) +target_link_libraries(LINTER_lib PUBLIC COMMON_lib) +target_link_libraries(LINTER_exe PRIVATE COMMON_lib) -target_link_libraries(LINTER_spawner_lib PRIVATE COMMON_lib) -target_link_libraries(LINTER_spawner_exe PUBLIC COMMON_lib) +target_link_libraries(LINTER_spawner_lib PUBLIC COMMON_lib) +target_link_libraries(LINTER_spawner_exe PRIVATE COMMON_lib) target_link_libraries(WRAPPER_lib PUBLIC COMMON_lib) target_link_libraries(WRAPPER_exe PRIVATE COMMON_lib) @@ -358,6 +357,8 @@ if(lto_supported) set_target_properties(COMMON_lib PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) set_target_properties(LINTER_exe PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) set_target_properties(LINTER_lib PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_target_properties(LINTER_spawner_lib PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_target_properties(LINTER_spawner_exe PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) else() message(WARNING "LTO is not supported: ${error}") endif() diff --git a/exchange/docker/dev/grafana_data/grafana.db b/exchange/docker/dev/grafana_data/grafana.db index 820a4bf6..0d3083a0 100644 Binary files a/exchange/docker/dev/grafana_data/grafana.db and b/exchange/docker/dev/grafana_data/grafana.db differ diff --git a/exchange/src/common/compilation/compile_cpp.cpp b/exchange/src/common/compilation/compile_cpp.cpp index a23afc21..2cbb9a54 100644 --- a/exchange/src/common/compilation/compile_cpp.cpp +++ b/exchange/src/common/compilation/compile_cpp.cpp @@ -2,6 +2,8 @@ #include "common/file_operations/file_operations.hpp" +#include + namespace nutc::common { namespace { std::string @@ -19,6 +21,25 @@ get_cpp_template_path() return template_path_env; } + +std::optional +exec_command(const std::string& command) +{ + std::array buffer{}; + std::string result; + + FILE* pipe = popen((command + " 2>&1").c_str(), "r"); + if (pipe == nullptr) { + throw std::runtime_error("popen() failed!"); + } + while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) { + result += buffer.data(); + } + if (pclose(pipe) != 0) { + return result; + } + return std::nullopt; +} } // namespace std::string @@ -33,12 +54,13 @@ compile_cpp(const std::filesystem::path& filepath) filepath.string(), get_cpp_template_path() ); - int result = system(command.c_str()); + std::optional result = exec_command(command); - if (result != 0) { - throw std::runtime_error( - fmt::format("Compilation of {} failed", filepath.string()) - ); + if (result) { + throw std::runtime_error(fmt::format( + "Compilation of {} failed. Compiler output below:\n {}", filepath.string(), + result.value() + )); } return binary_output; } diff --git a/exchange/src/linter/runtime/cpp/cpp_runtime.cpp b/exchange/src/linter/runtime/cpp/cpp_runtime.cpp index fdca5cdb..58f064cd 100644 --- a/exchange/src/linter/runtime/cpp/cpp_runtime.cpp +++ b/exchange/src/linter/runtime/cpp/cpp_runtime.cpp @@ -1,7 +1,6 @@ #include "cpp_runtime.hpp" #include "common/compilation/compile_cpp.hpp" -#include "common/file_operations/file_operations.hpp" #include #include @@ -13,42 +12,8 @@ #include #include -#include #include #include -#include - -namespace { -namespace fs = std::filesystem; - -std::pair -get_temp_file() -{ -#ifdef __APPLE__ - std::string template_path = (fs::temp_directory_path() / "algoXXXXXX").string(); - std::vector writable_template_path( - template_path.begin(), template_path.end() - ); - writable_template_path.push_back('\0'); - int fd = mkstemp(writable_template_path.data()); - if (fd == -1) { - throw std::runtime_error("Failed to get file descriptor for temporary file"); - } - - return {fd, writable_template_path.data()}; - -#else - int memfd = memfd_create("algo", MFD_CLOEXEC); - if (memfd == -1) { - throw std::runtime_error("Failed to create memfd"); - } - - return {memfd, "/proc/self/fd/" + std::to_string(memfd)}; - -#endif -} - -} // namespace namespace nutc::lint { @@ -64,9 +29,9 @@ CppRuntime::CppRuntime( CppRuntime::~CppRuntime() { - // TODO: shoudl not do - dlclose(dl_handle_); - close(fd_); + if (dl_handle_ != nullptr) { + dlclose(dl_handle_); + } } std::optional @@ -80,12 +45,17 @@ CppRuntime::init() algo_file << algo_ << std::flush; algo_file.close(); - std::string compiled_binary_path = common::compile_cpp(temp_file.string()); + // TODO: improve + std::string compiled_binary_path; + try { + compiled_binary_path = common::compile_cpp(temp_file.string()); + } catch (const std::exception& e) { + return fmt::format("[linter] failed to compile C++ code: {}", e.what()); + } dl_handle_ = dlopen(compiled_binary_path.c_str(), RTLD_NOW); if (dl_handle_ == nullptr) { std::string err = dlerror(); - close(fd_); return fmt::format("[linter] failed to dlopen: {}", err); } @@ -100,8 +70,6 @@ CppRuntime::init() if (init_func == nullptr || on_trade_update_func_ == nullptr || on_orderbook_update_func_ == nullptr || on_account_update_func_ == nullptr) { - dlclose(dl_handle_); - close(fd_); return fmt::format("[linter] failed to dynamically load functions"); } strategy_object_ = diff --git a/exchange/src/linter/runtime/cpp/cpp_runtime.hpp b/exchange/src/linter/runtime/cpp/cpp_runtime.hpp index b410a8ce..d7e70f4b 100644 --- a/exchange/src/linter/runtime/cpp/cpp_runtime.hpp +++ b/exchange/src/linter/runtime/cpp/cpp_runtime.hpp @@ -50,7 +50,6 @@ class CppRuntime : public Runtime { OnAccountUpdateFunc on_account_update_func_; Strategy* strategy_object_; - void* dl_handle_; - int fd_; + void* dl_handle_{}; }; } // namespace nutc::lint diff --git a/exchange/src/linter/spawner/main.cpp b/exchange/src/linter/spawner/main.cpp index c6fd695b..5fe04229 100644 --- a/exchange/src/linter/spawner/main.cpp +++ b/exchange/src/linter/spawner/main.cpp @@ -37,11 +37,17 @@ mock_cancel_func(nutc::common::Ticker, std::int64_t) } } // namespace +void +send_response(const std::string& response) +{ + std::cout << nutc::common::base64_encode(response) << std::endl; +} + int main(int argc, char* argv[]) { if (argc < 2) { - std::cout << "[linter] no language provided\n"; + send_response("[linter] no language provided"); return 1; } @@ -65,18 +71,18 @@ main(int argc, char* argv[]) lint_result = nutc::lint::lint(runtime); } else { - std::cout << "[linter] no language provided\n"; + send_response("[linter] no language provided"); return 1; } auto output = glz::write_json(lint_result); if (output) { - std::cout << *output << std::endl; + send_response(output.value()); } else { - std::cout << fmt::format( + send_response(fmt::format( "[linter] ERROR WRITING LINT RESULT: {}", glz::format_error(output.error()) - ) << std::endl; + )); } return 0; diff --git a/exchange/src/linter/spawning/spawning.cpp b/exchange/src/linter/spawning/spawning.cpp index bcb7c02e..00fd1bd0 100644 --- a/exchange/src/linter/spawning/spawning.cpp +++ b/exchange/src/linter/spawning/spawning.cpp @@ -14,11 +14,14 @@ namespace spawning { const std::filesystem::path& LintProcessManager::spawner_binary_path() { + static constexpr auto LINTER_SPAWNER_BINARY_ENV_VAR = + "NUTC_LINTER_SPAWNER_BINARY_PATH"; static const char* spawner_binary_location = - std::getenv("NUTC_LINTER_SPAWNER_BINARY_PATH"); // NOLINT + std::getenv(LINTER_SPAWNER_BINARY_ENV_VAR); // NOLINT if (spawner_binary_location == nullptr) [[unlikely]] { - throw std::runtime_error("NUTC_SPAWNER_BINARY_PATH environment variable not set" - ); + throw std::runtime_error(fmt::format( + "{} environment variable not set", LINTER_SPAWNER_BINARY_ENV_VAR + )); } static const std::filesystem::path spawner_binary_path{spawner_binary_location}; @@ -96,8 +99,10 @@ LintProcessManager::spawn_client(const std::string& algo_code, AlgoLanguage lang if (child->running()) { child->terminate(); } + std::string decoded_message = common::base64_decode(message); - auto error = glz::read_json(res, message); + auto error = + glz::read_json(res, decoded_message); if (error) { res = { false, "Internal server error. Reach out to " diff --git a/exchange/template.cpp b/exchange/template.cpp index a9d02fab..ea9f3b22 100644 --- a/exchange/template.cpp +++ b/exchange/template.cpp @@ -5,6 +5,8 @@ // #include "template.hpp" #include +#include +#include using PlaceMarketOrder = std::function; using PlaceLimitOrder = std::function; diff --git a/exchange/test/CMakeLists.txt b/exchange/test/CMakeLists.txt index 72b0cee5..04ef1f3d 100644 --- a/exchange/test/CMakeLists.txt +++ b/exchange/test/CMakeLists.txt @@ -31,7 +31,8 @@ add_executable(NUTC_tests src/unit/types/decimal.cpp src/integration/tests/basic.cpp - src/integration/tests/linter_test.cpp + src/integration/tests/linter_py_test.cpp + src/integration/tests/linter_cpp_test.cpp src/integration/tests/cancellation.cpp ) diff --git a/exchange/test/src/integration/tests/linter_cpp_test.cpp b/exchange/test/src/integration/tests/linter_cpp_test.cpp new file mode 100644 index 00000000..b046b29a --- /dev/null +++ b/exchange/test/src/integration/tests/linter_cpp_test.cpp @@ -0,0 +1,106 @@ +#include "linter/spawning/spawning.hpp" + +#include + +#include + +class IntegrationLinterCppTest : public ::testing::Test { +protected: + nutc::spawning::LintProcessManager manager; +}; + +const std::string basic_algo = R"(#include +#include +enum class Side { buy = 0, sell = 1 }; +enum class Ticker : std::uint8_t { ETH = 0, BTC = 1, LTC = 2 }; // NOLINT +bool place_market_order(Side side, Ticker ticker, float quantity); +std::int64_t place_limit_order(Side side, Ticker ticker, float quantity, + float price, bool ioc = false); +bool cancel_order(Ticker ticker, std::int64_t order_id); + +class Strategy { +public: + Strategy() { + place_market_order(Side::buy, Ticker::ETH, 1.0); + } + + void on_trade_update(Ticker ticker, Side side, float quantity, float price) {place_limit_order(Side::buy, Ticker::LTC, 1.0, 1.0);} + void on_orderbook_update(Ticker ticker, Side side, float quantity, + float price) {place_market_order(Side::sell, Ticker::BTC, 1.0);} + + void on_account_update(Ticker ticker, Side side, float price, float quantity, + float capital_remaining) {cancel_order(Ticker::BTC, 5);} +}; +)"; + +const std::string syntax_error = R"(#include +#include +enum class Side { buy = 0, sell = 1 }; +enum class Ticker : std::uint8_t { ETH = 0, BTC = 1, LTC = 2 }; // NOLINT +bool place_market_order(Side side, Ticker ticker, float quantity); +std::int64_t place_limit_order(Side side, Ticker ticker, float quantity, + float price, bool ioc = false); +bool cancel_order(Ticker ticker, std::int64_t order_id); + +class Strategy { +public: + Strategy() { + } + + void on_trade_update(Ticker ticker, Side side, float quantity, float price) {} + void on_orderbook_update(Ticker ticker, Side side, float quantity, + float price) { SYNTAX ERROR } + + void on_account_update(Ticker ticker, Side side, float price, float quantity, + float capital_remaining) {} +}; +)"; + +const std::string exception_thrown = R"(#include +#include +#include +enum class Side { buy = 0, sell = 1 }; +enum class Ticker : std::uint8_t { ETH = 0, BTC = 1, LTC = 2 }; // NOLINT +bool place_market_order(Side side, Ticker ticker, float quantity); +std::int64_t place_limit_order(Side side, Ticker ticker, float quantity, + float price, bool ioc = false); +bool cancel_order(Ticker ticker, std::int64_t order_id); + +class Strategy { +public: + Strategy() { + } + + void on_trade_update(Ticker ticker, Side side, float quantity, float price) {} + void on_orderbook_update(Ticker ticker, Side side, float quantity, + float price) { throw std::runtime_error("This is an error"); } + + void on_account_update(Ticker ticker, Side side, float price, float quantity, + float capital_remaining) {} +}; +)"; + +using nutc::spawning::AlgoLanguage; + +TEST_F(IntegrationLinterCppTest, basic) +{ + auto lint_result = manager.spawn_client(basic_algo, AlgoLanguage::Cpp); + std::cout << "first " << lint_result.message << "\n"; + ASSERT_TRUE(lint_result.success); +} + +TEST_F(IntegrationLinterCppTest, SyntaxErrorDetection) +{ + auto lint_result = manager.spawn_client(syntax_error, AlgoLanguage::Cpp); + ASSERT_FALSE(lint_result.success); + EXPECT_TRUE(lint_result.message.contains("‘SYNTAX’ was not declared in this scope") + ); +} + +TEST_F(IntegrationLinterCppTest, RuntimeError) +{ + auto lint_result = manager.spawn_client(exception_thrown, AlgoLanguage::Cpp); + ASSERT_FALSE(lint_result.success); + EXPECT_TRUE(lint_result.message.contains("Failed to run on_orderbook_update")); + EXPECT_TRUE(lint_result.message.contains("This is an error")); +} diff --git a/exchange/test/src/integration/tests/linter_test.cpp b/exchange/test/src/integration/tests/linter_py_test.cpp similarity index 82% rename from exchange/test/src/integration/tests/linter_test.cpp rename to exchange/test/src/integration/tests/linter_py_test.cpp index 11b6d73c..fea18206 100644 --- a/exchange/test/src/integration/tests/linter_test.cpp +++ b/exchange/test/src/integration/tests/linter_py_test.cpp @@ -2,9 +2,7 @@ #include -#include - -class IntegrationLinterTest : public ::testing::Test { +class IntegrationLinterPyTest : public ::testing::Test { protected: nutc::spawning::LintProcessManager manager; }; @@ -32,29 +30,6 @@ const std::string basic_algo = R"(class Strategy: pass )"; -const std::string incorrect_arguments_algo = R"(class Strategy: - def __init__(self) -> None: - pass - - def on_trade_update(self, ticker: Ticker, side: Side, price: float, quantity: float) -> None: - place_limit_order(Side.BUY, Ticker.ETH, 5, 5) - - def on_orderbook_update( - self, ticker: Ticker, side: Side, price: float, quantity: float - ) -> None: - pass - - def on_account_update( - self, - ticker: Ticker, - side: Side, - price: float, - quantity: float, - capital_remaining: float, - ) -> None: - pass -)"; - // TODO: fix/restore const std::string timeout_algo = R"(import time class Strategy: @@ -128,15 +103,14 @@ const std::string missing_on_account_update_algo = R"(class Strategy: pass )"; -TEST_F(IntegrationLinterTest, basic) +TEST_F(IntegrationLinterPyTest, basic) { auto lint_result = manager.spawn_client(basic_algo, nutc::spawning::AlgoLanguage::Python); - std::cout << lint_result.message << "\n"; ASSERT_TRUE(lint_result.success); } -TEST_F(IntegrationLinterTest, timeout) +TEST_F(IntegrationLinterPyTest, timeout) { // TODO: complete return; @@ -149,7 +123,7 @@ TEST_F(IntegrationLinterTest, timeout) ); } -TEST_F(IntegrationLinterTest, invalidAlgo) +TEST_F(IntegrationLinterPyTest, invalidAlgo) { std::string algo = R"(not_valid_python)"; auto lint_result = manager.spawn_client(algo, nutc::spawning::AlgoLanguage::Python); @@ -159,7 +133,7 @@ TEST_F(IntegrationLinterTest, invalidAlgo) ); } -TEST_F(IntegrationLinterTest, noStrategyClass) +TEST_F(IntegrationLinterPyTest, noStrategyClass) { std::string algo = R"(import math)"; auto lint_result = manager.spawn_client(algo, nutc::spawning::AlgoLanguage::Python); @@ -170,7 +144,7 @@ TEST_F(IntegrationLinterTest, noStrategyClass) ); } -TEST_F(IntegrationLinterTest, missingRequiredFunction) +TEST_F(IntegrationLinterPyTest, missingRequiredFunction) { auto lint_result = manager.spawn_client( missing_on_trade_update_algo, nutc::spawning::AlgoLanguage::Python diff --git a/web/app/api/protected/db/user/createUser/generateApplicationEmail.ts b/web/app/api/protected/db/user/createUser/generateApplicationEmail.ts index 47182576..85201b75 100644 --- a/web/app/api/protected/db/user/createUser/generateApplicationEmail.ts +++ b/web/app/api/protected/db/user/createUser/generateApplicationEmail.ts @@ -22,7 +22,7 @@ export async function GenerateApplicationEmail(user: User, profile: Profile) { const link = process.env.AUTH0_BASE_URL + `/api/handleReview?token=${token}`; const accept_link = link + "&accept=true"; const deny_link = link + "&accept=false"; - const resume_link = `${process.env.EXTERNAL_S3_ENDPOINT}/nutc/${s3Key?.Resume?.at(-1)?.s3Key + const resume_link = `${process.env.S3_ENDPOINT}/nutc/${s3Key?.Resume?.at(-1)?.s3Key }`; const mailOptions = { diff --git a/web/app/dash/submissions/[id]/page.tsx b/web/app/dash/submissions/[id]/page.tsx index cd722744..5f3f7dc2 100644 --- a/web/app/dash/submissions/[id]/page.tsx +++ b/web/app/dash/submissions/[id]/page.tsx @@ -59,7 +59,7 @@ export default async function SubmissionPage(props: { Download Submission