Improved async model and refactored linter

exchange/CMakeLists.txt

@@ -223,6 +223,7 @@ add_library(
     src/linter/runtime/cpp/cpp_runtime.cpp
     src/linter/runtime/python/python_runtime.cpp
     src/linter/spawning/spawning.cpp
+	src/linter/spawning/async_read_with_timeout.cpp
     src/linter/crow/crow.cpp
     # Utils
 )

exchange/src/common/types/algorithm/base_algorithm.hpp

@@ -1,9 +1,10 @@
 #pragma once
 
+#include "common/util.hpp"
+
 #include <fmt/format.h>
 
 namespace nutc::common {
-enum class AlgoLanguage { python, cpp };
 enum class AlgoLocation { local, s3 };
 
 class BaseAlgorithm {

exchange/src/common/util.hpp

@@ -15,15 +15,17 @@ std::string trader_id(const std::string& user_id, const std::string& algo_id);
 
 std::string get_firebase_endpoint(const std::string& params);
 
+std::string base64_encode(const std::string& data);
+
+std::string base64_decode(const std::string& data);
+
 // NOTE: this must be the same as Side in template.hpp
 enum class Side { buy = 0, sell = 1 };
 
 std::string to_string(Side side);
 
-std::string base64_encode(const std::string& data);
-
-std::string base64_decode(const std::string& data);
-
 enum class Mode { dev, sandbox, normal, bots_only };
 
+enum class AlgoLanguage { python, cpp };
+
 } // namespace nutc::common

exchange/src/exchange/sandbox_server/crow.cpp

@@ -3,7 +3,6 @@
 #include "common/fetching/fetching.hpp"
 #include "common/logging/logging.hpp"
 #include "common/messages_exchange_to_wrapper.hpp"
-#include "common/types/algorithm/base_algorithm.hpp"
 #include "exchange/config/dynamic/config.hpp"
 #include "exchange/config/static/config.hpp"
 #include "exchange/traders/trader_types/algo_trader.hpp"

exchange/src/linter/config.h

@@ -29,7 +29,7 @@
 #define S3_BUCKET "nutc"
 
 // Linting
-#define LINT_AUTO_TIMEOUT_SECONDS 10
+#define LINT_AUTO_TIMEOUT_SECONDS 4
 
 /**
  * If we are in debug mode.

exchange/src/linter/crow/crow.cpp

@@ -1,90 +1,81 @@
 #include "crow.hpp"
 
-#include "common/logging/logging.hpp"
 #include "common/fetching/fetching.hpp"
+#include "common/logging/logging.hpp"
 #include "linter/spawning/spawning.hpp"
 
-namespace nutc {
-namespace crow {
+namespace nutc::linter {
 
-std::thread
-get_server_thread()
+::crow::SimpleApp
+get_crow_app()
 {
-    std::thread server_thread([]() {
-        spawning::LintProcessManager spawner_manager;
-        namespace crow = ::crow;
-        ::crow::SimpleApp app;
-        CROW_ROUTE(app, "/")
-        ([&](const crow::request& req) {
-            crow::response res;
-
-            // Set CORS headers
-            res.add_header("Access-Control-Allow-Origin", "*");
-            res.add_header(
-                "Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS"
-            );
-            res.add_header(
-                "Access-Control-Allow-Headers", "Origin, Content-Type, Accept"
-            );
-
-            // Handle preflight request for OPTIONS method
-            if (req.method == crow::HTTPMethod::OPTIONS) {
-                res.code = 204; // No Content
-                return res;
-            }
-
-            if (!req.url_params.get("algo_url")) {
-                log_e(main, "No algo_id provided");
-                return crow::response(400);
-            }
-            if (!req.url_params.get("language")) {
-                log_e(main, "No language provided");
-                return crow::response(400);
-            }
-            std::string algo_url = req.url_params.get("algo_url");
-            std::string language = req.url_params.get("language");
+    namespace crow = ::crow;
+    ::crow::SimpleApp app;
+    CROW_ROUTE(app, "/")
+    ([&](const crow::request& req) {
+        crow::response res;
 
-            spawning::AlgoLanguage algo_language;
-            if (language == "Python") {
-                algo_language = spawning::AlgoLanguage::Python;
-            }
-            else if (language == "Cpp") {
-                algo_language = spawning::AlgoLanguage::Cpp;
-            }
-            else {
-                log_e(main, "Invalid language provided: {}", language);
-                return crow::response(400);
-            }
+        // Set CORS headers
+        res.add_header("Access-Control-Allow-Origin", "*");
+        res.add_header(
+            "Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS"
+        );
+        res.add_header("Access-Control-Allow-Headers", "Origin, Content-Type, Accept");
 
-            auto algo_code = client::storage_request(algo_url);
-            if (!algo_code.has_value()) {
-                crow::json::wvalue response = crow::json::wvalue({
-                    {"success", false                },
-                    {"message", "Algo file not found"}
-                });
+        // Handle preflight request for OPTIONS method
+        if (req.method == crow::HTTPMethod::OPTIONS) {
+            res.code = 204; // No Content
+            return res;
+        }
 
-                res.body = response.dump();
-                res.code = 200;
+        if (req.url_params.get("algo_url") == nullptr) {
+            log_e(main, "No algo_id provided");
+            return crow::response(400);
+        }
+        if (req.url_params.get("language") == nullptr) {
+            log_e(main, "No language provided");
+            return crow::response(400);
+        }
+        std::string algo_url = req.url_params.get("algo_url");
+        std::string language = req.url_params.get("language");
 
-                return res;
-            }
+        common::AlgoLanguage algo_language;
+        if (language == "Python") {
+            algo_language = common::AlgoLanguage::python;
+        }
+        else if (language == "Cpp") {
+            algo_language = common::AlgoLanguage::cpp;
+        }
+        else {
+            log_e(main, "Invalid language provided: {}", language);
+            return crow::response(400);
+        }
 
-            auto lint_res =
-                spawner_manager.spawn_client(algo_code.value(), algo_language);
-            crow::json::wvalue response({
-                {"success", lint_res.success                                 },
-                {"message", client::replaceDisallowedValues(lint_res.message)}
+        auto algo_code = client::storage_request(algo_url);
+        if (!algo_code.has_value()) {
+            crow::json::wvalue response = crow::json::wvalue({
+                {"success", false                },
+                {"message", "Algo file not found"}
             });
 
             res.body = response.dump();
             res.code = 200;
 
             return res;
+        }
+
+        auto lint_res = spawn_client(algo_code.value(), algo_language);
+        crow::json::wvalue response({
+            {"success", lint_res.success                                 },
+            {"message", client::replaceDisallowedValues(lint_res.message)}
         });
-        app.port(18081).run();
+
+        res.body = response.dump();
+        res.code = 200;
+
+        return res;
     });
-    return server_thread;
+    return app;
 }
 
-} // namespace crow
-} // namespace nutc
+} // namespace nutc::linter

exchange/src/linter/crow/crow.hpp

@@ -1,12 +1,8 @@
 #pragma once
 #include <crow/app.h>
 
-#include <thread>
+namespace nutc::linter {
 
-namespace nutc {
-namespace crow {
+::crow::SimpleApp get_crow_app();
 
-std::thread get_server_thread();
-
-}
-} // namespace nutc
+} // namespace nutc::linter

exchange/src/linter/lint/lint.cpp

@@ -8,8 +8,7 @@
 
 #include <string>
 
-namespace nutc {
-namespace lint {
+namespace nutc::linter {
 
 lint_result
 lint(Runtime& runtime)
@@ -100,5 +99,4 @@ lint(Runtime& runtime)
     return {true, out_message};
 }
 
-} // namespace lint
-} // namespace nutc
+} // namespace nutc::linter

exchange/src/linter/lint/lint.hpp

@@ -3,10 +3,8 @@
 #include "linter/lint/lint_result.hpp"
 #include "linter/runtime/runtime.hpp"
 
-namespace nutc {
-namespace lint {
+namespace nutc::linter {
 
 [[nodiscard]] lint_result lint(Runtime& runtime);
 
-} // namespace lint
-} // namespace nutc
+} // namespace nutc::linter

exchange/src/linter/lint/lint_result.hpp

@@ -4,20 +4,18 @@
 
 #include <string>
 
-namespace nutc {
-namespace lint {
+namespace nutc::linter {
 
 struct lint_result {
     bool success;
     std::string message;
 };
 
-} // namespace lint
-} // namespace nutc
+} // namespace nutc::linter
 
 template <>
-struct glz::meta<nutc::lint::lint_result> {
-    using t = nutc::lint::lint_result;
+struct glz::meta<nutc::linter::lint_result> {
+    using t = nutc::linter::lint_result;
     static constexpr auto value =
         object("success", &t::success, "message", &t::message);
 };

exchange/src/linter/main.cpp

@@ -1,3 +1,5 @@
+#include "linter/spawning/spawning.hpp"
+
 #include <quill/LogLevel.h>
 #define CROW_MAIN
 #include "common/logging/logging.hpp"
@@ -9,7 +11,6 @@
 
 #include <iostream>
 #include <string>
-#include <thread>
 
 static void
 process_arguments(int argc, const char** argv)
@@ -46,9 +47,7 @@ main(int argc, const char** argv)
     nutc::logging::init("linter.log", quill::LogLevel::Info);
     log_i(main, "Starting NUTC Linter");
 
-    auto server_thread = nutc::crow::get_server_thread();
-
-    server_thread.join();
+    nutc::linter::get_crow_app().port(18081).run();
 
     return 0;
 }

exchange/src/linter/runtime/cpp/cpp_runtime.cpp

@@ -15,7 +15,7 @@
 #include <fstream>
 #include <iostream>
 
-namespace nutc::lint {
+namespace nutc::linter {
 
 CppRuntime::CppRuntime(
     std::string algo, LimitOrderFunction limit_order, MarketOrderFunction market_order,
@@ -34,6 +34,7 @@ CppRuntime::~CppRuntime()
     }
 }
 
+// TODO: should be in the constructor
 std::optional<std::string>
 CppRuntime::init()
 {
@@ -103,4 +104,4 @@ CppRuntime::fire_on_account_update(
     on_account_update_func_(strategy_object_, ticker, side, quantity, price, capital);
 }
 
-} // namespace nutc::lint
+} // namespace nutc::linter

exchange/src/linter/runtime/cpp/cpp_runtime.hpp

@@ -2,7 +2,7 @@
 
 #include "linter/runtime/runtime.hpp"
 
-namespace nutc::lint {
+namespace nutc::linter {
 
 class CppRuntime : public Runtime {
 public:
@@ -38,11 +38,11 @@ class CppRuntime : public Runtime {
     using OnAccountUpdateFunc =
         void (*)(Strategy*, common::Ticker, common::Side, float, float, float);
 
-    OnTradeUpdateFunc on_trade_update_func_;
-    OnOrderBookUpdateFunc on_orderbook_update_func_;
-    OnAccountUpdateFunc on_account_update_func_;
+    OnTradeUpdateFunc on_trade_update_func_{};
+    OnOrderBookUpdateFunc on_orderbook_update_func_{};
+    OnAccountUpdateFunc on_account_update_func_{};
 
-    Strategy* strategy_object_;
+    Strategy* strategy_object_{};
     void* dl_handle_{};
 };
-} // namespace nutc::lint
+} // namespace nutc::linter

exchange/src/linter/runtime/python/python_runtime.cpp

@@ -4,7 +4,7 @@
 #include <pybind11/embed.h>
 #include <pybind11/pybind11.h>
 
-namespace nutc::lint {
+namespace nutc::linter {
 
 namespace py = pybind11;
 
@@ -51,8 +51,7 @@ PyRuntime::fire_on_orderbook_update(
 
 void
 PyRuntime::fire_on_account_update(
-    common::Ticker ticker, common::Side side, float price, float quantity,
-    float capital
+    common::Ticker ticker, common::Side side, float price, float quantity, float capital
 ) const
 {
     py::globals()["strategy"].attr("on_account_update")(
@@ -158,4 +157,4 @@ PyRuntime::run_initialization_code(const std::string& py_code)
     return std::nullopt;
 }
 
-} // namespace nutc::lint
+} // namespace nutc::linter

exchange/src/linter/runtime/python/python_runtime.hpp

@@ -4,7 +4,7 @@
 
 #include <pybind11/embed.h>
 
-namespace nutc::lint {
+namespace nutc::linter {
 
 class PyRuntime : public Runtime {
 public:
@@ -46,4 +46,4 @@ class PyRuntime : public Runtime {
     );
 };
 
-} // namespace nutc::lint
+} // namespace nutc::linter