add logging to sandbox

exchange/CMakeLists.txt

@@ -220,7 +220,6 @@ target_link_libraries(WRAPPER_exe PRIVATE ${Python3_LIBRARIES})
 add_library(
 	LINTER_lib OBJECT 
 
-    src/linter/fetching/fetching.cpp
     src/linter/runtime/cpp/cpp_runtime.cpp
     src/linter/runtime/python/python_runtime.cpp
     src/linter/spawning/spawning.cpp
@@ -313,6 +312,7 @@ add_library(COMMON_lib OBJECT
     src/common/types/algorithm/local_algorithm.cpp
     src/common/logging/logging.cpp
     src/common/compilation/compile_cpp.cpp
+    src/common/fetching/fetching.cpp
 )
 
 target_include_directories(

exchange/src/common/fetching/fetching.cpp

@@ -0,0 +1,119 @@
+#include "fetching.hpp"
+
+#include <curl/curl.h>
+#include <fmt/core.h>
+
+#include <optional>
+#include <regex>
+
+namespace nutc {
+namespace client {
+std::string
+replaceDisallowedValues(const std::string& input)
+{
+    std::regex newlinePattern("\\n");
+    std::string input2 = std::regex_replace(input, newlinePattern, "\\n");
+    std::regex disallowedPattern("[.$#\\[\\]]");
+
+    return std::regex_replace(input2, disallowedPattern, "");
+}
+
+static size_t
+write_callback(void* contents, size_t size, size_t nmemb, void* userp)
+{
+    auto* str = reinterpret_cast<std::string*>(userp);
+    auto* data = static_cast<char*>(contents);
+
+    str->append(data, size * nmemb);
+    return size * nmemb;
+}
+
+std::optional<std::string>
+storage_request(const std::string& url)
+{
+    std::string readBuffer;
+
+    CURL* curl = curl_easy_init();
+    if (curl) {
+        CURLcode res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+        if (res != CURLE_OK) {
+            return std::nullopt;
+        }
+        res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        if (res != CURLE_OK) {
+            return std::nullopt;
+        }
+        res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
+        if (res != CURLE_OK) {
+            return std::nullopt;
+        }
+
+        res = curl_easy_perform(curl);
+        if (res != CURLE_OK) {
+            return std::nullopt;
+        }
+        curl_easy_cleanup(curl);
+    }
+
+    return readBuffer;
+}
+
+std::optional<std::string>
+put_request(const std::string& url, const std::string& body)
+{
+    std::string responseBuffer;
+
+    CURL* curl = curl_easy_init();
+    if (curl) {
+        CURLcode res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        res = curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseBuffer);
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        res = curl_easy_perform(curl);
+        if (res != CURLE_OK) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        // Check HTTP status code
+        long http_code = 0;
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+        if (http_code >= 400) {
+            curl_easy_cleanup(curl);
+            return std::nullopt;
+        }
+
+        curl_easy_cleanup(curl);
+    }
+
+    return responseBuffer;
+}
+
+} // namespace client
+} // namespace nutc

exchange/src/linter/fetching/fetching.hpp → exchange/src/common/fetching/fetching.hpp

@@ -16,6 +16,7 @@ struct SetLintBody {
 
 std::optional<std::string> storage_request(const std::string& url);
 std::string replaceDisallowedValues(const std::string& input);
+std::optional<std::string> put_request(const std::string& url, const std::string& body);
 
 } // namespace client
 } // namespace nutc

exchange/src/exchange/sandbox_server/crow.cpp

@@ -1,12 +1,16 @@
 #include "crow.hpp"
 
+#include "common/fetching/fetching.hpp"
 #include "common/logging/logging.hpp"
 #include "common/messages_exchange_to_wrapper.hpp"
 #include "common/types/algorithm/base_algorithm.hpp"
 #include "exchange/config/dynamic/config.hpp"
+#include "exchange/config/static/config.hpp"
 #include "exchange/traders/trader_types/algo_trader.hpp"
 
+#include <boost/filesystem/operations.hpp>
 #include <crow/common.h>
+#include <unistd.h>
 
 namespace nutc::exchange {
 
@@ -34,17 +38,27 @@ CrowServer::CrowServer() :
                     return res;
                 }
 
+                if (!req.url_params.get("logfile_url")) {
+                    log_e(main, "No logfile_url provided");
+                    return crow::response(400);
+                }
+
+                std::string logfile_url = req.url_params.get("logfile_url");
+
                 try {
                     log_i(
                         sandbox_server,
-                        "Received sandbox request with algo_id {} and language {}",
-                        algo_id, language
+                        "Received sandbox request with algo_id {} and language {}. "
+                        "Pre-signed url: {}",
+                        algo_id, language, logfile_url
                     );
                     common::AlgoLanguage language_enum =
                         language == "Python" ? common::AlgoLanguage::python
                                              : common::AlgoLanguage::cpp;
                     std::string algorithm_data = req.body;
-                    add_pending_trader_(algo_id, language_enum, algorithm_data);
+                    add_pending_trader_(
+                        algo_id, language_enum, algorithm_data, logfile_url
+                    );
                     crow::json::wvalue response_json({
                         {"success", true                              },
                         {"message", "Algorithm Successfully Submitted"}
@@ -68,7 +82,7 @@ CrowServer::CrowServer() :
 void
 CrowServer::add_pending_trader_(
     const std::string& algo_id, common::AlgoLanguage language,
-    const std::string& algorithm_data
+    const std::string& algorithm_data, const std::string& logfile_url
 )
 {
     static const auto STARTING_CAPITAL = Config::get().constants().STARTING_CAPITAL;
@@ -82,7 +96,7 @@ CrowServer::add_pending_trader_(
     trader_lock.unlock();
 
     static auto trial_secs = Config::get().constants().SANDBOX_TRIAL_SECS;
-    start_remove_timer_(trial_secs, trader);
+    start_remove_timer_(trial_secs, trader, algo_id, logfile_url);
 
     auto get_start_message = []() {
         static auto start_message = glz::write_json(common::start_time{0});
@@ -112,17 +126,17 @@ CrowServer::~CrowServer()
 
 void
 CrowServer::start_remove_timer_(
-    unsigned int time_s, std::weak_ptr<GenericTrader> trader_ptr
+    unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr,
+    const std::string& algo_id, const std::string& logfile_url
 )
 {
-    auto timer = ba::steady_timer{io_context_, std::chrono::seconds(time_s)};
+    auto timer = ba::steady_timer{io_context_, std::chrono::seconds(time_ms)};
 
-    timer.async_wait([trader_ptr](const boost::system::error_code& err_code) {
+    timer.async_wait([trader_ptr, logfile_url,
+                      algo_id](const boost::system::error_code& err_code) {
         auto trader = trader_ptr.lock();
         if (trader == nullptr) {
-            log_i(
-                sandbox_server, "Trader already removed: {}", trader->get_display_name()
-            );
+            log_i(sandbox_server, "Trader already removed: {}", algo_id);
             return;
         }
         if (err_code) {
@@ -134,6 +148,19 @@ CrowServer::start_remove_timer_(
             log_i(sandbox_server, "Removing trader {}", trader->get_display_name());
             trader->disable();
         }
+
+        std::ifstream log_file(fmt::format("{}/{}.log", LOG_DIR, algo_id));
+        if (!log_file.is_open()) {
+            log_e(
+                sandbox_server, "Unable to open log file for trader {}",
+                trader->get_display_name()
+            );
+            return;
+        }
+        std::stringstream log_ss;
+        log_ss << log_file.rdbuf();
+
+        client::put_request(logfile_url, log_ss.str());
     });
     timers_.push_back(std::move(timer));
 }

exchange/src/exchange/sandbox_server/crow.hpp

@@ -50,10 +50,12 @@ class CrowServer {
 private:
     void add_pending_trader_(
         const std::string& algo_id, common::AlgoLanguage language,
-        const std::string& algorithm_data
+        const std::string& algorithm_data, const std::string& logfile_url
+    );
+    void start_remove_timer_(
+        unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr,
+        const std::string& algo_id, const std::string& logfile_url
     );
-    void
-    start_remove_timer_(unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr);
 };
 
 } // namespace nutc::exchange

exchange/src/linter/crow/crow.cpp

@@ -1,7 +1,7 @@
 #include "crow.hpp"
 
 #include "common/logging/logging.hpp"
-#include "linter/fetching/fetching.hpp"
+#include "common/fetching/fetching.hpp"
 #include "linter/spawning/spawning.hpp"
 
 namespace nutc {

web/app/dash/submissions/[id]/graphs.tsx

@@ -1,15 +1,19 @@
 "use client";
 
+import { ArrowDownTrayIcon } from "@heroicons/react/16/solid";
 import { Algo, AlgoFile } from "@prisma/client";
 import React, { useEffect, useState } from "react";
 
 export function AlgoGraphs({
   algo,
   userId,
+  s3Endpoint,
 }: {
   algo: Algo & { algoFile: AlgoFile };
   userId: string;
+  s3Endpoint: string;
 }) {
+  const [showLogs, setShowLogs] = useState(false);
   const upTime = new Date(algo?.algoFile?.createdAt).getTime() + 1000;
   const sandboxTimeMs = 300000;
   const baseEndpoint = `${process.env.NEXT_PUBLIC_NGINX_ENDPOINT}/d-solo/cdk4teh4zl534a/nutc-dev?orgId=1&var-traderid=${algo.algoFileS3Key}&from=${upTime}&theme=dark`;
@@ -20,14 +24,27 @@ export function AlgoGraphs({
       setUrl(baseEndpoint + "&refresh=5s");
       setTimeout(() => {
         setUrl(baseEndpoint + `&to=${upTime + sandboxTimeMs}`);
+        setShowLogs(true);
       }, upTime + sandboxTimeMs - Date.now());
     } else {
       setUrl(baseEndpoint + `&to=${upTime + sandboxTimeMs}`);
+      setShowLogs(true);
     }
-  }, [baseEndpoint, upTime]);
+  }, [baseEndpoint, setShowLogs, upTime]);
 
   return (
     <>
+      {showLogs ? (
+        <a
+          type="button"
+          target="_blank"
+          href={`${s3Endpoint}/nutc/${algo.logFileS3Key}`}
+          className="mt-2 inline-flex items-center gap-x-1.5 rounded-md bg-indigo-700 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-600 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-700">
+          Download Logs
+          <ArrowDownTrayIcon className="-mr-0.5 h-5 w-5" aria-hidden="true" />
+        </a>
+      ) : null}
+
       <div className="my-8">
         <h2 className="text-xl font-semibold mb-2">Profit and Loss</h2>
         <iframe