Skip to content

Commit

Permalink
add logging to sandbox
Browse files Browse the repository at this point in the history
  • Loading branch information
glasss13 committed Oct 10, 2024
1 parent a8228c8 commit bea5aae
Show file tree
Hide file tree
Showing 14 changed files with 1,168 additions and 106 deletions.
2 changes: 1 addition & 1 deletion exchange/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,6 @@ target_link_libraries(WRAPPER_exe PRIVATE ${Python3_LIBRARIES})
add_library(
LINTER_lib OBJECT

src/linter/fetching/fetching.cpp
src/linter/runtime/cpp/cpp_runtime.cpp
src/linter/runtime/python/python_runtime.cpp
src/linter/spawning/spawning.cpp
Expand Down Expand Up @@ -313,6 +312,7 @@ add_library(COMMON_lib OBJECT
src/common/types/algorithm/local_algorithm.cpp
src/common/logging/logging.cpp
src/common/compilation/compile_cpp.cpp
src/common/fetching/fetching.cpp
)

target_include_directories(
Expand Down
119 changes: 119 additions & 0 deletions exchange/src/common/fetching/fetching.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
#include "fetching.hpp"

#include <curl/curl.h>
#include <fmt/core.h>

#include <optional>
#include <regex>

namespace nutc {
namespace client {
std::string
replaceDisallowedValues(const std::string& input)
{
std::regex newlinePattern("\\n");
std::string input2 = std::regex_replace(input, newlinePattern, "\\n");
std::regex disallowedPattern("[.$#\\[\\]]");

return std::regex_replace(input2, disallowedPattern, "");
}

static size_t
write_callback(void* contents, size_t size, size_t nmemb, void* userp)
{
auto* str = reinterpret_cast<std::string*>(userp);
auto* data = static_cast<char*>(contents);

str->append(data, size * nmemb);
return size * nmemb;
}

std::optional<std::string>
storage_request(const std::string& url)
{
std::string readBuffer;

CURL* curl = curl_easy_init();
if (curl) {
CURLcode res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if (res != CURLE_OK) {
return std::nullopt;
}
res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
if (res != CURLE_OK) {
return std::nullopt;
}
res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
if (res != CURLE_OK) {
return std::nullopt;
}

res = curl_easy_perform(curl);
if (res != CURLE_OK) {
return std::nullopt;
}
curl_easy_cleanup(curl);
}

return readBuffer;
}

std::optional<std::string>
put_request(const std::string& url, const std::string& body)
{
std::string responseBuffer;

CURL* curl = curl_easy_init();
if (curl) {
CURLcode res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

res = curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str());
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &responseBuffer);
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

res = curl_easy_perform(curl);
if (res != CURLE_OK) {
curl_easy_cleanup(curl);
return std::nullopt;
}

// Check HTTP status code
long http_code = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code >= 400) {
curl_easy_cleanup(curl);
return std::nullopt;
}

curl_easy_cleanup(curl);
}

return responseBuffer;
}

} // namespace client
} // namespace nutc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ struct SetLintBody {

std::optional<std::string> storage_request(const std::string& url);
std::string replaceDisallowedValues(const std::string& input);
std::optional<std::string> put_request(const std::string& url, const std::string& body);

} // namespace client
} // namespace nutc
Expand Down
49 changes: 38 additions & 11 deletions exchange/src/exchange/sandbox_server/crow.cpp
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
#include "crow.hpp"

#include "common/fetching/fetching.hpp"
#include "common/logging/logging.hpp"
#include "common/messages_exchange_to_wrapper.hpp"
#include "common/types/algorithm/base_algorithm.hpp"
#include "exchange/config/dynamic/config.hpp"
#include "exchange/config/static/config.hpp"
#include "exchange/traders/trader_types/algo_trader.hpp"

#include <boost/filesystem/operations.hpp>
#include <crow/common.h>
#include <unistd.h>

namespace nutc::exchange {

Expand Down Expand Up @@ -34,17 +38,27 @@ CrowServer::CrowServer() :
return res;
}

if (!req.url_params.get("logfile_url")) {
log_e(main, "No logfile_url provided");
return crow::response(400);
}

std::string logfile_url = req.url_params.get("logfile_url");

try {
log_i(
sandbox_server,
"Received sandbox request with algo_id {} and language {}",
algo_id, language
"Received sandbox request with algo_id {} and language {}. "
"Pre-signed url: {}",
algo_id, language, logfile_url
);
common::AlgoLanguage language_enum =
language == "Python" ? common::AlgoLanguage::python
: common::AlgoLanguage::cpp;
std::string algorithm_data = req.body;
add_pending_trader_(algo_id, language_enum, algorithm_data);
add_pending_trader_(
algo_id, language_enum, algorithm_data, logfile_url
);
crow::json::wvalue response_json({
{"success", true },
{"message", "Algorithm Successfully Submitted"}
Expand All @@ -68,7 +82,7 @@ CrowServer::CrowServer() :
void
CrowServer::add_pending_trader_(
const std::string& algo_id, common::AlgoLanguage language,
const std::string& algorithm_data
const std::string& algorithm_data, const std::string& logfile_url
)
{
static const auto STARTING_CAPITAL = Config::get().constants().STARTING_CAPITAL;
Expand All @@ -82,7 +96,7 @@ CrowServer::add_pending_trader_(
trader_lock.unlock();

static auto trial_secs = Config::get().constants().SANDBOX_TRIAL_SECS;
start_remove_timer_(trial_secs, trader);
start_remove_timer_(trial_secs, trader, algo_id, logfile_url);

auto get_start_message = []() {
static auto start_message = glz::write_json(common::start_time{0});
Expand Down Expand Up @@ -112,17 +126,17 @@ CrowServer::~CrowServer()

void
CrowServer::start_remove_timer_(
unsigned int time_s, std::weak_ptr<GenericTrader> trader_ptr
unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr,
const std::string& algo_id, const std::string& logfile_url
)
{
auto timer = ba::steady_timer{io_context_, std::chrono::seconds(time_s)};
auto timer = ba::steady_timer{io_context_, std::chrono::seconds(time_ms)};

timer.async_wait([trader_ptr](const boost::system::error_code& err_code) {
timer.async_wait([trader_ptr, logfile_url,
algo_id](const boost::system::error_code& err_code) {
auto trader = trader_ptr.lock();
if (trader == nullptr) {
log_i(
sandbox_server, "Trader already removed: {}", trader->get_display_name()
);
log_i(sandbox_server, "Trader already removed: {}", algo_id);
return;
}
if (err_code) {
Expand All @@ -134,6 +148,19 @@ CrowServer::start_remove_timer_(
log_i(sandbox_server, "Removing trader {}", trader->get_display_name());
trader->disable();
}

std::ifstream log_file(fmt::format("{}/{}.log", LOG_DIR, algo_id));
if (!log_file.is_open()) {
log_e(
sandbox_server, "Unable to open log file for trader {}",
trader->get_display_name()
);
return;
}
std::stringstream log_ss;
log_ss << log_file.rdbuf();

client::put_request(logfile_url, log_ss.str());
});
timers_.push_back(std::move(timer));
}
Expand Down
8 changes: 5 additions & 3 deletions exchange/src/exchange/sandbox_server/crow.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ class CrowServer {
private:
void add_pending_trader_(
const std::string& algo_id, common::AlgoLanguage language,
const std::string& algorithm_data
const std::string& algorithm_data, const std::string& logfile_url
);
void start_remove_timer_(
unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr,
const std::string& algo_id, const std::string& logfile_url
);
void
start_remove_timer_(unsigned int time_ms, std::weak_ptr<GenericTrader> trader_ptr);
};

} // namespace nutc::exchange
1 change: 0 additions & 1 deletion exchange/src/linter/crow/crow.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#include "crow.hpp"

#include "common/logging/logging.hpp"
#include "linter/fetching/fetching.hpp"
#include "linter/spawning/spawning.hpp"

namespace nutc {
Expand Down
62 changes: 0 additions & 62 deletions exchange/src/linter/fetching/fetching.cpp

This file was deleted.

19 changes: 18 additions & 1 deletion web/app/dash/submissions/[id]/graphs.tsx
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
"use client";

import { ArrowDownTrayIcon } from "@heroicons/react/16/solid";
import { Algo, AlgoFile } from "@prisma/client";
import React, { useEffect, useState } from "react";

export function AlgoGraphs({
algo,
userId,
s3Endpoint,
}: {
algo: Algo & { algoFile: AlgoFile };
userId: string;
s3Endpoint: string;
}) {
const [showLogs, setShowLogs] = useState(false);
const upTime = new Date(algo?.algoFile?.createdAt).getTime() + 1000;
const sandboxTimeMs = 300000;
const baseEndpoint = `${process.env.NEXT_PUBLIC_NGINX_ENDPOINT}/d-solo/cdk4teh4zl534a/nutc-dev?orgId=1&var-traderid=${algo.algoFileS3Key}&from=${upTime}&theme=dark`;
Expand All @@ -20,14 +24,27 @@ export function AlgoGraphs({
setUrl(baseEndpoint + "&refresh=5s");
setTimeout(() => {
setUrl(baseEndpoint + `&to=${upTime + sandboxTimeMs}`);
setShowLogs(true);
}, upTime + sandboxTimeMs - Date.now());
} else {
setUrl(baseEndpoint + `&to=${upTime + sandboxTimeMs}`);
setShowLogs(true);
}
}, [baseEndpoint, upTime]);
}, [baseEndpoint, setShowLogs, upTime]);

return (
<>
{showLogs ? (
<a
type="button"
target="_blank"
href={`${s3Endpoint}/nutc/${algo.logFileS3Key}`}
className="inline-flex items-center gap-x-1.5 rounded-md bg-indigo-700 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-600 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-700">
Download Logs
<ArrowDownTrayIcon className="-mr-0.5 h-5 w-5" aria-hidden="true" />
</a>
) : null}

<div className="my-8">
<h2 className="text-xl font-semibold mb-2">Profit and Loss</h2>
<iframe
Expand Down
Loading

0 comments on commit bea5aae

Please sign in to comment.