From e3591a6784b8ab30ac88aaac1b84a02da403a9a9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 23 Sep 2022 10:48:35 +0200 Subject: [PATCH 1/2] Replace CloudStore with s5cmd. --- Project.toml | 13 ++----------- src/BugReporting.jl | 39 +++++++++++++++++++-------------------- test/Project.toml | 4 ++++ test/rr.jl | 17 +++++++++++------ 4 files changed, 36 insertions(+), 37 deletions(-) create mode 100644 test/Project.toml diff --git a/Project.toml b/Project.toml index 722d39f..38300f6 100644 --- a/Project.toml +++ b/Project.toml @@ -4,8 +4,6 @@ authors = ["Keno Fischer ", "Tim Besard access_key_id, + "AWS_SECRET_ACCESS_KEY" => secret_access_key, + "AWS_SESSION_TOKEN" => session_token) + run(cmd) end end diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 0000000..3d6c5cd --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,4 @@ +[deps] +CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/rr.jl b/test/rr.jl index 0f5d9ab..3194445 100644 --- a/test/rr.jl +++ b/test/rr.jl @@ -1,4 +1,4 @@ -using Pkg, CloudStore, CloudBase.CloudTest +using Pkg, CloudBase.CloudTest # helper functions to run a command or a block of code while capturing all output function communicate(f::Function) @@ -113,11 +113,16 @@ end mkpath(cache_dir) mktempdir(cache_dir) do temp_srv_dir Minio.with(; public=true, dir=temp_srv_dir) do conf - credentials, bucket = conf - url = bucket.baseurl * "test.tar.zst" - BugReporting.upload_rr_trace(temp_trace_dir; credentials, url) - - test_replay(url) + creds, bucket = conf + s3_url = "s3://$(bucket.name)/test.tar.zst" + http_url = bucket.baseurl * "test.tar.zst" + endpoint_url = replace(bucket.baseurl, "$(bucket.name)/"=>"") + withenv("S3_ENDPOINT_URL" => endpoint_url) do + BugReporting.upload_rr_trace(temp_trace_dir, s3_url; creds.access_key_id, + creds.secret_access_key, creds.session_token) + end + + test_replay(http_url) end end end From 10ab343991e8eeb889c37610ba0fc2afa8e8cfae Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 23 Sep 2022 11:00:48 +0200 Subject: [PATCH 2/2] Reorder output messages. --- src/BugReporting.jl | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/BugReporting.jl b/src/BugReporting.jl index 6da1e96..91a74e1 100644 --- a/src/BugReporting.jl +++ b/src/BugReporting.jl @@ -465,12 +465,14 @@ function make_interactive_report(report_arg, ARGS=[]) elseif report_type == "rr" proc = mktempdir() do trace_dir proc = rr_record(cmd, ARGS...; trace_dir=trace_dir, timeout=timeout, extras=true) - @info "Preparing trace directory for upload (if your trace is large this may take a few minutes)" + "Preparing trace for upload (if your trace is large this may take a few minutes)..." rr_pack(trace_dir) path, creds = get_upload_params() - println("Uploading trace directory...") - upload_rr_trace(trace_dir, "s3://$TRACE_BUCKET/$path"; creds...) + println("Uploading trace...") + withenv("AWS_REGION" => "us-east-1") do + upload_rr_trace(trace_dir, "s3://$TRACE_BUCKET/$path"; creds...) + end println("Uploaded to https://$TRACE_BUCKET.s3.amazonaws.com/$path") proc @@ -485,6 +487,21 @@ end include("sync_compat.jl") function get_upload_params() + # big disclaimer + println() + printstyled("### IMPORTANT =============================================================\n", blink = true) + println(""" + You are about to upload a trace directory to a publicly accessible location. + Such traces contain any information that was accessed by the traced + executable during its execution. This includes any code loaded, any + secrets entered, the contents of any configuration files, etc. + + DO NOT proceed, if you do not wish to make this information publicly available. + By proceeding you explicitly agree to waive any privacy interest in the + uploaded information.""") + printstyled("### =======================================================================\n", blink = true) + println() + c = Channel() t = @async HTTP.WebSockets.open(WSS_ENDPOINT) do ws HTTP.send(ws, "Hello Server, if it's not too much trouble, please send me S3 credentials") @@ -520,22 +537,12 @@ function get_upload_params() end bind(c, t) connectionId = take!(c) - println() - println(""" - ### IMPORTANT ============================================================= - You are about to upload a trace directory to a publicly accessible location. - Such traces contain any information that was accessed by the traced - executable during its execution. This includes any code loaded, any - secrets entered, the contents of any configuration files, etc. - DO NOT proceed, if you do not wish to make this information publicly available. - By proceeding you explicitly agree to waive any privacy interest in the - uploaded information. - ### ======================================================================= - """) - println("To upload a trace, please authenticate, by visiting:\n") + + println("To upload a trace, please authenticate by visiting:\n") println("\thttps://github.com/login/oauth/authorize?client_id=$GITHUB_APP_ID&state=$(HTTP.escapeuri(connectionId))") println("\n", "You can cancel upload by `Ctrl-C`.") flush(stdout) + s3creds = try take!(c) catch err @@ -558,7 +565,7 @@ function upload_rr_trace(trace_directory, url; access_key_id, secret_access_key, # Auto-pack this trace directory if it hasn't already been: sample_directory = joinpath(trace_directory, "latest-trace") if isdir(sample_directory) && uperm(sample_directory) & 0x2 == 0 - @info "`$sample_directory` not writable. Skipping `rr pack`." + println("`$sample_directory` not writable. Skipping `rr pack`.") else rr_pack(trace_directory) end