Merge pull request #92 from JuliaLang/tb/s5cmd

Switch to s5cmd

Project.toml

@@ -4,8 +4,6 @@ authors = ["Keno Fischer <[email protected]>", "Tim Besard <tim.besard@gma
 version = "0.2.7"
 
 [deps]
-CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"
-CloudStore = "3365d9ee-d53b-4a56-812d-5344d5b716d7"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 Elfutils_jll = "ab5a07f8-06af-567f-a878-e8bb879eba5a"
 GDB_jll = "a8b33d9f-0f8b-5197-9986-f85cbaa50c6c"
@@ -17,10 +15,9 @@ Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4"
 rr_jll = "e86bdf43-55f7-5ea2-9fd0-e7daa2c0f2b4"
+s5cmd_jll = "9c7924da-da29-52d4-adf6-81e6cfbf5c74"
 
 [compat]
-CloudBase = "1.2"
-CloudStore = "1"
 Elfutils_jll = "0.182"
 GDB_jll = "12.1"
 Git = "1.0"
@@ -31,10 +28,4 @@ Scratch = "1"
 Tar = "1.3.1"
 julia = "1.3"
 rr_jll = "5.3.1"
-
-[extras]
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test", "Pkg"]
+s5cmd_jll = "2.0.0"

src/BugReporting.jl

@@ -14,12 +14,11 @@ using Zstd_jll: zstdmt
 using Elfutils_jll: eu_readelf
 import HTTP
 using JSON
-using CloudStore: S3
-using CloudBase: AWS
 import Tar
 using Git: git
 import Downloads
 using ProgressMeter: Progress, update!
+using s5cmd_jll: s5cmd
 
 # https://github.com/JuliaLang/julia/pull/29411
 if isdefined(Base, :exit_on_sigint)
@@ -31,7 +30,7 @@ end
 
 const WSS_ENDPOINT = "wss://53ly7yebjg.execute-api.us-east-1.amazonaws.com/test"
 const GITHUB_APP_ID = "Iv1.c29a629771fe63c4"
-const TRACE_BUCKET = "https://julialang-dumps.s3.amazonaws.com"
+const TRACE_BUCKET = "julialang-dumps"
 const METADATA_VERSION = v"1"
 
 function check_rr_available()
@@ -466,10 +465,16 @@ function make_interactive_report(report_arg, ARGS=[])
     elseif report_type == "rr"
         proc = mktempdir() do trace_dir
             proc = rr_record(cmd, ARGS...; trace_dir=trace_dir, timeout=timeout, extras=true)
-            @info "Preparing trace directory for upload (if your trace is large this may take a few minutes)"
+            "Preparing trace for upload (if your trace is large this may take a few minutes)..."
             rr_pack(trace_dir)
-            params = get_upload_params()
-            upload_rr_trace(trace_dir; params...)
+            path, creds = get_upload_params()
+
+            println("Uploading trace...")
+            withenv("AWS_REGION" => "us-east-1") do
+                upload_rr_trace(trace_dir, "s3://$TRACE_BUCKET/$path"; creds...)
+            end
+            println("Uploaded to https://$TRACE_BUCKET.s3.amazonaws.com/$path")
+
             proc
         end
         handle_child_error(proc)
@@ -482,6 +487,21 @@ end
 include("sync_compat.jl")
 
 function get_upload_params()
+    # big disclaimer
+    println()
+    printstyled("### IMPORTANT =============================================================\n", blink = true)
+    println("""
+        You are about to upload a trace directory to a publicly accessible location.
+        Such traces contain any information that was accessed by the traced
+        executable during its execution. This includes any code loaded, any
+        secrets entered, the contents of any configuration files, etc.
+
+        DO NOT proceed, if you do not wish to make this information publicly available.
+        By proceeding you explicitly agree to waive any privacy interest in the
+        uploaded information.""")
+    printstyled("### =======================================================================\n", blink = true)
+    println()
+
     c = Channel()
     t = @async HTTP.WebSockets.open(WSS_ENDPOINT) do ws
         HTTP.send(ws, "Hello Server, if it's not too much trouble, please send me S3 credentials")
@@ -517,22 +537,12 @@ function get_upload_params()
     end
     bind(c, t)
     connectionId = take!(c)
-    println()
-    println("""
-    ### IMPORTANT =============================================================
-    You are about to upload a trace directory to a publicly accessible location.
-    Such traces contain any information that was accessed by the traced
-    executable during its execution. This includes any code loaded, any
-    secrets entered, the contents of any configuration files, etc.
-    DO NOT proceed, if you do not wish to make this information publicly available.
-    By proceeding you explicitly agree to waive any privacy interest in the
-    uploaded information.
-    ### =======================================================================
-    """)
-    println("To upload a trace, please authenticate, by visiting:\n")
+
+    println("To upload a trace, please authenticate by visiting:\n")
     println("\thttps://github.com/login/oauth/authorize?client_id=$GITHUB_APP_ID&state=$(HTTP.escapeuri(connectionId))")
     println("\n", "You can cancel upload by `Ctrl-C`.")
     flush(stdout)
+
     s3creds = try
         take!(c)
     catch err
@@ -546,40 +556,36 @@ function get_upload_params()
 
     println()
 
-    url = "$TRACE_BUCKET/$(s3creds["UPLOAD_PATH"])"
-
-    credentials = AWS.Credentials(
-        s3creds["AWS_ACCESS_KEY_ID"],
-        s3creds["AWS_SECRET_ACCESS_KEY"],
-        s3creds["AWS_SESSION_TOKEN"])
-
-    return (; credentials, url)
+    return s3creds["UPLOAD_PATH"], (; access_key_id=s3creds["AWS_ACCESS_KEY_ID"],
+                                      secret_access_key=s3creds["AWS_SECRET_ACCESS_KEY"],
+                                      session_token=s3creds["AWS_SESSION_TOKEN"])
 end
 
-function upload_rr_trace(trace_directory; credentials, url)
+function upload_rr_trace(trace_directory, url; access_key_id, secret_access_key, session_token)
     # Auto-pack this trace directory if it hasn't already been:
     sample_directory = joinpath(trace_directory, "latest-trace")
     if isdir(sample_directory) && uperm(sample_directory) & 0x2 == 0
-        @info "`$sample_directory` not writable. Skipping `rr pack`."
+        println("`$sample_directory` not writable. Skipping `rr pack`.")
     else
         rr_pack(trace_directory)
     end
 
     mktempdir() do dir
         # Create a compressed tarball
+        # TODO: stream (peak/s5cmd#182)
         tarball = joinpath(dir, "trace.tar")
         Tar.create(trace_directory, tarball)
         zstdmt() do zstdp
             run(`$zstdp --quiet $tarball`)
         end
 
         # Upload
-        println("Uploading trace directory...")
-        S3.put(url, tarball * ".zst"; credentials, region="us-east-1")
-        println("Uploaded to $url")
-
-        # NOTE: we used to perform this in a streaming fashion,
-        #       but CloudStore.jl doesn't support that very well.
+        # TODO: progress bar (peak/s5cmd#51)
+        cmd = `$(s5cmd()) --log error cp $(tarball).zst $url`
+        cmd = addenv(cmd, "AWS_ACCESS_KEY_ID" => access_key_id,
+                          "AWS_SECRET_ACCESS_KEY" => secret_access_key,
+                          "AWS_SESSION_TOKEN" => session_token)
+        run(cmd)
     end
 end
 

test/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/rr.jl

@@ -1,4 +1,4 @@
-using Pkg, CloudStore, CloudBase.CloudTest
+using Pkg, CloudBase.CloudTest
 
 # helper functions to run a command or a block of code while capturing all output
 function communicate(f::Function)
@@ -113,11 +113,16 @@ end
         mkpath(cache_dir)
         mktempdir(cache_dir) do temp_srv_dir
             Minio.with(; public=true, dir=temp_srv_dir) do conf
-                credentials, bucket = conf
-                url = bucket.baseurl * "test.tar.zst"
-                BugReporting.upload_rr_trace(temp_trace_dir; credentials, url)
-
-                test_replay(url)
+                creds, bucket = conf
+                s3_url = "s3://$(bucket.name)/test.tar.zst"
+                http_url = bucket.baseurl * "test.tar.zst"
+                endpoint_url = replace(bucket.baseurl, "$(bucket.name)/"=>"")
+                withenv("S3_ENDPOINT_URL" => endpoint_url) do
+                    BugReporting.upload_rr_trace(temp_trace_dir, s3_url; creds.access_key_id,
+                                                 creds.secret_access_key, creds.session_token)
+                end
+
+                test_replay(http_url)
             end
         end
     end