diff --git a/README.md b/README.md index b1cbb3b..4504e73 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,13 @@ Julia 1.3 through 1.5 as well. ## API -The public API of `Downloads` consists of three functions and three types: +The public API of `Downloads` consists of the following functions and types: - `download` — download a file from a URL, erroring if it can't be downloaded - `request` — request a URL, returning a `Response` object indicating success - `default_downloader!` - set the default `Downloader` object +- `pushhook!` — add a hook which allows for customizing downloading parameters +- `deletehook!` — remove a previously added parameter customization hook - `Response` — a type capturing the status and other metadata about a request - `RequestError` — an error type thrown by `download` and `request` on error - `Downloader` — an object encapsulating shared resources for downloading @@ -141,6 +143,52 @@ with getting a response at all, then a `RequestError` is thrown or returned. Set the default `Downloader`. If no argument is provided, resets the default downloader so that a fresh one is created the next time the default downloader is needed. +### pushhook! +```jl + pushhook!(hook) -> key +``` +- `hook :: Function` +- `key :: HookKey` + +Add a hook to customize download parameters for all downloads. + +The signature `hook` should be `(easy::Easy, info::Dict) -> Nothing`. +Multiple hooks can be added with repeated calls to `pushhook!`. Hooks are +applied in the order they were added. + +The returned `key` maybe used to remove a previously added `hook` cf. `deletehook!` + +Examples: +```jl +# define hook +hook = (easy, info) -> begin + # allow for long pauses during downloads + # (perhaps for malware scanning) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_LIMIT, 1 #= bytes =#) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME , 200 #= seconds =#) + # other possibilities + # set ca_roots + # disable certificate verification + # block or rewrite URLs +end + +# add hook +key = pushhook!(hook) + +# would fail with default download parameters... +download("https://httpbingo.julialang.org/delay/40", "test.txt") + +# cleanup +deletehook!(key) +``` + +### deletehook! +```jl + deletehook!(key) +``` +- `key :: HookKey` + +Remove a hook previously added with `pushhook!. ### Response ```jl diff --git a/src/Downloads.jl b/src/Downloads.jl index 58e0422..9beebf5 100644 --- a/src/Downloads.jl +++ b/src/Downloads.jl @@ -7,6 +7,8 @@ More generally, the module exports functions and types that provide lower-level for file downloading: - [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded - [`request`](@ref) — request a URL, returning a `Response` object indicating success +- [`pushhook!`](@ref) — add a hook which allows for customizing downloading parameters +- [`deletehook!`](@ref) — remove a previously added parameter customization hook - [`Response`](@ref) — a type capturing the status and other metadata about a request - [`RequestError`](@ref) — an error type thrown by `download` and `request` on error - [`Downloader`](@ref) — an object encapsulating shared resources for downloading @@ -20,7 +22,7 @@ using ArgTools include("Curl/Curl.jl") using .Curl -export download, request, Downloader, Response, RequestError, default_downloader! +export download, request, Downloader, Response, RequestError, default_downloader!, pushhook!, deletehook! ## public API types ## @@ -74,6 +76,90 @@ It is expected to be function taking two arguments: an `Easy` struct and an """ const EASY_HOOK = Ref{Union{Function, Nothing}}(nothing) +## Allow for a set of global hooks that can customize each download (via setting parameters on the +## `Easy` object associated with a request +const HookKey = Int +CURRENT_KEY = 0 +GlobalHookEntry = Tuple{HookKey, Function} +const GLOBAL_HOOK_LOCK = ReentrantLock() +const GLOBAL_HOOKS = Array{GlobalHookEntry,1}(undef, 0) + +## Add hook +""" + pushhook!(hook) -> key + + hook :: Function + key :: HookKey +Add a hook to customize download parameters for all downloads. + +The signature `hook` should be `(easy::Easy, info::Dict) -> Nothing``. +Mulitple hooks can be added with repeated calls to `pushhook!`. Hooks are +applied in the order they were added. + +The returned `key` maybe used to remove a previously added `hook` cf. [deletehook!](@ref) + +Examples: +```jl +# define hook +hook = (easy, info) -> begin + # allow for long pauses during downloads (perhaps for malware scanning) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_LIMIT, 1 #= bytes =#) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME , 200 #= seconds =#) + + # other possibilities + # set ca_roots + # disable certificate verification + # block or rewrite URLs + +end + +# add hook +key = pushhook!(hook) + +# would fail with default download parameters... +download("https://httpbingo.julialang.org/delay/40", "test.txt") + +# cleanup +deletehook!(key) +``` +""" +function pushhook!(hook::Function) :: HookKey + global CURRENT_KEY + key = -1 + lock(GLOBAL_HOOK_LOCK) do + key = CURRENT_KEY + push!(GLOBAL_HOOKS, (key, hook)) + CURRENT_KEY += 1 + end + key +end + +""" + deletehook!(key) + key :: HookKey + +Remove a hook previously added with [`pushhook!`](@ref). +""" +function deletehook!(key::HookKey) + keep = x -> x[1] != key + lock(GLOBAL_HOOK_LOCK) do + count(keep, GLOBAL_HOOKS) < length(GLOBAL_HOOKS) || + warn("Downloads.jl: Hook key $(key) not found in global hooks") + filter!(keep, GLOBAL_HOOKS) + end + nothing +end + +function apply_global_hooks(easy::Easy, info::NamedTuple) + lock(GLOBAL_HOOK_LOCK) do + for (_,h) in GLOBAL_HOOKS + h(easy, info) + end + end + nothing +end + + """ struct Response proto :: String @@ -367,6 +453,8 @@ function request( progress !== nothing && enable_progress(easy) set_ca_roots(downloader, easy) info = (url = url, method = method, headers = headers) + + apply_global_hooks(easy, info) easy_hook(downloader, easy, info) # do the request diff --git a/test/runtests.jl b/test/runtests.jl index 530adcf..919f48d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -586,6 +586,27 @@ include("setup.jl") @test Downloads.content_length(["Accept"=>"*/*",]) === nothing @test Downloads.content_length(["Accept"=>"*/*", "Content-Length"=>"100"]) == 100 end + + @testset "Global easy hooks" begin + trip_wire = 0 + original_hook_count = length(Downloads.GLOBAL_HOOKS) + url = "$server/get" + hook = (easy, info) -> trip_wire += 1 + key1 = pushhook!(hook) + _ = download_body(url) + @test trip_wire == 1 + key2 = pushhook!(hook) + _ = download_body(url) + @test trip_wire == 3 + deletehook!(key1) + _ = download_body(url) + @test trip_wire == 4 + deletehook!(key2) + _ = download_body(url) + @test trip_wire == 4 + + @test length(Downloads.GLOBAL_HOOKS) == original_hook_count + end end Downloads.DOWNLOADER[] = nothing