Skip to content

Commit

Permalink
Updates to the CDSAPI I didn't realize it had changed lol
Browse files Browse the repository at this point in the history
  • Loading branch information
@natgeo-wong
natgeo-wong committed Jan 31, 2025
1 parent 044682e commit ffaed35
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 57 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "ERA5Reanalysis"
uuid = "99c076f5-c00e-46eb-a0bd-7ba31777c146"
authors = ["Nathanael Wong <[email protected]>"]
version = "0.3.1"
version = "0.3.2"

[deps]
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
Expand Down
95 changes: 52 additions & 43 deletions src/downloads/cdsapi.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,69 +20,79 @@ function retrieve(
pause :: Real = 120.
)

apikey = string("Basic ", base64encode(ckeys["key"]))

@info "$(now()) - CDSAPI - Sending request to https://cds.climate.copernicus.eu/api/v2/resources/$(dset) ..."
@info "$(now()) - CDSAPI - Sending request to https://cds.climate.copernicus.eu/api/retrieve/v1/processes/$dset/execute ..."
response = HTTP.request(
"POST", ckeys["url"] * "/resources/$(dset)",
["Authorization" => apikey],
body = JSON.json(dkeys),
"POST", ckeys["url"] * "/retrieve/v1/processes/$dset/execute",
["PRIVATE-TOKEN" => ckeys["key"]],
body = JSON.json(Dict("inputs" => dkeys)),
verbose = 0,
retry = true, retries = 19
)
resp_dict = JSON.parse(String(response.body))
data = Dict("state" => "queued")
location = Dict(response.headers)["location"]
data = Dict("status" => "queued")

@info "$(now()) - CDSAPI - Request is queued"; flush(stderr)
sleep_seconds = 1.
while data["state"] == "queued"
data = parserequest(ckeys,resp_dict,apikey)
while data["status"] == "queued"
data = parserequest(ckeys,resp_dict)
sleep_seconds = min(1.5 * sleep_seconds,5)
sleep(sleep_seconds)
end

@info "$(now()) - CDSAPI - Request is running"; flush(stderr)
sleep_seconds = 1.
while data["state"] == "running"
data = parserequest(ckeys,resp_dict,apikey)
sleep_seconds = min(1.5 * sleep_seconds,pause)
while data["status"] != "successful"

data = parserequest(ckeys,resp_dict)
sleep_seconds = min(1.5 * sleep_seconds,5)
sleep(sleep_seconds)
end

if data["state"] == "completed"
if data["status"] == "failed"
@error "$(now()) - CDSAPI - Request failed"
end

end

@info "$(now()) - CDSAPI - Request is completed"; flush(stderr)
if data["status"] == "successful"

sleep(10)
response = HTTP.request(
"GET", location * "/results",
["PRIVATE-TOKEN" => ckeys["key"]]
)
data = JSON.parse(String(response.body))["asset"]["value"]
fsize = data["file:size"]
url = data["href"]

@info """$(now()) - CDSAPI - Downloading $(uppercase(dset)) data ...
URL: $(data["location"])
URL: $(url)
Destination: $(fnc)
"""
flush(stderr)

tries = 0
while isinteger(tries) && (tries < 10)
try
dt1 = now()
HTTP.download(data["location"],fnc,update_period=Inf)
dt2 = now()
tries += 0.5
@info "$(now()) - CDSAPI - Downloaded $(@sprintf("%.1f",data["content_length"]/1e6)) MB in $(@sprintf("%.1f",Dates.value(dt2-dt1)/1000)) seconds (Rate: $(@sprintf("%.1f",data["content_length"]/1e3/Dates.value(dt2-dt1))) MB/s)"
catch
tries += 1
@info "$(now()) - CDSAPI - Failed to download on Attempt $(tries) of 10"
end
flush(stderr)
end

if tries == 10
@warn "$(now()) - CDSAPI - Failed to download data, skipping to next request"
end

elseif data["state"] == "failed"

@error "$(now()) - CDSAPI - Request failed"
dt1 = now()
HTTP.download(url,fnc,update_period=Inf)
dt2 = now()
@info "$(now()) - CDSAPI - Downloaded $(@sprintf("%.1f",fsize/1024^2)) MB in $(@sprintf("%.1f",Dates.value(dt2-dt1)/1000)) seconds (Rate: $(@sprintf("%.1f",fsize/1024^2/Dates.value(dt2-dt1)*1000)) MB/s)"

# tries = 0
# while isinteger(tries) && (tries < 10)
# try
# dt1 = now()
# HTTP.download(url,fnc,update_period=Inf)
# dt2 = now()
# tries += 0.5
# @info "$(now()) - CDSAPI - Downloaded $(@sprintf("%.1f",fsize/1024^2)) MB in $(@sprintf("%.1f",Dates.value(dt2-dt1)/1000)) seconds (Rate: $(@sprintf("%.1f",fsize/1024^2/Dates.value(dt2-dt1)*1000)) MB/s)"
# catch
# tries += 1
# @info "$(now()) - CDSAPI - Failed to download on Attempt $(tries) of 10"
# end
# flush(stderr)
# end

# if tries == 10
# @warn "$(now()) - CDSAPI - Failed to download data, skipping to next request"
# end

end

Expand Down Expand Up @@ -129,12 +139,11 @@ Arguments
function parserequest(
ckeys :: AbstractDict,
resp :: AbstractDict,
api :: AbstractString
)

data = HTTP.request(
"GET", ckeys["url"] * "/tasks/" * string(resp["request_id"]),
["Authorization" => api]
"GET", ckeys["url"] * "/retrieve/v1/jobs/" * string(resp["jobID"]),
["PRIVATE-TOKEN" => ckeys["key"]]
)
data = JSON.parse(String(data.body))

Expand Down Expand Up @@ -164,7 +173,7 @@ Keyword Arguments
"""
function addCDSAPIkey(
key :: AbstractString;
url :: AbstractString = "https://cds.climate.copernicus.eu/api/v2",
url :: AbstractString = "https://cds.climate.copernicus.eu/api",
path :: AbstractString = homedir(),
filename :: AbstractString = ".cdsapirc",
overwrite :: Bool = false
Expand Down
27 changes: 14 additions & 13 deletions src/downloads/cdsretrieve.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,20 @@ function cdsretrieve(
cdsretrieve_area!(e5dkey,ereg)

if !isfile(fnc) || overwrite
tryretrieve = 0
while isinteger(tryretrieve) && (tryretrieve < 20)
try
retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys)
tryretrieve += 0.5
catch
tryretrieve += 1
@info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20"
end
end
if tryretrieve == 20
@warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests"
end
retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys)
# tryretrieve = 0
# while isinteger(tryretrieve) && (tryretrieve < 20)
# try
# retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys)
# tryretrieve += 0.5
# catch
# tryretrieve += 1
# @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20"
# end
# end
# if tryretrieve == 20
# @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests"
# end
end

flush(stderr)
Expand Down

2 comments on commit ffaed35

@natgeo-wong
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

Release Notes:

  • Updating to the new version of CDSAPI, will go back and make things more streamlined later if necessary

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/124128

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.3.2 -m "<description of version>" ffaed350ef9fe84ad258932ff2008a0723af1a19
git push origin v0.3.2

Please sign in to comment.