Skip to content

Commit

Permalink
Set Content-Type when upload Zarr entries
Browse files Browse the repository at this point in the history
  • Loading branch information
jwodder committed Feb 12, 2025
1 parent 1bd44f0 commit 5b502eb
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 8 deletions.
19 changes: 14 additions & 5 deletions dandi/dandiapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -2015,18 +2015,27 @@ def match(self, pattern: str) -> bool:
return False
return True

@property
def download_url(self) -> str:
"""
.. versionadded:: 0.67.0
The URL from which the entry can be downloaded
"""
return str(
URL(self.client.get_url(f"/zarr/{self.zarr_id}/files/")).with_query(
{"prefix": str(self), "download": "true"}
)
)

def get_download_file_iter(
self, chunk_size: int = MAX_CHUNK_SIZE
) -> Callable[[int], Iterator[bytes]]:
"""
Returns a function that when called (optionally with an offset into the
file to start downloading at) returns a generator of chunks of the file
"""
url = str(
URL(self.client.get_url(f"/zarr/{self.zarr_id}/files/")).with_query(
{"prefix": str(self), "download": "true"}
)
)
url = self.download_url

def downloader(start_at: int = 0) -> Iterator[bytes]:
lgr.debug("Starting download from %s", url)
Expand Down
26 changes: 23 additions & 3 deletions dandi/files/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from contextlib import closing
from dataclasses import dataclass, field, replace
from datetime import datetime
import json
import os
import os.path
from pathlib import Path
Expand Down Expand Up @@ -558,13 +559,16 @@ def _upload_zarr_file(
storage_session: RESTFullAPIClient, upload_url: str, item: UploadItem
) -> int:
try:
headers = {"Content-MD5": item.base64_digest}
if item.content_type is not None:
headers["Content-Type"] = item.content_type
with item.filepath.open("rb") as fp:
storage_session.put(
upload_url,
data=fp,
json_resp=False,
retry_if=_retry_zarr_file,
headers={"Content-MD5": item.base64_digest},
headers=headers,
)
except Exception:
post_upload_size_check(item.filepath, item.size, True)
Expand Down Expand Up @@ -643,22 +647,38 @@ class UploadItem:
filepath: Path
digest: str
size: int
content_type: str | None

@classmethod
def from_entry(cls, e: LocalZarrEntry, digest: str) -> UploadItem:
if e.name in {".zarray", ".zattrs", ".zgroup", ".zmetadata"}:
try:
with e.filepath.open("rb") as fp:
json.load(fp)
except Exception:
content_type = None

Check warning on line 659 in dandi/files/zarr.py

View check run for this annotation

Codecov / codecov/patch

dandi/files/zarr.py#L658-L659

Added lines #L658 - L659 were not covered by tests
else:
content_type = "application/json"
else:
content_type = None
return cls(
entry_path=str(e),
filepath=e.filepath,
digest=digest,
size=pre_upload_size_check(e.filepath),
content_type=content_type,
)

@property
def base64_digest(self) -> str:
return b64encode(bytes.fromhex(self.digest)).decode("us-ascii")

def upload_request(self) -> dict[str, str]:
return {"path": self.entry_path, "base64md5": self.base64_digest}
def upload_request(self) -> dict[str, str | None]:
return {
"path": self.entry_path,
"base64md5": self.base64_digest,
"content_type": self.content_type,
}


def _cmp_digests(
Expand Down
12 changes: 12 additions & 0 deletions dandi/tests/test_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,18 @@ def test_upload_zarr_with_excluded_dotfiles(
]


def test_upload_zarr_entry_content_type(new_dandiset, tmp_path):
filepath = tmp_path / "example.zarr"
zarr.save(filepath, np.arange(1000), np.arange(1000, 0, -1))
zf = dandi_file(filepath)
assert isinstance(zf, ZarrAsset)
asset = zf.upload(new_dandiset.dandiset, {"description": "A test Zarr"})
assert isinstance(asset, RemoteZarrAsset)
e = asset.get_entry_by_path(".zgroup")
r = new_dandiset.client.get(e.download_url, json_resp=False)
assert r.headers["Content-Type"] == "application/json"


def test_validate_deep_zarr(tmp_path: Path) -> None:
zarr_path = tmp_path / "foo.zarr"
zarr.save(zarr_path, np.arange(1000), np.arange(1000, 0, -1))
Expand Down

0 comments on commit 5b502eb

Please sign in to comment.