Skip to content

Commit

Permalink
Merge branch 'HDFGroup:develop' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@hyoklee
hyoklee authored Jan 17, 2025
2 parents b9d65d2 + 3e8aa54 commit 0535c3d
Show file tree
Hide file tree
Showing 13 changed files with 604 additions and 939 deletions.
13 changes: 9 additions & 4 deletions src/H5Cimage.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@
/* Helper routines */
static size_t H5C__cache_image_block_entry_header_size(const H5F_t *f);
static size_t H5C__cache_image_block_header_size(const H5F_t *f);
static herr_t H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf);
static herr_t H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf,
size_t buf_size);
#ifndef NDEBUG /* only used in assertions */
static herr_t H5C__decode_cache_image_entry(const H5F_t *f, const H5C_t *cache_ptr, const uint8_t **buf,
unsigned entry_num);
Expand Down Expand Up @@ -297,7 +298,7 @@ H5C__construct_cache_image_buffer(H5F_t *f, H5C_t *cache_ptr)
/* needed for sanity checks */
fake_cache_ptr->image_len = cache_ptr->image_len;
q = (const uint8_t *)cache_ptr->image_buffer;
status = H5C__decode_cache_image_header(f, fake_cache_ptr, &q);
status = H5C__decode_cache_image_header(f, fake_cache_ptr, &q, cache_ptr->image_len + 1);
assert(status >= 0);

assert(NULL != p);
Expand Down Expand Up @@ -1267,7 +1268,7 @@ H5C__cache_image_block_header_size(const H5F_t *f)
*-------------------------------------------------------------------------
*/
static herr_t
H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf)
H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf, size_t buf_size)
{
uint8_t version;
uint8_t flags;
Expand All @@ -1287,6 +1288,10 @@ H5C__decode_cache_image_header(const H5F_t *f, H5C_t *cache_ptr, const uint8_t *
/* Point to buffer to decode */
p = *buf;

/* Ensure buffer has enough data for signature comparison */
if (H5_IS_BUFFER_OVERFLOW(p, H5C__MDCI_BLOCK_SIGNATURE_LEN, *buf + buf_size - 1))
HGOTO_ERROR(H5E_CACHE, H5E_OVERFLOW, FAIL, "Insufficient buffer size for signature");

/* Check signature */
if (memcmp(p, H5C__MDCI_BLOCK_SIGNATURE, (size_t)H5C__MDCI_BLOCK_SIGNATURE_LEN) != 0)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad metadata cache image header signature");
Expand Down Expand Up @@ -2386,7 +2391,7 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)

/* Decode metadata cache image header */
p = (uint8_t *)cache_ptr->image_buffer;
if (H5C__decode_cache_image_header(f, cache_ptr, &p) < 0)
if (H5C__decode_cache_image_header(f, cache_ptr, &p, cache_ptr->image_len + 1) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTDECODE, FAIL, "cache image header decode failed");
assert((size_t)(p - (uint8_t *)cache_ptr->image_buffer) < cache_ptr->image_len);

Expand Down
2 changes: 1 addition & 1 deletion src/H5Epublic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#include "H5public.h" /* Generic Functions */
#include "H5Ipublic.h" /* Identifiers */

/* Value for the default error stack */
/** Value for the default error stack \since 1.12.2 */
#define H5E_DEFAULT 0 /* (hid_t) */

/**
Expand Down
102 changes: 42 additions & 60 deletions src/H5FDros3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,28 +77,35 @@ typedef struct H5FD_ros3_stats_bin {
* Stores all information needed to maintain access to a single HDF5 file
* that has been stored as a S3 object.
*
* `pub` (H5FD_t)
* pub
*
* Instance of H5FD_t which contains all fields common to all VFDs.
* It must be the first item in this structure, since at higher levels,
* this structure will be treated as an instance of H5FD_t.
*
* `fa` (H5FD_ros3_fapl_t)
* fa
*
* Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data
* needed to "open" the HDF5 file.
*
* `eoa` (haddr_t)
* eoa
*
* End of addressed space in file. After open, it should always
* equal the file size.
*
* `s3r_handle` (s3r_t *)
* s3r_handle
*
* Instance of S3 Request handle associated with the target resource.
* Responsible for communicating with remote host and presenting file
* contents as indistinguishable from a file on the local filesystem.
*
* cache
* cache_size (in bytes)
*
* A simple cache of the first N bytes of the file. Especially useful
* at file open, when we perform several reads that would otherwise
* be uncached.
*
* *** present only if ROS3_SATS is set to enable stats collection ***
*
* `meta` (H5FD_ros3_stats_bin_t[])
Expand All @@ -118,8 +125,8 @@ typedef struct H5FD_ros3_stats_bin {
***************************************************************************/
typedef struct H5FD_ros3_t {
H5FD_t pub;
H5FD_ros3_fapl_t fa;
haddr_t eoa;
H5FD_ros3_fapl_t fa;
s3r_t *s3r_handle;
uint8_t *cache;
size_t cache_size;
Expand Down Expand Up @@ -676,9 +683,9 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_open
*
* Purpose: Create and/or open a file as an HDF5 file.
* Purpose: Create and/or open a file as an HDF5 file
*
* Any flag except H5F_ACC_RDONLY will cause an error.
* Any flag except H5F_ACC_RDONLY will cause an error
*
* `url` param (as received from `H5FD_open()`) must conform to web url:
* NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ]
Expand All @@ -695,16 +702,12 @@ H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
static H5FD_t *
H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
H5FD_ros3_t *file = NULL;
struct tm *now = NULL;
char iso8601now[ISO8601_SIZE];
unsigned char signing_key[SHA256_DIGEST_LENGTH];
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
htri_t token_exists;
char *token;
H5FD_t *ret_value = NULL;
H5FD_ros3_t *file = NULL;
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
char *fapl_token = NULL;
H5FD_t *ret_value = NULL;

FUNC_ENTER_PACKAGE

Expand Down Expand Up @@ -733,47 +736,26 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if (NULL == (fa = (const H5FD_ros3_fapl_t *)H5P_peek_driver_info(plist)))
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "could not get ros3 VFL driver info");

/* Session/security token */
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "failed check for property token in plist");
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get token value");
/* Get the token, if it exists */
if (fa->authenticate) {
htri_t token_exists;

/* Does the token exist in the fapl? */
if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "failed check for property token in plist");

/* If so, get it */
if (token_exists) {
if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &fapl_token) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get token value");
}
}

/* Open file; procedure depends on whether or not the fapl instructs to
* authenticate requests or not.
*/
if (fa->authenticate == true) {
/* Compute signing key (part of AWS/S3 REST API). Can be re-used by
* user/key for 7 days after creation.
*
* TODO: Find way to reuse/share?
*/
now = gmnow();
assert(now != NULL);
if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp");
if (H5FD_s3comms_make_aws_signing_key(signing_key, (const char *)fa->secret_key,
(const char *)fa->aws_region, (const char *)iso8601now) < 0)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "problem while computing signing key");

if (token_exists)
handle = H5FD_s3comms_s3r_open(url, (const char *)fa->aws_region, (const char *)fa->secret_id,
(const unsigned char *)signing_key, (const char *)token);
else
handle = H5FD_s3comms_s3r_open(url, (const char *)fa->aws_region, (const char *)fa->secret_id,
(const unsigned char *)signing_key, "");
}
else
handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL);

if (handle == NULL)
/* If we want to check CURL's say on the matter in a controlled
* fashion, this is the place to do it, but would need to make a
* few minor changes to s3comms `s3r_t` and `s3r_read()`.
*/
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open");
if (NULL == (handle = H5FD__s3comms_s3r_open(url, fa, fapl_token)))
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "s3r_open failed");

/* Create new file struct */
if (NULL == (file = H5FL_CALLOC(H5FD_ros3_t)))
Expand All @@ -789,13 +771,13 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)

/* Cache the initial bytes of the file */
{
size_t filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
size_t filesize = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);

file->cache_size = (filesize < ROS3_MAX_CACHE_SIZE) ? filesize : ROS3_MAX_CACHE_SIZE;

if (NULL == (file->cache = (uint8_t *)H5MM_calloc(file->cache_size)))
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate cache memory");
if (H5FD_s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) < 0)
if (H5FD__s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, NULL, "unable to execute read");
}

Expand All @@ -804,7 +786,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
done:
if (ret_value == NULL) {
if (handle != NULL)
if (H5FD_s3comms_s3r_close(handle) < 0)
if (H5FD__s3comms_s3r_close(handle) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle");
if (file != NULL) {
H5MM_xfree(file->cache);
Expand Down Expand Up @@ -841,7 +823,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
#endif

/* Close the underlying request handle */
if (H5FD_s3comms_s3r_close(file->s3r_handle) < 0)
if (H5FD__s3comms_s3r_close(file->s3r_handle) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle");

/* Release the file info */
Expand Down Expand Up @@ -1054,7 +1036,7 @@ H5FD__ros3_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)

FUNC_ENTER_PACKAGE_NOERR

FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle))
FUNC_LEAVE_NOAPI(H5FD__s3comms_s3r_get_filesize(file->s3r_handle))
} /* end H5FD__ros3_get_eof() */

/*-------------------------------------------------------------------------
Expand Down Expand Up @@ -1107,7 +1089,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
assert(file->s3r_handle);
assert(buf);

filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
filesize = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);

if ((addr > filesize) || ((addr + size) > filesize))
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address");
Expand All @@ -1119,7 +1101,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
memcpy(buf, file->cache + addr, size);
}
else {
if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) < 0)
if (H5FD__s3comms_s3r_read(file->s3r_handle, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");

#ifdef ROS3_STATS
Expand Down
3 changes: 1 addition & 2 deletions src/H5FDros3.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
#define H5FD_ROS3_MAX_SECRET_TOK_LEN 4096

/**
*\struct H5FD_ros3_fapl_t
* \struct H5FD_ros3_fapl_t
* \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3().
*
* \details H5FD_ros_fapl_t is a public structure that is used to pass
Expand Down Expand Up @@ -100,7 +100,6 @@
*
* \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]
* A string which specifies the security key.
*
*/
typedef struct H5FD_ros3_fapl_t {
int32_t version;
Expand Down
Loading

0 comments on commit 0535c3d

Please sign in to comment.