Skip to content

Commit

Permalink
Refactor curl requests into helper functions
Browse files Browse the repository at this point in the history 
Most curl requests now use a local curl handle that's
created for that request, then destroyed afterwards.

The dataset multi read/write still uses the global handle for now.
  • Loading branch information
@mattjala
mattjala committed Oct 24, 2023
1 parent 9872181 commit da6f361
Show file tree
Hide file tree
Showing 9 changed files with 844 additions and 2,311 deletions.
549 changes: 487 additions & 62 deletions src/rest_vol.c
View file

Large diffs are not rendered by default.

15 changes: 15 additions & 0 deletions src/rest_vol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,12 @@ typedef struct dataset_read_info {

typedef enum transfer_type_t { UNINIT = 0, READ = 1, WRITE = 2 } transfer_type_t;

typedef enum content_type_t {
CONTENT_TYPE_UNINIT = 0,
CONTENT_TYPE_JSON = 1,
CONTENT_TYPE_OCTET_STREAM = 2
} content_type_t;

typedef struct dataset_transfer_info {
struct curl_slist *curl_headers;
char *host_headers;
Expand Down Expand Up @@ -725,6 +731,15 @@ herr_t RV_curl_multi_perform(CURL *curl_multi_ptr, dataset_transfer_info *transf
hid_t file_space_id, void *buf,
struct response_buffer resp_buffer));

/* Helper functions for cURL requests to the server */
long RV_curl_delete(server_info_t *server_info, const char *request_endpoint, const char *filename);
long RV_curl_put(server_info_t *server_info, const char *request_endpoint, const char *filename,
upload_info *uinfo, content_type_t content_type);
long RV_curl_get(server_info_t *server_info, const char *request_endpoint, const char *filename,
content_type_t content_type);
long RV_curl_post(server_info_t *server_info, const char *request_endpoint, const char *filename,
const char *post_data, size_t post_size, content_type_t content_type);

#define SERVER_VERSION_MATCHES_OR_EXCEEDS(version, major_needed, minor_needed, patch_needed) \
(version.major > major_needed) || (version.major == major_needed && version.minor > minor_needed) || \
(version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed)
Expand Down
1,141 changes: 143 additions & 998 deletions src/rest_vol_attr.c
View file

Large diffs are not rendered by default.

140 changes: 20 additions & 120 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,15 +129,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
{
RV_object_t *parent = (RV_object_t *)obj;
RV_object_t *new_dataset = NULL;
curl_off_t create_request_body_len = 0;
size_t host_header_len = 0;
size_t create_request_body_len = 0;
size_t path_size = 0;
size_t path_len = 0;
char *host_header = NULL;
char *create_request_body = NULL;
char request_url[URL_MAX_LENGTH];
const char *base_URL = NULL;
int url_len = 0;
char request_endpoint[URL_MAX_LENGTH];
int url_len = 0;
long http_response;
void *ret_value = NULL;

#ifdef RV_CONNECTOR_DEBUG
Expand All @@ -154,8 +152,6 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na

if (H5I_FILE != parent->obj_type && H5I_GROUP != parent->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object not a file or group");
if ((base_URL = parent->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object does not have valid server URL");

/* Check for write access */
if (!(parent->domain->u.file.intent & H5F_ACC_RDWR))
Expand Down Expand Up @@ -204,42 +200,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
new_dataset->u.dataset.dcpl_id = H5P_DATASET_CREATE_DEFAULT;

/* Form the request body to give the new Dataset its properties */
{
size_t tmp_len = 0;

if (RV_setup_dataset_create_request_body(obj, name, type_id, space_id, lcpl_id, dcpl_id,
&create_request_body, &tmp_len) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL,
"can't convert dataset creation parameters to JSON");

/* Check to make sure that the size of the create request HTTP body can safely be cast to a curl_off_t
*/
if (sizeof(curl_off_t) < sizeof(size_t))
ASSIGN_TO_SMALLER_SIZE(create_request_body_len, curl_off_t, tmp_len, size_t)
else if (sizeof(curl_off_t) > sizeof(size_t))
create_request_body_len = (curl_off_t)tmp_len;
else
ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(create_request_body_len, curl_off_t, tmp_len, size_t)
}

/* Setup the host header */
host_header_len = strlen(parent->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers = curl_slist_append(curl_headers, strncat(host_header, parent->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");
if (RV_setup_dataset_create_request_body(obj, name, type_id, space_id, lcpl_id, dcpl_id,
&create_request_body, &create_request_body_len) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL,
"can't convert dataset creation parameters to JSON");

/* Redirect cURL from the base URL to "/datasets" to create the dataset */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datasets", base_URL)) < 0)
if ((url_len = snprintf(request_endpoint, URL_MAX_LENGTH, "/datasets")) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, NULL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
Expand All @@ -250,33 +217,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
printf("-> Dataset creation request URL: %s\n\n", request_url);
#endif

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, new_dataset->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, new_dataset->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL HTTP headers: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POST, 1))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set up cURL to make HTTP POST request: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDS, create_request_body))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL POST data: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, create_request_body_len))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL POST data size: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL request URL: %s", curl_err_buf);
http_response = RV_curl_post(&new_dataset->domain->u.file.server_info, request_endpoint,
new_dataset->domain->u.file.filepath_name, (const char *)create_request_body,
create_request_body_len, CONTENT_TYPE_JSON);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Creating dataset\n\n");

printf(" /***********************************\\\n");
printf("-> | Making POST request to the server |\n");
printf(" \\***********************************/\n\n");
#endif

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTCREATE, NULL);
if (!HTTP_SUCCESS(http_response))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, NULL, "can't create dataset, response HTTP %ld",
http_response);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Created dataset\n\n");
Expand Down Expand Up @@ -307,19 +254,11 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na

if (create_request_body)
RV_free(create_request_body);
if (host_header)
RV_free(host_header);

/* Clean up allocated dataset object if there was an issue */
if (new_dataset && !ret_value)
if (RV_dataset_close(new_dataset, FAIL, NULL) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close dataset");

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
} /* end if */

PRINT_ERROR_STACK;

return ret_value;
Expand Down Expand Up @@ -1245,7 +1184,7 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
H5VL_file_specific_args_t vol_flush_args;
size_t host_header_len = 0;
char *host_header = NULL;
char request_url[URL_MAX_LENGTH];
char request_endpoint[URL_MAX_LENGTH];
const char *base_URL = NULL;

#ifdef RV_CONNECTOR_DEBUG
Expand All @@ -1258,8 +1197,6 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r

if (H5I_DATASET != dset->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a dataset");
if ((base_URL = dset->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataset does not have valid server URL");

switch (args->op_type) {
/* H5Dget_access_plist */
Expand Down Expand Up @@ -1313,41 +1250,11 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "can't flush datase's domain");

/* Make GET request to dataset with 'verbose' parameter for HSDS. */
snprintf(request_url, URL_MAX_LENGTH, "%s%s%s%s", base_URL, "/datasets/", dset->URI,
"?verbose=1");
snprintf(request_endpoint, URL_MAX_LENGTH, "%s%s%s", "/datasets/", dset->URI, "?verbose=1");

/* Setup the host header */
host_header_len = strlen(dset->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers =
curl_slist_append(curl_headers, strncat(host_header, dset->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, dset->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, dset->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPGET, 1))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL,
"can't set up cURL to make HTTP GET request: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL request URL: %s",
curl_err_buf);

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTGET, FAIL);
if (RV_curl_get(&dset->domain->u.file.server_info, request_endpoint,
dset->domain->u.file.filepath_name, CONTENT_TYPE_JSON) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset");

if (RV_parse_allocated_size_callback(response_buffer.buffer, NULL,
args->args.get_storage_size.storage_size) < 0)
Expand All @@ -1372,13 +1279,6 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
} /* end switch */

done:
if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
}

RV_free(host_header);

PRINT_ERROR_STACK;

return ret_value;
Expand Down
68 changes: 10 additions & 58 deletions src/rest_vol_datatype.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,21 +70,19 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
RV_object_t *new_datatype = NULL;
size_t commit_request_nalloc = 0;
size_t link_body_nalloc = 0;
size_t host_header_len = 0;
size_t datatype_body_len = 0;
size_t path_size = 0;
size_t path_len = 0;
const char *base_URL = NULL;
char *host_header = NULL;
char *commit_request_body = NULL;
char *datatype_body = NULL;
char *link_body = NULL;
char *path_dirname = NULL;
char request_url[URL_MAX_LENGTH];
char request_endpoint[URL_MAX_LENGTH];
int commit_request_len = 0;
int link_body_len = 0;
int url_len = 0;
void *ret_value = NULL;
long http_response;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received datatype commit call with following parameters:\n");
Expand All @@ -103,9 +101,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
if (H5I_FILE != parent->obj_type && H5I_GROUP != parent->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object not a file or group");

if ((base_URL = parent->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object does not have valid server URL");

/* Check for write access */
if (!(parent->domain->u.file.intent & H5F_ACC_RDWR))
FUNC_GOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "no write intent on file");
Expand Down Expand Up @@ -230,61 +225,25 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
printf("-> Datatype commit request body:\n%s\n\n", commit_request_body);
#endif

/* Setup the host header */
host_header_len = strlen(parent->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, NULL, "can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers = curl_slist_append(curl_headers, strncat(host_header, parent->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");

/* Redirect cURL from the base URL to "/datatypes" to commit the datatype */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datatypes", base_URL)) < 0)
if ((url_len = snprintf(request_endpoint, URL_MAX_LENGTH, "/datatypes")) < 0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, NULL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, NULL,
"datatype create URL size exceeded maximum URL size");

#ifdef RV_CONNECTOR_DEBUG
printf("-> Datatype commit URL: %s\n\n", request_url);
printf("-> Datatype commit URL: %s\n\n", request_endpoint);
#endif

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, new_datatype->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, new_datatype->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL HTTP headers: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POST, 1))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set up cURL to make HTTP POST request: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDS, commit_request_body))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL POST data: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)commit_request_len))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL POST data size: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL request URL: %s", curl_err_buf);
http_response = RV_curl_post(&parent->domain->u.file.server_info, request_endpoint,
parent->domain->u.file.filepath_name, (const char *)commit_request_body,
(size_t)commit_request_len, CONTENT_TYPE_JSON);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Committing datatype\n\n");

printf(" /***********************************\\\n");
printf("-> | Making POST request to the server |\n");
printf(" \\***********************************/\n\n");
#endif

CURL_PERFORM(curl, H5E_DATATYPE, H5E_BADVALUE, NULL);
if (!HTTP_SUCCESS(http_response))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, NULL, "couldn't commit datatype: received HTTP %ld",
http_response);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Committed datatype\n\n");
Expand Down Expand Up @@ -312,8 +271,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
RV_free(path_dirname);
if (commit_request_body)
RV_free(commit_request_body);
if (host_header)
RV_free(host_header);
if (datatype_body)
RV_free(datatype_body);
if (link_body)
Expand All @@ -324,11 +281,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
if (RV_datatype_close(new_datatype, FAIL, NULL) < 0)
FUNC_DONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, NULL, "can't close datatype");

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
} /* end if */

PRINT_ERROR_STACK;

return ret_value;
Expand Down
Loading

0 comments on commit da6f361

Please sign in to comment.