From 35b0ffb9f684beedf4ef041ce2762dae3473b6e2 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 21 Aug 2024 14:52:18 +0200 Subject: [PATCH 01/24] feature-scda: Introduce more section header macros --- src/sc_scda.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index 6f0675eb..540aaa28 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -33,11 +33,19 @@ #define SC_SCDA_VENDOR_STRING_BYTES 20 /**< maximal number of vendor string bytes */ #define SC_SCDA_USER_STRING_FIELD 62 /**< byte count for user string entry including the padding */ +#define SC_SCDA_COUNT_ENTRY 30 /**< byte count of the count variable entry in + the section header including padding */ #define SC_SCDA_COMMON_FIELD (2 + SC_SCDA_USER_STRING_FIELD) /**< byte count for common part of the file section headers */ +#define SC_SCDA_COUNT_FIELD (2 + SC_SCDA_COUNT_ENTRY) /**< byte count of the + complete count + variable entry in + the section header */ +#define SC_SCDA_COUNT_MAX_DIGITS 26 /**< maximal decimal digits count of a count + variable in a section header */ #define SC_SCDA_PADDING_MOD 32 /**< divisor for variable length padding */ /** get a random double in the range [A,B) */ From db724acbff44bc010430f64e9903185bf4ca5565 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 21 Aug 2024 15:46:28 +0200 Subject: [PATCH 02/24] feature-scda: Introduce two versions of mod padding Since we do not want to copy the potentially very big data, we also offer a padding function that only sets the padding bytes but does not copy the input data. For the fixed-length padding the data sizes are smaller and known in advance. That is why, we prefer the inplace padding for the fixed-length padding. However, for the sakes of flexibility and symmetry we will also add non-inplace function for the fixed-length padding. --- src/sc_scda.c | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 540aaa28..8007c796 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -373,18 +373,20 @@ sc_scda_pad_to_mod_len (size_t input_len) /** Pad data to a length that is congurent to 0 modulo \ref SC_SCDA_PADDING_MOD. * * \param [in] input_data The input data. At least \b input_len bytes. + * The input data is required since the padding byte + * content dependt on the trailing input data byte. * \param [in] input_len The length of \b input_data in number of bytes. - * \param [out] output_data On output the padded input data. Must be at least + * \param [out] padding On output the padding bytes. Must be at least * \ref sc_scda_pad_to_mod_len (\b input_len) bytes. */ static void sc_scda_pad_to_mod (const char *input_data, size_t input_len, - char *output_data) + char *padding) { size_t num_pad_bytes; SC_ASSERT (input_len == 0 || input_data != NULL); - SC_ASSERT (output_data != NULL); + SC_ASSERT (padding != NULL); /* compute the number of padding bytes */ num_pad_bytes = sc_scda_pad_to_mod_len (input_len); @@ -392,23 +394,43 @@ sc_scda_pad_to_mod (const char *input_data, size_t input_len, SC_ASSERT (num_pad_bytes >= 6); SC_ASSERT (num_pad_bytes <= SC_SCDA_PADDING_MOD + 6); - sc_scda_copy_bytes (output_data, input_data, input_len); - /* check for last byte to decide on padding format */ if (input_len > 0 && input_data[input_len - 1] == '\n') { /* input data ends with a line break */ - output_data[input_len] = '='; + padding[input_len] = '='; } else { /* add a line break add the beginning of the padding */ - output_data[input_len] = '\n'; + padding[input_len] = '\n'; } - output_data[input_len + 1] = '='; + padding[input_len + 1] = '='; /* append the remaining padding bytes */ - sc_scda_set_bytes (&output_data[input_len + 2], '=', num_pad_bytes - 4); - output_data[input_len + num_pad_bytes - 2] = '\n'; - output_data[input_len + num_pad_bytes - 1] = '\n'; + sc_scda_set_bytes (&padding[2], '=', num_pad_bytes - 4); + padding[num_pad_bytes - 2] = '\n'; + padding[num_pad_bytes - 1] = '\n'; +} + +/** Pad data inplace to a len. that is cong. to 0 mod. \ref SC_SCDA_PADDING_MOD. + * + * \param [in] input_data The input data. At least \b input_len bytes. + * \param [in] input_len The length of \b input_data in number of bytes. + * \param [out] output_data On output the padded input data. Must be at least + * \ref sc_scda_pad_to_mod_len (\b input_len) + + * \b input_len bytes. + */ +static void +sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, + char *output_data) +{ + SC_ASSERT (input_len == 0 || input_data != NULL); + SC_ASSERT (output_data != NULL); + + /* copy the input data */ + sc_scda_copy_bytes (output_data, input_data, input_len); + + /* append the padding */ + sc_scda_pad_to_mod (input_data, input_len, &output_data[input_len]); } /** Checks if \b padded_data is actually padded with respect to @@ -1068,7 +1090,7 @@ sc_scda_fopen_write_header_internal (sc_scda_fcontext_t *fc, current_len += SC_SCDA_COMMON_FIELD; /* pad the file header section */ - sc_scda_pad_to_mod (NULL, 0, &file_header_data[current_len]); + sc_scda_pad_to_mod_inplace (NULL, 0, &file_header_data[current_len]); current_len += SC_SCDA_PADDING_MOD; SC_ASSERT (current_len == SC_SCDA_HEADER_BYTES); From 44562fb0f09cb67b7008ce9cc78fbedc440e0ef2 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 21 Aug 2024 16:08:11 +0200 Subject: [PATCH 03/24] feature-scda: Introduce two versions of fixed padding As announced in the commit description of db724acb we add the analogous changes for the fixed-length paddding. --- src/sc_scda.c | 51 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 8007c796..2460dba4 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -260,6 +260,29 @@ sc_scda_init_nul (char *dest, size_t len) } /** Pad the input data to a fixed length. + * + * \param [in] input_len The length of \b input_len in number of bytes. + * \b input_len must be less or equal to + * \b pad_len - 4. + * \param [in] pad_len The target padding length. + * \param [out] padding On output the padded bytes. + * \b pad_len - \b input_len many bytes. + */ +static void +sc_scda_pad_to_fix_len (size_t input_len, size_t pad_len, char *padding) +{ + SC_ASSERT (padding != NULL); + SC_ASSERT (input_len <= pad_len - 4); + + /* We assume that padding has at least pad_len - input_len allocated bytes. */ + + /* set padding */ + padding[0] = ' '; + sc_scda_set_bytes (&padding[1], '-', pad_len - input_len - 2); + padding[pad_len - input_len - 1] = '\n'; +} + +/** Pad the input data inplace to a fixed length. * * \param [in] input_data The data that is padded. \b input_len many bytes. * \param [in] input_len The length of \b input_len in number of bytes. @@ -270,13 +293,9 @@ sc_scda_init_nul (char *dest, size_t len) * \b output_data in number of bytes. */ static void -sc_scda_pad_to_fix_len (const char *input_data, size_t input_len, - char *output_data, size_t pad_len) +sc_scda_pad_to_fix_len_inplace (const char *input_data, size_t input_len, + char *output_data, size_t pad_len) { -#if 0 - uint8_t *byte_arr; -#endif - SC_ASSERT (input_data != NULL); SC_ASSERT (output_data != NULL); SC_ASSERT (input_len <= pad_len - 4); @@ -287,13 +306,7 @@ sc_scda_pad_to_fix_len (const char *input_data, size_t input_len, sc_scda_copy_bytes (output_data, input_data, input_len); /* append padding */ -#if 0 - byte_arr = (uint8_t *) padded_data; -#endif - output_data[input_len] = ' '; - sc_scda_set_bytes (&output_data[input_len + 1], '-', - pad_len - input_len - 2); - output_data[pad_len - 1] = '\n'; + sc_scda_pad_to_fix_len (input_len, pad_len, &output_data[input_len]); } /** This function checks if \b padded_data is actually padded to \b pad_len. @@ -1018,8 +1031,8 @@ sc_scda_get_common_section_header (char section_char, const char* user_string, output[1] = ' '; /* write \b user_string to \b output including padding */ - sc_scda_pad_to_fix_len (user_string, user_string_len, - &output[2], SC_SCDA_USER_STRING_FIELD); + sc_scda_pad_to_fix_len_inplace (user_string, user_string_len, &output[2], + SC_SCDA_USER_STRING_FIELD); return invalid_user_string; } @@ -1056,10 +1069,10 @@ sc_scda_fopen_write_header_internal (sc_scda_fcontext_t *fc, file_header_data[current_len++] = ' '; /* vendor string */ - sc_scda_pad_to_fix_len (SC_SCDA_VENDOR_STRING, - strlen (SC_SCDA_VENDOR_STRING), - &file_header_data[current_len], - SC_SCDA_VENDOR_STRING_FIELD); + sc_scda_pad_to_fix_len_inplace (SC_SCDA_VENDOR_STRING, + strlen (SC_SCDA_VENDOR_STRING), + &file_header_data[current_len], + SC_SCDA_VENDOR_STRING_FIELD); current_len += SC_SCDA_VENDOR_STRING_FIELD; /* get common file section header part */ From f8f8b08d98a3f6c2d6b6e975db7bd2a97d431be7 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 21 Aug 2024 16:27:27 +0200 Subject: [PATCH 04/24] feature-scda: Fix offsets --- src/sc_scda.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 2460dba4..c7ca1e8f 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -410,13 +410,13 @@ sc_scda_pad_to_mod (const char *input_data, size_t input_len, /* check for last byte to decide on padding format */ if (input_len > 0 && input_data[input_len - 1] == '\n') { /* input data ends with a line break */ - padding[input_len] = '='; + padding[0] = '='; } else { /* add a line break add the beginning of the padding */ - padding[input_len] = '\n'; + padding[0] = '\n'; } - padding[input_len + 1] = '='; + padding[1] = '='; /* append the remaining padding bytes */ sc_scda_set_bytes (&padding[2], '=', num_pad_bytes - 4); From b4a333dbff9d2d574fb7b278636b9aee19305fc4 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 21 Aug 2024 16:47:22 +0200 Subject: [PATCH 05/24] feature-scda: Implement fwrite_block --- src/sc_scda.c | 209 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_scda.c | 7 ++ 2 files changed, 216 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index c7ca1e8f..a9c3c9a4 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1341,6 +1341,215 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, return fc; } +/** Write the specified count field to \b output. + * + * Since this function is dedicated to be executed in serial we assume that + * the length of the variable is collectively checked before this function is + * called. + * + * \param [in] ident The char that identifies the count variable. + * \param [in] var The count variable, which must be representable by + * at most \ref SC_SCDA_COUNT_MAX_DIGITS decimal + * digits. + * \param [out] output The specified count field. Must be at least \ref + * SC_SCDA_COUNT_FIELD bytes. + */ +static void +sc_scda_get_section_header_entry (char ident, size_t var, char *output) +{ + char var_str[SC_SCDA_COUNT_MAX_DIGITS + 1]; /* + 1 for trailing nul */ +#ifdef SC_ENABLE_DEBUG + long long unsigned cmp; +#endif + + SC_ASSERT (output != NULL); + + /* write the identifier */ + output[0] = ident; + output[1] = ' '; + + /* get var as string */ + snprintf (var_str, SC_SCDA_COUNT_MAX_DIGITS + 1, "%llu", + (long long unsigned) var); + + SC_ASSERT (strlen (var_str) > 0); + SC_ASSERT (strlen (var_str) <= SC_SCDA_COUNT_MAX_DIGITS); + +#ifdef SC_ENABLE_DEBUG + /* verify content of var_str */ + SC_ASSERT (sscanf (var_str, "%llu", &cmp) == 1); + SC_ASSERT (cmp == (unsigned long long) var); +#endif + + /* pad var_str */ + sc_scda_pad_to_fix_len_inplace (var_str, strlen (var_str), &output[2], + SC_SCDA_COUNT_ENTRY); +} + +/** Internal function to write the block section header. + * + * This function is dedicated to be called in \ref sc_scda_fwrite_block. + * + * \param [in] fc The file context as in \ref sc_scda_fwrite_block + * before running the first serial code part. + * \param [in] user_string As in the documentation of \ref + * sc_scda_fwrite_block. + * \param [in] len As in the documentation of \ref + * sc_scda_fwrite_block. + * \param [in] block_size As in the documentation of \ref + * sc_scda_fwrite_block. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fwrite_block_header_internal (sc_scda_fcontext_t *fc, + const char *user_string, size_t *len, + size_t block_size, int *count_err, + sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + int current_len; + int invalid_user_string; + int header_len; + char header_data[SC_SCDA_COMMON_FIELD + SC_SCDA_COUNT_FIELD]; + + *count_err = 0; + + header_len = SC_SCDA_COMMON_FIELD + SC_SCDA_COUNT_FIELD; + + /* get block file section header */ + + /* section-identifying character */ + current_len = 0; + + invalid_user_string = + sc_scda_get_common_section_header ('B', user_string, len, header_data); + /* We always translate the error code to have full coverage for the fuzzy + * error testing. + */ + sc_scda_scdaret_to_errcode (invalid_user_string ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid user string"); + + current_len += SC_SCDA_COMMON_FIELD; + + /* get count variable entry */ + sc_scda_get_section_header_entry ('E', block_size, + &header_data[current_len]); + current_len += SC_SCDA_COUNT_FIELD; + + SC_ASSERT (current_len == header_len); + + /* write block section header */ + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, header_data, + header_len, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block section header"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (header_len, count, count_err); +} + +/** Internal function to write the block section data. + * + * This function is dedicated to be called in \ref sc_scda_fwrite_block. + * + * \param [in] fc The file context as in \ref sc_scda_fwrite_block + * before running the first serial code part. + * \param [in] block_data As in the documentation of \ref + * sc_scda_fwrite_block. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fwrite_block_data_internal (sc_scda_fcontext_t *fc, + sc_array_t * block_data, size_t block_size, + int *count_err, sc_scda_ferror_t * errcode) +{ + int mpiret; + int count; + int invalid_block_data; + size_t num_pad_bytes; + /* \ref SC_SCDA_PADDING_MOD + 6 is the maximum number of mod padding bytes */ + char padding[SC_SCDA_PADDING_MOD + 6]; + + *count_err = 0; + + /* check block data */ + invalid_block_data = !(block_data->elem_size == block_size && + block_data->elem_count == 1); + sc_scda_scdaret_to_errcode (invalid_block_data ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid block data"); + + /* write the block data to the file section */ + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes, block_data->array, + block_size, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block data"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); + + /* get the padding bytes */ + num_pad_bytes = sc_scda_pad_to_mod_len (block_size); + sc_scda_pad_to_mod (block_data->array, block_size, padding); + + /* write the padding bytes */ + mpiret = sc_io_write_at (fc->file, fc->accessed_bytes + block_size, + padding, (int) num_pad_bytes, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block data padding"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); +} + +sc_scda_fcontext_t * +sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, + size_t *len, sc_array_t * block_data, size_t block_size, + int root, int encode, sc_scda_ferror_t * errcode) +{ + int count_err; + size_t num_pad_bytes; + + SC_ASSERT (fc != NULL); + SC_ASSERT (user_string != NULL); + SC_ASSERT (root >= 0); + /* block_data is ignored on all ranks except of root */ + SC_ASSERT (fc->mpirank != root || block_data != NULL); + SC_ASSERT (errcode != NULL); + + /* TODO: Check if block_size is collective. */ + + /* TODO: respect encode parameter */ + + /* The file header section is always written and read on rank 0. */ + if (fc->mpirank == 0) { + sc_scda_fwrite_block_header_internal (fc, user_string, len, block_size, + &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + + /* add number of written bytes */ + fc->accessed_bytes += SC_SCDA_COMMON_FIELD + SC_SCDA_COUNT_FIELD; + + /* The block data is written on the the user-defined rank root. */ + if (fc->mpirank == root) { + sc_scda_fwrite_block_data_internal (fc, block_data, block_size, + &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); + + /* get number of padding bytes to update internal file pointer */ + num_pad_bytes = sc_scda_pad_to_mod_len (block_size); + + fc->accessed_bytes += (sc_MPI_Offset) (block_size + num_pad_bytes); + + return fc; +} + /** Check a read file header section and extract the user string. * * \param [in] file_header_data The read file header data as a byte buffer. diff --git a/test/test_scda.c b/test/test_scda.c index d5cab3e7..6108edbb 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -169,6 +169,13 @@ main (int argc, char **argv) SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fwrite_inline with empty user string failed"); + /* write a block section */ + fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, 32, + mpisize - 1, 0, &errcode); + /* TODO: check errcode */ + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "scda_fwrite_block failed"); + sc_scda_fclose (fc, &errcode); /* TODO: check errcode and return value */ SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), From a30bd070ce76a04eadb69ea7f408318f4c618f6a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 22 Aug 2024 10:38:58 +0200 Subject: [PATCH 06/24] feature-scda: Check count length --- src/sc_scda.c | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index a9c3c9a4..c7893e4d 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1343,9 +1343,9 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, /** Write the specified count field to \b output. * - * Since this function is dedicated to be executed in serial we assume that - * the length of the variable is collectively checked before this function is - * called. + * The number of decimal digits is checked in this function. This function is + * only called in serial code places but we assume that it was checked in + * advance if the count variable is collective. * * \param [in] ident The char that identifies the count variable. * \param [in] var The count variable, which must be representable by @@ -1353,11 +1353,13 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, * digits. * \param [out] output The specified count field. Must be at least \ref * SC_SCDA_COUNT_FIELD bytes. + * \return True if count is too large and false otherwise. */ -static void +static int sc_scda_get_section_header_entry (char ident, size_t var, char *output) { - char var_str[SC_SCDA_COUNT_MAX_DIGITS + 1]; /* + 1 for trailing nul */ + char var_str[BUFSIZ]; + size_t len; #ifdef SC_ENABLE_DEBUG long long unsigned cmp; #endif @@ -1369,11 +1371,18 @@ sc_scda_get_section_header_entry (char ident, size_t var, char *output) output[1] = ' '; /* get var as string */ - snprintf (var_str, SC_SCDA_COUNT_MAX_DIGITS + 1, "%llu", - (long long unsigned) var); + /* BUFSIZ must be larger than \ref SC_SCDA_COUNT_MAX_DIGITS + 1 to ensure that + * this code is working. + * According to C89 section 4.9.2 BUFSIZ shall be at least 256. + */ + snprintf (var_str, BUFSIZ, "%llu", (long long unsigned) var); + len = strlen (var_str); - SC_ASSERT (strlen (var_str) > 0); - SC_ASSERT (strlen (var_str) <= SC_SCDA_COUNT_MAX_DIGITS); + SC_ASSERT (len > 0); + if (len > SC_SCDA_COUNT_MAX_DIGITS) { + /* count value is too large */ + return -1; + } #ifdef SC_ENABLE_DEBUG /* verify content of var_str */ @@ -1382,8 +1391,10 @@ sc_scda_get_section_header_entry (char ident, size_t var, char *output) #endif /* pad var_str */ - sc_scda_pad_to_fix_len_inplace (var_str, strlen (var_str), &output[2], + sc_scda_pad_to_fix_len_inplace (var_str, len, &output[2], SC_SCDA_COUNT_ENTRY); + + return 0; } /** Internal function to write the block section header. @@ -1412,7 +1423,7 @@ sc_scda_fwrite_block_header_internal (sc_scda_fcontext_t *fc, int mpiret; int count; int current_len; - int invalid_user_string; + int invalid_user_string, invalid_count; int header_len; char header_data[SC_SCDA_COMMON_FIELD + SC_SCDA_COUNT_FIELD]; @@ -1437,8 +1448,12 @@ sc_scda_fwrite_block_header_internal (sc_scda_fcontext_t *fc, current_len += SC_SCDA_COMMON_FIELD; /* get count variable entry */ - sc_scda_get_section_header_entry ('E', block_size, - &header_data[current_len]); + invalid_count = sc_scda_get_section_header_entry ('E', block_size, + &header_data[current_len]); + sc_scda_scdaret_to_errcode (invalid_count ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count"); + current_len += SC_SCDA_COUNT_FIELD; SC_ASSERT (current_len == header_len); From a4457164ae0ae85484f51c695d90a071627f1136 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 22 Aug 2024 17:38:39 +0200 Subject: [PATCH 07/24] feature-scda: Read block section header --- src/sc_scda.c | 173 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 9 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index c7893e4d..ac0f77e8 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1806,6 +1806,9 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, case 'I': *type = 'I'; break; + case 'B': + *type = 'B'; + break; default: /* an invalid/unsupported format */ wrong_format = 1; @@ -1838,6 +1841,126 @@ sc_scda_fread_section_header_common_internal (sc_scda_fcontext_t *fc, "Invalid user string in section header"); } +/** Internal function to read a count entry in a section header. + * + * This code is only valid to be run in serial. It is crucial that + * fc->accessed_bytes is not updated inside of this function. Therefore, + * fc->accessed_bytes must be updated accordingly (and collectivly) afterwards. + * + * This function is already prepared to be adjusted in the future to read a + * specified number of count entries. This will change the interface of this + * function. The reason that we will read multiple count entries in one function + * call instead of calling one function multiple times is that the error + * management for serial code parts relies on having one function that executes + * the serial code. + * + * \param [in] fc The file context as in \ref + * sc_scda_fread_section_header before running the + * second serial code part. + * \param [out] ident The character that identifies the count entry. + * If this character is not conforming to the scda + * convention, the function call is not completed and + * results in \ref SC_SCDA_FERR_FORMAT as error, + * cf. \b errcode. + * \param [in] expc_ident The expected count entry identifier. If the \b ident + * is not as expected the error \ref + * SC_SCDA_FERR_FORMAT is set and the function flow is + * not completed. + * \param [out] count_var The count variable read from the count entry. + * \param [out] count_err A Boolean indicating if a count error occurred. + * For this parameter the term count refers to the + * expected byte count for reading count (different + * count) entry. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, + char expc_ident, size_t *count_var, + int *count_err, sc_scda_ferror_t *errcode) +{ + char count_entry[SC_SCDA_COUNT_FIELD]; + char var_str[SC_SCDA_COUNT_MAX_DIGITS + 1]; + int mpiret; + int count; + int wrong_format, wrong_ident; + long long unsigned read_count; + size_t len; + + SC_ASSERT (fc != NULL); + SC_ASSERT (ident != NULL); + SC_ASSERT (count_var != NULL); + + *count_err = 0; + *count_var = 0; + + /* read a count entry */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, + SC_SCDA_COUNT_FIELD, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read a count entry in a section header"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (SC_SCDA_COUNT_FIELD, count, count_err); + + wrong_format = 0; + /* check and get the count variable identifier */ + switch (count_entry[0]) + { + case 'E': + *ident = 'E'; + break; + default: + /* invalid/unsupported count identifier */ + wrong_format = 1; + break; + } + sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count identifier"); + + /* compare read count identifier to the expected count identifier */ + wrong_ident = (*ident != expc_ident); + sc_scda_scdaret_to_errcode (wrong_ident ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Wrong count identifier in count entry"); + + /* check count entry format */ + if (count_entry[1] != ' ') { + /* wrong format */ + wrong_format = 1; + } + sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Missing space in count entry"); + + /* check padding and extract count variable string */ + sc_scda_init_nul (var_str, SC_SCDA_COUNT_MAX_DIGITS + 1); + if (sc_scda_get_pad_to_fix_len (&count_entry[2], SC_SCDA_COUNT_ENTRY, var_str, + &len)) { + wrong_format = 1; + } + sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid count variable padding"); + + /* If the padding to the length \ref SC_SCDA_COUNT_ENTRY was valid this + * assertion must hold. + */ + SC_ASSERT (len <= SC_SCDA_COUNT_MAX_DIGITS); + + /* get count variable value */ + /* The initialization above guarantees that var_str is nul-terminated. */ + if (len == 0 || sscanf (var_str, "%llu", &read_count) != 1) { + /* conversion failed or is not possible */ + wrong_format = 1; + } + sc_scda_scdaret_to_errcode (wrong_format ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Extraction of count value failed"); + + *count_var = (size_t) read_count; +} + sc_scda_fcontext_t * sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, size_t *len, char *type, size_t *elem_count, @@ -1846,6 +1969,8 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, { int count_err; int mpiret; + const int header_root = 0; + char var_ident; SC_ASSERT (fc != NULL); SC_ASSERT (user_string != NULL); @@ -1856,33 +1981,63 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, SC_ASSERT (errcode != NULL); /* read the common section header part first */ - if (fc->mpirank == 0) { + if (fc->mpirank == header_root) { sc_scda_fread_section_header_common_internal (fc, type, user_string, len, &count_err, errcode); } - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, header_root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, header_root, fc); fc->accessed_bytes += SC_SCDA_COMMON_FIELD; + if (fc->mpirank == header_root) { + /* read count entries */ + switch (*type) + { + case 'I': + /* no count entries to read */ + break; + case 'B': + /* one count entry to read */ + sc_scda_fread_count_entry_internal (fc, &var_ident, 'E', elem_size, + &count_err, errcode); + break; + default: + /* rank header_root already checked if type is valid/supported */ + SC_ABORT_NOT_REACHED (); + } + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, header_root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, header_root, fc); + /* Bcast type and user string */ - mpiret = sc_MPI_Bcast (type, 1, sc_MPI_CHAR, 0, fc->mpicomm); + mpiret = sc_MPI_Bcast (type, 1, sc_MPI_CHAR, header_root, fc->mpicomm); SC_CHECK_MPI (mpiret); mpiret = sc_MPI_Bcast (user_string, SC_SCDA_USER_STRING_BYTES + 1, - sc_MPI_BYTE, 0, fc->mpicomm); + sc_MPI_BYTE, header_root, fc->mpicomm); SC_CHECK_MPI (mpiret); - /* further code flow depends on the file section type */ + /* set global outputs and Bcast the counts if it is necessary */ switch (*type) { + /* set elem_count and elem_size according to the scda convention */ + /* TODO: Handle decode parameter */ case 'I': /* inline */ - /* set elem_count and elem_size according to the scda convention */ *elem_count = 0; *elem_size = 0; - /* TODO: Handle decode parameter */ + break; + case 'B': + /* block */ + *elem_count = 0; + /* elem_size was read on rank header_root */ + mpiret = sc_MPI_Bcast (elem_size, sizeof (size_t), sc_MPI_BYTE, header_root, + fc->mpicomm); + SC_CHECK_MPI (mpiret); + /* one count entry was read */ + fc->accessed_bytes += SC_SCDA_COUNT_FIELD; break; default: - /* rank 0 already checked if type is valid/supported */ + /* rank header_root already checked if type is valid/supported */ SC_ABORT_NOT_REACHED (); } From edc9f94db8a5dba9a27de3ff6fe4f3cf0563a9a1 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Thu, 22 Aug 2024 19:17:31 +0200 Subject: [PATCH 08/24] feature-scda: Start fread_block_data --- src/sc_scda.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index ac0f77e8..5c92ad8f 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2122,6 +2122,88 @@ sc_scda_fread_inline_data (sc_scda_fcontext_t *fc, sc_array_t *data, int root, return fc; } +/** Internal function to read the block data. + * + * \param [in] fc The file context as in \ref + * sc_scda_fread_block_data before running the + * serial code part. + * \param [out] data As in the documentation of \ref + * sc_scda_fread_block_data. + * \param [in] block_size As in the documentation of \ref + * sc_scda_fread_block_data. + * \param [out] count_err A Boolean indicating if a count error occurred. + * \param [out] errcode An errcode that can be interpreted by \ref + * sc_scda_ferror_string or mapped to an error class + * by \ref sc_scda_ferror_class. + */ +static void +sc_scda_fread_block_data_serial_internal (sc_scda_fcontext_t *fc, + sc_array_t *data, size_t block_size, + int *count_err, + sc_scda_ferror_t *errcode) +{ + int mpiret; + int count; + int invalid_array, invalid_padding; + size_t num_pad_bytes; + + *count_err = 0; + + /* check the passed sc_array */ + invalid_array = !(data->elem_count == 1 && data->elem_size == block_size); + sc_scda_scdaret_to_errcode (invalid_array ? SC_SCDA_FERR_ARG : + SC_SCDA_FERR_SUCCESS, errcode, fc); + + /* TODO: Check the padding before reading the data */ + /* to this end we need a function that checks this given the padding length */ + + /* read block data */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, data->array, + (int) block_size, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read inline data"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); +} + +sc_scda_fcontext_t * +sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, + size_t block_size, int root, + sc_scda_ferror_t *errcode) +{ + int count_err; + int wrong_usage; + + SC_ASSERT (fc != NULL); + SC_ASSERT (root >= 0); + SC_ASSERT (errcode != NULL); + + /* It is necessary that sc_scda_fread_section_header was called as last + * function call on fc and that it returned the block section type. + */ + wrong_usage = !(fc->header_before && fc->last_type == 'B'); + sc_scda_scdaret_to_errcode (wrong_usage ? SC_SCDA_FERR_USAGE : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "Wrong usage of scda functions"); + + if (block_data != NULL) { + /* the data is not skipped */ + if (fc->mpirank == root) { + sc_scda_fread_block_data_serial_internal (fc, block_data, block_size, + &count_err, errcode); + } + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, root, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, root, fc); + } + + /* if no error occurred, we move the internal file pointer */ + fc->accessed_bytes += (sc_MPI_Offset) block_size; + + /* last function call can not be \ref sc_scda_fread_section_header anymore */ + fc->header_before = 0; + + return fc; +} + int sc_scda_fclose (sc_scda_fcontext_t * fc, sc_scda_ferror_t * errcode) { From ccec211251fe4cca816133e81a06a0be9554b775 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 26 Aug 2024 10:13:34 +0200 Subject: [PATCH 09/24] feature-scda: Typo --- src/sc_scda.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sc_scda.h b/src/sc_scda.h index 48a94eaa..e5aead2e 100644 --- a/src/sc_scda.h +++ b/src/sc_scda.h @@ -830,7 +830,7 @@ sc_scda_fcontext_t *sc_scda_fread_inline_data (sc_scda_fcontext_t * fc, * sc_scda_fread_section_header. This preceding call gives also the required * \b block_size. * \note - * All parameters except of \b data_block are collective. + * All parameters except of \b block_data are collective. * * This function returns NULL on I/O errors. * From 3077b7f73509f0f1172ffd0aadff97644b9905ee Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 14:14:24 +0200 Subject: [PATCH 10/24] feature-scda: Padding macro and assert fix --- src/sc_scda.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 5c92ad8f..fe4ac106 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -47,6 +47,8 @@ #define SC_SCDA_COUNT_MAX_DIGITS 26 /**< maximal decimal digits count of a count variable in a section header */ #define SC_SCDA_PADDING_MOD 32 /**< divisor for variable length padding */ +#define SC_SCDA_PADDING_MOD_MAX (6 + SC_SCDA_PADDING_MOD) /**< maximal count of + mod padding bytes */ /** get a random double in the range [A,B) */ #define SC_SCDA_RAND_RANGE(A, B, state) ((A) + sc_rand (state) * ((B) - (A))) @@ -404,8 +406,8 @@ sc_scda_pad_to_mod (const char *input_data, size_t input_len, /* compute the number of padding bytes */ num_pad_bytes = sc_scda_pad_to_mod_len (input_len); - SC_ASSERT (num_pad_bytes >= 6); - SC_ASSERT (num_pad_bytes <= SC_SCDA_PADDING_MOD + 6); + SC_ASSERT (num_pad_bytes >= 7); + SC_ASSERT (num_pad_bytes <= SC_SCDA_PADDING_MOD_MAX); /* check for last byte to decide on padding format */ if (input_len > 0 && input_data[input_len - 1] == '\n') { From 3947d4f25dc45329b27c2487ea3aa3b52ebc0088 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 14:51:35 +0200 Subject: [PATCH 11/24] feature-scda: Factor out mod padding check --- src/sc_scda.c | 89 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index fe4ac106..d0b3cd61 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -448,64 +448,99 @@ sc_scda_pad_to_mod_inplace (const char *input_data, size_t input_len, sc_scda_pad_to_mod (input_data, input_len, &output_data[input_len]); } -/** Checks if \b padded_data is actually padded with respect to - * \ref SC_SCDA_PADDING_MOD. +/** Check if the padding bytes are correct w.r.t. \ref SC_SCDA_PADDING_MOD. * - * Given the raw data length, this function checks if the padding format is - * correct and extracts the raw data. + * Since the mod padding depends on the trailing data byte this function also + * requires the raw data. * - * \param [in] padded_data The padded data with byte count \b padded_len. - * \param [in] padded_len The length of \b padded_data in number of bytes. + * \param [in] data The raw data with byte count \b data_len. + * \param [in] data_len The length of \b data in number of bytes. + * \param [in] pad The padding bytes with byte count \b pad_len. + * \param [in] pad_len The length of \b pad in number of bytes. * Must be at least 7. - * \param [in] raw_len The length of the raw data in \b padded_data - * in number of bytes. This value must be known by - * the user. - * \param [out] raw_data On output the raw data from \b padded_data if - * the function returns true and undefined otherwise. - * \return True if \b padded_data does not satisfy the scda + * \return True if \b pad does not satisfy the scda * padding convention for padding to a modulo * condition. False, otherwise. */ static int -sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, - size_t raw_len, char *raw_data) +sc_scda_check_pad_to_mod (const char* data, size_t data_len, const char *pad, + size_t pad_len) { size_t si; size_t num_pad_bytes; - SC_ASSERT (padded_data != NULL); - SC_ASSERT (raw_len == 0 || raw_data != NULL); + SC_ASSERT (pad != NULL); - num_pad_bytes = sc_scda_pad_to_mod_len (raw_len); + num_pad_bytes = sc_scda_pad_to_mod_len (data_len); /* check if padding data length conforms to the padding format */ - if (num_pad_bytes + raw_len != padded_len) { - /* raw_len and padded_len are not consistent */ + if (num_pad_bytes != pad_len) { + /* data_len and pad_len are not consistent */ return -1; } - SC_ASSERT (padded_len >= 7); + SC_ASSERT (pad_len >= 7); /* check the content of the padding bytes */ - if (padded_data[padded_len - 1] != '\n' || - padded_data[padded_len - 2] != '\n') { + if (pad[pad_len - 1] != '\n' || + pad[pad_len - 2] != '\n') { /* terminating line breaks are missing */ return -1; } - for (si = padded_len - 3; si != padded_len - num_pad_bytes; --si) { - if (padded_data[si] != '=') { + for (si = pad_len - 3; si != 0; --si) { + if (pad[si] != '=') { /* wrong padding character */ return -1; } } - SC_ASSERT (si == raw_len); - if ((!((padded_data[si] == '=' && raw_len != 0 && - padded_data[si - 1] == '\n') || padded_data[si] == '\n'))) { + /* padding depends on the trailing data byte */ + if ((!((pad[si] == '=' && data_len != 0 && + data[data_len - 1] == '\n') || pad[si] == '\n'))) { /* wrong padding start */ return -1; } + /* correct padding bytes */ + return 0; +} + +/** Checks if \b padded_data is actually padded with respect to + * \ref SC_SCDA_PADDING_MOD. + * + * Given the raw data length, this function checks if the padding format is + * correct and extracts the raw data. + * + * \param [in] padded_data The padded data with byte count \b padded_len. + * \param [in] padded_len The length of \b padded_data in number of bytes. + * Must be at least 7. + * \param [in] raw_len The length of the raw data in \b padded_data + * in number of bytes. This value must be known by + * the user. + * \param [out] raw_data On output the raw data from \b padded_data if + * the function returns true and undefined otherwise. + * \return True if \b padded_data does not satisfy the scda + * padding convention for padding to a modulo + * condition. False, otherwise. + */ +static int +sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, + size_t raw_len, char *raw_data) +{ + SC_ASSERT (padded_data != NULL); + SC_ASSERT (raw_len == 0 || raw_data != NULL); + + if (padded_len < raw_len || padded_len - raw_len < 7) { + /* invalid lengths */ + return -1; + } + + if (sc_scda_check_pad_to_mod (padded_data, raw_len, &padded_data[raw_len], + padded_len - raw_len)) { + /* invalid padding bytes */ + return -1; + } + if (raw_len != 0) { /* get the raw data if we required raw_data != NULL */ sc_scda_copy_bytes (raw_data, padded_data, raw_len); From 5d9e7ffec3b1fe7e9d73ed3d1dd8b09d3031757b Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 14:54:08 +0200 Subject: [PATCH 12/24] feature-scda: Check padding in fread_block_data --- src/sc_scda.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index d0b3cd61..b34ceba4 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2183,6 +2183,7 @@ sc_scda_fread_block_data_serial_internal (sc_scda_fcontext_t *fc, int count; int invalid_array, invalid_padding; size_t num_pad_bytes; + char paddding[SC_SCDA_PADDING_MOD_MAX]; *count_err = 0; @@ -2190,9 +2191,7 @@ sc_scda_fread_block_data_serial_internal (sc_scda_fcontext_t *fc, invalid_array = !(data->elem_count == 1 && data->elem_size == block_size); sc_scda_scdaret_to_errcode (invalid_array ? SC_SCDA_FERR_ARG : SC_SCDA_FERR_SUCCESS, errcode, fc); - - /* TODO: Check the padding before reading the data */ - /* to this end we need a function that checks this given the padding length */ + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid block array during reading"); /* read block data */ mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, data->array, @@ -2200,6 +2199,24 @@ sc_scda_fread_block_data_serial_internal (sc_scda_fcontext_t *fc, sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read inline data"); SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); + + num_pad_bytes = sc_scda_pad_to_mod_len (block_size); + + /* read the padding the bytes */ + /* the padding depends on the trailing byte of the data */ + mpiret = sc_io_read_at (fc->file, fc->accessed_bytes + + (sc_MPI_Offset) block_size, paddding, + (int) num_pad_bytes, sc_MPI_BYTE, &count); + sc_scda_mpiret_to_errcode (mpiret, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Read inline data padding"); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); + + /* check the padding */ + invalid_padding = sc_scda_check_pad_to_mod (data->array, block_size, paddding, + num_pad_bytes); + sc_scda_scdaret_to_errcode (invalid_padding ? SC_SCDA_FERR_FORMAT : + SC_SCDA_FERR_SUCCESS, errcode, fc); + SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Invalid block data padding"); } sc_scda_fcontext_t * From 9b0b3a277df2dedbcb31e5e2679ecbf26c62be82 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 15:42:06 +0200 Subject: [PATCH 13/24] feature-scda: First test of reading block sec. --- test/test_scda.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 6108edbb..3afc4f5f 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -47,7 +47,7 @@ main (int argc, char **argv) sc_options_t *opt; sc_array_t data; const char *inline_data = "Test inline data \n"; - char read_inline_data[SC_SCDA_INLINE_FIELD]; + char read_data[SC_SCDA_INLINE_FIELD]; mpiret = sc_MPI_Init (&argc, &argv); SC_CHECK_MPI (mpiret); @@ -200,12 +200,12 @@ main (int argc, char **argv) "Identifying section type"); /* read inline data */ - sc_array_init_data (&data, read_inline_data, SC_SCDA_INLINE_FIELD, 1); + sc_array_init_data (&data, read_data, SC_SCDA_INLINE_FIELD, 1); fc = sc_scda_fread_inline_data (fc, &data, 0, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fread_inline_data failed"); SC_CHECK_ABORT (mpirank != 0 - || !strncmp (read_inline_data, inline_data, + || !strncmp (read_data, inline_data, SC_SCDA_INLINE_FIELD), "inline data mismatch"); SC_INFOF ("Read file section header of type %c with user string: %s\n", @@ -217,13 +217,32 @@ main (int argc, char **argv) &elem_count, &elem_size, &decode, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), - "sc_scda_fread_section_header failed"); + "sc_scda_fread_section_header for inline failed"); SC_CHECK_ABORT (section_type == 'I' && elem_count == 0 && elem_size == 0, "Identifying section type"); fc = sc_scda_fread_inline_data (fc, NULL, mpisize - 1, &errcode); SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "sc_scda_fread_inline_data skip failed"); + /* read the block section header */ + fc = sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header for block failed"); + SC_CHECK_ABORT (section_type == 'B' && elem_count == 0 && elem_size == 32, + "Identifying section type"); + + /* read the block data */ + (void) memset (read_data, '\0', SC_SCDA_INLINE_FIELD); + fc = sc_scda_fread_block_data (fc, &data, SC_SCDA_INLINE_FIELD, mpisize - 1, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_block_data failed"); + SC_CHECK_ABORT (mpirank != mpisize - 1 + || !strncmp (read_data, inline_data, + SC_SCDA_INLINE_FIELD), "block data mismatch"); + sc_scda_fclose (fc, &errcode); /* TODO: check errcode and return value */ SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), From e889d9c364c2294976bda493beff2b8b8d3e6b14 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 15:50:53 +0200 Subject: [PATCH 14/24] feature-scda: Introduce and use HEADER_ROOT macro --- src/sc_scda.c | 76 ++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index b34ceba4..bbee7f98 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -49,6 +49,7 @@ #define SC_SCDA_PADDING_MOD 32 /**< divisor for variable length padding */ #define SC_SCDA_PADDING_MOD_MAX (6 + SC_SCDA_PADDING_MOD) /**< maximal count of mod padding bytes */ +#define SC_SCDA_HEADER_ROOT 0 /**< root rank for header I/O operations */ /** get a random double in the range [A,B) */ #define SC_SCDA_RAND_RANGE(A, B, state) ((A) + sc_rand (state) * ((B) - (A))) @@ -1207,7 +1208,7 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, */ SC_SCDA_CHECK_COLL_ERR (errcode, fc, "File open write"); - if (fc->mpirank == 0) { + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fopen_write_header_internal (fc, user_string, len, &count_err, errcode); } @@ -1218,12 +1219,12 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, * error, i.e. it broadcasts the errcode, which may encode success, from * rank 0 to all other ranks and in case of an error it closes the file, * frees the file context and returns NULL. Hence, it is valid that errcode - * is only initialized on rank 0 before calling this macro. This macro is only - * valid to be called once in a function and this macro is only valid to be - * called directly after a non-collective code part that contains at least one - * call \ref SC_SCDA_CHECK_NONCOLL_ERR. + * is only initialized on rank SC_SCDA_HEADER_ROOT before calling this macro. + * This macro is only valid to be called once in a function and this macro is + * only valid to be called directly after a non-collective code part that + * contains at least one call \ref SC_SCDA_CHECK_NONCOLL_ERR. */ - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); /* The macro to check potential non-collective count errors. It is only valid * to be called directly after \ref SC_SCDA_HANDLE_NONCOLL_ERR and only * if the preceding non-collective code block contains at least one call of @@ -1232,9 +1233,11 @@ sc_scda_fopen_write (sc_MPI_Comm mpicomm, * prints an error message using \ref SC_LERRORF. This means in particular * that it is valid that errcode is only initialized on rank 0 before calling * this macro. The macro argument count_err must point to the count error - * Boolean that was set on rank 0 by \ref sc_scda_fopen_write_header_internal. + * Boolean that was set on rank SC_SCDA_HEADER_ROOT by \ref + * sc_scda_fopen_write_header_internal. */ - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); /* store number of written bytes */ fc->accessed_bytes = SC_SCDA_HEADER_BYTES; @@ -1355,12 +1358,13 @@ sc_scda_fwrite_inline (sc_scda_fcontext_t *fc, const char *user_string, SC_ASSERT (errcode != NULL); /* The file header section is always written and read on rank 0. */ - if (fc->mpirank == 0) { + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fwrite_inline_header_internal (fc, user_string, len, &count_err, errcode); } - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); /* add number of written bytes */ fc->accessed_bytes += SC_SCDA_COMMON_FIELD; @@ -1575,13 +1579,14 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, /* TODO: respect encode parameter */ - /* The file header section is always written and read on rank 0. */ - if (fc->mpirank == 0) { + /* file header section is always written and read on rank SC_SCDA_HEADER_ROOT */ + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fwrite_block_header_internal (fc, user_string, len, block_size, &count_err, errcode); } - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); /* add number of written bytes */ fc->accessed_bytes += SC_SCDA_COMMON_FIELD + SC_SCDA_COUNT_FIELD; @@ -1769,8 +1774,8 @@ sc_scda_fopen_read (sc_MPI_Comm mpicomm, */ SC_SCDA_CHECK_COLL_ERR (errcode, fc, "File open read"); - /* read file header section on rank 0 */ - if (fc->mpirank == 0) { + /* read file header section on rank SC_SCDA_HEADER_ROOT */ + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fopen_read_header_internal (fc, user_string, len, &count_err, errcode); } @@ -1779,16 +1784,17 @@ sc_scda_fopen_read (sc_MPI_Comm mpicomm, * More information can be found in the comments in \ref sc_scda_fopen_write * and in the documentation of the \ref SC_SCDA_HANDLE_NONCOLL_ERR. */ - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, 0, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); /* The macro to handle a non-collective count error that is associated to a * preceding call of \ref SC_SCDA_CHECK_NONCOLL_COUNT_ERR. * More information can be found in the comments in \ref sc_scda_fopen_write * and in the documentation of the \ref SC_SCDA_HANDLE_NONCOLL_COUNT_ERR. */ - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, 0, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); /* Bcast the user string */ mpiret = sc_MPI_Bcast (user_string, SC_SCDA_USER_STRING_BYTES + 1, - sc_MPI_BYTE, 0, mpicomm); + sc_MPI_BYTE, SC_SCDA_HEADER_ROOT, mpicomm); SC_CHECK_MPI (mpiret); /* store the number of read bytes */ @@ -2006,7 +2012,6 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, { int count_err; int mpiret; - const int header_root = 0; char var_ident; SC_ASSERT (fc != NULL); @@ -2018,16 +2023,17 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, SC_ASSERT (errcode != NULL); /* read the common section header part first */ - if (fc->mpirank == header_root) { + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fread_section_header_common_internal (fc, type, user_string, len, &count_err, errcode); } - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, header_root, fc); - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, header_root, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); fc->accessed_bytes += SC_SCDA_COMMON_FIELD; - if (fc->mpirank == header_root) { + if (fc->mpirank == SC_SCDA_HEADER_ROOT) { /* read count entries */ switch (*type) { @@ -2040,18 +2046,20 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, &count_err, errcode); break; default: - /* rank header_root already checked if type is valid/supported */ + /* rank SC_SCDA_HEADER_ROOT already checked if type is valid/supported */ SC_ABORT_NOT_REACHED (); } } - SC_SCDA_HANDLE_NONCOLL_ERR (errcode, header_root, fc); - SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, header_root, fc); + SC_SCDA_HANDLE_NONCOLL_ERR (errcode, SC_SCDA_HEADER_ROOT, fc); + SC_SCDA_HANDLE_NONCOLL_COUNT_ERR (errcode, &count_err, SC_SCDA_HEADER_ROOT, + fc); /* Bcast type and user string */ - mpiret = sc_MPI_Bcast (type, 1, sc_MPI_CHAR, header_root, fc->mpicomm); + mpiret = sc_MPI_Bcast (type, 1, sc_MPI_CHAR, SC_SCDA_HEADER_ROOT, + fc->mpicomm); SC_CHECK_MPI (mpiret); mpiret = sc_MPI_Bcast (user_string, SC_SCDA_USER_STRING_BYTES + 1, - sc_MPI_BYTE, header_root, fc->mpicomm); + sc_MPI_BYTE, SC_SCDA_HEADER_ROOT, fc->mpicomm); SC_CHECK_MPI (mpiret); /* set global outputs and Bcast the counts if it is necessary */ @@ -2066,15 +2074,15 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, case 'B': /* block */ *elem_count = 0; - /* elem_size was read on rank header_root */ - mpiret = sc_MPI_Bcast (elem_size, sizeof (size_t), sc_MPI_BYTE, header_root, - fc->mpicomm); + /* elem_size was read on rank SC_SCDA_HEADER_ROOT */ + mpiret = sc_MPI_Bcast (elem_size, sizeof (size_t), sc_MPI_BYTE, + SC_SCDA_HEADER_ROOT, fc->mpicomm); SC_CHECK_MPI (mpiret); /* one count entry was read */ fc->accessed_bytes += SC_SCDA_COUNT_FIELD; break; default: - /* rank header_root already checked if type is valid/supported */ + /* rank SC_SCDA_HEADER_ROOT already checked if type is valid/supported */ SC_ABORT_NOT_REACHED (); } From 5efe2d9e48dc70b4d6148869e10f212eb5a77fa9 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 17:30:34 +0200 Subject: [PATCH 15/24] feature-scda: Add buffer helper function --- src/sc_scda.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index bbee7f98..c2713352 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -237,6 +237,36 @@ sc_scda_copy_bytes (char *dest, const char *src, size_t n) (void) memcpy (dest, src, n); } +/** Merge up to three buffers into one contiguous buffer. + * + * \param [in] d1 The first buffer. Must be not NULL. + * \param [in] len1 The byte count of the first buffer. + * \param [in] d2 The second buffer. May be NULL. + * \param [in] len2 The byte count of the second buffer. Must be 0 if + * \b d2 is NULL. + * \param [in] d3 The third buffer. May be NULL and must be NULL if + * \b d2 is NULL. + * \param [in] len3 The byte count of the third buffer. Must be 0 if + * \b d3 is NULL. + * \param [out] out At least \b len1 + \b len2 + \b len3 bytes. + */ +static void +sc_scda_merge_data_to_buf (const char *d1, size_t len1, const char *d2, + size_t len2, const char *d3, size_t len3, + char *out) +{ + SC_ASSERT (d1 != NULL); + + sc_scda_copy_bytes (out, d1, len1); + if (d2 != NULL) { + sc_scda_copy_bytes (&out[len1], d2, len2); + } + if (d3 != NULL) { + SC_ASSERT (d2 != NULL); + sc_scda_copy_bytes (&out[len2], d3, len3); + } +} + /** Set \b n bytes in \b dest to \b c. * \b dest must have at least \b n bytes. */ From 3db39d429c5bbc63d97beac5740540e59dd8d69d Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 17:35:47 +0200 Subject: [PATCH 16/24] feature-scda: Implement checker func. for coll. param. --- src/sc_scda.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/sc_scda.c b/src/sc_scda.c index c2713352..f8ee8483 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -580,6 +580,86 @@ sc_scda_get_pad_to_mod (const char *padded_data, size_t padded_len, return 0; } +/** Check if the given parameters are collective. + * + * This function assumes that the parameters \b len1, \b len2 and \b len3 are + * collective. + * + * \param [in] fc A file context with filled MPI data. + * \param [in] param1 Pointer to the first parameter to check. + * Must be not NULL. + * \param [in] len1 The byte count of \b param1. + * \param [in] param2 Pointer to the second parameter to check. + * May be NULL. + * \param [in] len2 The byte count of \b param2. If \b param2 is NULL, + * \b len2 must be 0. + * \param [in] param3 Pointer to the third parameter to check. + * May be NULL and must be NULL if \b param2 is NULL. + * \param [in] len3 The byte count of \b param3. If \b param3 is NULL, + * \b len3 must be 0. + * \return \ref SC_SCDA_FERR_ARG if the at least one parameter + * does not match in parallel. Otherwise, \ref + * SC_SCDA_FERR_SUCCESS. + */ +static sc_scda_ret_t +sc_scda_check_coll_params (sc_scda_fcontext_t *fc, const char *param1, + size_t len1, const char *param2, size_t len2, + const char *param3, size_t len3) +{ + int mpiret; + int mismatch, collective_mismatch; + char *buffer, *recv_buf = NULL; + size_t len; + + SC_ASSERT (fc != NULL); + SC_ASSERT (param1 != NULL); + SC_ASSERT (param2 != NULL || len2 == 0); + SC_ASSERT (param3 != NULL || len3 == 0); + + len = len1 + len2 + len3; + + /* allocate contiguous buffer */ + buffer = (char *) SC_ALLOC (char, len); + + /* get buffer with parameter data */ + sc_scda_merge_data_to_buf (param1, len1, param2, len2, param3, len3, buffer); + + /* For the sake of simplicity, we use a Bcast followed by an Allreduce + * instead of one Allreduce call with a custom reduction function. + */ + /* In the future we may want to use a checksum on the buffer data if the data + * is large. + */ + if (fc->mpirank == 0) { + mpiret = sc_MPI_Bcast (buffer, (int) len, sc_MPI_BYTE, 0, fc->mpicomm); + } + else { + recv_buf = (char *) SC_ALLOC (char, len); + mpiret = sc_MPI_Bcast (recv_buf, (int) len, sc_MPI_BYTE, 0, fc->mpicomm); + } + SC_CHECK_MPI (mpiret); + + if (fc->mpirank > 0) { + /* compare data */ + mismatch = memcmp (recv_buf, buffer, len); + + /* free data buffer */ + SC_FREE (buffer); + SC_FREE (recv_buf); + } + else { + mismatch = 0; + SC_FREE (buffer); + } + + /* synchronize comparison results */ + mpiret = sc_MPI_Allreduce (&mismatch, &collective_mismatch, 1, sc_MPI_INT, + sc_MPI_LOR, fc->mpicomm); + SC_CHECK_MPI (mpiret); + + return collective_mismatch ? SC_SCDA_FERR_ARG : SC_SCDA_FERR_SUCCESS; +} + /** * This function is for creating and reading a file. * The passed \b fc must have filled MPI information (cf. \ref From 1ffa859687aa5ec59b384be52bd4d76e7d2253ac Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 17:37:51 +0200 Subject: [PATCH 17/24] feature-scda: Use general func. to check for coll. params. --- src/sc_scda.c | 49 ++++++++++++++----------------------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index f8ee8483..7f5b7232 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -674,43 +674,16 @@ sc_scda_examine_options (sc_scda_fopen_options_t * opt, sc_scda_fcontext_t *fc, SC_ASSERT (fc != NULL); if (opt != NULL) { - int mpiret; - int local_fuzzy_params_cmp, collective_fuzzy_params; - /* byte buffer since it is not clear which data type is larger */ - /* we use a char array as a byte buffer, cf. \ref sc_array_index */ - char buf[sizeof (unsigned) + sizeof (sc_rand_state_t)]; - unsigned bcast_everyn; - sc_rand_state_t bcast_seed; + sc_scda_ret_t ret; /* check if fuzzy_everyn and fuzzy_seed are collective */ + ret = sc_scda_check_coll_params (fc, (const char *) &opt->fuzzy_everyn, + sizeof (unsigned), + (const char *) &opt->fuzzy_seed, + sizeof (sc_rand_state_t), NULL, 0); + SC_ASSERT (ret == SC_SCDA_FERR_SUCCESS || ret == SC_SCDA_FERR_ARG); - /* copy fuzzy parameters to byte buffer */ - sc_scda_copy_bytes (buf, (char *) &opt->fuzzy_everyn, sizeof (unsigned)); - sc_scda_copy_bytes (&buf[sizeof (unsigned)], (char *) &opt->fuzzy_seed, - sizeof (sc_rand_state_t)); - - /* For the sake of simplicity, we use a Bcast followed by an Allreduce - * instead of one Allreduce call with a custom reduction function. - */ - mpiret = sc_MPI_Bcast (buf, sizeof (unsigned) + sizeof (sc_rand_state_t), - sc_MPI_BYTE, 0, fc->mpicomm); - SC_CHECK_MPI (mpiret); - - /* get actual data from the byte buffer */ - bcast_everyn = *((unsigned *) buf); - bcast_seed = *((sc_rand_state_t *) & buf[sizeof (unsigned)]); - - /* compare fuzzy parameters */ - local_fuzzy_params_cmp = bcast_everyn != opt->fuzzy_everyn - || bcast_seed != opt->fuzzy_seed; - - /* synchronize comparison results */ - mpiret = sc_MPI_Allreduce (&local_fuzzy_params_cmp, - &collective_fuzzy_params, 1, sc_MPI_INT, - sc_MPI_LOR, fc->mpicomm); - SC_CHECK_MPI (mpiret); - - if (collective_fuzzy_params) { + if (ret == SC_SCDA_FERR_ARG) { /* non-collective fuzzy parameters */ /* no fuzzy error testing in case of an error */ fc->fuzzy_everyn = 0; @@ -1677,6 +1650,7 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, { int count_err; size_t num_pad_bytes; + sc_scda_ret_t ret; SC_ASSERT (fc != NULL); SC_ASSERT (user_string != NULL); @@ -1685,7 +1659,12 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, SC_ASSERT (fc->mpirank != root || block_data != NULL); SC_ASSERT (errcode != NULL); - /* TODO: Check if block_size is collective. */ + /* check if block_size is collective */ + ret = sc_scda_check_coll_params (fc, (const char*) &block_size, + sizeof (size_t), NULL, 0, NULL, 0); + sc_scda_scdaret_to_errcode (ret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_block: block_size not " + "collective"); /* TODO: respect encode parameter */ From b2098bbe47c52c9797c04dc4b6734a5d50147be8 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 17:40:41 +0200 Subject: [PATCH 18/24] feature-scda: Test coll. parameter checking --- test/test_scda.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/test_scda.c b/test/test_scda.c index 3afc4f5f..91f88489 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -176,10 +176,30 @@ main (int argc, char **argv) SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fwrite_block failed"); - sc_scda_fclose (fc, &errcode); - /* TODO: check errcode and return value */ - SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), - "scda_fclose after write failed"); + /* intentionally try to write with non-collective block size */ + if (mpisize > 1) { + SC_GLOBAL_ESSENTIAL + ("We expect an invalid scda function parameter error." + " This is just for testing purposes and do not imply" + " erroneous code behavior.\n"); + } + fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, + (mpirank == 0) ? 32 : 33, mpisize - 1, 0, + &errcode); + SC_CHECK_ABORT (mpisize == 1 || (!sc_scda_ferror_is_success (errcode) && + errcode.scdaret == SC_SCDA_FERR_ARG), "scda_fwrite_block " + "check catch non-collective block size"); + + if (mpisize == 1) { + sc_scda_fclose (fc, &errcode); + /* TODO: check errcode and return value */ + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "scda_fclose after write failed"); + } + else { + /* fc was closed due to an intentionally triggered error */ + SC_CHECK_ABORT (fc == NULL, "fc closed after error in fwrite_block"); + } fc = sc_scda_fopen_read (mpicomm, filename, read_user_string, &len, &scda_opt, From ded3d9a5c418e778d57f3443017cefb3fb7db2a5 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 18:56:25 +0200 Subject: [PATCH 19/24] feature-scda: Print time-dependent seed --- test/test_scda.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_scda.c b/test/test_scda.c index 91f88489..de909e33 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -145,6 +145,8 @@ main (int argc, char **argv) mpiret = sc_MPI_Bcast (&scda_opt.fuzzy_seed, 1, sc_MPI_UNSIGNED, 0, mpicomm); SC_CHECK_MPI (mpiret); + SC_GLOBAL_INFOF ("Fuzzy error return with time-dependent seed activated. " + "The seed is %lld.\n", (long long) scda_opt.fuzzy_seed); } else { scda_opt.fuzzy_seed = (sc_rand_state_t) int_seed; From 19a0179dafb0154b11176a4cd83e9c7efcbff40a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 27 Aug 2024 18:57:16 +0200 Subject: [PATCH 20/24] feature-scda: Check block_size in fread_block_data --- src/sc_scda.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 7f5b7232..b7c71e76 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1663,7 +1663,7 @@ sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, ret = sc_scda_check_coll_params (fc, (const char*) &block_size, sizeof (size_t), NULL, 0, NULL, 0); sc_scda_scdaret_to_errcode (ret, errcode, fc); - SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_block: block_size not " + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fwrite_block: block_size is not " "collective"); /* TODO: respect encode parameter */ @@ -2018,7 +2018,7 @@ sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, int count; int wrong_format, wrong_ident; long long unsigned read_count; - size_t len; + size_t len = 0; SC_ASSERT (fc != NULL); SC_ASSERT (ident != NULL); @@ -2323,11 +2323,19 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, { int count_err; int wrong_usage; + sc_scda_ret_t ret; SC_ASSERT (fc != NULL); SC_ASSERT (root >= 0); SC_ASSERT (errcode != NULL); + /* check if block_size is collective */ + ret = sc_scda_check_coll_params (fc, (const char*) &block_size, + sizeof (size_t), NULL, 0, NULL, 0); + sc_scda_scdaret_to_errcode (ret, errcode, fc); + SC_SCDA_CHECK_COLL_ERR (errcode, fc, "fread_block_data: block_size is not " + "collective"); + /* It is necessary that sc_scda_fread_section_header was called as last * function call on fc and that it returned the block section type. */ From 245dba7b9ca189fd26a1d2e500d3b44018c62962 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 28 Aug 2024 10:55:57 +0200 Subject: [PATCH 21/24] feature-scda: Print seed only for activated fuzzy testing --- test/test_scda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_scda.c b/test/test_scda.c index de909e33..952723f2 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -140,7 +140,7 @@ main (int argc, char **argv) * able to handle this situations properly. */ scda_opt.fuzzy_everyn = (unsigned) int_everyn; - if (int_seed < 0) { + if (scda_opt.fuzzy_everyn > 0 && int_seed < 0) { scda_opt.fuzzy_seed = (sc_rand_state_t) sc_MPI_Wtime (); mpiret = sc_MPI_Bcast (&scda_opt.fuzzy_seed, 1, sc_MPI_UNSIGNED, 0, mpicomm); From ea25a0db92c01626ab38389dff979fcdbf232d96 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 28 Aug 2024 11:04:51 +0200 Subject: [PATCH 22/24] feature-scda: Generalize pad_to_mod_len --- src/sc_scda.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index b7c71e76..44c5589e 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -410,8 +410,11 @@ sc_scda_pad_to_mod_len (size_t input_len) if (num_pad_bytes < 7) { /* not sufficient number of padding bytes for the padding format */ - num_pad_bytes += SC_SCDA_PADDING_MOD; + num_pad_bytes += SC_SCDA_PADDING_MOD * + (size_t) ceil (((7. - num_pad_bytes) / ((double) SC_SCDA_PADDING_MOD))); + /* The factor is necessary for the case that SC_SCDA_PADDING_MOD < 7. */ } + SC_ASSERT (num_pad_bytes >= 7); return num_pad_bytes; } From cdf43fe03053107b39f90e88e53cf5ba0fdb7615 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 28 Aug 2024 12:12:52 +0200 Subject: [PATCH 23/24] feature-scda: Extend tests --- src/sc_scda.c | 9 +++++---- test/test_scda.c | 47 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 44c5589e..34ff311d 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -1643,13 +1643,13 @@ sc_scda_fwrite_block_data_internal (sc_scda_fcontext_t *fc, padding, (int) num_pad_bytes, sc_MPI_BYTE, &count); sc_scda_mpiret_to_errcode (mpiret, errcode, fc); SC_SCDA_CHECK_NONCOLL_ERR (errcode, "Writing block data padding"); - SC_SCDA_CHECK_NONCOLL_COUNT_ERR (block_size, count, count_err); + SC_SCDA_CHECK_NONCOLL_COUNT_ERR (num_pad_bytes, count, count_err); } sc_scda_fcontext_t * sc_scda_fwrite_block (sc_scda_fcontext_t *fc, const char *user_string, - size_t *len, sc_array_t * block_data, size_t block_size, - int root, int encode, sc_scda_ferror_t * errcode) + size_t *len, sc_array_t * block_data, size_t block_size, + int root, int encode, sc_scda_ferror_t * errcode) { int count_err; size_t num_pad_bytes; @@ -2358,7 +2358,8 @@ sc_scda_fread_block_data (sc_scda_fcontext_t *fc, sc_array_t *block_data, } /* if no error occurred, we move the internal file pointer */ - fc->accessed_bytes += (sc_MPI_Offset) block_size; + fc->accessed_bytes += (sc_MPI_Offset) (block_size + + sc_scda_pad_to_mod_len (block_size)); /* last function call can not be \ref sc_scda_fread_section_header anymore */ fc->header_before = 0; diff --git a/test/test_scda.c b/test/test_scda.c index 952723f2..0811b4fc 100644 --- a/test/test_scda.c +++ b/test/test_scda.c @@ -44,9 +44,11 @@ main (int argc, char **argv) sc_scda_ferror_t errcode; size_t len; size_t elem_count, elem_size; + size_t block_size; sc_options_t *opt; sc_array_t data; const char *inline_data = "Test inline data \n"; + const char *block_data = "Test block data"; char read_data[SC_SCDA_INLINE_FIELD]; mpiret = sc_MPI_Init (&argc, &argv); @@ -159,6 +161,14 @@ main (int argc, char **argv) SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fopen_write failed"); + /* write a block section to the file */ + block_size = strlen (block_data); + sc_array_init_data (&data, (void *) block_data, block_size, 1); + fc = sc_scda_fwrite_block (fc, "Block section test", NULL, &data, block_size, + mpisize - 1, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "scda_fwrite_block failed"); + /* write an inline section to the file */ sc_array_init_data (&data, (void *) inline_data, SC_SCDA_INLINE_FIELD, 1); fc = sc_scda_fwrite_inline (fc, "Inline section test without user-defined " @@ -172,8 +182,8 @@ main (int argc, char **argv) "scda_fwrite_inline with empty user string failed"); /* write a block section */ - fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, 32, - mpisize - 1, 0, &errcode); + fc = sc_scda_fwrite_block (fc, "A block section with the inline data", NULL, + &data, 32, mpisize - 1, 0, &errcode); /* TODO: check errcode */ SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), "scda_fwrite_block failed"); @@ -184,13 +194,13 @@ main (int argc, char **argv) ("We expect an invalid scda function parameter error." " This is just for testing purposes and do not imply" " erroneous code behavior.\n"); + fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, + (mpirank == 0) ? 32 : 33, mpisize - 1, 0, + &errcode); + SC_CHECK_ABORT (!sc_scda_ferror_is_success (errcode) && + errcode.scdaret == SC_SCDA_FERR_ARG, "scda_fwrite_block " + "check catch non-collective block size"); } - fc = sc_scda_fwrite_block (fc, "A block section", NULL, &data, - (mpirank == 0) ? 32 : 33, mpisize - 1, 0, - &errcode); - SC_CHECK_ABORT (mpisize == 1 || (!sc_scda_ferror_is_success (errcode) && - errcode.scdaret == SC_SCDA_FERR_ARG), "scda_fwrite_block " - "check catch non-collective block size"); if (mpisize == 1) { sc_scda_fclose (fc, &errcode); @@ -213,6 +223,27 @@ main (int argc, char **argv) SC_INFOF ("File header user string: %s\n", read_user_string); /* read first section header */ + fc = sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, + &elem_count, &elem_size, &decode, + &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_section_header failed"); + SC_CHECK_ABORT (section_type == 'B' && elem_count == 0 && + elem_size == block_size, "Identifying section type"); + + /* read block data */ + sc_array_init_data (&data, read_data, block_size, 1); + fc = sc_scda_fread_block_data (fc, &data, block_size, 0, &errcode); + SC_CHECK_ABORT (sc_scda_ferror_is_success (errcode), + "sc_scda_fread_block_data failed"); + SC_CHECK_ABORT (mpirank != 0 + || !strncmp (read_data, block_data, block_size), + "inline data mismatch"); + + SC_INFOF ("Read file section header of type %c with user string: %s\n", + section_type, read_user_string); + + /* read second section header */ fc = sc_scda_fread_section_header (fc, read_user_string, &len, §ion_type, &elem_count, &elem_size, &decode, &errcode); From 662314e0288d3a65e6337a2c3452480ded524ebd Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Wed, 28 Aug 2024 13:45:35 +0200 Subject: [PATCH 24/24] feature-scda: Init output variables --- src/sc_scda.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sc_scda.c b/src/sc_scda.c index 34ff311d..6cf833f3 100644 --- a/src/sc_scda.c +++ b/src/sc_scda.c @@ -2027,8 +2027,9 @@ sc_scda_fread_count_entry_internal (sc_scda_fcontext_t *fc, char *ident, SC_ASSERT (ident != NULL); SC_ASSERT (count_var != NULL); - *count_err = 0; + *ident = ' '; *count_var = 0; + *count_err = 0; /* read a count entry */ mpiret = sc_io_read_at (fc->file, fc->accessed_bytes, count_entry, @@ -2114,6 +2115,9 @@ sc_scda_fread_section_header (sc_scda_fcontext_t *fc, char *user_string, SC_ASSERT (decode != NULL); SC_ASSERT (errcode != NULL); + *elem_count = 0; + *elem_size = 0; + /* read the common section header part first */ if (fc->mpirank == SC_SCDA_HEADER_ROOT) { sc_scda_fread_section_header_common_internal (fc, type, user_string, len,