From c0e7e466db0cb1b89a5cbe33be577b4c3888e705 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 18 May 2022 17:00:46 +0100 Subject: [PATCH 01/26] Bump up version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1f28ff84..6a8a8619c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) # Initialise project -project( eccodes VERSION 2.26.0 LANGUAGES C ) +project( eccodes VERSION 2.26.1 LANGUAGES C ) ############################################################################### # system checks needed for eccodes_config.h and some options like MEMFS From d94f3e4e7c7c45d27fa2b593efd41097573af39a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 18 May 2022 17:00:55 +0100 Subject: [PATCH 02/26] ECC-1388: GRIB2 CCSDS: Unpacked values different from simple packing --- src/grib_accessor_class_data_ccsds_packing.c | 77 +++++++++++++++----- tests/grib_ccsds.sh | 24 +++++- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/grib_accessor_class_data_ccsds_packing.c b/src/grib_accessor_class_data_ccsds_packing.c index 42ba99408..7cd984d51 100644 --- a/src/grib_accessor_class_data_ccsds_packing.c +++ b/src/grib_accessor_class_data_ccsds_packing.c @@ -396,7 +396,12 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) Assert(val[i] == val[0]); } #endif - if ((err = grib_set_double_internal(hand, self->reference_value, val[0])) != GRIB_SUCCESS) + if (grib_get_nearest_smaller_value(hand, self->reference_value, val[0], &reference_value) != GRIB_SUCCESS) { + grib_context_log(a->context, GRIB_LOG_ERROR, + "CCSDS pack_double: unable to find nearest_smaller_value of %g for %s", min, self->reference_value); + return GRIB_INTERNAL_ERROR; + } + if ((err = grib_set_double_internal(hand, self->reference_value, reference_value)) != GRIB_SUCCESS) return err; if ((err = grib_set_long_internal(hand, self->number_of_values, n_vals)) != GRIB_SUCCESS) @@ -413,21 +418,60 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) if ((err = grib_get_long_internal(hand, self->number_of_data_points, &number_of_data_points)) != GRIB_SUCCESS) return err; - d = grib_power(decimal_scale_factor, 10); - min *= d; - max *= d; + if (bits_per_value == 0 || (binary_scale_factor == 0 && decimal_scale_factor != 0)) { + d = grib_power(decimal_scale_factor, 10); + min *= d; + max *= d; - if (grib_get_nearest_smaller_value(hand, self->reference_value, min, &reference_value) != GRIB_SUCCESS) { - grib_context_log(a->context, GRIB_LOG_ERROR, - "CCSDS pack_double: unable to find nearest_smaller_value of %g for %s", min, self->reference_value); - return GRIB_INTERNAL_ERROR; - } + if (grib_get_nearest_smaller_value(hand, self->reference_value, min, &reference_value) != GRIB_SUCCESS) { + grib_context_log(a->context, GRIB_LOG_ERROR, + "CCSDS pack_double: unable to find nearest_smaller_value of %g for %s", min, self->reference_value); + return GRIB_INTERNAL_ERROR; + } - if (reference_value > min) { - grib_context_log(a->context, GRIB_LOG_ERROR, - "CCSDS pack_double: reference_value=%g min_value=%g diff=%g", reference_value, min, reference_value - min); - DebugAssert(reference_value <= min); - return GRIB_INTERNAL_ERROR; + if (reference_value > min) { + grib_context_log(a->context, GRIB_LOG_ERROR, + "CCSDS pack_double: reference_value=%g min_value=%g diff=%g", reference_value, min, reference_value - min); + DebugAssert(reference_value <= min); + return GRIB_INTERNAL_ERROR; + } + } + else { + int last = 127; /* last must be a parameter coming from the def file*/ + double range = 0; + double minrange = 0, maxrange = 0; + double unscaled_max = 0; + double unscaled_min = 0; + double f = 0; + double decimal = 1; + + range = (max - min); + unscaled_min = min; + unscaled_max = max; + f = (grib_power(bits_per_value, 2) - 1); + minrange = grib_power(-last, 2) * f; + maxrange = grib_power(last, 2) * f; + + while (range < minrange) { + decimal_scale_factor += 1; + decimal *= 10; + min = unscaled_min * decimal; + max = unscaled_max * decimal; + range = (max - min); + } + while (range > maxrange) { + decimal_scale_factor -= 1; + decimal /= 10; + min = unscaled_min * decimal; + max = unscaled_max * decimal; + range = (max - min); + } + if (grib_get_nearest_smaller_value(hand, self->reference_value, min, &reference_value) != GRIB_SUCCESS) { + grib_context_log(a->context, GRIB_LOG_ERROR, + "CCSDS pack_double: unable to find nearest_smaller_value of %g for %s", min, self->reference_value); + return GRIB_INTERNAL_ERROR; + } + d = grib_power(decimal_scale_factor, 10); } binary_scale_factor = grib_get_binary_scale_fact(max, reference_value, bits_per_value, &err); @@ -445,7 +489,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) p = encoded; for (i = 0; i < n_vals; i++) { long blen = bits8; - unsigned long unsigned_val = (unsigned long)((((val[i] * d) - (reference_value)) * divisor) + 0.5); + unsigned long unsigned_val = (unsigned long)((((val[i] * d) - reference_value) * divisor) + 0.5); while (blen >= 8) { blen -= 8; *p = (unsigned_val >> blen); @@ -455,8 +499,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) } /* buflen = n_vals*(bits_per_value/8);*/ - grib_context_log(a->context, GRIB_LOG_DEBUG, - "CCSDS pack_double: packing %s, %d values", a->name, n_vals); + grib_context_log(a->context, GRIB_LOG_DEBUG,"CCSDS pack_double: packing %s, %d values", a->name, n_vals); buflen += 10240; buf = (unsigned char*)grib_context_buffer_malloc_clear(a->context, buflen); diff --git a/tests/grib_ccsds.sh b/tests/grib_ccsds.sh index e235555b3..f9a0f2239 100755 --- a/tests/grib_ccsds.sh +++ b/tests/grib_ccsds.sh @@ -33,8 +33,30 @@ grib_check_key_equals $outfile2 packingType,const "grid_ccsds 1" grib_check_key_equals $outfile2 accuracy 0 rm -f $outfile1 $outfile2 +# ECC-1388: Tiny values +# --------------------- +tempFilt=temp.$label.filter +IFS_SAMPLES_ROOT=${proj_dir}/ifs_samples +IFS_SAMPLE_SIMPLE=$IFS_SAMPLES_ROOT/grib1_mlgrib2/gg_ml.tmpl +IFS_SAMPLE_CCSDS=$IFS_SAMPLES_ROOT/grib1_mlgrib2_ccsds/gg_ml.tmpl +echo 'set bitsPerValue=24; set values={ 1.00000007851413483E-25, 1.00000007851413529E-25 }; write;' > $tempFilt +${tools_dir}/grib_filter -o $outfile1 $tempFilt $IFS_SAMPLE_SIMPLE +${tools_dir}/grib_filter -o $outfile2 $tempFilt $IFS_SAMPLE_CCSDS +${tools_dir}/grib_compare -c data:n $outfile1 $outfile2 +rm -f $tempFilt + +# ECC-1388: Constant field variant +# -------------------------------- +echo 'set bitsPerValue=24; set values={ 1.00000007851413483E-25, 1.00000007851413483E-25 }; write;' > $tempFilt +${tools_dir}/grib_filter -o $outfile1 $tempFilt $IFS_SAMPLE_SIMPLE +${tools_dir}/grib_filter -o $outfile2 $tempFilt $IFS_SAMPLE_CCSDS +# Must compare the referenceValue with 0 tolerance (override the referenceValueError key) +${tools_dir}/grib_compare -R referenceValue=0 -c referenceValue $outfile1 $outfile2 +rm -f $tempFilt + + # ECC-1387 -# -------- +# --------- echo 'set values={6, 6, 6};write;' | ${tools_dir}/grib_filter -o $outfile2 - ${data_dir}/ccsds.grib2 grib_check_key_equals $outfile2 referenceValue,bitsPerValue '6 0' From d5b8c1962853aa0787c084132bed3f2b856537e0 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 30 Jun 2022 12:49:13 +0100 Subject: [PATCH 03/26] Bump up version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a8a8619c..2ff926b6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) # Initialise project -project( eccodes VERSION 2.26.1 LANGUAGES C ) +project( eccodes VERSION 2.26.2 LANGUAGES C ) ############################################################################### # system checks needed for eccodes_config.h and some options like MEMFS From fbb940799b5e39d73201fec4350260da26d6ac66 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 30 Jun 2022 12:55:48 +0100 Subject: [PATCH 04/26] ECC-1413: GRIB: Add iteration key in the MARS namespace for stream=elda, type=4i --- definitions/mars/grib.elda.4i.def | 1 + tests/grib_local.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/definitions/mars/grib.elda.4i.def b/definitions/mars/grib.elda.4i.def index ce5e03eaf..852d42dcb 100644 --- a/definitions/mars/grib.elda.4i.def +++ b/definitions/mars/grib.elda.4i.def @@ -1,2 +1,3 @@ alias mars.anoffset=offsetToEndOf4DvarWindow; alias mars.number=perturbationNumber; +alias mars.iteration = iterationNumber; diff --git a/tests/grib_local.sh b/tests/grib_local.sh index 9bc9bf8bb..e513a9fd0 100755 --- a/tests/grib_local.sh +++ b/tests/grib_local.sh @@ -189,5 +189,15 @@ result=`echo 'print "[ccccIdentifiers]";' | ${tools_dir}/grib_filter - $temp` [ "$result" = "kwbc ecmf sabm" ] +# ECC-1413: Local Definition 38 +# -------------------------------- +${tools_dir}/grib_set -s \ + setLocalDefinition=1,localDefinitionNumber=38,type=4i,stream=elda,iterationNumber=23 \ + $sample_g2 $temp +grib_check_key_equals $temp 'mars.iteration' '23' +${tools_dir}/grib_ls -jm $temp > $temp.1 +grep -q "iteration.* 23" $temp.1 + +# Clean up rm -f $temp $temp.1 $temp.2 $temp.3 From a2d2784b5b3cbc77154567d7eae3ff2154604945 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 9 Jun 2022 12:17:43 +0100 Subject: [PATCH 05/26] ECC-1400: GRIB2: Update shortname of 260318 - 260321 --- definitions/grib2/cfVarName.def | 8 ++++---- definitions/grib2/shortName.def | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/definitions/grib2/cfVarName.def b/definitions/grib2/cfVarName.def index d7537b7b5..fd2609df6 100644 --- a/definitions/grib2/cfVarName.def +++ b/definitions/grib2/cfVarName.def @@ -2682,7 +2682,7 @@ parameterNumber = 0 ; } #Precipitation type (most severe) in the last 1 hour -'sev_ptype1h' = { +'ptype_sev1h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2691,7 +2691,7 @@ lengthOfTimeRange = 1 ; } #Precipitation type (most severe) in the last 3 hours -'sev_ptype3h' = { +'ptype_sev3h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2700,7 +2700,7 @@ lengthOfTimeRange = 3 ; } #Precipitation type (most frequent) in the last 1 hour -'freq_ptype1h' = { +'ptype_freq1h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2709,7 +2709,7 @@ lengthOfTimeRange = 1 ; } #Precipitation type (most frequent) in the last 3 hours -'freq_ptype3h' = { +'ptype_freq3h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; diff --git a/definitions/grib2/shortName.def b/definitions/grib2/shortName.def index 8e434fdc8..e92d3563e 100644 --- a/definitions/grib2/shortName.def +++ b/definitions/grib2/shortName.def @@ -2682,7 +2682,7 @@ parameterNumber = 0 ; } #Precipitation type (most severe) in the last 1 hour -'sev_ptype1h' = { +'ptype_sev1h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2691,7 +2691,7 @@ lengthOfTimeRange = 1 ; } #Precipitation type (most severe) in the last 3 hours -'sev_ptype3h' = { +'ptype_sev3h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2700,7 +2700,7 @@ lengthOfTimeRange = 3 ; } #Precipitation type (most frequent) in the last 1 hour -'freq_ptype1h' = { +'ptype_freq1h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; @@ -2709,7 +2709,7 @@ lengthOfTimeRange = 1 ; } #Precipitation type (most frequent) in the last 3 hours -'freq_ptype3h' = { +'ptype_freq3h' = { discipline = 0 ; parameterCategory = 1 ; parameterNumber = 19 ; From e9a15ef4ea9fee9ac56211f4d548ea4f71363251 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 8 Jun 2022 18:16:56 +0100 Subject: [PATCH 06/26] ECC-1399: GRIB2: Add parameters for additional precipitation type severity and frequency --- definitions/grib2/cfVarName.def | 18 ++++++++++++++++++ definitions/grib2/name.def | 18 ++++++++++++++++++ definitions/grib2/paramId.def | 18 ++++++++++++++++++ definitions/grib2/shortName.def | 18 ++++++++++++++++++ definitions/grib2/units.def | 18 ++++++++++++++++++ 5 files changed, 90 insertions(+) diff --git a/definitions/grib2/cfVarName.def b/definitions/grib2/cfVarName.def index fd2609df6..2ca1e5028 100644 --- a/definitions/grib2/cfVarName.def +++ b/definitions/grib2/cfVarName.def @@ -2717,6 +2717,24 @@ typeOfStatisticalProcessing = 101 ; lengthOfTimeRange = 3 ; } +#Precipitation type (most severe) in the last 6 hours +'ptype_sev6h' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 100 ; + lengthOfTimeRange = 6 ; + } +#Precipitation type (most frequent) in the last 6 hours +'ptype_freq6h' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 101 ; + lengthOfTimeRange = 6 ; + } #Soil temperature 'sot' = { discipline = 2 ; diff --git a/definitions/grib2/name.def b/definitions/grib2/name.def index b20646216..3d68aac9c 100644 --- a/definitions/grib2/name.def +++ b/definitions/grib2/name.def @@ -2717,6 +2717,24 @@ typeOfStatisticalProcessing = 101 ; lengthOfTimeRange = 3 ; } +#Precipitation type (most severe) in the last 6 hours +'Precipitation type (most severe) in the last 6 hours' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 100 ; + lengthOfTimeRange = 6 ; + } +#Precipitation type (most frequent) in the last 6 hours +'Precipitation type (most frequent) in the last 6 hours' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 101 ; + lengthOfTimeRange = 6 ; + } #Soil temperature 'Soil temperature' = { discipline = 2 ; diff --git a/definitions/grib2/paramId.def b/definitions/grib2/paramId.def index 488ccd426..24641d428 100644 --- a/definitions/grib2/paramId.def +++ b/definitions/grib2/paramId.def @@ -2717,6 +2717,24 @@ typeOfStatisticalProcessing = 101 ; lengthOfTimeRange = 3 ; } +#Precipitation type (most severe) in the last 6 hours +'260338' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 100 ; + lengthOfTimeRange = 6 ; + } +#Precipitation type (most frequent) in the last 6 hours +'260339' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 101 ; + lengthOfTimeRange = 6 ; + } #Soil temperature '260360' = { discipline = 2 ; diff --git a/definitions/grib2/shortName.def b/definitions/grib2/shortName.def index e92d3563e..0b1cf3e0c 100644 --- a/definitions/grib2/shortName.def +++ b/definitions/grib2/shortName.def @@ -2717,6 +2717,24 @@ typeOfStatisticalProcessing = 101 ; lengthOfTimeRange = 3 ; } +#Precipitation type (most severe) in the last 6 hours +'ptype_sev6h' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 100 ; + lengthOfTimeRange = 6 ; + } +#Precipitation type (most frequent) in the last 6 hours +'ptype_freq6h' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 101 ; + lengthOfTimeRange = 6 ; + } #Soil temperature 'sot' = { discipline = 2 ; diff --git a/definitions/grib2/units.def b/definitions/grib2/units.def index 56c4c1c0d..bb6545433 100644 --- a/definitions/grib2/units.def +++ b/definitions/grib2/units.def @@ -2717,6 +2717,24 @@ typeOfStatisticalProcessing = 101 ; lengthOfTimeRange = 3 ; } +#Precipitation type (most severe) in the last 6 hours +'code table (4.201)' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 100 ; + lengthOfTimeRange = 6 ; + } +#Precipitation type (most frequent) in the last 6 hours +'code table (4.201)' = { + discipline = 0 ; + parameterCategory = 1 ; + parameterNumber = 19 ; + indicatorOfUnitForTimeRange = 1 ; + typeOfStatisticalProcessing = 101 ; + lengthOfTimeRange = 6 ; + } #Soil temperature 'K' = { discipline = 2 ; From 0b24e78d10e3e2e704b53aef9ddb82e0e25dedd1 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Tue, 30 Aug 2022 14:37:57 +0000 Subject: [PATCH 07/26] Test permissions to repo --- experimental/show_compile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/experimental/show_compile.sh b/experimental/show_compile.sh index ad54fc20a..8ded1f34e 100755 --- a/experimental/show_compile.sh +++ b/experimental/show_compile.sh @@ -47,3 +47,4 @@ echo echo "Compilation for C ..." echo "$CCMP myprog.c $LIBS $INCL" echo +echo From ca5718f38b193ec0a677a874507790c1d7bbb115 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 25 Oct 2022 11:25:02 +0100 Subject: [PATCH 08/26] ECC-1451: GRIB: key 'level' not equal to 'mars.levelist' for soil levels --- definitions/grib2/template.4.horizontal.def | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/definitions/grib2/template.4.horizontal.def b/definitions/grib2/template.4.horizontal.def index d6aa9fbee..5f1c61f85 100755 --- a/definitions/grib2/template.4.horizontal.def +++ b/definitions/grib2/template.4.horizontal.def @@ -82,11 +82,13 @@ if (extraDim) { } } -# See ECC-854, ECC-1435 +# See ECC-854, ECC-1435, ECC-1451 if( (typeOfFirstFixedSurface == 151 && typeOfSecondFixedSurface == 151) || (typeOfFirstFixedSurface == 152 && typeOfSecondFixedSurface == 152) || (typeOfFirstFixedSurface == 114 && typeOfSecondFixedSurface == 114) ) { alias mars.levelist = bottomLevel; + alias ls.level = bottomLevel; + alias vertical.level = bottomLevel; } alias ls.typeOfLevel=typeOfLevel; From fbec511dcb9d473b7dd352ffefb347f1d80ebce6 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 25 Oct 2022 11:26:29 +0100 Subject: [PATCH 09/26] Bump up version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cd226c30..78382b33c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) # Initialise project -project( eccodes VERSION 2.27.0 LANGUAGES C ) +project( eccodes VERSION 2.27.1 LANGUAGES C ) ############################################################################### # system checks needed for eccodes_config.h and some options like MEMFS From 0ed9b503cbfc0311cd53d4399fb33a038eb2c4c9 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 22 Nov 2022 15:12:02 +0000 Subject: [PATCH 10/26] ECC-1462, ECC-1449, ECC-1448 --- definitions/grib2/cfVarName.def | 66 +++++++++++++++++++++ definitions/grib2/name.def | 66 +++++++++++++++++++++ definitions/grib2/paramId.def | 66 +++++++++++++++++++++ definitions/grib2/shortName.def | 66 +++++++++++++++++++++ definitions/grib2/template.4.horizontal.def | 3 +- definitions/grib2/units.def | 66 +++++++++++++++++++++ definitions/mars/grib.mmsf.fc.def | 5 +- 7 files changed, 336 insertions(+), 2 deletions(-) diff --git a/definitions/grib2/cfVarName.def b/definitions/grib2/cfVarName.def index ab93cb803..ddf1413f0 100644 --- a/definitions/grib2/cfVarName.def +++ b/definitions/grib2/cfVarName.def @@ -2199,6 +2199,72 @@ typeOfFirstFixedSurface = 103 ; scaleFactorOfFirstFixedSurface = 0 ; } +#Urban cover +'cur' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 0 ; + } +#Road Cover +'cro' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 1 ; + } +#Building cover +'cbu' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 2 ; + } +#Building height +'bldh' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 3 ; + } +#Vertical-to-horizontal area ratio +'hwr' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 4 ; + } +#Standard deviation of building height +'bhstd' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 5 ; + } +#Wetland cover +'cwe' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 45 ; + } +#Wetland type +'twe' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 46 ; + } +#Irrigation cover +'cirr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 47 ; + } +#C4 crop cover +'c4cr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 48 ; + } +#C4 grass cover +'c4gr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 49 ; + } #Mean temperature tendency due to short-wave radiation 'mttswr' = { discipline = 0 ; diff --git a/definitions/grib2/name.def b/definitions/grib2/name.def index a98012218..726fbba69 100644 --- a/definitions/grib2/name.def +++ b/definitions/grib2/name.def @@ -2199,6 +2199,72 @@ scaledValueOfFirstFixedSurface = 100 ; scaleFactorOfFirstFixedSurface = 0 ; } +#Urban cover +'Urban cover' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 0 ; + } +#Road Cover +'Road Cover' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 1 ; + } +#Building cover +'Building cover' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 2 ; + } +#Building height +'Building height' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 3 ; + } +#Vertical-to-horizontal area ratio +'Vertical-to-horizontal area ratio' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 4 ; + } +#Standard deviation of building height +'Standard deviation of building height' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 5 ; + } +#Wetland cover +'Wetland cover' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 45 ; + } +#Wetland type +'Wetland type' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 46 ; + } +#Irrigation cover +'Irrigation cover' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 47 ; + } +#C4 crop cover +'C4 crop cover' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 48 ; + } +#C4 grass cover +'C4 grass cover' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 49 ; + } #Mean temperature tendency due to short-wave radiation 'Mean temperature tendency due to short-wave radiation' = { discipline = 0 ; diff --git a/definitions/grib2/paramId.def b/definitions/grib2/paramId.def index ddffa4448..8b5a0ed40 100644 --- a/definitions/grib2/paramId.def +++ b/definitions/grib2/paramId.def @@ -2199,6 +2199,72 @@ scaledValueOfFirstFixedSurface = 100 ; scaleFactorOfFirstFixedSurface = 0 ; } +#Urban cover +'229001' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 0 ; + } +#Road Cover +'229002' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 1 ; + } +#Building cover +'229003' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 2 ; + } +#Building height +'229004' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 3 ; + } +#Vertical-to-horizontal area ratio +'229005' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 4 ; + } +#Standard deviation of building height +'229006' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 5 ; + } +#Wetland cover +'229007' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 45 ; + } +#Wetland type +'229008' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 46 ; + } +#Irrigation cover +'229009' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 47 ; + } +#C4 crop cover +'229010' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 48 ; + } +#C4 grass cover +'229011' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 49 ; + } #Mean temperature tendency due to short-wave radiation '235001' = { discipline = 0 ; diff --git a/definitions/grib2/shortName.def b/definitions/grib2/shortName.def index 4c7702fff..1982a740f 100644 --- a/definitions/grib2/shortName.def +++ b/definitions/grib2/shortName.def @@ -2199,6 +2199,72 @@ scaledValueOfFirstFixedSurface = 100 ; scaleFactorOfFirstFixedSurface = 0 ; } +#Urban cover +'cur' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 0 ; + } +#Road Cover +'cro' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 1 ; + } +#Building cover +'cbu' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 2 ; + } +#Building height +'bldh' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 3 ; + } +#Vertical-to-horizontal area ratio +'hwr' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 4 ; + } +#Standard deviation of building height +'bhstd' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 5 ; + } +#Wetland cover +'cwe' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 45 ; + } +#Wetland type +'twe' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 46 ; + } +#Irrigation cover +'cirr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 47 ; + } +#C4 crop cover +'c4cr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 48 ; + } +#C4 grass cover +'c4gr' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 49 ; + } #Mean temperature tendency due to short-wave radiation 'mttswr' = { discipline = 0 ; diff --git a/definitions/grib2/template.4.horizontal.def b/definitions/grib2/template.4.horizontal.def index 5f1c61f85..3749b84b4 100755 --- a/definitions/grib2/template.4.horizontal.def +++ b/definitions/grib2/template.4.horizontal.def @@ -82,9 +82,10 @@ if (extraDim) { } } -# See ECC-854, ECC-1435, ECC-1451 +# See ECC-854, ECC-1435, ECC-1451, ECC-1462 if( (typeOfFirstFixedSurface == 151 && typeOfSecondFixedSurface == 151) || (typeOfFirstFixedSurface == 152 && typeOfSecondFixedSurface == 152) || + (typeOfFirstFixedSurface == 168 && typeOfSecondFixedSurface == 168) || (typeOfFirstFixedSurface == 114 && typeOfSecondFixedSurface == 114) ) { alias mars.levelist = bottomLevel; alias ls.level = bottomLevel; diff --git a/definitions/grib2/units.def b/definitions/grib2/units.def index 987dcf86d..04b39e072 100644 --- a/definitions/grib2/units.def +++ b/definitions/grib2/units.def @@ -2199,6 +2199,72 @@ scaledValueOfFirstFixedSurface = 100 ; scaleFactorOfFirstFixedSurface = 0 ; } +#Urban cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 0 ; + } +#Road Cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 1 ; + } +#Building cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 2 ; + } +#Building height +'m' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 3 ; + } +#Vertical-to-horizontal area ratio +'m**2 m**-2' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 4 ; + } +#Standard deviation of building height +'m' = { + discipline = 2 ; + parameterCategory = 6 ; + parameterNumber = 5 ; + } +#Wetland cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 45 ; + } +#Wetland type +'Code table 4.239' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 46 ; + } +#Irrigation cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 47 ; + } +#C4 crop cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 48 ; + } +#C4 grass cover +'(0 - 1)' = { + discipline = 2 ; + parameterCategory = 0 ; + parameterNumber = 49 ; + } #Mean temperature tendency due to short-wave radiation 'K s**-1' = { discipline = 0 ; diff --git a/definitions/mars/grib.mmsf.fc.def b/definitions/mars/grib.mmsf.fc.def index 3dc2f9a36..9d47177d1 100644 --- a/definitions/mars/grib.mmsf.fc.def +++ b/definitions/mars/grib.mmsf.fc.def @@ -5,7 +5,10 @@ if (class is "en") { alias mars.system = systemNumber; } if (class is "c3") { alias mars.system = systemNumber; } alias mars.number = perturbationNumber; alias mars.method = methodNumber; -alias mars.origin = centre; + +if (!(class is "gw")) { # ECC-1448 + alias mars.origin = centre; +} # See ECC-624 if (centre == 80 && subCentre == 98 && class is "c3") { From 5e34bfbc0466bc5babc682af981ccb80049ce050 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 22 Nov 2022 15:43:15 +0000 Subject: [PATCH 11/26] ECC-1456: MARS: Create 2 new classes EF (for EFAS) and GF (for GLOFAS) --- definitions/mars/class.table | 2 ++ 1 file changed, 2 insertions(+) diff --git a/definitions/mars/class.table b/definitions/mars/class.table index 3dcfe05d2..446f13833 100644 --- a/definitions/mars/class.table +++ b/definitions/mars/class.table @@ -40,6 +40,8 @@ 39 gw Global Wildfire Information System 40 e6 ERA6 41 l6 ERA6/LAND +42 ef EFAS (European flood awareness system) +43 gf GLOFAS (Global flood awareness system) 99 te Test 100 at Austria 101 be Belgium From 8eaa1631fc3872674977263d5cc2d4bfdd5eae15 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 25 Nov 2022 14:48:24 +0000 Subject: [PATCH 12/26] ECC-1475: Check return size of md5 keys --- src/grib_accessor_class_md5.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grib_accessor_class_md5.c b/src/grib_accessor_class_md5.c index ed5474be5..14f2f9bc5 100644 --- a/src/grib_accessor_class_md5.c +++ b/src/grib_accessor_class_md5.c @@ -271,6 +271,6 @@ static void destroy(grib_context* c, grib_accessor* a) static int value_count(grib_accessor* a, long* count) { - *count = 16; + *count = 1; /* ECC-1475 */ return 0; } From 3fb11b3348b236ff7ee7dad43f98df304e9eb842 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 28 Nov 2022 18:17:03 +0000 Subject: [PATCH 13/26] ECC-1477: Compilation fails with libjasper v4.x.x --- src/grib_jasper_encoding.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/grib_jasper_encoding.c b/src/grib_jasper_encoding.c index fe8facf92..d51c57147 100644 --- a/src/grib_jasper_encoding.c +++ b/src/grib_jasper_encoding.c @@ -22,7 +22,7 @@ static int ecc_jasper_initialise() { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 int jaserr = 0; jas_conf_clear(); jas_conf_set_max_mem_usage(jas_get_total_mem_size()); @@ -36,7 +36,7 @@ static int ecc_jasper_initialise() static jas_image_t* ecc_jasper_decode(jas_stream_t *in) { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 /* Second argument (=fmt) < 0 means "If possible, try to determine the format of the input data" */ return jas_image_decode(in, -1, 0); #else @@ -46,7 +46,7 @@ static jas_image_t* ecc_jasper_decode(jas_stream_t *in) static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *optstr) { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 const int fmt = jas_image_strtofmt("jpc"); return jas_image_encode(image, jpcstream, fmt, optstr); #else @@ -56,7 +56,7 @@ static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char * static void ecc_jasper_cleanup() { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 jas_cleanup_thread(); jas_cleanup_library(); #endif From f5e24997764e32efb8659cf2b5ba77ec2ba008ec Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 30 Nov 2022 14:23:45 +0000 Subject: [PATCH 14/26] ECC-1484: GRIB: New MARS class for Greenhouse gases dataset --- definitions/mars/class.table | 1 + 1 file changed, 1 insertion(+) diff --git a/definitions/mars/class.table b/definitions/mars/class.table index 446f13833..516e43217 100644 --- a/definitions/mars/class.table +++ b/definitions/mars/class.table @@ -42,6 +42,7 @@ 41 l6 ERA6/LAND 42 ef EFAS (European flood awareness system) 43 gf GLOFAS (Global flood awareness system) +44 gg Greenhouse Gases 99 te Test 100 at Austria 101 be Belgium From 1f4a6ba97388191fca97188d2e6ffa9fb2857cce Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 5 Jan 2023 12:10:33 +0000 Subject: [PATCH 15/26] ECC-1491: Pseudo GRIB: The offset and count keys are incorrect --- src/grib_io.c | 2 ++ tests/pseudo_budg.sh | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/grib_io.c b/src/grib_io.c index 08d4f14b9..6c82c7319 100644 --- a/src/grib_io.c +++ b/src/grib_io.c @@ -437,6 +437,8 @@ static int read_PSEUDO(reader* r, const char* type) i++; } + r->offset = r->tell(r->read_data) - 4; + for (j = 0; j < 3; j++) { if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err) return err; diff --git a/tests/pseudo_budg.sh b/tests/pseudo_budg.sh index c4db067e8..033b51e07 100755 --- a/tests/pseudo_budg.sh +++ b/tests/pseudo_budg.sh @@ -47,5 +47,14 @@ ${tools_dir}/grib_ls -jm $tempOut ms=`${tools_dir}/grib_get -p mars.step $tempOut` [ "$ms" = "19" ] +# ECC-1491 +cat ${data_dir}/budg ${data_dir}/budg > $tempBud +${tools_dir}/grib_get -p count,offset $tempBud > $tempOut +cat > $tempRef << EOF +1 0 +2 6000 +EOF +diff $tempRef $tempOut + rm -f $tempRef $tempOut $tempBud From 77f3c2ff7b882e810f7a6288d4e6f23a4bba3577 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 5 Jan 2023 12:11:11 +0000 Subject: [PATCH 16/26] Bump up version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78382b33c..e708666e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) # Initialise project -project( eccodes VERSION 2.27.1 LANGUAGES C ) +project( eccodes VERSION 2.27.2 LANGUAGES C ) ############################################################################### # system checks needed for eccodes_config.h and some options like MEMFS From c179245304f6b4203a1b501e9bb0c8a5c4eb6e5d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 6 Feb 2023 11:34:56 +0000 Subject: [PATCH 17/26] ECC-1081: 'level' is not edition-independent for potential vorticity levels --- CMakeLists.txt | 2 +- definitions/grib1/section.1.def | 4 ++++ src/grib_accessor_class_g2level.c | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 724c9130d..d0f2860a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) # Initialise project -project( eccodes VERSION 2.28.0 LANGUAGES C ) +project( eccodes VERSION 2.28.1 LANGUAGES C ) ############################################################################### # system checks needed for eccodes_config.h and some options like MEMFS diff --git a/definitions/grib1/section.1.def b/definitions/grib1/section.1.def index 974977dfd..25829af6c 100644 --- a/definitions/grib1/section.1.def +++ b/definitions/grib1/section.1.def @@ -112,6 +112,10 @@ if( indicatorOfTypeOfLevel == 109 || { alias mars.levelist = level; } +if (indicatorOfTypeOfLevel == 117) { + # See ECC-1081 + constant levelFactor = 6; +} unsigned[1] yearOfCentury : edition_specific; unsigned[1] month; diff --git a/src/grib_accessor_class_g2level.c b/src/grib_accessor_class_g2level.c index 4931f2f2b..d62152f53 100644 --- a/src/grib_accessor_class_g2level.c +++ b/src/grib_accessor_class_g2level.c @@ -292,6 +292,7 @@ static int pack_long(grib_accessor* a, const long* val, size_t* len) long value_first = *val; long scale_first = 0; long type_first = 0; + long levelFactor = 1; char pressure_units[10] = {0,}; size_t pressure_units_len = 10; @@ -324,6 +325,13 @@ static int pack_long(grib_accessor* a, const long* val, size_t* len) if (!strcmp(pressure_units, "hPa")) value_first *= 100; break; + case 109: + //printf("type1 = 109\n"); + if ((ret = grib_get_long(hand, "levelFactor", &levelFactor)) == GRIB_SUCCESS) { + // See ECC-1081 + // printf(" Special case - change scale_first to %ld\n", levelFactor); + scale_first = levelFactor; + } default: break; From 33e6174199868eeb4c46dec90ae2af690c927b4c Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 17 Feb 2023 12:55:24 +0000 Subject: [PATCH 18/26] ECC-1529: grib_util_set_spec: Input packing type of grid_ccsds changed to grid_simple --- src/grib_util.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/grib_util.c b/src/grib_util.c index e017e6d4e..eb328bd54 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -990,6 +990,9 @@ grib_handle* grib_util_set_spec2(grib_handle* h, if (STR_EQUAL(input_packing_type, "grid_ieee")) { SET_STRING_VALUE("packingType", input_packing_type); } + if (STR_EQUAL(input_packing_type, "grid_ccsds")) { + setCcsdsPacking = 1; + } } /*if ( (*err=check_values(data_values, data_values_count))!=GRIB_SUCCESS ) { From f89129d102392656522a793d2dbcfa8cf9dc6b8a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 17 Feb 2023 13:02:19 +0000 Subject: [PATCH 19/26] ECC-1529: grib_util_set_spec: Input packing type of grid_ccsds changed to grid_simple --- tests/grib_util_set_spec.c | 83 +++++++++++++++++++++---------------- tests/grib_util_set_spec.sh | 21 ++++++++-- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/tests/grib_util_set_spec.c b/tests/grib_util_set_spec.c index a2a4b4756..6d65653a1 100644 --- a/tests/grib_util_set_spec.c +++ b/tests/grib_util_set_spec.c @@ -17,19 +17,20 @@ static int get_packing_type_code(const char* packingType) { int result = GRIB_UTIL_PACKING_TYPE_GRID_SIMPLE; if (packingType == NULL) - return result; + return GRIB_UTIL_PACKING_TYPE_SAME_AS_INPUT; if (STR_EQUAL(packingType, "grid_jpeg")) - result = GRIB_UTIL_PACKING_TYPE_JPEG; - if (STR_EQUAL(packingType, "grid_ccsds")) - result = GRIB_UTIL_PACKING_TYPE_CCSDS; + return GRIB_UTIL_PACKING_TYPE_JPEG; + else if (STR_EQUAL(packingType, "grid_ccsds")) + return GRIB_UTIL_PACKING_TYPE_CCSDS; else if (STR_EQUAL(packingType, "grid_simple")) - result = GRIB_UTIL_PACKING_TYPE_GRID_SIMPLE; + return GRIB_UTIL_PACKING_TYPE_GRID_SIMPLE; else if (STR_EQUAL(packingType, "grid_second_order")) - result = GRIB_UTIL_PACKING_TYPE_GRID_SECOND_ORDER; + return GRIB_UTIL_PACKING_TYPE_GRID_SECOND_ORDER; else if (STR_EQUAL(packingType, "grid_ieee")) - result = GRIB_UTIL_PACKING_TYPE_IEEE; + return GRIB_UTIL_PACKING_TYPE_IEEE; + Assert(!"Invalid packingType"); return result; } @@ -47,8 +48,8 @@ static void test_reduced_gg(int remove_local_def, int edition, const char* packi const void* buffer = NULL; char gridType[128] = {0,}; - codes_handle* handle = 0; - codes_handle* finalh = 0; + grib_handle* handle = 0; + grib_handle* finalh = 0; grib_util_grid_spec spec = {0,}; grib_util_packing_spec packing_spec = {0,}; @@ -70,6 +71,9 @@ static void test_reduced_gg(int remove_local_def, int edition, const char* packi CODES_CHECK(grib_get_size(handle, "values", &inlen), 0); values = (double*)malloc(sizeof(double) * inlen); CODES_CHECK(grib_get_double_array(handle, "values", values, &inlen), 0); + // make sure values are not constant + values[0] = 4.4; + values[1] = 5.5; for (i = 0; i < inlen; ++i) { values[i] *= 1.10; } @@ -90,7 +94,10 @@ static void test_reduced_gg(int remove_local_def, int edition, const char* packi packing_spec.packing_type = get_packing_type_code(packingType); packing_spec.bitsPerValue = 24; packing_spec.accuracy = GRIB_UTIL_ACCURACY_USE_PROVIDED_BITS_PER_VALUES; - packing_spec.packing = GRIB_UTIL_PACKING_USE_PROVIDED; + if (packingType) + packing_spec.packing = GRIB_UTIL_PACKING_USE_PROVIDED; + else + packing_spec.packing = GRIB_UTIL_PACKING_SAME_AS_INPUT; /*Extra settings packing_spec.extra_settings_count++; @@ -112,7 +119,7 @@ static void test_reduced_gg(int remove_local_def, int edition, const char* packi /* Try some invalid inputs and check it is handled */ { - codes_handle* h2 = 0; + grib_handle* h2 = 0; packing_spec.accuracy = 999; h2 = grib_util_set_spec(handle, &spec, &packing_spec, set_spec_flags, values, outlen, &err); Assert(err == GRIB_INTERNAL_ERROR); @@ -158,18 +165,18 @@ static void test_regular_ll(int remove_local_def, int edition, const char* packi char gridType[128] = {0,}; long input_edition = 0; - codes_handle* handle = 0; - codes_handle* finalh = 0; + grib_handle* handle = 0; + grib_handle* finalh = 0; grib_util_grid_spec spec = {0,}; grib_util_packing_spec packing_spec = {0,}; Assert(input_filename); in = fopen(input_filename, "rb"); Assert(in); - handle = codes_handle_new_from_file(0, in, PRODUCT_GRIB, &err); + handle = grib_handle_new_from_file(0, in, &err); Assert(handle); - CODES_CHECK(codes_get_long(handle, "edition", &input_edition), 0); + CODES_CHECK(grib_get_long(handle, "edition", &input_edition), 0); CODES_CHECK(grib_get_string(handle, "gridType", gridType, &slen), 0); if (!STR_EQUAL(gridType, "regular_ll")) { @@ -180,9 +187,12 @@ static void test_regular_ll(int remove_local_def, int edition, const char* packi out = fopen(output_filename, "wb"); Assert(out); - CODES_CHECK(codes_get_size(handle, "values", &inlen), 0); + CODES_CHECK(grib_get_size(handle, "values", &inlen), 0); values = (double*)malloc(sizeof(double) * inlen); - CODES_CHECK(codes_get_double_array(handle, "values", values, &inlen), 0); + CODES_CHECK(grib_get_double_array(handle, "values", values, &inlen), 0); + // make sure values are not constant + values[0] = 4.4; + values[1] = 5.5; spec.grid_type = GRIB_UTIL_GRID_SPEC_REGULAR_LL; spec.Nj = 14; @@ -201,7 +211,10 @@ static void test_regular_ll(int remove_local_def, int edition, const char* packi packing_spec.packing_type = get_packing_type_code(packingType); packing_spec.bitsPerValue = 24; packing_spec.accuracy = GRIB_UTIL_ACCURACY_USE_PROVIDED_BITS_PER_VALUES; - packing_spec.packing = GRIB_UTIL_PACKING_USE_PROVIDED; + if (packingType) + packing_spec.packing = GRIB_UTIL_PACKING_USE_PROVIDED; + else + packing_spec.packing = GRIB_UTIL_PACKING_SAME_AS_INPUT; packing_spec.extra_settings_count++; packing_spec.extra_settings[0].type = GRIB_TYPE_LONG; @@ -215,7 +228,7 @@ static void test_regular_ll(int remove_local_def, int edition, const char* packi packing_spec.deleteLocalDefinition = 1; } - finalh = codes_grib_util_set_spec( + finalh = grib_util_set_spec( handle, &spec, &packing_spec, @@ -232,19 +245,19 @@ static void test_regular_ll(int remove_local_def, int edition, const char* packi if (input_edition == 1) { const double expected_lat1 = 60.001; double lat1 = 0; - CODES_CHECK(codes_get_double(finalh, "latitudeOfFirstGridPointInDegrees", &lat1), 0); + CODES_CHECK(grib_get_double(finalh, "latitudeOfFirstGridPointInDegrees", &lat1), 0); Assert(fabs(lat1 - expected_lat1) < 1e-10); } /* Write out the message to the output file */ - CODES_CHECK(codes_get_message(finalh, &buffer, &size), 0); + CODES_CHECK(grib_get_message(finalh, &buffer, &size), 0); CODES_CHECK(codes_check_message_header(buffer, size, PRODUCT_GRIB), 0); CODES_CHECK(codes_check_message_footer(buffer, size, PRODUCT_GRIB), 0); if (fwrite(buffer, 1, size, out) != size) { Assert(0); } - codes_handle_delete(handle); - codes_handle_delete(finalh); + grib_handle_delete(handle); + grib_handle_delete(finalh); free(values); fclose(in); fclose(out); @@ -266,13 +279,13 @@ static void test_grid_complex_spatial_differencing(int remove_local_def, int edi char gridType[128] = {0,}; double theMax,theMin,theAverage; - codes_handle *handle = 0; - codes_handle *finalh = 0; + grib_handle *handle = 0; + grib_handle *finalh = 0; grib_util_grid_spec spec={0,}; grib_util_packing_spec packing_spec={0,}; in = fopen(input_filename,"rb"); Assert(in); - handle = codes_handle_new_from_file(0, in, PRODUCT_GRIB, &err); Assert(handle); + handle = grib_handle_new_from_file(0, in, &err); Assert(handle); CODES_CHECK(grib_get_string(handle, "packingType", gridType, &slen),0); if (!STR_EQUAL(gridType, "grid_complex_spatial_differencing")) { @@ -281,13 +294,13 @@ static void test_grid_complex_spatial_differencing(int remove_local_def, int edi } out = fopen(output_filename,"wb"); Assert(out); - CODES_CHECK(codes_get_size(handle,"values",&inlen), 0); + CODES_CHECK(grib_get_size(handle,"values",&inlen), 0); values = (double*)malloc(sizeof(double)*inlen); - CODES_CHECK(codes_get_double_array(handle, "values", values,&inlen), 0); + CODES_CHECK(grib_get_double_array(handle, "values", values,&inlen), 0); - CODES_CHECK(codes_get_double(handle, "max", &theMax),0); - CODES_CHECK(codes_get_double(handle, "min", &theMin),0); - CODES_CHECK(codes_get_double(handle, "average",&theAverage),0); + CODES_CHECK(grib_get_double(handle, "max", &theMax),0); + CODES_CHECK(grib_get_double(handle, "min", &theMin),0); + CODES_CHECK(grib_get_double(handle, "average",&theAverage),0); printf("inlen=%lu \t max=%g \t min=%g \t avg=%g\n", inlen, theMax, theMin, theAverage); spec.grid_type = GRIB_UTIL_GRID_SPEC_REGULAR_LL; @@ -315,7 +328,7 @@ static void test_grid_complex_spatial_differencing(int remove_local_def, int edi packing_spec.deleteLocalDefinition = 1; } - finalh = codes_grib_util_set_spec( + finalh = grib_util_set_spec( handle, &spec, &packing_spec, @@ -327,12 +340,12 @@ static void test_grid_complex_spatial_differencing(int remove_local_def, int edi Assert(err == 0); /* Write out the message to the output file */ - CODES_CHECK(codes_get_message(finalh, &buffer, &size),0); + CODES_CHECK(grib_get_message(finalh, &buffer, &size),0); if(fwrite(buffer,1,size,out) != size) { Assert(0); } - codes_handle_delete(handle); - codes_handle_delete(finalh); + grib_handle_delete(handle); + grib_handle_delete(finalh); fclose(in); fclose(out); } diff --git a/tests/grib_util_set_spec.sh b/tests/grib_util_set_spec.sh index b132091ad..c89543a57 100755 --- a/tests/grib_util_set_spec.sh +++ b/tests/grib_util_set_spec.sh @@ -14,7 +14,7 @@ # -------------------------------------------------- # Regular Lat/Lon Grid # -------------------------------------------------- -infile=../data/latlon.grib +infile=${data_dir}/latlon.grib outfile=out.grib_util_set_spec.grib tempOut=temp.grib_util_set_spec.grib grib_util_set_spec=${test_dir}/grib_util_set_spec @@ -84,7 +84,7 @@ $EXEC $grib_util_set_spec -p grid_second_order $infile $outfile # Check output file. Values are scaled up by 1.1 grib_check_key_equals $outfile packingType grid_second_order stats_new=`${tools_dir}/grib_get -F%.2f -p min,max $outfile` -[ "$stats_new" = "176.28 246.90" ] +[ "$stats_new" = "4.84 246.90" ] ${tools_dir}/grib_get_data $outfile > /dev/null CHECK_TOOL="${tools_dir}/grib_check_gaussian_grid" @@ -93,15 +93,28 @@ if [ -x $CHECK_TOOL ]; then fi ### Constant field N=32 -########################################### +# --------------------------- infile=$ECCODES_SAMPLES_PATH/reduced_gg_pl_32_grib2.tmpl rm -f $outfile $EXEC $grib_util_set_spec $infile $outfile -grib_check_key_equals $outfile "packingType,const" "grid_simple 1" +grib_check_key_equals $outfile "packingType,const" "grid_simple 0" ${tools_dir}/grib_get_data $outfile > /dev/null +# CCSDS input +# --------------------------- +infile=${data_dir}/ccsds.grib2 +$EXEC $grib_util_set_spec $infile $outfile +grib_check_key_equals $outfile packingType grid_ccsds + +$EXEC $grib_util_set_spec -p grid_simple $infile $outfile +grib_check_key_equals $outfile packingType grid_simple + +$EXEC $grib_util_set_spec -p grid_second_order $infile $outfile +grib_check_key_equals $outfile packingType grid_second_order + + ### Clean up rm -f $outfile $tempOut rm -f error.data From 1f0620a5642858c3d21261117134985536b2fb0e Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 17 Feb 2023 13:44:12 +0000 Subject: [PATCH 20/26] ECC-1529: Test --- tests/grib_util_set_spec.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/grib_util_set_spec.sh b/tests/grib_util_set_spec.sh index c89543a57..1c83ba0d0 100755 --- a/tests/grib_util_set_spec.sh +++ b/tests/grib_util_set_spec.sh @@ -104,15 +104,17 @@ ${tools_dir}/grib_get_data $outfile > /dev/null # CCSDS input # --------------------------- -infile=${data_dir}/ccsds.grib2 -$EXEC $grib_util_set_spec $infile $outfile -grib_check_key_equals $outfile packingType grid_ccsds +if [ $HAVE_AEC -eq 1 ]; then + infile=${data_dir}/ccsds.grib2 + $EXEC $grib_util_set_spec $infile $outfile + grib_check_key_equals $outfile packingType grid_ccsds -$EXEC $grib_util_set_spec -p grid_simple $infile $outfile -grib_check_key_equals $outfile packingType grid_simple + $EXEC $grib_util_set_spec -p grid_simple $infile $outfile + grib_check_key_equals $outfile packingType grid_simple -$EXEC $grib_util_set_spec -p grid_second_order $infile $outfile -grib_check_key_equals $outfile packingType grid_second_order + $EXEC $grib_util_set_spec -p grid_second_order $infile $outfile + grib_check_key_equals $outfile packingType grid_second_order +fi ### Clean up From a7e8286c9ee4e62461597b209bfb502337267ecf Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 18 Feb 2023 14:41:52 +0000 Subject: [PATCH 21/26] ECC-1530: grib_util_set_spec: Input packing type of grid_second_order changed to grid_simple --- src/grib_util.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/grib_util.c b/src/grib_util.c index eb328bd54..37c118703 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -993,6 +993,9 @@ grib_handle* grib_util_set_spec2(grib_handle* h, if (STR_EQUAL(input_packing_type, "grid_ccsds")) { setCcsdsPacking = 1; } + if (STR_EQUAL(input_packing_type, "grid_second_order")) { + setSecondOrder = 1; + } } /*if ( (*err=check_values(data_values, data_values_count))!=GRIB_SUCCESS ) { From 13bd64a940736618cc4d169b771ef94b38ccf87f Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 22 Feb 2023 17:59:21 +0000 Subject: [PATCH 22/26] ECC-1532: GRIB: Implement feature/carra_expver --- definitions/grib2/crraLocalVersion.table | 1 + definitions/grib2/local.crra.2.def | 7 +++++++ definitions/grib2/products_10.def | 4 +++- tests/grib_uerra.sh | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 definitions/grib2/local.crra.2.def diff --git a/definitions/grib2/crraLocalVersion.table b/definitions/grib2/crraLocalVersion.table index 3b7d11367..b84a43579 100644 --- a/definitions/grib2/crraLocalVersion.table +++ b/definitions/grib2/crraLocalVersion.table @@ -1 +1,2 @@ 1 CRRA Copernicus regional reanalysis +2 CRRA Copernicus regional reanalysis with expver diff --git a/definitions/grib2/local.crra.2.def b/definitions/grib2/local.crra.2.def new file mode 100644 index 000000000..bb3c1b8ae --- /dev/null +++ b/definitions/grib2/local.crra.2.def @@ -0,0 +1,7 @@ +# CARRA/CERRA local + +codetable[2] suiteName "grib2/crra_suiteName.table" : dump; +alias crraSuiteID = suiteName; + +ksec1expver[4] experimentVersionNumber = "0002" : dump; +alias marsExpver = experimentVersionNumber ; diff --git a/definitions/grib2/products_10.def b/definitions/grib2/products_10.def index bafd04a6f..df1f00e8a 100644 --- a/definitions/grib2/products_10.def +++ b/definitions/grib2/products_10.def @@ -1,5 +1,7 @@ # (C) Copyright 2005- ECMWF. # Copernicus regional reanalysis (CARRA/CERRA) -constant marsExpver = 'prod'; +if (!defined(marsExpver)) { + constant marsExpver = 'prod'; +} include "grib2/products_crra.def" diff --git a/tests/grib_uerra.sh b/tests/grib_uerra.sh index a0b827cbf..cdd7beb87 100755 --- a/tests/grib_uerra.sh +++ b/tests/grib_uerra.sh @@ -73,5 +73,23 @@ status=$? set -e [ $status -ne 0 ] +# ECC-1532 +# --------- +# By default crraLocalVersion=1 which does not allow expver to be set +# because it is a constant (prod or test) +set +e +${tools_dir}/grib_set -s \ + productionStatusOfProcessedData=10,grib2LocalSectionPresent=1,marsExpver=coco \ +$grib2_sample $temp1 +status=$? +set -e +[ $status -ne 0 ] + +# crraLocalVersion=2 has a coded key which can be set +${tools_dir}/grib_set -s \ + productionStatusOfProcessedData=10,grib2LocalSectionPresent=1,crraLocalVersion=2,marsExpver=coco \ +$grib2_sample $temp1 +grib_check_key_equals $temp1 'marsExpver,mars.expver' 'coco coco' + # Clean up rm -f $temp1 $temp2 $tempSample From 35cb72061bacc8eaf09df3e3418b27f0f3d47089 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 24 Feb 2023 11:00:03 +0000 Subject: [PATCH 23/26] Remove C++ comments --- src/grib_accessor_class_g2level.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/grib_accessor_class_g2level.c b/src/grib_accessor_class_g2level.c index d62152f53..9cb95ecb0 100644 --- a/src/grib_accessor_class_g2level.c +++ b/src/grib_accessor_class_g2level.c @@ -326,10 +326,8 @@ static int pack_long(grib_accessor* a, const long* val, size_t* len) value_first *= 100; break; case 109: - //printf("type1 = 109\n"); if ((ret = grib_get_long(hand, "levelFactor", &levelFactor)) == GRIB_SUCCESS) { - // See ECC-1081 - // printf(" Special case - change scale_first to %ld\n", levelFactor); + /* See ECC-1081 */ scale_first = levelFactor; } From a5d0e3f5b0ba05d42fa9d910594c7e2e139a338d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 24 Feb 2023 12:44:05 +0000 Subject: [PATCH 24/26] grib_util_set_spec: tests --- tests/grib_util_set_spec.sh | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/grib_util_set_spec.sh b/tests/grib_util_set_spec.sh index 1c83ba0d0..3daf5d0db 100755 --- a/tests/grib_util_set_spec.sh +++ b/tests/grib_util_set_spec.sh @@ -14,7 +14,7 @@ # -------------------------------------------------- # Regular Lat/Lon Grid # -------------------------------------------------- -infile=${data_dir}/latlon.grib +infile=${data_dir}/latlon.grib # edition 1 outfile=out.grib_util_set_spec.grib tempOut=temp.grib_util_set_spec.grib grib_util_set_spec=${test_dir}/grib_util_set_spec @@ -42,7 +42,7 @@ res=`${tools_dir}/grib_get -p edition,section2Used $outfile` [ "$res" = "2 0" ] # GRIB2 input with local definition -infile=../data/regular_latlon_surface.grib2 +infile=${data_dir}/regular_latlon_surface.grib2 $EXEC $grib_util_set_spec -r $infile $outfile > /dev/null grib_check_key_equals $outfile section2Used 0 # GRIB2 input without local definition @@ -52,18 +52,26 @@ grib_check_key_equals $outfile section2Used 0 # Convert to edition2 and use JPEG for packing if [ $HAVE_JPEG -eq 1 ]; then - infile=../data/latlon.grib + infile=${data_dir}/latlon.grib $EXEC $grib_util_set_spec -e 2 -p grid_jpeg $infile $outfile > /dev/null res=`${tools_dir}/grib_get -p edition,section2Used,packingType $outfile` [ "$res" = "2 1 grid_jpeg" ] fi -# Convert to edition2 and use CCSDS for packing +# CCSDS for packing and different editions if [ $HAVE_AEC -eq 1 ]; then - infile=../data/latlon.grib + infile=${data_dir}/sample.grib2 + $EXEC $grib_util_set_spec -p grid_ccsds $infile $outfile + grib_check_key_equals $outfile packingType grid_ccsds + + infile=${data_dir}/latlon.grib #grib1 $EXEC $grib_util_set_spec -e 2 -p grid_ccsds $infile $outfile > /dev/null res=`${tools_dir}/grib_get -p edition,section2Used,packingType $outfile` [ "$res" = "2 1 grid_ccsds" ] + + # If we don't convert, then should leave it as grid_simple (No CCSDS in grib1) + $EXEC $grib_util_set_spec -p grid_ccsds $infile $outfile + grib_check_key_equals $outfile packingType grid_simple fi # -------------------------------------------------- @@ -72,7 +80,7 @@ fi # The gaussian tests intentionally cause an error so need to stop it failing unset ECCODES_FAIL_IF_LOG_MESSAGE -infile=../data/reduced_gaussian_model_level.grib2 +infile=${data_dir}/reduced_gaussian_model_level.grib2 outfile=out.grib_util_set_spec.grib rm -f $outfile @@ -116,6 +124,23 @@ if [ $HAVE_AEC -eq 1 ]; then grib_check_key_equals $outfile packingType grid_second_order fi +# Second order input/output +# --------------------------- +${tools_dir}/grib_set -r -s packingType=grid_second_order ${data_dir}/sample.grib2 $tempOut +grib_check_key_equals $tempOut packingType grid_second_order +$EXEC $grib_util_set_spec $tempOut $outfile +grib_check_key_equals $outfile packingType grid_second_order + +$EXEC $grib_util_set_spec -p grid_second_order ${data_dir}/simple.grib $outfile +grib_check_key_equals $outfile packingType grid_second_order + + +# Check DEBUG output +# --------------------------- +export ECCODES_DEBUG=-1 +$EXEC $grib_util_set_spec ${data_dir}/sample.grib2 $outfile > $tempOut 2>&1 +grep -q "ECCODES DEBUG grib_util:" $tempOut + ### Clean up rm -f $outfile $tempOut From dadd1d8ef911ac660906634ec1b3e589c54e6bf1 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 24 Feb 2023 12:49:29 +0000 Subject: [PATCH 25/26] Testing: JasPer issues --- tests/grib_threads_ecc-604.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/grib_threads_ecc-604.sh b/tests/grib_threads_ecc-604.sh index 47c1b5d26..c2d491ffb 100755 --- a/tests/grib_threads_ecc-604.sh +++ b/tests/grib_threads_ecc-604.sh @@ -71,7 +71,8 @@ GRIB2_INPUTS=" ${data_dir}/test_file.grib2 ${data_dir}/sample.grib2" -if [ $HAVE_JPEG -eq 1 ]; then +# There is a problem with multi-threading and Jasper versions > 2 +if [ $HAVE_JPEG -eq 1 -a $HAVE_LIBJASPER -eq 0 ]; then echo "Adding extra files (HAVE_JPEG=1)" GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS fi From bc105ef53e92512720f6b59a3bc9c4e8ff578b93 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 24 Feb 2023 13:38:09 +0000 Subject: [PATCH 26/26] ECC-1532: GRIB: Allow setting 'expver' on CARRA/CERRA data --- definitions/grib2/products_11.def | 6 ++++-- tests/grib_uerra.sh | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/definitions/grib2/products_11.def b/definitions/grib2/products_11.def index 6084a0b37..bad30182a 100644 --- a/definitions/grib2/products_11.def +++ b/definitions/grib2/products_11.def @@ -1,5 +1,7 @@ # (C) Copyright 2005- ECMWF. -# Copernicus regional reanalysis (CARRA/CERRA) -constant marsExpver = 'test'; +# Copernicus regional reanalysis test (CARRA/CERRA) +if (!defined(marsExpver)) { + constant marsExpver = 'test'; +} include "grib2/products_crra.def" diff --git a/tests/grib_uerra.sh b/tests/grib_uerra.sh index cdd7beb87..15f9523e6 100755 --- a/tests/grib_uerra.sh +++ b/tests/grib_uerra.sh @@ -85,11 +85,18 @@ status=$? set -e [ $status -ne 0 ] -# crraLocalVersion=2 has a coded key which can be set +# ECC-1532 +# crraLocalVersion=2 has a coded key for experimentVersionNumber ${tools_dir}/grib_set -s \ productionStatusOfProcessedData=10,grib2LocalSectionPresent=1,crraLocalVersion=2,marsExpver=coco \ $grib2_sample $temp1 grib_check_key_equals $temp1 'marsExpver,mars.expver' 'coco coco' +${tools_dir}/grib_set -s \ + productionStatusOfProcessedData=11,grib2LocalSectionPresent=1,crraLocalVersion=2,experimentVersionNumber=0078 \ +$grib2_sample $temp1 +grib_check_key_equals $temp1 'marsExpver,mars.expver' '0078 0078' + + # Clean up rm -f $temp1 $temp2 $tempSample