IOSS: Add support for lossy compression specifying NSD for netCDF qua…

…ntize method
sandialabs · Jun 19, 2024 · 62d5ee7 · 62d5ee7
1 parent 47474bd
commit 62d5ee7
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 17 deletions.
diff --git a/packages/seacas/libraries/ioss/src/exodus/Ioex_DatabaseIO.C b/packages/seacas/libraries/ioss/src/exodus/Ioex_DatabaseIO.C
@@ -401,6 +401,11 @@ namespace Ioex {
         ex_set_option(m_exodusFilePtr, EX_OPT_COMPRESSION_TYPE, exo_method);
       }
 
+      if (properties.exists("COMPRESSION_QUANTIZE_NSD")) {
+        int quant_level = properties.get("COMPRESSION_QUANTIZE_NSD").get_int();
+        ex_set_option(m_exodusFilePtr, EX_OPT_QUANTIZE_NSD, quant_level);
+      }
+
       if (properties.exists("COMPRESSION_LEVEL")) {
         int comp_level = properties.get("COMPRESSION_LEVEL").get_int();
         ex_set_option(m_exodusFilePtr, EX_OPT_COMPRESSION_LEVEL, comp_level);

diff --git a/packages/seacas/libraries/ioss/src/exodus/Ioex_ParallelDatabaseIO.C b/packages/seacas/libraries/ioss/src/exodus/Ioex_ParallelDatabaseIO.C
@@ -724,6 +724,12 @@ namespace Ioex {
         }
         ex_set_option(m_exodusFilePtr, EX_OPT_COMPRESSION_TYPE, exo_method);
       }
+
+      if (properties.exists("COMPRESSION_QUANTIZE_NSD")) {
+        int quant_level = properties.get("COMPRESSION_QUANTIZE_NSD").get_int();
+        ex_set_option(m_exodusFilePtr, EX_OPT_QUANTIZE_NSD, quant_level);
+      }
+
       if (properties.exists("COMPRESSION_LEVEL")) {
         int comp_level = properties.get("COMPRESSION_LEVEL").get_int();
         ex_set_option(m_exodusFilePtr, EX_OPT_COMPRESSION_LEVEL, comp_level);
@@ -4809,7 +4815,7 @@ namespace Ioex {
   void ParallelDatabaseIO::output_processor_id_map(Ioss::Region *region, INT /*dummy*/)
   {
     std::vector<INT> proc_id(elementCount, myProcessor);
-    const auto &blocks = region->get_element_blocks();
+    const auto      &blocks = region->get_element_blocks();
     for (const auto &block : blocks) {
       put_field_internal(block, block->get_field("proc_id"), Data(proc_id), -1);
     }
@@ -4870,10 +4876,10 @@ namespace Ioex {
       add_processor_id_map(region);
       output_other_metadata();
       if (int_byte_size_api() == 8) {
-	output_processor_id_map(region, int64_t(0));
+        output_processor_id_map(region, int64_t(0));
       }
       else {
-	output_processor_id_map(region, int(0));
+        output_processor_id_map(region, int(0));
       }
     }
   }

diff --git a/packages/seacas/libraries/ioss/src/main/io_shell.C b/packages/seacas/libraries/ioss/src/main/io_shell.C
@@ -631,7 +631,8 @@ namespace {
       properties.add(Ioss::Property("IGNORE_INFO_RECORDS", "YES"));
     }
 
-    if (interFace.compression_level > 0 || interFace.shuffle || interFace.szip) {
+    if (interFace.compression_level > 0 || interFace.shuffle || interFace.szip || interFace.quant ||
+        interFace.zlib) {
       properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
       properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compression_level));
       properties.add(Ioss::Property("COMPRESSION_SHUFFLE", static_cast<int>(interFace.shuffle)));
@@ -642,6 +643,9 @@ namespace {
       else if (interFace.zlib) {
         properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib"));
       }
+      else if (interFace.quant) {
+        properties.add(Ioss::Property("COMPRESSION_QUANTIZE_NSD", interFace.quantize_nsd));
+      }
     }
 
     if (interFace.compose_output == "default") {

diff --git a/packages/seacas/libraries/ioss/src/main/shell_interface.C b/packages/seacas/libraries/ioss/src/main/shell_interface.C
@@ -129,7 +129,12 @@ void IOShell::Interface::enroll_options()
   options_.enroll(
       "szip", Ioss::GetLongOption::NoValue,
       "Use the SZip library if compression is enabled. Not as portable as zlib [exodus only]",
-      nullptr, nullptr, true);
+      nullptr);
+
+  options_.enroll("quantize_nsd", Ioss::GetLongOption::MandatoryValue,
+                  "Use the lossy quantize compression method.  Value specifies number of digits to "
+                  "retain (1..15) [exodus only]",
+                  nullptr, nullptr, true);
 
 #if defined(SEACAS_HAVE_MPI)
   options_.enroll(
@@ -215,10 +220,10 @@ void IOShell::Interface::enroll_options()
                   "Files are decomposed externally into a file-per-processor in a parallel run.",
                   nullptr);
 
-  options_.enroll(
-      "add_processor_id_field", Ioss::GetLongOption::NoValue,
-      "Add a cell-centered field whose value is the processor id of that cell", nullptr);
-  
+  options_.enroll("add_processor_id_field", Ioss::GetLongOption::NoValue,
+                  "Add a cell-centered field whose value is the processor id of that cell",
+                  nullptr);
+
   options_.enroll("serialize_io_size", Ioss::GetLongOption::MandatoryValue,
                   "Number of processors that can perform simultaneous IO operations in "
                   "a parallel run;\n\t\t0 to disable",
@@ -233,10 +238,11 @@ void IOShell::Interface::enroll_options()
       "If non-zero, then put <$val> timesteps in each file. Then close file and start new file.",
       nullptr);
 
-  options_.enroll("split_cyclic", Ioss::GetLongOption::MandatoryValue,
-                  "If non-zero, then the `split_times` timesteps will be put into <$val> files\n\t\tand "
-                  "then recycle filenames.",
-                  nullptr);
+  options_.enroll(
+      "split_cyclic", Ioss::GetLongOption::MandatoryValue,
+      "If non-zero, then the `split_times` timesteps will be put into <$val> files\n\t\tand "
+      "then recycle filenames.",
+      nullptr);
 
   options_.enroll("file_per_state", Ioss::GetLongOption::NoValue,
                   "put transient data for each timestep in separate file (EXPERIMENTAL)", nullptr);
@@ -320,7 +326,7 @@ void IOShell::Interface::enroll_options()
 
   options_.enroll("omit_sets", Ioss::GetLongOption::MandatoryValue,
                   "comma-separated list of nodeset/edgeset/faceset/elemset/sideset names\n"
-		  "\t\tthat should NOT be transferred to output database",
+                  "\t\tthat should NOT be transferred to output database",
                   nullptr);
 
   options_.enroll("boundary_sideset", Ioss::GetLongOption::NoValue,
@@ -436,11 +442,20 @@ bool IOShell::Interface::parse_options(int argc, char **argv, int my_processor)
 
   shuffle = (options_.retrieve("shuffle") != nullptr);
   if (options_.retrieve("szip") != nullptr) {
-    szip = true;
-    zlib = false;
+    szip  = true;
+    zlib  = false;
+    quant = false;
   }
   zlib = (options_.retrieve("zlib") != nullptr);
 
+  {
+    const char *temp = options_.retrieve("quantize_nsd");
+    if (temp != nullptr) {
+      quant        = true;
+      quantize_nsd = std::strtol(temp, nullptr, 10);
+    }
+  }
+
   if (szip && zlib) {
     if (my_processor == 0) {
       fmt::print(stderr, "ERROR: Only one of 'szip' or 'zlib' can be specified.\n");
@@ -560,7 +575,7 @@ bool IOShell::Interface::parse_options(int argc, char **argv, int my_processor)
   }
 
   if (options_.retrieve("line_decomp") != nullptr) {
-    line_decomp = true;
+    line_decomp  = true;
     decomp_extra = options_.get_option_value("line_decomp", decomp_extra);
   }
 

diff --git a/packages/seacas/libraries/ioss/src/main/shell_interface.h b/packages/seacas/libraries/ioss/src/main/shell_interface.h
@@ -53,6 +53,7 @@ namespace IOShell {
     int                      surface_split_type{-1};
     int                      data_storage_type{0};
     int                      compression_level{0};
+    int                      quantize_nsd{0};
     int                      serialize_io_size{0};
     int                      flush_interval{0};
 
@@ -80,6 +81,7 @@ namespace IOShell {
     bool shuffle{false};
     bool zlib{true};
     bool szip{false};
+    bool quant{false};
     bool debug{false};
     bool detect_nans{false};
     bool statistics{false};