diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 9a67fc8058..dc6c2ceab1 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -48,6 +48,9 @@ add_subdirectory(cpp/benchmarks/latency_rec) add_subdirectory(cpp/benchmarks/latency_snd) add_subdirectory(cpp/benchmarks/many_connections_rec) add_subdirectory(cpp/benchmarks/many_connections_snd) +if(HAS_HDF5) +add_subdirectory(cpp/benchmarks/measurement) +endif() add_subdirectory(cpp/benchmarks/multiple_rec_cb) add_subdirectory(cpp/benchmarks/multiple_snd) add_subdirectory(cpp/benchmarks/performance_rec) diff --git a/samples/cpp/benchmarks/measurement/CMakeLists.txt b/samples/cpp/benchmarks/measurement/CMakeLists.txt new file mode 100644 index 0000000000..4d8a44a28f --- /dev/null +++ b/samples/cpp/benchmarks/measurement/CMakeLists.txt @@ -0,0 +1,37 @@ +# ========================= eCAL LICENSE ================================= +# +# Copyright (C) 2016 - 2019 Continental Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ========================= eCAL LICENSE ================================= + +cmake_minimum_required(VERSION 3.10) + +set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON) + +project(benchmark_hdf5) + +find_package(eCAL REQUIRED) + +ecal_add_sample(${PROJECT_NAME} src/main.cpp) + +target_link_libraries(${PROJECT_NAME} + eCAL::hdf5 +) + +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) + +ecal_install_sample(${PROJECT_NAME}) + +set_property(TARGET ${PROJECT_NAME} PROPERTY FOLDER samples/cpp/benchmarks/measurement) diff --git a/samples/cpp/benchmarks/measurement/src/main.cpp b/samples/cpp/benchmarks/measurement/src/main.cpp new file mode 100644 index 0000000000..d589e9a663 --- /dev/null +++ b/samples/cpp/benchmarks/measurement/src/main.cpp @@ -0,0 +1,148 @@ +/* ========================= eCAL LICENSE ================================= + * + * Copyright (C) 2016 - 2019 Continental Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ========================= eCAL LICENSE ================================= +*/ + +#include +#include +#include +#include +#include + +#include + +#define TEST_RAW_PERF 1 +#define TEST_HDF5_PERF 1 + +#define USE_OFSTREAM 0 +#define TEST_POST_SLEEP 2 // seconds + +#define DATA_SET_SIZE_1 (1024) // 1 kB +#define DATA_SET_NUMBER_1 (64*1024) + +#define DATA_SET_SIZE_2 (128*1024) // 128 kB +#define DATA_SET_NUMBER_2 (8*1024) + +#define DATA_SET_SIZE_3 (1024*1024) // 1 MB +#define DATA_SET_NUMBER_3 (1024) + +#define DATA_SET_SIZE_4 (4096*1024) // 4 MB +#define DATA_SET_NUMBER_4 (256) + +namespace +{ + // properties + std::string output_dir = "measurement_dir"; + const size_t max_size_per_file = 500; // MB + std::vector data; +} + +void MeasPerf(const std::string& file_name, const size_t pkg_size, const size_t pkg_num) +{ + data.resize(pkg_size); + const size_t write_loops(pkg_num); + +#if TEST_RAW_PERF + // Test raw performance + { + // start time + auto start = std::chrono::high_resolution_clock::now(); + +#if USE_OFSTREAM + + std::ofstream rfile(file_name + "_raw", std::ios::trunc | std::ios::out | std::ios::binary); + if (rfile) + { + for (size_t loops = 0; loops < write_loops; ++loops) + { + rfile.write(data.data(), data.size()); + } + } + rfile.close(); + +#else // USE_OFSTREAM + + FILE* pFile(nullptr); + std::string fname = file_name + "_raw"; + pFile = fopen(fname.c_str(), "wb"); + if (pFile) + { + for (size_t loops = 0; loops < write_loops; ++loops) + { + fwrite(data.data(), 1, data.size(), pFile); + } + fclose(pFile); + } + +#endif // USE_OFSTREAM + + // end time + auto finish = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = finish - start; + size_t sum_data = data.size() * write_loops; + std::cout << std::endl; + std::cout << "Packages number : " << write_loops << std::endl; + std::cout << "Packages size : " << data.size()/1024 << " kB" << std::endl; + std::cout << "Sum payload : " << sum_data / (1024*1024) << " MB" << std::endl; + std::cout << "Throughput RAW : " << int((sum_data / (1024.0 * 1024.0)) / elapsed.count()) << " MB/s " << std::endl; + std::cout << std::endl; + + std::this_thread::sleep_for(std::chrono::seconds(TEST_POST_SLEEP)); + } +#endif + +#if TEST_HDF5_PERF + // Test HDF5 performance + { + // start time + auto start = std::chrono::high_resolution_clock::now(); + + eCAL::eh5::HDF5Meas writer(output_dir, eCAL::eh5::CREATE); + writer.SetFileBaseName(file_name + "_hdf5"); + writer.SetMaxSizePerFile(max_size_per_file); + for (size_t loop = 0; loop < write_loops; ++loop) + { + writer.AddEntryToFile(static_cast(data.data()), data.size(), 0, 0, "myChannel", 0, loop); + } + writer.Close(); + + // end time + auto finish = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = finish - start; + size_t sum_data = data.size() * write_loops; + std::cout << std::endl; + std::cout << "Packages number : " << write_loops << std::endl; + std::cout << "Packages size : " << data.size() / 1024 << " kB" << std::endl; + std::cout << "Sum payload : " << sum_data / (1024 * 1024) << " MB" << std::endl; + std::cout << "Throughput HDF5 : " << int((sum_data / (1024.0 * 1024.0)) / elapsed.count()) << " MB/s " << std::endl; + std::cout << std::endl; + + std::this_thread::sleep_for(std::chrono::seconds(TEST_POST_SLEEP)); + } +#endif +} + +int main() +{ + MeasPerf("meas_1_pkg", DATA_SET_SIZE_1, DATA_SET_NUMBER_1); + + MeasPerf("meas_128_pkg", DATA_SET_SIZE_2, DATA_SET_NUMBER_2); + + MeasPerf("meas_1024_pkg", DATA_SET_SIZE_3, DATA_SET_NUMBER_3); + + MeasPerf("meas_4096_pkg", DATA_SET_SIZE_4, DATA_SET_NUMBER_4); +} diff --git a/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp b/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp index be1f3c430f..bba0185736 100644 --- a/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp +++ b/testing/contrib/ecalhdf5/hdf5_test/src/hdf5_test.cpp @@ -20,37 +20,148 @@ #include #include -#include #include +#include +#include #include #include // This header file is usually not available as public include! -#define TEST_RAW_PERF 1 -#define TEST_HDF5_PERF 1 +using namespace eCAL::experimental::measurement::base; -#define USE_OFSTREAM 0 -#define TEST_POST_SLEEP 2 // seconds +namespace +{ +struct TestingMeasEntry +{ + std::string channel_name = "topic"; + std::string data = "Hello World"; + long long snd_timestamp = 1001LL; + long long rcv_timestamp = 2001LL; + long long id = 1LL; + long long clock = 11LL; +}; + +bool MeasEntryEqualsEntryInfo(const TestingMeasEntry& meas_entry, const EntryInfo entry_info) +{ + return meas_entry.id == entry_info.SndID + && meas_entry.snd_timestamp == entry_info.SndTimestamp + && meas_entry.rcv_timestamp == entry_info.RcvTimestamp + && meas_entry.clock == entry_info.SndClock; +} -#define TEST_SIZE_1 1 -#define DATA_SET_SIZE_1 (1024) // 1 kB -#define DATA_SET_NUMBER_1 (64*1024) +bool WriteToHDF(eCAL::eh5::HDF5Meas& writer, const TestingMeasEntry& entry) +{ + return writer.AddEntryToFile( + entry.data.data(), // data + entry.data.size(), // data size + entry.snd_timestamp, // snd_timestamp + entry.rcv_timestamp, // rcv_timestamp + entry.channel_name, // channel name + entry.id, // id + entry.clock // clock + ); +} -#define TEST_SIZE_2 1 -#define DATA_SET_SIZE_2 (128*1024) // 128 kB -#define DATA_SET_NUMBER_2 (8*1024) -#define TEST_SIZE_3 1 -#define DATA_SET_SIZE_3 (1024*1024) // 1 MB -#define DATA_SET_NUMBER_3 (1024) +// Validates, that if all entries contained in TestingMeasEntry were added to a measurement, they are later contained in the read measurement. +void ValidateChannelsInMeasurement(eCAL::eh5::HDF5Meas& hdf5_reader, const std::vector& entries_in_measurement) +{ + std::set expected_channel_names; + std::transform(entries_in_measurement.begin(), entries_in_measurement.end(), std::inserter(expected_channel_names, expected_channel_names.begin()), + [](const TestingMeasEntry& entry) { return entry.channel_name; }); -#define TEST_SIZE_4 1 -#define DATA_SET_SIZE_4 (4096*1024) // 4 MB -#define DATA_SET_NUMBER_4 (256) + EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); +} -namespace +// Finds an EntryInfo corresponding to the TestingMeasEntry in the EntryInfoSet +// Return default EntryInfo if it cannot be Found +EntryInfo FindInSet(const eCAL::eh5::EntryInfoSet& info_set, const TestingMeasEntry& to_find) +{ + auto it = std::find_if(info_set.begin(), info_set.end(), [&to_find](const EntryInfo& entry) { return entry.SndClock == to_find.clock; }); + + if (it != info_set.end()) { + return *it; + } + else { + return EntryInfo{}; + } +} + +// Validate, that the TestingMeasEntry can be read from the measurement +void ValidateDataInMeasurement(eCAL::eh5::HDF5Meas& hdf5_reader, const TestingMeasEntry& entry) +{ + eCAL::eh5::EntryInfoSet entries_info_set; + EXPECT_TRUE(hdf5_reader.GetEntriesInfo(entry.channel_name, entries_info_set)); + + // Get Entry info from measurement and see that it checks out + EntryInfo info = FindInSet(entries_info_set, entry); + // entry was not in the dataset + if (info == EntryInfo{}) + { + FAIL() << "Entry info not found in set / measurement"; + return; + } + + EXPECT_TRUE(MeasEntryEqualsEntryInfo(entry, info)); + + size_t data_size; + EXPECT_TRUE(hdf5_reader.GetEntryDataSize(info.ID, data_size)); + + std::string data_read(data_size, ' '); + EXPECT_TRUE(hdf5_reader.GetEntryData(info.ID, const_cast(data_read.data()))); + + EXPECT_EQ(data_read, entry.data); +} + +TestingMeasEntry m1{ + "topic / with / slash", + "Hello World", + 1001LL, + 2001LL, + 1LL, + 11LL, +}; + +TestingMeasEntry m2{ + "another,topic", + "", + 1002LL, + 2002LL, + 2LL, + 12LL +}; + +TestingMeasEntry m3{ + " ASCII and beyond!\a\b\t\n\v\f\r\"#$%&\'()*+,-./0123456789:,<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~üöäÜÖÄâÂôÔûÛáàÁÀúÙ", + "o.O", + 1003LL, + 2003LL, + 3LL, + 13LL +}; + +TestingMeasEntry topic_1 +{ + "topic_1", + "topic1: test data", + 1001LL, + 2001LL, + 1LL, + 11LL +}; + +TestingMeasEntry topic_2 { + "topic_2", + "topic2: test data", + 1002LL, + 2002LL, + 2LL, + 12LL, +}; + + // properties std::string output_dir = "measurement_dir"; const size_t max_size_per_file = 500; // MB @@ -72,39 +183,21 @@ TEST(HDF5, EscapeUnescape) EXPECT_EQ(eCAL::eh5::GetUnescapedString(faulty_escaped_string), expected_unescaped_string); } + + TEST(HDF5, WriteReadIntegrity) { // Define data that will be written to the file - const std::string t1_name = "topic / with / slash"; - const std::string t1_data = "Hello World"; - const long long t1_snd_timestamp = 1001LL; - const long long t1_rcv_timestamp = 2001LL; - const long long t1_id = 1LL; - const long long t1_clock = 11LL; - - const std::string t2_name = "another,topic"; - const std::string t2_data = ""; - const long long t2_snd_timestamp = 1002LL; - const long long t2_rcv_timestamp = 2002LL; - const long long t2_id = 2LL; - const long long t2_clock = 12LL; - - const std::string t3_name = " ASCII and beyond!\a\b\t\n\v\f\r\"#$%&\'()*+,-./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~üöäÜÖÄâÂôÔûÛáàÁÀúÙ"; - const std::string t3_data = "o.O"; - const long long t3_snd_timestamp = 1003LL; - const long long t3_rcv_timestamp = 2003LL; - const long long t3_id = 3LL; - const long long t3_clock = 13LL; - std::string base_name = "integrity_meas"; std::string meas_root_dir = output_dir + "/" + base_name; + std::vector meas_entries{ m1, m2, m3 }; + // Write HDF5 file { eCAL::eh5::HDF5Meas hdf5_writer; - if (hdf5_writer.Open(meas_root_dir, eCAL::eh5::eAccessType::CREATE)) { hdf5_writer.SetFileBaseName(base_name); @@ -115,35 +208,10 @@ TEST(HDF5, WriteReadIntegrity) FAIL() << "Failed to open HDF5 Writer"; } - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t1_data.data(), // data - t1_data.size(), // data size - t1_snd_timestamp, // snd_timestamp - t1_rcv_timestamp, // rcv_timestamp - t1_name, // channel name - t1_id, // id - t1_clock // clock - )); - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t2_data.data(), // data - t2_data.size(), // data size - t2_snd_timestamp, // snd_timestamp - t2_rcv_timestamp, // rcv_timestamp - t2_name, // channel name - t2_id, // id - t2_clock // clock - )); - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t3_data.data(), // data - t3_data.size(), // data size - t3_snd_timestamp, // snd_timestamp - t3_rcv_timestamp, // rcv_timestamp - t3_name, // channel name - t3_id, // id - t3_clock // clock - )); + for (const auto& entry : meas_entries) + { + EXPECT_TRUE(WriteToHDF(hdf5_writer, entry)); + } EXPECT_TRUE(hdf5_writer.Close()); } @@ -151,112 +219,25 @@ TEST(HDF5, WriteReadIntegrity) // Read HDF5 file { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + ".hdf5")); - std::set expected_channel_names { t1_name, t2_name, t3_name }; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t1; - eCAL::eh5::EntryInfoSet entries_info_set_t2; - eCAL::eh5::EntryInfoSet entries_info_set_t3; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t3_name, entries_info_set_t3)); - - EXPECT_EQ(entries_info_set_t1.size(), 1); - EXPECT_EQ(entries_info_set_t2.size(), 1); - EXPECT_EQ(entries_info_set_t3.size(), 1); - - EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); - EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); - - EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); - EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); - - EXPECT_EQ(entries_info_set_t3.begin()->SndTimestamp, t3_snd_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->RcvTimestamp, t3_rcv_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->SndID, t3_id); - EXPECT_EQ(entries_info_set_t3.begin()->SndClock, t3_clock); - - size_t t1_data_size; - size_t t2_data_size; - size_t t3_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t3.begin()->ID, t3_data_size)); - - std::string t1_data_read(t1_data_size, ' '); - std::string t2_data_read(t2_data_size, ' '); - std::string t3_data_read(t3_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t3.begin()->ID, const_cast(t3_data_read.data()))); - - EXPECT_EQ(t1_data_read, t1_data); - EXPECT_EQ(t2_data_read, t2_data); - EXPECT_EQ(t3_data_read, t3_data); - } + ValidateChannelsInMeasurement(hdf5_reader, meas_entries); + for (const auto& entry : meas_entries) + { + ValidateDataInMeasurement(hdf5_reader, entry); + } + } // Read entries with HDF5 dir API { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(meas_root_dir)); - eCAL::eh5::EntryInfoSet entries_info_set_t1; - eCAL::eh5::EntryInfoSet entries_info_set_t2; - eCAL::eh5::EntryInfoSet entries_info_set_t3; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t3_name, entries_info_set_t3)); - - EXPECT_EQ(entries_info_set_t1.size(), 1); - EXPECT_EQ(entries_info_set_t2.size(), 1); - EXPECT_EQ(entries_info_set_t3.size(), 1); - - EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); - EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); - - EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); - EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); - - EXPECT_EQ(entries_info_set_t3.begin()->SndTimestamp, t3_snd_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->RcvTimestamp, t3_rcv_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->SndID, t3_id); - EXPECT_EQ(entries_info_set_t3.begin()->SndClock, t3_clock); - - size_t t1_data_size; - size_t t2_data_size; - size_t t3_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t3.begin()->ID, t3_data_size)); - - std::string t1_data_read(t1_data_size, ' '); - std::string t2_data_read(t2_data_size, ' '); - std::string t3_data_read(t3_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t3.begin()->ID, const_cast(t3_data_read.data()))); - - EXPECT_EQ(t1_data_read, t1_data); - EXPECT_EQ(t2_data_read, t2_data); - EXPECT_EQ(t3_data_read, t3_data); + ValidateChannelsInMeasurement(hdf5_reader, meas_entries); + for (const auto& entry : meas_entries) + { + ValidateDataInMeasurement(hdf5_reader, entry); + } } } @@ -269,124 +250,13 @@ TEST(HDF5, ReadWrite) writer.SetFileBaseName(file_name); writer.SetMaxSizePerFile(max_size_per_file); - data.resize(DATA_SET_SIZE_1); + data.resize(1024); EXPECT_TRUE(writer.AddEntryToFile(static_cast(data.data()), data.size(), 0, 0, "myChannel", 0, 0)); EXPECT_TRUE(writer.Close()); } -void MeasPerf(const std::string& file_name, const size_t pkg_size, const size_t pkg_num) -{ - data.resize(pkg_size); - const size_t write_loops(pkg_num); - -#if TEST_RAW_PERF - // Test raw performance - { - // start time - auto start = std::chrono::high_resolution_clock::now(); - -#if USE_OFSTREAM - - std::ofstream rfile(file_name + "_raw", std::ios::trunc | std::ios::out | std::ios::binary); - if (rfile) - { - for (size_t loops = 0; loops < write_loops; ++loops) - { - rfile.write(data.data(), data.size()); - } - } - rfile.close(); - -#else // USE_OFSTREAM - - FILE* pFile(nullptr); - std::string fname = file_name + "_raw"; - pFile = fopen(fname.c_str(), "wb"); - if (pFile) - { - for (size_t loops = 0; loops < write_loops; ++loops) - { - fwrite(data.data(), 1, data.size(), pFile); - } - fclose(pFile); - } - -#endif // USE_OFSTREAM - - // end time - auto finish = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed = finish - start; - size_t sum_data = data.size() * write_loops; - std::cout << std::endl; - std::cout << "Packages number : " << write_loops << std::endl; - std::cout << "Packages size : " << data.size()/1024 << " kB" << std::endl; - std::cout << "Sum payload : " << sum_data / (1024*1024) << " MB" << std::endl; - std::cout << "Throughput RAW : " << int((sum_data / (1024.0 * 1024.0)) / elapsed.count()) << " MB/s " << std::endl; - std::cout << std::endl; - - std::this_thread::sleep_for(std::chrono::seconds(TEST_POST_SLEEP)); - } -#endif - -#if TEST_HDF5_PERF - // Test HDF5 performance - { - // start time - auto start = std::chrono::high_resolution_clock::now(); - - eCAL::eh5::HDF5Meas writer(output_dir, eCAL::eh5::CREATE); - writer.SetFileBaseName(file_name + "_hdf5"); - writer.SetMaxSizePerFile(max_size_per_file); - for (size_t loop = 0; loop < write_loops; ++loop) - { - EXPECT_TRUE(writer.AddEntryToFile(static_cast(data.data()), data.size(), 0, 0, "myChannel", 0, loop)); - } - EXPECT_TRUE(writer.Close()); - - // end time - auto finish = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed = finish - start; - size_t sum_data = data.size() * write_loops; - std::cout << std::endl; - std::cout << "Packages number : " << write_loops << std::endl; - std::cout << "Packages size : " << data.size() / 1024 << " kB" << std::endl; - std::cout << "Sum payload : " << sum_data / (1024 * 1024) << " MB" << std::endl; - std::cout << "Throughput HDF5 : " << int((sum_data / (1024.0 * 1024.0)) / elapsed.count()) << " MB/s " << std::endl; - std::cout << std::endl; - - std::this_thread::sleep_for(std::chrono::seconds(TEST_POST_SLEEP)); - } -#endif -} - -#if TEST_SIZE_1 -TEST(HDF5, Performance_1kb) -{ - MeasPerf("meas_1_pkg", DATA_SET_SIZE_1, DATA_SET_NUMBER_1); -} -#endif // TEST_SIZE_1 - -#if TEST_SIZE_2 -TEST(HDF5, Performance_128kb) -{ - MeasPerf("meas_128_pkg", DATA_SET_SIZE_2, DATA_SET_NUMBER_2); -} -#endif // TEST_SIZE_2 - -#if TEST_SIZE_3 -TEST(HDF5, Performance_1024kb) -{ - MeasPerf("meas_1024_pkg", DATA_SET_SIZE_3, DATA_SET_NUMBER_3); -} -#endif // TEST_SIZE_3 -#if TEST_SIZE_4 -TEST(HDF5, Performance_4096kb) -{ - MeasPerf("meas_4096_pkg", DATA_SET_SIZE_4, DATA_SET_NUMBER_4); -} -#endif // TEST_SIZE_4 TEST(HDF5, IsOneFilePerChannelEnabled) { @@ -412,23 +282,8 @@ TEST(HDF5, IsOneFilePerChannelEnabled) TEST(HDF5, SetOneFilePerChannelEnabled) { - // Define data that will be written to the file - const std::string t1_name = "topic_1"; - const std::string t1_data = "topic1: test data"; - const long long t1_snd_timestamp = 1001LL; - const long long t1_rcv_timestamp = 2001LL; - const long long t1_id = 1LL; - const long long t1_clock = 11LL; - - const std::string t2_name = "topic_2"; - const std::string t2_data = "topic2: test data"; - const long long t2_snd_timestamp = 1002LL; - const long long t2_rcv_timestamp = 2002LL; - const long long t2_id = 2LL; - const long long t2_clock = 12LL; - std::string base_name = "output"; - + // Write HDF5 file { eCAL::eh5::HDF5Meas hdf5_writer; @@ -444,25 +299,8 @@ TEST(HDF5, SetOneFilePerChannelEnabled) FAIL() << "Failed to open HDF5 Writer"; } - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t1_data.data(), // data - t1_data.size(), // data size - t1_snd_timestamp, // snd_timestamp - t1_rcv_timestamp, // rcv_timestamp - t1_name, // channel name - t1_id, // id - t1_clock // clock - )); - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t2_data.data(), // data - t2_data.size(), // data size - t2_snd_timestamp, // snd_timestamp - t2_rcv_timestamp, // rcv_timestamp - t2_name, // channel name - t2_id, // id - t2_clock // clock - )); + EXPECT_TRUE(WriteToHDF(hdf5_writer, topic_1)); + EXPECT_TRUE(WriteToHDF(hdf5_writer, topic_2)); EXPECT_TRUE(hdf5_writer.IsOneFilePerChannelEnabled()); @@ -473,52 +311,19 @@ TEST(HDF5, SetOneFilePerChannelEnabled) { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + t1_name + ".hdf5")); - - std::set expected_channel_names {t1_name}; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t1; - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); - EXPECT_EQ(entries_info_set_t1.size(), 1); - - EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); - EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); - - size_t t1_data_size; - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); - - std::string t1_data_read(t1_data_size, ' '); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); - EXPECT_EQ(t1_data_read, t1_data); + EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + topic_1.channel_name + ".hdf5")); + ValidateChannelsInMeasurement(hdf5_reader, { topic_1 }); + ValidateDataInMeasurement(hdf5_reader, topic_1); } // Read HDF5 topic 2 file { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + t2_name + ".hdf5")); + EXPECT_TRUE(hdf5_reader.Open(output_dir + "/" + base_name + "_" + topic_2.channel_name + ".hdf5")); - std::set expected_channel_names {t2_name}; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t2; - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); - EXPECT_EQ(entries_info_set_t2.size(), 1); - - EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); - EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); - - size_t t2_data_size; - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); - - std::string t2_data_read(t2_data_size, ' '); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); - EXPECT_EQ(t2_data_read, t2_data); + ValidateChannelsInMeasurement(hdf5_reader, { topic_2 }); + ValidateDataInMeasurement(hdf5_reader, topic_2); } } @@ -526,32 +331,17 @@ TEST(HDF5, SetOneFilePerChannelEnabled) TEST(HDF5, EscapeFilenamesForOneFilePerChannel) { // Define data that will be written to the file + TestingMeasEntry normal_ascii = topic_1; + normal_ascii.channel_name = "NormalAscii"; + const std::string normal_ascii_escaped_filename = "NormalAscii"; - const std::string t1_name = "NormalAscii"; - const std::string t1_data = "Normal Ascii channel content"; - const long long t1_snd_timestamp = 1001LL; - const long long t1_rcv_timestamp = 2001LL; - const long long t1_id = 1LL; - const long long t1_clock = 11LL; - const std::string t1_escaped_file_name = "NormalAscii"; - - const std::string t2_name = "Escape these: \\/*\"<>|%"; - const std::string t2_data = "Data for channel with some escaped characters"; - const long long t2_snd_timestamp = 1002LL; - const long long t2_rcv_timestamp = 2002LL; - const long long t2_id = 2LL; - const long long t2_clock = 12LL; - const std::string t2_escaped_file_name = "Escape these%3A %5C%2F%2A%22%3C%3E%7C%25"; - - // IMPORTANT: This file must be UTF-8 encoded... - const std::string t3_name = "Don't escape these +~#,;.²³🍺"; - const std::string t3_data = "😐"; - const long long t3_snd_timestamp = 1002LL; - const long long t3_rcv_timestamp = 2002LL; - const long long t3_id = 2LL; - const long long t3_clock = 12LL; - const std::string t3_escaped_file_name = "Don't escape these +~#,;.²³\xF0\x9F\x8D\xBA"; + TestingMeasEntry escape_ascii = topic_1; + escape_ascii.channel_name = "Escape these: \\/*\"<>|%"; + const std::string escape_ascii_escaped_filename = "Escape these%3A %5C%2F%2A%22%3C%3E%7C%25"; + TestingMeasEntry escape_ascii_2 = topic_1; + escape_ascii_2.channel_name = "Don't escape these +~#,;.²³🍺"; + const std::string escape_ascii_2_escaped_filename = "Don't escape these +~#,;.²³\xF0\x9F\x8D\xBA"; std::string base_name = "escape_filename_meas"; std::string meas_root_dir = output_dir + "/" + base_name; @@ -560,7 +350,6 @@ TEST(HDF5, EscapeFilenamesForOneFilePerChannel) { eCAL::eh5::HDF5Meas hdf5_writer; - if (hdf5_writer.Open(meas_root_dir, eCAL::eh5::eAccessType::CREATE)) { hdf5_writer.SetFileBaseName(base_name); @@ -572,36 +361,9 @@ TEST(HDF5, EscapeFilenamesForOneFilePerChannel) FAIL() << "Failed to open HDF5 Writer"; } - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t1_data.data(), // data - t1_data.size(), // data size - t1_snd_timestamp, // snd_timestamp - t1_rcv_timestamp, // rcv_timestamp - t1_name, // channel name - t1_id, // id - t1_clock // clock - )); - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t2_data.data(), // data - t2_data.size(), // data size - t2_snd_timestamp, // snd_timestamp - t2_rcv_timestamp, // rcv_timestamp - t2_name, // channel name - t2_id, // id - t2_clock // clock - )); - - EXPECT_TRUE(hdf5_writer.AddEntryToFile( - t3_data.data(), // data - t3_data.size(), // data size - t3_snd_timestamp, // snd_timestamp - t3_rcv_timestamp, // rcv_timestamp - t3_name, // channel name - t3_id, // id - t3_clock // clock - )); + EXPECT_TRUE(WriteToHDF(hdf5_writer, normal_ascii)); + EXPECT_TRUE(WriteToHDF(hdf5_writer, escape_ascii)); + EXPECT_TRUE(WriteToHDF(hdf5_writer, escape_ascii_2)); EXPECT_TRUE(hdf5_writer.Close()); } @@ -610,93 +372,24 @@ TEST(HDF5, EscapeFilenamesForOneFilePerChannel) { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + t1_escaped_file_name + ".hdf5")); - - std::set expected_channel_names { t1_name}; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t1; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); - - EXPECT_EQ(entries_info_set_t1.size(), 1); - - EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); - EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); - - size_t t1_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); - - std::string t1_data_read(t1_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); - - EXPECT_EQ(t1_data_read, t1_data); + EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + normal_ascii_escaped_filename + ".hdf5")); + ValidateDataInMeasurement(hdf5_reader, normal_ascii); } // Read File 2 { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + t2_escaped_file_name + ".hdf5")); - - std::set expected_channel_names { t2_name}; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t2; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); - - EXPECT_EQ(entries_info_set_t2.size(), 1); - - EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); - EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); - - size_t t2_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); - - std::string t2_data_read(t2_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); - - EXPECT_EQ(t2_data_read, t2_data); + EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + escape_ascii_escaped_filename + ".hdf5")); + ValidateDataInMeasurement(hdf5_reader, escape_ascii); } // Read File 3 { eCAL::eh5::HDF5Meas hdf5_reader; - EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + t3_escaped_file_name + ".hdf5")); - - std::set expected_channel_names { t3_name}; - EXPECT_EQ(hdf5_reader.GetChannelNames(), expected_channel_names); - - eCAL::eh5::EntryInfoSet entries_info_set_t3; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t3_name, entries_info_set_t3)); - - EXPECT_EQ(entries_info_set_t3.size(), 1); - - EXPECT_EQ(entries_info_set_t3.begin()->SndTimestamp, t3_snd_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->RcvTimestamp, t3_rcv_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->SndID, t3_id); - EXPECT_EQ(entries_info_set_t3.begin()->SndClock, t3_clock); - - size_t t3_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t3.begin()->ID, t3_data_size)); - - std::string t3_data_read(t3_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t3.begin()->ID, const_cast(t3_data_read.data()))); - - EXPECT_EQ(t3_data_read, t3_data); + EXPECT_TRUE(hdf5_reader.Open(meas_root_dir + "/" + base_name + "_" + escape_ascii_2_escaped_filename + ".hdf5")); + ValidateDataInMeasurement(hdf5_reader, escape_ascii_2); } // Read entries with HDF5 dir API @@ -704,52 +397,8 @@ TEST(HDF5, EscapeFilenamesForOneFilePerChannel) eCAL::eh5::HDF5Meas hdf5_reader; EXPECT_TRUE(hdf5_reader.Open(meas_root_dir)); - - eCAL::eh5::EntryInfoSet entries_info_set_t1; - eCAL::eh5::EntryInfoSet entries_info_set_t2; - eCAL::eh5::EntryInfoSet entries_info_set_t3; - - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t1_name, entries_info_set_t1)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t2_name, entries_info_set_t2)); - EXPECT_TRUE(hdf5_reader.GetEntriesInfo(t3_name, entries_info_set_t3)); - - EXPECT_EQ(entries_info_set_t1.size(), 1); - EXPECT_EQ(entries_info_set_t2.size(), 1); - EXPECT_EQ(entries_info_set_t3.size(), 1); - - EXPECT_EQ(entries_info_set_t1.begin()->SndTimestamp, t1_snd_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->RcvTimestamp, t1_rcv_timestamp); - EXPECT_EQ(entries_info_set_t1.begin()->SndID, t1_id); - EXPECT_EQ(entries_info_set_t1.begin()->SndClock, t1_clock); - - EXPECT_EQ(entries_info_set_t2.begin()->SndTimestamp, t2_snd_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->RcvTimestamp, t2_rcv_timestamp); - EXPECT_EQ(entries_info_set_t2.begin()->SndID, t2_id); - EXPECT_EQ(entries_info_set_t2.begin()->SndClock, t2_clock); - - EXPECT_EQ(entries_info_set_t3.begin()->SndTimestamp, t3_snd_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->RcvTimestamp, t3_rcv_timestamp); - EXPECT_EQ(entries_info_set_t3.begin()->SndID, t3_id); - EXPECT_EQ(entries_info_set_t3.begin()->SndClock, t3_clock); - - size_t t1_data_size; - size_t t2_data_size; - size_t t3_data_size; - - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t1.begin()->ID, t1_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t2.begin()->ID, t2_data_size)); - EXPECT_TRUE(hdf5_reader.GetEntryDataSize(entries_info_set_t3.begin()->ID, t3_data_size)); - - std::string t1_data_read(t1_data_size, ' '); - std::string t2_data_read(t2_data_size, ' '); - std::string t3_data_read(t3_data_size, ' '); - - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t1.begin()->ID, const_cast(t1_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t2.begin()->ID, const_cast(t2_data_read.data()))); - EXPECT_TRUE(hdf5_reader.GetEntryData(entries_info_set_t3.begin()->ID, const_cast(t3_data_read.data()))); - - EXPECT_EQ(t1_data_read, t1_data); - EXPECT_EQ(t2_data_read, t2_data); - EXPECT_EQ(t3_data_read, t3_data); + ValidateDataInMeasurement(hdf5_reader, normal_ascii); + ValidateDataInMeasurement(hdf5_reader, escape_ascii); + ValidateDataInMeasurement(hdf5_reader, escape_ascii_2); } } \ No newline at end of file