From 4a7ed6f3899340698894c6851d1ccde2da26bcaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Martinez?= Date: Wed, 19 Jun 2019 15:06:28 +0200 Subject: [PATCH] Error Correction Code --- Project/GNU/CLI/Makefile.am | 6 +- Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj | 14 +- .../Lib/RAWcooked_Lib.vcxproj.filters | 24 + Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj | 14 +- .../Lib/RAWcooked_Lib.vcxproj.filters | 24 + Source/CLI/Global.cpp | 42 ++ Source/CLI/Global.h | 2 + Source/CLI/Help.cpp | 7 + Source/CLI/Input.cpp | 4 +- Source/CLI/Input.h | 2 +- Source/CLI/Main.cpp | 24 +- Source/Lib/FileIO.cpp | 20 +- Source/Lib/FileIO.h | 6 +- Source/Lib/HashSum/HashSum.cpp | 2 +- Source/Lib/Input_Base.cpp | 18 + Source/Lib/Input_Base.h | 3 + Source/Lib/Matroska/Matroska_Common.cpp | 605 +++++++++++++++++- Source/Lib/Matroska/Matroska_Common.h | 71 ++ Source/Lib/RAWcooked/RAWcooked.cpp | 320 ++++++++- Source/Lib/RAWcooked/RAWcooked.h | 14 +- Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp | 195 ++++++ Source/Lib/ThirdParty/Reed-Solomon/Galois.h | 58 ++ Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp | 186 ++++++ Source/Lib/ThirdParty/Reed-Solomon/Matrix.h | 86 +++ .../ThirdParty/Reed-Solomon/Reed-Solomon.cpp | 502 +++++++++++++++ .../ThirdParty/Reed-Solomon/Reed-Solomon.h | 129 ++++ 26 files changed, 2309 insertions(+), 69 deletions(-) create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Galois.h create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Matrix.h create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp create mode 100644 Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h diff --git a/Project/GNU/CLI/Makefile.am b/Project/GNU/CLI/Makefile.am index cb11835b..0f9bfe1b 100644 --- a/Project/GNU/CLI/Makefile.am +++ b/Project/GNU/CLI/Makefile.am @@ -27,6 +27,9 @@ rawcooked_SOURCES = \ ../../../Source/Lib/RawFrame/RawFrame.cpp \ ../../../Source/Lib/TIFF/TIFF.cpp \ ../../../Source/Lib/WAV/WAV.cpp \ + ../../../Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp \ + ../../../Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp \ + ../../../Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp \ ../../../Source/Lib/ThirdParty/flac/src/libFLAC/bitmath.c \ ../../../Source/Lib/ThirdParty/flac/src/libFLAC/bitreader.c \ ../../../Source/Lib/ThirdParty/flac/src/libFLAC/cpu.c \ @@ -59,7 +62,8 @@ AM_CPPFLAGS = -I../../../Source \ -I../../../Source/Lib/ThirdParty/flac/src/libFLAC/include \ -I../../../Source/Lib/ThirdParty/md5 \ -I../../../Source/Lib/ThirdParty/thread-pool/include\ - -I../../../Source/Lib/ThirdParty/zlib + -I../../../Source/Lib/ThirdParty/zlib \ + -I../../../Source/Lib/ThirdParty AM_CXXFLAGS = -std=c++0x diff --git a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj index 9036455b..f44cccbc 100644 --- a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj +++ b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj @@ -42,6 +42,9 @@ + + + @@ -105,6 +108,9 @@ + + + @@ -214,7 +220,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true false MultiThreadedDebugDLL @@ -230,7 +236,7 @@ Level3 Disabled _DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true false MultiThreadedDebugDLL @@ -248,7 +254,7 @@ true true WIN32;NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true MultiThreadedDLL @@ -267,7 +273,7 @@ true true NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true MultiThreadedDLL AnySuitable diff --git a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters index 208fe363..a40dafd4 100644 --- a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters +++ b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters @@ -103,6 +103,12 @@ {889f93d5-50da-4606-8480-a1ec82aa200f} + + {bea62b93-d3d5-4cf1-a957-65810ce3ac52} + + + {e97f8d7f-a3f3-4d1b-87c3-c17fcf86decf} + @@ -303,6 +309,15 @@ Header Files\HashSum + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + @@ -440,6 +455,15 @@ Source Files\HashSum + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + diff --git a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj index 7012af7d..5c489a4e 100644 --- a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj +++ b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj @@ -42,6 +42,9 @@ + + + @@ -105,6 +108,9 @@ + + + @@ -214,7 +220,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true false MultiThreadedDebugDLL @@ -230,7 +236,7 @@ Level3 Disabled _DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true false MultiThreadedDebugDLL @@ -248,7 +254,7 @@ true true WIN32;NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true MultiThreadedDLL @@ -267,7 +273,7 @@ true true NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions) - ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib + ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty true MultiThreadedDLL AnySuitable diff --git a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters index 208fe363..a40dafd4 100644 --- a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters +++ b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters @@ -103,6 +103,12 @@ {889f93d5-50da-4606-8480-a1ec82aa200f} + + {bea62b93-d3d5-4cf1-a957-65810ce3ac52} + + + {e97f8d7f-a3f3-4d1b-87c3-c17fcf86decf} + @@ -303,6 +309,15 @@ Header Files\HashSum + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + @@ -440,6 +455,15 @@ Source Files\HashSum + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + + + ThirdParty\Reed-Solomon + diff --git a/Source/CLI/Global.cpp b/Source/CLI/Global.cpp index 7672615f..05a6bc8e 100644 --- a/Source/CLI/Global.cpp +++ b/Source/CLI/Global.cpp @@ -127,6 +127,20 @@ int global::SetHash(bool Value) return 0; } +//--------------------------------------------------------------------------- +int global::SetEcc(bool Value) +{ + Actions.set(Action_Ecc, Value); + return 0; +} + +//--------------------------------------------------------------------------- +int global::SetFix(bool Value) +{ + Actions.set(Action_Fix, Value); + return 0; +} + //--------------------------------------------------------------------------- int global::SetAll(bool Value) { @@ -142,6 +156,10 @@ int global::SetAll(bool Value) return ReturnValue; if (int ReturnValue = SetHash(Value)) return ReturnValue; + if (int ReturnValue = SetEcc(Value)) + return ReturnValue; + if (int ReturnValue = SetFix(Value)) + return ReturnValue; return 0; } @@ -441,12 +459,24 @@ int global::ManageCommandLine(const char* argv[], int argc) return Value; License.Feature(Feature_GeneralOptions); } + else if (strcmp(argv[i], "--ecc") == 0) + { + int Value = SetEcc(true); + if (Value) + return Value; + } else if (strcmp(argv[i], "--encode") == 0) { int Value = SetEncode(true); if (Value) return Value; } + else if (strcmp(argv[i], "--fix") == 0) + { + int Value = SetFix(true); + if (Value) + return Value; + } else if (strcmp(argv[i], "--hash") == 0) { int Value = SetHash(true); @@ -497,12 +527,24 @@ int global::ManageCommandLine(const char* argv[], int argc) if (Value) return Value; } + else if (strcmp(argv[i], "--no-ecc") == 0) + { + int Value = SetEcc(false); + if (Value) + return Value; + } else if (strcmp(argv[i], "--no-encode") == 0) { int Value = SetEncode(false); if (Value) return Value; } + else if (strcmp(argv[i], "--no-fix") == 0) + { + int Value = SetFix(false); + if (Value) + return Value; + } else if (strcmp(argv[i], "--no-hash") == 0) { int Value = SetHash(false); diff --git a/Source/CLI/Global.h b/Source/CLI/Global.h index 1dcabcfe..7ea3a182 100644 --- a/Source/CLI/Global.h +++ b/Source/CLI/Global.h @@ -86,6 +86,8 @@ class global int SetConch(bool Value); int SetEncode(bool Value); int SetHash(bool Value); + int SetEcc(bool Value); + int SetFix(bool Value); int SetAll(bool Value); // Progress indicator diff --git a/Source/CLI/Help.cpp b/Source/CLI/Help.cpp index 8255674b..20538bac 100644 --- a/Source/CLI/Help.cpp +++ b/Source/CLI/Help.cpp @@ -143,6 +143,13 @@ ReturnValue Help(const char* Name) cout << " Don't do conformance check (see above)." << endl; cout << " Is default (it may change in the future)" << endl; cout << endl; + cout << " --ecc" << endl; + cout << " Add Error Correction Codes to the resulting Matroska file." << endl; + cout << " --no-ecc" << endl; + cout << " Don't add Error Correction Codes to the resulting Matroska file" << endl; + cout << " (see above)." << endl; + cout << " Is default (it may change in the future)" << endl; + cout << endl; cout << " --encode" << endl; cout << " Encode audio-visual RAW data into a compressed stream." << endl; cout << " Is default" << endl; diff --git a/Source/CLI/Input.cpp b/Source/CLI/Input.cpp index 0628f611..052c70ed 100644 --- a/Source/CLI/Input.cpp +++ b/Source/CLI/Input.cpp @@ -398,9 +398,9 @@ void input::CheckDurations(vector const& Durations, vector const } //--------------------------------------------------------------------------- -bool input::OpenInput(filemap& FileMap, const string& Name, errors* Errors) +bool input::OpenInput(filemap& FileMap, const string& Name, bool Write, errors* Errors) { - if (FileMap.Open_ReadMode(Name)) + if (FileMap.Open(Name, Write)) { if (Errors) { diff --git a/Source/CLI/Input.h b/Source/CLI/Input.h index 1b9a755e..8206ec18 100644 --- a/Source/CLI/Input.h +++ b/Source/CLI/Input.h @@ -30,7 +30,7 @@ class input static void CheckDurations(vector const& Durations, vector const& Durations_FileName, errors* Errors = nullptr); // I/O - static bool OpenInput(filemap& FileMap, const string& Name, errors* Errors); + static bool OpenInput(filemap& FileMap, const string& Name, bool Write, errors* Errors); }; #endif diff --git a/Source/CLI/Main.cpp b/Source/CLI/Main.cpp index 678d14e5..7a9ba4ca 100644 --- a/Source/CLI/Main.cpp +++ b/Source/CLI/Main.cpp @@ -67,6 +67,7 @@ user_mode Ask_Callback(user_mode* Mode, const string& FileName, const string& Ex struct parse_info { string* Name; + string Name2; // TODO: simplify code related to file names filemap FileMap; vector RemovedFiles; string FileName_Template; @@ -144,7 +145,7 @@ bool parse_info::ParseFile_Input(input_base_uncompressed& SingleFile, input& Inp for (size_t i = 1; i < SingleFile.InputInfo->FrameCount; i++) { Name = &RemovedFiles[i]; - if (input::OpenInput(FileMap, *Name, &Global.Errors)) + if (input::OpenInput(FileMap, *Name, Global.Actions[Action_Fix], &Global.Errors)) return true; RAWcooked.OutputFileName = Name->substr(Global.Path_Pos_Global); FormatPath(RAWcooked.OutputFileName); @@ -321,6 +322,13 @@ int ParseFile_Compressed(parse_info& ParseInfo) { ReturnValue = 1; } + if (ParseInfo.IsDetected && Global.Actions[Action_Ecc]) + { + ParseInfo.FileMap.Close(); + const string* Name = ParseInfo.Name2.empty() ? ParseInfo.Name : &ParseInfo.Name2 ; + if (Name) + M.Erasure_Write(Name->c_str()); + } } // End @@ -345,7 +353,7 @@ int ParseFile(size_t Files_Pos) ParseInfo.Name = &Input.Files[Files_Pos]; // Open file - if (input::OpenInput(ParseInfo.FileMap, *ParseInfo.Name, &Global.Errors)) + if (input::OpenInput(ParseInfo.FileMap, *ParseInfo.Name, Global.Actions[Action_Fix], &Global.Errors)) return 1; // Compressed content @@ -424,7 +432,7 @@ int main(int argc, const char* argv[]) case AlwaysYes: break; default: if ((!Value && Global.Actions[Action_Encode] && !Output.Streams.empty()) - || (Global.Check && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty())) + || ((Global.Check || Global.Actions[Action_Ecc]) && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty())) { cerr << "Do you want to continue despite warnings ? [y/N] "; string Result; @@ -435,23 +443,25 @@ int main(int argc, const char* argv[]) } } - // FFmpeg + // FFmpeg if (!Value && Global.Actions[Action_Encode]) - Value = Output.Process(Global); + Value = Output.Process (Global); // RAWcooked file if (!Global.DisplayCommand) RAWcooked.Delete(); // Check result - if (Global.Check && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty()) + if ((Global.Check || Global.Actions[Action_Ecc]) && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty()) { parse_info ParseInfo; - Value = ParseInfo.FileMap.Open_ReadMode(Global.OutputFileName); + Value = ParseInfo.FileMap.Open(Global.OutputFileName, Global.Actions[Action_Fix]); if (!Value) { // Configure for a 2nd pass ParseInfo.Name = NULL; + ParseInfo.Name2 = Global.OutputFileName; + Global.Check = true; Global.OutputFileName = Global.Inputs[0]; if (!Global.Actions[Action_Hash]) Global.OutputFileName_IsProvided = true; diff --git a/Source/Lib/FileIO.cpp b/Source/Lib/FileIO.cpp index c9aad2ec..8513829f 100644 --- a/Source/Lib/FileIO.cpp +++ b/Source/Lib/FileIO.cpp @@ -5,7 +5,9 @@ */ //--------------------------------------------------------------------------- -#define _GNU_SOURCE // Needed for ftruncate on GNU compiler +#if !defined(_GNU_SOURCE) + #define _GNU_SOURCE // Needed for ftruncate on GNU compiler +#endif #include "Lib/FileIO.h" #include #include @@ -27,7 +29,7 @@ //--------------------------------------------------------------------------- //--------------------------------------------------------------------------- -int filemap::Open_ReadMode(const char* FileName) +int filemap::Open(const char* FileName, bool Write) { Close(); @@ -35,7 +37,7 @@ int filemap::Open_ReadMode(const char* FileName) #if defined(_WIN32) || defined(_WINDOWS) HANDLE& File = (HANDLE&)Private; - File = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + File = CreateFileA(FileName, Write ? (GENERIC_READ | GENERIC_WRITE) : (GENERIC_READ), FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (File != INVALID_HANDLE_VALUE) { DWORD FileSizeHigh; @@ -46,7 +48,7 @@ int filemap::Open_ReadMode(const char* FileName) if (Buffer_Size) { HANDLE& Mapping = (HANDLE&)Private2; - Mapping = CreateFileMapping(File, 0, PAGE_READONLY, 0, 0, 0); + Mapping = CreateFileMapping(File, 0, Write ? PAGE_READWRITE : PAGE_READONLY , 0, 0, 0); if (Mapping) FileIsOpen = true; else @@ -68,7 +70,7 @@ int filemap::Open_ReadMode(const char* FileName) FileIsOpen = false; #else int& P = (int&)Private; - P = open(FileName, O_RDONLY, 0); + P = open(FileName, Write ? O_RDWR : O_RDONLY, 0); if (P != -1) { struct stat Fstat; @@ -89,7 +91,7 @@ int filemap::Open_ReadMode(const char* FileName) #endif if (FileIsOpen) - FileIsOpen = Remap() ? false : true; + FileIsOpen = Remap(Write) ? false : true; if (!FileIsOpen) return 1; @@ -98,7 +100,7 @@ int filemap::Open_ReadMode(const char* FileName) } //--------------------------------------------------------------------------- -int filemap::Remap() +int filemap::Remap(bool Write) { if (Buffer) { @@ -115,7 +117,7 @@ int filemap::Remap() #if defined(_WIN32) || defined(_WINDOWS) HANDLE& Mapping = (HANDLE&)Private2; - Buffer = (unsigned char*)MapViewOfFile(Mapping, FILE_MAP_READ, 0, 0, 0); + Buffer = (unsigned char*)MapViewOfFile(Mapping, Write ? (FILE_MAP_READ | FILE_MAP_WRITE) : (FILE_MAP_READ), 0, 0, 0); if (!Buffer) { CloseHandle(Mapping); @@ -128,7 +130,7 @@ int filemap::Remap() } #else int& P = (int&)Private; - Buffer = (unsigned char*)mmap(NULL, Buffer_Size, PROT_READ, MAP_FILE | MAP_PRIVATE, P, 0); + Buffer = (unsigned char*)mmap(NULL, Buffer_Size, Write ? (PROT_READ | PROT_WRITE) : (PROT_READ), MAP_SHARED, P, 0); if (Buffer == MAP_FAILED) { Buffer = NULL; diff --git a/Source/Lib/FileIO.h b/Source/Lib/FileIO.h index 82649b56..ee9aa8bb 100644 --- a/Source/Lib/FileIO.h +++ b/Source/Lib/FileIO.h @@ -28,10 +28,10 @@ class filemap size_t Buffer_Size = 0; // Actions - int Open_ReadMode(const char* FileName); - int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); } + int Open(const char* FileName, bool Write = false); + int Open(const string& FileName, bool Write = false) { return Open(FileName.c_str(), Write); } bool IsOpen() { return Private == (void*)-1 ? false : true; } - int Remap(); + int Remap(bool Write = false); int Close(); private: diff --git a/Source/Lib/HashSum/HashSum.cpp b/Source/Lib/HashSum/HashSum.cpp index 8301a6bc..fe3c290f 100644 --- a/Source/Lib/HashSum/HashSum.cpp +++ b/Source/Lib/HashSum/HashSum.cpp @@ -60,8 +60,8 @@ const char** ErrorTexts[] = { undecodable::MessageText, nullptr, - invalid::MessageText, nullptr, + invalid::MessageText, }; static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), IncoherencyMessage); diff --git a/Source/Lib/Input_Base.cpp b/Source/Lib/Input_Base.cpp index c0b4b525..4e3c95d5 100644 --- a/Source/Lib/Input_Base.cpp +++ b/Source/Lib/Input_Base.cpp @@ -232,6 +232,24 @@ uint64_t input_base::Get_EB() return ToReturn; } + +//--------------------------------------------------------------------------- +int64_t input_base::Get_BXs(size_t Size) +{ + TEST_BUFFEROVERFLOW(Size); + + int64_t ToReturn = 0; + int64_t Mask = ((int64_t)-1) << (Size * 8); + while (Size) + { + ToReturn = (ToReturn << 8) | Buffer[Buffer_Offset]; + Size--; + Buffer_Offset++; + } + + return ToReturn | Mask; +} + //--------------------------------------------------------------------------- void input_base::Error(error::type Type, error::generic::code Code) { diff --git a/Source/Lib/Input_Base.h b/Source/Lib/Input_Base.h index 9ad47a29..43677a15 100644 --- a/Source/Lib/Input_Base.h +++ b/Source/Lib/Input_Base.h @@ -29,6 +29,8 @@ enum action : uint8_t Action_Conch, Action_CheckPadding, Action_AcceptTruncated, + Action_Ecc, + Action_Fix, Action_Max }; @@ -101,6 +103,7 @@ class input_base uint64_t Get_B8(); long double Get_BF10(); uint64_t Get_EB(); + int64_t Get_BXs(size_t Size); // Error message void Undecodable(error::undecodable::code Code) { Error(error::Undecodable, (error::generic::code)Code); } diff --git a/Source/Lib/Matroska/Matroska_Common.cpp b/Source/Lib/Matroska/Matroska_Common.cpp index 9ed1cadc..9f125ff1 100644 --- a/Source/Lib/Matroska/Matroska_Common.cpp +++ b/Source/Lib/Matroska/Matroska_Common.cpp @@ -14,6 +14,7 @@ #include "Lib/RawFrame/RawFrame.h" #include "Lib/Config.h" #include "Lib/FileIO.h" +#include "Reed-Solomon/Reed-Solomon.h" #include #include #include @@ -93,25 +94,47 @@ static const char* MessageText[] = { "can not open file for reading", "files are not same", + "input file, it is corrupted, can not correct so many errors", }; enum code : uint8_t { FileOpen, FileComparison, + Erasure_TooManyErrors, + Max +}; + +static_assert(Max == sizeof(MessageText) / sizeof(const char*), IncoherencyMessage); + +} // undecodable + +namespace invalid +{ + +static const char* MessageText[] = +{ + "input file, it is corrupted, use --fix for fixing it", + "input file, it is corrupted (parity part), use --fix for rebuilding", +}; + +enum code : uint8_t +{ + Erasure_DataCorrupted, + Erasure_ParityCorrupted, Max }; namespace undecodable { static_assert(Max == sizeof(MessageText) / sizeof(const char*), IncoherencyMessage); } -} // unparsable +} // invalid const char** ErrorTexts[] = { undecodable::MessageText, nullptr, nullptr, - nullptr, + invalid::MessageText, }; static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), IncoherencyMessage); @@ -161,7 +184,7 @@ void frame_writer::FrameCall(raw_frame* RawFrame, const string& OutputFileName) if (!Mode[NoOutputCheck] && !File_Write.IsOpen()) { // File already exists, we want to check it - if (File_Read.Open_ReadMode(BaseDirectory + OutputFileName)) + if (File_Read.Open(BaseDirectory + OutputFileName)) { Offset = (size_t)-1; SizeOnDisk = (size_t)-1; @@ -655,6 +678,7 @@ matroska::call matroska::SubElements_##_VALUE(uint64_t Name) \ ELEMENT_BEGIN(_) ELEMENT_CASE( 8538067, Segment) +ELEMENT_CASE( 7273, Rawcooked_Segment) ELEMENT_END() ELEMENT_BEGIN(Segment) @@ -737,6 +761,103 @@ ELEMENT_VOID( 30, Segment_Tracks_TrackEntry_Video_PixelWidth) ELEMENT_VOID( 3A, Segment_Tracks_TrackEntry_Video_PixelHeight) ELEMENT_END() +ELEMENT_BEGIN(Rawcooked_Segment) +ELEMENT_VOID( 70, Rawcooked_Segment_LibraryName) +ELEMENT_VOID( 71, Rawcooked_Segment_LibraryVersion) +ELEMENT_CASE( 726365, Rawcooked_Segment_Erasure) +ELEMENT_END() + +ELEMENT_BEGIN(Rawcooked_Segment_Erasure) +ELEMENT_VOID( 726368, Rawcooked_Segment_Erasure_ShardHashes) +ELEMENT_VOID( 726369, Rawcooked_Segment_Erasure_ShardInfo) +ELEMENT_VOID( 72636C, Rawcooked_Segment_Erasure_ParityShardsLocation) +ELEMENT_VOID( 726370, Rawcooked_Segment_Erasure_ParityShards) +ELEMENT_VOID( 726373, Rawcooked_Segment_Erasure_EbmlStartLocation) +ELEMENT_END() + +//--------------------------------------------------------------------------- +// CRC_32_Table (Little Endian bitstream, ) +// The CRC in use is the IEEE-CRC-32 algorithm as used in the ISO 3309 standard and in section 8.1.1.6.2 of ITU-T recommendation V.42, with initial value of 0xFFFFFFFF. The CRC value MUST be computed on a little endian bitstream and MUST use little endian storage. +// A CRC is computed like this: +// Init: uint32_t CRC32 ^= 0; +// for each data byte do +// CRC32=(CRC32>>8) ^ Mk_CRC32_Table[(CRC32&0xFF)^*Buffer_Current++]; +// End: CRC32 ^= 0; +static const uint32_t Matroska_CRC32_Table[256] = +{ + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02Ef8D, +}; + +//--------------------------------------------------------------------------- +void Matroska_CRC32_Compute(uint32_t& CRC32, const uint8_t* Buffer_Current, const uint8_t* Buffer_End) +{ + while (Buffer_Current < Buffer_End) + CRC32 = (CRC32 >> 8) ^ Matroska_CRC32_Table[(CRC32 & 0xFF) ^ *Buffer_Current++]; +} + //--------------------------------------------------------------------------- // Glue void matroska_ProgressIndicator_Show(matroska* M) @@ -818,6 +939,305 @@ matroska::call matroska::SubElements_Void(uint64_t Name) Levels[Level].SubElements = &matroska::SubElements_Void; return &matroska::Void; } +//--------------------------------------------------------------------------- +matroska::erasure_encode::erasure_encode(erasure_info& Info_) +{ + Info = Info_; + + size_t DataHashes_Count = (Info.erasureLength + Info.shardSize - 1) / (Info.shardSize); + size_t ParityHashes_Count = (DataHashes_Count + Info.dataShardCount - 1) / Info.dataShardCount; + ParityHashes_Count *= Info.parityShardCount; + + RS = new ReedSolomon(Info.dataShardCount, Info.parityShardCount); + HashValues = new rawcooked::hash_value[DataHashes_Count + ParityHashes_Count]; + ParityShards = new uint8_t[ParityHashes_Count * Info.shardSize]; +} + +//--------------------------------------------------------------------------- +size_t matroska::erasure_encode::Encode(uint8_t* Buffer, size_t Buffer_Max, size_t Offset) +{ + // Init + uint8_t* Shards[256]; + size_t Package_Pos = Offset / Info.shardSize / Info.dataShardCount; + size_t Hash_Pos = Package_Pos * (Info.dataShardCount + Info.parityShardCount); + size_t Parity_Pos = Package_Pos * Info.parityShardCount; + + // Data shards init + for (uint8_t i = 0; i < Info.dataShardCount; i++) + { + size_t Size = Buffer_Max - Offset; + if (Size > Info.shardSize) + Size = Info.shardSize; + + if (Size == Info.shardSize) + { + Shards[i] = Buffer + Offset; + } + else + { + Shards[i] = new uint8_t[Info.shardSize]; + memcpy(Shards[i], Buffer + Offset, Size); + memset(Shards[i] + Size, 0x00, Info.shardSize - Size); + } + + if (Size) + { + HashValues[Hash_Pos++] = Compute_MD5(Buffer + Offset, Size); + Offset += Size; + } + } + + // Parity shards init + auto ParityShards_Begin = ParityShards + Parity_Pos * Info.shardSize; + auto ParityShards_Temp = ParityShards_Begin; + for (uint8_t i = 0; i < Info.parityShardCount; i++) + { + Shards[Info.dataShardCount + i] = ParityShards_Temp; + ParityShards_Temp += Info.shardSize; + } + memset(ParityShards_Begin, 0x00, ParityShards_Temp - ParityShards_Begin); + + // Process + RS->encode(Shards, Info.shardSize); + + // Store hashes + for (uint8_t i = 0; i < Info.parityShardCount; i++) + { + HashValues[Hash_Pos++] = Compute_MD5(Shards[Info.dataShardCount + i], Info.shardSize); + } + + return Offset; +} + +//--------------------------------------------------------------------------- +bool matroska::erasure_check::Init() +{ + DataHashes_Count = (Info.erasureLength + Info.shardSize - 1) / (Info.shardSize); + ParityHashes_Count = (DataHashes_Count + Info.dataShardCount - 1) / Info.dataShardCount; + ParityHashes_Count *= Info.parityShardCount; + if (16 * (DataHashes_Count + ParityHashes_Count) != HashValues_Size) + { + return true; + } + + return false; +} + +//--------------------------------------------------------------------------- +size_t matroska::erasure_check::Check(uint8_t* Buffer, size_t Buffer_Max, size_t Offset) +{ + // Coherency + size_t isNOK = 0; + if (HashValues_Size != 16 * (DataHashes_Count + ParityHashes_Count)) + { + isNOK = 1; + } + if (ParityShards_Size != Info.shardSize * ParityHashes_Count) + { + if (ParityShards_Size || !Info.shardSize || !ParityHashes_Count) + isNOK = 1; + else + ParityShards_Size = Info.shardSize * ParityHashes_Count; + } + if (isNOK) + return Offset; // TEMP TODO: Issues + + // Init + size_t Offset_Base = Offset; + size_t Package_Pos = Offset / Info.shardSize / Info.dataShardCount; + size_t Hash_Pos = Package_Pos * (Info.dataShardCount + Info.parityShardCount); + size_t Parity_Pos = Package_Pos * Info.parityShardCount; + + // Compute hash of data shards + std::vector shardPresent; + for (uint8_t i = 0; i < Info.dataShardCount; i++) + { + size_t Size = Buffer_Max - Offset; + if (Size > Info.shardSize) + Size = Info.shardSize; + if (!Size) + break; + + shardPresent.push_back(memcmp(HashValues[Hash_Pos + i].Values, Compute_MD5(Buffer + Offset, Size).Values, 16) ? false : true); + if (!shardPresent.back()) + isNOK++; + + Offset += Size; + } + + //if (!isNOK) + // return Offset; // All is fine + + // Compute hash of parity shards + size_t dataShardTempCount = shardPresent.size(); + shardPresent.resize(248, true); + for (uint8_t i = 0; i < Info.parityShardCount; i++) + { + shardPresent.push_back(memcmp(HashValues[Hash_Pos + dataShardTempCount + i].Values, Compute_MD5(ParityShards + (Parity_Pos + i) * Info.shardSize, Info.shardSize).Values, 16) ? false : true); + if (!shardPresent.back()) + isNOK++; + } + + if (isNOK) + { + if (isNOK > Info.parityShardCount) + { + if (Errors) + { + std::stringstream in, out; + in << std::hex << Offset_Base; + out << std::hex << Offset_Base + Info.dataShardCount * Info.shardSize - 1; + Errors->Error(IO_FileChecker, error::Undecodable, (error::generic::code)filechecker_issue::undecodable::Erasure_TooManyErrors, "at 0x" + in.str() + "-0x" + out.str()); + } + + return (uint64_t)-1; + } + + uint8_t** Shards = new uint8_t* [Info.dataShardCount + Info.parityShardCount]; + Offset = Offset_Base; + + for (uint8_t i = 0; i < Info.dataShardCount; i++) + { + size_t Size = Buffer_Max - Offset; + if (Size > Info.shardSize) + Size = Info.shardSize; + + if (Size == Info.shardSize) + { + Shards[i] = Buffer + Offset; + } + else + { + Shards[i] = new uint8_t[Info.shardSize]; + memcpy(Shards[i], Buffer + Offset, Size); + memset(Shards[i] + Size, 0x00, Info.shardSize - Size); + } + + Offset += Size; + } + + for (uint8_t i = 0; i < Info.parityShardCount; i++) + { + Shards[Info.dataShardCount + i] = ParityShards + (Parity_Pos + i) * Info.shardSize ; + } + + if (Write) + { + ReedSolomon RS(Info.dataShardCount, Info.parityShardCount); + if (!RS.repair(Shards, Info.shardSize, shardPresent)) + { + if (Errors) + { + std::stringstream in, out; + in << std::hex << Offset_Base; + out << std::hex << Offset_Base + Info.dataShardCount * Info.shardSize - 1; + Errors->Error(IO_FileChecker, error::Undecodable, (error::generic::code)filechecker_issue::undecodable::Erasure_TooManyErrors, "at 0x" + in.str() + "-0x" + out.str()); + } + + return (uint64_t)-1; + } + } + for (size_t i = 0; i < shardPresent.size(); i++) + if (!shardPresent[i]) + { + if (i < Info.dataShardCount) // Data shard + { + size_t Write_Offset = Offset_Base + i * Info.shardSize; + size_t Write_Size = Buffer_Max - Write_Offset; + if (Write_Size > Info.shardSize) + Write_Size = Info.shardSize; + + if (Write) + { + memcpy(Buffer + Write_Offset, Shards[i], Write_Size); + } + else + { + if (Errors) + { + std::stringstream in, out; + in << std::hex << Write_Offset; + out << std::hex << Write_Offset + Write_Size - 1; + Errors->Error(IO_FileChecker, error::Invalid, (error::generic::code)filechecker_issue::invalid::Erasure_DataCorrupted, "at 0x" + in.str() + "-0x" + out.str()); + } + } + } + else // Parity shard + { + size_t Offset_Temp = (Parity_Pos + i - Info.dataShardCount) * Info.shardSize; + + if (Write) + { + memcpy(ParityShards + Offset_Temp, Shards[i], Info.shardSize); + } + else + { + if (Errors) + { + std::stringstream in, out; + in << std::hex << ParityShards - Buffer + Offset_Temp; + out << std::hex << ParityShards - Buffer + Offset_Temp + Info.shardSize - 1; + Errors->Error(IO_FileChecker, error::Invalid, (error::generic::code)filechecker_issue::invalid::Erasure_ParityCorrupted, "at 0x" + in.str() + "-0x" + out.str()); + } + } + } + } + + if (!Write) + return (uint64_t)-1; + } + + return Offset; +} + +//--------------------------------------------------------------------------- +rawcooked::hash_value matroska::erasure::Compute_MD5(uint8_t* Buffer, size_t Buffer_Size) +{ + MD5_CTX Hash; + MD5_Init(&Hash); + MD5_Update(&Hash, Buffer, (unsigned long)Buffer_Size); + rawcooked::hash_value HashValue; + MD5_Final(HashValue.Values, &Hash); + return HashValue; +} + +//--------------------------------------------------------------------------- +void matroska::Erasure_Init() +{ + if (Buffer_Size_Matroska) + return; // Already donc + Buffer_Size_Matroska = Levels[Level].Offset_End; + + // Erasure encode? + if (!Erasure_Encode && Actions[Action_Ecc]) + { + erasure_info Info; + Info.dataShardCount = 248; + Info.parityShardCount = 8; + Info.shardSize = 1024 * 1024; + Info.erasureStart = 0; + Info.erasureLength = Buffer_Size_Matroska; + + Erasure_Encode = new erasure_encode(Info); + } + + // Erasure check? + if (Buffer_Size_Matroska != Buffer_Size) + { + const uint8_t* SearchBuffer = Buffer + Buffer_Size - 4; + const uint8_t* MatroskaEnd = Buffer + Buffer_Size_Matroska; + for (; SearchBuffer >= MatroskaEnd; SearchBuffer--) + { + if (SearchBuffer[0] != 0x1A || SearchBuffer[1] != 0x45 || SearchBuffer[2] != 0xDF || SearchBuffer[3] != 0xA3) + continue; + Buffer_Offset = SearchBuffer - Buffer; + Levels[Level].Offset_End = Buffer_Offset; + Matroska_ShouldBeParsed = true; + break; + } + } +} + //--------------------------------------------------------------------------- void matroska::ParseBuffer() { @@ -845,15 +1265,47 @@ void matroska::ParseBuffer() Level++; size_t Buffer_Offset_LowerLimit = 0; // Used for indicating the system that we'll not need anymore memory below this value + size_t Buffer_Offset_HigherLimit = 0; // Used for indicating the system that we didn't use memory beyond this value + size_t Erasure_Encode_Offset = 0; + size_t Erasure_Check_Offset = 0; while (Buffer_Offset < Buffer_Size) { + // Erasure encode + if (Erasure_Encode && Buffer_Size_Matroska && Buffer_Offset + 16 >= Erasure_Encode_Offset && Buffer_Offset < Buffer_Size_Matroska) + { + Erasure_Encode_Offset = Erasure_Encode->Encode(Buffer, Buffer_Size_Matroska, Erasure_Encode_Offset); + if (Buffer_Offset_HigherLimit < Erasure_Encode_Offset) + Buffer_Offset_HigherLimit = Erasure_Encode_Offset; + } + + // Erasure check + if (Erasure_Check && Buffer_Size_Matroska && Buffer_Offset >= Erasure_Check_Offset && Buffer_Offset < Buffer_Size_Matroska) + { + Erasure_Check_Offset = Erasure_Check->Check(Buffer, Buffer_Size_Matroska, Erasure_Check_Offset); + if (Erasure_Check_Offset == (uint64_t)-1) + break; // Error detected, we stop + if (Buffer_Offset_HigherLimit < Erasure_Check_Offset) + Buffer_Offset_HigherLimit = Erasure_Check_Offset; + } + + // Parse header uint64_t Name = Get_EB(); uint64_t Size = Get_EB(); if (Size <= Levels[Level - 1].Offset_End - Buffer_Offset) Levels[Level].Offset_End = Buffer_Offset + Size; else Levels[Level].Offset_End = Levels[Level - 1].Offset_End; + + // Erasure encode + if (Erasure_Encode && Buffer_Size_Matroska && Levels[Level].Offset_End >= Erasure_Encode_Offset && Levels[Level].Offset_End < Buffer_Size_Matroska) + { + Erasure_Encode_Offset = Erasure_Encode->Encode(Buffer, Buffer_Size_Matroska, Erasure_Encode_Offset); + if (Buffer_Offset_HigherLimit < Erasure_Encode_Offset) + Buffer_Offset_HigherLimit = Erasure_Encode_Offset; + } + + // Parse data call Call = (this->*Levels[Level - 1].SubElements)(Name); IsList = false; (this->*Call)(); @@ -873,10 +1325,42 @@ void matroska::ParseBuffer() // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value { - FileMap->Remap(); + FileMap->Remap(Actions[Action_Fix]); Buffer = FileMap->Buffer; Buffer_Offset_LowerLimit = Buffer_Offset; } + + if (Matroska_ShouldBeParsed && Buffer_Offset >= Buffer_Size) + { + Matroska_ShouldBeParsed = false; + Buffer_Offset = 0; + Buffer_Size = Buffer_Size_Matroska; + Levels[0].Offset_End = Buffer_Size; + Level = 1; + + if (Erasure_Check) + { + if (Erasure_Check->Init()) + { + // TODO + delete Erasure_Check; + Erasure_Check = nullptr; + } + + if (Erasure_Encode) + { + delete Erasure_Encode; + Erasure_Encode = nullptr; + } + } + } + } + + // Erasure encode + if (Erasure_Encode) + { + Erasure.Erasure(Erasure_Encode->HashValues, Erasure_Encode->ParityShards, Buffer_Size); + delete[] Erasure_Encode->HashValues; } // Progress indicator @@ -904,6 +1388,8 @@ void matroska::Void() void matroska::Segment() { SetDetected(); + Erasure_Init(); + IsList = true; } @@ -984,7 +1470,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedAttachment_Fil { if (!RAWcooked_FileNameIsValid) return; // File name should come first. TODO: support when file name comes after - if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00) + if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80) return; // MD5 support only Buffer_Offset++; @@ -1017,7 +1503,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedBlock_FileHash { if (!RAWcooked_FileNameIsValid) return; // File name should come first. TODO: support when file name comes after - if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00) + if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80) return; // MD5 support only Buffer_Offset++; @@ -1297,7 +1783,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_FileHash { if (!RAWcooked_FileNameIsValid) return; // File name should come first. TODO: support when file name comes after - if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00) + if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80) return; // MD5 support only Buffer_Offset++; @@ -1717,6 +2203,104 @@ void matroska::Segment_Tracks_TrackEntry_Video_PixelHeight() TrackInfo_Current->Frame.SetHeight(Data); } +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment() +{ + IsList = true; +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_LibraryName() +{ +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_LibraryVersion() +{ +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure() +{ + IsList = true; +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure_EbmlStartLocation() +{ + size_t Size = Levels[Level].Offset_End - Buffer_Offset; + int64_t Data = Get_BXs(Size); +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure_ShardHashes() +{ + uint64_t Kind = Get_EB(); + size_t Size = (Levels[Level].Offset_End - Buffer_Offset); + if (Kind == 0 && !(Size % 16)) //MD5 + { + if (!Erasure_Check) + Erasure_Check = new erasure_check(Actions[Action_Fix], Errors); + Erasure_Check->HashValues_Size = Size; + Erasure_Check->HashValues = new rawcooked::hash_value[Size / 16]; + memcpy(Erasure_Check->HashValues, Buffer + Buffer_Offset, Size); + Buffer_Offset += Size; + } +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure_ShardInfo() +{ + uint64_t dataShardCount = Get_EB(); + uint64_t parityShardCount = Get_EB(); + uint64_t shardSize = Get_EB(); + uint64_t erasureStart = Get_EB(); + uint64_t erasureLength = Get_EB(); + + if (Buffer_Offset <= Levels[Level].Offset_End + && dataShardCount && dataShardCount <= 0xFF + && parityShardCount && parityShardCount <= 0xFF + && dataShardCount + parityShardCount <= 0x100 + && erasureStart < Buffer_Size + && erasureLength < Buffer_Size + && erasureStart < Buffer_Size - erasureLength) + { + if (!Erasure_Check) + Erasure_Check = new erasure_check(Actions[Action_Fix], Errors); + Erasure_Check->Info.dataShardCount = (uint8_t)dataShardCount; + Erasure_Check->Info.parityShardCount = (uint8_t)parityShardCount; + Erasure_Check->Info.shardSize = shardSize; + Erasure_Check->Info.erasureStart = erasureStart; + Erasure_Check->Info.erasureLength = erasureLength; + } +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure_ParityShards() +{ + size_t Size = Levels[Level].Offset_End - Buffer_Offset; + if (!Erasure_Check) + Erasure_Check = new erasure_check(Actions[Action_Fix], Errors); + Erasure_Check->ParityShards = Buffer + Buffer_Offset; + Erasure_Check->ParityShards_Size = Size; + Buffer_Offset += Size; +} + +//--------------------------------------------------------------------------- +void matroska::Rawcooked_Segment_Erasure_ParityShardsLocation() +{ + size_t Size = Levels[Level].Offset_End - Buffer_Offset; + int64_t Data = Get_BXs(Size); + + uint64_t Remaining = Buffer_Size - Buffer_Offset; + if ((Data <= 0 || (Remaining < INT64_MAX && Data < (int64_t)Remaining)) && (Data >= 0 || (Buffer_Offset >= (uint64_t)-Data))) + { + if (!Erasure_Check) + Erasure_Check = new erasure_check(Actions[Action_Fix], Errors); + Erasure_Check->ParityShards = Buffer + (size_t) (Buffer_Offset + Data); + } +} + //*************************************************************************** // Utils //*************************************************************************** @@ -2050,3 +2634,10 @@ void matroska::ProcessFrame_FLAC() break; } } + +//--------------------------------------------------------------------------- +void matroska::Erasure_Write(const char* FileName) +{ + Erasure.FileName = FileName; + Erasure.Erasure_AppendToFile(); +} diff --git a/Source/Lib/Matroska/Matroska_Common.h b/Source/Lib/Matroska/Matroska_Common.h index 3b17aabd..953391f9 100644 --- a/Source/Lib/Matroska/Matroska_Common.h +++ b/Source/Lib/Matroska/Matroska_Common.h @@ -11,6 +11,7 @@ //--------------------------------------------------------------------------- #include "Lib/FFV1/FFV1_Frame.h" +#include "Lib/RAWcooked/RAWcooked.h" #include "Lib/Input_Base.h" #include "Lib/FileIO.h" #include @@ -30,6 +31,7 @@ class matroska_mapping; class ThreadPool; class flac_info; class hashes; +class ReedSolomon; class frame_writer { @@ -91,6 +93,9 @@ class matroska : public input_base hashes* Hashes_FromRAWcooked; hashes* Hashes_FromAttachments; + // Erasure + void Erasure_Write(const char* FileName); + // Theading relating functions void ProgressIndicator_Show(); @@ -170,6 +175,15 @@ class matroska : public input_base MATROSKA_ELEMENT(Segment_Tracks_TrackEntry_Video_PixelWidth); MATROSKA_ELEMENT(Segment_Tracks_TrackEntry_Video_PixelHeight); MATROSKA_ELEMENT(Void); + MATROSKA_ELEMENT(Rawcooked_Segment); + MATROSKA_ELEMENT(Rawcooked_Segment_LibraryName); + MATROSKA_ELEMENT(Rawcooked_Segment_LibraryVersion); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_EbmlStartLocation); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ShardHashes); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ShardInfo); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ParityShards); + MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ParityShardsLocation); enum format { @@ -255,6 +269,63 @@ class matroska : public input_base int16_t Block_Timestamp; friend class frame_writer; + // Erasure code + size_t Buffer_Size_Matroska = 0; + bool Matroska_ShouldBeParsed = false; + struct erasure_info + { + uint8_t dataShardCount = 0; + uint8_t parityShardCount = 0; + size_t shardSize = 0; + size_t erasureStart = 0; + size_t erasureLength = 0; + }; + struct erasure + { + erasure_info Info; + rawcooked::hash_value Compute_MD5(uint8_t* Buffer, size_t Buffer_Size); + }; + struct erasure_encode : public erasure + { + erasure_encode(erasure_info& Info); + + ReedSolomon* RS = nullptr; + rawcooked::hash_value* HashValues = nullptr; + uint8_t* ParityShards = nullptr; + + size_t Encode(uint8_t* Buffer, size_t Buffer_Max, size_t Offset); + }; + struct erasure_check : public erasure + { + erasure_check(bool Write_Source = false, errors* Errors_Source = nullptr) : + Write(Write_Source), + Errors(Errors_Source) + { + } + + bool Init(); + + // + rawcooked::hash_value* HashValues = nullptr; + size_t HashValues_Size = 0; // In bytes + uint8_t* ParityShards = nullptr; + size_t ParityShards_Size = 0; // In bytes + + // + size_t DataHashes_Count = 0; + size_t ParityHashes_Count = 0; + + size_t Check(uint8_t* Buffer, size_t Buffer_Max, size_t Offset); + + private: + errors* Errors; + bool Write; + }; + erasure_encode* Erasure_Encode = nullptr; + erasure_check* Erasure_Check = nullptr; + rawcooked Erasure; + void Erasure_Init(); + //Utils bool GetFormatAndFlavor(trackinfo* TrackInfo, input_base_uncompressed* PotentialParser, raw_frame::flavor Flavor); void ParseDecodedFrame(trackinfo* TrackInfo); diff --git a/Source/Lib/RAWcooked/RAWcooked.cpp b/Source/Lib/RAWcooked/RAWcooked.cpp index 83ba34a9..f1868408 100644 --- a/Source/Lib/RAWcooked/RAWcooked.cpp +++ b/Source/Lib/RAWcooked/RAWcooked.cpp @@ -55,12 +55,30 @@ static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), Inco } // intermediatewrite_issue +//--------------------------------------------------------------------------- +extern void Matroska_CRC32_Compute(uint32_t& CRC32, const uint8_t* Buffer_Current, const uint8_t* Buffer_End); +static void CRC32_Fill(uint8_t* Buffer, size_t Buffer_Size) +{ + // Compute and fill CRC32 + uint32_t CRC32Computed = 0xFFFFFFFF; + Matroska_CRC32_Compute(CRC32Computed, Buffer, Buffer + Buffer_Size); + CRC32Computed ^= 0xFFFFFFFF; + Buffer[-4] = (uint8_t)(CRC32Computed); + Buffer[-3] = (uint8_t)(CRC32Computed >> 8); + Buffer[-2] = (uint8_t)(CRC32Computed >> 16); + Buffer[-1] = (uint8_t)(CRC32Computed >> 24); +} + //--------------------------------------------------------------------------- // Library name and version const char* LibraryName = "RAWcooked"; const char* LibraryVersion = "18.10.1"; +// Global +static const uint32_t CRC32 = 0x3F; +static const uint32_t Void = 0x6C; + // EBML static const uint32_t Name_EBML = 0x0A45DFA3; static const uint32_t Name_EBML_Doctype = 0x0282; @@ -93,14 +111,21 @@ static const uint32_t Name_RawCooked_FileHash = 0x20; static const uint32_t Name_RawCooked_FileSize = 0x30; static const uint32_t Name_RawCooked_MaskBaseFileSize = 0x31; // In BlockGroup only static const uint32_t Name_RawCooked_MaskAdditionFileSize = 0x32; // In Track only -// Global information +// Segment information static const uint32_t Name_RawCooked_LibraryName = 0x70; static const uint32_t Name_RawCooked_LibraryVersion = 0x71; static const uint32_t Name_RawCooked_PathSeparator = 0x72; +// Erasure +static const uint32_t Name_Rawcooked_Erasure = 0x726365; // "rce", RAWcooked global part, EBML class D +static const uint32_t Name_Rawcooked_Erasure_ShardInfo = 0x726369; // "rci", RAWcooked global part, EBML class D +static const uint32_t Name_Rawcooked_Erasure_ShardHashes = 0x726368; // "rch", RAWcooked global part, EBML class D +static const uint32_t Name_Rawcooked_Erasure_ParityShards = 0x726370; // "rcp", RAWcooked global part, EBML class D +static const uint32_t Name_Rawcooked_Erasure_EbmlStartLocation = 0x726373; // "rcs", RAWcooked global part, EBML class D +static const uint32_t Name_Rawcooked_Erasure_ParityShardsLocation = 0x72636C; // "rcl", RAWcooked global part, EBML class D // Parameters static const char* DocType = "rawcooked"; -static const uint8_t DocTypeVersion = 1; -static const uint8_t DocTypeReadVersion = 1; +static const uint64_t DocTypeVersion = 1; +static const uint64_t DocTypeReadVersion = 1; //--------------------------------------------------------------------------- enum hashformat @@ -112,18 +137,40 @@ enum hashformat static size_t Size_EB(uint64_t Value) { size_t S_l = 1; - while (Value >> (S_l * 7)) + while (S_l < 7 && Value >> (S_l * 7)) S_l++; if (Value == (uint64_t)-1 >> ((sizeof(Value) - S_l) * 8 + S_l)) S_l++; return S_l; } +//--------------------------------------------------------------------------- +static void Write_EB(uint8_t* Buffer, size_t& Buffer_Offset, uint64_t Value) +{ + size_t S_l = Size_EB(Value); + uint64_t S2 = Value | (((uint64_t)1) << (S_l * 7)); + while (S_l) + { + Buffer[Buffer_Offset] = (uint8_t)(S2 >> ((S_l - 1) * 8)); + Buffer_Offset++; + S_l--; + } +} + //--------------------------------------------------------------------------- static size_t Size_Number(uint64_t Value) { size_t S_l = 1; - while (Value >> (S_l * 8)) + while (S_l < 7 && Value >> (S_l * 8)) + S_l++; + return S_l; +} + +//--------------------------------------------------------------------------- +static size_t Size_Number(int64_t Value) +{ + size_t S_l = 1; + while (S_l < 7 && Value >> (S_l * 8) == ((int64_t)-1) >> (S_l * 8)) S_l++; return S_l; } @@ -136,10 +183,11 @@ class ebml_writer ~ebml_writer(); // Types - void EB(uint64_t Size); - void Block(uint32_t Name, uint64_t Size, const uint8_t* Data); + void EB(uint64_t Value, size_t Length = (size_t)-1); + void Block(uint32_t Name, uint64_t Size, const uint8_t* Data, size_t BlockSize_Length = (size_t)-1); void String(uint32_t Name, const char* Data); - void Number(uint32_t Name, uint64_t Number); + void Number(uint32_t Name, uint64_t Number, size_t BlockSize_Length = (size_t)-1); + void Number(uint32_t Name, int64_t Number, size_t BlockSize_Length = (size_t)-1); void DataWithEncodedPrefix(uint32_t Name, uint64_t Prefix, uint64_t Size, const uint8_t* Data); void CompressableData(uint32_t Name, const uint8_t* Data, uint64_t Size, uint64_t CompressedSize); @@ -153,37 +201,88 @@ class ebml_writer size_t GetBufferSize(); private: - std::vector BlockSizes; + struct block_info + { + size_t Buffer_PreviousOffset = 0; + size_t Size = 0; + std::vector Blocks; + + size_t GetSize(size_t Level) + { + if (Level) + return Blocks.back().GetSize(Level - 1); + else + return Blocks.back().Size; + } + + void Remove(size_t Level) + { + if (Level) + return Blocks.back().Remove(Level - 1); + else + Blocks.pop_back(); + } + + void Begin(size_t Offset, size_t Level) + { + if (Level) + return Blocks.back().Begin(Offset, Level - 1); + else + { + Blocks.push_back(block_info()); + Blocks.back().Buffer_PreviousOffset = Offset; + } + } + + void End(size_t Offset, size_t Level) + { + if (Level) + return Blocks.back().End(Offset, Level - 1); + else + { + Blocks.back().Size = Offset - Blocks.back().Buffer_PreviousOffset; + } + } + + void Set2ndPass() + { + Buffer_PreviousOffset = 0; + reverse(Blocks.begin(), Blocks.end()); // We'll decrement size for keeping track about where we are + for (auto Block : Blocks) + Block.Set2ndPass(); + } + }; + block_info BlockInfo; + size_t Level = (size_t)-1; uint8_t* Buffer = NULL; - size_t Buffer_PreviousOffset = 0; size_t Buffer_Offset = 0; }; //--------------------------------------------------------------------------- -void ebml_writer::EB(uint64_t Size) +void ebml_writer::EB(uint64_t Value, size_t S_l) { - size_t S_l = Size_EB(Size); + if (S_l > 7) + S_l = Size_EB(Value); if (Buffer) { - uint64_t S2 = Size | (((uint64_t)1) << (S_l * 7)); + uint64_t S2 = Value | (((uint64_t)1) << (S_l * 7)); while (S_l) { Buffer[Buffer_Offset] = (uint8_t)(S2 >> ((S_l - 1) * 8)); Buffer_Offset++; S_l--; } - } else Buffer_Offset += S_l; } //--------------------------------------------------------------------------- -void ebml_writer::Block(uint32_t Name, uint64_t Size, const uint8_t* Data) +void ebml_writer::Block(uint32_t Name, uint64_t Size, const uint8_t* Data, size_t S_l) { EB(Name); - EB(Size); + EB(Size, S_l); if (Buffer) { @@ -199,10 +298,32 @@ void ebml_writer::String(uint32_t Name, const char* Data) } //--------------------------------------------------------------------------- -void ebml_writer::Number(uint32_t Name, uint64_t Number) +void ebml_writer::Number(uint32_t Name, uint64_t Number, size_t S_l) { EB(Name); - size_t S_l = Size_Number(Number); + if (S_l > 7) + S_l = Size_Number(Number); + EB(S_l); + + if (Buffer) + { + while (S_l) + { + Buffer[Buffer_Offset] = (uint8_t)(Number >> ((S_l - 1) * 8)); + Buffer_Offset++; + S_l--; + } + } + else + Buffer_Offset += S_l; +} + +//--------------------------------------------------------------------------- +void ebml_writer::Number(uint32_t Name, int64_t Number, size_t S_l) +{ + EB(Name); + if (S_l > 7) + S_l = Size_Number(Number); EB(S_l); if (Buffer) @@ -246,14 +367,17 @@ void ebml_writer::CompressableData(uint32_t Name, const uint8_t* Data, uint64_t //--------------------------------------------------------------------------- void ebml_writer::Block_Begin(uint32_t Name) { + Level++; + EB(Name); if (Buffer) { - EB(BlockSizes.back()); - BlockSizes.pop_back(); + EB(BlockInfo.GetSize(Level)); } else - Buffer_PreviousOffset = Buffer_Offset; + { + BlockInfo.Begin(Buffer_Offset, Level); + } } //--------------------------------------------------------------------------- @@ -261,9 +385,15 @@ void ebml_writer::Block_End() { if (!Buffer) { - BlockSizes.push_back(Buffer_Offset - Buffer_PreviousOffset); - EB(Buffer_Offset - Buffer_PreviousOffset); + BlockInfo.End(Buffer_Offset, Level); + EB(BlockInfo.GetSize(Level)); } + else + { + BlockInfo.Remove(Level); + } + + Level--; } //--------------------------------------------------------------------------- @@ -272,9 +402,8 @@ void ebml_writer::Set2ndPass() if (Buffer) return; - reverse(BlockSizes.begin(), BlockSizes.end()); // We'll decrement size for keeping track about where we are + BlockInfo.Set2ndPass(); Buffer = new uint8_t[Buffer_Offset]; - Buffer_PreviousOffset = 0; Buffer_Offset = 0; } @@ -310,6 +439,7 @@ rawcooked::rawcooked() : rawcooked::~rawcooked() { Close(); + delete EbmlWriter; } //--------------------------------------------------------------------------- @@ -470,8 +600,8 @@ void rawcooked::Parse() { Writer.Block_Begin(Name_EBML); Writer.String(Name_EBML_Doctype, DocType); - Writer.Number(Name_EBML_DoctypeVersion, 1); - Writer.Number(Name_EBML_DoctypeReadVersion, 1); + Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion); + Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion); Writer.Block_End(); } @@ -568,11 +698,11 @@ void rawcooked::Delete() } //--------------------------------------------------------------------------- -void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size) +void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size, bool Append) { if (!File_WasCreated) { - bool RejectIfExists = (Mode && *Mode == AlwaysYes) ? false : true; + bool RejectIfExists = (Append || (Mode && *Mode == AlwaysYes)) ? false : true; if (file::return_value Result = File.Open_WriteMode(string(), FileName, RejectIfExists)) { if (RejectIfExists && Result == file::Error_FileAlreadyExists && Mode && *Mode == Ask && Ask_Callback) @@ -595,9 +725,141 @@ void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size) return; } } + if (Append) + { + if (File.Seek(0, file::End)) + { + if (Errors) + Errors->Error(IO_IntermediateWriter, error::Undecodable, (error::generic::code)intermediatewrite_issue::undecodable::FileWrite, FileName); + return; + } + } File_WasCreated = true; } if (File.Write(Buffer, Buffer_Size)) return; } + +//--------------------------------------------------------------------------- +void rawcooked::Erasure(hash_value* HashValues, uint8_t* ParityShards, size_t Buffer_Size) +{ + // Create + size_t DataHashes_Count = (Buffer_Size + 1024 * 1024 - 1) / (1024 * 1024); + size_t ParityHashes_Count = (DataHashes_Count + 284 - 1) / 248; + ParityHashes_Count *= 8; + + size_t Offset; + size_t ShardInfo_Size = Size_EB(248) + Size_EB(8) + Size_EB(1024 * 1024) + Size_EB(0) + Size_EB(Buffer_Size); + uint8_t* ShardInfo = new uint8_t[ShardInfo_Size]; + Offset = 0; + Write_EB(ShardInfo, Offset, 248); + Write_EB(ShardInfo, Offset, 8); + Write_EB(ShardInfo, Offset, 1024 * 1024); + Write_EB(ShardInfo, Offset, 0); + Write_EB(ShardInfo, Offset, Buffer_Size); + size_t ShardHashes_Size = Size_EB(0) + 16 * (DataHashes_Count + ParityHashes_Count); + uint8_t* ShardHashes = new uint8_t[ShardHashes_Size]; + Offset = 0; + Write_EB(ShardHashes, Offset, 0); + for (size_t i = 0; i < DataHashes_Count; i++) + { + memcpy(ShardHashes + Offset, HashValues[i].Values, 16); + Offset += 16; + } + for (size_t i = 0; i < ParityHashes_Count; i++) + { + memcpy(ShardHashes + Offset, HashValues[DataHashes_Count + i].Values, 16); + Offset += 16; + } + + delete EbmlWriter; + EbmlWriter = new ebml_writer; + auto& Writer = *EbmlWriter; + size_t Padding_Size, Padding_Size2; + vector Locations; + for (uint8_t Pass = 0; Pass < 2; Pass++) + { + // EBML header + if (!Pass) + Locations.push_back(Writer.GetBufferSize()); + Writer.Block_Begin(Name_EBML); + Writer.String(Name_EBML_Doctype, DocType); + Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion); + Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion); + Writer.Block_End(); + + // Segment - before padding + Writer.Block_Begin(Name_RawCookedSegment); + Writer.String(Name_RawCooked_LibraryName, LibraryName); + Writer.String(Name_RawCooked_LibraryVersion, LibraryVersion); + Writer.Block_Begin(Name_Rawcooked_Erasure); + Writer.Block(Name_Rawcooked_Erasure_ShardInfo, ShardInfo_Size, ShardInfo); + Writer.Block(Name_Rawcooked_Erasure_ShardHashes, ShardHashes_Size, ShardHashes); + + // Segment - padding + if (!Pass) + Padding_Size = 1024 * 1024 - ((Buffer_Size + Writer.GetBufferSize() + 12 /*TEMP*/ + Size_EB(Name_Rawcooked_Erasure_ParityShards) + Size_EB(ParityHashes_Count * 1024 * 1024)) % (1024 * 1024)); + Writer.Block(Void, Padding_Size, ParityShards, 3); //TEMP ParityShards + + // Segment - after padding + Writer.Block(Name_Rawcooked_Erasure_ParityShards, ParityHashes_Count * 1024 * 1024, ParityShards); + Writer.Block_End(); + Writer.Block_End(); + if (!Pass) + Locations.push_back(Writer.GetBufferSize()); + + for (uint8_t i = 0; i < 8 + 1; i++) + { + // EBML header + Writer.Block_Begin(Name_EBML); + Writer.String(Name_EBML_Doctype, DocType); + Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion); + Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion); + Writer.Block_End(); + + // Segment - before padding + Writer.Block_Begin(Name_RawCookedSegment); + Writer.String(Name_RawCooked_LibraryName, LibraryName); + Writer.String(Name_RawCooked_LibraryVersion, LibraryVersion); + Writer.Block_Begin(Name_Rawcooked_Erasure); + const uint32_t Zero = 0; + Writer.Block(CRC32, 4, (const uint8_t*)&Zero); + size_t CRC32_Start = Writer.GetBufferSize(); + Writer.Block(Name_Rawcooked_Erasure_ShardInfo, ShardInfo_Size, ShardInfo); + Writer.Block(Name_Rawcooked_Erasure_ShardHashes, ShardHashes_Size, ShardHashes); + + // Segment - padding + size_t LastPartBlockSize = (Size_EB(Name_Rawcooked_Erasure_EbmlStartLocation) + Size_EB(4) + 4); + if (!i && !Pass) + Padding_Size2 = 1024 * 1024 - ((Writer.GetBufferSize() - Locations[1] + 10 /*TEMP*/ + LastPartBlockSize * (1 + 8 + 1 + 1)) % (1024 * 1024)); + Writer.Block(Void, Padding_Size2, ParityShards, 3); //TEMP ParityShards + + // Segment - after padding + Writer.Number(Name_Rawcooked_Erasure_EbmlStartLocation, Pass ? (Locations[0] - Locations[1 + i + 1] + LastPartBlockSize * (1 + 8 + 1)) : 0, 4); + for (uint8_t j = 0; j < 8 + 1; j++) + Writer.Number(Name_Rawcooked_Erasure_EbmlStartLocation, Pass ? (Locations[1 + j] - Locations[1 + i + 1] + LastPartBlockSize * (1 + 8 - j)) : 0, 4); + Writer.Number(Name_Rawcooked_Erasure_ParityShardsLocation, Pass ? (Locations[1] - ParityHashes_Count * 1024 * 1024 - Locations[1 + i + 1]) : 0, 4); + Writer.Block_End(); + Writer.Block_End(); + if (!Pass) + Locations.push_back(Writer.GetBufferSize()); + else + CRC32_Fill(Writer.GetBuffer() + CRC32_Start, Writer.GetBufferSize() - CRC32_Start); // Compute and fill CRC32 + } + + // Init 2nd pass + Writer.Set2ndPass(); + } +} + +//--------------------------------------------------------------------------- +void rawcooked::Erasure_AppendToFile() +{ + if (!EbmlWriter) + return; // Nothing to do + + // Write + auto& Writer = *EbmlWriter; + WriteToDisk(Writer.GetBuffer(), Writer.GetBufferSize(), true); +} diff --git a/Source/Lib/RAWcooked/RAWcooked.h b/Source/Lib/RAWcooked/RAWcooked.h index af3acd97..d96e39f5 100644 --- a/Source/Lib/RAWcooked/RAWcooked.h +++ b/Source/Lib/RAWcooked/RAWcooked.h @@ -17,6 +17,7 @@ #include #include using namespace std; +class ebml_writer; //--------------------------------------------------------------------------- class rawcooked @@ -48,6 +49,14 @@ class rawcooked string OutputFileName; uint64_t FileSize; + // Erasure + struct hash_value + { + unsigned char Values[16]; + }; + void Erasure(hash_value* Hashes_Values, uint8_t* ParityShards, size_t Buffer_Size); + void Erasure_AppendToFile(); + // Errors user_mode* Mode = nullptr; ask_callback Ask_Callback = nullptr; @@ -60,7 +69,7 @@ class rawcooked file File; size_t BlockCount; bool File_WasCreated = false; - void WriteToDisk(uint8_t* Buffer, size_t Buffer_Size); + void WriteToDisk(uint8_t* Buffer, size_t Buffer_Size, bool Append = false); // First frame info uint8_t* FirstFrame_Before; @@ -69,6 +78,9 @@ class rawcooked size_t FirstFrame_After_Size; uint8_t* FirstFrame_FileName; size_t FirstFrame_FileName_Size; + + // Erasure + ebml_writer* EbmlWriter = nullptr; }; //--------------------------------------------------------------------------- diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp new file mode 100644 index 00000000..a31d0caf --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp @@ -0,0 +1,195 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a MIT-style license that can + * be found in the License.html file in the root of the source tree. + */ + +/* + * Based on https://www.backblaze.com/blog/reed-solomon + * Based on https://github.com/Backblaze/JavaReedSolomon + * Based on https://github.com/DrPizza/reed-solomon + * Initial copyrights: + * copyright 2015 Peter Bright, Backblaze. + */ + +//--------------------------------------------------------------------------- +#include "Reed-Solomon/Galois.h" +//--------------------------------------------------------------------------- + +Galois galois; + +/** + * The polynomial used to generate the logarithm table. + * + * There are a number of polynomials that work to generate + * a Galois field of 256 elements. The choice is arbitrary, + * and we just use the first one. + * + * The possibilities are: 29, 43, 45, 77, 95, 99, 101, 105, + * 113, 135, 141, 169, 195, 207, 231, and 245. + */ + +/** + * Mapping from members of the Galois Field to their + * integer logarithms. The entry for 0 is meaningless + * because there is no log of 0. + */ +static uint8_t LogTable[256] = { + 0, 0, 1, 25, 2, 50, 26, 198, + 3, 223, 51, 238, 27, 104, 199, 75, + 4, 100, 224, 14, 52, 141, 239, 129, + 28, 193, 105, 248, 200, 8, 76, 113, + 5, 138, 101, 47, 225, 36, 15, 33, + 53, 147, 142, 218, 240, 18, 130, 69, + 29, 181, 194, 125, 106, 39, 249, 185, + 201, 154, 9, 120, 77, 228, 114, 166, + 6, 191, 139, 98, 102, 221, 48, 253, + 226, 152, 37, 179, 16, 145, 34, 136, + 54, 208, 148, 206, 143, 150, 219, 189, + 241, 210, 19, 92, 131, 56, 70, 64, + 30, 66, 182, 163, 195, 72, 126, 110, + 107, 58, 40, 84, 250, 133, 186, 61, + 202, 94, 155, 159, 10, 21, 121, 43, + 78, 212, 229, 172, 115, 243, 167, 87, + 7, 112, 192, 247, 140, 128, 99, 13, + 103, 74, 222, 237, 49, 197, 254, 24, + 227, 165, 153, 119, 38, 184, 180, 124, + 17, 68, 146, 217, 35, 32, 137, 46, + 55, 63, 209, 91, 149, 188, 207, 205, + 144, 135, 151, 178, 220, 252, 190, 97, + 242, 86, 211, 171, 20, 42, 93, 158, + 132, 60, 57, 83, 71, 109, 65, 162, + 31, 45, 67, 216, 183, 123, 164, 118, + 196, 23, 73, 236, 127, 12, 111, 246, + 108, 161, 59, 82, 41, 157, 85, 170, + 251, 96, 134, 177, 187, 204, 62, 90, + 203, 89, 95, 176, 156, 169, 160, 81, + 11, 245, 22, 235, 122, 117, 44, 215, + 79, 174, 213, 233, 230, 231, 173, 232, + 116, 214, 244, 234, 168, 80, 88, 175 +}; + +/** + * Inverse of the logarithm table. Maps integer logarithms + * to members of the field. There is no entry for 255 + * because the highest log is 254. + */ +static uint8_t ExpTable[255] = { + 1, 2, 4, 8, 16, 32, 64, 128, 29, + 58, 116, 232, 205, 135, 19, 38, 76, + 152, 45, 90, 180, 117, 234, 201, 143, + 3, 6, 12, 24, 48, 96, 192, 157, + 39, 78, 156, 37, 74, 148, 53, 106, + 212, 181, 119, 238, 193, 159, 35, 70, + 140, 5, 10, 20, 40, 80, 160, 93, + 186, 105, 210, 185, 111, 222, 161, 95, + 190, 97, 194, 153, 47, 94, 188, 101, + 202, 137, 15, 30, 60, 120, 240, 253, + 231, 211, 187, 107, 214, 177, 127, 254, + 225, 223, 163, 91, 182, 113, 226, 217, + 175, 67, 134, 17, 34, 68, 136, 13, + 26, 52, 104, 208, 189, 103, 206, 129, + 31, 62, 124, 248, 237, 199, 147, 59, + 118, 236, 197, 151, 51, 102, 204, 133, + 23, 46, 92, 184, 109, 218, 169, 79, + 158, 33, 66, 132, 21, 42, 84, 168, + 77, 154, 41, 82, 164, 85, 170, 73, + 146, 57, 114, 228, 213, 183, 115, 230, + 209, 191, 99, 198, 145, 63, 126, 252, + 229, 215, 179, 123, 246, 241, 255, 227, + 219, 171, 75, 150, 49, 98, 196, 149, + 55, 110, 220, 165, 87, 174, 65, 130, + 25, 50, 100, 200, 141, 7, 14, 28, + 56, 112, 224, 221, 167, 83, 166, 81, + 162, 89, 178, 121, 242, 249, 239, 195, + 155, 43, 86, 172, 69, 138, 9, 18, + 36, 72, 144, 61, 122, 244, 245, 247, + 243, 251, 235, 203, 139, 11, 22, 44, + 88, 176, 125, 250, 233, 207, 131, 27, + 54, 108, 216, 173, 71, 142, +}; + +Galois::Galois() +{ + uint8_t a = 0; + do + { + uint8_t b = 0; + do + { + MultiplicationTable[a][b] = multiply(a, b); + if (!(b & 0x0F)) + { + uint8_t result = multiply(a, b); + MultiplicationTableSplit[a].High[b >> 4] = result; + MultiplicationTableSplit[a].High[(b >> 4) + 16] = result; + MultiplicationTableSplit[a].High[(b >> 4) + 32] = result; + MultiplicationTableSplit[a].High[(b >> 4) + 48] = result; + } + if (!(b & 0xF0)) + { + uint8_t result = multiply(a, b); + MultiplicationTableSplit[a].Low[b] = result; + MultiplicationTableSplit[a].Low[b + 16] = result; + MultiplicationTableSplit[a].Low[b + 32] = result; + MultiplicationTableSplit[a].Low[b + 48] = result; + } + } + while (b++ != 255); + } + while (a++ != 255); +} + +/** + * Multiplies to elements of the field. + */ +uint8_t Galois::multiply(uint8_t a, uint8_t b) +{ + if (!a || !b) + return 0; + + uint16_t logA = LogTable[a]; + uint16_t logB = LogTable[b]; + uint16_t logResult = logA + logB; + if (255 <= logResult) + logResult -= 255; + return ExpTable[logResult]; +} + +/** + * Inverse of multiplication. + */ +uint8_t Galois::divide(uint8_t a, uint8_t b) +{ + if (!a) + return 0; + if (!b) + throw; // b should never be 0 + int16_t logA = LogTable[a]; + int16_t logB = LogTable[b]; + int16_t logResult = logA - logB; + if (logResult < 0) + logResult += 255; + return ExpTable[logResult]; +} + +/** + * Computes a**n. + * + * The result will be the same as multiplying a times itself n times. + * + * @param a A member of the field. + * @param n A plain-old integer. + * @return The result of multiplying a by itself n times. + */ +uint8_t Galois::exp(uint8_t a, uint8_t n) +{ + if (!n) + return 1; + if (!a) + return 0; + uint16_t logResult = LogTable[a] * n; + while (255 <= logResult) + logResult -= 255; + return ExpTable[logResult]; +} diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Galois.h b/Source/Lib/ThirdParty/Reed-Solomon/Galois.h new file mode 100644 index 00000000..17451acd --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Galois.h @@ -0,0 +1,58 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a MIT-style license that can + * be found in the License.html file in the root of the source tree. + */ + + //--------------------------------------------------------------------------- +#ifndef GaloisH +#define GaloisH +//--------------------------------------------------------------------------- + +/** + * 8-bit Galois Field + */ + + //--------------------------------------------------------------------------- +#include +//--------------------------------------------------------------------------- + +class Galois +{ +public: + Galois(); + + /** + * Multiplies to elements of the field. + */ + uint8_t multiply(uint8_t a, uint8_t b); + + /** + * Inverse of multiplication. + */ + uint8_t divide(uint8_t a, uint8_t b); + + /** + * Computes a**n. + * + * The result will be the same as multiplying a times itself n times. + * + * @param a A member of the field. + * @param n A plain-old integer. + * @return The result of multiplying a by itself n times. + */ + uint8_t exp(uint8_t a, uint8_t n); + + // Members + uint8_t MultiplicationTable[256][256]; + struct multiplication_table_split + { + uint8_t High[4 * 16]; // 4 high bits + uint8_t Low[4 * 16]; // 4 low bits + }; + multiplication_table_split MultiplicationTableSplit[256]; +}; + +extern Galois galois; + +#endif diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp new file mode 100644 index 00000000..a8049d46 --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp @@ -0,0 +1,186 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a BSD-style license that can + * be found in the License.html file in the root of the source tree. + */ + +/* + * Based on https://www.backblaze.com/blog/reed-solomon + * Based on https://github.com/Backblaze/JavaReedSolomon + * Based on https://github.com/DrPizza/reed-solomon + * Initial copyrights: + * copyright 2015 Peter Bright, Backblaze. + */ + +//--------------------------------------------------------------------------- +#include "Reed-Solomon/Galois.h" +#include "Reed-Solomon/Matrix.h" +#include +#include +#include +using namespace std; +//--------------------------------------------------------------------------- + +Matrix::Matrix(uint8_t rows_minus1, uint8_t columns_minus1) : + rowCountMinus1{ rows_minus1 }, + columnCountMinus1{ columns_minus1 }, + data{ new uint8_t * [rowCountMinus1 + 1] }, + dataPointer{ new uint8_t[(rowCountMinus1 + 1) * (columnCountMinus1 + 1) * 2] } // Twice the needed size in order to anticipate invert() +{ + uint8_t* dataTemp = dataPointer; + uint16_t columnCount = columnCountMinus1 + 1; + uint8_t r = 0; + do + { + data[r] = dataTemp; + dataTemp += columnCount; + } + while (r++ != rowCountMinus1); +} + +// Move +Matrix::Matrix(Matrix&& rhs) : + rowCountMinus1{ rhs.rowCountMinus1 }, + columnCountMinus1{ rhs.columnCountMinus1 }, + dataPointer{ rhs.dataPointer }, + data{ rhs.data } +{ + rhs.dataPointer = nullptr; + rhs.data = nullptr; +} + +Matrix::~Matrix() +{ + delete[] data; + delete[] dataPointer; +} + +Matrix Matrix::submatrix(uint8_t rmin, uint8_t cmin, uint8_t rmax, uint8_t cmax) const +{ + Matrix result{ static_cast(rmax - rmin - 1), static_cast(cmax - cmin - 1) }; + for (uint8_t r = rmin; r < rmax; ++r) + for (uint8_t c = cmin; c < cmax; ++c) + result.data[r - rmin][c - cmin] = data[r][c]; + return result; +} + +Matrix Matrix::times(const Matrix & rhs) const +{ + Matrix result{ rowCountMinus1, rhs.columnCountMinus1 }; + uint8_t r = 0; + do + { + uint8_t c = 0; + do + { + uint8_t value = 0; + uint8_t i = 0; + do + { + value ^= galois.multiply(data[r][i], rhs.data[i][c]); + } + while (i++ != columnCountMinus1); + result.data[r][c] = value; + } + while (c++ != rhs.columnCountMinus1); + } + while (r++ != rowCountMinus1); + return result; +} + +void Matrix::invert() +{ + // Augmenting this matrix with + // an identity matrix on the right. + const uint16_t size = (rowCountMinus1 + 1) * (columnCountMinus1 + 1); + uint8_t * dataTemp = dataPointer + size; + uint8_t * dataTemp2 = dataTemp + size; + uint16_t r = rowCountMinus1 + 1; + uint16_t columnCount = columnCountMinus1 + 1; + do + { + dataTemp2 -= columnCount; + memset(dataTemp2, 0, columnCount); + dataTemp2[--r] = 1; + dataTemp2 -= columnCount; + dataTemp -= columnCount; + memcpy(dataTemp2, dataTemp, columnCount); + data[r] = dataTemp2; + } + while (r); + + // Do Gaussian elimination to transform the left half into + // an identity matrix. + gaussianElimination(); + + // The right half is now the inverse. + dataTemp2 += columnCount; + uint16_t columnCount2 = columnCount * 2; + while (r <= rowCountMinus1) + { + memcpy(dataTemp, dataTemp2, columnCount); + data[r++] = dataTemp; + dataTemp += columnCount; + dataTemp2 += columnCount2; + } +} + +void Matrix::gaussianElimination() +{ + const uint16_t columnCount2 = (columnCountMinus1 + 1) * 2; + + // Clear out the part below the main diagonal and scale the main + // diagonal to be 1. + uint8_t r = 0; + do + { + // If the element on the diagonal is 0, find a row below + // that has a non-zero and swap them. + if (!data[r][r]) + { + uint8_t rowBelow = r + 1; + do + { + if (data[rowBelow][r]) + { + //Swap + for (uint16_t c = 0; c < columnCount2; c++) + swap(data[r][c], data[rowBelow][c]); + } + } + while (rowBelow++ != rowCountMinus1); + + // If we couldn't find one, the matrix is singular. + if (!data[r][r]) + { + throw runtime_error("matrix is singular"); + } + } + + // Scale to 1. + if (data[r][r] != 1) + { + uint8_t scale = galois.divide(1, data[r][r]); + for (uint16_t c = 0; c < columnCount2; ++c) + data[r][c] = galois.multiply(data[r][c], scale); + } + + // Make everything below the 1 be a 0 by subtracting + // a multiple of it. (Subtraction and addition are + // both exclusive or in the Galois field.) + uint8_t r2 = 0; + do + { + if (r2 == r) + continue; + uint8_t scale = data[r2][r]; + if (scale) + { + for (uint16_t c = 0; c < columnCount2; c++) + data[r2][c] ^= galois.multiply(scale, data[r][c]); + } + } + while (r2++ != rowCountMinus1); + } + while (r++ != rowCountMinus1); +} diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h new file mode 100644 index 00000000..17503ec4 --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h @@ -0,0 +1,86 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a MIT-style license that can + * be found in the License.html file in the root of the source tree. + */ + +//--------------------------------------------------------------------------- +#ifndef MatrixH +#define MatrixH +//--------------------------------------------------------------------------- + + //--------------------------------------------------------------------------- +#include +//--------------------------------------------------------------------------- + +/** + * A matrix over the 8-bit Galois field. + * + * Note: input is not verified + */ + +class Matrix +{ +public: + /** + * Initialize a matrix of zeros. + * + * @param rows_minus1 The number of rows in the matrix minus 1. + * @param columns_minus1 The number of columns in the matrix minus 1. + */ + Matrix(uint8_t rows_minus1, uint8_t columns_minus1); + Matrix(Matrix&& rhs); + ~Matrix(); + + /** + * Returns pointer on one row. + */ + inline const uint8_t* getRow(uint8_t r) const { return data[r]; } + + /** + * Returns the value at row r, column c. + */ + inline uint8_t get(uint8_t r, uint8_t c) const { return data[r][c]; } + + /** + * Sets the value at row r, column c. + */ + inline void set(uint8_t r, uint8_t c, uint8_t value) { data[r][c] = value; } + + /** + * Returns a part of this matrix. + */ + Matrix submatrix(uint8_t rmin, uint8_t cmin, uint8_t rmax, uint8_t cmax) const; + + /** + * Multiplies this matrix (the one on the left) by another + * matrix (the one on the right). + */ + Matrix times(const Matrix& rhs) const; + + /** + * Inverse this matrix. + */ + void invert(); + +private: + /** + * Does the work of matrix inversion. + * + * Assumes that this is an r by 2r matrix. + */ + void gaussianElimination(); + + // Members + const uint8_t rowCountMinus1; + const uint8_t columnCountMinus1; + uint8_t** data; + uint8_t* dataPointer; + + // Forbidden + Matrix(const Matrix& rhs) = delete; + bool operator==(const Matrix& rhs) const = delete; + bool operator!=(const Matrix& rhs) const = delete; +}; + +#endif diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp new file mode 100644 index 00000000..3509dac1 --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp @@ -0,0 +1,502 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a BSD-style license that can + * be found in the License.html file in the root of the source tree. + */ + +/* + * Based on https://www.backblaze.com/blog/reed-solomon + * Based on https://github.com/Backblaze/JavaReedSolomon + * Based on https://github.com/DrPizza/reed-solomon + * Initial copyrights: + * copyright 2015 Peter Bright, Backblaze. + */ + +//--------------------------------------------------------------------------- +#include "Reed-Solomon/Galois.h" +#include "Reed-Solomon/Matrix.h" +#include "Reed-Solomon/Reed-Solomon.h" +#include +#if defined(__x86_64) || defined(_M_AMD64) + #if defined(_MSC_VER) + #include + #else + #include + #endif + #include +#endif +using namespace std; + +//--------------------------------------------------------------------------- + +/** + * Create a Vandermonde matrix, which is guaranteed to have the + * property that any subset of rows that forms a square matrix + * is invertible. + * + * @param rows Number of rows in the result. + * @param cols Number of columns in the result. + * @return A Matrix. + */ +static Matrix vandermonde(uint8_t rows_minus1, uint8_t cols) +{ + uint8_t cols_minus1 = cols - 1; + Matrix result{ rows_minus1, cols_minus1 }; + uint8_t r = 0; + do + { + uint8_t c = 0; + do + { + result.set(r, c, galois.exp(r, c)); + } + while (c++ != cols_minus1); + } + while (r++ != rows_minus1); + return result; +} + +/** + * Create the matrix to use for encoding, given the number of + * data shards and the number of total shards. + * + * The top square of the matrix is guaranteed to be an identity + * matrix, which means that the data shards are unchanged after + * encoding. + */ +static Matrix buildMatrix(uint8_t dataShards, uint8_t totalShardsMinus1) +{ + if (!totalShardsMinus1) + return Matrix(0, 0); // Fake one, it will be trashed + + // Start with a Vandermonde matrix. This matrix would work, + // in theory, but doesn't have the property that the data + // shards are unchanged after encoding. + Matrix v = vandermonde(totalShardsMinus1, dataShards); + + // Multiple by the inverse of the top square of the matrix. + // This will make the top square be the identity matrix, but + // preserve the property that any square subset of rows is + // invertible. + Matrix top = v.submatrix(0, 0, dataShards, dataShards); + top.invert(); + return v.times(top); +} + +ReedSolomon::ReedSolomon(uint8_t dataShardCount_, uint8_t parityShardCount_) : + dataShardCount{ dataShardCount_ }, + parityShardCount{ parityShardCount_ }, + totalShardCountMinus1{ !dataShardCount || !parityShardCount || dataShardCount + parityShardCount > 256 ? static_cast(0) : static_cast(dataShardCount + parityShardCount - 1) }, + matrixRows{ new const uint8_t * [parityShardCount] }, + matrix{ buildMatrix(dataShardCount, totalShardCountMinus1) } +{ + if (!totalShardCountMinus1) + return; + + // Fill parity_rows + for (uint8_t i = 0; i < parityShardCount; i++) + matrixRows[i] = matrix.getRow(dataShardCount + i); +} + +ReedSolomon::~ReedSolomon() +{ + delete[] matrixRows; +} + +static size_t singleInstructionMultipleData = 0; +#if defined(__x86_64) || defined(_M_AMD64) + #if defined(_MSC_VER) + inline void cpuid(unsigned int cpuInfo[4], unsigned int id) { __cpuid((int*)cpuInfo, (int)id); } + #else + inline void cpuid(unsigned int cpuInfo[4], unsigned int id) { __get_cpuid(id, &cpuInfo[0], &cpuInfo[1], &cpuInfo[2], &cpuInfo[3]); } + #endif +#endif +bool ReedSolomon::setSingleInstructionMultipleData(uint8_t exponent) +{ + if (exponent == (uint8_t )-1) + { + /* + if (setSingleInstructionMultipleData(6)) + return true; + if (setSingleInstructionMultipleData(5)) + return true; + if (setSingleInstructionMultipleData(4)) + return true; + */ + if (setSingleInstructionMultipleData(0)) + return true; + return false; + } + + #if defined(__x86_64) || defined(_M_AMD64) + unsigned int cpuInfo[4]; + #endif + + switch (exponent) + { + case 0: break; // No SIMD, supported everywhere + case 4: + #if defined(__x86_64) || defined(_M_AMD64) + break; // SSE3 everywhere + #else + return false; + #endif + case 5: + #if defined(__x86_64) || defined(_M_AMD64) + cpuid(cpuInfo, 7); + if (cpuInfo[1] && (1 << 5)) + break; // AVX2 supported + #else + return false; + #endif + case 6: + #if defined(__x86_64) || defined(_M_AMD64) + cpuid(cpuInfo, 7); + if (cpuInfo[1] && ((1 << 16) | (1 << 30))) + break; // AVX512F + AVX512BW supported + #else + return false; + #endif + default: + return false; // Supported nowhere + } + + singleInstructionMultipleData = exponent; + return true; +} + +static int processedBufferSize = 4096; +bool ReedSolomon::setProcessedBufferSize(size_t bufferSize) +{ + processedBufferSize = static_cast(bufferSize); + return true; +} + +static size_t processedOrder = 0; +bool ReedSolomon::setProcessedOrder(uint8_t order) +{ + processedOrder = order; + return true; +} + +void ReedSolomon::encode(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const +{ + // Build the array of data/parity shards + auto dataShards = &shards[0]; + auto parityShards = &shards[dataShardCount]; + + // Do the coding + process(matrixRows, dataShards, parityShards, parityShardCount, shardByteSize); +} + +bool ReedSolomon::verify(uint8_t const* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const +{ + // Build the array of data/parity shards + auto dataShards = &shards[0]; + auto parityShards = &shards[dataShardCount]; + + // Create fake parity shards + uint8_t* newParityShardsPointer = new uint8_t[parityShardCount * shardByteSize]; + memset(newParityShardsPointer, 0, parityShardCount * shardByteSize); + uint8_t** newParityShards = new uint8_t * [parityShardCount]; + for (uint8_t i = 0; i < parityShardCount; i++) + newParityShards[i] = &newParityShardsPointer[i * shardByteSize]; + + // Do the coding as if there is no parity shards in the input + process(matrixRows, dataShards, newParityShards, parityShardCount, shardByteSize); + + bool ok = true; + for (uint8_t r = 0; r < parityShardCount; r++) + if (memcmp(newParityShards[r], parityShards[r], shardByteSize)) + ok = false; + + delete[] newParityShards; + delete[] newParityShardsPointer; + return ok; + +} + +bool ReedSolomon::repair(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize, std::vector shardPresent) const +{ + // Integrity check + if (shardPresent.size() != totalShardCountMinus1 + 1) + return false; + + // Quick check: are all of the shards present? If so, there's nothing to do. + uint16_t number_present = 0; + { + uint8_t i = 0; + do + { + if (shardPresent[i]) + number_present++; + } + while (i++ != totalShardCountMinus1); + } + if (number_present == totalShardCountMinus1 + 1) + { + // All of the shards data data. We don't need to do anything. + return true; + } + if (number_present < dataShardCount) + { + // Not enough valid shards. We don't need to do anything. + return false; + } + + // Init invalid shard content to 0, + // needed for process() + { + uint8_t i = 0; + do + { + if (!shardPresent[i]) + memset(shards[i], 0, shardByteSize); + } + while (i++ != totalShardCountMinus1); + } + + // Pull out the rows of the matrix that correspond to the + // shards that we have and build a square matrix. This + // matrix could be used to generate the shards that we have + // from the original data. + // + // Also, pull out an array holding just the shards that + // correspond to the rows of the submatrix. These shards + // will be the input to the decoding process that re-creates + // the missing data shards. + Matrix dataDecodeMatrix{ static_cast(dataShardCount - 1), static_cast(dataShardCount - 1) }; + const uint8_t ** validDataShards = new const uint8_t * [dataShardCount]; + uint8_t validDataShardCount = 0; + { + uint8_t r = 0; + do + { + if (shardPresent[r]) + { + for (uint8_t c = 0; c < dataShardCount; ++c) + dataDecodeMatrix.set(validDataShardCount, c, matrix.get(r, c)); + validDataShards[validDataShardCount] = shards[r]; + validDataShardCount++; + } + } + while (r++ <= totalShardCountMinus1 && validDataShardCount < dataShardCount); + } + + // Invert the matrix, so we can go from the encoded shards + // back to the original data. Then pull out the row that + // generates the shard that we want to decode. Note that + // since this matrix maps back to the orginal data, it can + // be used to create a data shard, but not a parity shard. + dataDecodeMatrix.invert(); + + // Re-create any data shards that were missing. + // + // The input to the coding is all of the shards we actually + // have, and the output is the missing data shards. The computation + // is done using the special decode matrix we just built. + uint8_t** newParityShards = new uint8_t * [parityShardCount]; + const uint8_t** validMatrixRows = new const uint8_t * [parityShardCount]; + uint8_t newParityShardCount = 0; + for (uint8_t i = 0; i < dataShardCount; i++) + { + if (!shardPresent[i]) + { + newParityShards[newParityShardCount] = shards[i]; + validMatrixRows[newParityShardCount] = dataDecodeMatrix.getRow(i); + newParityShardCount++; + } + } + process(validMatrixRows, validDataShards, newParityShards, newParityShardCount, shardByteSize); + + // Now that we have all of the data shards intact, we can + // compute any of the parity that is missing. + // + // The input to the coding is ALL of the data shards, including + // any that we just calculated. The output is whichever of the + // data shards were missing. + newParityShardCount = 0; + { + uint16_t i = dataShardCount; + do + { + if (!shardPresent[i]) + { + newParityShards[newParityShardCount] = shards[i]; + validMatrixRows[newParityShardCount] = matrixRows[i - dataShardCount]; + newParityShardCount++; + } + } + while (i++ != totalShardCountMinus1); + } + const uint8_t* __restrict__ * __restrict__ dataShards = const_cast(&shards[0]); + process(validMatrixRows, dataShards, newParityShards, newParityShardCount, shardByteSize); + + delete[] validDataShards; + delete[] newParityShards; + delete[] validMatrixRows; + + return true; +} + +/** + * Multiplies a subset of rows from a coding matrix by a full set of + * input shards to produce some output shards. + * + * @param validMatrixRows The rows from the matrix to use. + * @param validDataShards An array of byte arrays, each of which is one input shard. + * The inputs array may have extra buffers after the ones + * that are used. They will be ignored. The number of + * inputs used is determined by the length of the + * each matrix row. + * @param newParityShards Byte arrays where the computed shards are stored. The + * outputs array may also have extra, unused, elements + * at the end. The number of outputs computed, and the + * number of matrix rows used, is determined by + * outputCount. + * @param newParityShardCount The number of newParityShards to compute. + * @param shardByteSize The number of bytes to process. + */ +void ReedSolomon::process(uint8_t const* __restrict__ const* __restrict__ const validMatrixRows, uint8_t const* __restrict__ const* __restrict__ const validDataShards, uint8_t* __restrict__ const* __restrict__ const newParityShards, uint8_t newParityShardCount, size_t shardByteSize) const +{ + size_t loop1, loop2, loop3; + switch (processedOrder) + { + case 3 : + loop1 = shardByteSize / processedBufferSize; + shardByteSize = processedBufferSize; + loop2 = dataShardCount; + loop3 = newParityShardCount; + break; + case 2: + loop1 = shardByteSize / processedBufferSize; + shardByteSize = processedBufferSize; + loop2 = newParityShardCount; + loop3 = dataShardCount; + break; + case 1 : + loop1 = dataShardCount; + loop2 = newParityShardCount; + loop3 = 1; + break; + default : + loop1 = newParityShardCount; + loop2 = dataShardCount; + loop3 = 1; + break; + } + + for (int l1 = 0; l1 < loop1; l1++) + for (int l2 = 0; l2 < loop2; l2++) + for (int l3 = 0; l3 < loop3; l3++) + { + int r, c, o; + switch (processedOrder) + { + case 3 : + r = l3; + c = l2; + o = l1 * processedBufferSize; + break; + case 2: + r = l2; + c = l3; + o = l1 * processedBufferSize; + break; + case 1 : + r = l2; + c = l1; + o = 0; + break; + default : + r = l1; + c = l2; + o = 0; + break; + } + + uint8_t matrixValue = validMatrixRows[r][c]; + const uint8_t* __restrict__ dataShard = validDataShards[c] + o; + uint8_t * __restrict__ const newParityShard = newParityShards[r] + o; + + if (singleInstructionMultipleData == 0) + { + for (size_t i = 0; i < shardByteSize; i++) + newParityShard[i] ^= galois.MultiplicationTable[matrixValue][dataShard[i]]; + continue; + } + + /* + size_t end = shardByteSize & (~((size_t(1) << singleInstructionMultipleData) - 1)); + + if (singleInstructionMultipleData == 4) + { + const __m128i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]); + __m128i* __restrict__ output_ptr = reinterpret_cast<__m128i*>(&newParityShard[0]); + const __m128i* __restrict__ input_end = input_ptr + end / 16; + + const __m128i low_table = _mm_loadu_si128(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low)); + const __m128i high_table = _mm_loadu_si128(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High)); + const __m128i mask = _mm_set1_epi8(0x0F); + while (input_ptr < input_end) + { + __m128i input = _mm_loadu_si128(input_ptr++); + __m128i low_indices = _mm_and_si128(input, mask); + __m128i low_parts = _mm_shuffle_epi8(low_table, low_indices); + __m128i high_indices = _mm_and_si128(_mm_srli_epi32(input, 4), mask); // Equivalent to _mm_srli_epi8(input, 4); + __m128i high_parts = _mm_shuffle_epi8(high_table, high_indices); + __m128i intermediate = _mm_xor_si128(low_parts, high_parts); + _mm_storeu_si128(output_ptr, _mm_xor_si128(_mm_loadu_si128(output_ptr), intermediate)); // output = output XOR intermediate + output_ptr++; + } + } + + if (singleInstructionMultipleData == 5) + { + const __m256i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]); + __m256i* __restrict__ output_ptr = reinterpret_cast<__m256i*>(&newParityShard[0]); + const __m256i* __restrict__ input_end = input_ptr + end / 32; + + const __m256i low_table = _mm256_loadu_si256(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low)); + const __m256i high_table = _mm256_loadu_si256(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High)); + const __m256i mask = _mm256_set1_epi8(0x0F); + while (input_ptr < input_end) + { + __m256i input = _mm256_loadu_si256(input_ptr++); + __m256i low_indices = _mm256_and_si256(input, mask); + __m256i low_parts = _mm256_shuffle_epi8(low_table, low_indices); + __m256i high_indices = _mm256_and_si256(_mm256_srli_epi32(input, 4), mask); // Equivalent to _mm256_srli_epi8(input, 4); + __m256i high_parts = _mm256_shuffle_epi8(high_table, high_indices); + __m256i intermediate = _mm256_xor_si256(low_parts, high_parts); + _mm256_storeu_si256(output_ptr, _mm256_xor_si256(_mm256_loadu_si256(output_ptr), intermediate)); // output = output XOR intermediate + output_ptr++; + } + } + + if (singleInstructionMultipleData == 6) + { + const __m512i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]); + __m512i* __restrict__ output_ptr = reinterpret_cast<__m512i*>(&newParityShard[0]); + const __m512i* __restrict__ input_end = input_ptr + end / 64; + + const __m512i low_table = _mm512_loadu_si512(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low)); + const __m512i high_table = _mm512_loadu_si512(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High)); + const __m512i mask = _mm512_set1_epi8(0x0F); + while (input_ptr < input_end) + { + __m512i input = _mm512_loadu_si512(input_ptr++); + __m512i low_indices = _mm512_and_si512(input, mask); + __m512i low_parts = _mm512_shuffle_epi8(low_table, low_indices); + __m512i high_indices = _mm512_and_si512(_mm512_srli_epi32(input, 4), mask); // Equivalent to _mm512_srli_epi8(input, 4); + __m512i high_parts = _mm512_shuffle_epi8(high_table, high_indices); + __m512i intermediate = _mm512_xor_si512(low_parts, high_parts); + _mm512_storeu_si512(output_ptr, _mm512_xor_si512(_mm512_loadu_si512(output_ptr), intermediate)); // output = output XOR intermediate + output_ptr++; + } + } + + for (; end < shardByteSize; end++) + newParityShard[end] ^= galois.MultiplicationTable[matrixValue][dataShard[end]]; + */ + } +} diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h new file mode 100644 index 00000000..3e3ec59d --- /dev/null +++ b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h @@ -0,0 +1,129 @@ +/* Copyright (c) MediaArea.net SARL. + * + * Use of this source code is governed by a MIT-style license that can + * be found in the License.html file in the root of the source tree. + */ + + //--------------------------------------------------------------------------- +#ifndef ReedSolomonH +#define ReedSolomonH +//--------------------------------------------------------------------------- + +//--------------------------------------------------------------------------- +#include "Reed-Solomon/Matrix.h" +#include +using size_t = decltype(sizeof 1); +//--------------------------------------------------------------------------- + +//--------------------------------------------------------------------------- +#if defined (_MSC_VER) +#define __restrict__ __restrict +#endif +//--------------------------------------------------------------------------- + +/** + * Reed-Solomon Coding over 8-bit values. + */ + +class ReedSolomon +{ +public: + /** + * Initializes a new encoder/decoder. + */ + ReedSolomon(uint8_t dataShardCount_, uint8_t parityShardCount_); + ~ReedSolomon(); + + /** + * Set SIMD (Single Instruction Multiple Data) + * @param exponent Exponent of 2 of count of bytes in a single instruction + * e.g. 4 for SSE (128-bit) + * if set to -1, the highest exponent avalaible is selected + * @return false if exponent is not supported, else true + */ + static bool setSingleInstructionMultipleData(uint8_t exponent = -1); + + /** + * Set processed buffer size + * @return false if size is not supported, else true + */ + static bool setProcessedBufferSize(size_t bufferSize); + + /** + * Set encoding order + * @param order 0 = + * + * @return false if order is not supported, else true + */ + static bool setProcessedOrder(uint8_t order); + + /** + * Returns false if constructor failed to initialize. + */ + bool isValid() const { return !totalShardCountMinus1; } + + /** + * Returns the number of data shards. + */ + uint8_t getDataShardCount() const { return dataShardCount; } + + /** + * Returns the number of parity shards. + */ + uint8_t getParityShardCount() const { return parityShardCount; } + + /** + * Returns the total number of shards. + */ + uint16_t getTotalShardCount() const { return totalShardCountMinus1 + 1; } + + /** + * Encodes parity for a set of data shards. + * + * @param shards An array containing data shards followed by parity shards. + * Each shard is a byte array, and they must all be the same + * size. + * @param shardByteSize The number of bytes to encode in each shard. + * + */ + void encode(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const; + + /** + * Returns true if the parity shards contain the right data. + * + * @param shards An array containing data shards followed by parity shards. + * Each shard is a byte array, and they must all be the same + * size. + * @param shardByteSize The number of bytes to check in each shard. + */ + bool verify(uint8_t const* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const; + + /** + * Given a list of shards, some of which contain data, fills in the + * ones that don't have data. + * + * Quickly does nothing if all of the shards are present. + * + * If any shards are missing (based on the flags in shardsPresent), + * the data in those shards is recomputed and filled in. + */ + bool repair(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize, std::vector shardPresent) const; + +private: + void process(uint8_t const* __restrict__ const* __restrict__ const validMatrixRows, uint8_t const* __restrict__ const* __restrict__ const validDataShards, uint8_t* __restrict__ const* __restrict__ const newParityShards, uint8_t newParityShardCount, size_t shardByteSize) const; + +private: + // Members + const uint8_t dataShardCount; + const uint8_t parityShardCount; + const uint8_t totalShardCountMinus1; + + /** + * Rows from the matrix for encoding parity, each one as its own + * byte array to allow for efficient access while encoding. + */ + uint8_t const* __restrict* __restrict__ matrixRows; + Matrix matrix; +}; + +#endif \ No newline at end of file