diff --git a/Source/CLI/Global.cpp b/Source/CLI/Global.cpp index c53cea70..94a5208a 100644 --- a/Source/CLI/Global.cpp +++ b/Source/CLI/Global.cpp @@ -242,6 +242,40 @@ int global::SetHash(bool Value) return 0; } +//--------------------------------------------------------------------------- +int global::SetFileOpenMethod(const char* Value) +{ + if (strcmp(Value, "mmap") == 0) + { + FileOpenMethod = filemap::method::mmap; + return 0; + } + if (strcmp(Value, "fstream") == 0) + { + FileOpenMethod = filemap::method::fstream; + return 0; + } + if (strcmp(Value, "fopen") == 0) + { + FileOpenMethod = filemap::method::fopen; + return 0; + } + if (strcmp(Value, "open") == 0) + { + FileOpenMethod = filemap::method::open; + return 0; + } + #if defined(_WIN32) || defined(_WINDOWS) + if (strcmp(Value, "createfile") == 0) + { + FileOpenMethod = filemap::method::createfile; + return 0; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + cerr << "Error: unknown io value '" << Value << "'." << endl; + return 1; +} + //--------------------------------------------------------------------------- int global::SetAll(bool Value) { @@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc) IgnoreLicenseKey = !License.IsSupported_License(); SubLicenseId = 0; SubLicenseDur = 1; + FileOpenMethod = filemap::method::mmap; ShowLicenseKey = false; StoreLicenseKey = false; DisplayCommand = false; @@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc) if (auto Value = SetAcceptFiles()) return Value; } + else if (strcmp(argv[i], "--io") == 0) + { + if (i + 1 == argc) + return Error_Missing(argv[i]); + int Value = SetFileOpenMethod(argv[++i]); + if (Value) + return Value; + } else if (!strcmp(argv[i], "-framerate")) { if (OptionsForOtherFiles) diff --git a/Source/CLI/Global.h b/Source/CLI/Global.h index e06e1f18..76eabd53 100644 --- a/Source/CLI/Global.h +++ b/Source/CLI/Global.h @@ -38,6 +38,7 @@ class global string LicenseKey; uint64_t SubLicenseId; uint64_t SubLicenseDur; + filemap::method FileOpenMethod; bool IgnoreLicenseKey; bool ShowLicenseKey; bool StoreLicenseKey; @@ -100,6 +101,7 @@ class global int SetFrameMd5An(bool Value); int SetFrameMd5FileName(const char* FileName); int SetHash(bool Value); + int SetFileOpenMethod(const char* Value); int SetAll(bool Value); private: diff --git a/Source/CLI/Main.cpp b/Source/CLI/Main.cpp index 53efb45e..84e9f00f 100644 --- a/Source/CLI/Main.cpp +++ b/Source/CLI/Main.cpp @@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos) } //--------------------------------------------------------------------------- -int ParseFile_Compressed(parse_info& ParseInfo) +int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName) { // Init string OutputDirectoryName; @@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo) matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors); M->Quiet = Global.Quiet; M->NoOutputCheck = NoOutputCheck; + M->OpenName = FileOpenName; + M->OpenStyle = Global.FileOpenMethod; if (ParseInfo.ParseFile_Input(*M)) { ReturnValue = 1; @@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos) return 1; // Compressed content - if (int Value = ParseFile_Compressed(ParseInfo)) + if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name)) return Value; if (ParseInfo.IsDetected) return 0; @@ -755,6 +757,7 @@ int main(int argc, const char* argv[]) if (!Value) { // Configure for a 2nd pass + auto FileOpenName = Global.OutputFileName; ParseInfo.Name = NULL; Global.OutputFileName = Global.Inputs[0]; if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes @@ -772,7 +775,7 @@ int main(int argc, const char* argv[]) // Parse (check mode) Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]); Global.Actions.set(Action_Decode, false); // Override config - Value = ParseFile_Compressed(ParseInfo); + Value = ParseFile_Compressed(ParseInfo, &FileOpenName); if (!Value && !ParseInfo.IsDetected) { cout << '\n' << "Error: " << Global.OutputFileName << endl; diff --git a/Source/Lib/Compressed/Matroska/Matroska.cpp b/Source/Lib/Compressed/Matroska/Matroska.cpp index 550397f4..924ef8c8 100644 --- a/Source/Lib/Compressed/Matroska/Matroska.cpp +++ b/Source/Lib/Compressed/Matroska/Matroska.cpp @@ -359,13 +359,16 @@ void matroska::ParseBuffer() // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value { - FileMap->Remap(); + FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024); Buffer = *FileMap; + if (OpenStyle == filemap::method::mmap) + { if (ReversibilityData) ReversibilityData->SetBaseData(Buffer.Data()); for (const auto& TrackInfo_Current : TrackInfo) if (TrackInfo_Current && TrackInfo_Current->ReversibilityData) TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data()); + } Buffer_Offset_LowerLimit = Buffer_Offset; } @@ -376,7 +379,7 @@ void matroska::ParseBuffer() Buffer_Offset = Cluster_Offset; Cluster_Level = (size_t)-1; - FileMap->Remap(); + FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024); Buffer = *FileMap; if (ReversibilityData) ReversibilityData->SetBaseData(Buffer.Data()); @@ -850,6 +853,19 @@ void matroska::Segment_Cluster() Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string()); if (ReversibilityData && !FrameWriter_Template->Compound) InitOutput_Find(); + + Actions[Action_Hash] = false; + + if (!FileMap2) + { + FileMap2 = FileMap; + if (OpenStyle != filemap::method::mmap && OpenName) + { + FileMap = new filemap; + FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024); + Buffer = *FileMap; + } + } } //--------------------------------------------------------------------------- diff --git a/Source/Lib/Utils/FileIO/FileIO.cpp b/Source/Lib/Utils/FileIO/FileIO.cpp index 0c543a94..73cec021 100644 --- a/Source/Lib/Utils/FileIO/FileIO.cpp +++ b/Source/Lib/Utils/FileIO/FileIO.cpp @@ -11,8 +11,11 @@ #include "Lib/Utils/FileIO/FileIO.h" #include #include +#include #if defined(_WIN32) || defined(_WINDOWS) #include "windows.h" + #include + #include #include // File existence #include // Directory creation #define access _access_s @@ -29,10 +32,106 @@ //--------------------------------------------------------------------------- //--------------------------------------------------------------------------- -int filemap::Open_ReadMode(const char* FileName) +struct private_buffered +{ + union f + { + FILE* File; + ifstream* Ifstream; + int Int; + #if defined(_WIN32) || defined(_WINDOWS) + HANDLE Handle; + #endif + }; + f F; + size_t Data_Shift = 0; + size_t MaxSize = 0; +}; + +//--------------------------------------------------------------------------- +int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End) { Close(); + if (NewStyle != method::mmap) + { + Method = NewStyle; + private_buffered* P = new private_buffered; + P->MaxSize = End - Begin; + size_t FileSize; + + switch (Method) + { + default: // case style::fstream: + { + auto F = new ifstream(FileName, ios::binary); + F->seekg(0, F->end); + FileSize = F->tellg(); + F->seekg(Begin, F->beg); + P->F.Ifstream = F; + break; + } + case method::fopen: + { + struct stat Fstat; + if (stat(FileName, &Fstat)) + return 1; + FileSize = Fstat.st_size; + auto F = fopen(FileName, "rb"); + P->F.File = F; + break; + } + case method::open: + { + struct stat Fstat; + if (stat(FileName, &Fstat)) + return 1; + FileSize = Fstat.st_size; + #if defined(_WIN32) || defined(_WINDOWS) + auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD); + #else //defined(_WIN32) || defined(_WINDOWS) + auto F = open(FileName, O_RDONLY); + #endif //defined(_WIN32) || defined(_WINDOWS) + if (F == -1) + return 1; + P->F.Int = F; + break; + } + #if defined(_WIN32) || defined(_WINDOWS) + case method::createfile: + { + DWORD FileSizeHigh; + auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh); + if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value) + && (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode + { + FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow; + } + else + return 1; + if (Begin) + { + LARGE_INTEGER GoTo; + GoTo.QuadPart = Begin; + if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0)) + return 1; + P->Data_Shift = Begin; + } + P->F.Handle = NewFile; + break; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + } + + auto Buffer = new uint8_t[P->MaxSize]; + P->Data_Shift -= P->MaxSize; + AssignBase(Buffer - P->Data_Shift, FileSize); + Private2 = (decltype(Private2))P; + + return Remap(Begin, End); + } + size_t NewSize; #if defined(_WIN32) || defined(_WINDOWS) auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); @@ -99,12 +198,63 @@ inline int munmap_const(const void* addr, size_t length) #pragma GCC diagnostic pop #endif #endif -int filemap::Remap() +int filemap::Remap(size_t Begin, size_t End) { // Special case for 0-byte files if (Empty()) return 0; + if (Method != method::mmap) + { + auto P = (private_buffered*)Private2; + auto Buffer = Data() + P->Data_Shift; + auto Buffer_MaxSize = P->MaxSize; + Begin -= P->Data_Shift; + if (!End) + End = Size(); + End -= P->Data_Shift; + auto Buffer_Middle = Buffer + Begin; + auto Buffer_Middle_Size = Buffer_MaxSize - Begin; + memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size); + P->Data_Shift += Begin; + AssignKeepSizeBase(Buffer - P->Data_Shift); + Buffer += Buffer_Middle_Size; + Buffer_MaxSize -= Buffer_Middle_Size; + + switch (Method) + { + default: // case style::fstream: + { + auto F = P->F.Ifstream; + F->read((char*)Buffer, Buffer_MaxSize); + break; + } + case method::fopen: + { + auto F = P->F.File; + if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1) + return 1; + break; + } + case method::open: + { + auto F = P->F.Int; + read(F, (void*)Buffer, Buffer_MaxSize); + break; + } + #if defined(_WIN32) || defined(_WINDOWS) + case method::createfile: + { + auto F = P->F.Handle; + ReadFile(F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0); + break; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + } + + return 0; + } + // Close previous map if (Data()) { diff --git a/Source/Lib/Utils/FileIO/FileIO.h b/Source/Lib/Utils/FileIO/FileIO.h index 45590707..75be259c 100644 --- a/Source/Lib/Utils/FileIO/FileIO.h +++ b/Source/Lib/Utils/FileIO/FileIO.h @@ -23,19 +23,30 @@ class filemap : public buffer_view ~filemap() { Close(); } // Actions - int Open_ReadMode(const char* FileName); - int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); } + enum class method + { + mmap, + fstream, + fopen, + open, + #if defined(_WIN32) || defined(_WINDOWS) + createfile, + #endif //defined(_WIN32) || defined(_WINDOWS) + }; + int Open_ReadMode(const char* FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}); + int Open_ReadMode(const string& FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}) { return Open_ReadMode(FileName.c_str(), NewMethod, Begin, End); } bool IsOpen() { return Private == (decltype(Private))-1 ? false : true; } - int Remap(); + int Remap(size_t Begin = 0, size_t End = 0); int Close(); private: #if defined(_WIN32) || defined(_WINDOWS) void* Private = (void*)-1; - void* Private2 = (void*)-1; #else //defined(_WIN32) || defined(_WINDOWS) int Private = (int)-1; #endif //defined(_WIN32) || defined(_WINDOWS) + void* Private2 = (void*)-1; + method Method = {}; }; class file diff --git a/Source/Lib/Utils/FileIO/Input_Base.cpp b/Source/Lib/Utils/FileIO/Input_Base.cpp index 1bd39e7f..d21e38a5 100644 --- a/Source/Lib/Utils/FileIO/Input_Base.cpp +++ b/Source/Lib/Utils/FileIO/Input_Base.cpp @@ -38,6 +38,7 @@ input_base::~input_base() bool input_base::Parse(filemap* FileMap_Source, const buffer_view& Buffer_Source, size_t FileSize_Source) { FileMap = FileMap_Source; + FileMap2 = nullptr; FileSize = FileSize_Source == (size_t)-1 ? Buffer_Source.Size() : FileSize_Source; Buffer = Buffer_Source; HashComputed = false; diff --git a/Source/Lib/Utils/FileIO/Input_Base.h b/Source/Lib/Utils/FileIO/Input_Base.h index 0b0383e4..e38d0736 100644 --- a/Source/Lib/Utils/FileIO/Input_Base.h +++ b/Source/Lib/Utils/FileIO/Input_Base.h @@ -76,7 +76,9 @@ class input_base // Config bitset Actions; hashes* Hashes = nullptr; - string* FileName = nullptr; + const string* FileName = nullptr; + const string* OpenName = nullptr; // TODO: merge with FileName + filemap::method OpenStyle = {}; // Parse bool Parse(const buffer_view& Buffer, size_t FileSize = (size_t)-1) { return Parse(nullptr, Buffer, FileSize); } @@ -95,6 +97,7 @@ class input_base virtual void ParseBuffer() = 0; virtual void BufferOverflow() = 0; filemap* FileMap; + filemap* FileMap2; size_t FileSize; buffer_view Buffer; size_t Buffer_Offset;