Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve MD5 parsing #300

Merged
merged 2 commits into from
Oct 29, 2024
Merged

Improve MD5 parsing #300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af53886
Preserve computed data MD5 after file save
g-maxime Sep 28, 2024
49bd1bf
Disable MD5 computation when no needed
g-maxime Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions Source/CLI/CommandLine_Parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,6 @@ CL_OPTION(MD5_Generate)
//---------------------------------------------------------------------------
CL_OPTION(MD5_Verify)
{
C.GenerateMD5=true;
C.VerifyMD5=true;
C.VerifyMD5_Force=true;

Expand All @@ -788,7 +787,6 @@ CL_OPTION(MD5_Verify)
//---------------------------------------------------------------------------
CL_OPTION(MD5_Embed)
{
C.GenerateMD5=true;
C.EmbedMD5=true;

return -2; //Continue
Expand All @@ -797,7 +795,6 @@ CL_OPTION(MD5_Embed)
//---------------------------------------------------------------------------
CL_OPTION(MD5_Embed_Overwrite)
{
C.GenerateMD5=true;
C.EmbedMD5=true;
C.EmbedMD5_AuthorizeOverWritting=true;

Expand Down
24 changes: 0 additions & 24 deletions Source/GUI/Qt/GUI_Main_Menu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1358,11 +1358,6 @@ void GUI_Main::OnMenu_Options_GenerateMD5(bool)
break;
case QMessageBox::No : // No was clicked
C->GenerateMD5=true;
if (C->GenerateMD5==false)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Embed]->setChecked(false);
//OnMenu_Options_EmbedMD5();
}
break;
case QMessageBox::Cancel : // Cancel was clicked
C->GenerateMD5=false;
Expand All @@ -1374,13 +1369,6 @@ void GUI_Main::OnMenu_Options_GenerateMD5(bool)
else
{
C->GenerateMD5=Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Generate]->isChecked();
if (C->GenerateMD5==false)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Verify]->setChecked(false);
//OnMenu_Options_VerifyMD5();
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Embed]->setChecked(false);
//OnMenu_Options_EmbedMD5();
}
}
}

Expand All @@ -1390,13 +1378,6 @@ void GUI_Main::OnMenu_Options_VerifyMD5(bool)
C->VerifyMD5=Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Verify]->isChecked();
if (C->VerifyMD5==true)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Generate]->setChecked(true);
//OnMenu_Options_GenerateMD5();
if (!C->GenerateMD5)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Verify]->setChecked(false);
//OnMenu_Options_VerifyMD5();
}
C->Menu_File_Options_Update();

//Showing
Expand All @@ -1418,11 +1399,6 @@ void GUI_Main::OnMenu_Options_EmbedMD5(bool)
C->EmbedMD5=Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Embed]->isChecked();
if (C->EmbedMD5==true)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Generate]->setChecked(true);
if (!C->GenerateMD5)
{
Menu_Fields_CheckBoxes[Group_MD5*options::MaxCount+Option_MD5_Embed]->setChecked(false);
}
C->Menu_File_Options_Update();
View_Refresh();
}
Expand Down
3 changes: 2 additions & 1 deletion Source/Riff/Riff_Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,8 @@ void Riff_Base::Read_Internal_ReadAllInBuffer ()
if (BytesRead==0)
break; //Read is finished
Global->CS.Enter();
Global->Progress=(float)Global->In.Position_Get()/Global->In.Size_Get();
int64u Skipped=Global->data?Global->data->Size:0;
Global->Progress=(float)(Global->In.Position_Get()-Skipped)/Global->In.Size_Get();
if (Global->Canceling)
{
Global->CS.Leave();
Expand Down
52 changes: 0 additions & 52 deletions Source/Riff/Riff_Chunks_WAVE_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@

//---------------------------------------------------------------------------
#include "Riff/Riff_Chunks.h"
extern "C"
{
#include "MD5/md5.h"
}
#include "ZenLib/Utils.h"
//---------------------------------------------------------------------------

Expand All @@ -39,54 +35,6 @@ void Riff_WAVE_data::Read_Internal ()
Global->data=new Riff_Base::global::chunk_data;
Global->data->File_Offset=Global->In.Position_Get();
Global->data->Size=Chunk.Content.Size;

//MD5
try
{
Chunk.Content.Buffer=new int8u[65536];
}
catch(...)
{
throw exception_read_chunk("Problem during memory allocation");
}

//Reading
if (Global->GenerateMD5)
{
MD5Context MD5;
MD5Init(&MD5);
while(Chunk.Content.Buffer_Offset<Chunk.Content.Size)
{
size_t ToRead=(size_t)Chunk.Content.Size-Chunk.Content.Buffer_Offset;
if (ToRead>65536)
ToRead=65536;
size_t BytesRead=Global->In.Read(Chunk.Content.Buffer, ToRead);
if (BytesRead==0)
break; //Read is finished
Global->CS.Enter();
Global->Progress=(float)Global->In.Position_Get()/Global->In.Size_Get();
if (Global->Canceling)
{
Global->CS.Leave();
throw exception_canceled();
}
Global->CS.Leave();
//SleeperThread::msleep(200);
Chunk.Content.Buffer_Offset+=BytesRead;
MD5Update(&MD5, Chunk.Content.Buffer, (unsigned int)BytesRead);
}
if (Chunk.Content.Buffer_Offset<Chunk.Content.Size)
throw exception_read();
delete[] Chunk.Content.Buffer; Chunk.Content.Buffer=NULL;
Chunk.Content.Buffer_Offset=0;
int8u Digest[16];
MD5Final(Digest, &MD5);
int128u DigestI=BigEndian2int128u(Digest);
Global->MD5Generated=new Riff_Base::global::chunk_strings;
Global->MD5Generated->Strings["md5generated"]=Ztring().From_Number(DigestI, 16).To_UTF8();
while (Global->MD5Generated->Strings["md5generated"].size()<32)
Global->MD5Generated->Strings["md5generated"].insert(Global->MD5Generated->Strings["md5generated"].begin(), '0'); //Padding with 0, this must be a 32-byte string
}
}

//***************************************************************************
Expand Down
105 changes: 100 additions & 5 deletions Source/Riff/Riff_Handler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
#include "ZenLib/File.h"
#include "ZenLib/Dir.h"
#include "TinyXml2/tinyxml2.h"
extern "C"
{
#include "MD5/md5.h"
}

#ifdef MACSTORE
#include "Common/Mac_Helpers.h"
Expand Down Expand Up @@ -503,11 +507,22 @@ bool Riff_Handler::Open_Internal(const string &FileName)
File_IsValid=false;
File_IsCanceled=false;
bool ReturnValue=true;


// Preserve already computed data md5
string MD5Generated;
if (Chunks && Chunks->Global && Chunks->Global->MD5Generated)
MD5Generated = Chunks->Global->MD5Generated->Strings["md5generated"];

//Global info
delete Chunks; Chunks=new Riff();
Chunks->Global->File_Name=Ztring().From_UTF8(FileName);

if (!MD5Generated.empty())
{
Chunks->Global->MD5Generated=new Riff_Base::global::chunk_strings;
Chunks->Global->MD5Generated->Strings["md5generated"]=MD5Generated;
}

//Opening file
if (!File::Exists(Ztring().From_UTF8(FileName)) || !Chunks->Global->In.Open(Ztring().From_UTF8(FileName)))
{
Expand Down Expand Up @@ -549,6 +564,90 @@ bool Riff_Handler::Open_Internal(const string &FileName)
ReturnValue=false;
}

//Compute MD5
if (Chunks->Global->data &&
((Chunks->Global->GenerateMD5 && (!Chunks->Global->MD5Generated || Chunks->Global->MD5Generated->Strings["md5generated"].empty())) ||
(Chunks->Global->VerifyMD5 && (Chunks->Global->MD5Stored && !Chunks->Global->MD5Stored->Strings["md5stored"].empty())) ||
(Chunks->Global->EmbedMD5 && ((!Chunks->Global->MD5Stored || Chunks->Global->MD5Stored->Strings["md5stored"].empty()) || Chunks->Global->EmbedMD5_AuthorizeOverWritting))))
{
size_t Buffer_Offset=0;
int8u* Buffer=nullptr;
try
{
Buffer=new int8u[65536];
}
catch(...)
{
Errors<<Chunks->Global->File_Name.To_UTF8()<<": Problem during memory allocation"<<endl;
PerFile_Error<<"Problem during memory allocation"<<endl;
ReturnValue=false;
}

if (Buffer)
{
if (Chunks->Global->In.GoTo(Chunks->Global->data->File_Offset))
{
MD5Context MD5;
MD5Init(&MD5);

while(Buffer_Offset<Chunks->Global->data->Size)
{
size_t ToRead=(size_t)Chunks->Global->data->Size-Buffer_Offset;
if (ToRead>65536)
ToRead=65536;
size_t BytesRead=Chunks->Global->In.Read(Buffer, ToRead);
if (BytesRead==0)
break; //Read is finished
Buffer_Offset+=BytesRead;

Chunks->Global->CS.Enter();
int64u Skipped=Chunks->Global->data->Size-Buffer_Offset;
Chunks->Global->Progress=(float)(Chunks->Global->In.Size_Get()-Skipped)/Chunks->Global->In.Size_Get();
if (Chunks->Global->Canceling)
{
Errors<<Chunks->Global->File_Name.To_UTF8()<<": canceled"<<endl;
PerFile_Error<<"canceled"<<endl;
File_IsCanceled=true;
Chunks->Global->Canceling=false;
ReturnValue=false;
Chunks->Global->CS.Leave();
break;
}
Chunks->Global->CS.Leave();
MD5Update(&MD5, Buffer, (unsigned int)BytesRead);
}
delete[] Buffer;

if (File_IsCanceled)
{
//Nothing to do
}
else if (Buffer_Offset<Chunks->Global->data->Size)
{
Errors<<Chunks->Global->File_Name.To_UTF8()<<": Problem during reading"<<endl;
PerFile_Error<<"Problem during reading"<<endl;
ReturnValue=false;
}
else
{
int8u Digest[16];
MD5Final(Digest, &MD5);
int128u DigestI=BigEndian2int128u(Digest);
Chunks->Global->MD5Generated=new Riff_Base::global::chunk_strings;
Chunks->Global->MD5Generated->Strings["md5generated"]=Ztring().From_Number(DigestI, 16).To_UTF8();
while (Chunks->Global->MD5Generated->Strings["md5generated"].size()<32)
Chunks->Global->MD5Generated->Strings["md5generated"].insert(Chunks->Global->MD5Generated->Strings["md5generated"].begin(), '0'); //Padding with 0, this must be a 32-byte string
}
}
else
{
Errors<<Chunks->Global->File_Name.To_UTF8()<<": Problem during seeking"<<endl;
PerFile_Error<<"Problem during seeking"<<endl;
ReturnValue=false;
}
}
}

//Cleanup
Chunks->Global->In.Close();

Expand Down Expand Up @@ -1087,16 +1186,12 @@ bool Riff_Handler::Save()

//Loading the new file (we are verifying the integraty of the generated file)
string FileName=Chunks->Global->File_Name.To_UTF8();
bool GenerateMD5_Temp=Chunks->Global->GenerateMD5;
Chunks->Global->GenerateMD5=false;
if (!Open_Internal(FileName) && Chunks==NULL) //There may be an error but file is open (eg MD5 error)
{
Errors<<FileName<<": WARNING, the resulting file can not be validated, file may be CORRUPTED"<<endl;
PerFile_Error<<"WARNING, the resulting file can not be validated, file may be CORRUPTED"<<endl;
Chunks->Global->GenerateMD5=GenerateMD5_Temp;
return false;
}
Chunks->Global->GenerateMD5=GenerateMD5_Temp;

CriticalSectionLocker(Chunks->Global->CS);
Chunks->Global->Progress=1;
Expand Down
Loading