Skip to content

Commit

Permalink
* hasher bugfixing, Find decade old CRC/SHA1/MD5 bug. Added correct u…
Browse files Browse the repository at this point in the history 
…nicode support. Tested on x86 and X64. TODO: MD4 asm version for X64.
  • Loading branch information
@maxpiva
maxpiva committed Mar 26, 2015
1 parent 1bd5d8a commit ff70a1b
Show file tree
Hide file tree
Showing 6 changed files with 59 additions and 50 deletions.
2 changes: 1 addition & 1 deletion JMMFileHelper/FileHashHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public static void GetVideoInfo(string fileName, ref Hashes hashInfo, ref MediaI

public static Hashes GetHashInfo(string fileName, bool forceRefresh, JMMFileHelper.Hasher.OnHashProgress hashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
return Hasher.CalculateHashes(fileName, hashProgress, true, getCRC32, getMD5, getSHA1);
return Hasher.CalculateHashes(fileName, hashProgress, getCRC32, getMD5, getSHA1);
}

public static MediaInfoResult GetMediaInfo(string fileName, bool forceRefresh)
Expand Down
37 changes: 17 additions & 20 deletions JMMFileHelper/Hasher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace JMMFileHelper
public class Hasher
{
public static Logger logger = LogManager.GetCurrentClassLogger();
public delegate int OnHashProgress([MarshalAs(UnmanagedType.LPTStr)]string strFileName, int nProgressPct);
public delegate int OnHashProgress([MarshalAs(UnmanagedType.LPWStr)]string strFileName, int nProgressPct);

[System.Flags]
internal enum LoadLibraryFlags : uint
Expand Down Expand Up @@ -62,14 +62,14 @@ static Hasher()
}

#region DLL functions
[DllImport("hasher.dll", EntryPoint = "CalculateHashes_AsyncIO", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Auto)]
[DllImport("hasher.dll", EntryPoint = "CalculateHashes_AsyncIO", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)]
private static extern int CalculateHashes_callback_dll(
[MarshalAs(UnmanagedType.LPTStr)] string szFileName,
[MarshalAs(UnmanagedType.LPArray)] byte[] hash,
[MarshalAs(UnmanagedType.LPWStr)]string szFileName,
[MarshalAs(UnmanagedType.LPArray)]byte[] hash,
[MarshalAs(UnmanagedType.FunctionPtr)] OnHashProgress lpHashProgressFunc,
[MarshalAs(UnmanagedType.Bool)] bool getCRC32,
[MarshalAs(UnmanagedType.Bool)] bool getMD5,
[MarshalAs(UnmanagedType.Bool)] bool getSHA1
[MarshalAs(UnmanagedType.Bool)]bool getCRC32,
[MarshalAs(UnmanagedType.Bool)]bool getMD5,
[MarshalAs(UnmanagedType.Bool)]bool getSHA1
);

// Calculates hash immediately (with progress)
Expand Down Expand Up @@ -106,29 +106,25 @@ public static string HashToString(byte[] hash, int start, int length)

public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgress)
{
return CalculateHashes(strPath, onHashProgress, true, true, true, true);
return CalculateHashes(strPath, onHashProgress, true, true, true);
}

public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgress, bool getED2k, bool getCRC32, bool getMD5, bool getSHA1)
public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
Hashes rhash = new Hashes();
if (Finalise.ModuleHandle != IntPtr.Zero)
{
byte[] hash = new byte[56];

// Disable other hashing as it is currently broken when using the DLL

bool gotHash = false;
try
{
if (CalculateHashes_dll(strPath, ref hash, onHashProgress, false, false, false))
if (CalculateHashes_dll(strPath, ref hash, onHashProgress, getCRC32, getMD5, getSHA1))
{
rhash.ed2k = HashToString(hash, 0, 16);
if (!string.IsNullOrEmpty(rhash.ed2k)) gotHash = true;

//if (getCRC32) rhash.crc32 = HashToString(hash, 16, 4);
//if (getMD5) rhash.md5 = HashToString(hash, 20, 16);
//if (getSHA1) rhash.sha1 = HashToString(hash, 36, 20);
rhash.crc32 = HashToString(hash, 16, 4);
rhash.md5 = HashToString(hash, 20, 16);
rhash.sha1 = HashToString(hash, 36, 20);
}
}
catch (Exception ex)
Expand All @@ -139,15 +135,16 @@ public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgre
if (!gotHash)
{
logger.Error("Error using DLL to get hash (Functon returned FALSE), trying C# code instead: {0}", strPath);
return CalculateHashes_here(strPath, onHashProgress, getED2k, getCRC32, getMD5, getSHA1);
return CalculateHashes_here(strPath, onHashProgress, getCRC32, getMD5, getSHA1);
}
return rhash;
}
return CalculateHashes_here(strPath, onHashProgress, getED2k, getCRC32, getMD5, getSHA1);
return CalculateHashes_here(strPath, onHashProgress, getCRC32, getMD5, getSHA1);
}

protected static Hashes CalculateHashes_here(string strPath, OnHashProgress onHashProgress, bool getED2k, bool getCRC32, bool getMD5, bool getSHA1)
public static Hashes CalculateHashes_here(string strPath, OnHashProgress onHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
bool getED2k = true;
logger.Trace("Using C# code to has file: {0}", strPath);

FileStream fs;
Expand Down
16 changes: 8 additions & 8 deletions JMMServer/MainWindow.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2446,31 +2446,31 @@ private static void HashTest()
//string fileName = @"M:\[ Anime Test ]\Code_Geass_R2_Ep14_Geass_Hunt_[720p,BluRay,x264]_-_THORA.mkv";

DateTime start = DateTime.Now;
Hashes hashes = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false, false);
Hashes hashes = Hasher.CalculateHashes(fileName, OnHashProgress, false, false, false);
TimeSpan ts = DateTime.Now - start;

double doubleED2k = ts.TotalMilliseconds;

start = DateTime.Now;
Hashes hashes2 = Hasher.CalculateHashes(fileName, OnHashProgress, true, true, false, false);
Hashes hashes2 = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false);
ts = DateTime.Now - start;

double doubleCRC32 = ts.TotalMilliseconds;

start = DateTime.Now;
Hashes hashes3 = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, true, false);
Hashes hashes3 = Hasher.CalculateHashes(fileName, OnHashProgress, false, true, false);
ts = DateTime.Now - start;

double doubleMD5 = ts.TotalMilliseconds;

start = DateTime.Now;
Hashes hashes4 = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false, true);
Hashes hashes4 = Hasher.CalculateHashes(fileName, OnHashProgress, false, false, true);
ts = DateTime.Now - start;

double doubleSHA1 = ts.TotalMilliseconds;

start = DateTime.Now;
Hashes hashes5 = Hasher.CalculateHashes(fileName, OnHashProgress, true, true, true, true);
Hashes hashes5 = Hasher.CalculateHashes(fileName, OnHashProgress, true, true, true);
ts = DateTime.Now - start;

double doubleAll = ts.TotalMilliseconds;
Expand All @@ -2488,7 +2488,7 @@ private static void HashTest2()
FileInfo fi = new FileInfo(fileName);
string fileSize1 = Utils.FormatByteSize(fi.Length);
DateTime start = DateTime.Now;
Hashes hashes = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false, false);
Hashes hashes = Hasher.CalculateHashes(fileName, OnHashProgress, false, false, false);
TimeSpan ts = DateTime.Now - start;

double doubleFile1 = ts.TotalMilliseconds;
Expand All @@ -2497,7 +2497,7 @@ private static void HashTest2()
fi = new FileInfo(fileName);
string fileSize2 = Utils.FormatByteSize(fi.Length);
start = DateTime.Now;
Hashes hashes2 = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false, false);
Hashes hashes2 = Hasher.CalculateHashes(fileName, OnHashProgress, false, false, false);
ts = DateTime.Now - start;

double doubleFile2 = ts.TotalMilliseconds;
Expand All @@ -2507,7 +2507,7 @@ private static void HashTest2()
fi = new FileInfo(fileName);
string fileSize3 = Utils.FormatByteSize(fi.Length);
start = DateTime.Now;
Hashes hashes3 = Hasher.CalculateHashes(fileName, OnHashProgress, true, false, false, false);
Hashes hashes3 = Hasher.CalculateHashes(fileName, OnHashProgress, false, false, false);
ts = DateTime.Now - start;

double doubleFile3 = ts.TotalMilliseconds;
Expand Down
14 changes: 9 additions & 5 deletions hasher/Hasher.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseOfMfc>false</UseOfMfc>
<CharacterSet>MultiByte</CharacterSet>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
Expand Down Expand Up @@ -223,8 +223,8 @@ ml.exe crc32x86.asm /c /Cx /coff</Command>
</HeaderFileName>
</Midl>
<ClCompile>
<Optimization>MaxSpeed</Optimization>
<InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;HASHER_EXPORTS;HASHLIB_USE_ASM;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<StringPooling>true</StringPooling>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
Expand All @@ -237,6 +237,8 @@ ml.exe crc32x86.asm /c /Cx /coff</Command>
<ProgramDataBaseFileName>.\Release/</ProgramDataBaseFileName>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
Expand Down Expand Up @@ -279,8 +281,8 @@ ml.exe crc32x86.asm /c /Cx /coff</Command>
</HeaderFileName>
</Midl>
<ClCompile>
<Optimization>MaxSpeed</Optimization>
<InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<PreprocessorDefinitions>WIN64;NDEBUG;_WINDOWS;x64;_USRDLL;HASHER_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<StringPooling>true</StringPooling>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
Expand All @@ -293,6 +295,8 @@ ml.exe crc32x86.asm /c /Cx /coff</Command>
<ProgramDataBaseFileName>.\Release_x64/</ProgramDataBaseFileName>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
Expand Down
32 changes: 20 additions & 12 deletions hasher/hasher.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
#define ED2K_CHUNK_SIZE 9728000
#define SIZE_HASH_BUFFER 16384

extern "C" __declspec(dllexport) int __cdecl CalculateHashes_SyncIO(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
extern "C" __declspec(dllexport) int __cdecl CalculateHashes_SyncIO(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
struct _stat64 statFile;
if (_tstat64(pszFile, &statFile) != 0)
if (_wstat64(pszFile, &statFile) != 0)
return 1;
if (statFile.st_size <= 0)
return 6;
Expand All @@ -33,7 +33,7 @@ extern "C" __declspec(dllexport) int __cdecl CalculateHashes_SyncIO(const TCHAR
else
uChunkSizeLast = uChunkSize;

HANDLE hFile = CreateFile(pszFile, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
HANDLE hFile = CreateFileW(pszFile, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
if (hFile == INVALID_HANDLE_VALUE)
return 2;

Expand Down Expand Up @@ -147,11 +147,11 @@ static const unsigned int uChunkSize = 9728000;
#define HIDWORD(l) ((DWORD)(((DWORDLONG)(l)>>32)&0xFFFFFFFF))
#define MAKEDWORDLONG(a,b) ((DWORDLONG)(((DWORD)(a))|(((DWORDLONG)((DWORD)(b)))<<32)))

extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
//get file size
struct _stat64 statFile;
if (_tstat64(pszFile, &statFile) != 0)
if (_wstat64(pszFile, &statFile) != 0)
return 1;
if (statFile.st_size <= 0)
return 6;
Expand All @@ -163,7 +163,7 @@ extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR
nChunks++;

//open file
HANDLE hFile = CreateFile(pszFile, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING,
HANDLE hFile = CreateFileW(pszFile, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING,
FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN, 0);
if (hFile == INVALID_HANDLE_VALUE)
return 2;
Expand Down Expand Up @@ -265,10 +265,13 @@ extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR
{
//update MD4 of current chunk
DWORD dwBytesChunkLeft = (DWORD)(uChunkEnd - iPos);
MD4Engine.Add(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getSHA1) sha1.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getCRC32) crc32.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getMD5) md5.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (dwBytesChunkLeft>0)
{
MD4Engine.Add(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getSHA1) sha1.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getCRC32) crc32.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
if (getMD5) md5.update(blocks[iMaskedReaderPos], dwBytesChunkLeft);
}
//calculate MD4 of chunk
MD4Engine.Finish();
MD4Engine.GetHash((uchar *)&md4);
Expand All @@ -286,9 +289,14 @@ extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR
//update MD4 of next chunk if data was left on the block
DWORD dwBytesChunkNext = dwBytesRead - dwBytesChunkLeft;
if (dwBytesChunkNext > 0)
{
MD4Engine.Add(blocks[iMaskedReaderPos] + dwBytesChunkLeft, dwBytesChunkNext);
if (getSHA1) sha1.update(blocks[iMaskedReaderPos] + dwBytesChunkLeft, dwBytesChunkNext);
if (getCRC32) crc32.update(blocks[iMaskedReaderPos] + dwBytesChunkLeft, dwBytesChunkNext);
if (getMD5) md5.update(blocks[iMaskedReaderPos] + dwBytesChunkLeft, dwBytesChunkNext);
}
}

//prepare for next block
iReaderPos++;
iPos += dwBytesRead;
Expand Down Expand Up @@ -352,7 +360,7 @@ extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR
return nStatus;
}

extern "C" __declspec(dllexport) int __cdecl CalculateHashes(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
extern "C" __declspec(dllexport) int __cdecl CalculateHashes(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
{
return (0 == CalculateHashes_AsyncIO(pszFile, pResult, pHashProgress, getCRC32, getMD5, getSHA1)) ? TRUE : FALSE;
}
8 changes: 4 additions & 4 deletions hasher/hasher.h
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

typedef int (__stdcall *HASHCALLBACK)(const TCHAR * pszFile, int nProgress);
typedef int(__stdcall *HASHCALLBACK)(LPCWSTR pszFile, int nProgress);

extern "C" __declspec(dllexport) int __cdecl CalculateHashes_SsyncIO(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);
extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);
extern "C" __declspec(dllexport) int __cdecl CalculateHashes(const TCHAR * pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);
extern "C" __declspec(dllexport) int __cdecl CalculateHashes_SsyncIO(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);
extern "C" __declspec(dllexport) int __cdecl CalculateHashes_AsyncIO(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);
extern "C" __declspec(dllexport) int __cdecl CalculateHashes(LPCWSTR pszFile, unsigned char * pResult, HASHCALLBACK pHashProgress, bool getCRC32, bool getMD5, bool getSHA1);

0 comments on commit ff70a1b

Please sign in to comment.