Skip to content

Commit

Permalink
Massively improve start time for cached archives with 1000's of files (…
Browse files Browse the repository at this point in the history 
…fix #10)
  • Loading branch information
@fraganator
fraganator committed Jan 8, 2022
1 parent fabe7e2 commit 8da8c6a
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 29 deletions.
21 changes: 16 additions & 5 deletions src/ArchiveCacheManager/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,16 +113,27 @@ static void ExtractArchive(string[] args)
// Any other path is likely to be either Metadata, or some other non-game archive
if (outputPath.EndsWith(@"7-Zip\Temp", StringComparison.InvariantCultureIgnoreCase))
{
// MinArchiveSize is megabytes, multiply to convert to bytes
// TODO: Check if archive in cache first before anything else. Avoids unnecessary calls to 7z.
if (Archive.DecompressedSize > Config.MinArchiveSize * 1048576)
// Check if already cached to avoid calculating the decompressed archive size
if (CacheManager.ArchiveInCache())
{
// Calling this function won't actually extract archive as it will check if archive is cached, but will update last play time.
CacheManager.ExtractArchive();
}
// MinArchiveSize is megabytes, multiply to convert to bytes
// TODO: Check if archive in cache first before anything else. Avoids unnecessary calls to 7z.
else
{
Logger.Log("Archive smaller than MinArchiveSize, bypassing extraction to cache.");
Zip.Call7z(args);
// Only called when required, as can be very time consuming when large number of files in archive.
Archive.UpdateDecompressedSize();
if (Archive.DecompressedSize > Config.MinArchiveSize * 1048576)
{
CacheManager.ExtractArchive();
}
else
{
Logger.Log("Archive smaller than MinArchiveSize, bypassing extraction to cache.");
Zip.Call7z(args);
}
}
}
else
Expand Down
6 changes: 5 additions & 1 deletion src/Core/Archive.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,14 @@ public static void Init(string archivePath)
{
mPath = archivePath;
mCachePath = PathUtils.ArchiveCachePath(archivePath);
mDecompressedSize = Zip.GetDecompressedSize(archivePath);

Logger.Log(string.Format("Archive path set to \"{0}\".", mPath));
Logger.Log(string.Format("Archive cache path set to \"{0}\".", mCachePath));
}

public static void UpdateDecompressedSize()
{
mDecompressedSize = Zip.GetDecompressedSize(Path);
Logger.Log(string.Format("Decompressed archive size is {0} bytes.", mDecompressedSize));
}

Expand Down
82 changes: 68 additions & 14 deletions src/Core/CacheManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,35 +162,48 @@ public static void VerifyCacheIntegrity()
/// </summary>
public static void ListArchive()
{
string stdout = string.Empty;
string stderr = string.Empty;
int exitCode = 0;
string selectedFilePath = string.Empty;
string fileList = string.Empty;

if (ArchiveInCache())
{
stdout = ListCacheArchive();
selectedFilePath = "Path = " + Path.Combine(PathUtils.ArchiveCachePath(Archive.Path), launchGameInfo.SelectedFile);
fileList = ListCacheArchive();
}
else
{
Zip.List(Archive.Path, ref stdout, ref stderr, ref exitCode);
selectedFilePath = "Path = " + launchGameInfo.SelectedFile;
fileList = ListFileArchive(ref stderr, ref exitCode);
}

Console.Write(fileList);
Console.Write(stderr);
Environment.ExitCode = exitCode;
}

/// <summary>
/// Lists all of the files in the archive.
/// </summary>
/// <returns>The file list for the archive in "Path = absolute\file\path.ext" format, with one entry per line.</returns>
public static string ListFileArchive(ref string stderr, ref int exitCode)
{
string stdout = string.Empty;
string selectedFilePath = string.Empty;
string fileList = string.Empty;

Zip.ListVerbose(Archive.Path, ref stdout, ref stderr, ref exitCode);
selectedFilePath = "Path = " + launchGameInfo.SelectedFile;

if (!launchGameInfo.SelectedFile.Equals(string.Empty) && stdout.Contains(selectedFilePath))
{
stdout = selectedFilePath;
fileList = selectedFilePath;
Logger.Log(string.Format("Selected individual file from archive \"{0}\".", launchGameInfo.SelectedFile));
}
else
{
stdout = ApplyExtensionPriority(stdout);
fileList = ApplyExtensionPriority(stdout);
}

Console.Write(stdout);
Console.Write(stderr);
Environment.ExitCode = exitCode;
return fileList;
}

/// <summary>
Expand All @@ -203,13 +216,54 @@ public static string ListCacheArchive()
string[] exclude = new string[] { PathUtils.GetArchiveCachePlaytimePath(Archive.CachePath),
PathUtils.GetArchiveCacheGameInfoPath(Archive.CachePath),
PathUtils.GetArchiveCacheExtractingFlagPath(Archive.CachePath) };
string selectedFilePath = string.Empty;

if (!launchGameInfo.SelectedFile.Equals(string.Empty))
{
if (Directory.GetFiles(Archive.CachePath, launchGameInfo.SelectedFile, SearchOption.AllDirectories).Length > 0)
{
fileList = "Path = " + Path.Combine(PathUtils.ArchiveCachePath(Archive.Path), launchGameInfo.SelectedFile);
Logger.Log(string.Format("Selected individual file from archive \"{0}\".", launchGameInfo.SelectedFile));
}
}

foreach (string filePath in Directory.EnumerateFiles(Archive.CachePath, "*", SearchOption.AllDirectories))
if (fileList == string.Empty)
{
if (!exclude.Contains(filePath))
try
{
fileList = string.Format("{0}Path = {1}\r\n", fileList, filePath);
string[] extensionPriority = Config.ExtensionPriority[string.Format(@"{0} \ {1}", launchGameInfo.Emulator, launchGameInfo.Platform)].Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);

// Search the extensions in priority order
foreach (string extension in extensionPriority)
{
List<string> filePaths = Directory.GetFiles(Archive.CachePath, string.Format("*{0}", extension.Trim()), SearchOption.AllDirectories).ToList();

foreach (string ex in exclude)
{
filePaths.Remove(ex);
}

fileList = string.Join("\r\nPath = ", filePaths);
fileList = string.Format("Path = {0}\r\n", fileList);
}
}
catch (KeyNotFoundException)
{

}
}

if (fileList == string.Empty)
{
List<string> filePaths = Directory.GetFiles(Archive.CachePath, "*", SearchOption.AllDirectories).ToList();

foreach (string ex in exclude)
{
filePaths.Remove(ex);
}

fileList = string.Join("\r\nPath = ", filePaths);
fileList = string.Format("Path = {0}\r\n", fileList);
}

return fileList;
Expand Down
30 changes: 21 additions & 9 deletions src/Core/Zip.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,22 @@ public static void Extract(string archivePath, string cachePath, ref string stdo
/// <param name="stderr"></param>
/// <param name="exitCode"></param>
public static void List(string archivePath, ref string stdout, ref string stderr, ref int exitCode)
{
// l = list
// {0} = archive path
string args = string.Format("l \"{0}\"", archivePath);

run7z(args, ref stdout, ref stderr, ref exitCode);
}

/// <summary>
/// Run the 7z list command on the specified archive. Listing contains technical data on every file.
/// </summary>
/// <param name="archivePath"></param>
/// <param name="stdout"></param>
/// <param name="stderr"></param>
/// <param name="exitCode"></param>
public static void ListVerbose(string archivePath, ref string stdout, ref string stderr, ref int exitCode)
{
// This command is a duplicate of that used by LaunchBox.
// l = list
Expand All @@ -81,7 +97,7 @@ public static string[] GetFileList(string archivePath)
List<string> fileList = new List<string>();
bool foundFirstPath = false;

List(archivePath, ref stdout, ref stderr, ref exitCode);
ListVerbose(archivePath, ref stdout, ref stderr, ref exitCode);

if (exitCode == 0)
{
Expand Down Expand Up @@ -134,15 +150,11 @@ public static long GetDecompressedSize(string archivePath)
if (exitCode == 0)
{
// TODO: Replace logic with regex?
string[] stdoutArray = stdout.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
string[] stdoutArray = stdout.Split(new string[] { "------------------------" }, StringSplitOptions.RemoveEmptyEntries);

foreach (string line in stdoutArray)
{
if (line.StartsWith("Size ="))
{
size += Convert.ToInt64(line.Split(" ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)[2]);
}
}
string summary = stdoutArray[stdoutArray.Length - 1];

size = Convert.ToInt64(summary.Split(" ".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)[2]);
}

return size;
Expand Down

0 comments on commit 8da8c6a

Please sign in to comment.