Skip to content

Commit

Permalink
More improvements to the import process when re-sorting files. More s…
Browse files Browse the repository at this point in the history 
…trict about HTTP API
  • Loading branch information
@da3dsoul
da3dsoul committed Sep 22, 2017
1 parent 1304743 commit 5561d7a
Show file tree
Hide file tree
Showing 7 changed files with 80 additions and 45 deletions.
4 changes: 2 additions & 2 deletions ...API/v1/Implementations/ShokoServiceImplementation/ShokoServiceImplementation_Utilities.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ public CL_VideoLocal_Renamed RenameAndMoveFile(int videoLocalID, string scriptNa
ret.VideoLocal = null;
ret.Success = false;
ret.NewFileName = errorString;
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return ret;
}
vid.FileName = name;
Expand All @@ -630,7 +630,7 @@ public CL_VideoLocal_Renamed RenameAndMoveFile(int videoLocalID, string scriptNa
ret.NewFileName = $"ERROR: {ex.Message}";
ret.Success = false;
}
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return ret;
}

Expand Down
16 changes: 11 additions & 5 deletions Shoko.Server/AniDBHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1028,13 +1028,19 @@ public SVR_AniDB_Anime GetAnimeInfoHTTP(ISession session, int animeID, bool forc
SVR_AniDB_Anime anime;
ISessionWrapper sessionWrapper = session.Wrap();

anime = RepoFactory.AniDB_Anime.GetByAnimeID(sessionWrapper, animeID);
bool skip = true;
if (forceRefresh)
skip = false;
else
bool animeRecentlyUpdated = false;
if (anime != null)
{
anime = RepoFactory.AniDB_Anime.GetByAnimeID(sessionWrapper, animeID);
if (anime == null) skip = false;
TimeSpan ts = DateTime.Now - anime.DateTimeUpdated;
if (ts.TotalHours < 4) animeRecentlyUpdated = true;
}
if (!animeRecentlyUpdated)
{
if (forceRefresh)
skip = false;
else if (anime == null) skip = false;
}

if (skip)
Expand Down
19 changes: 14 additions & 5 deletions Shoko.Server/Commands/Import/CommandRequest_HashFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ private SVR_VideoLocal_Place ProcessFile_LocalInfo()
{
numAttempts++;
Thread.Sleep(1000);
Console.WriteLine($"Attempt # {numAttempts}");
Console.WriteLine($@"Attempt # {numAttempts}");
}

// if we failed to access the file, get ouuta here
Expand Down Expand Up @@ -181,18 +181,22 @@ private SVR_VideoLocal_Place ProcessFile_LocalInfo()

if (vlocalplace.FullServerPath == null)
{
if (vlocal.Places.Count == 1) RepoFactory.VideoLocal.Delete(vlocal);
if (vlocal.Places.Count == 1)
{
RepoFactory.VideoLocal.Delete(vlocal);
vlocal = null;
}
RepoFactory.VideoLocalPlace.Delete(vlocalplace);
vlocalplace = null;
vlocal = null;
} else if (ForceHash)
}
if (vlocal != null && ForceHash)
{
vlocal.FileSize = filesize;
vlocal.DateTimeUpdated = DateTime.Now;
}
}

if (vlocalplace == null)
if (vlocal == null)
{
logger.Trace("VideoLocal, creating temporary record");
vlocal = new SVR_VideoLocal
Expand All @@ -208,13 +212,18 @@ private SVR_VideoLocal_Place ProcessFile_LocalInfo()
IsIgnored = 0,
IsVariation = 0
};
}

if (vlocalplace == null)
{
vlocalplace = new SVR_VideoLocal_Place
{
FilePath = filePath,
ImportFolderID = nshareID,
ImportFolderType = folder.ImportFolderType
};
// Male sure we have an ID
RepoFactory.VideoLocalPlace.Save(vlocalplace);
}

// check if we need to get a hash this file
Expand Down
58 changes: 37 additions & 21 deletions Shoko.Server/Commands/Import/CommandRequest_ProcessFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Shoko.Models.Server;
using Shoko.Server.Commands.AniDB;
using NutzCode.CloudFileSystem;
using Pri.LongPath;
using Shoko.Commons.Queue;
using Shoko.Models.Queue;
using Shoko.Server.Models;
Expand Down Expand Up @@ -113,36 +114,49 @@ private void ProcessFile_AniDB(SVR_VideoLocal vidLocal)
{
// get info from AniDB
logger.Debug("Getting AniDB_File record from AniDB....");
Raw_AniDB_File fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vidLocal);
if (fileInfo != null)
{
// check if we already have a record
aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);

if (aniFile == null)
aniFile = new SVR_AniDB_File();
// check if we already have a record
aniFile = RepoFactory.AniDB_File.GetByHashAndFileSize(vidLocal.Hash, vlocal.FileSize);

if (aniFile == null)
{
aniFile = new SVR_AniDB_File();
ForceAniDB = true;
}

SVR_AniDB_File.Populate(aniFile, fileInfo);
if (ForceAniDB)
{
Raw_AniDB_File fileInfo = ShokoService.AnidbProcessor.GetFileInfo(vidLocal);
if (fileInfo != null)
{
SVR_AniDB_File.Populate(aniFile, fileInfo);
if (!string.IsNullOrEmpty(fileInfo.OtherEpisodesRAW))
{
string[] epIDs = fileInfo.OtherEpisodesRAW.Split(',');
foreach (string epid in epIDs)
{
if (!int.TryParse(epid, out int id)) continue;
CommandRequest_GetEpisode cmdEp = new CommandRequest_GetEpisode(id);
cmdEp.Save();
}
}
}
else aniFile = null;
}

if (aniFile != null)
{
//overwrite with local file name
string localFileName = vidLocal.FileName;
string localFileName = vidLocal.GetBestVideoLocalPlace().FullServerPath;
localFileName = !string.IsNullOrEmpty(localFileName)
? Path.GetFileName(localFileName)
: vidLocal.FileName;
aniFile.FileName = localFileName;

RepoFactory.AniDB_File.Save(aniFile, false);
aniFile.CreateLanguages();
aniFile.CreateCrossEpisodes(localFileName);

if (!string.IsNullOrEmpty(fileInfo.OtherEpisodesRAW))
{
string[] epIDs = fileInfo.OtherEpisodesRAW.Split(',');
foreach (string epid in epIDs)
{
if (!int.TryParse(epid, out int id)) continue;
CommandRequest_GetEpisode cmdEp = new CommandRequest_GetEpisode(id);
cmdEp.Save();
}
}

animeID = aniFile.AnimeID;
}
}
Expand Down Expand Up @@ -170,12 +184,14 @@ private void ProcessFile_AniDB(SVR_VideoLocal vidLocal)
vidLocal.ED2KHash);
return;
}
string fileName = vidLocal.GetBestVideoLocalPlace().FullServerPath;
fileName = !string.IsNullOrEmpty(fileName) ? Path.GetFileName(fileName) : vidLocal.FileName;
foreach (Shoko.Models.Azure.Azure_CrossRef_File_Episode xref in xrefs)
{
CrossRef_File_Episode xrefEnt = new CrossRef_File_Episode
{
Hash = vidLocal.ED2KHash,
FileName = vidLocal.FileName,
FileName = fileName,
FileSize = vidLocal.FileSize,
CrossRefSource = (int)CrossRefSource.WebCache,
AnimeID = xref.AnimeID,
Expand Down
16 changes: 8 additions & 8 deletions Shoko.Server/Models/SVR_VideoLocal_Place.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ private bool RenameFile()
{
logger.Info(
$"Renaming file FAILED! From ({fullFileName}) to ({newFullName}) - {r?.Error ?? "Result is null"}");
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return false;
}

Expand All @@ -131,7 +131,7 @@ private bool RenameFile()
if (tup == null)
{
logger.Error($"Unable to LOCATE file {newFullName} inside the import folders");
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return false;
}

Expand Down Expand Up @@ -178,7 +178,7 @@ private bool RenameFile()
logger.Info($"Renaming file FAILED! From ({fullFileName}) to ({newFullName}) - {ex.Message}");
logger.Error(ex, ex.ToString());
}
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return true;
}

Expand Down Expand Up @@ -693,8 +693,8 @@ public string MoveWithResultString(FileSystemResult<IObject> fileSystemResult, s
{
logger.Error("Unable to MOVE file: {0} to {1} error {2}", FullServerPath,
newFullServerPath, fr?.Error ?? "No Error String");
ShokoServer.StartWatchingFiles();
return "ERROR: " + fr?.Error ?? "Error moving filebut no error string";
ShokoServer.StartWatchingFiles(false);
return "ERROR: " + (fr?.Error ?? "Error moving file, but no error string");
}

string originalFileName = FullServerPath;
Expand Down Expand Up @@ -746,7 +746,7 @@ public string MoveWithResultString(FileSystemResult<IObject> fileSystemResult, s
if (dd != null && dd.IsOk && dd.Result is IDirectory)
RecursiveDeleteEmptyDirectories((IDirectory) dd.Result, true);
}
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return newFolderPath;
}

Expand Down Expand Up @@ -905,7 +905,7 @@ private bool MoveFileIfRequired()
{
logger.Error("Unable to MOVE file: {0} to {1} error {2}", FullServerPath,
newFullServerPath, fr?.Error ?? "No Error String");
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
return false;
}

Expand Down Expand Up @@ -958,7 +958,7 @@ private bool MoveFileIfRequired()
if (dd != null && dd.IsOk && dd.Result is IDirectory)
RecursiveDeleteEmptyDirectories((IDirectory) dd.Result, true);
}
ShokoServer.StartWatchingFiles();
ShokoServer.StartWatchingFiles(false);
}
}
catch (Exception ex)
Expand Down
4 changes: 4 additions & 0 deletions Shoko.Server/Repositories/Cached/VideoLocal_PlaceRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ public List<SVR_VideoLocal_Place> GetByFilePathAndImportFolderType(string filePa

public override void Delete(SVR_VideoLocal_Place obj)
{
// Remove associated duplicate file records
var dups = RepoFactory.DuplicateFile.GetByFilePathAndImportFolder(obj.FilePath, obj.ImportFolderID);
if (dups != null && dups.Count > 0) dups.ForEach(RepoFactory.DuplicateFile.Delete);

base.Delete(obj);
foreach (SVR_AnimeEpisode ep in obj.VideoLocal.GetAnimeEpisodes())
{
Expand Down
8 changes: 4 additions & 4 deletions Shoko.Server/ShokoServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1351,7 +1351,7 @@ static void AutoUpdateTimer_Elapsed(object sender, ElapsedEventArgs e)
Importer.SendUserInfoUpdate(false);
}

public static void StartWatchingFiles()
public static void StartWatchingFiles(bool log = true)
{
StopWatchingFiles();
StopCloudWatchTimer();
Expand All @@ -1361,13 +1361,13 @@ public static void StartWatchingFiles()
{
try
{
if (share.FolderIsWatched)
if (share.FolderIsWatched && log)
{
logger.Info($"Watching ImportFolder: {share.ImportFolderName} || {share.ImportFolderLocation}");
}
if (share.CloudID == null && Directory.Exists(share.ImportFolderLocation) && share.FolderIsWatched)
{
logger.Info($"Parsed ImportFolderLocation: {share.ImportFolderLocation}");
if (log) logger.Info($"Parsed ImportFolderLocation: {share.ImportFolderLocation}");
RecoveringFileSystemWatcher fsw = new RecoveringFileSystemWatcher
{
Path = share.ImportFolderLocation
Expand All @@ -1385,7 +1385,7 @@ public static void StartWatchingFiles()
}
else if (!share.FolderIsWatched)
{
logger.Info("ImportFolder found but not watching: {0} || {1}", share.ImportFolderName,
if (log) logger.Info("ImportFolder found but not watching: {0} || {1}", share.ImportFolderName,
share.ImportFolderLocation);
}
}
Expand Down

0 comments on commit 5561d7a

Please sign in to comment.