Skip to content

Commit

Permalink
Merge pull request #133 from C9Glax/cuttingedge
Browse files Browse the repository at this point in the history 
RateLimits, FileNames, Volume/Chapter Numbers
  • Loading branch information
@C9Glax
C9Glax authored Feb 28, 2024
2 parents 1f24a23 + 597abde commit 9b2a6de
Show file tree
Hide file tree
Showing 16 changed files with 150 additions and 156 deletions.
45 changes: 25 additions & 20 deletions Tranga/Chapter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,28 @@ namespace Tranga;

private static readonly Regex LegalCharacters = new (@"([A-z]*[0-9]* *\.*-*,*\]*\[*'*\'*\)*\(*~*!*)*");
private static readonly Regex IllegalStrings = new(@"Vol(ume)?.?", RegexOptions.IgnoreCase);
private static readonly Regex Digits = new(@"[0-9\.]*");
public Chapter(Manga parentManga, string? name, string? volumeNumber, string chapterNumber, string url)
{
this.parentManga = parentManga;
this.name = name;
this.volumeNumber = volumeNumber ?? "0";
this.chapterNumber = chapterNumber;
this.volumeNumber = volumeNumber is not null ? string.Concat(Digits.Matches(volumeNumber).Select(x => x.Value)) : "0";
this.chapterNumber = string.Concat(Digits.Matches(chapterNumber).Select(x => x.Value));
this.url = url;

string chapterVolNumStr;
if (volumeNumber is not null && volumeNumber.Length > 0)
chapterVolNumStr = $"Vol.{volumeNumber} Ch.{chapterNumber}";
else
chapterVolNumStr = $"Ch.{chapterNumber}";

string chapterName = string.Concat(LegalCharacters.Matches(name ?? ""));
string volStr = volumeNumber is not null ? $"Vol.{volumeNumber} " : "";
string chNumberStr = $"Ch.{chapterNumber} ";
string chNameStr = chapterName.Length > 0 ? $"- {chapterName}" : "";
chNameStr = IllegalStrings.Replace(chNameStr, "");
this.fileName = $"{volStr}{chNumberStr}{chNameStr}";
if (name is not null && name.Length > 0)
{
string chapterName = IllegalStrings.Replace(string.Concat(LegalCharacters.Matches(name)), "");
this.fileName = $"{chapterVolNumStr} - {chapterName}";
}
else
this.fileName = chapterVolNumStr;
}

public override string ToString()
Expand Down Expand Up @@ -81,23 +89,20 @@ public int CompareTo(object? obj)
/// <returns>true if chapter is present</returns>
internal bool CheckChapterIsDownloaded(string downloadLocation)
{
string newFilePath = GetArchiveFilePath(downloadLocation);
if (!Directory.Exists(Path.Join(downloadLocation, parentManga.folderName)))
return false;
FileInfo[] archives = new DirectoryInfo(Path.Join(downloadLocation, parentManga.folderName)).GetFiles();
Regex chapterInfoRex = new(@"Ch\.[0-9.]+");
Regex chapterRex = new(@"[0-9]+(\.[0-9]+)?");

if (File.Exists(newFilePath))
return true;
Regex volChRex = new(@"(?:Vol(?:ume)?\.([0-9]+)\D*)?Ch(?:apter)?\.([0-9]+(?:\.[0-9]+)*)");

string cn = this.chapterNumber;
if (archives.FirstOrDefault(archive => chapterRex.Match(chapterInfoRex.Match(archive.Name).Value).Value == cn) is { } path)
Chapter t = this;
return archives.Select(archive => archive.Name).Any(archiveFileName =>
{
File.Move(path.FullName, newFilePath);
return true;
}
return false;
Match m = volChRex.Match(archiveFileName);
string archiveVolNum = m.Groups[1].Success ? m.Groups[1].Value : "0";
string archiveChNum = m.Groups[2].Value;
return archiveVolNum == t.volumeNumber &&
archiveChNum == t.chapterNumber;
});
}
/// <summary>
/// Creates full file path of chapter-archive
Expand Down
5 changes: 4 additions & 1 deletion Tranga/Jobs/JobJsonConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis
jo.GetValue("parentJobId")!.Value<string?>());
}else if ((jo.ContainsKey("jobType") && jo["jobType"]!.Value<byte>() == (byte)Job.JobType.DownloadNewChaptersJob) || jo.ContainsKey("translatedLanguage"))//TODO change to jobType
{
DateTime lastExecution = jo.GetValue("lastExecution") is {} le
? le.ToObject<DateTime>()
: DateTime.UnixEpoch; //TODO do null checks on all variables
return new DownloadNewChapters(this._clone,
jo.GetValue("mangaConnector")!.ToObject<MangaConnector>(JsonSerializer.Create(new JsonSerializerSettings()
{
Expand All @@ -47,7 +50,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object? exis
}
}))!,
jo.GetValue("manga")!.ToObject<Manga>(),
jo.GetValue("lastExecution")!.ToObject<DateTime>(),
lastExecution,
jo.GetValue("recurring")!.Value<bool>(),
jo.GetValue("recurrenceTime")!.ToObject<TimeSpan?>(),
jo.GetValue("parentJobId")!.Value<string?>());
Expand Down
23 changes: 10 additions & 13 deletions Tranga/MangaConnectors/Bato.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,10 @@ namespace Tranga.MangaConnectors;

public class Bato : MangaConnector
{

public Bato(GlobalBase clone) : base(clone, "Bato")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{1, 60}
});
this.downloadClient = new HttpDownloadClient(clone);
}

public override Manga[] GetManga(string publicationTitle = "")
Expand All @@ -21,7 +19,7 @@ public override Manga[] GetManga(string publicationTitle = "")
string sanitizedTitle = string.Join(' ', Regex.Matches(publicationTitle, "[A-z]*").Where(m => m.Value.Length > 0)).ToLower();
string requestUrl = $"https://bato.to/v3x-search?word={sanitizedTitle}&lang=en";
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Manga>();

Expand All @@ -43,8 +41,7 @@ public override Manga[] GetManga(string publicationTitle = "")

public override Manga? GetMangaFromUrl(string url)
{
RequestResult requestResult =
downloadClient.MakeRequest(url, 1);
RequestResult requestResult = downloadClient.MakeRequest(url, RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
if (requestResult.htmlDocument is null)
Expand Down Expand Up @@ -89,7 +86,7 @@ private Manga ParseSinglePublicationFromHtml(HtmlDocument document, string publi

string posterUrl = document.DocumentNode.SelectNodes("//img")
.First(child => child.GetAttributeValue("data-hk", "") == "0-1-0").GetAttributeValue("src", "").Replace("&amp;", "&");
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, 1);
string coverFileNameInCache = SaveCoverImageToCache(posterUrl, RequestType.MangaCover);

List<HtmlNode> genreNodes = document.DocumentNode.SelectSingleNode("//b[text()='Genres:']/..").SelectNodes("span").ToList();
string[] tags = genreNodes.Select(node => node.FirstChild.InnerText).ToArray();
Expand Down Expand Up @@ -129,7 +126,7 @@ public override Chapter[] GetChapters(Manga manga, string language="en")
string requestUrl = $"https://bato.to/title/{manga.publicationId}";
// Leaving this in for verification if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return Array.Empty<Chapter>();

Expand All @@ -141,7 +138,7 @@ public override Chapter[] GetChapters(Manga manga, string language="en")

private List<Chapter> ParseChaptersFromHtml(Manga manga, string mangaUrl)
{
RequestResult result = downloadClient.MakeRequest(mangaUrl, 1);
RequestResult result = downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)result.statusCode < 200 || (int)result.statusCode >= 300 || result.htmlDocument is null)
{
Log("Failed to load site");
Expand Down Expand Up @@ -184,7 +181,7 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p
string requestUrl = chapter.url;
// Leaving this in to check if the page exists
RequestResult requestResult =
downloadClient.MakeRequest(requestUrl, 1);
downloadClient.MakeRequest(requestUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
Expand All @@ -196,13 +193,13 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p
string comicInfoPath = Path.GetTempFileName();
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());

return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), 1, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
return DownloadChapterImages(imageUrls, chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, "https://mangakatana.com/", progressToken:progressToken);
}

private string[] ParseImageUrlsFromHtml(string mangaUrl)
{
RequestResult requestResult =
downloadClient.MakeRequest(mangaUrl, 1);
downloadClient.MakeRequest(mangaUrl, RequestType.Default);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
return Array.Empty<string>();
Expand Down
2 changes: 1 addition & 1 deletion Tranga/MangaConnectors/ChromiumDownloadClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ private async Task<IBrowser> DownloadBrowser()
});
}

public ChromiumDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public ChromiumDownloadClient(GlobalBase clone) : base(clone)
{
this.browser = DownloadBrowser().Result;
}
Expand Down
32 changes: 12 additions & 20 deletions Tranga/MangaConnectors/DownloadClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,29 @@ namespace Tranga.MangaConnectors;

internal abstract class DownloadClient : GlobalBase
{
private readonly Dictionary<byte, DateTime> _lastExecutedRateLimit;
private readonly Dictionary<byte, TimeSpan> _rateLimit;
private readonly Dictionary<RequestType, DateTime> _lastExecutedRateLimit;

protected DownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone)
protected DownloadClient(GlobalBase clone) : base(clone)
{
this._lastExecutedRateLimit = new();
_rateLimit = new();
foreach (KeyValuePair<byte, int> limit in rateLimitRequestsPerMinute)
_rateLimit.Add(limit.Key, TimeSpan.FromMinutes(1).Divide(limit.Value));
}

internal void SetCustomRequestLimit(byte requestType, int limit)
{
if (_rateLimit.ContainsKey(requestType))
_rateLimit[requestType] = TimeSpan.FromMinutes(1).Divide(limit);
else
_rateLimit.Add(requestType, TimeSpan.FromMinutes(1).Divide(limit));
}

public RequestResult MakeRequest(string url, byte requestType, string? referrer = null, string? clickButton = null)
public RequestResult MakeRequest(string url, RequestType requestType, string? referrer = null, string? clickButton = null)
{
if (_rateLimit.TryGetValue(requestType, out TimeSpan value))
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(value));
else
if (!settings.requestLimits.ContainsKey(requestType))
{
Log("RequestType not configured for rate-limit.");
return new RequestResult(HttpStatusCode.NotAcceptable, null, Stream.Null);
}

TimeSpan rateLimitTimeout = _rateLimit[requestType]
.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));
int rateLimit = settings.userAgent == TrangaSettings.DefaultUserAgent
? TrangaSettings.DefaultRequestLimits[requestType]
: settings.requestLimits[requestType];

TimeSpan timeBetweenRequests = TimeSpan.FromMinutes(1).Divide(rateLimit);
_lastExecutedRateLimit.TryAdd(requestType, DateTime.Now.Subtract(timeBetweenRequests));

TimeSpan rateLimitTimeout = timeBetweenRequests.Subtract(DateTime.Now.Subtract(_lastExecutedRateLimit[requestType]));

if (rateLimitTimeout > TimeSpan.Zero)
{
Expand Down
3 changes: 1 addition & 2 deletions Tranga/MangaConnectors/HttpDownloadClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ internal class HttpDownloadClient : DownloadClient
Timeout = TimeSpan.FromSeconds(10)
};


public HttpDownloadClient(GlobalBase clone, Dictionary<byte, int> rateLimitRequestsPerMinute) : base(clone, rateLimitRequestsPerMinute)
public HttpDownloadClient(GlobalBase clone) : base(clone)
{
Client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", settings.userAgent);
}
Expand Down
6 changes: 3 additions & 3 deletions Tranga/MangaConnectors/MangaConnector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public void CopyCoverFromCacheToDownloadLocation(Manga manga, int? retries = 1)
/// <param name="fullPath"></param>
/// <param name="requestType">RequestType for Rate-Limit</param>
/// <param name="referrer">referrer used in html request header</param>
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requestType, string? referrer = null)
private HttpStatusCode DownloadImage(string imageUrl, string fullPath, RequestType requestType, string? referrer = null)
{
RequestResult requestResult = downloadClient.MakeRequest(imageUrl, requestType, referrer);

Expand All @@ -217,7 +217,7 @@ private HttpStatusCode DownloadImage(string imageUrl, string fullPath, byte requ
return requestResult.statusCode;
}

protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, byte requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveArchiveFilePath, RequestType requestType, string? comicInfoPath = null, string? referrer = null, ProgressToken? progressToken = null)
{
if (progressToken?.cancellationRequested ?? false)
return HttpStatusCode.RequestTimeout;
Expand Down Expand Up @@ -274,7 +274,7 @@ protected HttpStatusCode DownloadChapterImages(string[] imageUrls, string saveAr
return HttpStatusCode.OK;
}

protected string SaveCoverImageToCache(string url, byte requestType)
protected string SaveCoverImageToCache(string url, RequestType requestType)
{
string filetype = url.Split('/')[^1].Split('?')[0].Split('.')[^1];
string filename = $"{DateTime.Now.Ticks.ToString()}.{filetype}";
Expand Down
34 changes: 9 additions & 25 deletions Tranga/MangaConnectors/MangaDex.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,9 @@
namespace Tranga.MangaConnectors;
public class MangaDex : MangaConnector
{
private enum RequestType : byte
{
Manga,
Feed,
AtHomeServer,
CoverUrl,
Author,
}

public MangaDex(GlobalBase clone) : base(clone, "MangaDex")
{
this.downloadClient = new HttpDownloadClient(clone, new Dictionary<byte, int>()
{
{(byte)RequestType.Manga, 250},
{(byte)RequestType.Feed, 250},
{(byte)RequestType.AtHomeServer, 40},
{(byte)RequestType.CoverUrl, 250},
{(byte)RequestType.Author, 250}
});
this.downloadClient = new HttpDownloadClient(clone);
}

public override Manga[] GetManga(string publicationTitle = "")
Expand All @@ -41,7 +25,7 @@ public override Manga[] GetManga(string publicationTitle = "")
//Request next Page
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", (byte)RequestType.Manga);
$"https://api.mangadex.org/manga?limit={limit}&title={publicationTitle}&offset={offset}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
Expand Down Expand Up @@ -75,7 +59,7 @@ public override Manga[] GetManga(string publicationTitle = "")
public override Manga? GetMangaFromId(string publicationId)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", (byte)RequestType.Manga);
downloadClient.MakeRequest($"https://api.mangadex.org/manga/{publicationId}", RequestType.MangaInfo);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
Expand Down Expand Up @@ -149,7 +133,7 @@ public override Manga[] GetManga(string publicationTitle = "")
string? coverUrl = GetCoverUrl(publicationId, posterId);
string? coverCacheName = null;
if (coverUrl is not null)
coverCacheName = SaveCoverImageToCache(coverUrl, (byte)RequestType.AtHomeServer);
coverCacheName = SaveCoverImageToCache(coverUrl, RequestType.MangaCover);

List<string> authors = GetAuthors(authorIds);

Expand Down Expand Up @@ -216,7 +200,7 @@ public override Chapter[] GetChapters(Manga manga, string language="en")
//Request next "Page"
RequestResult requestResult =
downloadClient.MakeRequest(
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", (byte)RequestType.Feed);
$"https://api.mangadex.org/manga/{manga.publicationId}/feed?limit={limit}&offset={offset}&translatedLanguage%5B%5D={language}", RequestType.MangaDexFeed);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
break;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
Expand Down Expand Up @@ -268,7 +252,7 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p
Log($"Retrieving chapter-info {chapter} {chapterParentManga}");
//Request URLs for Chapter-Images
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", (byte)RequestType.AtHomeServer);
downloadClient.MakeRequest($"https://api.mangadex.org/at-home/server/{chapter.url}?forcePort443=false'", RequestType.MangaDexImage);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
{
progressToken?.Cancel();
Expand All @@ -293,7 +277,7 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p
File.WriteAllText(comicInfoPath, chapter.GetComicInfoXmlString());

//Download Chapter-Images
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), (byte)RequestType.AtHomeServer, comicInfoPath, progressToken:progressToken);
return DownloadChapterImages(imageUrls.ToArray(), chapter.GetArchiveFilePath(settings.downloadLocation), RequestType.MangaImage, comicInfoPath, progressToken:progressToken);
}

private string? GetCoverUrl(string publicationId, string? posterId)
Expand All @@ -307,7 +291,7 @@ public override HttpStatusCode DownloadChapter(Chapter chapter, ProgressToken? p

//Request information where to download Cover
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", (byte)RequestType.CoverUrl);
downloadClient.MakeRequest($"https://api.mangadex.org/cover/{posterId}", RequestType.MangaCover);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return null;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
Expand All @@ -328,7 +312,7 @@ private List<string> GetAuthors(IEnumerable<string> authorIds)
foreach (string authorId in authorIds)
{
RequestResult requestResult =
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", (byte)RequestType.Author);
downloadClient.MakeRequest($"https://api.mangadex.org/author/{authorId}", RequestType.MangaDexAuthor);
if ((int)requestResult.statusCode < 200 || (int)requestResult.statusCode >= 300)
return ret;
JsonObject? result = JsonSerializer.Deserialize<JsonObject>(requestResult.result);
Expand Down
Loading

0 comments on commit 9b2a6de

Please sign in to comment.