HttpTools: Read Limit, HandleURL: more universal solution for invalid…

… html's
LaineZ · Oct 18, 2024 · a0d236d · a0d236d
1 parent ac9b5a2
commit a0d236d
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 22 deletions.
diff --git a/fs24bot3/Core/HttpTools.cs b/fs24bot3/Core/HttpTools.cs
@@ -1,15 +1,10 @@
-using fs24bot3.Core;
-using HtmlAgilityPack;
-using Newtonsoft.Json;
+using Newtonsoft.Json;
 using Serilog;
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.NetworkInformation;
-using System.Runtime.Serialization;
 using System.Text;
 using System.Threading.Tasks;
 using System.Web;
@@ -22,10 +17,11 @@ public class HttpTools
     readonly CookieContainer Cookies = new CookieContainer();
     public readonly HttpClient Client = new HttpClient();
 
-    public HttpTools()
+    public HttpTools(uint maxContentSize = 8)
     {
         Client.DefaultRequestHeaders.UserAgent.ParseAdd(
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0");
+        Client.MaxResponseContentBufferSize = maxContentSize * 1024 * 1024;
     }
 
     public string RecursiveHtmlDecode(string str)
@@ -157,19 +153,11 @@ public async Task<string> GetTextPlainResponse(string rawurl)
         {
             if (response.Content.Headers.ContentType.MediaType == "text/plain")
             {
-                if (response.Content.Headers.ContentLength.GetValueOrDefault() > 10000)
-                {
-                    throw new InvalidDataException(
-                        $"Ошибка в Content-Length запроса: Слишком большой размер, максимальный размер: 10000 байт");
-                }
-
                 return await response.Content.ReadAsStringAsync();
             }
-            else
-            {
-                throw new InvalidDataException(
-                    $"Ошибка в Content-Type запроса: Необходимый Content-Type: text/plain получилось: {response.Content.Headers.ContentType.MediaType}");
-            }
+
+            throw new InvalidDataException(
+                $"Ошибка в Content-Type запроса: Необходимый Content-Type: text/plain получилось: {response.Content.Headers.ContentType.MediaType}");
         }
 
         return null;

diff --git a/fs24bot3/EventProcessors/OnMsgEvent.cs b/fs24bot3/EventProcessors/OnMsgEvent.cs
@@ -113,18 +113,29 @@ await BotContext.Client.SendMessage(message.Target,
 
             try
             {
-                var document = await web.LoadFromWebAsync(url);
-                string title = document?.DocumentNode?.SelectSingleNode("//head/title")?.InnerText ?? "Нет заголовка";
+                var http = new HttpTools(2);
+                var request = await http.GetResponseAsync(url);
+                var text = await request.Content.ReadAsStringAsync();
+
+                var document = new HtmlDocument();
+                document.LoadHtml(text);
+                string title = document.DocumentNode?.SelectSingleNode("//title")?.InnerText;
+
+                if (string.IsNullOrWhiteSpace(title))
+                {
+                    return;
+                }
+
                 var domain = url.Split("/");
 
                 if (domain.Length >= 3)
                 {
                     await BotContext.Client.SendMessage(message.Target, $"[b][ {title} ][r] - {domain[2]}");
                 }
             }
-            catch (HttpRequestException)
+            catch (Exception e)
             {
-                Log.Warning("Unable to handle URL: {0} due to request error", url);
+                Log.Warning("Unable to handle URL: {0} due to error: {1}", url, e);
             }
         }
     }