build(deps): update dependencies

Cargo.toml

@@ -23,6 +23,7 @@ tonic-reflection = "0.10"
 tower = { version = "0.4" }
 tonic-async-interceptor = "0.10"
 prost = "0.12"
+prost-types = "0.12"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 tokio-stream = "0.1"
 migration = { path = "migration" }
@@ -48,6 +49,7 @@ derive_more = "0"
 futures = { version = "0" }
 futures-util = { version = "0" }
 manga_parser = { git = "https://github.com/hubble459/manga_parser" }
+# manga_parser = { path = "../manga_parser" }
 fcm = "0"
 hyper = { version = "0" }
 

build.rs

@@ -15,6 +15,7 @@ fn main() -> io::Result<()> {
     tonic_build::configure()
         .file_descriptor_set_path(descriptor_path)
         .build_client(false)
+        .compile_well_known_types(false)
         .protoc_arg("--experimental_allow_proto3_optional")
         .compile(protos.as_slice(), &["proto"])?;
 

configs/madara.yaml

@@ -99,7 +99,40 @@ images:
 
 search:
   - hostnames:
+      - 1stkissmanga.club
+      - 1stkissmanga.io
+      - 1stkissmanga.com
+      - 1stkissmanga.love
+      - 247manga.com
+      - aquamanga.com
+      - azmanhwa.net
+      - isekaiscanmanga.com
+      - isekaiscan.com
       - isekaiscan.top
+      - lhtranslation.net
+      - manga68.com
+      - mangabat.best
+      - mangaboat.com
+      - mangachill.net
+      - mangachill.io
+      - mangafoxfull.com
+      - mangahz.com
+      - mangaonlineteam.com
+      - mangarockteam.com
+      - mangasushi.org
+      - mangatx.com
+      - mangaweebs.in
+      - mangazukiteam.com
+      - manhuadex.com
+      - manhuaplus.com
+      - manhwatop.com
+      - mixedmanga.com
+      - s2manga.com
+      - topmanhua.com
+      - yaoi.mobi
+      - zinmanga.com
+      - hubmanga.com
+      - manhuafast.com
     search_url: "{hostname}/search?s={query}"
     query_format:
       - replace_regex: '\+'

proto/rumgap/v1/scrape_error.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package rumgap.v1;
+
+enum ScrapeErrorType {
+    Unknown = 0;
+    ReqwestError = 1;
+    ReqwestMiddlewareError = 2;
+    IoError = 3;
+    ConfigError = 4;
+    ConfigDeserializeError = 5;
+    WebScrapingError = 6;
+    NotAValidURL = 7;
+    SelectorError = 8;
+    WebsiteNotSupported = 9;
+    SearchNotSupported = 10;
+    MultipleScrapingErrors = 11;
+    CloudflareIUAM = 12;
+}
+
+message ScrapeError {
+    ScrapeErrorType type = 1;
+    string message = 2;
+}
+

proto/rumgap/v1/v1.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package rumgap.v1;
 
+import "google/protobuf/any.proto";
+
 import "rumgap/v1/user.proto";
 import "rumgap/v1/friend.proto";
 import "rumgap/v1/manga.proto";
@@ -66,4 +68,10 @@ message Id {
 }
 
 message Empty {
+}
+
+message DetailedError {
+    int32 status = 1;
+    string message = 2;
+    repeated google.protobuf.Any details = 3;
 }

src/interceptor/logger.rs

@@ -1,9 +1,13 @@
 use std::task::{Context, Poll};
 
 use hyper::Body;
+use prost::Message;
 use tonic::body::BoxBody;
+use tonic::Status;
 use tower::{Layer, Service};
 
+use crate::proto::ScrapeError;
+
 #[derive(Debug, Clone, Default)]
 pub struct LoggerLayer;
 
@@ -44,7 +48,20 @@ where
         req.extensions_mut().insert(target);
 
         Box::pin(async move {
-            inner.call(req).await
+            let response = inner.call(req).await;
+
+            if let Ok(response) = &response {
+                let grpc_status = Status::from_header_map(response.headers());
+                if let Some(grpc_status) = grpc_status {
+                    error!("Response error: {:#?}", grpc_status);
+                    let details = grpc_status.details();
+                    if !details.is_empty() {
+                        error!("Response error details: {:#?}", ScrapeError::decode(details).ok());
+                    }
+                }
+            }
+
+            response
         })
     }
 }

src/service/v1/manga.rs

@@ -1,13 +1,11 @@
 use std::pin::Pin;
 use std::time::Duration;
 
-use chrono::Utc;
+use chrono::{NaiveDateTime, Utc};
 use futures::Stream;
-use manga_parser::error::ScrapeError;
 use manga_parser::scraper::MangaScraper;
 use manga_parser::Url;
 use migration::{Expr, IntoCondition, JoinType, OnConflict};
-use sea_orm::prelude::DateTimeWithTimeZone;
 use sea_orm::ActiveValue::{NotSet, Set};
 use sea_orm::{
     ActiveModelTrait, ColumnTrait, DatabaseConnection, DeriveColumn, EntityTrait, EnumIter,
@@ -22,6 +20,7 @@ use crate::proto::manga_server::{Manga, MangaServer};
 use crate::proto::{
     Id, MangaReply, MangaRequest, MangasReply, MangasRequest, PaginateReply, PaginateSearchQuery,
 };
+use crate::util::scrape_error_proto::StatusWrapper;
 use crate::util::search::manga::lucene_filter;
 use crate::{data, util, MANGA_PARSER};
 
@@ -85,13 +84,10 @@ pub async fn save_manga(
 
     // TODO: backtick and probably other special characters
     // TODO: should be replaced with normal characters
-    let manga: manga_parser::model::Manga = MANGA_PARSER.manga(&url).await.map_err(|e| {
-        if let ScrapeError::WebsiteNotSupported(e) = e {
-            Status::unavailable(e.to_string())
-        } else {
-            Status::internal(e.to_string())
-        }
-    })?;
+    let manga: manga_parser::model::Manga = MANGA_PARSER
+        .manga(&url)
+        .await
+        .map_err(StatusWrapper::from)?;
 
     let saved = entity::manga::ActiveModel {
         id: id.map_or(NotSet, Set),
@@ -304,7 +300,7 @@ impl Manga for MangaController {
         let req = request.get_ref();
         let manga_id = req.id;
 
-        let (url, updated_at): (String, DateTimeWithTimeZone) =
+        let (url, updated_at): (String, NaiveDateTime) =
             entity::manga::Entity::find_by_id(manga_id)
                 .select_only()
                 .column(entity::manga::Column::Url)
@@ -321,7 +317,9 @@ impl Manga for MangaController {
             .unwrap_or(3600000);
 
         // Check if it should be updated
-        let manga = if (Utc::now() - chrono::Duration::milliseconds(interval_ms)) > updated_at {
+        let manga = if (Utc::now().naive_utc() - chrono::Duration::milliseconds(interval_ms))
+            > updated_at
+        {
             // Update
             info!("Updating manga with id '{}' [{}]", manga_id, url);
             save_manga(db, logged_in, Some(manga_id), Url::parse(&url).unwrap()).await?

src/service/v1/search.rs

@@ -12,6 +12,7 @@ use tonic::{Request, Response, Status};
 use crate::proto::search_server::{Search, SearchServer};
 use crate::proto::{SearchManga, SearchReply, SearchRequest};
 use crate::MANGA_PARSER;
+use crate::util::scrape_error_proto::StatusWrapper;
 
 #[derive(Debug, Default)]
 pub struct SearchController;
@@ -40,7 +41,7 @@ impl Search for SearchController {
         )
         .await
         .map_err(|e| Status::deadline_exceeded(e.to_string()))?
-        .map_err(|e| Status::internal(e.to_string()))?;
+        .map_err(StatusWrapper::from)?;
 
         let urls: Vec<String> = search_results
             .iter()

src/util/mod.rs

@@ -1,6 +1,7 @@
 pub mod argon;
+pub mod auth;
 pub mod order;
+pub mod scrape_error_proto;
 pub mod search;
 pub mod updater;
-pub mod auth;
 pub mod verify;