From b6cdd221a334465ab0cc29bbc1371a3def375b31 Mon Sep 17 00:00:00 2001
From: Yuukiy <76897913+Yuukiy@users.noreply.github.com>
Date: Sun, 7 Jan 2024 19:12:20 +0800
Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E6=B5=8B=E8=AF=95=E8=84=9A?=
 =?UTF-8?q?=E6=9C=AC=EF=BC=8C=E6=9B=B4=E6=96=B0=E6=B5=8B=E8=AF=95=E7=94=A8?=
 =?UTF-8?q?=E4=BE=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 unittest/data/KQBD-089 (msin).json | 21 +++++++--------------
 unittest/test_crawlers.py          | 20 +++++++++++++++-----
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/unittest/data/KQBD-089 (msin).json b/unittest/data/KQBD-089 (msin).json
index b4f7f31d0..178322820 100644
--- a/unittest/data/KQBD-089 (msin).json	
+++ b/unittest/data/KQBD-089 (msin).json	
@@ -6,21 +6,14 @@
   "cover": "https://pics.dmm.co.jp/digital/video/244kqbd00089/244kqbd00089pl.jpg",
   "big_cover": null,
   "genre": [
+    "4K",
+    "ハイビジョン",
+    "単体作品",
     "フェラ",
-    "ごっくん",
-    "お掃除フェラ",
-    "美少女",
-    "中出し",
-    "中出し",
-    "バック",
-    "生",
-    "中",
-    "制服",
-    "中出し",
-    "ベロ",
-    "キス",
-    "顔",
-    "舐め"
+    "電マ",
+    "女子校生",
+    "拘束",
+    "セーラー服"
   ],
   "genre_id": null,
   "genre_norm": null,
diff --git a/unittest/test_crawlers.py b/unittest/test_crawlers.py
index 4dee220b4..d5b165957 100644
--- a/unittest/test_crawlers.py
+++ b/unittest/test_crawlers.py
@@ -10,7 +10,7 @@
 sys.path.insert(0, os.path.abspath(os.path.join(file_dir, '..')))
 
 from core.datatype import MovieInfo
-from web.exceptions import CrawlerError
+from web.exceptions import CrawlerError, SiteBlocked
 
 
 logger = logging.getLogger(__name__)
@@ -47,12 +47,15 @@ def compare(avid, scraper, file):
         parse_data = getattr(mod, 'parse_clean_data')
     else:
         parse_data = getattr(mod, 'parse_data')
+
     try:
         parse_data(online)
-    except CrawlerError as e:
-        logger.info(e)
-    except requests.exceptions.ReadTimeout as e:
+    except SiteBlocked as e:
+        logger.warning(e)
+        return
+    except (CrawlerError, requests.exceptions.ReadTimeout) as e:
         logger.info(e)
+
     try:
         # 解包数据再进行比较，以便测试不通过时快速定位不相等的键值
         local_vars = vars(local)
@@ -68,12 +71,19 @@ def compare(avid, scraper, file):
                 assert urlsplit(v).path == urlsplit(local_vars.get(k, None)).path
             elif k == 'actress_pics' and scraper == 'javbus':
                 local_tmp = online_tmp = {}
-                local_pics = local_vars.get('actress_pics')
+                local_pics = local_vars.get(k)
                 if local_pics:
                     local_tmp = {name: urlsplit(url).path for name, url in local_pics.items()}
                 if v:
                     online_tmp = {name: urlsplit(url).path for name, url in v.items()}
                 assert local_tmp == online_tmp
+            elif k == 'preview_pics' and scraper == 'javbus':
+                local_pics = local_vars.get(k)
+                if local_pics:
+                    local_tmp = [urlsplit(i).path for i in local_pics]
+                if v:
+                    online_tmp = [urlsplit(i).path for i in v]
+                assert local_tmp == online_tmp
             # 对顺序没有要求的list型字段，比较时也应该忽略顺序信息
             elif k in ['genre', 'genre_id', 'genre_norm', 'actress']:
                 if isinstance(v, list):