Merge pull request #217 from miyouzi/dev

修復Web解析 #215

Anime.py

@@ -7,6 +7,7 @@
 import shutil
 import traceback
 import Config
+import pyhttpx
 from Danmu import Danmu
 from bs4 import BeautifulSoup
 import re, time, os, platform, subprocess, requests, random, sys
@@ -31,6 +32,10 @@ def __init__(self, sn, debug_mode=False, gost_port=34173):
         self._gost_port = str(gost_port)
 
         self._session = requests.session()
+        if 'firefox' in self._settings['ua'].lower():
+            self._pyhttpx_session = pyhttpx.HttpSession(browser_type='firefox')
+        else:
+            self._pyhttpx_session = pyhttpx.HttpSession(browser_type='chrome')
         self._title = ''
         self._sn = sn
         self._bangumi_name = ''
@@ -48,6 +53,8 @@ def __init__(self, sn, debug_mode=False, gost_port=34173):
         self.realtime_show_file_size = False
         self.upload_succeed_flag = False
         self._danmu = False
+        self._proxies = {}
+        self._proxy_auth = ()
 
         self.season_title_filter = re.compile('第[零一二三四五六七八九十]{1,3}季$')
         self.extra_title_filter = re.compile('\[(特別篇|中文配音)\]$')
@@ -75,11 +82,22 @@ def __init_proxy(self):
             # 需要使用 gost 的情况, 代理到 gost
             os.environ['HTTP_PROXY'] = 'http://127.0.0.1:' + self._gost_port
             os.environ['HTTPS_PROXY'] = 'http://127.0.0.1:' + self._gost_port
+            self._proxies = {'https': '127.0.0.1:' + self._gost_port,
+                             'http': '127.0.0.1:' + self._gost_port}
         else:
             # 无需 gost 的情况
             os.environ['HTTP_PROXY'] = self._settings['proxy']
             os.environ['HTTPS_PROXY'] = self._settings['proxy']
-        os.environ['NO_PROXY'] = "127.0.0.1,localhost,bahamut.akamaized.net"
+            proxy_info = Config.parse_proxy(self._settings['proxy'])
+            proxy_without_protocol = proxy_info['proxy_ip'] + ':' + proxy_info['proxy_port']
+            self._proxies = {'https': proxy_without_protocol,
+                             'http': proxy_without_protocol}
+            self._proxy_auth = (proxy_info['proxy_user'], proxy_info['proxy_passwd'])
+
+        if self._settings['no_proxy_akamai']:
+            os.environ['NO_PROXY'] = "127.0.0.1,localhost,bahamut.akamaized.net"
+        else:
+            os.environ['NO_PROXY'] = "127.0.0.1,localhost"
 
     def renew(self):
         self.__get_src()
@@ -117,10 +135,10 @@ def get_filename(self):
 
     def __get_src(self):
         if self._settings['use_mobile_api']:
-            self._src = self.__request(f'https://api.gamer.com.tw/mobile_app/anime/v2/video.php?sn={self._sn}', no_cookies=True).json()
+            self._src = self.__request_json(f'https://api.gamer.com.tw/mobile_app/anime/v2/video.php?sn={self._sn}', no_cookies=True)
         else:
             req = f'https://ani.gamer.com.tw/animeVideo.php?sn={self._sn}'
-            f = self.__request(req, no_cookies=True)
+            f = self.__request(req, no_cookies=True, use_pyhttpx=True)
             self._src = BeautifulSoup(f.content, "lxml")
 
     def __get_title(self):
@@ -233,20 +251,20 @@ def __init_header(self):
             "Connection": "Keep-Alive"
         }
         self._web_header = {
-                "user-agent": ua,
+                "User-Agent": ua,
                 "referer": ref,
-                "accept-language": lang,
-                "accept": accept,
-                "accept-encoding": accept_encoding,
-                "cache-control": cache_control,
-                "origin": origin
+                "Accept-Language": lang,
+                "Accept": accept,
+                "Accept-Encoding": accept_encoding,
+                "Cache-Control": cache_control,
+                "Origin": origin
             }
         if self._settings['use_mobile_api']:
             self._req_header = self._mobile_header
         else:
             self._req_header = self._web_header
 
-    def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition_header=None):
+    def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition_header=None, use_pyhttpx = False):
         # 设置 header
         current_header = self._req_header
         if addition_header is None:
@@ -257,12 +275,17 @@ def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition
 
         # 获取页面
         error_cnt = 0
+        if self._cookies and not no_cookies:
+            cookies = self._cookies
+        else:
+            cookies = {}
         while True:
             try:
-                if self._cookies and not no_cookies:
-                    f = self._session.get(req, headers=current_header, cookies=self._cookies, timeout=10)
+                if use_pyhttpx:
+                    f = self._pyhttpx_session.get(req, headers=current_header, cookies=cookies, timeout=10,
+                                                  proxies=self._proxies, proxy_auth=self._proxy_auth)
                 else:
-                    f = self._session.get(req, headers=current_header, cookies={}, timeout=10)
+                    f = self._session.get(req, headers=current_header, cookies=cookies, timeout=10)
             except requests.exceptions.RequestException as e:
                 if error_cnt >= max_retry >= 0:
                     raise TryTooManyTimeError('任務狀態: sn=' + str(self._sn) + ' 请求失败次数过多！请求链接：\n%s' % req)
@@ -277,12 +300,12 @@ def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition
         # 处理 cookie
         if not self._cookies:
             # 当实例中尚无 cookie, 则读取
-            self._cookies = f.cookies.get_dict()
+            self._cookies = self._session.cookies
         elif 'nologinuser' not in self._cookies.keys() and 'BAHAID' not in self._cookies.keys():
             # 处理游客cookie
-            if 'nologinuser' in f.cookies.get_dict().keys():
-                # self._cookies['nologinuser'] = f.cookies.get_dict()['nologinuser']
-                self._cookies = f.cookies.get_dict()
+            if 'nologinuser' in self._session.cookies.keys():
+                # self._cookies['nologinuser'] = self._session.cookies['nologinuser']
+                self._cookies = self._session.cookies
         else:  # 如果用户提供了 cookie, 则处理cookie刷新
             if 'set-cookie' in f.headers.keys():  # 发现server响应了set-cookie
                 if 'deleted' in f.headers.get('set-cookie'):
@@ -329,10 +352,10 @@ def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition
                     # 20220115 简化 cookie 刷新逻辑
                     err_print(self._sn, '收到新cookie', display=False)
 
-                    self._cookies.update(f.cookies.get_dict())
+                    self._cookies.update(self._session.cookies)
                     Config.renew_cookies(self._cookies, log=False)
 
-                    key_list_str = ', '.join(f.cookies.get_dict().keys())
+                    key_list_str = ', '.join(self._session.cookies.keys())
                     err_print(self._sn, f'用戶cookie刷新 {key_list_str} ', display=False)
 
                     self.__request('https://ani.gamer.com.tw/')
@@ -346,21 +369,25 @@ def __request(self, req, no_cookies=False, show_fail=True, max_retry=3, addition
 
         return f
 
+    def __request_json(self, req, no_cookies=False, show_fail=True, max_retry=3, addition_header=None, use_pyhttpx = False):
+        if use_pyhttpx:
+            return self.__request(req, no_cookies, show_fail, max_retry, addition_header, use_pyhttpx).json
+        else:
+            return self.__request(req, no_cookies, show_fail, max_retry, addition_header, use_pyhttpx).json()
+
     def __get_m3u8_dict(self):
         # m3u8获取模块参考自 https://github.com/c0re100/BahamutAnimeDownloader
         def get_device_id():
             req = 'https://ani.gamer.com.tw/ajax/getdeviceid.php'
-            f = self.__request(req)
-            self._device_id = f.json()['deviceid']
+            self._device_id = self.__request_json(req)['deviceid']
             return self._device_id
 
         def get_playlist():
             if self._settings['use_mobile_api']:
                 req = f'https://api.gamer.com.tw/mobile_app/anime/v2/m3u8.php?sn={str(self._sn)}&device={self._device_id}'
             else:
                 req = 'https://ani.gamer.com.tw/ajax/m3u8.php?sn=' + str(self._sn) + '&device=' + self._device_id
-            f = self.__request(req)
-            self._playlist = f.json()
+            self._playlist = self.__request_json(req)
 
         def random_string(num):
             chars = 'abcdefghijklmnopqrstuvwxyz0123456789'
@@ -377,7 +404,7 @@ def gain_access():
                 req = 'https://ani.gamer.com.tw/ajax/token.php?adID=0&sn=' + str(
                     self._sn) + "&device=" + self._device_id + "&hash=" + random_string(12)
             # 返回基础信息, 用于判断是不是VIP
-            return self.__request(req).json()
+            return self.__request_json(req)
 
         def unlock():
             req = 'https://ani.gamer.com.tw/ajax/unlock.php?sn=' + str(self._sn) + "&ttl=0"
@@ -412,8 +439,7 @@ def check_no_ad(error_count=10):
 
             req = "https://ani.gamer.com.tw/ajax/token.php?sn=" + str(
                 self._sn) + "&device=" + self._device_id + "&hash=" + random_string(12)
-            f = self.__request(req)
-            resp = f.json()
+            resp = self.__request_json(req)
             if 'time' in resp.keys():
                 if not resp['time'] == 1:
                     err_print(self._sn, '廣告似乎還沒去除, 追加等待2秒, 剩餘重試次數 ' + str(error_count), status=1)
@@ -1003,7 +1029,7 @@ def download(self, resolution='', save_dir='', bangumi_tag='', realtime_show_fil
                     else:
                         apiMethod = "getUpdates"
                         api_url = "https://api.telegram.org/bot" + vApiTokenTelegram + "/" + apiMethod # Telegram bot api url
-                        response = self.__request(api_url).json()
+                        response = self.__request_json(api_url)
                         chat_id = response["result"][0]["message"]["chat"]["id"] # Get chat id
                     try:
                         api_method = "sendMessage"
@@ -1341,4 +1367,6 @@ def set_resolution(self, resolution):
 
 
 if __name__ == '__main__':
-    pass
+    a = Anime('31724')
+    print(a.get_m3u8_dict())
+    a.download('360')

Config.py

@@ -21,8 +21,8 @@
 sn_list_path = os.path.join(working_dir, 'sn_list.txt')
 cookie_path = os.path.join(working_dir, 'cookie.txt')
 logs_dir = os.path.join(working_dir, 'logs')
-aniGamerPlus_version = 'v24.2'
-latest_config_version = 17.0
+aniGamerPlus_version = 'v24.3'
+latest_config_version = 17.1
 latest_database_version = 2.0
 cookie = None
 max_multi_thread = 5
@@ -101,7 +101,7 @@ def __init_settings():
                 'video_filename_extension': 'mp4',  # 视频扩展名/封装格式
                 'zerofill': 1,  # 剧集名补零, 此项填补足位数, 小于等于 1 即不补零
                 # cookie的自动刷新对 UA 有检查
-                'ua': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
+                'ua': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
                 'use_proxy': False,
                 'proxy': 'http://user:[email protected]:1000',  # 代理功能, config_version v13.0 删除链式代理
                 'upload_to_server': False,
@@ -145,7 +145,7 @@ def __init_settings():
                 'read_sn_list_when_checking_update': True,
                 'read_config_when_checking_update': True,
                 'ads_time': 25,
-                'mobile_ads_time': 3,
+                'mobile_ads_time': 25,
                 'use_dashboard': True,
                 'dashboard': {
                     'host': '127.0.0.1',
@@ -338,7 +338,7 @@ def __update_settings(old_settings):  # 升级配置文件
         new_settings['use_mobile_api'] = False
 
     if 'mobile_ads_time' not in new_settings.keys():
-        new_settings['mobile_ads_time'] = 3  # 使用APP API非会员广告等待时间可低至 3s
+        new_settings['mobile_ads_time'] = 25  # 使用APP API非会员广告等待时间可低至 3s
 
     if 'message_suffix' not in new_settings['coolq_settings'].keys():
         # v21.1 新增
@@ -371,6 +371,10 @@ def __update_settings(old_settings):  # 升级配置文件
     if 'only_use_vip' not in new_settings.keys():
         new_settings['only_use_vip'] = False
 
+    if 'no_proxy_akamai' not in new_settings.keys():
+        # 是否代理 akamai CDN （视频流）
+        new_settings['no_proxy_akamai'] = False
+
     new_settings['config_version'] = latest_config_version
     with open(config_path, 'w', encoding='utf-8') as f:
         json.dump(new_settings, f, ensure_ascii=False, indent=4)
@@ -674,6 +678,9 @@ def read_cookie(log=False):
         os.rename(error_cookie_path, cookie_path)
     # 用户可以将cookie保存在程序所在目录下，保存为 cookies.txt ，UTF-8 编码
     if os.path.exists(cookie_path):
+        # 防止 Cookie 文件为空报错
+        if os.path.getsize(cookie_path) == 0:
+            return None
         # del_bom(cookie_path)  # 移除 bom
         check_encoding(cookie_path)  # 移除 bom
         if log:
@@ -816,5 +823,32 @@ def get_local_ip():
     return local_ip
 
 
+def parse_proxy(proxy_str: str) -> dict:
+    if len(proxy_str) == 0 or proxy_str.isspace():
+        return {}
+
+    result = {}
+
+    if re.match(r'.*@.*', proxy_str):
+        proxy_user = re.sub(r':(\/\/)?', '', re.findall(r':\/\/.*?:', proxy_str)[0])
+        proxy_passwd = re.sub(r'(:\/\/:)?@?', '', re.sub(proxy_user, '', re.findall(r':.*@', proxy_str)[0]))
+        result['proxy_user'] = proxy_user
+        result['proxy_passwd'] = proxy_passwd
+        proxy_str = proxy_str.replace(proxy_user + ':' + proxy_passwd + '@', '')
+    else:
+        result['proxy_user'] = None
+        result['proxy_passwd'] = None
+
+    proxy_protocol = re.sub(r':\/\/.*', '', proxy_str).upper()
+    proxy_ip = re.sub(r':(\/\/)?', '', re.findall(r':.*:', proxy_str)[0])
+    proxy_port = re.sub(r':', '', re.findall(r':\d+', proxy_str)[0])
+
+    result['proxy_protocol'] = proxy_protocol
+    result['proxy_ip'] = proxy_ip
+    result['proxy_port'] = proxy_port
+
+    return result
+
+
 if __name__ == '__main__':
     pass

README.md

@@ -204,7 +204,7 @@ docker run -td --name anigamerplus \
     "read_sn_list_when_checking_update": true,  // 是否在檢查更新時讀取sn_list.txt, 開啓後對sn_list.txt的更改將會在下次檢查更新時生效而不用重啓程序
     "read_config_when_checking_update": true,  // 是否在檢查更新時讀取配置文件, 開啓後對配置文件的更改將會在下次檢查時更新生效而不用重啓程序
     "ads_time": 25,  // 非VIP廣告等待時間, 如果等待時間不足, 程式會自行追加時間 (最大20秒)
-    "mobile_ads_time":3  // 使用移動端API解析的廣告等待時間
+    "mobile_ads_time": 25  // 使用移動端API解析的廣告等待時間
     "use_dashboard": true  // Web 控制台開關
     "dashboard": {  // Web控制面板配置
         "host": "127.0.0.1",  // 監聽地址, 如果需要允許外部訪問, 請填寫 "0.0.0.0"
@@ -302,7 +302,7 @@ v8.0 影片下載模式新增分段下載, 其工作流程: 由 aniGamerPlus 讀
 
  - 在程序所在目錄新建一個名爲**cookie.txt**的文本文件, 打開將上面的Cookie複製貼上保存即可
     ![](screenshot/CookiesFormat.png)
-   
+
 #### （推薦自動獲取UA）通過獲取Web控制臺如何獲取 UA:
 
  - 開啓 Web 控制臺功能（默認開啓），打開控制臺，找到`取得當前UA`按鈕，點擊後會自動填入當前瀏覽器UA，然後保存即可
@@ -455,7 +455,7 @@ optional arguments:
     - **list** 讀取 sn_list 中的内容進行下載, 並會將任務狀態記錄在資料庫中, 重啓自動下載未完成的集數, 該功能用於單次大量下載. **此模式無法通過```-r```參數指定解析度**
 
     - **sn-list** 讀取 sn_list 中的指定sn進行下載, sn後面的模式設定會被忽略，僅下載單個sn, 並會將任務狀態記錄在資料庫中. **此模式無法通過```-r```參數指定解析度**
-    
+
     - **sn-range** 下載此番据指定sn範圍的劇集, 對於劇集名稱不是正整數的番劇, 可以用此模式
 
  - **-t** 接最大并發下載數, 可空, 空則讀取**config.json**中的定義
@@ -484,7 +484,7 @@ optional arguments:
     - 指定不連續劇集或sn時, 請用英文逗號```,```分隔, 中間無空格
 
     - 在 ```range``` 模式下, 指定連續劇集格式: 起始劇集-終止劇集. 舉例想下載第5到9集, 則格式為 5-9
-    
+
     - 在 ```sn-range``` 模式下, 格式同 ```range``` 模式, 不過將劇集改成 sn 碼
 
     - 將會按sn順序下載
@@ -499,7 +499,7 @@ optional arguments:
 
         - 想下載某番劇第2集, 第5到8集, 第12集
         ```python3 aniGamerPlus.py -s 10218 -e 2,5-8,12```
-        
+
         - 想下載某番劇sn範圍 14440 到 14459 的劇集, 外加 sn 為 14670 和 14746 的兩集
         ```python3 aniGamerPlus.py -s 14440 -m sn-range -e 14670,14746,14440-14459```
 

config-sample.json

@@ -20,9 +20,10 @@
     "customized_video_filename_suffix": "",
     "video_filename_extension": "mp4",
     "zerofill": 1,
-    "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
+    "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
     "use_proxy": false,
     "proxy": "http://user:[email protected]:1000",
+    "no_proxy_akamai": false,
     "upload_to_server": false,
     "ftp": {
         "server": "",
@@ -52,7 +53,7 @@
     "read_sn_list_when_checking_update": true,
     "read_config_when_checking_update": true,
     "ads_time": 25,
-	"mobile_ads_time": 3,
+	"mobile_ads_time": 25,
     "use_dashboard": true,
     "dashboard": {
         "host": "127.0.0.1",
@@ -66,4 +67,4 @@
     "quantity_of_logs": 7,
     "config_version": 13.0,
     "database_version": 2.0
-}
+}

requirements.txt

@@ -11,3 +11,4 @@ pysocks==1.7.1
 lxml==4.9.1
 markupsafe<2.1.0
 greenlet==1.1.3
+pyhttpx==1.3.30