From 90286e64d28c9d996084215c830dbfb731754e61 Mon Sep 17 00:00:00 2001 From: "Y.D.X." <73375426+YDX-2147483647@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:28:21 +0800 Subject: [PATCH 1/3] =?UTF-8?q?refactor:=20`fetch`=20=E2=86=92=20`ky`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deno.jsonc | 1 + deno.lock | 4 +++- src/core/fetch_wrapper.ts | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/deno.jsonc b/deno.jsonc index ecf8477..2b67b26 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -12,6 +12,7 @@ }, "imports": { "std/": "https://deno.land/std@0.205.0/", + "ky": "https://esm.sh/ky@1.1.3", "npm:aes-js": "npm:aes-js@^3.1.2", "npm:before-after-hook": "npm:before-after-hook@^2.2.2", "npm:chalk": "npm:chalk@^5.3.0", diff --git a/deno.lock b/deno.lock index abc1e21..fff3e47 100644 --- a/deno.lock +++ b/deno.lock @@ -2560,6 +2560,8 @@ "https://deno.land/x/deno_dom@v0.1.42/src/dom/selectors/sizzle.js": "c3aed60c1045a106d8e546ac2f85cc82e65f62d9af2f8f515210b9212286682a", "https://deno.land/x/deno_dom@v0.1.42/src/dom/utils-types.ts": "96db30e3e4a75b194201bb9fa30988215da7f91b380fca6a5143e51ece2a8436", "https://deno.land/x/deno_dom@v0.1.42/src/dom/utils.ts": "4c6206516fb8f61f37a209c829e812c4f5a183e46d082934dd14c91bde939263", - "https://deno.land/x/deno_dom@v0.1.42/src/parser.ts": "e06b2300d693e6ae7564e53dfa5c9a9e97fdb8c044c39c52c8b93b5d60860be3" + "https://deno.land/x/deno_dom@v0.1.42/src/parser.ts": "e06b2300d693e6ae7564e53dfa5c9a9e97fdb8c044c39c52c8b93b5d60860be3", + "https://esm.sh/ky@1.1.3": "7bc4a737f22591792a42989de03efeace690158962ead46f622b73fc3ddf85b1", + "https://esm.sh/v135/ky@1.1.3/denonext/ky.mjs": "6babfbef9debcf3a28dda275c0c477206002c82635f8ef9a3173ae69002f25f6" } } diff --git a/src/core/fetch_wrapper.ts b/src/core/fetch_wrapper.ts index 28baa09..fd0ff15 100644 --- a/src/core/fetch_wrapper.ts +++ b/src/core/fetch_wrapper.ts @@ -6,11 +6,13 @@ * @module */ +import ky from 'ky' + import type { HookCollectionType } from './hooks_type.ts' function fetch_wrapper(request: RequestInit & { url: string }) { const { url, ...init } = request - return fetch(url, init) + return ky(url, init) } /** From 89fd771079685ad688bc2159a0e033942e907ae4 Mon Sep 17 00:00:00 2001 From: "Y.D.X." <73375426+YDX-2147483647@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:03:25 +0800 Subject: [PATCH 2/3] =?UTF-8?q?refactor:=20`Deno.errors.TimedOut`=20?= =?UTF-8?q?=E2=86=92=20`ky.TimeoutError`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugin/cli/hooks.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugin/cli/hooks.ts b/src/plugin/cli/hooks.ts index 5b33da2..7cfd28e 100644 --- a/src/plugin/cli/hooks.ts +++ b/src/plugin/cli/hooks.ts @@ -1,4 +1,6 @@ +import { TimeoutError } from 'ky' import cliProgress from 'npm:cli-progress' + import type { HookCollectionType } from '../../core/index.ts' import { logger } from '../../util/logger.ts' import { recent_checker } from '../../util/my_date.ts' @@ -19,9 +21,9 @@ export function verbose(hook: HookCollectionType) { logger.warn(`未能访问“${source.name}”(NotFound)。将忽略。`, { plugin: 'cli', }) - } else if (err instanceof Deno.errors.TimedOut) { + } else if (err instanceof TimeoutError) { logger.error( - `访问“${source.name}”(TimedOut)超时,可能因为访问太频繁。将忽略。`, + `访问“${source.name}”(Timeout)超时,可能因为访问太频繁。将忽略。`, { plugin: 'cli' }, ) } else { From 9e6458c5c9706e6afa3a44221f622596995c541b Mon Sep 17 00:00:00 2001 From: "Y.D.X." <73375426+YDX-2147483647@users.noreply.github.com> Date: Tue, 12 Dec 2023 21:40:32 +0800 Subject: [PATCH 3/3] feat(plugin): Retry after timeout --- config/config.schema.json | 13 ++++++++- deno.lock | 1 + src/examples/cli.ts | 2 ++ src/examples/ding-cli.ts | 2 ++ src/examples/proxy-cli.ts | 4 ++- src/examples/rss-cli.ts | 2 ++ src/examples/server-cli.ts | 4 ++- src/examples/server-ding-cli.ts | 4 ++- src/plugin/cli/hooks.ts | 10 +++++++ src/plugin/retry/index.ts | 48 +++++++++++++++++++++++++++++++++ 10 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 src/plugin/retry/index.ts diff --git a/config/config.schema.json b/config/config.schema.json index b7e111a..2456125 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft-07/schema", - "$id": "config.schema.json-v0.3.1", + "$id": "config.schema.json-v0.3.2", "title": "Bulletin IT 设置", "type": "object", "properties": { @@ -77,6 +77,17 @@ "output_path" ] }, + "retry": { + "title": "超时后重试插件", + "type": "object", + "properties": { + "delay": { + "title": "超时后等待的毫秒数", + "type": "number", + "default": 10000 + } + } + }, "proxy": { "title": "虚拟校园网插件", "type": "object", diff --git a/deno.lock b/deno.lock index fff3e47..03e7e0c 100644 --- a/deno.lock +++ b/deno.lock @@ -2538,6 +2538,7 @@ "https://deno.land/std@0.205.0/assert/mod.ts": "37c49a26aae2b254bbe25723434dc28cd7532e444cf0b481a97c045d110ec085", "https://deno.land/std@0.205.0/assert/unimplemented.ts": "d56fbeecb1f108331a380f72e3e010a1f161baa6956fd0f7cf3e095ae1a4c75a", "https://deno.land/std@0.205.0/assert/unreachable.ts": "4600dc0baf7d9c15a7f7d234f00c23bca8f3eba8b140286aaca7aa998cf9a536", + "https://deno.land/std@0.205.0/async/delay.ts": "a6142eb44cdd856b645086af2b811b1fcce08ec06bb7d50969e6a872ee9b8659", "https://deno.land/std@0.205.0/fmt/colors.ts": "c51c4642678eb690dcf5ffee5918b675bf01a33fba82acf303701ae1a4f8c8d9", "https://deno.land/x/deno_dom@v0.1.42/build/deno-wasm/deno-wasm.js": "d6841a06342eb6a2798ef28de79ad69c0f2fa349fa04d3ca45e5fcfbf50a9340", "https://deno.land/x/deno_dom@v0.1.42/deno-dom-wasm.ts": "a33d160421bbb6e3104285ea5ebf33352b7ad50d82ea8765e3cf65f972b25119", diff --git a/src/examples/cli.ts b/src/examples/cli.ts index 0e9e0c3..35ccdfb 100644 --- a/src/examples/cli.ts +++ b/src/examples/cli.ts @@ -1,11 +1,13 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' add_hook.verbose(hook) add_hook.preview_output(hook) add_hook.progress_bar(hook) add_hook.recent_filter(hook, 90) +add_retry_hook(hook) add_normalize_hook(hook) await update_notices() diff --git a/src/examples/ding-cli.ts b/src/examples/ding-cli.ts index bf17524..7151307 100644 --- a/src/examples/ding-cli.ts +++ b/src/examples/ding-cli.ts @@ -6,10 +6,12 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' import robot from '../plugin/ding/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' import { logger } from '../util/logger.ts' add_hook.verbose(hook) add_hook.progress_bar(hook) +add_retry_hook(hook) add_normalize_hook(hook) const { all_notices, new_notices, change } = await update_notices() diff --git a/src/examples/proxy-cli.ts b/src/examples/proxy-cli.ts index d5c81ce..cb0ff8c 100644 --- a/src/examples/proxy-cli.ts +++ b/src/examples/proxy-cli.ts @@ -1,12 +1,14 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' -import add_proxy_hook from '../plugin/proxy/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_proxy_hook from '../plugin/proxy/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' add_hook.verbose(hook) add_hook.preview_output(hook) add_hook.progress_bar(hook) add_proxy_hook(hook) +add_retry_hook(hook) add_normalize_hook(hook) await update_notices() diff --git a/src/examples/rss-cli.ts b/src/examples/rss-cli.ts index 0aa3673..85d438a 100644 --- a/src/examples/rss-cli.ts +++ b/src/examples/rss-cli.ts @@ -1,12 +1,14 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' import add_rss_hook from '../plugin/rss/index.ts' add_hook.verbose(hook) add_hook.progress_bar(hook) add_hook.recent_filter(hook, 90) add_normalize_hook(hook) +add_retry_hook(hook) add_rss_hook(hook) await update_notices() diff --git a/src/examples/server-cli.ts b/src/examples/server-cli.ts index 5627d21..9b5cae8 100644 --- a/src/examples/server-cli.ts +++ b/src/examples/server-cli.ts @@ -4,13 +4,15 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' -import add_proxy_hook from '../plugin/proxy/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_proxy_hook from '../plugin/proxy/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' import add_rss_hook from '../plugin/rss/index.ts' add_hook.verbose(hook) add_hook.progress_bar(hook) add_proxy_hook(hook) +add_retry_hook(hook) add_normalize_hook(hook) add_rss_hook(hook) diff --git a/src/examples/server-ding-cli.ts b/src/examples/server-ding-cli.ts index b46b952..ef240d9 100644 --- a/src/examples/server-ding-cli.ts +++ b/src/examples/server-ding-cli.ts @@ -5,14 +5,16 @@ import { hook, update_notices } from '../core/index.ts' import { add_hook } from '../plugin/cli/index.ts' import robot from '../plugin/ding/index.ts' -import add_proxy_hook from '../plugin/proxy/index.ts' import add_normalize_hook from '../plugin/normalize/index.ts' +import add_proxy_hook from '../plugin/proxy/index.ts' +import add_retry_hook from '../plugin/retry/index.ts' import add_rss_hook from '../plugin/rss/index.ts' import { logger } from '../util/logger.ts' add_hook.verbose(hook) add_hook.progress_bar(hook) add_proxy_hook(hook) +add_retry_hook(hook) add_normalize_hook(hook) add_rss_hook(hook) diff --git a/src/plugin/cli/hooks.ts b/src/plugin/cli/hooks.ts index 7cfd28e..0077b5d 100644 --- a/src/plugin/cli/hooks.ts +++ b/src/plugin/cli/hooks.ts @@ -11,6 +11,12 @@ export function verbose(hook: HookCollectionType) { const { sources } = options logger.info(`发现${sources.length}个通知来源。`, { plugin: 'cli' }) }) + hook.before('fetch_each', ({ source }) => { + logger.debug(`开始获取“${source.name}”。`, { + plugin: 'cli', + hook: 'fetch_each.before', + }) + }) hook.error('fetch_each', ( err, // @ts-ignore If `fetch_each` has an error hook, the after hook may get `undefined` @@ -39,6 +45,10 @@ export function verbose(hook: HookCollectionType) { plugin: 'cli', }) } + logger.debug(`结束获取“${source.name}”。`, { + plugin: 'cli', + hook: 'fetch_each.after', + }) }) hook.after('update', (_result, { write_json_path }) => { logger.info(`已按需保存到“${write_json_path}”。`, { plugin: 'cli' }) diff --git a/src/plugin/retry/index.ts b/src/plugin/retry/index.ts new file mode 100644 index 0000000..3ca5d20 --- /dev/null +++ b/src/plugin/retry/index.ts @@ -0,0 +1,48 @@ +/** + * 超时后重试 + * + * ky 遇到超时并不会重试。 + * https://github.com/sindresorhus/ky/discussions/279#discussioncomment-187602 + * https://github.com/sindresorhus/ky/issues/546 + * @module + */ + +import ky, { TimeoutError } from 'ky' +import { delay } from 'std/async/delay.ts' + +import { config, type HookCollectionType } from '../../core/index.ts' +import { logger } from '../../util/logger.ts' + +const retry_options = Object.assign({ + delay: 10_000, +}, config.retry) as { + delay: number +} + +/** + * `request`出错时用 ky 重试一次 + * + * 建议在其它`request` hook 之后调用。 + */ +export default function add_retry_hook(hook: HookCollectionType) { + hook.error('request', (error, options) => { + if (error instanceof TimeoutError) { + logger.http( + `Time is out when requesting “${options.url}”. Retry later.`, + { + plugin: 'retry', + }, + ) + + return delay(retry_options.delay).then(() => { + logger.http(`Retry “${options.url}”.`, { + plugin: 'retry', + }) + const { url, ...init } = options + return ky(url, init) + }) + } else { + throw error + } + }) +}