diff --git a/core/download.ts b/core/download.ts index 9a269884..2a350353 100644 --- a/core/download.ts +++ b/core/download.ts @@ -6,6 +6,7 @@ export interface TextResponse { readonly ok: boolean parseJson(): null | unknown parseXml(): Document | null | XMLDocument + readonly redirected: boolean readonly status: number readonly text: string readonly url: string @@ -97,6 +98,7 @@ export function createTextResponse( } return bodyCache }, + redirected: other.redirected ?? false, status, text, url: other.url ?? 'https://example.com' @@ -124,6 +126,7 @@ export function createDownloadTask(): DownloadTask { } return createTextResponse(text, { headers: response.headers, + redirected: response.redirected, status: response.status, url: response.url }) diff --git a/core/test/download.test.ts b/core/test/download.test.ts index d218a7a3..9ccf6f68 100644 --- a/core/test/download.test.ts +++ b/core/test/download.test.ts @@ -29,6 +29,7 @@ test('makes requests', async () => { let response = await task.request('https://example.com') equal(response.status, 200) + equal(response.redirected, false) equal(await response.text(), 'Hi') }) diff --git a/loader-tests/check-opml.ts b/loader-tests/check-opml.ts index 8a480c03..6b48cd34 100644 --- a/loader-tests/check-opml.ts +++ b/loader-tests/check-opml.ts @@ -59,7 +59,9 @@ cli.run(async args => { } let feeds = await parseFeedsFromFile(opmlFile) - await completeTasks(feeds.map(feed => () => fetchAndParsePosts(feed.url))) + await completeTasks( + feeds.map(feed => () => fetchAndParsePosts(feed.url, true)) + ) if (home) { for (let feed of feeds) { await findRSSfromHome(feed) diff --git a/loader-tests/utils.ts b/loader-tests/utils.ts index 58b40792..acc116f5 100644 --- a/loader-tests/utils.ts +++ b/loader-tests/utils.ts @@ -128,22 +128,51 @@ export function success(msg: string, details?: string): void { print(styleText('green', styleText('bold', '✓ ') + msg)) } -export async function fetchAndParsePosts(url: string): Promise { +export function semiSuccess(msg: string, note: string): void { + print( + styleText( + 'yellow', + styleText('bold', '✓ ') + msg + ' ' + styleText('bold', note) + ) + ) +} + +export async function fetchAndParsePosts( + url: string, + badSource = false +): Promise { try { let task = createDownloadTask() - let textResponse = await task.text(url) - let candidate: false | PreviewCandidate = getLoaderForText(textResponse) + let response = await task.text(url) + if (badSource && response.status >= 400) { + semiSuccess(url, `${response.status}`) + return + } + if ( + badSource && + response.redirected && + response.contentType === 'text/html' && + response.text.toLocaleLowerCase().includes(' 1 ? ' posts' : ' post')) } - success(url, list.length + (list.length > 1 ? ' posts' : ' post')) } catch (e) { error(e, `During loading posts for ${url}`) }