From b20f58f3039f7f78d9be0e5eb03311c820a98065 Mon Sep 17 00:00:00 2001 From: Andrey Sitnik Date: Fri, 2 Feb 2024 13:19:59 +0100 Subject: [PATCH] Add slow categories --- core/fast.ts | 4 +- core/feed.ts | 2 +- core/index.ts | 1 + core/slow.ts | 110 +++++++++++++++++++++++++++++++++++++++++ core/test/slow.test.ts | 104 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 core/slow.ts create mode 100644 core/test/slow.test.ts diff --git a/core/fast.ts b/core/fast.ts index ca534635..3419c497 100644 --- a/core/fast.ts +++ b/core/fast.ts @@ -9,7 +9,7 @@ import { } from './category.js' import { client } from './client.js' import { onEnvironment } from './environment.js' -import { type FeedValue, loadFeed, loadFeeds, MISSED_FEED } from './feed.js' +import { BROKEN_FEED, type FeedValue, loadFeed, loadFeeds } from './feed.js' import { loadFilters } from './filter.js' import { deletePost, getPost, loadPosts } from './post.js' import type { PostValue } from './post.js' @@ -162,7 +162,7 @@ async function load(categoryId: string, since?: number): Promise { let entries = await Promise.all( posts.map(async post => { - return { feed: (await loadFeed(post.feedId)) ?? MISSED_FEED, post } + return { feed: (await loadFeed(post.feedId)) ?? BROKEN_FEED, post } }) ) diff --git a/core/feed.ts b/core/feed.ts index 1efff2bc..dba17014 100644 --- a/core/feed.ts +++ b/core/feed.ts @@ -127,7 +127,7 @@ export function testFeed(feed: Partial = {}): FeedValue { } } -export const MISSED_FEED: FeedValue = { +export const BROKEN_FEED: FeedValue = { categoryId: 'general', id: 'missed', lastOriginId: undefined, diff --git a/core/index.ts b/core/index.ts index 4feaa76a..416c5c70 100644 --- a/core/index.ts +++ b/core/index.ts @@ -20,3 +20,4 @@ export * from './feed.js' export * from './post.js' export * from './i18n.js' export * from './html.js' +export * from './slow.js' diff --git a/core/slow.ts b/core/slow.ts new file mode 100644 index 00000000..ebeb2a6d --- /dev/null +++ b/core/slow.ts @@ -0,0 +1,110 @@ +import { atom, onMount } from 'nanostores' + +import { + BROKEN_CATEGORY, + type CategoryValue, + GENERAL_CATEGORY, + loadCategories +} from './category.js' +import { client } from './client.js' +import { BROKEN_FEED, type FeedValue, loadFeed } from './feed.js' +import { loadPosts } from './post.js' +import { readonlyExport } from './utils/stores.js' + +export type SlowCategoriesTree = [CategoryValue, [FeedValue, number][]][] + +export type SlowCategoriesValue = + | { + isLoading: false + tree: SlowCategoriesTree + } + | { isLoading: true } + +let $categories = atom({ isLoading: true }) + +async function findSlowCategories(): Promise { + let [posts, categories] = await Promise.all([ + loadPosts({ reading: 'slow' }), + loadCategories() + ]) + + let general: [FeedValue, number][] = [] + let byCategory: Record = {} + let broken: [FeedValue, number][] = [] + + let postsByFeed: Record = {} + for (let post of posts) { + postsByFeed[post.feedId] = (postsByFeed[post.feedId] ?? 0) + 1 + } + + await Promise.all( + Object.entries(postsByFeed).map(async ([feedId, unread]) => { + let feed = (await loadFeed(feedId)) ?? BROKEN_FEED + let category = feed.categoryId + if (feed.categoryId === 'general') { + general.push([feed, unread]) + } + if (category === 'general' || categories.find(i => i.id === category)) { + let list = byCategory[category] ?? (byCategory[category] = []) + list.push([feed, unread]) + } else { + broken.push([feed, unread]) + } + }) + ) + + let categoriesByName = categories.sort((a, b) => { + return a.title.localeCompare(b.title) + }) + + let result: SlowCategoriesTree = [] + if (general.length > 0) { + result.push([GENERAL_CATEGORY, general]) + } + for (let category of categoriesByName) { + let list = byCategory[category.id] + if (list) { + result.push([category, list]) + } + } + if (broken.length > 0) { + result.push([BROKEN_CATEGORY, broken]) + } + + return result +} + +onMount($categories, () => { + $categories.set({ isLoading: true }) + + let unbindLog: (() => void) | undefined + let unbindClient = client.subscribe(loguxClient => { + unbindLog?.() + unbindLog = undefined + + if (loguxClient) { + findSlowCategories().then(tree => { + $categories.set({ isLoading: false, tree }) + }) + + unbindLog = loguxClient.log.on('add', action => { + if ( + action.type.startsWith('categories/') || + action.type.startsWith('feeds/') || + action.type.startsWith('posts/') + ) { + findSlowCategories().then(tree => { + $categories.set({ isLoading: false, tree }) + }) + } + }) + } + }) + + return () => { + unbindLog?.() + unbindClient() + } +}) + +export const slowCategories = readonlyExport($categories) diff --git a/core/test/slow.test.ts b/core/test/slow.test.ts new file mode 100644 index 00000000..49c28285 --- /dev/null +++ b/core/test/slow.test.ts @@ -0,0 +1,104 @@ +import { cleanStores, keepMount } from 'nanostores' +import { deepStrictEqual } from 'node:assert' +import { afterEach, beforeEach, test } from 'node:test' +import { setTimeout } from 'node:timers/promises' + +import { + addCategory, + addFeed, + addPost, + BROKEN_CATEGORY, + BROKEN_FEED, + changeCategory, + deletePost, + GENERAL_CATEGORY, + loadCategory, + loadFeed, + slowCategories, + testFeed, + testPost +} from '../index.js' +import { cleanClientTest, enableClientTest } from './utils.js' + +beforeEach(() => { + enableClientTest() +}) + +afterEach(async () => { + cleanStores(slowCategories) + await setTimeout(10) + await cleanClientTest() +}) + +test('has empty slow categories from beginning', async () => { + deepStrictEqual(slowCategories.get(), { isLoading: true }) + await setTimeout(100) + deepStrictEqual(slowCategories.get(), { + isLoading: false, + tree: [] + }) +}) + +test('returns slow feeds', async () => { + keepMount(slowCategories) + let categoryB = await addCategory({ title: 'B' }) + let categoryA = await addCategory({ title: 'A' }) + let feedA1 = await addFeed(testFeed({ categoryId: categoryA })) + let feedA2 = await addFeed(testFeed({ categoryId: categoryA })) + let feedA3 = await addFeed(testFeed({ categoryId: categoryA })) + let feedB = await addFeed(testFeed({ categoryId: categoryB })) + let feedC = await addFeed(testFeed()) + let feedD = await addFeed(testFeed({ categoryId: 'unknown' })) + await addPost(testPost({ feedId: feedA1, reading: 'slow' })) + await addPost(testPost({ feedId: feedA1, reading: 'slow' })) + await addPost(testPost({ feedId: feedA1, reading: 'fast' })) + await addPost(testPost({ feedId: feedA2, reading: 'slow' })) + await addPost(testPost({ feedId: feedA3, reading: 'fast' })) + await addPost(testPost({ feedId: feedB, reading: 'slow' })) + await addPost(testPost({ feedId: feedC, reading: 'slow' })) + await addPost(testPost({ feedId: feedD, reading: 'slow' })) + let post9 = await addPost(testPost({ feedId: 'unknown', reading: 'slow' })) + + await setTimeout(10) + deepStrictEqual(slowCategories.get(), { + isLoading: false, + tree: [ + [ + GENERAL_CATEGORY, + [ + [await loadFeed(feedC), 1], + [BROKEN_FEED, 1] + ] + ], + [ + await loadCategory(categoryA), + [ + [await loadFeed(feedA1), 2], + [await loadFeed(feedA2), 1] + ] + ], + [await loadCategory(categoryB), [[await loadFeed(feedB), 1]]], + [BROKEN_CATEGORY, [[await loadFeed(feedD), 1]]] + ] + }) + + await changeCategory(categoryA, { title: 'New A' }) + await deletePost(post9) + + await setTimeout(10) + deepStrictEqual(slowCategories.get(), { + isLoading: false, + tree: [ + [GENERAL_CATEGORY, [[await loadFeed(feedC), 1]]], + [await loadCategory(categoryB), [[await loadFeed(feedB), 1]]], + [ + await loadCategory(categoryA), + [ + [await loadFeed(feedA1), 2], + [await loadFeed(feedA2), 1] + ] + ], + [BROKEN_CATEGORY, [[await loadFeed(feedD), 1]]] + ] + }) +})