Merge pull request #13 from maicss/async-pool

post: add promise pool

.gitignore

@@ -4,3 +4,4 @@ node_modules
 .idea
 .DS_Store
 tmp
+coverage

.husky/commit-msg

@@ -2,7 +2,7 @@
 . "$(dirname -- "$0")/_/husky.sh"
 
 message="$(cat $1)"
-requiredPattern="^(feat|chore|WIP|add|cut|fix|bump|make|start|stop|refactor|reformat|optimise|document|merge): .*$"
+requiredPattern="^(feat|chore|WIP|fix|docs|ci|refactor|style|test|revert|post)(\([\w]+\))?!?: .*$"
 if ! [[ $message =~ $requiredPattern ]];
 then
   echo "-"
@@ -11,8 +11,7 @@ then
   echo "🚨 Wrong commit message! 😕"
   echo "The commit message must have this format:"
   echo "<verb in imperative mood> <what was done>"
-  echo "Allowed verbs in imperative mood: add, cut, fix, bump, make, start, stop, refactor, reformat, optimise, document, merge"
-  echo "Example: add login button"
+  echo "Allowed verbs in imperative mood:feat|chore|WIP|fix|docs|ci|refactor|style|test|revert"
   echo "-"
   echo "Your commit message was:"
   echo $message

.vitepress/theme/BlogIndex.vue

@@ -63,4 +63,4 @@ const changePage = (page:number|'prev'|'next') => {
   content: ''
 }
 
-</style>
+</style>

.vitepress/theme/GiscusLayout.vue

@@ -36,7 +36,9 @@ const foldCode = () => {
           const codeFoldMask = document.createElement('div')
           codeFoldMask.className = 'code-fold-mask'
           codeFoldMask.innerHTML =
-            '<svg viewBox="0 0 1024 1024" width="24px" class="code-fold-button" xmlns="http://www.w3.org/2000/svg"><path fill="currentColor" d="M104.704 338.752a64 64 0 0 1 90.496 0l316.8 316.8 316.8-316.8a64 64 0 0 1 90.496 90.496L557.248 791.296a64 64 0 0 1-90.496 0L104.704 429.248a64 64 0 0 1 0-90.496z"></path></svg>'
+            `<svg viewBox="0 0 1024 1024" width="24px" class="code-fold-button" xmlns="http://www.w3.org/2000/svg">
+              <path fill="currentColor" d="M104.704 338.752a64 64 0 0 1 90.496 0l316.8 316.8 316.8-316.8a64 64 0 0 1 90.496 90.496L557.248 791.296a64 64 0 0 1-90.496 0L104.704 429.248a64 64 0 0 1 0-90.496z"/>
+            </svg>`
           codeFoldMask.addEventListener('click', () => {
             const svg = codeFoldMask.querySelector('svg')!
             if (svg.classList.contains('reverse')) {

.vitepress/theme/blog.data.ts

@@ -14,7 +14,7 @@ declare const data: Post[]
 export { data }
 
 export default createContentLoader('/pages/blog/*.md', {
-  excerpt: '<!--more-->',
+  excerpt: '<!-- more -->',
   render: false,
   transform(raw): Post[] {
     return raw
@@ -41,4 +41,4 @@ function formatDate(raw: string): Post['date'] {
       day: 'numeric'
     })
   }
-}
+}

README.md

@@ -13,8 +13,8 @@ based on [VitePress](https://vitepress.dev/)
     - [ ] pyqt6
 - [x] 添加评论
 - [ ] 添加路由级别 transition
-- [ ] 添加 blog 列表自动生成，并输出到blog index，并添加分页
-- [ ] 添加 blog 上一篇下一篇页脚
+- [x] 添加 blog 列表自动生成，并输出到blog index，并添加分页
+- [x] 添加 blog 上一篇下一篇页脚
 - [ ] 添加 tag
 - [ ] 添加 tag 过滤
 - [ ] 添加 代码块横向过长处理，纵向过长处理

pages/blog/change-locales.md

@@ -19,7 +19,7 @@ Some commands used in my daily life
  - Install debconf (i.e. run apt-get update then apt-get install debconf，as root)
  - Run dpkg-reconfigure locales as root
 
-<!--more-->
+<!-- more -->
 ## The Hard Way
 Edit `/etc/locale.gen` as root。If `/etc/locale.gen` does not exist，create it。An example `/etc/locale.gen` is below。
 Run `/usr/sbin/locale-gen` as root

pages/blog/comment-test.md

@@ -24,7 +24,7 @@ next:
 白雲一片去悠悠，青楓浦上不勝愁。
 誰家今夜扁舟子？何處相思明月樓？
 可憐樓上月徘徊，應照離人妝鏡台。
-<!--more-->
+<!-- more -->
 玉戶簾中卷不去，擣衣砧上拂還來。
 此時相望不相聞，願逐月華流照君。
 鴻雁長飛光不度，魚龍潛躍水成文。
@@ -353,4 +353,4 @@ class WebParser:
 
 ```
 
-这里是正文
+这里是正文

pages/blog/promise-pool.md

@@ -0,0 +1,222 @@
+---
+date: 2023/11/19
+description: javascript promise pool
+image: promise-pool.webp
+prev:
+  text: Frontmatter Config
+  link: /blog/test
+next:
+  text: Blog 首页
+  link: /blog/
+---
+
+# javascript promise pool
+
+在日常搬砖中或者面试题都遇到过异步控制的需求，简单的比如直接处理成二维数组和 `Promise.all` 就完事了。
+但是更好的方法是创建一个异步池效率会更佳，下面这个图片更加形象的说明了异步池和 `Promise.all` 分批的区别：
+![promise pool figure](./images/promise-pool.webp)
+图片来自网络搜索。
+使用浏览器加载大量资源的时候，也会看到类似的网络请求瀑布图。
+
+控制并发数的有个很典型的场景，爬虫。`nodejs` 并没有像浏览器一样会限制并发数量，所以所有的请求一股脑全都发出去是非常不明智的行为。
+
+最近搬砖的时候遇到了一个场景，页面上有很多图片需要加载，分批+懒加载优化之后，还是可能遇到同时加载超过六张图片。众所周知，Chrome 并发只有 6 个，如果所有的资源都给图片的话，会导致其他可能需要加载的资源就会卡住，而且可能会卡很久，所以很需要异步控制，趁着这个机会，把之前写过的代码整理一下。
+
+<!-- more -->
+
+## 代码
+
+直接上 Typescript 的版本，适用的场景更多，给兄弟们复制粘贴的时候更加方便：wink：。
+
+```typescript
+interface AsyncData<T> {
+  success: boolean
+  data?: T
+}
+
+function asyncPool<T>(asyncFns: (() => PromiseLike<T>)[], type = 'all', concurrency = 3, timeout = 0) {
+  if (!asyncFns.length) return []
+  let currentIndex = 0
+  let runningJobs = 0
+  const result: AsyncData<T>[] = []
+  let timer: null | number = null
+  return new Promise((resolve, reject) => {
+    const responseHandler = (index: number, success: boolean) => (data: T) => {
+      if (type === 'all' && !success) {
+        if (timer) clearTimeout(timer)
+        return reject(data)
+      }
+      result[index] = { data, success }
+      runningJobs--
+      getNewJob()
+    }
+    const getNewJob = () => {
+      // clear timeout when all jobs are done
+      if (currentIndex === asyncFns.length && runningJobs === 0) {
+        if (timer) clearTimeout(timer)
+        resolve(result)
+      }
+      while (runningJobs < concurrency && currentIndex < asyncFns.length) {
+        asyncFns[currentIndex]().then(responseHandler(currentIndex, true), responseHandler(currentIndex, false))
+        currentIndex++
+        runningJobs++
+      }
+    }
+    if (timeout > 0) {
+      timer = setTimeout(() => reject('timeout'), timeout)
+    }
+    getNewJob()
+  })
+}
+```
+
+## 工具函数
+
+为了测试方便，我们写一个生成 promise 的函数。
+
+```typescript
+const createAsyncFn = (time: number, fail?: boolean) => () =>
+  new Promise((resolve, reject) =>
+    setTimeout(() => {
+      if (fail) {
+        reject(time)
+      } else {
+        resolve(time)
+      }
+    }, time)
+  )
+```
+
+## 测试用例
+
+使用 `vitest`，覆盖率 100%。
+
+```javascript
+import { asyncPool, createAsyncFn } from './asyncPool.mjs'
+import { describe, expect, vi, it } from 'vitest'
+const maxAllowedDeviation = 100 // 允许最大误差。实际运行下来，50也是可以的
+describe('tests of asyncPool', () => {
+  it('should return [] when gives []', () => {
+    expect(asyncPool([])).toEqual([])
+  })
+
+  it('all mode, should use time less than 980ms + 100ms', async () => {
+    const fns = [
+      createAsyncFn(300),
+      createAsyncFn(100),
+      createAsyncFn(200),
+
+      createAsyncFn(300),
+      createAsyncFn(280),
+      createAsyncFn(150),
+
+      createAsyncFn(300),
+      createAsyncFn(400),
+      createAsyncFn(500),
+    ]
+    // 300 + 150 + 400 = 850
+    // 100 + 300 + 300 = 700
+    // 200 + 280 + 500 = 980
+    const startTime = Date.now()
+    await asyncPool(fns)
+    const endTime = Date.now()
+    const elapsedTime = endTime - startTime
+    const expectedTime = 980
+
+    expect(elapsedTime).toBeGreaterThanOrEqual(expectedTime)
+    expect(elapsedTime).toBeLessThanOrEqual(expectedTime + maxAllowedDeviation)
+  })
+  it('all mode, reject, should use time less than 850ms + 100ms', async () => {
+    const fns = [
+      createAsyncFn(300),
+      createAsyncFn(100),
+      createAsyncFn(200),
+
+      createAsyncFn(300),
+      createAsyncFn(280),
+      createAsyncFn(150),
+
+      createAsyncFn(300),
+      createAsyncFn(400, true),
+      createAsyncFn(500),
+    ]
+    // 300 + 150 + 400 = 850 reject here
+    // 100 + 300 + 300 = 700
+    // 200 + 280 + 500 = 980
+    const startTime = Date.now()
+    try {
+      await asyncPool(fns)
+    } catch (e) {}
+    const endTime = Date.now()
+    const elapsedTime = endTime - startTime
+    const expectedTime = 850
+
+    expect(elapsedTime).toBeGreaterThanOrEqual(expectedTime)
+    expect(elapsedTime).toBeLessThanOrEqual(expectedTime + maxAllowedDeviation)
+  })
+
+  it('all mode with timeout', async () => {
+    const fns = [
+      createAsyncFn(300),
+      createAsyncFn(100),
+      createAsyncFn(200),
+
+      createAsyncFn(300),
+      createAsyncFn(280),
+      createAsyncFn(150),
+
+      createAsyncFn(300),
+      createAsyncFn(400, true),
+      createAsyncFn(500),
+    ]
+    // 300 + 150 + 400 = 850 reject here
+    // 100 + 300 + 300 = 700
+    // 200 + 280 + 500 = 980
+    const startTime = Date.now()
+    try {
+      await asyncPool(fns, 'all', 3, 400)
+    } catch (e) {}
+    const endTime = Date.now()
+    const elapsedTime = endTime - startTime
+    const expectedTime = 400
+
+    expect(elapsedTime).toBeGreaterThanOrEqual(expectedTime)
+    expect(elapsedTime).toBeLessThanOrEqual(expectedTime + maxAllowedDeviation)
+  })
+
+  it('settle mode, should use time less than 1610ms + 100ms', async () => {
+    const fns = [
+      createAsyncFn(300),
+      createAsyncFn(200),
+      createAsyncFn(100),
+
+      createAsyncFn(500),
+      createAsyncFn(600),
+      createAsyncFn(400, true),
+
+      createAsyncFn(700),
+      createAsyncFn(1000, true),
+      createAsyncFn(800),
+    ]
+    // 0,300 + 5,400 + 8,1000 = 1710
+    // 1,200 + 4,600 + 7,800 = 1600
+    // 2,100 + 3,500 + 6,700 = 1300
+    const startTime = Date.now()
+    await asyncPool(fns, 'settle')
+    const endTime = Date.now()
+    const elapsedTime = endTime - startTime
+    const expectedTime = 1710
+
+    expect(elapsedTime).toBeGreaterThanOrEqual(expectedTime)
+    expect(elapsedTime).toBeLessThanOrEqual(expectedTime + maxAllowedDeviation)
+  })
+})
+```
+
+## 总结
+
+这个示例是拿到全部的 Promise 结果之后再统一返回，现实场景可能会更加复杂，比如可能需要维护一个 queue，动态的添加 promise，而且要每个 promise 执行完毕之后都直接返回，方便后续处理。
+
+这里只实现了 Promise 异步控制的 `Promise.all` 和 `Promise.allSettled` 两个场景的实现。其他实现比如 `Promise.race` 或 `Promise.any` 一般来说很少有使用并发池的场景，可以尝试自己实现。
+
+总之，这里的实现只是一个玩具类型的，实际项目推荐使用 [promise-pool](https://superchargejs.com/docs/3.x/promise-pool)。

pages/blog/test.md

@@ -6,8 +6,8 @@ prev:
   text: comment test page
   link: /blog/comment-test
 next:
-  text: Blog 首页
-  link: /blog/
+  text: javascript promise pool
+  link: /blog/promise-pool
 ---
 # Frontmatter Config