feat: queue scrapes

cardboard-ink · May 5, 2024 · 3c1278f · 3c1278f
1 parent 83fff4a
commit 3c1278f
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 16 deletions.
diff --git a/app/package.json b/app/package.json
@@ -12,6 +12,7 @@
     "chalk": "^5.3.0",
     "elysia": "latest",
     "minio": "^8.0.0",
+    "p-queue": "^8.0.1",
     "puppeteer": "^22.7.1"
   },
   "devDependencies": {

diff --git a/app/src/controllers/server-banner.controller.ts b/app/src/controllers/server-banner.controller.ts
@@ -1,18 +1,18 @@
 import { Elysia, t } from "elysia"
-import { serverBannerBucket, guildedServerProfileScrape, streamToBuffer } from "../libs"
+import { serverBannerBucket, guildedServerProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
 
 export const serverBannerController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await serverBannerBucket.checkAssetExists(params.id)) {
             const lastModified = await serverBannerBucket.getAssetLastModified(params.id)
             if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                guildedServerProfileScrape(params.id, 'banner')
+                scrapeQueue.add(async () => guildedServerProfileScrape(params.id, 'banner'))
             }
             const banner = await serverBannerBucket.getAsset(params.id)
             return new Response(await streamToBuffer(banner), { headers: { 'Content-Type': 'image/webp' } })
         }
-        const imageBlob = await guildedServerProfileScrape(params.id, 'banner')
-        if (imageBlob instanceof Error) {
+        const imageBlob = await scrapeQueue.add(async () => await guildedServerProfileScrape(params.id, 'banner'), {priority: 1})
+        if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }
         return new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } })

diff --git a/app/src/controllers/server-icon.controller.ts b/app/src/controllers/server-icon.controller.ts
@@ -1,19 +1,19 @@
 import { Elysia, t } from "elysia"
-import { serverIconBucket, guildedServerProfileScrape, streamToBuffer } from "../libs"
+import { serverIconBucket, guildedServerProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
 
 export const serverIconController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await serverIconBucket.checkAssetExists(params.id)) {
             const lastModified = await serverIconBucket.getAssetLastModified(params.id)
             if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                guildedServerProfileScrape(params.id, 'icon')
+                scrapeQueue.add(() => guildedServerProfileScrape(params.id, 'icon'))
             }
             const avatar = await serverIconBucket.getAsset(params.id)
             const res = new Response(await streamToBuffer(avatar), { headers: { 'Content-Type': 'image/webp' } })
             return res
         }
-        const imageBlob = await guildedServerProfileScrape(params.id, 'icon')
-        if (imageBlob instanceof Error) {
+        const imageBlob = await scrapeQueue.add(async () => await guildedServerProfileScrape(params.id, 'icon'), {priority: 1})
+        if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }
         const res = new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } })

diff --git a/app/src/controllers/user-avatar.controller.ts b/app/src/controllers/user-avatar.controller.ts
@@ -1,19 +1,19 @@
 import { Elysia, t } from "elysia"
-import { userAvatarBucket, guildedUserProfileScrape, streamToBuffer } from "../libs"
+import { userAvatarBucket, guildedUserProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
 
 export const userAvatarController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await userAvatarBucket.checkAssetExists(params.id)) {
             const lastModified = await userAvatarBucket.getAssetLastModified(params.id)
             if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                guildedUserProfileScrape(params.id, 'avatar')
+                scrapeQueue.add(() => guildedUserProfileScrape(params.id, 'avatar'))
             }
             const avatar = await userAvatarBucket.getAsset(params.id)
             const res = new Response(await streamToBuffer(avatar), { headers: { 'Content-Type': 'image/webp' } })
             return res
         }
-        const imageBlob = await guildedUserProfileScrape(params.id, 'avatar')
-        if (imageBlob instanceof Error) {
+        const imageBlob = await scrapeQueue.add(async () => await guildedUserProfileScrape(params.id, 'avatar'), {priority: 1})
+        if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }
         const res = new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } })

diff --git a/app/src/controllers/user-banner.controller.ts b/app/src/controllers/user-banner.controller.ts
@@ -1,17 +1,20 @@
 import { Elysia, t } from "elysia"
-import { userBannerBucket, guildedUserProfileScrape, streamToBuffer } from "../libs"
+import { userBannerBucket, guildedUserProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
 
 export const userBannerController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await userBannerBucket.checkAssetExists(params.id)) {
             const lastModified = await userBannerBucket.getAssetLastModified(params.id)
             if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                guildedUserProfileScrape(params.id, 'banner')
+                scrapeQueue.add(() => guildedUserProfileScrape(params.id, 'banner'))
             }
             const banner = await userBannerBucket.getAsset(params.id)
             return new Response(await streamToBuffer(banner), { headers: { 'Content-Type': 'image/webp' } })
         }
-        const imageBlob = await guildedUserProfileScrape(params.id, 'banner')
+        const imageBlob = await scrapeQueue.add(async () => await guildedUserProfileScrape(params.id, 'banner'), {priority: 1})
+        if (imageBlob instanceof Error || !imageBlob) {
+            return new Response('User not found', { status: 404 })
+        }
         return new Response(imageBlob, { headers: { 'Content-Type': 'image/webp' } })
     }, {
         params: t.Object({

diff --git a/app/src/libs/guilded-scrape.ts b/app/src/libs/guilded-scrape.ts
@@ -1,12 +1,16 @@
 import { userAvatarBucket, userBannerBucket, serverBannerBucket, serverIconBucket } from "./minio"
 import puppeteer from "puppeteer";
 
+import PQueue from 'p-queue';
+
 const browser = await puppeteer.launch({ 
     headless: true,
     args: ['--no-sandbox', '--disable-setuid-sandbox'],
     executablePath: "/usr/bin/chromium",
 })
 
+export const scrapeQueue = new PQueue({concurrency: 3});
+
 export const guildedUserProfileScrape: (id: string, getElement: 'avatar' | 'banner') => Promise<Blob|Error> = async (id: string, getElement: 'avatar' | 'banner') => {
     try {
         const getClass = getElement === 'avatar' ? '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded>.UserProfilePictureControl-picture' : '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded.ProgressiveLoadedImage-container-cover>.UserProfileBackground-image'
@@ -43,4 +47,4 @@ export const guildedServerProfileScrape: (id: string, getElement: 'icon' | 'bann
     } catch (e) {
         return new Error('Server not found')
     }
-}
+}