feat: use API

cardboard-ink · May 7, 2024 · 670934b · 670934b
1 parent 3c1278f
commit 670934b
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 51 deletions.
diff --git a/.env.example b/.env.example
@@ -1,7 +1,9 @@
 MINIO_ROOT_USER=hammer
 MINIO_ROOT_PASSWORD=hammerthatwhammer
 MINIO_CONSOLE_PORT=9002
+MINIO_PORT=9000
 
+G_TOKEN=gapi_abc123
 PORT=3000
-MINIO_ACCESS_KEY=G50xHZi7QDNNh2SY6ziY
-MINIO_SECRET_KEY=Oc6zqYDmULb3uzR957KpW8l1iyjOWHSOZRRPl7QX
+MINIO_ACCESS_KEY=KTec8GmMOXz4BhKDZsQh
+MINIO_SECRET_KEY=jMHddZjNXju1WfBUdxgdxAMgczZfWwGwwCDCt0y0
diff --git a/app/src/controllers/server-banner.controller.ts b/app/src/controllers/server-banner.controller.ts
@@ -1,17 +1,17 @@
 import { Elysia, t } from "elysia"
-import { serverBannerBucket, guildedServerProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
+import { serverBannerBucket, guildedServerProfileScrape, streamToBuffer } from "../libs"
 
 export const serverBannerController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await serverBannerBucket.checkAssetExists(params.id)) {
             const lastModified = await serverBannerBucket.getAssetLastModified(params.id)
-            if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                scrapeQueue.add(async () => guildedServerProfileScrape(params.id, 'banner'))
+            if (Date.now() - lastModified.valueOf() > 24 * 60 * 60 * 1000) {
+                guildedServerProfileScrape(params.id, 'banner')
             }
             const banner = await serverBannerBucket.getAsset(params.id)
             return new Response(await streamToBuffer(banner), { headers: { 'Content-Type': 'image/webp' } })
         }
-        const imageBlob = await scrapeQueue.add(async () => await guildedServerProfileScrape(params.id, 'banner'), {priority: 1})
+        const imageBlob = await guildedServerProfileScrape(params.id, 'banner')
         if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }

diff --git a/app/src/controllers/server-icon.controller.ts b/app/src/controllers/server-icon.controller.ts
@@ -1,18 +1,18 @@
 import { Elysia, t } from "elysia"
-import { serverIconBucket, guildedServerProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
+import { serverIconBucket, guildedServerProfileScrape, streamToBuffer } from "../libs"
 
 export const serverIconController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await serverIconBucket.checkAssetExists(params.id)) {
             const lastModified = await serverIconBucket.getAssetLastModified(params.id)
-            if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                scrapeQueue.add(() => guildedServerProfileScrape(params.id, 'icon'))
+            if (Date.now() - lastModified.valueOf() > 24 * 60 * 60 * 1000) {
+                guildedServerProfileScrape(params.id, 'icon')
             }
             const avatar = await serverIconBucket.getAsset(params.id)
             const res = new Response(await streamToBuffer(avatar), { headers: { 'Content-Type': 'image/webp' } })
             return res
         }
-        const imageBlob = await scrapeQueue.add(async () => await guildedServerProfileScrape(params.id, 'icon'), {priority: 1})
+        const imageBlob = await guildedServerProfileScrape(params.id, 'icon')
         if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }
@@ -39,3 +39,4 @@ export const serverIconController = new Elysia()
             tags: ['server'],
         }
     })
+
diff --git a/app/src/controllers/user-avatar.controller.ts b/app/src/controllers/user-avatar.controller.ts
@@ -1,18 +1,21 @@
 import { Elysia, t } from "elysia"
-import { userAvatarBucket, guildedUserProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
+import { userAvatarBucket, guildedUserProfileScrape, streamToBuffer } from "../libs"
 
 export const userAvatarController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await userAvatarBucket.checkAssetExists(params.id)) {
+            console.log('User avatar exists')
             const lastModified = await userAvatarBucket.getAssetLastModified(params.id)
-            if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                scrapeQueue.add(() => guildedUserProfileScrape(params.id, 'avatar'))
+            if (Date.now() - lastModified.valueOf() > 24 * 60 * 60 * 1000) {
+                console.log('User avatar is older than 24 hours')
+                guildedUserProfileScrape(params.id, 'avatar')
             }
+            console.log('Sending cached avatar')
             const avatar = await userAvatarBucket.getAsset(params.id)
             const res = new Response(await streamToBuffer(avatar), { headers: { 'Content-Type': 'image/webp' } })
             return res
         }
-        const imageBlob = await scrapeQueue.add(async () => await guildedUserProfileScrape(params.id, 'avatar'), {priority: 1})
+        const imageBlob = await guildedUserProfileScrape(params.id, 'avatar')
         if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }

diff --git a/app/src/controllers/user-banner.controller.ts b/app/src/controllers/user-banner.controller.ts
@@ -1,17 +1,17 @@
 import { Elysia, t } from "elysia"
-import { userBannerBucket, guildedUserProfileScrape, streamToBuffer, scrapeQueue } from "../libs"
+import { userBannerBucket, guildedUserProfileScrape, streamToBuffer } from "../libs"
 
 export const userBannerController = new Elysia()
     .get('/:id', async ({ params }) => {
         if (await userBannerBucket.checkAssetExists(params.id)) {
             const lastModified = await userBannerBucket.getAssetLastModified(params.id)
-            if (Date.now() - lastModified.valueOf() > 5 * 60 * 1000) {
-                scrapeQueue.add(() => guildedUserProfileScrape(params.id, 'banner'))
+            if (Date.now() - lastModified.valueOf() > 24 * 60 * 60 * 1000) {
+                guildedUserProfileScrape(params.id, 'banner')
             }
             const banner = await userBannerBucket.getAsset(params.id)
             return new Response(await streamToBuffer(banner), { headers: { 'Content-Type': 'image/webp' } })
         }
-        const imageBlob = await scrapeQueue.add(async () => await guildedUserProfileScrape(params.id, 'banner'), {priority: 1})
+        const imageBlob = await guildedUserProfileScrape(params.id, 'banner')
         if (imageBlob instanceof Error || !imageBlob) {
             return new Response('User not found', { status: 404 })
         }

diff --git a/app/src/libs/guilded-scrape.ts b/app/src/libs/guilded-scrape.ts
@@ -1,49 +1,78 @@
 import { userAvatarBucket, userBannerBucket, serverBannerBucket, serverIconBucket } from "./minio"
-import puppeteer from "puppeteer";
 
-import PQueue from 'p-queue';
-
-const browser = await puppeteer.launch({ 
-    headless: true,
-    args: ['--no-sandbox', '--disable-setuid-sandbox'],
-    executablePath: "/usr/bin/chromium",
-})
-
-export const scrapeQueue = new PQueue({concurrency: 3});
+export const guildedMediaLink = (awsUrl?: string) => {
+    // replace https://s3-us-west-2.amazonaws.com/www.guilded.gg with https://cdn.gilcdn.com 
+    // keep everything else same
+    if (!awsUrl) return awsUrl;
+    return awsUrl.replace('https://s3-us-west-2.amazonaws.com/www.guilded.gg', 'https://cdn.gilcdn.com');
+  }
 
 export const guildedUserProfileScrape: (id: string, getElement: 'avatar' | 'banner') => Promise<Blob|Error> = async (id: string, getElement: 'avatar' | 'banner') => {
-    try {
-        const getClass = getElement === 'avatar' ? '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded>.UserProfilePictureControl-picture' : '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded.ProgressiveLoadedImage-container-cover>.UserProfileBackground-image'
-        const page = await browser.newPage()
-        await page.goto(`https://www.guilded.gg/profile/${id}`)
-        await page.waitForSelector(getClass, { timeout: 7000 })
-        const src = await page.$eval(getClass, (el: any) => el.src)
-        if (getElement === 'avatar') {
-            await userAvatarBucket.uploadImage(id, src)
-        } else if (getElement === 'banner') {
-            await userBannerBucket.uploadImage(id, src)
-        }
-        await page.close()
-        return await (await fetch(src)).blob()
-    } catch (e) {
+    console.log(`Scraping user: ${id} ${getElement}`)
+    console.log('trying profile')
+    const profile = await (await fetch(`https://www.guilded.gg/api/users/${id}/profilev3`, {keepalive: false})).json()
+    console.log(profile)
+    if (!profile) {
         return new Error('User not found')
     }
+    const src = getElement === 'avatar' ? guildedMediaLink(profile.profilePictureLg) : guildedMediaLink(profile.profileBannerLg)
+    console.log('signing', process.env.G_TOKEN)
+    const signed = await(await fetch(`https://www.guilded.gg/api/v1/url-signatures`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'Authorization': `Bearer ${process.env.G_TOKEN}`
+        },
+        body: JSON.stringify({
+            urls: [src]
+        }),
+        keepalive: false
+    })).json()
+    console.log(signed)
+    if (!signed) {
+        return new Error('Failed to sign URL')
+    }
+    const signedSrc = signed.urlSignatures[0].url
+    console.log(signedSrc)
+    if (getElement === 'avatar') {
+        await userAvatarBucket.uploadImage(id, signedSrc)
+    } else if (getElement === 'banner') {
+        await userBannerBucket.uploadImage(id, signedSrc)
+    }
+    return await (await fetch(signedSrc)).blob()
 }
 
 export const guildedServerProfileScrape: (id: string, getElement: 'icon' | 'banner') => Promise<Blob | Error> = async (id, getElement) => {
+    console.log(`Scraping server: ${id} ${getElement}`)
     try {
-        const getClass = getElement === 'icon' ? '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded>.TeamPlaqueV2-profile-pic' : '.ProgressiveLoadedImage-container.ProgressiveLoadedImage-container-progressive-loaded.ProgressiveLoadedImage-container-src-loaded>.TeamOverviewBanner-banner.TeamPageBanner-overview-banner'
-        const page = await browser.newPage()
-        await page.goto(`https://www.guilded.gg/teams/${id}/overview`)
-        await page.waitForSelector(getClass, { timeout: 7000 })
-        const src = await page.$eval(getClass, (el: any) => el.src)
+        const server = await (await fetch(`https://www.guilded.gg/api/teams/${id}/info`, {keepalive: false})).json()
+        if (!server) {
+            return new Error('Server not found')
+        }
+        const src = getElement === 'icon' ? guildedMediaLink(server.profilePicture) : guildedMediaLink(server.teamDashImage)
+        const signed = await(await fetch(`https://www.guilded.gg/api/v1/url-signatures`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'Authorization': `Bearer ${process.env.G_TOKEN}`
+            },
+            body: JSON.stringify({
+                urls: [src]
+            }),
+            keepalive: false
+        })).json()
+        if (!signed) {
+            return new Error('Failed to sign URL')
+        }
+        const signedSrc = signed.urlSignatures[0].url
         if (getElement === 'icon') {
-            await serverIconBucket.uploadImage(id, src)
+            await serverIconBucket.uploadImage(id, signedSrc)
         } else if (getElement === 'banner') {
-            await serverBannerBucket.uploadImage(id, src)
+            await serverBannerBucket.uploadImage(id, signedSrc)
         }
-        await page.close()
-        return await (await fetch(src)).blob()
+        return await (await fetch(signedSrc)).blob()
     } catch (e) {
         return new Error('Server not found')
     }

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -32,6 +32,7 @@ services:
       MINIO_PORT: 9000
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:?Enter the Minio Access Key from the Admin Dashboard}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:?Enter the Minio Secret Key from the Admin Dashboard}
+      G_TOKEN: ${G_TOKEN:?Enter the Guilded Token}
     build: 
       context: .
       dockerfile: ./docker/app.dockerfile