Merge pull request #541 from mchangrh/etagTest

add etag and other tests
ajayyy · Feb 22, 2023 · d76ee7c · d76ee7c
2 parents f4286b1 + 436e75e
commit d76ee7c
Show file tree

Hide file tree

Showing 42 changed files with 1,626 additions and 1,160 deletions.
diff --git a/ci.json b/ci.json
@@ -56,7 +56,6 @@
             ]
         }
     ],
-    "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {
         "vote": {

diff --git a/package.json b/package.json
@@ -9,8 +9,8 @@
     "cover:report": "nyc report",
     "dev": "nodemon",
     "dev:bash": "nodemon -x 'npm test ; npm start'",
-    "postgres:docker": "docker run --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:14-alpine",
-    "redis:docker": "docker run --rm -p 6379:6379 redis:7-alpine --save '' --appendonly no",
+    "postgres:docker": "docker run --init -it --rm -p 5432:5432 -e POSTGRES_USER=ci_db_user -e POSTGRES_PASSWORD=ci_db_pass postgres:14-alpine",
+    "redis:docker": "docker run --init -it --rm -p 6379:6379 redis:7-alpine --save '' --appendonly no",
     "start": "ts-node src/index.ts",
     "tsc": "tsc -p tsconfig.json",
     "lint": "eslint src test",

diff --git a/src/middleware/userCounter.ts b/src/middleware/userCounter.ts
@@ -14,7 +14,7 @@ export function userCounter(req: Request, res: Response, next: NextFunction): vo
                 method: "post",
                 url: `${config.userCounterURL}/api/v1/addIP?hashedIP=${getIP(req)}`,
                 httpAgent
-            }).catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+            }).catch(() => /* instanbul skip next */ Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
         }
     }
 

diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts
@@ -34,11 +34,11 @@ async function handleGetSegmentInfo(req: Request, res: Response): Promise<DBSegm
     // deduplicate with set
     UUIDs = [ ...new Set(UUIDs)];
     // if more than 10 entries, slice
-    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
-    if (!Array.isArray(UUIDs) || !UUIDs) {
+    if (!Array.isArray(UUIDs) || !UUIDs?.length) {
         res.status(400).send("UUIDs parameter does not match format requirements.");
         return;
     }
+    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
     const DBSegments = await getSegmentsByUUID(UUIDs);
     // all uuids failed lookup
     if (!DBSegments?.length) {

diff --git a/src/routes/getSkipSegmentsByHash.ts b/src/routes/getSkipSegmentsByHash.ts
@@ -25,13 +25,13 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
     try {
         await getEtag("skipSegmentsHash", hashPrefix, service)
             .then(etag => res.set("ETag", etag))
-            .catch(() => null);
+            .catch(/* istanbul ignore next */ () => null);
         const output = Object.entries(segments).map(([videoID, data]) => ({
             videoID,
             segments: data.segments,
         }));
         return res.status(output.length === 0 ? 404 : 200).json(output);
-    } catch(e) {
+    } catch (e) /* istanbul ignore next */ {
         Logger.error(`skip segments by hash error: ${e}`);
 
         return res.status(500).send("Internal server error");

diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts
@@ -2,10 +2,12 @@ import { db } from "../databases/databases";
 import { createMemoryCache } from "../utils/createMemoryCache";
 import { config } from "../config";
 import { Request, Response } from "express";
+import { validateCategories } from "../utils/parseParams";
 
 const MILLISECONDS_IN_MINUTE = 60000;
 // eslint-disable-next-line @typescript-eslint/no-misused-promises
 const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+/* istanbul ignore next */
 const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
 
 interface DBSegment {
@@ -38,7 +40,6 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
         }
     }
 
-
     return {
         userNames,
         viewCounts,
@@ -51,7 +52,7 @@ export async function getTopCategoryUsers(req: Request, res: Response): Promise<
     const sortType = parseInt(req.query.sortType as string);
     const category = req.query.category as string;
 
-    if (sortType == undefined || !config.categoryList.includes(category) ) {
+    if (sortType == undefined || !validateCategories([category]) ) {
         //invalid request
         return res.sendStatus(400);
     }

diff --git a/src/routes/getTotalStats.ts b/src/routes/getTotalStats.ts
@@ -3,6 +3,7 @@ import { config } from "../config";
 import { Request, Response } from "express";
 import axios from "axios";
 import { Logger } from "../utils/logger";
+import { getCWSUsers } from "../utils/getCWSUsers";
 
 // A cache of the number of chrome web store users
 let chromeUsersCache = 0;
@@ -29,30 +30,30 @@ let lastFetch: DBStatsData = {
 updateExtensionUsers();
 
 export async function getTotalStats(req: Request, res: Response): Promise<void> {
-
-    const row = await getStats(!!req.query.countContributingUsers);
+    const countContributingUsers = Boolean(req.query?.countContributingUsers == "true");
+    const row = await getStats(countContributingUsers);
     lastFetch = row;
 
-    if (row !== undefined) {
-        const extensionUsers = chromeUsersCache + firefoxUsersCache;
-
-        //send this result
-        res.send({
-            userCount: row.userCount,
-            activeUsers: extensionUsers,
-            apiUsers: Math.max(apiUsersCache, extensionUsers),
-            viewCount: row.viewCount,
-            totalSubmissions: row.totalSubmissions,
-            minutesSaved: row.minutesSaved,
-        });
-
-        // Check if the cache should be updated (every ~14 hours)
-        const now = Date.now();
-        if (now - lastUserCountCheck > 5000000) {
-            lastUserCountCheck = now;
-
-            updateExtensionUsers();
-        }
+    /* istanbul ignore if */
+    if (!row) res.sendStatus(500);
+    const extensionUsers = chromeUsersCache + firefoxUsersCache;
+
+    //send this result
+    res.send({
+        userCount: row.userCount ?? 0,
+        activeUsers: extensionUsers,
+        apiUsers: Math.max(apiUsersCache, extensionUsers),
+        viewCount: row.viewCount,
+        totalSubmissions: row.totalSubmissions,
+        minutesSaved: row.minutesSaved,
+    });
+
+    // Check if the cache should be updated (every ~14 hours)
+    const now = Date.now();
+    if (now - lastUserCountCheck > 5000000) {
+        lastUserCountCheck = now;
+
+        updateExtensionUsers();
     }
 }
 
@@ -67,42 +68,53 @@ function getStats(countContributingUsers: boolean): Promise<DBStatsData> {
     }
 }
 
-
 function updateExtensionUsers() {
+    /* istanbul ignore else */
     if (config.userCounterURL) {
         axios.get(`${config.userCounterURL}/api/v1/userCount`)
-            .then(res => {
-                apiUsersCache = Math.max(apiUsersCache, res.data.userCount);
-            })
-            .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+            .then(res => apiUsersCache = Math.max(apiUsersCache, res.data.userCount))
+            .catch( /* istanbul ignore next */ () => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
     }
 
     const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/";
     const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone";
+    const chromeExtId = "mnjggcdmjocbbbhaepdhchncahnbgone";
 
     axios.get(mozillaAddonsUrl)
-        .then(res => {
-            firefoxUsersCache = res.data.average_daily_users;
-            axios.get(chromeExtensionUrl)
-                .then(res => {
-                    const body = res.data;
-                    // 2021-01-05
-                    // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
-                    const matchingString = '"UserDownloads:';
-                    const matchingStringLen = matchingString.length;
-                    const userDownloadsStartIndex = body.indexOf(matchingString);
-                    if (userDownloadsStartIndex >= 0) {
-                        const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
-                        const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
-                        chromeUsersCache = parseInt(userDownloadsStr);
-                    }
-                    else {
-                        lastUserCountCheck = 0;
-                    }
-                })
-                .catch(() => Logger.debug(`Failing to connect to ${chromeExtensionUrl}`));
-        })
-        .catch(() => {
+        .then(res => firefoxUsersCache = res.data.average_daily_users )
+        .catch( /* istanbul ignore next */ () => {
             Logger.debug(`Failing to connect to ${mozillaAddonsUrl}`);
+            return 0;
         });
+    getCWSUsers(chromeExtId)
+        .then(res => chromeUsersCache = res)
+        .catch(/* istanbul ignore next */ () =>
+            getChromeUsers(chromeExtensionUrl)
+                .then(res => chromeUsersCache = res)
+        );
 }
+
+/* istanbul ignore next */
+function getChromeUsers(chromeExtensionUrl: string): Promise<number> {
+    return axios.get(chromeExtensionUrl)
+        .then(res => {
+            const body = res.data;
+            // 2021-01-05
+            // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
+            const matchingString = '"UserDownloads:';
+            const matchingStringLen = matchingString.length;
+            const userDownloadsStartIndex = body.indexOf(matchingString);
+            /* istanbul ignore else */
+            if (userDownloadsStartIndex >= 0) {
+                const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
+                const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
+                return parseInt(userDownloadsStr);
+            } else {
+                lastUserCountCheck = 0;
+            }
+        })
+        .catch(/* istanbul ignore next */ () => {
+            Logger.debug(`Failing to connect to ${chromeExtensionUrl}`);
+            return 0;
+        });
+}
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
@@ -120,7 +120,7 @@ async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, video
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
 async function autoModerateSubmission(apiVideoDetails: videoDetails,
-    submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
+    submission: { videoID: VideoID; userID: HashedUserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
     // get duration from API
     const apiDuration = apiVideoDetails.duration;
     // if API fail or returns 0, get duration from client
@@ -156,7 +156,7 @@ async function autoModerateSubmission(apiVideoDetails: videoDetails,
     return false;
 }
 
-async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
+async function checkUserActiveWarning(userID: HashedUserID): Promise<CheckResult> {
     const MILLISECONDS_IN_HOUR = 3600000;
     const now = Date.now();
     const warnings = (await db.prepare("all",
@@ -337,10 +337,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
     return CHECK_PASS;
 }
 
-async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
+async function checkByAutoModerator(videoID: VideoID, userID: HashedUserID, segments: IncomingSegment[], service: Service, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
     // Auto moderator check
     if (service == Service.YouTube) {
-        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration });
+        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { videoID, userID, segments, service, videoDuration });
         if (autoModerateResult) {
             return {
                 pass: false,
@@ -492,7 +492,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);
 
     //hash the userID
-    const userID = await getHashCache(paramUserID || "");
+    if (!paramUserID) {
+        return res.status(400).send("No userID provided");
+    }
+    const userID: HashedUserID = await getHashCache(paramUserID);
 
     const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
     if (!invalidCheckResult.pass) {

diff --git a/src/utils/createMemoryCache.ts b/src/utils/createMemoryCache.ts
@@ -1,4 +1,5 @@
 export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeMs: number): any {
+    /* istanbul ignore if */
     if (isNaN(cacheTimeMs)) cacheTimeMs = 0;
 
     // holds the promise results

diff --git a/src/utils/getCWSUsers.ts b/src/utils/getCWSUsers.ts
@@ -0,0 +1,13 @@
+import axios from "axios";
+import { Logger } from "../utils/logger";
+
+export const getCWSUsers = (extID: string): Promise<number | undefined> =>
+    axios.post(`https://chrome.google.com/webstore/ajax/detail?pv=20210820&id=${extID}`)
+        .then(res => res.data.split("\n")[2])
+        .then(data => JSON.parse(data))
+        .then(data => (data[1][1][0][23]).replaceAll(/,|\+/g,""))
+        .then(data => parseInt(data))
+        .catch((err) => {
+            Logger.error(`Error getting chrome users - ${err}`);
+            return 0;
+        });
diff --git a/src/utils/getHashCache.ts b/src/utils/getHashCache.ts
@@ -28,7 +28,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
                 Logger.debug(`Got data from redis: ${reply}`);
                 return reply as T & HashedValue;
             }
-        } catch (err) {
+        } catch (err) /* istanbul ignore next */ {
             Logger.error(err as string);
         }
     }
@@ -37,7 +37,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
     const data = getHash(key, cachedHashTimes);
 
     if (!config.redis?.disableHashCache) {
-        redis.set(redisKey, data).catch((err) => Logger.error(err));
+        redis.set(redisKey, data).catch(/* istanbul ignore next */ (err) => Logger.error(err));
     }
 
     return data as T & HashedValue;

diff --git a/src/utils/innerTubeAPI.ts b/src/utils/innerTubeAPI.ts
@@ -70,15 +70,16 @@ export async function getPlayerData (videoID: string, ignoreCache = false): Prom
     }
     try {
         const data = await getFromITube(videoID)
-            .catch(err => {
+            .catch(/* istanbul ignore next */ err => {
                 Logger.warn(`InnerTube API Error for ${videoID}: ${err}`);
                 return Promise.reject(err);
             });
         DiskCache.set(cacheKey, data)
             .then(() => Logger.debug(`InnerTube API: video information cache set for: ${videoID}`))
-            .catch((err: any) => Logger.warn(err));
+            .catch(/* istanbul ignore next */ (err: any) => Logger.warn(err));
         return data;
     } catch (err) {
+        /* istanbul ignore next */
         return Promise.reject(err);
     }
 }
diff --git a/src/utils/isUserTempVIP.ts b/src/utils/isUserTempVIP.ts
@@ -11,7 +11,7 @@ export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID
     try {
         const reply = await redis.get(tempVIPKey(hashedUserID));
         return reply && reply == channelID;
-    } catch (e) {
+    } catch (e) /* istanbul ignore next */ {
         Logger.error(e as string);
         return false;
     }

diff --git a/src/utils/logger.ts b/src/utils/logger.ts
@@ -45,6 +45,7 @@ class Logger {
     };
 
     constructor() {
+        /* istanbul ignore if */
         if (config.mode === "development") {
             this._settings.INFO = true;
             this._settings.DEBUG = true;
@@ -73,9 +74,11 @@ class Logger {
 
         let color = colors.Bright;
         if (level === LogLevel.ERROR) color = colors.FgRed;
+        /* istanbul ignore if */
         if (level === LogLevel.WARN) color = colors.FgYellow;
 
         let levelStr = level.toString();
+        /* istanbul ignore if */
         if (levelStr.length === 4) {
             levelStr += " ";  // ensure logs are aligned
         }

diff --git a/src/utils/parseParams.ts b/src/utils/parseParams.ts
@@ -73,3 +73,6 @@ export const parseActionTypes = (req: Request, fallback: ActionType[]): ActionTy
 
 export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined =>
     syntaxErrorWrapper(getRequiredSegments, req, []); // never fall back
+
+export const validateCategories = (categories: string[]): boolean =>
+    categories.every((category: string) => config.categoryList.includes(category));
diff --git a/src/utils/redis.ts b/src/utils/redis.ts
@@ -135,17 +135,21 @@ if (config.redis?.enabled) {
             .then((reply) => resolve(reply))
             .catch((err) => reject(err))
     );
+    /* istanbul ignore next */
     client.on("error", function(error) {
         lastClientFail = Date.now();
         Logger.error(`Redis Error: ${error}`);
     });
+    /* istanbul ignore next */
     client.on("reconnect", () => {
         Logger.info("Redis: trying to reconnect");
     });
+    /* istanbul ignore next */
     readClient?.on("error", function(error) {
         lastReadFail = Date.now();
         Logger.error(`Redis Read-Only Error: ${error}`);
     });
+    /* istanbul ignore next */
     readClient?.on("reconnect", () => {
         Logger.info("Redis Read-Only: trying to reconnect");
     });

diff --git a/test.json b/test.json
@@ -50,7 +50,6 @@
             ]
         }
     ],
-    "maxNumberOfActiveWarnings": 3,
     "hoursAfterWarningExpires": 24,
     "rateLimit": {
         "vote": {