lukatarman · lukatarman · Apr 11, 2024 · Dec 18, 2023 · Dec 19, 2023 · Dec 19, 2023
diff --git a/backend/assets/html.details.pages.mock.js b/backend/assets/html.details.pages.mock.js
@@ -1,5 +1,21 @@
+import { parseHTML } from "linkedom";
+
+export function createHtmlDetailsPage(page) {
+  return { page, id: 1 };
+}
+
 export function createHtmlDetailsPages(pages) {
   return pages.map((page, index) => {
     return { page, id: index + 1 };
   });
 }
+
+export function getParsedHtmlPage(page) {
+  return parseHTML(page).document;
+}
+
+export function getParsedHtmlPages(pages) {
+  return pages.map((page, index) => {
+    return { page: parseHTML(page).document, id: index + 1 };
+  });
+}
diff --git a/backend/assets/steam-details-pages/feartress.game.html.details.page.js b/backend/assets/steam-details-pages/feartress.game.html.details.page.js
@@ -821,7 +821,7 @@ export const feartressGameHtmlDetailsPage = `<!DOCTYPE html>
 
 																			<div class="grid_label grid_date">Release</div>
 										<div class="grid_content grid_date">
-											TBD										</div>
+											10.10.2001										</div>
 																	</div>
 
 
@@ -842,7 +842,7 @@ export const feartressGameHtmlDetailsPage = `<!DOCTYPE html>
 
 															<div class="release_date">
 									<div class="subtitle column">Release Date:</div>
-									<div class="date">TBD</div>
+									<div class="date">1 January, 2001</div>
 								</div>
 
 																						<div class="dev_row">

diff --git a/backend/assets/steamdb-details-pages/elden.ring.html.details.page.js b/backend/assets/steamdb-details-pages/elden.ring.html.details.page.js
diff --git a/backend/src/adapters/driven/http/steam.client.js b/backend/src/adapters/driven/http/steam.client.js
@@ -1,4 +1,5 @@
 import { SteamApp } from "../../../core/models/steam.app.js";
+import { ValidDataSources } from "../../../core/models/valid.data.sources.js";
 
 export class SteamClient {
   #httpClient;
@@ -33,38 +34,27 @@ export class SteamClient {
     ).map((player) => (player ? player.data.response.player_count : 0));
   }
 
-  async getSteamAppHtmlDetailsPage(id) {
-    // TODO https://github.com/lukatarman/steam-game-stats/issues/192
-    // We add the try catch block here to not crash the whole backend when our application comes accross
-    // errors written into the Steam API
-    try {
-      const url = `https://store.steampowered.com/app/${id}`;
-
-      return (await this.#httpClient.get(url)).data;
-    } catch (err) {
-      return "";
-    }
-  }
+  // TODO https://github.com/lukatarman/steam-game-stats/issues/192
+  async getSourceHtmlDetailsPage(id, source) {
+    const url = this.getSourceUrl(id, source);
 
-  async getSteamchartsGameHtmlDetailsPage(id) {
-    // TODO https://github.com/lukatarman/steam-game-stats/issues/192
     try {
-      const url = `https://steamcharts.com/app/${id}`;
-
       return (await this.#httpClient.get(url)).data;
     } catch (err) {
       return "";
     }
   }
 
-  async getSteamDbHtmlDetailsPage(id) {
-    // TODO https://github.com/lukatarman/steam-game-stats/issues/192
-    try {
-      const url = `https://steamdb.info/app/${id}/info/`;
+  getSourceUrl(id, source) {
+    switch (source) {
+      case ValidDataSources.validDataSources.steamWeb:
+        return `https://store.steampowered.com/app/${id}`;
 
-      return (await this.#httpClient.get(url)).data;
-    } catch (error) {
-      return "";
+      case ValidDataSources.validDataSources.steacharts:
+        return `https://steamcharts.com/app/${id}`;
+
+      case ValidDataSources.validDataSources.steamDb:
+        return `https://steamdb.info/app/${id}/info/`;
     }
   }
 }
diff --git a/backend/src/core/features/game-identifier/game.identifier.js b/backend/src/core/features/game-identifier/game.identifier.js
@@ -1,14 +1,6 @@
-import {
-  discoverGamesFromSteamWeb,
-  updateTypeSideEffectFree,
-  identifyGames,
-  assignType,
-  updateMissingDetails,
-  updateMissingReleaseDates,
-  recordAttemptsViaSteamDb,
-} from "../../services/game.service.js";
 import { delay } from "../../../common/time.utils.js";
 import { HistoryCheck } from "../../models/history.check.js";
+import { ValidDataSources } from "../../models/valid.data.sources.js";
 
 export class GameIdentifier {
   #steamClient;
@@ -17,6 +9,7 @@ export class GameIdentifier {
   #historyChecksRepository;
   #logger;
   #options;
+  #htmlParser;
 
   constructor(
     steamClient,
@@ -25,203 +18,155 @@ export class GameIdentifier {
     historyChecksRepository,
     logger,
     options,
+    htmlParser,
   ) {
     this.#steamClient = steamClient;
     this.#steamAppsRepository = steamAppsRepository;
     this.#gamesRepository = gamesRepository;
     this.#historyChecksRepository = historyChecksRepository;
     this.#logger = logger;
     this.#options = options;
+    this.#htmlParser = htmlParser;
   }
 
-  tryViaSteamWeb = async () => {
-    this.#logger.debugc("identifying games via steam web");
+  checkIfGameViaSource = async (source) => {
+    this.#logger.debugc(`identifying games via ${source}`);
 
-    const steamApps = await this.#steamAppsRepository.getSteamWebUntriedFilteredSteamApps(
+    const steamApps = await this.#steamAppsRepository.getSourceUntriedFilteredSteamApps(
       this.#options.batchSize,
+      source,
     );
-    if (steamApps.length === 0) {
+
+    if (steamApps.isEmpty) {
       this.#logger.debugc(
         `no steam apps in db, retry in: ${this.#options.globalIterationDelay} ms`,
       );
+
       return;
     }
 
-    const [games, updatedSteamApps] = await this.#identifyViaSteamWeb(steamApps);
-
-    this.#persist(games, updatedSteamApps);
-  };
+    const htmlDetailsPages = await this.#getSteamAppsHtmlDetailsPages(
+      steamApps.content,
+      source,
+    );
 
-  async #identifyViaSteamWeb(steamApps) {
-    const htmlDetailsPages = await this.#getSteamAppsHtmlDetailsPages(steamApps);
+    steamApps.identifyTypes(htmlDetailsPages, source);
 
-    const games = discoverGamesFromSteamWeb(steamApps, htmlDetailsPages);
+    const games = steamApps.extractGames(htmlDetailsPages, source);
 
-    const updatedSteamApps = updateTypeSideEffectFree(steamApps, htmlDetailsPages);
+    await this.#persistGameCheckUpdates(games, steamApps.content);
+  };
 
-    return [games, updatedSteamApps];
-  }
+  async #getSteamAppsHtmlDetailsPages(steamApps, source) {
+    const htmlDetailsPages = [];
 
-  async #getSteamAppsHtmlDetailsPages(steamApps) {
-    const detailsPages = [];
     for (let steamApp of steamApps) {
-      detailsPages.push(
-        // TODO https://github.com/lukatarman/steam-game-stats/issues/192
-        await this.#steamClient.getSteamAppHtmlDetailsPage(steamApp.appid),
+      // TODO https://github.com/lukatarman/steam-game-stats/issues/192
+      const htmlPage = await this.#steamClient.getSourceHtmlDetailsPage(
+        steamApp.appid,
+        source,
       );
+
+      htmlDetailsPages.push({
+        page: this.#htmlParser(htmlPage).document,
+        id: steamApp.appid,
+      });
+
       await delay(this.#options.unitDelay);
     }
-    return detailsPages;
+
+    return htmlDetailsPages;
   }
 
-  async #persist(games, updatedSteamApps) {
+  async #persistGameCheckUpdates(games, steamApps) {
     if (games.length !== 0) {
       this.#logger.debugc(`persiting ${games.length} identified games`);
+
       await this.#gamesRepository.insertManyGames(games);
       await this.#historyChecksRepository.insertManyHistoryChecks(
         HistoryCheck.manyFromGames(games),
       );
     }
-    await this.#steamAppsRepository.updateSteamAppsById(updatedSteamApps);
-  }
-
-  tryViaSteamchartsWeb = async () => {
-    this.#logger.debugc("identifying games via steamcharts web");
-
-    const steamApps =
-      await this.#steamAppsRepository.getSteamchartsUntriedFilteredSteamApps(
-        this.#options.batchSize,
-      );
-    if (steamApps.length === 0) {
-      this.#logger.debugc(
-        `no steam apps in db, retry in: ${this.#options.globalIterationDelay} ms`,
-      );
-      return;
-    }
 
-    const updatedSteamApps = await this.#updateStatusViaSteamchartsWeb(steamApps);
-    const games = identifyGames(updatedSteamApps);
-
-    this.#persist(games, updatedSteamApps);
-  };
-
-  // this method is a mess right now, including the tests. Will be fixed in issue: #198
-  async #updateStatusViaSteamchartsWeb(steamApps) {
-    const updatedSteamApps = [];
-
-    for (let steamApp of steamApps) {
-      const steamAppCopy = steamApp.copy();
-      steamAppCopy.triedViaSteamchartsWeb();
-
-      // TODO https://github.com/lukatarman/steam-game-stats/issues/192
-      const result = await this.#steamClient.getSteamchartsGameHtmlDetailsPage(
-        steamApp.appid,
-      );
-
-      if (result === "") {
-        steamAppCopy.failedViaSteamchartsWeb();
-        this.#logger.debugc(`no entry on steamcharts web for appid: ${steamApp.appid}`);
-      }
-
-      assignType(result, steamAppCopy);
-
-      await delay(this.#options.unitDelay);
-      updatedSteamApps.push(steamAppCopy);
-    }
-
-    return updatedSteamApps;
+    await this.#steamAppsRepository.updateSteamAppsById(steamApps);
   }
 
   updateGamesWithoutDetails = async () => {
+    const source = ValidDataSources.validDataSources.steamDb;
     this.#logger.debugc("updating games without details");
 
     const games = await this.#gamesRepository.getGamesWithoutDetails(
       this.#options.batchSize,
     );
 
-    if (games.length === 0) {
+    if (games.isEmpty) {
       this.#logger.debugc(
-        `no games without details in db, retrying in ${
+        `no games without details in db, retry in: ${
           this.#options.globalIterationDelay
         } ms`,
       );
       return;
     }
 
-    const steamApps = await this.#steamAppsRepository.getSteamAppsById(
-      games.map((game) => game.id),
-    );
+    const steamApps = await this.#steamAppsRepository.getSteamAppsById(games.ids);
 
-    const htmlDetailsPages = await this.#getSteamDbHtmlDetailsPages(games);
+    const htmlDetailsPages = await this.#getSteamAppsHtmlDetailsPages(
+      steamApps.content,
+      source,
+    );
 
-    const updatedApps = recordAttemptsViaSteamDb(steamApps, htmlDetailsPages);
+    steamApps.recordAttemptsViaSource(htmlDetailsPages, source);
 
-    updateMissingDetails(games, htmlDetailsPages);
+    games.updateGameDetailsFrom(htmlDetailsPages);
 
-    this.#persistMissingProperties(games, updatedApps);
+    this.#persistUpdatedDetails(games.content, steamApps.content);
   };
 
-  async #getSteamDbHtmlDetailsPages(games) {
-    const htmlDetailsPages = [];
-
-    for (let game of games) {
-      // TODO https://github.com/lukatarman/steam-game-stats/issues/192
-      const htmlPage = await this.#steamClient.getSteamDbHtmlDetailsPage(game.id);
-      htmlDetailsPages.push({ page: htmlPage, id: game.id });
-
-      await delay(this.#options.unitDelay);
-    }
-
-    return htmlDetailsPages;
-  }
-
-  async #persistMissingProperties(games, appsWithoutPages) {
-    if (appsWithoutPages.length !== 0) {
-      this.#logger.debugc(`persisting ${appsWithoutPages.length} apps without pages`);
-      this.#steamAppsRepository.updateSteamAppsById(appsWithoutPages);
-    }
-
+  async #persistUpdatedDetails(games, steamApps) {
+    this.#logger.debugc(`persisting ${steamApps.length} apps with updated html attempts`);
     this.#logger.debugc(`persisting ${games.length} games with updated details`);
-    await this.#gamesRepository.updateGameDetails(games);
+
+    await this.#steamAppsRepository.updateSteamAppsById(steamApps);
+    await this.#gamesRepository.updateGameDetailsFrom(games);
   }
 
   updateGamesWithoutReleaseDates = async () => {
-    this.#logger.debugc("updating games without details");
+    const source = ValidDataSources.validDataSources.steamDb;
+    this.#logger.debugc(`updating games without release dates via ${source}`);
 
     const games = await this.#gamesRepository.getGamesWithoutReleaseDates(
       this.#options.batchSize,
     );
 
-    if (games.length === 0) {
+    if (games.isEmpty) {
       this.#logger.debugc(
-        `no games without release dates in db, retrying in ${
-          this.#options.iterationDelay
+        `no games without release dates in db, retry in: ${
+          this.#options.globalIterationDelay
         } ms`,
       );
+
       return;
     }
 
-    const steamApps = await this.#steamAppsRepository.getSteamAppsById(
-      games.map((game) => game.id),
-    );
+    const steamApps = await this.#steamAppsRepository.getSteamAppsById(games.ids);
 
-    const htmlDetailsPages = await this.#getSteamDbHtmlDetailsPages(games);
+    const htmlDetailsPages = await this.#getSteamAppsHtmlDetailsPages(
+      steamApps.content,
+      source,
+    );
 
-    const updatedApps = recordAttemptsViaSteamDb(steamApps, htmlDetailsPages);
+    steamApps.recordAttemptsViaSource(htmlDetailsPages, source);
 
-    updateMissingReleaseDates(games, htmlDetailsPages);
+    games.extractReleaseDatesFrom(htmlDetailsPages);
 
-    this.#persistReleaseDates(games, updatedApps);
+    this.#persistReleaseDates(games.content, steamApps.content);
   };
 
-  #persistReleaseDates = async (games, appsWithoutPages) => {
-    if (appsWithoutPages.length !== 0) {
-      this.#logger.debugc(`persisting ${appsWithoutPages.length} apps without pages`);
-      this.#steamAppsRepository.updateSteamAppsById(appsWithoutPages);
-    }
-
+  async #persistReleaseDates(games, steamApps) {
+    this.#logger.debugc(`persisting ${steamApps.length} apps with updated html attempts`);
     this.#logger.debugc(`persisting ${games.length} games with updated release dates`);
 
+    await this.#steamAppsRepository.updateSteamAppsById(steamApps);
     await this.#gamesRepository.updateReleaseDates(games);
-  };
+  }
 }