Feature/NR-59 Fix handling steam client html pages errors

.github/workflows/cicd-backend.yml

@@ -0,0 +1,59 @@
+name: Validate and deploy backend.
+
+on:
+  push:
+    branches: [master]
+    paths: ["backend/**"]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.12.1]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "npm"
+      - name: Install backend dependencies
+        run: npm ci -w backend
+      - name: Prettier check
+        run: npm run -w backend prettier:check
+      - name: Unit tests
+        run: npm run -w backend unit-test
+      - name: Integration tests
+        run: npm run -w backend integration-test
+
+  build-and-push:
+    needs: tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          file: backend/Dockerfile
+          tags: sjakusch/steam-game-stats-backend:latest
+
+  deploy:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy on Render
+        if: github.ref == 'refs/heads/master'
+        env:
+          deploy_url: ${{ secrets.RENDER_DEPLOY_HOOK_URL }}
+        run: |
+          curl "$deploy_url"

backend/.prettierignore

@@ -1,3 +1,5 @@
 .vscode
-assets
-db
+/assets/*
+db
+
+!assets/html.details.pages.mock.js

backend/assets/html.details.pages.mock.js

@@ -0,0 +1,5 @@
+export function createHtmlDetailsPages(pages) {
+  return pages.map((page, index) => {
+    return { page, id: index + 1 };
+  });
+}

backend/src/features/game-identifier/game.identifier.js

@@ -65,6 +65,7 @@ export class GameIdentifier {
     const detailsPages = [];
     for (let steamApp of steamApps) {
       detailsPages.push(
+        // TODO https://github.com/lukatarman/steam-game-stats/issues/192
         await this.#steamClient.getSteamAppHtmlDetailsPage(steamApp.appid),
       );
       await delay(this.#options.unitDelay);
@@ -115,6 +116,7 @@ export class GameIdentifier {
       await delay(this.#options.unitDelay);
 
       try {
+        // TODO https://github.com/lukatarman/steam-game-stats/issues/192
         const result = await this.#steamClient.getSteamchartsGameHtmlDetailsPage(
           steamApp.appid,
         );
@@ -146,26 +148,51 @@ export class GameIdentifier {
       return;
     }
 
+    const steamApps = await this.#steamAppsRepository.getSteamAppsById(
+      games.map((game) => game.id),
+    );
+
     const htmlDetailsPages = await this.#getSteamDbHtmlDetailsPages(games);
 
-    const updatedGames = updateMissingDetails(games, htmlDetailsPages);
+    const appsWithoutPages = this.#recordFailedAttempts(htmlDetailsPages, steamApps);
 
-    this.#persistMissingProperties(updatedGames);
+    updateMissingDetails(games, htmlDetailsPages);
+
+    this.#persistMissingProperties(games, appsWithoutPages);
   };
 
   async #getSteamDbHtmlDetailsPages(games) {
     const htmlDetailsPages = [];
 
     for (let game of games) {
-      htmlDetailsPages.push(await this.#steamClient.getSteamDbHtmlDetailsPage(game.id));
+      // TODO https://github.com/lukatarman/steam-game-stats/issues/192
+      const htmlPage = await this.#steamClient.getSteamDbHtmlDetailsPage(game.id);
+      htmlDetailsPages.push({ page: htmlPage, id: game.id });
 
       await delay(this.#options.unitDelay);
     }
 
     return htmlDetailsPages;
   }
 
-  async #persistMissingProperties(games) {
+  #recordFailedAttempts(htmlDetailsPages, steamApps) {
+    return htmlDetailsPages
+      .filter((page) => page.page === "")
+      .map((page) => {
+        const steamApp = steamApps.find((steamApp) => steamApp.appid === page.id);
+        const steamAppCopy = steamApp.copy();
+        steamAppCopy.triedViaSteamDb();
+
+        return steamAppCopy;
+      });
+  }
+
+  async #persistMissingProperties(games, appsWithoutPages) {
+    if (appsWithoutPages.length !== 0) {
+      this.#logger.debugc(`persisting ${appsWithoutPages.length} apps without pages`);
+      this.#steamAppsRepository.updateSteamAppsById(appsWithoutPages);
+    }
+
     this.#logger.debugc(`persisting ${games.length} games with updated details`);
     await this.#gamesRepository.updateGameDetails(games);
   }
@@ -186,14 +213,25 @@ export class GameIdentifier {
       return;
     }
 
+    const steamApps = await this.#steamAppsRepository.getSteamAppsById(
+      games.map((game) => game.id),
+    );
+
     const htmlDetailsPages = await this.#getSteamDbHtmlDetailsPages(games);
 
-    const updatedGames = updateMissingReleaseDates(games, htmlDetailsPages);
+    const appsWithoutPages = this.#recordFailedAttempts(htmlDetailsPages, steamApps);
 
-    this.#persistReleaseDates(updatedGames);
+    updateMissingReleaseDates(games, htmlDetailsPages);
+
+    this.#persistReleaseDates(games, appsWithoutPages);
   };
 
-  #persistReleaseDates = async (games) => {
+  #persistReleaseDates = async (games, appsWithoutPages) => {
+    if (appsWithoutPages.length !== 0) {
+      this.#logger.debugc(`persisting ${appsWithoutPages.length} apps without pages`);
+      this.#steamAppsRepository.updateSteamAppsById(appsWithoutPages);
+    }
+
     this.#logger.debugc(`persisting ${games.length} games with updated release dates`);
 
     await this.#gamesRepository.updateReleaseDates(games);