fix: save adblock hosts at compile time (#882)

package.json

@@ -4,12 +4,13 @@
   "description": "A custom JavaScript rendering engine based on Playwright",
   "main": "dist/index.js",
   "scripts": {
-    "build": "yarn clean && yarn tsc",
+    "build": "yarn clean && yarn tsc && yarn browser:adblocks",
     "ci:start": "ALLOW_LOCALHOST=true yarn start",
     "clean": "rm -rf dist/",
     "dev": "nodemon",
     "dev:run": "yarn build && NODE_ENV=development node -r dotenv/config dist/index.js",
     "docker:build": "./scripts/build.sh",
+    "browser:adblocks": "./scripts/update_adblock_hosts.sh",
     "lint": "eslint --ext=jsx,ts,tsx,js .",
     "start": "UV_THREADPOOL_SIZE=100 node dist/index.js",
     "semantic-release": "semantic-release",

scripts/update_adblock_hosts.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# adblock hosts file URL
+URL="https://raw.githubusercontent.com/badmojr/1Hosts/master/Pro/domains.txt"
+
+TARGET_DIR="./dist/lib/browser"
+TARGET_FILE="adblock_hosts.txt"
+
+if curl -o "${TARGET_DIR}/${TARGET_FILE}" "$URL" -s; then
+  echo "✅ adblock hosts download successful."
+else
+  echo "❌ adblock hosts download failed."
+  exit 1
+fi

src/lib/browser/Adblocker.ts

@@ -1,12 +1,8 @@
-import { request } from 'undici';
+import { promises as fs } from 'fs';
 
 import { report } from '../../helpers/errorReporting';
 import { log as mainLog } from '../../helpers/logger';
 
-// TO DO: Cronjob to update this list on our servers
-const list =
-  'https://raw.githubusercontent.com/badmojr/1Hosts/master/Pro/domains.txt';
-
 const log = mainLog.child({ svc: 'adbk' });
 
 /**
@@ -17,15 +13,8 @@ export class Adblocker {
 
   async load(): Promise<void> {
     try {
-      const res = await request(list, {
-        method: 'GET',
-      });
-
-      let body = '';
-      for await (const chunk of res.body) {
-        body += chunk.toString();
-      }
-      const lines = body.split(/[\r\n]+/);
+      const data = await fs.readFile(`${__dirname}/adblock_hosts.txt`, 'utf8');
+      const lines = data.split(/[\r\n]+/);
 
       for (const line of lines) {
         if (!line.startsWith('#')) {
@@ -35,7 +24,6 @@ export class Adblocker {
 
       log.info('Ready', {
         entries: this.#hostnames.size,
-        lastMod: res.headers['last-modified'],
       });
     } catch (err: any) {
       report(new Error('Error while setting up adblocker'), { err });