diff --git a/package.json b/package.json index cef40c15..0cf36e46 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,13 @@ "description": "A custom JavaScript rendering engine based on Playwright", "main": "dist/index.js", "scripts": { - "build": "yarn clean && yarn tsc", + "build": "yarn clean && yarn tsc && yarn browser:adblocks", "ci:start": "ALLOW_LOCALHOST=true yarn start", "clean": "rm -rf dist/", "dev": "nodemon", "dev:run": "yarn build && NODE_ENV=development node -r dotenv/config dist/index.js", "docker:build": "./scripts/build.sh", + "browser:adblocks": "./scripts/update_adblock_hosts.sh", "lint": "eslint --ext=jsx,ts,tsx,js .", "start": "UV_THREADPOOL_SIZE=100 node dist/index.js", "semantic-release": "semantic-release", diff --git a/scripts/update_adblock_hosts.sh b/scripts/update_adblock_hosts.sh new file mode 100755 index 00000000..ba095d08 --- /dev/null +++ b/scripts/update_adblock_hosts.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# adblock hosts file URL +URL="https://raw.githubusercontent.com/badmojr/1Hosts/master/Pro/domains.txt" + +TARGET_DIR="./dist/lib/browser" +TARGET_FILE="adblock_hosts.txt" + +if curl -o "${TARGET_DIR}/${TARGET_FILE}" "$URL" -s; then + echo "✅ adblock hosts download successful." +else + echo "❌ adblock hosts download failed." + exit 1 +fi diff --git a/src/lib/browser/Adblocker.ts b/src/lib/browser/Adblocker.ts index b353bdc8..498109ff 100644 --- a/src/lib/browser/Adblocker.ts +++ b/src/lib/browser/Adblocker.ts @@ -1,12 +1,8 @@ -import { request } from 'undici'; +import { promises as fs } from 'fs'; import { report } from '../../helpers/errorReporting'; import { log as mainLog } from '../../helpers/logger'; -// TO DO: Cronjob to update this list on our servers -const list = - 'https://raw.githubusercontent.com/badmojr/1Hosts/master/Pro/domains.txt'; - const log = mainLog.child({ svc: 'adbk' }); /** @@ -17,15 +13,8 @@ export class Adblocker { async load(): Promise { try { - const res = await request(list, { - method: 'GET', - }); - - let body = ''; - for await (const chunk of res.body) { - body += chunk.toString(); - } - const lines = body.split(/[\r\n]+/); + const data = await fs.readFile(`${__dirname}/adblock_hosts.txt`, 'utf8'); + const lines = data.split(/[\r\n]+/); for (const line of lines) { if (!line.startsWith('#')) { @@ -35,7 +24,6 @@ export class Adblocker { log.info('Ready', { entries: this.#hostnames.size, - lastMod: res.headers['last-modified'], }); } catch (err: any) { report(new Error('Error while setting up adblocker'), { err });