add script to check and generate format

.github/workflows/ci-validation.yml

@@ -5,11 +5,11 @@ name: CI validation
 on:
   # https://stackoverflow.com/questions/64635032/github-actions-run-on-push-to-all-branches
   push:
-    branches: 
+    branches:
       - '**'
   pull_request:
     branches:
-      - master  
+      - master
 
 jobs:
   build:
@@ -19,8 +19,9 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20
+      - run: node format.js --check
       - run: pip3 install -r requirements.txt
       - run: py.test -vv
-      - run: python3 validate.py  
+      - run: python3 validate.py
       - run: php validate.php
       - run: go test

crawler-user-agents.json

@@ -4244,8 +4244,8 @@
     "pattern": "virustotal",
     "addition_date": "2021/09/22",
     "instances": [
-      "Mozilla\/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http:\/\/code.google.com\/appengine; appid: s~virustotalcloud)",
-      "AppEngine-Google; (+http:\/\/code.google.com\/appengine; appid: s~virustotalcloud)"
+      "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)",
+      "AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)"
     ],
     "url": "https://www.virustotal.com/gui/home/url"
   },

format.js

@@ -0,0 +1,21 @@
+const fs = require("fs");
+const path = require("path");
+
+const jsonFilePath = path.join(__dirname, "crawler-user-agents.json");
+
+const original = fs.readFileSync(jsonFilePath, "utf-8");
+
+const updated = JSON.stringify(JSON.parse(original), null, 2) + '\n';
+
+if (process.argv[2] === "--generate") {
+    fs.writeFileSync(jsonFilePath, updated);
+    process.exit(0);
+}
+
+if (process.argv[2] === "--check") {
+    if (updated !== original) {
+        console.error("JSON file format is wrong. Run `node validate.js --generate` to update.");
+        process.exit(1);
+    }
+}
+