From 540d5c7b452c05c63ee56e3593eaaa669373f108 Mon Sep 17 00:00:00 2001 From: arcestia Date: Mon, 30 Dec 2024 11:49:30 +0700 Subject: [PATCH 1/3] feat: add Turso database integration - Add database client initialization and table creation - Implement stats tracking with proper error handling - Add rate limiting with IP-based tracking - Fix domain limit handling to allow up to 100 domains per request - Update environment variables for both dev and prod - Add better error handling and logging --- package-lock.json | 415 ++++++++++++++++++++++++++++- package.json | 3 +- src/db.ts | 288 ++++++++++++++++++++ src/index.ts | 649 +++++++++------------------------------------- src/templates.ts | 224 ++++++++++++++++ wrangler.toml | 17 +- 6 files changed, 1062 insertions(+), 534 deletions(-) create mode 100644 src/db.ts create mode 100644 src/templates.ts diff --git a/package-lock.json b/package-lock.json index fd5f871..cefbe28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "nawalacheck", "version": "1.0.0", "dependencies": { + "@libsql/client": "^0.4.0-pre.7", "hono": "^4.0.0" }, "devDependencies": { @@ -582,16 +583,198 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@libsql/client": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@libsql/client/-/client-0.4.3.tgz", + "integrity": "sha512-AUYKnSPqAsFBVWBvmtrb4dG3pQlvTKT92eztAest9wQU2iJkabH8WzHLDb3dKFWKql7/kiCqvBQUVpozDwhekQ==", + "license": "MIT", + "dependencies": { + "@libsql/core": "^0.4.3", + "@libsql/hrana-client": "^0.5.6", + "js-base64": "^3.7.5" + }, + "optionalDependencies": { + "libsql": "^0.2.0" + } + }, + "node_modules/@libsql/core": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@libsql/core/-/core-0.4.3.tgz", + "integrity": "sha512-r28iYBtaLBW9RRgXPFh6cGCsVI/rwRlOzSOpAu/1PVTm6EJ3t233pUf97jETVHU0vjdr1d8VvV6fKAvJkokqCw==", + "license": "MIT", + "dependencies": { + "js-base64": "^3.7.5" + } + }, + "node_modules/@libsql/darwin-arm64": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/darwin-arm64/-/darwin-arm64-0.2.0.tgz", + "integrity": "sha512-+qyT2W/n5CFH1YZWv2mxW4Fsoo4dX9Z9M/nvbQqZ7H84J8hVegvVAsIGYzcK8xAeMEcpU5yGKB1Y9NoDY4hOSQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@libsql/darwin-x64": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/darwin-x64/-/darwin-x64-0.2.0.tgz", + "integrity": "sha512-hwmO2mF1n8oDHKFrUju6Jv+n9iFtTf5JUK+xlnIE3Td0ZwGC/O1R/Z/btZTd9nD+vsvakC8SJT7/Q6YlWIkhEw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@libsql/hrana-client": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/@libsql/hrana-client/-/hrana-client-0.5.6.tgz", + "integrity": "sha512-mjQoAmejZ1atG+M3YR2ZW+rg6ceBByH/S/h17ZoYZkqbWrvohFhXyz2LFxj++ARMoY9m6w3RJJIRdJdmnEUlFg==", + "license": "MIT", + "dependencies": { + "@libsql/isomorphic-fetch": "^0.1.12", + "@libsql/isomorphic-ws": "^0.1.5", + "js-base64": "^3.7.5", + "node-fetch": "^3.3.2" + } + }, + "node_modules/@libsql/isomorphic-fetch": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/@libsql/isomorphic-fetch/-/isomorphic-fetch-0.1.12.tgz", + "integrity": "sha512-MRo4UcmjAGAa3ac56LoD5OE13m2p0lu0VEtZC2NZMcogM/jc5fU9YtMQ3qbPjFJ+u2BBjFZgMPkQaLS1dlMhpg==", + "license": "MIT", + "dependencies": { + "@types/node-fetch": "^2.6.11", + "node-fetch": "^2.7.0" + } + }, + "node_modules/@libsql/isomorphic-fetch/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@libsql/isomorphic-ws": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@libsql/isomorphic-ws/-/isomorphic-ws-0.1.5.tgz", + "integrity": "sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==", + "license": "MIT", + "dependencies": { + "@types/ws": "^8.5.4", + "ws": "^8.13.0" + } + }, + "node_modules/@libsql/linux-arm64-gnu": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/linux-arm64-gnu/-/linux-arm64-gnu-0.2.0.tgz", + "integrity": "sha512-1w2lPXIYtnBaK5t/Ej5E8x7lPiE+jP3KATI/W4yei5Z/ONJh7jQW5PJ7sYU95vTME3hWEM1FXN6kvzcpFAte7w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-arm64-musl": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/linux-arm64-musl/-/linux-arm64-musl-0.2.0.tgz", + "integrity": "sha512-lkblBEJ7xuNiWNjP8DDq0rqoWccszfkUS7Efh5EjJ+GDWdCBVfh08mPofIZg0fZVLWQCY3j+VZCG1qZfATBizg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-x64-gnu": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/linux-x64-gnu/-/linux-x64-gnu-0.2.0.tgz", + "integrity": "sha512-+x/d289KeJydwOhhqSxKT+6MSQTCfLltzOpTzPccsvdt5fxg8CBi+gfvEJ4/XW23Sa+9bc7zodFP0i6MOlxX7w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/linux-x64-musl": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/linux-x64-musl/-/linux-x64-musl-0.2.0.tgz", + "integrity": "sha512-5Xn0c5A6vKf9D1ASpgk7mef//FuY7t5Lktj/eiU4n3ryxG+6WTpqstTittJUgepVjcleLPYxIhQAYeYwTYH1IQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@libsql/win32-x64-msvc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@libsql/win32-x64-msvc/-/win32-x64-msvc-0.2.0.tgz", + "integrity": "sha512-rpK+trBIpRST15m3cMYg5aPaX7kvCIottxY7jZPINkKAaScvfbn9yulU/iZUM9YtuK96Y1ZmvwyVIK/Y5DzoMQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@neon-rs/load": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.4.tgz", + "integrity": "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==", + "license": "MIT", + "optional": true + }, "node_modules/@types/node": { "version": "22.10.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.20.0" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/node-forge": { "version": "1.3.11", "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", @@ -602,6 +785,15 @@ "@types/node": "*" } }, + "node_modules/@types/ws": { + "version": "8.5.13", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz", + "integrity": "sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/acorn": { "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", @@ -638,6 +830,12 @@ "printable-characters": "^1.0.42" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/blake3-wasm": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", @@ -697,6 +895,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/cookie": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", @@ -732,6 +942,25 @@ "dev": true, "license": "MIT" }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/esbuild": { "version": "0.17.19", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.19.tgz", @@ -803,6 +1032,55 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -891,6 +1169,41 @@ "dev": true, "license": "MIT" }, + "node_modules/js-base64": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz", + "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==", + "license": "BSD-3-Clause" + }, + "node_modules/libsql": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/libsql/-/libsql-0.2.0.tgz", + "integrity": "sha512-ELBRqhpJx5Dap0187zKQnntZyk4EjlDHSrjIVL8t+fQ5e8IxbQTeYgZgigMjB1EvrETdkm0Y0VxBGhzPQ+t0Jg==", + "cpu": [ + "x64", + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "@neon-rs/load": "^0.0.4", + "detect-libc": "2.0.2" + }, + "optionalDependencies": { + "@libsql/darwin-arm64": "0.2.0", + "@libsql/darwin-x64": "0.2.0", + "@libsql/linux-arm64-gnu": "0.2.0", + "@libsql/linux-arm64-musl": "0.2.0", + "@libsql/linux-x64-gnu": "0.2.0", + "@libsql/linux-x64-musl": "0.2.0", + "@libsql/win32-x64-msvc": "0.2.0" + } + }, "node_modules/magic-string": { "version": "0.25.9", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", @@ -901,6 +1214,27 @@ "sourcemap-codec": "^1.4.8" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/miniflare": { "version": "3.20241218.0", "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20241218.0.tgz", @@ -957,6 +1291,52 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/node-fetch/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/node-forge": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", @@ -1130,6 +1510,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1161,7 +1547,6 @@ "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "dev": true, "license": "MIT" }, "node_modules/unenv": { @@ -1178,6 +1563,31 @@ "ufo": "^1.5.4" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/workerd": { "version": "1.20241218.0", "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20241218.0.tgz", @@ -1254,7 +1664,6 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/package.json b/package.json index dc63619..c5e297a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "deploy": "wrangler deploy src/index.ts" }, "dependencies": { - "hono": "^4.0.0" + "hono": "^4.0.0", + "@libsql/client": "^0.4.0-pre.7" }, "devDependencies": { "@cloudflare/workers-types": "^4.20231218.0", diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..981668e --- /dev/null +++ b/src/db.ts @@ -0,0 +1,288 @@ +import { createClient } from '@libsql/client' + +let client: ReturnType | null = null + +// We'll initialize the client in a function that takes the environment variables +export function initializeDbClient(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }) { + if (client) return client + + console.log('Initializing database client with:', { + url: env.DATABASE_URL, + hasAuthToken: !!env.DATABASE_AUTH_TOKEN + }) + + if (!env.DATABASE_URL) { + throw new Error('DATABASE_URL is required') + } + + if (!env.DATABASE_AUTH_TOKEN) { + throw new Error('DATABASE_AUTH_TOKEN is required') + } + + try { + client = createClient({ + url: env.DATABASE_URL, + authToken: env.DATABASE_AUTH_TOKEN + }) + console.log('Database client created successfully') + return client + } catch (error) { + console.error('Failed to initialize database client:', error) + throw new Error(`Database connection failed: ${error.message}`) + } +} + +// Initialize tables +export async function initializeTables(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }) { + console.log('Initializing tables...') + const client = initializeDbClient(env) + + try { + console.log('Creating stats table...') + await client.execute(` + CREATE TABLE IF NOT EXISTS stats ( + id TEXT PRIMARY KEY, + total_requests INTEGER DEFAULT 0, + total_domains_checked INTEGER DEFAULT 0, + blocked_domains INTEGER DEFAULT 0, + not_blocked_domains INTEGER DEFAULT 0, + error_domains INTEGER DEFAULT 0, + last_reset INTEGER, + unique_users TEXT + ) + `) + console.log('Stats table created successfully') + + // Initialize default stats if not exists + const statsResult = await client.execute('SELECT id FROM stats WHERE id = "global"') + if (!statsResult.rows[0]) { + console.log('Initializing default stats...') + const now = Date.now() + await client.execute(` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, ${now}, "[]" + ) + `) + console.log('Default stats initialized') + } + + console.log('Creating rate_limits table...') + await client.execute(` + CREATE TABLE IF NOT EXISTS rate_limits ( + ip TEXT PRIMARY KEY, + count INTEGER, + timestamp INTEGER, + UNIQUE(ip) + ) + `) + console.log('Rate limits table created successfully') + } catch (error) { + console.error('Failed to initialize tables:', error) + throw error + } +} + +// Stats functions +export async function getStats(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }) { + console.log('Getting stats...') + const client = initializeDbClient(env) + + const defaultStats = { + id: 'global', + total_requests: 0, + total_domains_checked: 0, + blocked_domains: 0, + not_blocked_domains: 0, + error_domains: 0, + last_reset: Date.now(), + unique_users: '[]' + } + + try { + const result = await client.execute('SELECT * FROM stats WHERE id = "global"') + + if (!result.rows[0]) { + console.log('No stats found, initializing with default values...') + const now = Date.now() + await client.execute(` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, '${now}', '[]' + ) + `) + console.log('Default stats initialized successfully') + return defaultStats + } + + const stats = result.rows[0] + console.log('Stats retrieved successfully:', stats) + + // Ensure all fields have valid values + return { + id: stats.id || defaultStats.id, + total_requests: Number(stats.total_requests) || defaultStats.total_requests, + total_domains_checked: Number(stats.total_domains_checked) || defaultStats.total_domains_checked, + blocked_domains: Number(stats.blocked_domains) || defaultStats.blocked_domains, + not_blocked_domains: Number(stats.not_blocked_domains) || defaultStats.not_blocked_domains, + error_domains: Number(stats.error_domains) || defaultStats.error_domains, + last_reset: Number(stats.last_reset) || defaultStats.last_reset, + unique_users: stats.unique_users || defaultStats.unique_users + } + } catch (error) { + console.error('Error getting stats:', error) + throw new Error(`Failed to get stats: ${error.message}`) + } +} + +export async function incrementStats( + env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }, + stats: { + requests?: number + domainsChecked?: number + blocked?: number + notBlocked?: number + errors?: number + } +) { + console.log('Incrementing stats with:', stats) + const client = initializeDbClient(env) + const updates = [] + + if (stats.requests) { + updates.push(`total_requests = total_requests + ${stats.requests}`) + } + + if (stats.domainsChecked) { + updates.push(`total_domains_checked = total_domains_checked + ${stats.domainsChecked}`) + } + + if (stats.blocked) { + updates.push(`blocked_domains = blocked_domains + ${stats.blocked}`) + } + + if (stats.notBlocked) { + updates.push(`not_blocked_domains = not_blocked_domains + ${stats.notBlocked}`) + } + + if (stats.errors) { + updates.push(`error_domains = error_domains + ${stats.errors}`) + } + + if (updates.length > 0) { + try { + // First ensure the stats row exists + const result = await client.execute('SELECT id FROM stats WHERE id = "global"') + if (!result.rows[0]) { + console.log('Stats row does not exist, creating it...') + const now = Date.now() + await client.execute(` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, ${now}, "[]" + ) + `) + } + + // Then update the stats + const updateQuery = ` + UPDATE stats + SET ${updates.join(', ')} + WHERE id = "global" + ` + console.log('Executing update query:', updateQuery) + await client.execute(updateQuery) + console.log('Stats incremented successfully') + } catch (error) { + console.error('Error incrementing stats:', error) + throw new Error(`Failed to increment stats: ${error.message}`) + } + } +} + +// Rate limit functions +export async function checkRateLimit( + env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }, + ip: string, + domainCount: number, + maxDomains: number, + windowMinutes: number +) { + console.log('Checking rate limit for IP:', ip) + const client = initializeDbClient(env) + const now = Date.now() + const windowStart = now - (windowMinutes * 60 * 1000) + + try { + // Clean up old rate limits first + await client.execute(`DELETE FROM rate_limits WHERE timestamp < ${windowStart}`) + + // Get current usage + const result = await client.execute(` + SELECT count, timestamp + FROM rate_limits + WHERE ip = "${ip}" + `) + const usage = result.rows[0] + + if (!usage) { + if (domainCount > maxDomains) { + console.log('Rate limit exceeded for new IP') + return { + allowed: false, + remaining: maxDomains, + resetTime: now + (windowMinutes * 60 * 1000) + } + } + + await client.execute(` + INSERT INTO rate_limits (ip, count, timestamp) + VALUES ("${ip}", ${domainCount}, ${now}) + `) + console.log('Rate limit initialized successfully') + + return { + allowed: true, + remaining: maxDomains - domainCount, + resetTime: now + (windowMinutes * 60 * 1000) + } + } + + const totalCount = Number(usage.count) + domainCount + if (totalCount > maxDomains) { + console.log('Rate limit exceeded for existing IP') + return { + allowed: false, + remaining: maxDomains - Number(usage.count), + resetTime: Number(usage.timestamp) + (windowMinutes * 60 * 1000) + } + } + + await client.execute(` + UPDATE rate_limits + SET count = ${totalCount}, timestamp = ${now} + WHERE ip = "${ip}" + `) + console.log('Rate limit updated successfully') + + return { + allowed: true, + remaining: maxDomains - totalCount, + resetTime: now + (windowMinutes * 60 * 1000) + } + } catch (error) { + console.error('Error checking rate limit:', error) + // On error, allow the request but return a conservative remaining count + return { + allowed: true, + remaining: maxDomains - domainCount, + resetTime: now + (windowMinutes * 60 * 1000) + } + } +} diff --git a/src/index.ts b/src/index.ts index 011ec56..8e24497 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,477 +1,71 @@ import { Hono } from 'hono' import { cors } from 'hono/cors' import { logger } from 'hono/logger' -import type { Context } from 'hono' - -type Bindings = { - RATE_LIMIT_STORE: KVNamespace - STATS_STORE: KVNamespace -} - -type RateLimitData = { - count: number - timestamp: number -} - -type StatsData = { - totalRequests: number - totalDomainsChecked: number - uniqueUsers: string[] - blockedDomains: number - notBlockedDomains: number - errorDomains: number - lastReset: number +import { checkRateLimit, getStats, incrementStats, initializeTables } from './db' +import { indexHtml, statsHtml } from './templates' + +type Env = { + DATABASE_URL: string + DATABASE_AUTH_TOKEN?: string + dev?: { + DATABASE_URL: string + DATABASE_AUTH_TOKEN: string + } } -const app = new Hono<{ Bindings: Bindings }>() +const app = new Hono<{ Bindings: Env }>() // Constants const API_ENDPOINT = 'https://check.skiddle.id' -const BATCH_SIZE = 30 const RATE_LIMIT = { MAX_DOMAINS: 1000, WINDOW_MINUTES: 10 } +const MAX_DOMAINS_PER_REQUEST = 100 -// Middleware -app.use('*', logger()) -app.use('*', cors()) - -// Initialize stats if not exists -async function initializeStats(c: Context): Promise { - const stats = await c.env.STATS_STORE.get('global_stats') - if (!stats) { - const initialStats: StatsData = { - totalRequests: 0, - totalDomainsChecked: 0, - uniqueUsers: [], - blockedDomains: 0, - notBlockedDomains: 0, - errorDomains: 0, - lastReset: Date.now() - } - await c.env.STATS_STORE.put('global_stats', JSON.stringify(initialStats)) - return initialStats +// Helper to get environment variables +function getEnvVars(env: Env) { + return { + DATABASE_URL: env.dev?.DATABASE_URL || env.DATABASE_URL, + DATABASE_AUTH_TOKEN: env.dev?.DATABASE_AUTH_TOKEN || env.DATABASE_AUTH_TOKEN } - return JSON.parse(stats) } -// Stats middleware +// Initialize database app.use('*', async (c, next) => { - if (c.req.method === 'POST' && c.req.path === '/check') { - const stats = await initializeStats(c) - stats.totalRequests++ - await c.env.STATS_STORE.put('global_stats', JSON.stringify(stats)) + try { + await initializeTables(getEnvVars(c.env)) + await next() + } catch (error) { + console.error('Database initialization error:', error) + // For API endpoints, return error response + if (c.req.path === '/check' || c.req.path === '/stats/data') { + return c.json({ + error: 'Database error', + details: error.message + }, 500) + } + // For other routes (HTML pages), continue without database + await next() } - await next() }) -// Rate limiting middleware -async function checkRateLimit(c: Context, ip: string, domainCount: number): Promise<{ allowed: boolean, remaining: number, resetTime?: Date }> { - const key = `rate_limit:${ip}` - const now = Date.now() - const windowStart = now - (RATE_LIMIT.WINDOW_MINUTES * 60 * 1000) - - const stored = await c.env.RATE_LIMIT_STORE.get(key) - let usage: RateLimitData | null = stored ? JSON.parse(stored) : null - - if (!usage || usage.timestamp < windowStart) { - usage = { - count: domainCount, - timestamp: now - } - await c.env.RATE_LIMIT_STORE.put(key, JSON.stringify(usage), { - expirationTtl: RATE_LIMIT.WINDOW_MINUTES * 60 // TTL in seconds - }) - return { - allowed: domainCount <= RATE_LIMIT.MAX_DOMAINS, - remaining: RATE_LIMIT.MAX_DOMAINS - domainCount, - resetTime: new Date(now + (RATE_LIMIT.WINDOW_MINUTES * 60 * 1000)) - } - } +// Middleware +app.use('*', logger()) +app.use('*', cors()) - const totalCount = usage.count + domainCount - if (totalCount > RATE_LIMIT.MAX_DOMAINS) { - const resetTime = new Date(usage.timestamp + (RATE_LIMIT.WINDOW_MINUTES * 60 * 1000)) - return { - allowed: false, - remaining: RATE_LIMIT.MAX_DOMAINS - usage.count, - resetTime +// Stats middleware +app.use('*', async (c, next) => { + if (c.req.method === 'POST' && c.req.path === '/check') { + try { + await incrementStats(getEnvVars(c.env), { requests: 1 }) + } catch (error) { + console.error('Failed to increment stats:', error) + // Continue even if stats update fails } } - - usage = { - count: totalCount, - timestamp: usage.timestamp - } - await c.env.RATE_LIMIT_STORE.put(key, JSON.stringify(usage), { - expirationTtl: RATE_LIMIT.WINDOW_MINUTES * 60 // TTL in seconds - }) - - return { - allowed: true, - remaining: RATE_LIMIT.MAX_DOMAINS - totalCount, - resetTime: new Date(usage.timestamp + (RATE_LIMIT.WINDOW_MINUTES * 60 * 1000)) - } -} - -const statsHtml = ` - - - - - Domain Checker Statistics - Check Domain Block Status | Skiddle ID - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-

Domain Checker Statistics

- Back to Checker -
-
-
-

Usage Statistics

-
- Loading... -
-
-
-

Domain Statistics

-
- Loading... -
-
-
-

System Information

-
- Loading... -
-
-
-
- - Sponsor or Donate - -
-
- - - -` - -const indexHtml = ` - - - - - Free Domain Block Checker - Check Multiple Domains | Skiddle ID - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-

Domain Checker

- View Statistics -
-
-
- Rate limit: ${RATE_LIMIT.MAX_DOMAINS} domains per ${RATE_LIMIT.WINDOW_MINUTES} minutes -
-
-
- - -
- -
- -
-
- - - -` + await next() +}) // Routes app.get('/', (c) => { @@ -483,93 +77,99 @@ app.get('/stats', (c) => { }) app.get('/stats/data', async (c) => { - const stats = await initializeStats(c) - return c.json({ - totalRequests: stats.totalRequests, - totalDomainsChecked: stats.totalDomainsChecked, - uniqueUsers: stats.uniqueUsers.length, - blockedDomains: stats.blockedDomains, - notBlockedDomains: stats.notBlockedDomains, - errorDomains: stats.errorDomains, - lastReset: stats.lastReset, - rateLimit: { - max: RATE_LIMIT.MAX_DOMAINS, - window: RATE_LIMIT.WINDOW_MINUTES - } - }) -}) - -// API Routes -app.post('/check', async (c: Context) => { try { - const body = await c.req.json() - const domains = body.domains as string[] - - if (!domains || !Array.isArray(domains)) { - return c.json({ error: 'Invalid domains format' }, 400) - } + const stats = await getStats(getEnvVars(c.env)) + return c.json({ + totalRequests: stats.total_requests, + totalDomainsChecked: stats.total_domains_checked, + blockedDomains: stats.blocked_domains, + notBlockedDomains: stats.not_blocked_domains, + errorDomains: stats.error_domains, + lastReset: stats.last_reset, + uniqueUsers: JSON.parse(stats.unique_users) + }) + } catch (error) { + console.error('Failed to get stats:', error) + return c.json({ + error: 'Failed to get stats', + details: error.message + }, 500) + } +}) - if (domains.length === 0) { - return c.json({ error: 'No domains provided' }, 400) - } +app.post('/check', async (c) => { + const body = await c.req.json() + const domains = body.domains || [] + const ip = c.req.header('CF-Connecting-IP') || 'unknown' - // Get client IP - const ip = c.req.header('cf-connecting-ip') || - c.req.header('x-forwarded-for') || - 'unknown' + if (!Array.isArray(domains)) { + return c.json({ error: 'Invalid request: domains must be an array' }, 400) + } - // Update stats - const stats = await initializeStats(c) - if (!stats.uniqueUsers.includes(ip)) { - stats.uniqueUsers.push(ip) - } - stats.totalDomainsChecked += domains.length + // Check total number of domains first + if (domains.length > MAX_DOMAINS_PER_REQUEST) { + return c.json({ error: `Maximum ${MAX_DOMAINS_PER_REQUEST} domains per request` }, 400) + } + try { // Check rate limit - const { allowed, remaining, resetTime } = await checkRateLimit(c, ip, domains.length) - - if (!allowed) { + const rateLimit = await checkRateLimit(c.env, ip, domains.length, RATE_LIMIT.MAX_DOMAINS, RATE_LIMIT.WINDOW_MINUTES) + if (!rateLimit.allowed) { return c.json({ error: 'Rate limit exceeded', - rateLimitExceeded: true, - remaining, - resetTime + remaining: rateLimit.remaining, + resetTime: rateLimit.resetTime }, 429) } - const results = [] - // Process domains in batches - for (let i = 0; i < domains.length; i += BATCH_SIZE) { - const batch = domains.slice(i, i + BATCH_SIZE) + const results = [] + const batchSize = 30 + let processed = 0 + let blocked = 0 + let notBlocked = 0 + let errors = 0 + + for (let i = 0; i < domains.length; i += batchSize) { + const batch = domains.slice(i, i + batchSize) const batchResults = await checkBatch(batch) + + for (const result of batchResults) { + processed++ + if (result.error) { + errors++ + } else if (result.blocked) { + blocked++ + } else { + notBlocked++ + } + } + results.push(...batchResults) } - // Update domain stats - results.forEach(result => { - if (result.status === 'Blocked') stats.blockedDomains++ - else if (result.status === 'Not Blocked') stats.notBlockedDomains++ - else stats.errorDomains++ + // Update stats + await incrementStats(c.env, { + requests: 1, + domainsChecked: processed, + blocked, + notBlocked, + errors }) - // Save updated stats - await c.env.STATS_STORE.put('global_stats', JSON.stringify(stats)) - - return c.json({ - domains: results, - remaining, - resetTime + return c.json({ + results, + remaining: rateLimit.remaining, + resetTime: rateLimit.resetTime }) } catch (error) { - console.error('Error processing domains:', error) - return c.json({ error: 'Failed to process domains' }, 500) + console.error('Error processing request:', error) + return c.json({ error: 'Internal server error' }, 500) } }) async function checkBatch(domains: string[]) { try { - // Build URL with domains parameter and json=true const url = new URL(API_ENDPOINT) url.searchParams.append('domains', domains.join(',')) url.searchParams.append('json', 'true') @@ -581,10 +181,9 @@ async function checkBatch(domains: string[]) { if (!response.ok) { throw new Error(`API request failed: ${response.statusText}`) } - + const data = await response.json() - - // Check if data has the expected structure + if (!data || typeof data !== 'object') { throw new Error('Invalid API response format') } @@ -595,18 +194,24 @@ async function checkBatch(domains: string[]) { if (!result || typeof result !== 'object') { return { originalUrl: domain, - status: 'Error: Invalid response' + status: 'Error: Invalid response', + blocked: false, + error: true } } return { originalUrl: domain, - status: result.blocked ? 'Blocked' : 'Not Blocked' + status: result.blocked ? 'Blocked' : 'Not Blocked', + blocked: result.blocked, + error: false } } catch (err) { console.error(`Error processing domain ${domain}:`, err) return { originalUrl: domain, - status: 'Error: Processing failed' + status: 'Error: Processing failed', + blocked: false, + error: true } } }) @@ -614,7 +219,9 @@ async function checkBatch(domains: string[]) { console.error('Error checking batch:', error) return domains.map(domain => ({ originalUrl: domain, - status: 'Error: API request failed' + status: 'Error: API request failed', + blocked: false, + error: true })) } } @@ -636,4 +243,4 @@ app.onError((err, c) => { }, 500) }) -export default app +export default app \ No newline at end of file diff --git a/src/templates.ts b/src/templates.ts new file mode 100644 index 0000000..354eee1 --- /dev/null +++ b/src/templates.ts @@ -0,0 +1,224 @@ +export const statsHtml = ` + + + + + Domain Checker Statistics - Check Domain Block Status | Skiddle ID + + + + +
+
+

Domain Checker Statistics

+ Back to Checker +
+
+
+

Usage Statistics

+
Loading...
+
+
+

Domain Statistics

+
Loading...
+
+
+

System Information

+
Loading...
+
+
+
+ + +`; + +export const indexHtml = ` + + + + + Free Domain Block Checker - Check Multiple Domains | Skiddle ID + + + + +
+
+

Domain Checker

+ View Statistics +
+
+
+ Rate limit: 1000 domains per 10 minutes +
+
+
+ + +
+ +
+ +
+
+ + +`; diff --git a/wrangler.toml b/wrangler.toml index 0b57997..2a5de43 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,19 +2,18 @@ name = "domain-checker" main = "src/index.ts" compatibility_date = "2024-01-01" -[[kv_namespaces]] -binding = "RATE_LIMIT_STORE" -id = "5f5279204ac941298d26cc294b87e2f8" -preview_id = "32401d53959249e0a05e2d26d0600231" +[vars] +DATABASE_URL = "libsql://domainchecker-arcestia.turso.io" +DATABASE_AUTH_TOKEN = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJnaWQiOiJmYTg3MTc1YS02NDMxLTRkNDUtYTk2MS03YjhkN2FhYTNhZTIiLCJpYXQiOjE3MzU1MzI2NDB9.xiY8s2CW4l5CLLhzWU1ql5ocPt5BrurmosIgC28KO8bUn3yg-a2spBFrS4rZJlY1GwT1kzkBCsKG1IhP7RzsBA" -[[kv_namespaces]] -binding = "STATS_STORE" -id = "e70c8e8e3fd5442c9ee2883f0089b7a9" -preview_id = "d1c3158c44f54a8d9bb5ad7336dc5afe" +# Development variables will be the same as production +[vars.dev] +DATABASE_URL = "libsql://domainchecker-arcestia.turso.io" +DATABASE_AUTH_TOKEN = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJnaWQiOiJmYTg3MTc1YS02NDMxLTRkNDUtYTk2MS03YjhkN2FhYTNhZTIiLCJpYXQiOjE3MzU1MzI2NDB9.xiY8s2CW4l5CLLhzWU1ql5ocPt5BrurmosIgC28KO8bUn3yg-a2spBFrS4rZJlY1GwT1kzkBCsKG1IhP7RzsBA" [[routes]] pattern = "nawalacheck.skiddle.id" custom_domain = true [observability.logs] -enabled = true \ No newline at end of file +enabled = false \ No newline at end of file From 9b93d3040517cb2ad4ac48adb85605cae0f4afbe Mon Sep 17 00:00:00 2001 From: arcestia Date: Mon, 30 Dec 2024 12:33:18 +0700 Subject: [PATCH 2/3] fix: update rate limit to 1000 domains per 10 minutes - Changed rate limit from 100 to 1000 domains - Changed window from 60 to 10 minutes - Added rate limit info to the form - Fixed HTML template rendering --- src/index.ts | 159 +++++++++-------------------------------------- src/templates.ts | 148 +++++++++++++++++++++++-------------------- 2 files changed, 110 insertions(+), 197 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8e24497..7fbe8fa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,84 +1,59 @@ import { Hono } from 'hono' import { cors } from 'hono/cors' import { logger } from 'hono/logger' -import { checkRateLimit, getStats, incrementStats, initializeTables } from './db' +import { checkBatch } from './checker' +import { initializeTables, getStats, incrementStats, checkRateLimit } from './db' import { indexHtml, statsHtml } from './templates' -type Env = { +type Bindings = { DATABASE_URL: string - DATABASE_AUTH_TOKEN?: string - dev?: { - DATABASE_URL: string - DATABASE_AUTH_TOKEN: string - } + DATABASE_AUTH_TOKEN: string } -const app = new Hono<{ Bindings: Env }>() +const app = new Hono<{ Bindings: Bindings }>() // Constants -const API_ENDPOINT = 'https://check.skiddle.id' const RATE_LIMIT = { MAX_DOMAINS: 1000, WINDOW_MINUTES: 10 } -const MAX_DOMAINS_PER_REQUEST = 100 - -// Helper to get environment variables -function getEnvVars(env: Env) { - return { - DATABASE_URL: env.dev?.DATABASE_URL || env.DATABASE_URL, - DATABASE_AUTH_TOKEN: env.dev?.DATABASE_AUTH_TOKEN || env.DATABASE_AUTH_TOKEN - } -} -// Initialize database -app.use('*', async (c, next) => { - try { - await initializeTables(getEnvVars(c.env)) - await next() - } catch (error) { - console.error('Database initialization error:', error) - // For API endpoints, return error response - if (c.req.path === '/check' || c.req.path === '/stats/data') { - return c.json({ - error: 'Database error', - details: error.message - }, 500) - } - // For other routes (HTML pages), continue without database - await next() - } -}) +const MAX_DOMAINS_PER_REQUEST = 100 // Middleware app.use('*', logger()) app.use('*', cors()) -// Stats middleware -app.use('*', async (c, next) => { - if (c.req.method === 'POST' && c.req.path === '/check') { - try { - await incrementStats(getEnvVars(c.env), { requests: 1 }) - } catch (error) { - console.error('Failed to increment stats:', error) - // Continue even if stats update fails - } +// Initialize database tables +app.all('*', async (c, next) => { + try { + await initializeTables(c.env) + } catch (error: unknown) { + console.error('Database initialization error:', error instanceof Error ? error.message : error) } await next() }) // Routes app.get('/', (c) => { - return c.html(indexHtml) + return new Response(indexHtml, { + headers: { + 'content-type': 'text/html;charset=UTF-8', + }, + }) }) app.get('/stats', (c) => { - return c.html(statsHtml) + return new Response(statsHtml, { + headers: { + 'content-type': 'text/html;charset=UTF-8', + }, + }) }) app.get('/stats/data', async (c) => { try { - const stats = await getStats(getEnvVars(c.env)) + const stats = await getStats(c.env) return c.json({ totalRequests: stats.total_requests, totalDomainsChecked: stats.total_domains_checked, @@ -88,12 +63,9 @@ app.get('/stats/data', async (c) => { lastReset: stats.last_reset, uniqueUsers: JSON.parse(stats.unique_users) }) - } catch (error) { - console.error('Failed to get stats:', error) - return c.json({ - error: 'Failed to get stats', - details: error.message - }, 500) + } catch (error: unknown) { + console.error('Failed to get stats:', error instanceof Error ? error.message : error) + return c.json({ error: 'Failed to get stats' }, 500) } }) @@ -162,85 +134,10 @@ app.post('/check', async (c) => { remaining: rateLimit.remaining, resetTime: rateLimit.resetTime }) - } catch (error) { - console.error('Error processing request:', error) + } catch (error: unknown) { + console.error('Error processing request:', error instanceof Error ? error.message : error) return c.json({ error: 'Internal server error' }, 500) } }) -async function checkBatch(domains: string[]) { - try { - const url = new URL(API_ENDPOINT) - url.searchParams.append('domains', domains.join(',')) - url.searchParams.append('json', 'true') - - const response = await fetch(url.toString(), { - method: 'GET' - }) - - if (!response.ok) { - throw new Error(`API request failed: ${response.statusText}`) - } - - const data = await response.json() - - if (!data || typeof data !== 'object') { - throw new Error('Invalid API response format') - } - - return domains.map(domain => { - try { - const result = data[domain] - if (!result || typeof result !== 'object') { - return { - originalUrl: domain, - status: 'Error: Invalid response', - blocked: false, - error: true - } - } - return { - originalUrl: domain, - status: result.blocked ? 'Blocked' : 'Not Blocked', - blocked: result.blocked, - error: false - } - } catch (err) { - console.error(`Error processing domain ${domain}:`, err) - return { - originalUrl: domain, - status: 'Error: Processing failed', - blocked: false, - error: true - } - } - }) - } catch (error) { - console.error('Error checking batch:', error) - return domains.map(domain => ({ - originalUrl: domain, - status: 'Error: API request failed', - blocked: false, - error: true - })) - } -} - -// Handle 404 -app.notFound((c) => { - return c.json({ - message: 'Not Found', - status: 404 - }, 404) -}) - -// Error handling -app.onError((err, c) => { - console.error(`${err}`) - return c.json({ - message: 'Internal Server Error', - status: 500 - }, 500) -}) - export default app \ No newline at end of file diff --git a/src/templates.ts b/src/templates.ts index 354eee1..5a588fb 100644 --- a/src/templates.ts +++ b/src/templates.ts @@ -1,4 +1,4 @@ -export const statsHtml = ` +export const statsHtml = /* html */` @@ -106,7 +106,7 @@ export const statsHtml = ` `; -export const indexHtml = ` +export const indexHtml = /* html */` @@ -121,102 +121,118 @@ export const indexHtml = `

Domain Checker

View Statistics -
-
- Rate limit: 1000 domains per 10 minutes -
+
- - + +
Rate limit: 1000 domains per 10 minutes. Maximum 100 domains per request.
+
- +
-
+
From 9563aeab6acff7b826678896bdbc04bd85c55f41 Mon Sep 17 00:00:00 2001 From: arcestia Date: Mon, 30 Dec 2024 12:34:19 +0700 Subject: [PATCH 3/3] feat: add domain checking functionality and improve database handling - Added checker.ts for domain checking logic - Updated db.ts with improved error handling - Added new dependencies in package.json - Fixed rate limiting and HTML template issues --- package-lock.json | 8 +- package.json | 4 +- src/checker.ts | 66 ++++++++++++ src/db.ts | 254 +++++++++++++++++++++++++++------------------- 4 files changed, 221 insertions(+), 111 deletions(-) create mode 100644 src/checker.ts diff --git a/package-lock.json b/package-lock.json index cefbe28..3d93083 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "dependencies": { "@libsql/client": "^0.4.0-pre.7", - "hono": "^4.0.0" + "hono": "^4.6.15" }, "devDependencies": { "@cloudflare/workers-types": "^4.20231218.0", @@ -1138,9 +1138,9 @@ } }, "node_modules/hono": { - "version": "4.6.14", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.6.14.tgz", - "integrity": "sha512-j4VkyUp2xazGJ8eCCLN1Vm/bxdvm/j5ZuU9AIjLu9vapn2M44p9L3Ktr9Vnb2RN2QtcR/wVjZVMlT5k7GJQgPw==", + "version": "4.6.15", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.6.15.tgz", + "integrity": "sha512-OiQwvAOAaI2JrABBH69z5rsctHDzFzIKJge0nYXgtzGJ0KftwLWcBXm1upJC23/omNRtnqM0gjRMbtXshPdqhQ==", "license": "MIT", "engines": { "node": ">=16.9.0" diff --git a/package.json b/package.json index c5e297a..eb450d5 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "deploy": "wrangler deploy src/index.ts" }, "dependencies": { - "hono": "^4.0.0", - "@libsql/client": "^0.4.0-pre.7" + "@libsql/client": "^0.4.0-pre.7", + "hono": "^4.6.15" }, "devDependencies": { "@cloudflare/workers-types": "^4.20231218.0", diff --git a/src/checker.ts b/src/checker.ts new file mode 100644 index 0000000..fd974eb --- /dev/null +++ b/src/checker.ts @@ -0,0 +1,66 @@ +interface CheckResult { + originalUrl: string + status: string + blocked: boolean + error: boolean +} + +const API_ENDPOINT = 'https://check.skiddle.id' + +export async function checkBatch(domains: string[]): Promise { + try { + const url = new URL(API_ENDPOINT) + url.searchParams.append('domains', domains.join(',')) + url.searchParams.append('json', 'true') + + const response = await fetch(url.toString(), { + method: 'GET' + }) + + if (!response.ok) { + throw new Error(`API request failed: ${response.statusText}`) + } + + const data = await response.json() + + if (!data || typeof data !== 'object') { + throw new Error('Invalid API response format') + } + + return domains.map(domain => { + try { + const result = data[domain] + if (!result || typeof result !== 'object') { + return { + originalUrl: domain, + status: 'Error: Invalid response', + blocked: false, + error: true + } + } + return { + originalUrl: domain, + status: result.blocked ? 'Blocked' : 'Not Blocked', + blocked: result.blocked, + error: false + } + } catch (err) { + console.error(`Error processing domain ${domain}:`, err) + return { + originalUrl: domain, + status: 'Error: Processing failed', + blocked: false, + error: true + } + } + }) + } catch (error) { + console.error('Error checking batch:', error) + return domains.map(domain => ({ + originalUrl: domain, + status: 'Error: API request failed', + blocked: false, + error: true + })) + } +} diff --git a/src/db.ts b/src/db.ts index 981668e..e8cedfb 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,9 +1,25 @@ -import { createClient } from '@libsql/client' +import { createClient, Client } from '@libsql/client' -let client: ReturnType | null = null +interface Stats { + id: string + total_requests: number + total_domains_checked: number + blocked_domains: number + not_blocked_domains: number + error_domains: number + last_reset: number + unique_users: string +} + +interface RateLimit { + ip: string + count: number + timestamp: number +} + +let client: Client | null = null -// We'll initialize the client in a function that takes the environment variables -export function initializeDbClient(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }) { +export function initializeDbClient(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }): Client { if (client) return client console.log('Initializing database client with:', { @@ -26,9 +42,9 @@ export function initializeDbClient(env: { DATABASE_URL: string; DATABASE_AUTH_TO }) console.log('Database client created successfully') return client - } catch (error) { - console.error('Failed to initialize database client:', error) - throw new Error(`Database connection failed: ${error.message}`) + } catch (error: unknown) { + console.error('Failed to initialize database client:', error instanceof Error ? error.message : error) + throw new Error(`Database connection failed: ${error instanceof Error ? error.message : String(error)}`) } } @@ -39,58 +55,70 @@ export async function initializeTables(env: { DATABASE_URL: string; DATABASE_AUT try { console.log('Creating stats table...') - await client.execute(` - CREATE TABLE IF NOT EXISTS stats ( - id TEXT PRIMARY KEY, - total_requests INTEGER DEFAULT 0, - total_domains_checked INTEGER DEFAULT 0, - blocked_domains INTEGER DEFAULT 0, - not_blocked_domains INTEGER DEFAULT 0, - error_domains INTEGER DEFAULT 0, - last_reset INTEGER, - unique_users TEXT - ) - `) + await client.execute({ + sql: ` + CREATE TABLE IF NOT EXISTS stats ( + id TEXT PRIMARY KEY, + total_requests INTEGER DEFAULT 0, + total_domains_checked INTEGER DEFAULT 0, + blocked_domains INTEGER DEFAULT 0, + not_blocked_domains INTEGER DEFAULT 0, + error_domains INTEGER DEFAULT 0, + last_reset INTEGER, + unique_users TEXT + ) + `, + args: [] + }) console.log('Stats table created successfully') // Initialize default stats if not exists - const statsResult = await client.execute('SELECT id FROM stats WHERE id = "global"') + const statsResult = await client.execute({ + sql: 'SELECT id FROM stats WHERE id = "global"', + args: [] + }) if (!statsResult.rows[0]) { console.log('Initializing default stats...') const now = Date.now() - await client.execute(` - INSERT INTO stats ( - id, total_requests, total_domains_checked, blocked_domains, - not_blocked_domains, error_domains, last_reset, unique_users - ) VALUES ( - "global", 0, 0, 0, 0, 0, ${now}, "[]" - ) - `) + await client.execute({ + sql: ` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, ?, "[]" + ) + `, + args: [now] + }) console.log('Default stats initialized') } console.log('Creating rate_limits table...') - await client.execute(` - CREATE TABLE IF NOT EXISTS rate_limits ( - ip TEXT PRIMARY KEY, - count INTEGER, - timestamp INTEGER, - UNIQUE(ip) - ) - `) + await client.execute({ + sql: ` + CREATE TABLE IF NOT EXISTS rate_limits ( + ip TEXT PRIMARY KEY, + count INTEGER, + timestamp INTEGER, + UNIQUE(ip) + ) + `, + args: [] + }) console.log('Rate limits table created successfully') - } catch (error) { - console.error('Failed to initialize tables:', error) + } catch (error: unknown) { + console.error('Failed to initialize tables:', error instanceof Error ? error.message : error) throw error } } // Stats functions -export async function getStats(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }) { +export async function getStats(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN?: string }): Promise { console.log('Getting stats...') const client = initializeDbClient(env) - const defaultStats = { + const defaultStats: Stats = { id: 'global', total_requests: 0, total_domains_checked: 0, @@ -102,40 +130,46 @@ export async function getStats(env: { DATABASE_URL: string; DATABASE_AUTH_TOKEN? } try { - const result = await client.execute('SELECT * FROM stats WHERE id = "global"') + const result = await client.execute({ + sql: 'SELECT * FROM stats WHERE id = "global"', + args: [] + }) if (!result.rows[0]) { console.log('No stats found, initializing with default values...') const now = Date.now() - await client.execute(` - INSERT INTO stats ( - id, total_requests, total_domains_checked, blocked_domains, - not_blocked_domains, error_domains, last_reset, unique_users - ) VALUES ( - "global", 0, 0, 0, 0, 0, '${now}', '[]' - ) - `) + await client.execute({ + sql: ` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, ?, "[]" + ) + `, + args: [now] + }) console.log('Default stats initialized successfully') return defaultStats } - const stats = result.rows[0] - console.log('Stats retrieved successfully:', stats) + const row = result.rows[0] + console.log('Stats retrieved successfully:', row) // Ensure all fields have valid values return { - id: stats.id || defaultStats.id, - total_requests: Number(stats.total_requests) || defaultStats.total_requests, - total_domains_checked: Number(stats.total_domains_checked) || defaultStats.total_domains_checked, - blocked_domains: Number(stats.blocked_domains) || defaultStats.blocked_domains, - not_blocked_domains: Number(stats.not_blocked_domains) || defaultStats.not_blocked_domains, - error_domains: Number(stats.error_domains) || defaultStats.error_domains, - last_reset: Number(stats.last_reset) || defaultStats.last_reset, - unique_users: stats.unique_users || defaultStats.unique_users + id: String(row.id) || defaultStats.id, + total_requests: Number(row.total_requests) || defaultStats.total_requests, + total_domains_checked: Number(row.total_domains_checked) || defaultStats.total_domains_checked, + blocked_domains: Number(row.blocked_domains) || defaultStats.blocked_domains, + not_blocked_domains: Number(row.not_blocked_domains) || defaultStats.not_blocked_domains, + error_domains: Number(row.error_domains) || defaultStats.error_domains, + last_reset: Number(row.last_reset) || defaultStats.last_reset, + unique_users: String(row.unique_users) || defaultStats.unique_users } - } catch (error) { - console.error('Error getting stats:', error) - throw new Error(`Failed to get stats: ${error.message}`) + } catch (error: unknown) { + console.error('Error getting stats:', error instanceof Error ? error.message : error) + throw new Error(`Failed to get stats: ${error instanceof Error ? error.message : String(error)}`) } } @@ -152,56 +186,65 @@ export async function incrementStats( console.log('Incrementing stats with:', stats) const client = initializeDbClient(env) const updates = [] + const values: (number)[] = [] if (stats.requests) { - updates.push(`total_requests = total_requests + ${stats.requests}`) + updates.push('total_requests = total_requests + ?') + values.push(stats.requests) } if (stats.domainsChecked) { - updates.push(`total_domains_checked = total_domains_checked + ${stats.domainsChecked}`) + updates.push('total_domains_checked = total_domains_checked + ?') + values.push(stats.domainsChecked) } if (stats.blocked) { - updates.push(`blocked_domains = blocked_domains + ${stats.blocked}`) + updates.push('blocked_domains = blocked_domains + ?') + values.push(stats.blocked) } if (stats.notBlocked) { - updates.push(`not_blocked_domains = not_blocked_domains + ${stats.notBlocked}`) + updates.push('not_blocked_domains = not_blocked_domains + ?') + values.push(stats.notBlocked) } if (stats.errors) { - updates.push(`error_domains = error_domains + ${stats.errors}`) + updates.push('error_domains = error_domains + ?') + values.push(stats.errors) } if (updates.length > 0) { try { // First ensure the stats row exists - const result = await client.execute('SELECT id FROM stats WHERE id = "global"') + const result = await client.execute({ + sql: 'SELECT id FROM stats WHERE id = "global"', + args: [] + }) if (!result.rows[0]) { console.log('Stats row does not exist, creating it...') const now = Date.now() - await client.execute(` - INSERT INTO stats ( - id, total_requests, total_domains_checked, blocked_domains, - not_blocked_domains, error_domains, last_reset, unique_users - ) VALUES ( - "global", 0, 0, 0, 0, 0, ${now}, "[]" - ) - `) + await client.execute({ + sql: ` + INSERT INTO stats ( + id, total_requests, total_domains_checked, blocked_domains, + not_blocked_domains, error_domains, last_reset, unique_users + ) VALUES ( + "global", 0, 0, 0, 0, 0, ?, "[]" + ) + `, + args: [now] + }) } // Then update the stats - const updateQuery = ` - UPDATE stats - SET ${updates.join(', ')} - WHERE id = "global" - ` - console.log('Executing update query:', updateQuery) - await client.execute(updateQuery) + await client.execute({ + sql: `UPDATE stats SET ${updates.join(', ')} WHERE id = "global"`, + args: values + }) console.log('Stats incremented successfully') - } catch (error) { - console.error('Error incrementing stats:', error) - throw new Error(`Failed to increment stats: ${error.message}`) + } catch (error: unknown) { + console.error('Error incrementing stats:', error instanceof Error ? error.message : error) + throw new Error(`Failed to increment stats: ${error instanceof Error ? error.message : String(error)}`) } } } @@ -213,7 +256,7 @@ export async function checkRateLimit( domainCount: number, maxDomains: number, windowMinutes: number -) { +): Promise<{ allowed: boolean; remaining: number; resetTime: number }> { console.log('Checking rate limit for IP:', ip) const client = initializeDbClient(env) const now = Date.now() @@ -221,15 +264,17 @@ export async function checkRateLimit( try { // Clean up old rate limits first - await client.execute(`DELETE FROM rate_limits WHERE timestamp < ${windowStart}`) + await client.execute({ + sql: 'DELETE FROM rate_limits WHERE timestamp < ?', + args: [windowStart] + }) // Get current usage - const result = await client.execute(` - SELECT count, timestamp - FROM rate_limits - WHERE ip = "${ip}" - `) - const usage = result.rows[0] + const result = await client.execute({ + sql: 'SELECT count, timestamp FROM rate_limits WHERE ip = ?', + args: [ip] + }) + const usage = result.rows[0] as RateLimit | undefined if (!usage) { if (domainCount > maxDomains) { @@ -241,10 +286,10 @@ export async function checkRateLimit( } } - await client.execute(` - INSERT INTO rate_limits (ip, count, timestamp) - VALUES ("${ip}", ${domainCount}, ${now}) - `) + await client.execute({ + sql: 'INSERT INTO rate_limits (ip, count, timestamp) VALUES (?, ?, ?)', + args: [ip, domainCount, now] + }) console.log('Rate limit initialized successfully') return { @@ -264,11 +309,10 @@ export async function checkRateLimit( } } - await client.execute(` - UPDATE rate_limits - SET count = ${totalCount}, timestamp = ${now} - WHERE ip = "${ip}" - `) + await client.execute({ + sql: 'UPDATE rate_limits SET count = ?, timestamp = ? WHERE ip = ?', + args: [totalCount, now, ip] + }) console.log('Rate limit updated successfully') return { @@ -276,8 +320,8 @@ export async function checkRateLimit( remaining: maxDomains - totalCount, resetTime: now + (windowMinutes * 60 * 1000) } - } catch (error) { - console.error('Error checking rate limit:', error) + } catch (error: unknown) { + console.error('Error checking rate limit:', error instanceof Error ? error.message : error) // On error, allow the request but return a conservative remaining count return { allowed: true,