feat: 🎸 initial commit

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+insert_final_newline = false
+trim_trailing_whitespace = false
+
+[*.{js,jsx,json,ts,tsx,yml}]
+indent_size = 2
+indent_style = space

.env-example

@@ -0,0 +1,18 @@
+# See https://github.com/hoangvu12/kaguya-database
+SUPABASE_KEY=
+SUPABASE_URL=
+
+# Push notification (npm run webPush:generate)
+WEB_PUSH_PUBLIC_KEY=
+WEB_PUSH_PRIVATE_KEY=
+WEB_PUSH_EMAIL=
+
+# This will be your base route (https://example.com/BASE_ROUTE)
+BASE_ROUTE=
+
+# Discord
+# Discord new anime/manga update channel id
+DISCORD_UPDATE_CHANNEL_ID=
+DISCORD_GUILD_ID=
+DISCORD_CLIENT_ID=
+DISCORD_TOKEN=

.eslintignore

@@ -0,0 +1,2 @@
+/**/*.js
+src/cli/templates/*

.eslintrc.json

@@ -0,0 +1,25 @@
+{
+  "env": {
+    "browser": false,
+    "es6": true,
+    "node": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "tsconfig.json",
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint", "jest"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "plugin:jest/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "no-unused-vars": "off"
+  }
+}

.gitignore

@@ -0,0 +1,33 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Dependencies
+node_modules/
+
+# Coverage
+coverage
+
+# Transpiled files
+build/
+
+# VS Code
+.vscode
+!.vscode/tasks.js
+
+# JetBrains IDEs
+.idea/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Misc
+.DS_Store
+
+.env
+
+.yarn-lock

.prettierrc

@@ -0,0 +1,12 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all",
+  "overrides": [
+    {
+      "files": "*.ts",
+      "options": {
+        "parser": "typescript"
+      }
+    }
+  ]
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 hoangvu12
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,37 @@
+# kaguya-scraper
+
+Main data scraper for [Kaguya](https://github.com/hoangvu12/Kaguya)
+
+
+## Clone repository
+
+To clone the repository, use the following commands:
+
+```sh
+git clone https://github.com/hoangvu12/kaguya-scraper
+cd kaguya-scraper
+npm install
+```
+
+
+## How it works?
+*kaguya-scraper* is built to handle multiple anime/manga ```scrapers``` (Or we can call it ```sources```).
+
+It will look into each source per an amount of time (you can define it in the source), scrape new data, also notify to subscribed users, send new data message to discord and push to the database. 
+
+(Learn how to setup database here: [kaguya-database](https://github.com/hoangvu12/kaguya-database))
+
+
+## How do I create a source?
+You can create a source in ```src/scrapers/(anime|manga)```.
+
+Or you can just run ```npm run cli generate```. It will generate a source for you based on the answer you give it 
+
+Or you can look at examples at ```src/scrapers/anime/avs.ts``` and ```src/scrapers/manga/nt.ts```. These are two sources that I personally made for currently running [Kaguya](https://github.com/hoangvu12/Kaguya)
+
+### Initialize source.
+Everytime you create a new source, you have to initialize that source.
+
+Run ```npm run cli scraper:init``` to initialize source.
+
+#### Note: Because initialize source will scrape all the anime, it will mostly takes very long time to run. (based on how many anime/manga the source have, 2-3 hours for ~3k anime/manga).

jest.config.js

@@ -0,0 +1,20 @@
+module.exports = {
+  testEnvironment: 'node',
+  transform: {
+    "^.+\\.tsx?$": "ts-jest"
+  },
+  moduleFileExtensions: [
+    "ts",
+    "tsx",
+    "js",
+    "jsx",
+    "json",
+    "node",
+  ],
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(ts|js)x?$',
+  coverageDirectory: 'coverage',
+  collectCoverageFrom: [
+    'src/**/*.{ts,tsx,js,jsx}',
+    '!src/**/*.d.ts',
+  ],
+};

package.json

@@ -0,0 +1,85 @@
+{
+  "name": "kaguya-scraper",
+  "version": "1.0.0",
+  "description": "Scraper for Kaguya (kaguya.live)",
+  "engines": {
+    "node": ">= 16.13 <17"
+  },
+  "devDependencies": {
+    "@types/apicache": "^1.6.1",
+    "@types/cors": "^2.8.12",
+    "@types/cron": "^1.7.3",
+    "@types/express": "^4.17.13",
+    "@types/inquirer": "^8.2.0",
+    "@types/jest": "~27.0.2",
+    "@types/lodash": "^4.14.178",
+    "@types/node": "~16.11.6",
+    "@types/tough-cookie": "^4.0.1",
+    "@types/web-push": "^3.3.2",
+    "@typescript-eslint/eslint-plugin": "~5.7.0",
+    "@typescript-eslint/parser": "~5.7.0",
+    "eslint": "~8.4.1",
+    "eslint-config-prettier": "~8.3.0",
+    "eslint-plugin-jest": "~25.3.0",
+    "jest": "~27.4.4",
+    "prettier": "~2.5.1",
+    "rimraf": "~3.0.2",
+    "simple-graphql-to-typescript": "^0.10.21",
+    "ts-jest": "~27.1.1",
+    "tsc-alias": "^1.5.0",
+    "tsconfig-paths": "^3.12.0",
+    "tsutils": "~3.21.0",
+    "typescript": "~4.5.3"
+  },
+  "scripts": {
+    "start": "node build/src/main.js",
+    "clean": "rimraf coverage build tmp",
+    "prebuild": "npm run lint",
+    "build": "tsc -p tsconfig.release.json",
+    "watch": "tsc -w -p tsconfig.release.json && tsc-alias",
+    "lint": "eslint . --ext .ts,.tsx",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "dev": "nodemon",
+    "commands": "node build/src/deployCommands.js",
+    "permissions": "node build/src/permission.js",
+    "scrapers:init": "node build/src/initScrapers.js",
+    "webPush:generate": "node build/src/generateWebPush.js",
+    "cli": "node build/src/cli/index.js"
+  },
+  "author": "Nguyen Vu <[email protected]>",
+  "license": "MIT",
+  "dependencies": {
+    "@discordjs/rest": "^0.3.0",
+    "@supabase/supabase-js": "^1.30.0",
+    "aniep": "^0.2.2",
+    "apicache": "^1.6.3",
+    "axios": "^0.25.0",
+    "axios-cookiejar-support": "^2.0.4",
+    "axios-logger": "^2.6.0",
+    "axios-rate-limit": "^1.3.0",
+    "axios-retry": "^3.2.4",
+    "cheerio": "^1.0.0-rc.10",
+    "commander": "^9.0.0",
+    "cors": "^2.8.5",
+    "cron": "^1.8.2",
+    "dayjs": "^1.10.7",
+    "discord-api-types": "^0.26.1",
+    "discord.js": "^13.6.0",
+    "dotenv": "^16.0.0",
+    "express": "^4.17.2",
+    "inquirer": "^8.2.0",
+    "lodash": "^4.17.21",
+    "node-match-path": "^0.6.3",
+    "socket.io": "^4.4.1",
+    "tough-cookie": "^4.0.0",
+    "tslib": "~2.3.1",
+    "web-push": "^3.4.5",
+    "winston": "^3.6.0",
+    "winston-daily-rotate-file": "^4.6.1",
+    "ytdl-core": "^4.10.1"
+  },
+  "volta": {
+    "node": "16.13.0"
+  }
+}