From 089ba39358c2379909d79675ae6957ec722e271c Mon Sep 17 00:00:00 2001 From: CryptoBear Date: Sat, 14 Jan 2023 14:32:20 +0800 Subject: [PATCH] feat: scripts/spider-nansen --- .eslintrc.js | 1 + .gitignore | 1 + .prettierrc | 11 ++- package.json | 1 + pnpm-lock.yaml | 193 ++++++++++++++++++++++++++++++++------- scripts/spider-nansen.ts | 39 ++++++++ src/utils/index.ts | 4 + src/utils/puppeteer.ts | 12 +++ 8 files changed, 229 insertions(+), 33 deletions(-) create mode 100644 scripts/spider-nansen.ts create mode 100644 src/utils/puppeteer.ts diff --git a/.eslintrc.js b/.eslintrc.js index 975e163..e169c2b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -7,6 +7,7 @@ module.exports = { '@typescript-eslint/explicit-module-boundary-types': 'off', '@typescript-eslint/no-explicit-any': 'off', '@typescript-eslint/no-namespace': 'off', + '@typescript-eslint/no-empty-function': 'off', }, parser: '@typescript-eslint/parser', parserOptions: { diff --git a/.gitignore b/.gitignore index 07e7a78..d1bdf42 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Compiled output /dist /node_modules +/screenshots # Config .env diff --git a/.prettierrc b/.prettierrc index f00c7d1..01289f9 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,12 @@ { - "trailingComma": "all", + "printWidth": 150, "singleQuote": true, - "printWidth": 150 + "trailingComma": "all", + "proseWrap": "never", + "overrides": [ + { + "files": ".prettierrc", + "options": { "parser": "json" } + } + ] } diff --git a/package.json b/package.json index 84b3c06..bd7f667 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@nestjs/typeorm": "^9.0.1", "dotenv": "^16.0.3", "mysql2": "^2.3.3", + "puppeteer": "^19.5.0", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d360490..2a22024 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,7 @@ specifiers: mysql2: ^2.3.3 pm2: ^5.2.2 prettier: ^2.3.2 + puppeteer: ^19.5.0 reflect-metadata: ^0.1.13 rimraf: ^3.0.2 rxjs: ^7.2.0 @@ -44,6 +45,7 @@ dependencies: '@nestjs/typeorm': 9.0.1_hkapd3a3mi37wyf4vtt5y4i2ye dotenv: 16.0.3 mysql2: 2.3.3 + puppeteer: 19.5.0 reflect-metadata: 0.1.13 rimraf: 3.0.2 rxjs: 7.8.0 @@ -191,7 +193,6 @@ packages: engines: {node: '>=6.9.0'} dependencies: '@babel/highlight': 7.18.6 - dev: true /@babel/compat-data/7.20.5: resolution: {integrity: sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g==} @@ -314,7 +315,6 @@ packages: /@babel/helper-validator-identifier/7.19.1: resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} engines: {node: '>=6.9.0'} - dev: true /@babel/helper-validator-option/7.18.6: resolution: {integrity: sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==} @@ -339,7 +339,6 @@ packages: '@babel/helper-validator-identifier': 7.19.1 chalk: 2.4.2 js-tokens: 4.0.0 - dev: true /@babel/parser/7.20.7: resolution: {integrity: sha512-T3Z9oHybU+0vZlY9CiDSJQTD5ZapcW18ZctFMi0MOAl/4BjFF4ul7NVSARLdbGO5vDqy9eQiGTV0LtKfvCYvcg==} @@ -1385,6 +1384,14 @@ packages: '@types/yargs-parser': 21.0.0 dev: true + /@types/yauzl/2.10.0: + resolution: {integrity: sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==} + requiresBuild: true + dependencies: + '@types/node': 16.18.10 + dev: false + optional: true + /@typescript-eslint/eslint-plugin/5.47.0_ncmi6noazr3nzas7jxykisekym: resolution: {integrity: sha512-AHZtlXAMGkDmyLuLZsRpH3p4G/1iARIwc/T0vIem2YB+xW6pZaXYXzCBnZSF/5fdM97R9QqZWZ+h3iW10XgevQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -1667,7 +1674,6 @@ packages: debug: 4.3.4 transitivePeerDependencies: - supports-color - dev: true /ajv-formats/2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} @@ -1735,7 +1741,6 @@ packages: engines: {node: '>=4'} dependencies: color-convert: 1.9.3 - dev: true /ansi-styles/4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} @@ -1927,7 +1932,6 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.0 - dev: true /blessed/0.1.81: resolution: {integrity: sha512-LoF5gae+hlmfORcG1M5+5XZi4LBmvlXTzwJWzUlPryN/SJdSflZvROM2TwkT0GMpq7oqT48NRd4GS7BiVBc5OQ==} @@ -1995,6 +1999,10 @@ packages: node-int64: 0.4.0 dev: true + /buffer-crc32/0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + dev: false + /buffer-from/1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -2003,7 +2011,6 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: true /buffer/6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} @@ -2031,7 +2038,6 @@ packages: /callsites/3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - dev: true /camelcase/5.3.1: resolution: {integrity: sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==} @@ -2054,7 +2060,6 @@ packages: ansi-styles: 3.2.1 escape-string-regexp: 1.0.5 supports-color: 5.5.0 - dev: true /chalk/3.0.0: resolution: {integrity: sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==} @@ -2099,6 +2104,10 @@ packages: fsevents: 2.3.2 dev: true + /chownr/1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + dev: false + /chrome-trace-event/1.0.3: resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} engines: {node: '>=6.0'} @@ -2193,7 +2202,6 @@ packages: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} dependencies: color-name: 1.1.3 - dev: true /color-convert/2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} @@ -2203,7 +2211,6 @@ packages: /color-name/1.1.3: resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - dev: true /color-name/1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} @@ -2299,6 +2306,16 @@ packages: yaml: 1.10.2 dev: true + /cosmiconfig/8.0.0: + resolution: {integrity: sha512-da1EafcpH6b/TD8vDRaWV7xFINlHlF6zKsGwS1TsuVJTZRkquaS5HTMq7uq6h31619QjbsYl21gVDOm32KM1vQ==} + engines: {node: '>=14'} + dependencies: + import-fresh: 3.3.0 + js-yaml: 4.1.0 + parse-json: 5.2.0 + path-type: 4.0.0 + dev: false + /create-require/1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -2312,6 +2329,14 @@ packages: resolution: {integrity: sha512-/f6gpQuxDaqXu+1kwQYSckUglPaOrHdbIlBAu0YuW8/Cdb45XwXYNUBXg3r/9Mo6n540Kn/smKcZWko5x99KrQ==} dev: true + /cross-fetch/3.1.5: + resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==} + dependencies: + node-fetch: 2.6.7 + transitivePeerDependencies: + - encoding + dev: false + /cross-spawn/7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -2426,6 +2451,10 @@ packages: engines: {node: '>=8'} dev: true + /devtools-protocol/0.0.1068969: + resolution: {integrity: sha512-ATFTrPbY1dKYhPPvpjtwWKSK2mIwGmRwX54UASn9THEuIZCe2n9k3vVuMmt6jWeL+e5QaaguEv/pMyR+JQB7VQ==} + dev: false + /dezalgo/1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} dependencies: @@ -2490,7 +2519,6 @@ packages: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} dependencies: once: 1.4.0 - dev: true /enhanced-resolve/5.12.0: resolution: {integrity: sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==} @@ -2511,7 +2539,6 @@ packages: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: is-arrayish: 0.2.1 - dev: true /es-module-lexer/0.9.3: resolution: {integrity: sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==} @@ -2527,7 +2554,6 @@ packages: /escape-string-regexp/1.0.5: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} - dev: true /escape-string-regexp/2.0.0: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} @@ -2794,6 +2820,20 @@ packages: tmp: 0.0.33 dev: true + /extract-zip/2.0.1: + resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} + engines: {node: '>= 10.17.0'} + hasBin: true + dependencies: + debug: 4.3.4 + get-stream: 5.2.0 + yauzl: 2.10.0 + optionalDependencies: + '@types/yauzl': 2.10.0 + transitivePeerDependencies: + - supports-color + dev: false + /fast-deep-equal/3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true @@ -2840,6 +2880,12 @@ packages: resolution: {integrity: sha512-GDqVQezKzRABdeqflsgMr7ktzgF9CyS+p2oe0jJqUY6izSSbhPIQJDpoU4PtGcD7VPM9xh/dVrTu6z1nwgmEGw==} dev: true + /fd-slicer/1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + dependencies: + pend: 1.2.0 + dev: false + /figures/3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -2969,6 +3015,10 @@ packages: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + /fs-constants/1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + dev: false + /fs-extra/10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} engines: {node: '>=12'} @@ -3045,7 +3095,6 @@ packages: engines: {node: '>=8'} dependencies: pump: 3.0.0 - dev: true /get-stream/6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} @@ -3144,7 +3193,6 @@ packages: /has-flag/3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} - dev: true /has-flag/4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} @@ -3202,7 +3250,6 @@ packages: debug: 4.3.4 transitivePeerDependencies: - supports-color - dev: true /human-signals/1.1.1: resolution: {integrity: sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==} @@ -3241,7 +3288,6 @@ packages: dependencies: parent-module: 1.0.1 resolve-from: 4.0.0 - dev: true /import-local/3.1.0: resolution: {integrity: sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==} @@ -3329,7 +3375,6 @@ packages: /is-arrayish/0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} - dev: true /is-binary-path/2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} @@ -3886,7 +3931,6 @@ packages: /js-tokens/4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - dev: true /js-yaml/3.14.1: resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} @@ -3910,7 +3954,6 @@ packages: /json-parse-even-better-errors/2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} - dev: true /json-schema-traverse/0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -3990,7 +4033,6 @@ packages: /lines-and-columns/1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - dev: true /loader-runner/4.3.0: resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==} @@ -4160,6 +4202,10 @@ packages: /minimist/1.2.7: resolution: {integrity: sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==} + /mkdirp-classic/0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + dev: false + /mkdirp/0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} hasBin: true @@ -4460,7 +4506,6 @@ packages: engines: {node: '>=6'} dependencies: callsites: 3.1.0 - dev: true /parse-json/5.2.0: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} @@ -4470,7 +4515,6 @@ packages: error-ex: 1.3.2 json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 - dev: true /parse5-htmlparser2-tree-adapter/6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} @@ -4517,7 +4561,10 @@ packages: /path-type/4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - dev: true + + /pend/1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + dev: false /picocolors/1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -4680,6 +4727,11 @@ packages: /process-nextick-args/2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + /progress/2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + dev: false + /promptly/2.2.0: resolution: {integrity: sha512-aC9j+BZsRSSzEsXBNBwDnAxujdx19HycZoKgRgzWnS8eOHg1asuf9heuLprfbe739zY3IdUQx+Egv6Jn135WHA==} dependencies: @@ -4729,13 +4781,50 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: true /punycode/2.1.1: resolution: {integrity: sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==} engines: {node: '>=6'} dev: true + /puppeteer-core/19.5.0: + resolution: {integrity: sha512-s2EE2x/7UfvR4AP+6OokY7yEmIboZoQFc2InuWUu5grWG7uka3W2Nch6+R4ERQepH4NO8kkkEBzNO4PqtwU4lQ==} + engines: {node: '>=14.1.0'} + dependencies: + cross-fetch: 3.1.5 + debug: 4.3.4 + devtools-protocol: 0.0.1068969 + extract-zip: 2.0.1 + https-proxy-agent: 5.0.1 + proxy-from-env: 1.1.0 + rimraf: 3.0.2 + tar-fs: 2.1.1 + unbzip2-stream: 1.4.3 + ws: 8.11.0 + transitivePeerDependencies: + - bufferutil + - encoding + - supports-color + - utf-8-validate + dev: false + + /puppeteer/19.5.0: + resolution: {integrity: sha512-601tKJdKu/mlL6l5wklNJiQbIxyEU2N4pwZzFkwhr1VizNYV0Fly1aFMT7qAp7CvzklYLypQGKn7/Zgpy+HheA==} + engines: {node: '>=14.1.0'} + requiresBuild: true + dependencies: + cosmiconfig: 8.0.0 + https-proxy-agent: 5.0.1 + progress: 2.0.3 + proxy-from-env: 1.1.0 + puppeteer-core: 19.5.0 + transitivePeerDependencies: + - bufferutil + - encoding + - supports-color + - utf-8-validate + dev: false + /qs/6.11.0: resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} engines: {node: '>=0.6'} @@ -4803,7 +4892,6 @@ packages: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 - dev: true /readdirp/3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} @@ -4857,7 +4945,6 @@ packages: /resolve-from/4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} - dev: true /resolve-from/5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} @@ -5177,7 +5264,6 @@ packages: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} dependencies: safe-buffer: 5.2.1 - dev: true /strip-ansi/6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} @@ -5238,7 +5324,6 @@ packages: engines: {node: '>=4'} dependencies: has-flag: 3.0.0 - dev: true /supports-color/7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} @@ -5284,6 +5369,26 @@ packages: engines: {node: '>=6'} dev: true + /tar-fs/2.1.1: + resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 2.2.0 + dev: false + + /tar-stream/2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.4 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.0 + dev: false + /terminal-link/2.1.1: resolution: {integrity: sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==} engines: {node: '>=8'} @@ -5355,7 +5460,6 @@ packages: /through/2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} - dev: true /tmp/0.0.33: resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==} @@ -5651,6 +5755,13 @@ packages: engines: {node: '>=4.2.0'} hasBin: true + /unbzip2-stream/1.4.3: + resolution: {integrity: sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==} + dependencies: + buffer: 5.7.1 + through: 2.3.8 + dev: false + /universalify/0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -5883,6 +5994,19 @@ packages: optional: true dev: true + /ws/8.11.0: + resolution: {integrity: sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + /xml2js/0.4.23: resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} engines: {node: '>=4.0.0'} @@ -5966,6 +6090,13 @@ packages: y18n: 5.0.8 yargs-parser: 21.1.1 + /yauzl/2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + dev: false + /yn/3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} diff --git a/scripts/spider-nansen.ts b/scripts/spider-nansen.ts new file mode 100644 index 0000000..4d040e6 --- /dev/null +++ b/scripts/spider-nansen.ts @@ -0,0 +1,39 @@ +import { openUrl, delay } from '../src/utils'; + +const startTime = Date.now(); + +const url = 'https://pro.nansen.ai'; +const input = '0xab5c66752a9e8167967685f1450532fb96d5d24f'; + +openUrl(url, async (page) => { + // 填写登录表单 + await page.waitForSelector('input[name=password]'); + await page.keyboard.sendCharacter('43506575@qq.com'); + await page.keyboard.press('Tab'); + await page.keyboard.sendCharacter('Wokan1xia~'); + await page.screenshot({ path: 'screenshots/nansen/1-login.png' }); + + // 点击登录，跳转页面 + await page.keyboard.press('Tab'); + await page.keyboard.press('Enter'); + await page.waitForNavigation(); + await page.screenshot({ path: 'screenshots/nansen/2-login-success.png' }); + + // 打开 wallet profiler + await page.goto(`${url}/wallet-profiler?address=${input}`); + await delay(10000); + await page.screenshot({ path: 'screenshots/nansen/3-wallet-profiler.png' }); + + // 打开 labels 弹窗（要用会员账号才可以） + await (await page.waitForSelector('#address_labels button')).click(); + await page.screenshot({ path: 'screenshots/nansen/4-wallet-profiler-labels.png' }); + + // 解析 labels + const labels = await page.evaluate(() => + [...document.querySelectorAll('#portal-root span[data-value=label]')].map((s) => (s as HTMLSpanElement).innerText || ''), + ); + + console.log('\ninput:', input); + console.log('\noutput:', labels); + console.log(`\ndone 🎉 (use ${(Date.now() - startTime / 1000).toFixed(1)}s)\n`); +}); diff --git a/src/utils/index.ts b/src/utils/index.ts index 2d70768..0b47841 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1 +1,5 @@ +export * from './puppeteer'; + export const isDev = process.env.NODE_ENV !== 'production'; + +export const delay = (ms: number) => new Promise((r) => setTimeout(r, ms)); diff --git a/src/utils/puppeteer.ts b/src/utils/puppeteer.ts new file mode 100644 index 0000000..4147b3e --- /dev/null +++ b/src/utils/puppeteer.ts @@ -0,0 +1,12 @@ +import puppeteer, { Page, WaitForOptions } from 'puppeteer'; + +type Fn = (page: Page) => Promise | void; + +export async function openUrl(url: string, fn: Fn = () => {}, options: WaitForOptions = {}) { + console.log('[utils/puppeteer] open url:', url); + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto(url, options); + await fn(page); + await browser.close(); +}