diff --git a/README.md b/README.md index 5cc03728..49aea3c0 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,5 @@ _Status: alpha_ ## TODOs -- rename opencodegraphdata -> something shorter +- item.ui and item.ai +- add support for resolving annotations (because ) diff --git a/client/web-playground/package.json b/client/web-playground/package.json index 182a2a0c..af68c022 100644 --- a/client/web-playground/package.json +++ b/client/web-playground/package.json @@ -24,6 +24,7 @@ "@codemirror/lint": "^6.4.2", "@opencodegraph/client": "workspace:*", "@opencodegraph/codemirror-extension": "workspace:*", + "@opencodegraph/provider-docs": "workspace:*", "@opencodegraph/provider-links": "workspace:*", "@opencodegraph/provider-storybook": "workspace:*", "@opencodegraph/ui-react": "workspace:*", diff --git a/client/web-playground/src/demo/settings.ts b/client/web-playground/src/demo/settings.ts index b5462cd7..34603e63 100644 --- a/client/web-playground/src/demo/settings.ts +++ b/client/web-playground/src/demo/settings.ts @@ -1,8 +1,23 @@ import { type ProviderSettings } from '@opencodegraph/client' +const USE_STORED_CORPUS = true + async function getProviders(): Promise> { const providerSettings: Record = { - '../../../../provider/hello-world/index.ts': true, + '../../../../provider/hello-world/index.ts': false, + '../../../../provider/docs/src/provider/provider.ts': { + corpus: USE_STORED_CORPUS + ? { + url: new URL( + 'tmp-ocg-provider-docs/sourcegraph-docs-old-web-corpus.json', + import.meta.url + ).toString(), + } + : { + entryPage: 'http://localhost:5800/docs/start', + prefix: 'http://localhost:5800/docs', + }, + } satisfies import('@opencodegraph/provider-docs').Settings, '../../../../provider/links/index.ts': { links: [ { @@ -38,7 +53,9 @@ async function getProviders(): Promise ({ plugins: [react()], resolve: { - alias: - mode === 'development' + alias: [ + ...(mode === 'development' ? [ // In dev mode, build from TypeScript sources so we don't need to run `tsc -b` // in the background. @@ -17,8 +20,14 @@ export default defineConfig(({ mode }) => ({ replacement: '$1/src/index', }, ] - : [], + : []), + { + find: 'tmp-ocg-provider-docs', + replacement: docsProviderDataDir, + }, + ], }, + define: {}, css: { devSourcemap: true, modules: { @@ -27,6 +36,9 @@ export default defineConfig(({ mode }) => ({ }, server: { port: 5900, + fs: { + allow: [searchForWorkspaceRoot(process.cwd()), docsProviderDataDir], + }, }, build: { emptyOutDir: false, diff --git a/lib/client/src/api.ts b/lib/client/src/api.ts index c3fe52de..54eee0bd 100644 --- a/lib/client/src/api.ts +++ b/lib/client/src/api.ts @@ -56,6 +56,7 @@ export function observeAnnotations( emitPartial ? startWith(null) : tap(), catchError(error => { logger?.(`failed to get annotations: ${error}`) + console.error(error) return of(null) }) ) diff --git a/lib/protocol/package.json b/lib/protocol/package.json index 717c4a67..e1618178 100644 --- a/lib/protocol/package.json +++ b/lib/protocol/package.json @@ -17,7 +17,7 @@ ], "sideEffects": false, "scripts": { - "generate": "node --no-warnings=ExperimentalWarning --loader ts-node/esm/transpile-only ../schema/dev/generateJsonSchemaTypes.ts src/opencodegraph-protocol.schema.json \"import { Item, Annotation } from '@opencodegraph/schema'\" > src/opencodegraph-protocol.schema.ts && pnpm -w run format \"$PNPM_SCRIPT_SRC_DIR/src/opencodegraph-protocol.schema.ts\"", + "generate": "node --no-warnings=ExperimentalWarning --loader ts-node/esm/transpile-only ../schema/dev/generateJsonSchemaTypes.ts src/opencodegraph-protocol.schema.json \"import { Annotation } from '@opencodegraph/schema'\" > src/opencodegraph-protocol.schema.ts && pnpm -w run format \"$PNPM_SCRIPT_SRC_DIR/src/opencodegraph-protocol.schema.ts\"", "build": "pnpm run --silent generate && tsc --build", "test": "vitest", "prepublishOnly": "tsc --build --clean && pnpm run build" diff --git a/lib/schema/src/opencodegraph.schema.json b/lib/schema/src/opencodegraph.schema.json index cd0a0b01..d32b1ae0 100644 --- a/lib/schema/src/opencodegraph.schema.json +++ b/lib/schema/src/opencodegraph.schema.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "opencodegraph.schema.json#", - "title": "Data", + "title": "Schema", "description": "Metadata about code", "allowComments": true, "type": "object", @@ -14,35 +14,59 @@ }, "definitions": { "Annotation": { - "description": "An annotation describes information relevant to a specific range in a file.", + "description": "An annotation describes information relevant to a file (or a range within a file).", "type": "object", "additionalProperties": false, - "required": ["item", "range"], + "required": [], "properties": { - "item": { "$ref": "#/definitions/Item" }, - "range": { "$ref": "#/definitions/Range" } + "title": { + "description": "A descriptive title of the annotation.", + "type": "string" + }, + "url": { + "description": "An external URL with more information about the annotation.", + "type": "string", + "format": "uri" + }, + "ui": { + "description": "The human user interface of the annotation, with information for human consumption.", + "$ref": "#/definitions/AnnotationUI" + }, + "ai": { + "description": "Information from the annotation intended for consumption by AI, not humans.222", + "$ref": "#/definitions/AssistantInfo" + }, + "range": { + "description": "The range in the file that this annotation applies to. If not set, the annotation applies to the entire file.", + "$ref": "#/definitions/Range" + } } }, - "Item": { + "AnnotationUI": { + "description": "The human user interface of an annotation, with information for human consumption.", "type": "object", "additionalProperties": false, - "required": ["title"], "properties": { - "title": { "type": "string" }, - "detail": { "type": "string" }, - "url": { "description": "An external URL with more information.", "type": "string", "format": "uri" }, - "image": { "$ref": "#/definitions/ItemImage" } + "detail": { + "description": "Text containing additional details for the human, shown when they interact with the annotation.", + "type": "string" + }, + "format": { + "description": "The format of the title and description (Markdown or plain text).", + "type": "string", + "enum": ["markdown", "plaintext"] + } } }, - "ItemImage": { + "AssistantInfo": { + "description": "Information from an annotation intended for consumption by AI, not humans.", "type": "object", "additionalProperties": false, - "required": ["url"], "properties": { - "url": { "type": "string", "format": "uri" }, - "width": { "type": "number" }, - "height": { "type": "number" }, - "alt": { "type": "string" } + "content": { + "description": "Text content for AI to consume.", + "type": "string" + } } }, "Range": { diff --git a/lib/schema/src/opencodegraph.schema.ts b/lib/schema/src/opencodegraph.schema.ts index 71d0422b..d8a72e08 100644 --- a/lib/schema/src/opencodegraph.schema.ts +++ b/lib/schema/src/opencodegraph.schema.ts @@ -1,31 +1,50 @@ /** * Metadata about code */ -export interface Data { +export interface Schema { annotations?: Annotation[] } /** - * An annotation describes information relevant to a specific range in a file. + * An annotation describes information relevant to a file (or a range within a file). */ export interface Annotation { - item: Item - range: Range + /** + * A descriptive title of the annotation. + */ + title?: string + /** + * An external URL with more information about the annotation. + */ + url?: string + ui?: AnnotationUI + ai?: AssistantInfo + range?: Range } -export interface Item { - title: string +/** + * The human user interface of the annotation, with information for human consumption. + */ +export interface AnnotationUI { + /** + * Text containing additional details for the human, shown when they interact with the annotation. + */ detail?: string /** - * An external URL with more information. + * The format of the title and description (Markdown or plain text). */ - url?: string - image?: ItemImage + format?: 'markdown' | 'plaintext' } -export interface ItemImage { - url: string - width?: number - height?: number - alt?: string +/** + * Information from the annotation intended for consumption by AI, not humans.222 + */ +export interface AssistantInfo { + /** + * Text content for AI to consume. + */ + content?: string } +/** + * The range in the file that this annotation applies to. If not set, the annotation applies to the entire file. + */ export interface Range { start: Position end: Position diff --git a/package.json b/package.json index 817b031a..affeea48 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "ts-node": "^10.9.2", "typescript": "^5.3.3", "vite": "^5", - "vitest": "^0" + "vitest": "^1.1.0" }, "eslintConfig": { "extends": "./.config/eslintrc.js" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26ed6eda..7106f647 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,7 +54,7 @@ importers: version: 4.0.2(vite@5.0.4) '@vitest/coverage-v8': specifier: ^0.34.6 - version: 0.34.6(vitest@0.34.6) + version: 0.34.6(vitest@1.1.0) eslint: specifier: ^8.54.0 version: 8.54.0 @@ -101,8 +101,8 @@ importers: specifier: ^5 version: 5.0.4(@types/node@20.10.0) vitest: - specifier: ^0 - version: 0.34.6 + specifier: ^1.1.0 + version: 1.1.0(@types/node@20.10.0)(jsdom@23.0.1) client/browser: dependencies: @@ -336,6 +336,9 @@ importers: '@opencodegraph/codemirror-extension': specifier: workspace:* version: link:../codemirror + '@opencodegraph/provider-docs': + specifier: workspace:* + version: link:../../provider/docs '@opencodegraph/provider-links': specifier: workspace:* version: link:../../provider/links @@ -394,7 +397,7 @@ importers: version: 2.3.3 vitest-fetch-mock: specifier: ^0.2.2 - version: 0.2.2(vitest@0.34.6) + version: 0.2.2(vitest@1.1.0) lib/protocol: dependencies: @@ -462,6 +465,40 @@ importers: specifier: ^2.0.0 version: 2.0.0 + provider/docs: + dependencies: + '@mozilla/readability': + specifier: ^0.5.0 + version: 0.5.0 + '@opencodegraph/provider': + specifier: workspace:* + version: link:../../lib/provider + '@xenova/transformers': + specifier: ^2.12.1 + version: 2.12.1 + better-localstorage: + specifier: ^1.0.5 + version: 1.0.5 + buffer: + specifier: ^6.0.3 + version: 6.0.3 + env-paths: + specifier: ^3.0.0 + version: 3.0.0 + jsdom: + specifier: ^23.0.1 + version: 23.0.1 + onnxruntime-web: + specifier: '*' + version: 1.14.0 + devDependencies: + '@types/jsdom': + specifier: ^21.1.6 + version: 21.1.6 + vitest-fetch-mock: + specifier: ^0.2.2 + version: 0.2.2(vitest@1.1.0) + provider/hello-world: dependencies: '@opencodegraph/provider': @@ -488,7 +525,7 @@ importers: devDependencies: vitest-fetch-mock: specifier: ^0.2.2 - version: 0.2.2(vitest@0.34.6) + version: 0.2.2(vitest@1.1.0) web: dependencies: @@ -4044,6 +4081,11 @@ packages: resolution: {integrity: sha512-OfX7E2oUDYxtBvsuS4e/jSn4Q9Qb6DzgeYtsAdkPZ47znpoNsMgZw0+tVijiv3uGNR6dgNlty6r9rzIzHjtd/A==} dev: false + /@huggingface/jinja@0.1.2: + resolution: {integrity: sha512-x5mpbfJt1nKmVep5WNP5VjNsjWApWNj8pPYI+uYMkBWH9bWUJmQmHt2lbf0VCoQd54Oq3XuFEh/UyoVh7rPxmg==} + engines: {node: '>=18'} + dev: false + /@humanwhocodes/config-array@0.11.13: resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==} engines: {node: '>=10.10.0'} @@ -4118,6 +4160,13 @@ packages: '@sinclair/typebox': 0.27.8 dev: true + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: true + /@jest/transform@29.6.0: resolution: {integrity: sha512-bhP/KxPo3e322FJ0nKAcb6WVK76ZYyQd1lWygJzoSqP8SYMSLdxHqP4wnPTI4WvbB8PKPDV30y5y7Tya4RHOBA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -4342,6 +4391,11 @@ packages: - supports-color dev: false + /@mozilla/readability@0.5.0: + resolution: {integrity: sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==} + engines: {node: '>=14.0.0'} + dev: false + /@ndelangen/get-tarball@3.0.9: resolution: {integrity: sha512-9JKTEik4vq+yGosHYhZ1tiH/3WpUS0Nh0kej4Agndhox8pAdWhEx5knFVRcb/ya9knCRCs1rPxNrSXTDdfVqpA==} dependencies: @@ -4442,6 +4496,49 @@ packages: resolution: {integrity: sha512-C16M+IYz0rgRhWZdCmK+h58JMv8vijAA61gmz2rspCSwKwzBebpdcsiUmwrtJRdphuY30i6BSLEOP8ppbNLyLg==} dev: false + /@protobufjs/aspromise@1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + dev: false + + /@protobufjs/base64@1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + dev: false + + /@protobufjs/codegen@2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + dev: false + + /@protobufjs/eventemitter@1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + dev: false + + /@protobufjs/fetch@1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + dev: false + + /@protobufjs/float@1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + dev: false + + /@protobufjs/inquire@1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + dev: false + + /@protobufjs/path@1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + dev: false + + /@protobufjs/pool@1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + dev: false + + /@protobufjs/utf8@1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + dev: false + /@radix-ui/primitive@1.0.1: resolution: {integrity: sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==} dependencies: @@ -6530,16 +6627,6 @@ packages: '@types/connect': 3.4.32 '@types/node': 20.10.0 - /@types/chai-subset@1.3.3: - resolution: {integrity: sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==} - dependencies: - '@types/chai': 4.3.5 - dev: true - - /@types/chai@4.3.5: - resolution: {integrity: sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==} - dev: true - /@types/chrome@0.0.254: resolution: {integrity: sha512-svkOGKwA+6ZZuk9xtrYun8MYpNY/9hD17rgZ19v3KunhsK1ZOKaMESw12/1AXLh1u3UPA8jQIRi2370DXv9wgw==} dependencies: @@ -6699,6 +6786,14 @@ packages: resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} dev: true + /@types/jsdom@21.1.6: + resolution: {integrity: sha512-/7kkMsC+/kMs7gAYmmBR9P0vGTnOoLhQhyhQJSlXGI5bzTHp6xdo0TtKWQAsz6pmSAeVqKSbqeyP6hytqr9FDw==} + dependencies: + '@types/node': 20.10.0 + '@types/tough-cookie': 4.0.5 + parse5: 7.1.2 + dev: true + /@types/json-schema@7.0.14: resolution: {integrity: sha512-U3PUjAudAdJBeC2pgN8uTIKgxrb4nlDF3SF0++EldXQvQBGkpFZMSnwQiIoDU77tv45VgNkl/L4ouD+rEomujw==} dev: true @@ -6711,6 +6806,10 @@ packages: resolution: {integrity: sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==} dev: true + /@types/long@4.0.2: + resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} + dev: false + /@types/mdast@3.0.15: resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} dependencies: @@ -6842,6 +6941,10 @@ packages: resolution: {integrity: sha512-Hy6UMpxhE3j1tLpl27exp1XqHD7n8chAiNPzWfz16LPZoMMoSc4dzLl6w9qijkEb/r5O1ozdu1CWGA2L83ZeZg==} dev: true + /@types/tough-cookie@4.0.5: + resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} + dev: true + /@types/unist@2.0.3: resolution: {integrity: sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==} @@ -7216,7 +7319,7 @@ packages: - supports-color dev: true - /@vitest/coverage-v8@0.34.6(vitest@0.34.6): + /@vitest/coverage-v8@0.34.6(vitest@1.1.0): resolution: {integrity: sha512-fivy/OK2d/EsJFoEoxHFEnNGTg+MmdZBAVK9Ka4qhXR2K3J0DS08vcGVwzDtXSuUMabLv4KtPcpSKkcMXFDViw==} peerDependencies: vitest: '>=0.32.0 <1' @@ -7232,47 +7335,47 @@ packages: std-env: 3.3.3 test-exclude: 6.0.0 v8-to-istanbul: 9.2.0 - vitest: 0.34.6 + vitest: 1.1.0(@types/node@20.10.0)(jsdom@23.0.1) transitivePeerDependencies: - supports-color dev: true - /@vitest/expect@0.34.6: - resolution: {integrity: sha512-QUzKpUQRc1qC7qdGo7rMK3AkETI7w18gTCUrsNnyjjJKYiuUB9+TQK3QnR1unhCnWRC0AbKv2omLGQDF/mIjOw==} + /@vitest/expect@1.1.0: + resolution: {integrity: sha512-9IE2WWkcJo2BR9eqtY5MIo3TPmS50Pnwpm66A6neb2hvk/QSLfPXBz2qdiwUOQkwyFuuXEUj5380CbwfzW4+/w==} dependencies: - '@vitest/spy': 0.34.6 - '@vitest/utils': 0.34.6 + '@vitest/spy': 1.1.0 + '@vitest/utils': 1.1.0 chai: 4.3.10 dev: true - /@vitest/runner@0.34.6: - resolution: {integrity: sha512-1CUQgtJSLF47NnhN+F9X2ycxUP0kLHQ/JWvNHbeBfwW8CzEGgeskzNnHDyv1ieKTltuR6sdIHV+nmR6kPxQqzQ==} + /@vitest/runner@1.1.0: + resolution: {integrity: sha512-zdNLJ00pm5z/uhbWF6aeIJCGMSyTyWImy3Fcp9piRGvueERFlQFbUwCpzVce79OLm2UHk9iwaMSOaU9jVHgNVw==} dependencies: - '@vitest/utils': 0.34.6 - p-limit: 4.0.0 + '@vitest/utils': 1.1.0 + p-limit: 5.0.0 pathe: 1.1.1 dev: true - /@vitest/snapshot@0.34.6: - resolution: {integrity: sha512-B3OZqYn6k4VaN011D+ve+AA4whM4QkcwcrwaKwAbyyvS/NB1hCWjFIBQxAQQSQir9/RtyAAGuq+4RJmbn2dH4w==} + /@vitest/snapshot@1.1.0: + resolution: {integrity: sha512-5O/wyZg09V5qmNmAlUgCBqflvn2ylgsWJRRuPrnHEfDNT6tQpQ8O1isNGgo+VxofISHqz961SG3iVvt3SPK/QQ==} dependencies: - magic-string: 0.30.1 + magic-string: 0.30.5 pathe: 1.1.1 - pretty-format: 29.6.0 + pretty-format: 29.7.0 dev: true - /@vitest/spy@0.34.6: - resolution: {integrity: sha512-xaCvneSaeBw/cz8ySmF7ZwGvL0lBjfvqc1LpQ/vcdHEvpLn3Ff1vAvjw+CoGn0802l++5L/pxb7whwcWAw+DUQ==} + /@vitest/spy@1.1.0: + resolution: {integrity: sha512-sNOVSU/GE+7+P76qYo+VXdXhXffzWZcYIPQfmkiRxaNCSPiLANvQx5Mx6ZURJ/ndtEkUJEpvKLXqAYTKEY+lTg==} dependencies: - tinyspy: 2.1.1 + tinyspy: 2.2.0 dev: true - /@vitest/utils@0.34.6: - resolution: {integrity: sha512-IG5aDD8S6zlvloDsnzHw0Ut5xczlF+kv2BOTo+iXfPr54Yhi5qbVOgGB1hZaVq4iJ4C/MZ2J0y15IlsV/ZcI0A==} + /@vitest/utils@1.1.0: + resolution: {integrity: sha512-z+s510fKmYz4Y41XhNs3vcuFTFhcij2YF7F8VQfMEYAAUfqQh0Zfg7+w9xdgFGhPf3tX3TicAe+8BDITk6ampQ==} dependencies: - diff-sequences: 29.4.3 - loupe: 2.3.6 - pretty-format: 29.6.0 + diff-sequences: 29.6.3 + loupe: 2.3.7 + pretty-format: 29.7.0 dev: true /@vscode/test-electron@2.3.2: @@ -7343,6 +7446,16 @@ packages: resolution: {integrity: sha512-CqTpxOlUCPWRNUPZDxT5v2NnHXA4oox612iUGnmTUGQFhZ1Gkj8kirtl/2wcF6MqX7+PqqicZzOCBKKfIn0dww==} dev: true + /@xenova/transformers@2.12.1: + resolution: {integrity: sha512-b++KSLKezi9Vic9VPXBc/egE5dTw11fvqCcWRg+AQgS+hLGNc7E/sL6JRNhnZ4NmKW0Sx/2gKs33rYllTC1xKA==} + dependencies: + '@huggingface/jinja': 0.1.2 + onnxruntime-web: 1.14.0 + sharp: 0.32.6 + optionalDependencies: + onnxruntime-node: 1.14.0 + dev: false + /@yarnpkg/esbuild-plugin-pnp@3.0.0-rc.15(esbuild@0.17.19): resolution: {integrity: sha512-kYzDJO5CA9sy+on/s2aIW0411AklfCi8Ck/4QDivOqsMKpStZA2SsR+X27VTggGwpStWaLrjJcDcdDMowtG8MA==} engines: {node: '>=14.15.0'} @@ -7389,6 +7502,15 @@ packages: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 dependencies: acorn: 8.10.0 + dev: true + + /acorn-jsx@5.3.2(acorn@8.11.2): + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + dependencies: + acorn: 8.11.2 + dev: false /acorn-walk@7.2.0: resolution: {integrity: sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==} @@ -7399,6 +7521,11 @@ packages: resolution: {integrity: sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==} engines: {node: '>=0.4.0'} + /acorn-walk@8.3.1: + resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} + engines: {node: '>=0.4.0'} + dev: true + /acorn@7.4.1: resolution: {integrity: sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==} engines: {node: '>=0.4.0'} @@ -7442,7 +7569,6 @@ packages: debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color - dev: true /aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} @@ -7700,7 +7826,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /autoprefixer@10.4.16(postcss@8.4.32): resolution: {integrity: sha512-7vd3UC6xKp0HLfua5IjZlcXvGAGy7cBAXTg2lyQ/8WpNhd6SiZ8Be+xm3FyBSYJx5GKcpRCzBh7RH4/0dnY+uQ==} @@ -7740,7 +7865,6 @@ packages: /b4a@1.6.4: resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==} - dev: true /babel-core@7.0.0-bridge.0(@babel/core@7.22.8): resolution: {integrity: sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==} @@ -7848,7 +7972,6 @@ packages: /base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} requiresBuild: true - dev: true /basic-auth@2.0.1: resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} @@ -7865,6 +7988,10 @@ packages: is-stream: 2.0.0 dev: true + /better-localstorage@1.0.5: + resolution: {integrity: sha512-fRS8BjU/Fl2Aeq4HLY7QPN5qaasYpZ9B4GB3ljEoBuo46gj36gDw6Yfi33ZHEklI1n707c1JmdZI9BK9kAebPA==} + dev: false + /better-opn@2.1.1: resolution: {integrity: sha512-kIPXZS5qwyKiX/HcRvDYfmBQUa8XP17I0mYZZ0y4UhpYOSvtsLHDYqmomS+Mj20aDvD3knEiQ0ecQy2nhio3yA==} engines: {node: '>8.0.0'} @@ -7895,7 +8022,6 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 - dev: true /body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} @@ -8000,7 +8126,13 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: true + + /buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false /builtin-modules@3.3.0: resolution: {integrity: sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==} @@ -8210,7 +8342,6 @@ packages: /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} requiresBuild: true - dev: true /chownr@2.0.0: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} @@ -8350,7 +8481,6 @@ packages: engines: {node: '>=7.0.0'} dependencies: color-name: 1.1.4 - dev: true /color-name@1.1.3: resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} @@ -8358,13 +8488,27 @@ packages: /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - dev: true + + /color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.2 + dev: false /color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} hasBin: true dev: true + /color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + dev: false + /colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} dev: true @@ -8378,7 +8522,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /comma-separated-tokens@1.0.8: resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} @@ -8612,6 +8755,12 @@ packages: engines: {node: '>=4'} hasBin: true + /cssstyle@3.0.0: + resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==} + engines: {node: '>=14'} + dependencies: + rrweb-cssom: 0.6.0 + /csstype@3.1.0: resolution: {integrity: sha512-uX1KG+x9h5hIJsaKR9xHUeUraxf8IODOwq9JLNPq6BwB04a/xgpq3rcx47l5BZu5zBPlgD342tdke3Hom/nJRA==} @@ -8626,6 +8775,13 @@ packages: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} dev: true + /data-urls@5.0.0: + resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} + engines: {node: '>=18'} + dependencies: + whatwg-mimetype: 4.0.0 + whatwg-url: 14.0.0 + /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -8689,6 +8845,9 @@ packages: engines: {node: '>=10'} dev: true + /decimal.js@10.4.3: + resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} + /decode-named-character-reference@1.0.2: resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} dependencies: @@ -8700,8 +8859,6 @@ packages: requiresBuild: true dependencies: mimic-response: 3.1.0 - dev: true - optional: true /deep-eql@4.1.3: resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==} @@ -8742,8 +8899,6 @@ packages: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} requiresBuild: true - dev: true - optional: true /deep-is@0.1.3: resolution: {integrity: sha512-GtxAN4HvBachZzm4OnWqc45ESpUCMwkYcsjnsPs23FwJbsO+k4t0k9bQCgOmzIlpHO28+WPK/KRbRk0DDHuuDw==} @@ -8819,7 +8974,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -8851,8 +9005,11 @@ packages: resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} engines: {node: '>=8'} requiresBuild: true - dev: true - optional: true + + /detect-libc@2.0.2: + resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} + engines: {node: '>=8'} + dev: false /detect-node-es@1.1.0: resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} @@ -8884,8 +9041,8 @@ packages: /didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} - /diff-sequences@29.4.3: - resolution: {integrity: sha512-ofrBgwpPhCD85kMKtE9RYFFq6OC1A89oW2vvgWZNCwxrUpRUILopY7lsYyMDSjc8g6U6aiO0Qubg6r4Wgt5ZnA==} + /diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dev: true @@ -9007,7 +9164,6 @@ packages: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} dependencies: once: 1.4.0 - dev: true /entities@2.1.0: resolution: {integrity: sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==} @@ -9016,7 +9172,11 @@ packages: /entities@4.5.0: resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} engines: {node: '>=0.12'} - dev: true + + /env-paths@3.0.0: + resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false /envinfo@7.10.0: resolution: {integrity: sha512-ZtUjZO6l5mwTHvc1L9+1q5p/R3wTopcfqMW8r5t8SJSKqeVI/LtajORwRFEKpEFuekjD0VBjwu1HMxL4UalIRw==} @@ -9899,12 +10059,25 @@ packages: strip-final-newline: 3.0.0 dev: true + /execa@8.0.1: + resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} + engines: {node: '>=16.17'} + dependencies: + cross-spawn: 7.0.3 + get-stream: 8.0.1 + human-signals: 5.0.0 + is-stream: 3.0.0 + merge-stream: 2.0.0 + npm-run-path: 5.1.0 + onetime: 6.0.0 + signal-exit: 4.1.0 + strip-final-newline: 3.0.0 + dev: true + /expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} requiresBuild: true - dev: true - optional: true /express@4.18.2: resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} @@ -9985,7 +10158,6 @@ packages: /fast-fifo@1.3.2: resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - dev: true /fast-glob@3.3.1: resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==} @@ -10138,6 +10310,10 @@ packages: hasBin: true dev: true + /flatbuffers@1.12.0: + resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==} + dev: false + /flatted@3.2.1: resolution: {integrity: sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg==} dev: true @@ -10187,6 +10363,14 @@ packages: mime-types: 2.1.35 dev: true + /form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + /format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} @@ -10207,7 +10391,6 @@ packages: /fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} requiresBuild: true - dev: true /fs-extra@10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} @@ -10337,6 +10520,11 @@ packages: engines: {node: '>=10'} dev: true + /get-stream@8.0.1: + resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} + engines: {node: '>=16'} + dev: true + /get-symbol-description@1.0.0: resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} engines: {node: '>= 0.4'} @@ -10363,8 +10551,6 @@ packages: /github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} requiresBuild: true - dev: true - optional: true /github-slugger@1.5.0: resolution: {integrity: sha512-wIh+gKBI9Nshz2o46B0B3f5k/W+WI9ZAv6y5Dn5WJ5SK1t0TnDimB4WE5rmTD05ZAIn8HALCZVmCsvj0w0v0lw==} @@ -10529,6 +10715,10 @@ packages: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} dev: true + /guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + dev: false + /gunzip-maybe@1.4.2: resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} hasBin: true @@ -10710,6 +10900,12 @@ packages: lru-cache: 6.0.0 dev: true + /html-encoding-sniffer@4.0.0: + resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} + engines: {node: '>=18'} + dependencies: + whatwg-encoding: 3.1.1 + /html-escaper@2.0.0: resolution: {integrity: sha512-a4u9BeERWGu/S8JiWEAQcdrg9v4QArtP9keViQjGMdff20fBdd8waotXaNmODqBe6uZ3Nafi7K/ho4gCQHV3Ig==} dev: true @@ -10786,7 +10982,6 @@ packages: debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color - dev: true /https-proxy-agent@4.0.0: resolution: {integrity: sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==} @@ -10816,7 +11011,6 @@ packages: debug: 4.3.4(supports-color@8.1.1) transitivePeerDependencies: - supports-color - dev: true /human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} @@ -10828,16 +11022,26 @@ packages: engines: {node: '>=14.18.0'} dev: true + /human-signals@5.0.0: + resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} + engines: {node: '>=16.17.0'} + dev: true + /iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} dependencies: safer-buffer: 2.1.2 + /iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + /ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} requiresBuild: true - dev: true /ignore@5.2.4: resolution: {integrity: sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==} @@ -10896,7 +11100,6 @@ packages: /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} requiresBuild: true - dev: true /ini@4.1.1: resolution: {integrity: sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g==} @@ -10979,6 +11182,10 @@ packages: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} dev: true + /is-arrayish@0.3.2: + resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} + dev: false + /is-bigint@1.0.1: resolution: {integrity: sha512-J0ELF4yHFxHy0cmSxZuheDOz2luOdVvqjwmEcj8H/L1JHeuEDSDbeRP+Dk9kFVk5RTFzbucJ2Kb9F7ixY2QaCg==} @@ -11167,6 +11374,9 @@ packages: engines: {node: '>=0.10.0'} dev: true + /is-potential-custom-element-name@1.0.1: + resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + /is-promise@2.2.2: resolution: {integrity: sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==} dev: true @@ -11501,6 +11711,41 @@ packages: engines: {node: '>=12.0.0'} dev: true + /jsdom@23.0.1: + resolution: {integrity: sha512-2i27vgvlUsGEBO9+/kJQRbtqtm+191b5zAZrU/UezVmnC2dlDAFLgDYJvAEi94T4kjsRKkezEtLQTgsNEsW2lQ==} + engines: {node: '>=18'} + peerDependencies: + canvas: ^2.11.2 + peerDependenciesMeta: + canvas: + optional: true + dependencies: + cssstyle: 3.0.0 + data-urls: 5.0.0 + decimal.js: 10.4.3 + form-data: 4.0.0 + html-encoding-sniffer: 4.0.0 + http-proxy-agent: 7.0.0 + https-proxy-agent: 7.0.2 + is-potential-custom-element-name: 1.0.1 + nwsapi: 2.2.7 + parse5: 7.1.2 + rrweb-cssom: 0.6.0 + saxes: 6.0.0 + symbol-tree: 3.2.4 + tough-cookie: 4.1.3 + w3c-xmlserializer: 5.0.0 + webidl-conversions: 7.0.0 + whatwg-encoding: 3.1.1 + whatwg-mimetype: 4.0.0 + whatwg-url: 14.0.0 + ws: 8.15.1 + xml-name-validator: 5.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + /jsesc@0.5.0: resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==} hasBin: true @@ -11785,9 +12030,12 @@ packages: import-meta-resolve: 2.2.2 dev: true - /local-pkg@0.4.3: - resolution: {integrity: sha512-SFppqq5p42fe2qcZQqqEOiVRXl+WCP1MdT6k7BDEW1j++sp5fIY+/fdRQitvKgB5BrBcmrs5m/L0v2FrU5MY1g==} + /local-pkg@0.5.0: + resolution: {integrity: sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==} engines: {node: '>=14'} + dependencies: + mlly: 1.4.2 + pkg-types: 1.0.3 dev: true /locate-path@2.0.0: @@ -11844,6 +12092,10 @@ packages: is-unicode-supported: 0.1.0 dev: true + /long@4.0.0: + resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} + dev: false + /longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -11860,6 +12112,12 @@ packages: get-func-name: 2.0.2 dev: true + /loupe@2.3.7: + resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + dependencies: + get-func-name: 2.0.2 + dev: true + /lru-cache@10.1.0: resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==} engines: {node: 14 || >=16.14} @@ -11875,7 +12133,6 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 - dev: true /lru-queue@0.1.0: resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==} @@ -11912,6 +12169,13 @@ packages: '@jridgewell/sourcemap-codec': 1.4.15 dev: true + /magic-string@0.30.5: + resolution: {integrity: sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: true + /make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -12446,8 +12710,8 @@ packages: /micromark-extension-mdxjs@3.0.0: resolution: {integrity: sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==} dependencies: - acorn: 8.10.0 - acorn-jsx: 5.3.2(acorn@8.10.0) + acorn: 8.11.2 + acorn-jsx: 5.3.2(acorn@8.11.2) micromark-extension-mdx-expression: 3.0.0 micromark-extension-mdx-jsx: 3.0.0 micromark-extension-mdx-md: 2.0.0 @@ -12867,8 +13131,6 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} requiresBuild: true - dev: true - optional: true /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} @@ -12912,7 +13174,6 @@ packages: /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: true /minipass@3.3.5: resolution: {integrity: sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==} @@ -12943,7 +13204,6 @@ packages: /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: true /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -12958,13 +13218,13 @@ packages: hasBin: true dev: true - /mlly@1.4.0: - resolution: {integrity: sha512-ua8PAThnTwpprIaU47EPeZ/bPUVp2QYBbWMphUQpVdBI3Lgqzm5KZQ45Agm3YJedHXaIHl6pBGabaLSUPPSptg==} + /mlly@1.4.2: + resolution: {integrity: sha512-i/Ykufi2t1EZ6NaPLdfnZk2AX8cs0d+mTzVKuPfqPKPatxLApaBoxJQ9x1/uckXtrS/U5oisPMDkNs0yQTaBRg==} dependencies: - acorn: 8.10.0 + acorn: 8.11.2 pathe: 1.1.1 pkg-types: 1.0.3 - ufo: 1.1.2 + ufo: 1.3.2 dev: true /mocha@10.2.0: @@ -13065,8 +13325,6 @@ packages: /napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} requiresBuild: true - dev: true - optional: true /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -13090,8 +13348,6 @@ packages: requiresBuild: true dependencies: semver: 7.5.4 - dev: true - optional: true /node-addon-api@4.3.0: resolution: {integrity: sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==} @@ -13099,6 +13355,10 @@ packages: dev: true optional: true + /node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} + dev: false + /node-dir@0.1.17: resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==} engines: {node: '>= 0.10.5'} @@ -13203,6 +13463,9 @@ packages: boolbase: 1.0.0 dev: true + /nwsapi@2.2.7: + resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==} + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -13320,6 +13583,36 @@ packages: resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} dev: true + /onnx-proto@4.0.4: + resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==} + dependencies: + protobufjs: 6.11.4 + dev: false + + /onnxruntime-common@1.14.0: + resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==} + dev: false + + /onnxruntime-node@1.14.0: + resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==} + os: [win32, darwin, linux] + requiresBuild: true + dependencies: + onnxruntime-common: 1.14.0 + dev: false + optional: true + + /onnxruntime-web@1.14.0: + resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==} + dependencies: + flatbuffers: 1.12.0 + guid-typescript: 1.0.9 + long: 4.0.0 + onnx-proto: 4.0.4 + onnxruntime-common: 1.14.0 + platform: 1.3.6 + dev: false + /open@7.4.2: resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} engines: {node: '>=8'} @@ -13411,9 +13704,9 @@ packages: yocto-queue: 0.1.0 dev: true - /p-limit@4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + /p-limit@5.0.0: + resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==} + engines: {node: '>=18'} dependencies: yocto-queue: 1.0.0 dev: true @@ -13537,7 +13830,6 @@ packages: resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} dependencies: entities: 4.5.0 - dev: true /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} @@ -13672,10 +13964,14 @@ packages: resolution: {integrity: sha512-nN7pYi0AQqJnoLPC9eHFQ8AcyaixBUOwvqc5TDnIKCMEE6I0y8P7OKA7fPexsXGCGxQDl/cmrLAp26LhcwxZ4A==} dependencies: jsonc-parser: 3.2.0 - mlly: 1.4.0 + mlly: 1.4.2 pathe: 1.1.1 dev: true + /platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + dev: false + /playwright-core@1.39.0: resolution: {integrity: sha512-+k4pdZgs1qiM+OUkSjx96YiKsXsmb59evFoqv8SKO067qBA+Z2s/dCzJij/ZhdQcs2zlTAgRKfeiiLm8PQ2qvw==} engines: {node: '>=16'} @@ -13825,8 +14121,6 @@ packages: simple-get: 4.0.1 tar-fs: 2.1.1 tunnel-agent: 0.6.0 - dev: true - optional: true /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -13845,11 +14139,11 @@ packages: hasBin: true dev: true - /pretty-format@29.6.0: - resolution: {integrity: sha512-XH+D4n7Ey0iSR6PdAnBs99cWMZdGsdKrR33iUHQNr79w1szKTCIZDVdXuccAsHVwDBp0XeWPfNEoaxP9EZgRmQ==} + /pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: - '@jest/schemas': 29.6.0 + '@jest/schemas': 29.6.3 ansi-styles: 5.2.0 react-is: 18.2.0 dev: true @@ -13917,6 +14211,26 @@ packages: resolution: {integrity: sha512-9t5qARVofg2xQqKtytzt+lZ4d1Qvj8t5B8fEwXK6qOfgRLgH/b13QlgEyDh033NOS31nXeFbYv7CLUDG1CeifQ==} dev: false + /protobufjs@6.11.4: + resolution: {integrity: sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==} + hasBin: true + requiresBuild: true + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/long': 4.0.2 + '@types/node': 20.10.0 + long: 4.0.0 + dev: false + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -13928,6 +14242,9 @@ packages: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: true + /psl@1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + /pump@2.0.1: resolution: {integrity: sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==} dependencies: @@ -13940,7 +14257,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: true /pumpify@1.5.1: resolution: {integrity: sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==} @@ -13953,7 +14269,10 @@ packages: /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} - dev: true + + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} /puppeteer-core@2.1.1: resolution: {integrity: sha512-n13AWriBMPYxnpbb6bnaY5YoY6rGj8vPLrz6CZF3o0qJNEwlcfJVxBzYZ0NJsQ21UbdJoijPCDrM++SUVEz7+w==} @@ -14015,9 +14334,11 @@ packages: side-channel: 1.0.4 dev: true + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} + /queue-tick@1.0.1: resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==} - dev: true /quick-lru@5.1.1: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} @@ -14056,8 +14377,6 @@ packages: ini: 1.3.8 minimist: 1.2.8 strip-json-comments: 2.0.1 - dev: true - optional: true /react-colorful@5.6.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-1exovf0uGTGyq5mXQT0zgQ80uvj2PCwvF8zY1RN9/vbJVSjSo3fsB/4L3ObbF7u70NduSiK4xu4Y6q1MHoUGEw==} @@ -14306,7 +14625,6 @@ packages: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 - dev: true /readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} @@ -14533,6 +14851,9 @@ packages: engines: {node: '>=0.10.5'} dev: true + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + /resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -14645,6 +14966,9 @@ packages: '@rollup/rollup-win32-x64-msvc': 4.6.1 fsevents: 2.3.3 + /rrweb-cssom@0.6.0: + resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==} + /run-applescript@5.0.0: resolution: {integrity: sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg==} engines: {node: '>=12'} @@ -14707,6 +15031,12 @@ packages: resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} dev: true + /saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + dependencies: + xmlchars: 2.2.0 + /scheduler@0.23.0: resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} dependencies: @@ -14733,7 +15063,6 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - dev: true /send@0.18.0: resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} @@ -14822,6 +15151,21 @@ packages: kind-of: 6.0.3 dev: true + /sharp@0.32.6: + resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} + engines: {node: '>=14.15.0'} + requiresBuild: true + dependencies: + color: 4.2.3 + detect-libc: 2.0.2 + node-addon-api: 6.1.0 + prebuild-install: 7.1.1 + semver: 7.5.4 + simple-get: 4.0.1 + tar-fs: 3.0.4 + tunnel-agent: 0.6.0 + dev: false + /shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -14868,11 +15212,14 @@ packages: engines: {node: '>=14'} dev: true + /signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + dev: true + /simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} requiresBuild: true - dev: true - optional: true /simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} @@ -14881,8 +15228,12 @@ packages: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 - dev: true - optional: true + + /simple-swizzle@0.2.2: + resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} + dependencies: + is-arrayish: 0.3.2 + dev: false /simple-update-notifier@1.1.0: resolution: {integrity: sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==} @@ -14996,6 +15347,10 @@ packages: resolution: {integrity: sha512-Rz6yejtVyWnVjC1RFvNmYL10kgjC49EOghxWn0RFqlCHGFpQx+Xe7yW3I4ceK1SGrWIGMjD5Kbue8W/udkbMJg==} dev: true + /std-env@3.7.0: + resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==} + dev: true + /stop-iteration-iterator@1.0.0: resolution: {integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==} engines: {node: '>= 0.4'} @@ -15033,7 +15388,6 @@ packages: dependencies: fast-fifo: 1.3.2 queue-tick: 1.0.1 - dev: true /string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} @@ -15093,7 +15447,6 @@ packages: requiresBuild: true dependencies: safe-buffer: 5.2.1 - dev: true /stringify-entities@4.0.3: resolution: {integrity: sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g==} @@ -15148,18 +15501,16 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} requiresBuild: true - dev: true - optional: true /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} dev: true - /strip-literal@1.0.1: - resolution: {integrity: sha512-QZTsipNpa2Ppr6v1AmJHESqJ3Uz247MUS0OjrnnZjFAvEoWqxuyFuXn2xLgMtRnijJShAa1HL0gtJyUs7u7n3Q==} + /strip-literal@1.3.0: + resolution: {integrity: sha512-PugKzOsyXpArk0yWmUwqOZecSO0GH0bPoctLcqNDH9J04pVW3lflYE0ujElBGTloevcxF5MofAOZ7C5l2b+wLg==} dependencies: - acorn: 8.10.0 + acorn: 8.11.2 dev: true /style-mod@4.1.0: @@ -15325,6 +15676,9 @@ packages: - utf-8-validate dev: true + /symbol-tree@3.2.4: + resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} + /synchronous-promise@2.0.17: resolution: {integrity: sha512-AsS729u2RHUfEra9xJrE39peJcc2stq2+poBXX8bcM08Y6g9j/i/PUzwNQqkaJde7Ntg1TO7bSREbR5sdosQ+g==} dev: true @@ -15400,7 +15754,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 - dev: true /tar-fs@3.0.4: resolution: {integrity: sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==} @@ -15408,7 +15761,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 3.1.6 - dev: true /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} @@ -15420,7 +15772,6 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 - dev: true /tar-stream@3.1.6: resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==} @@ -15428,7 +15779,6 @@ packages: b4a: 1.6.4 fast-fifo: 1.3.2 streamx: 2.15.1 - dev: true /tar@6.1.13: resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==} @@ -15523,17 +15873,17 @@ packages: resolution: {integrity: sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==} dev: true - /tinybench@2.5.0: - resolution: {integrity: sha512-kRwSG8Zx4tjF9ZiyH4bhaebu+EDz1BOx9hOigYHlUW4xxI/wKIUQUqo018UlU4ar6ATPBsaMrdbKZ+tmPdohFA==} + /tinybench@2.5.1: + resolution: {integrity: sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==} dev: true - /tinypool@0.7.0: - resolution: {integrity: sha512-zSYNUlYSMhJ6Zdou4cJwo/p7w5nmAH17GRfU/ui3ctvjXFErXXkruT4MWW6poDeXgCaIBlGLrfU6TbTXxyGMww==} + /tinypool@0.8.1: + resolution: {integrity: sha512-zBTCK0cCgRROxvs9c0CGK838sPkeokNGdQVUUwHAbynHFlmyJYj825f/oRs528HaIJ97lo0pLIlDUzwN+IorWg==} engines: {node: '>=14.0.0'} dev: true - /tinyspy@2.1.1: - resolution: {integrity: sha512-XPJL2uSzcOyBMky6OFrusqWlzfFrXtE0hPuMgW8A2HmaqrPo4ZQHRN/V0QXN3FSjKxpsbRrFc5LI7KOwBsT1/w==} + /tinyspy@2.2.0: + resolution: {integrity: sha512-d2eda04AN/cPOR89F7Xv5bK/jrQEhmcLFe6HFldoeO9AJtps+fqEnh486vnT/8y4bw38pSyxDcTCAq+Ks2aJTg==} engines: {node: '>=14.0.0'} dev: true @@ -15580,9 +15930,24 @@ packages: engines: {node: '>=6'} dev: false + /tough-cookie@4.1.3: + resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} + engines: {node: '>=6'} + dependencies: + psl: 1.9.0 + punycode: 2.3.0 + universalify: 0.2.0 + url-parse: 1.5.10 + /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + /tr46@5.0.0: + resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==} + engines: {node: '>=18'} + dependencies: + punycode: 2.3.1 + /tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -15696,8 +16061,6 @@ packages: requiresBuild: true dependencies: safe-buffer: 5.2.1 - dev: true - optional: true /tunnel@0.0.6: resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} @@ -15800,8 +16163,8 @@ packages: resolution: {integrity: sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==} dev: true - /ufo@1.1.2: - resolution: {integrity: sha512-TrY6DsjTQQgyS3E3dBaOXf0TpPD8u9FVrVYmKVegJuFw51n/YB9XPt+U6ydzFG5ZIN7+DIjPbNmXoBj9esYhgQ==} + /ufo@1.3.2: + resolution: {integrity: sha512-o+ORpgGwaYQXgqGDwd+hkS4PuZ3QnmqMMxRuajK/a38L6fTpcE5GPIfrf+L/KemFzfUpeUQc1rRS1iDBozvnFA==} dev: true /uglify-js@3.17.4: @@ -16037,6 +16400,10 @@ packages: unist-util-visit-parents: 6.0.1 dev: false + /universalify@0.2.0: + resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} + engines: {node: '>= 4.0.0'} + /universalify@1.0.0: resolution: {integrity: sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==} engines: {node: '>= 10.0.0'} @@ -16054,7 +16421,7 @@ packages: /unplugin@0.10.2: resolution: {integrity: sha512-6rk7GUa4ICYjae5PrAllvcDeuT8pA9+j5J5EkxbMFaV+SalHhxZ7X2dohMzu6C3XzsMT+6jwR/+pwPNR3uK9MA==} dependencies: - acorn: 8.10.0 + acorn: 8.11.2 chokidar: 3.5.3 webpack-sources: 3.2.3 webpack-virtual-modules: 0.4.6 @@ -16106,6 +16473,12 @@ packages: resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==} dev: true + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + /use-callback-ref@1.3.0(@types/react@18.2.37)(react@18.2.0): resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==} engines: {node: '>=10'} @@ -16312,14 +16685,13 @@ packages: vite: 5.0.10(@types/node@20.10.0) dev: false - /vite-node@0.34.6(@types/node@20.10.0): - resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==} - engines: {node: '>=v14.18.0'} + /vite-node@1.1.0(@types/node@20.10.0): + resolution: {integrity: sha512-jV48DDUxGLEBdHCQvxL1mEh7+naVy+nhUUUaPAZLd3FJgXuxQiewHcfeZebbJ6onDqNGkP4r3MhQ342PRlG81Q==} + engines: {node: ^18.0.0 || >=20.0.0} hasBin: true dependencies: cac: 6.7.14 debug: 4.3.4(supports-color@8.1.1) - mlly: 1.4.0 pathe: 1.1.1 picocolors: 1.0.0 vite: 5.0.4(@types/node@20.10.0) @@ -16415,34 +16787,34 @@ packages: fsevents: 2.3.3 dev: true - /vitest-fetch-mock@0.2.2(vitest@0.34.6): + /vitest-fetch-mock@0.2.2(vitest@1.1.0): resolution: {integrity: sha512-XmH6QgTSjCWrqXoPREIdbj40T7i1xnGmAsTAgfckoO75W1IEHKR8hcPCQ7SO16RsdW1t85oUm6pcQRLeBgjVYQ==} engines: {node: '>=14.14.0'} peerDependencies: vitest: '>=0.16.0' dependencies: cross-fetch: 3.1.8 - vitest: 0.34.6 + vitest: 1.1.0(@types/node@20.10.0)(jsdom@23.0.1) transitivePeerDependencies: - encoding dev: true - /vitest@0.34.6: - resolution: {integrity: sha512-+5CALsOvbNKnS+ZHMXtuUC7nL8/7F1F2DnHGjSsszX8zCjWSSviphCb/NuS9Nzf4Q03KyyDRBAXhF/8lffME4Q==} - engines: {node: '>=v14.18.0'} + /vitest@1.1.0(@types/node@20.10.0)(jsdom@23.0.1): + resolution: {integrity: sha512-oDFiCrw7dd3Jf06HoMtSRARivvyjHJaTxikFxuqJjO76U436PqlVw1uLn7a8OSPrhSfMGVaRakKpA2lePdw79A==} + engines: {node: ^18.0.0 || >=20.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' - '@vitest/browser': '*' - '@vitest/ui': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': ^1.0.0 + '@vitest/ui': ^1.0.0 happy-dom: '*' jsdom: '*' - playwright: '*' - safaridriver: '*' - webdriverio: '*' peerDependenciesMeta: '@edge-runtime/vm': optional: true + '@types/node': + optional: true '@vitest/browser': optional: true '@vitest/ui': @@ -16451,36 +16823,29 @@ packages: optional: true jsdom: optional: true - playwright: - optional: true - safaridriver: - optional: true - webdriverio: - optional: true dependencies: - '@types/chai': 4.3.5 - '@types/chai-subset': 1.3.3 '@types/node': 20.10.0 - '@vitest/expect': 0.34.6 - '@vitest/runner': 0.34.6 - '@vitest/snapshot': 0.34.6 - '@vitest/spy': 0.34.6 - '@vitest/utils': 0.34.6 - acorn: 8.10.0 - acorn-walk: 8.2.0 + '@vitest/expect': 1.1.0 + '@vitest/runner': 1.1.0 + '@vitest/snapshot': 1.1.0 + '@vitest/spy': 1.1.0 + '@vitest/utils': 1.1.0 + acorn-walk: 8.3.1 cac: 6.7.14 chai: 4.3.10 debug: 4.3.4(supports-color@8.1.1) - local-pkg: 0.4.3 - magic-string: 0.30.1 + execa: 8.0.1 + jsdom: 23.0.1 + local-pkg: 0.5.0 + magic-string: 0.30.5 pathe: 1.1.1 picocolors: 1.0.0 - std-env: 3.3.3 - strip-literal: 1.0.1 - tinybench: 2.5.0 - tinypool: 0.7.0 + std-env: 3.7.0 + strip-literal: 1.3.0 + tinybench: 2.5.1 + tinypool: 0.8.1 vite: 5.0.4(@types/node@20.10.0) - vite-node: 0.34.6(@types/node@20.10.0) + vite-node: 1.1.0(@types/node@20.10.0) why-is-node-running: 2.2.2 transitivePeerDependencies: - less @@ -16499,6 +16864,12 @@ packages: /w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + /w3c-xmlserializer@5.0.0: + resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} + engines: {node: '>=18'} + dependencies: + xml-name-validator: 5.0.0 + /walk-up-path@3.0.1: resolution: {integrity: sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==} dev: true @@ -16530,6 +16901,10 @@ packages: /webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + /webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + /webpack-sources@3.2.3: resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} engines: {node: '>=10.13.0'} @@ -16543,6 +16918,23 @@ packages: resolution: {integrity: sha512-poXpCylU7ExuvZK8z+On3kX+S8o/2dQ/SVYueKA0D4WEMXROXgY8Ez50/bQEUmvoSMMrWcrJqCHuhAbsiwg7Dg==} dev: true + /whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + dependencies: + iconv-lite: 0.6.3 + + /whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + + /whatwg-url@14.0.0: + resolution: {integrity: sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==} + engines: {node: '>=18'} + dependencies: + tr46: 5.0.0 + webidl-conversions: 7.0.0 + /whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} dependencies: @@ -16700,6 +17092,22 @@ packages: optional: true dev: true + /ws@8.15.1: + resolution: {integrity: sha512-W5OZiCjXEmk0yZ66ZN82beM5Sz7l7coYxpRkzS+p9PP+ToQry8szKh+61eNktr7EA9DOwvFGhfC605jDHbP6QQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + /xml-name-validator@5.0.0: + resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} + engines: {node: '>=18'} + /xml2js@0.5.0: resolution: {integrity: sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==} engines: {node: '>=4.0.0'} @@ -16713,6 +17121,9 @@ packages: engines: {node: '>=4.0'} dev: true + /xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -16728,7 +17139,6 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: true /yaml@2.3.4: resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} diff --git a/provider/docs/README.md b/provider/docs/README.md new file mode 100644 index 00000000..4920c616 --- /dev/null +++ b/provider/docs/README.md @@ -0,0 +1,64 @@ +¿# Docs context provider for OpenCodeGraph + +This is a context provider for [OpenCodeGraph](https://opencodegraph.org) that adds contextual documentation to your code from an existing documentation corpus. + +## Screenshot + +![Screenshot of OpenCodeGraph docs annotations]() + +_TODO(sqs)_ + +Visit the [OpenCodeGraph playground](https://opencodegraph.org/playground) for a live example. + +## Usage + +Add the following to your settings in any OpenCodeGraph client: + +```json +"opencodegraph.providers": { + // ...other providers... + "https://opencodegraph.org/npm/@opencodegraph/provider-docs": { + // TODO(sqs) + } +}, +``` + +TODO(sqs) + +See "[Configuration](#configuration)" for more. + +Tips: + +- If you're using VS Code, you can put the snippet above in `.vscode/settings.json` in the repository or workspace root to configure per-repository links. +- Play around with the docs provider in realtime on the [OpenCodeGraph playground](https://opencodegraph.org/playground). + +## Configuration + + + +```typescript +/** Settings for the docs OpenCodeGraph provider. */ +export interface Settings { + // TODO(sqs) +} +``` + +## Development + +- [Source code](https://sourcegraph.com/github.com/sourcegraph/opencodegraph/-/tree/provider/docs) +- [Docs](https://opencodegraph.org/docs/providers/docs) +- License: Apache 2.0 + +``` +time p run -s docs-query 'redirect' $(find ~/src/github.com/vikejs/vike/docs/pages -type f) +time p run -s docs-query 'making provider work in vscode' $(find ../../web/content/docs -type f) +``` + +TODOs: + +- simplify cache interface +- deal with different content types (markdown/html) differently +- make it slurp up gdocs/confluence/markdown in repos +- show OCG annotations (but in a way that doesn't overlay lines in the file, is more passive?) +- show a demo of Cody working with this +- show docs most relevant to the current visible portion or the selection, not the whole file diff --git a/provider/docs/bin/create-file-corpus.ts b/provider/docs/bin/create-file-corpus.ts new file mode 100644 index 00000000..9192f6a1 --- /dev/null +++ b/provider/docs/bin/create-file-corpus.ts @@ -0,0 +1,30 @@ +import { readFile } from 'fs/promises' +import path from 'path' +import { corpusData } from '../src/corpus/data' +import { type Doc } from '../src/corpus/doc/doc' + +const args = process.argv.slice(2) + +const corpusFiles = args + +const USAGE = `\nUsage: ${path.basename(process.argv[1])} ` +if (corpusFiles.length === 0) { + console.error('Error: no corpus files specified') + console.error(USAGE) + process.exit(1) +} + +const data = corpusData( + await Promise.all( + corpusFiles.map(async (file, i) => { + const data = await readFile(file, 'utf8') + return { + id: i + 1, + text: data, + } satisfies Doc + }) + ) +) + +console.error(`# ${data.docs.length} docs`) +console.log(JSON.stringify(data, null, 2)) diff --git a/provider/docs/bin/create-web-corpus.ts b/provider/docs/bin/create-web-corpus.ts new file mode 100644 index 00000000..35a72fee --- /dev/null +++ b/provider/docs/bin/create-web-corpus.ts @@ -0,0 +1,28 @@ +import path from 'path' +import { corpusData } from '../src/corpus/data' +import { createWebCorpusSource } from '../src/corpus/source/web/webCorpusSource' + +const args = process.argv.slice(2) + +const entryPage = args[0] +const prefix = args[1] +const ignore = args.slice(2) + +const USAGE = `\nUsage: ${path.basename(process.argv[1])} [ignore]` +if (!entryPage || !prefix || args.length < 2) { + console.error('Error: invalid arguments') + console.error(USAGE) + process.exit(1) +} + +const corpusSource = createWebCorpusSource({ + entryPage: new URL(entryPage), + prefix: new URL(prefix), + ignore, + logger: message => console.error('# ' + message), +}) + +const data = corpusData(await corpusSource.docs()) + +console.error(`# ${data.docs.length} docs`) +console.log(JSON.stringify(data, null, 2)) diff --git a/provider/docs/bin/docs-query.ts b/provider/docs/bin/docs-query.ts new file mode 100644 index 00000000..3b10f938 --- /dev/null +++ b/provider/docs/bin/docs-query.ts @@ -0,0 +1,66 @@ +import { readFile } from 'fs/promises' +import path from 'path' +import envPaths from 'env-paths' +import { indexCorpus } from '../src/corpus' +import { createFileSystemCorpusCache } from '../src/corpus/cache/fs' +import { type CorpusData } from '../src/corpus/data' +import { extractContentUsingMozillaReadability } from '../src/corpus/doc/contentExtractor' + +const args = process.argv.slice(2) + +const corpusDataFile = args[0] +const query = args[1] + +const USAGE = `\nUsage: ${path.basename(process.argv[1])} ` +if (!corpusDataFile) { + console.error('Error: no corpus data file specified (use create-file-corpus or create-web-corpus to create one)') + console.error(USAGE) + process.exit(1) +} +if (!query) { + console.error('Error: no query specified') + console.error(USAGE) + process.exit(1) +} +if (args.length !== 2) { + console.error('Error: invalid arguments') + console.error(USAGE) + process.exit(1) +} + +const corpusData = JSON.parse(await readFile(corpusDataFile, 'utf8')) as CorpusData + +const cacheDir = envPaths('opencodegraph-provider-docs').cache +const fsCache = createFileSystemCorpusCache(cacheDir) + +const corpus = await indexCorpus(corpusData, { + cache: fsCache, + contentExtractor: extractContentUsingMozillaReadability, +}) +const results = await corpus.search(query) +console.error(`# ${corpus.docs.length} docs in corpus`) +console.error(`# Query: ${JSON.stringify(query)}`) +const MAX_RESULTS = 5 +console.error(`# ${results.length} results${results.length > MAX_RESULTS ? ` (showing top ${MAX_RESULTS})` : ''}`) +for (const [i, result] of results.slice(0, MAX_RESULTS).entries()) { + const doc = corpusData.docs[result.doc - 1] + if (i !== 0) { + console.log() + } + console.log(`#${i + 1} [${result.score.toFixed(3)}] ${doc.url ?? `doc${doc.id}`}#chunk${result.chunk}`) + console.log(`${indent(truncate(result.excerpt.replaceAll('\n\n', '\n'), 500), '\t')}`) +} + +function truncate(text: string, maxLength: number): string { + if (text.length > maxLength) { + return text.slice(0, maxLength) + '...' + } + return text +} + +function indent(text: string, indent: string): string { + if (text === '') { + return '' + } + return indent + text.replaceAll('\n', '\n' + indent) +} diff --git a/provider/docs/package.json b/provider/docs/package.json new file mode 100644 index 00000000..2c2be49b --- /dev/null +++ b/provider/docs/package.json @@ -0,0 +1,41 @@ +{ + "name": "@opencodegraph/provider-docs", + "version": "0.0.1", + "description": "Add contextual documentation to your code from an existing documentation corpus (OpenCodeGraph provider)", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/sourcegraph/opencodegraph", + "directory": "provider/docs" + }, + "type": "module", + "main": "dist/src/provider/provider.js", + "types": "dist/src/provider/provider.d.ts", + "files": [ + "dist", + "!**/*.test.*", + "README.md" + ], + "sideEffects": false, + "scripts": { + "build": "tsc --build", + "test": "vitest", + "docs-query": "node --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only bin/docs-query.ts", + "create-file-corpus": "node --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only bin/create-file-corpus.ts", + "create-web-corpus": "node --no-warnings=ExperimentalWarning --experimental-specifier-resolution=node --loader ts-node/esm/transpile-only bin/create-web-corpus.ts" + }, + "dependencies": { + "@mozilla/readability": "^0.5.0", + "@opencodegraph/provider": "workspace:*", + "@xenova/transformers": "^2.12.1", + "better-localstorage": "^1.0.5", + "buffer": "^6.0.3", + "env-paths": "^3.0.0", + "jsdom": "^23.0.1", + "onnxruntime-web": "*" + }, + "devDependencies": { + "@types/jsdom": "^21.1.6", + "vitest-fetch-mock": "^0.2.2" + } +} diff --git a/provider/docs/src/corpus/cache/cache.test.ts b/provider/docs/src/corpus/cache/cache.test.ts new file mode 100644 index 00000000..f426bd43 --- /dev/null +++ b/provider/docs/src/corpus/cache/cache.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'vitest' +import { contentID } from './cache' + +describe('contentID', () => { + test('returns the content ID', async () => { + expect(await contentID('abc')).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') + }) +}) diff --git a/provider/docs/src/corpus/cache/cache.ts b/provider/docs/src/corpus/cache/cache.ts new file mode 100644 index 00000000..3936849a --- /dev/null +++ b/provider/docs/src/corpus/cache/cache.ts @@ -0,0 +1,71 @@ +import { type webcrypto } from 'node:crypto' + +/** + * A unique identifier for a document's or chunk's content (based on a hash of the text). + */ +export type ContentID = string + +export async function contentID(text: string): Promise { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + const crypto: webcrypto.Crypto = (globalThis as any).crypto || (await import('node:crypto')).default.webcrypto + + return Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)))) + .map(b => b.toString(16).padStart(2, '0')) + .join('') +} + +export interface CorpusCache { + get(contentID: ContentID, key: string): Promise + set(contentID: ContentID, key: string, value: T): Promise +} + +export function scopedCache(cache: CorpusCache, scope: string): CorpusCache { + function scopedKey(key: string): string { + return `${scope}-${key}` + } + return { + get: async (contentID, key) => cache.get(contentID, scopedKey(key)) as Promise, + set: async (contentID, key, value) => cache.set(contentID, scopedKey(key), value), + } +} + +/** + * Memoize an operation using the cache. + * + * The alternate form with `toJSONValue` and `fromJSONValue` is for when the value is not directly + * JSON-serializable, such as `Float32Array`. + */ +export async function memo(cache: CorpusCache, text: string, key: string, fn: () => Promise): Promise +export async function memo( + cache: CorpusCache, + text: string, + key: string, + fn: () => Promise, + toJSONValue: (value: T) => any, + fromJSONValue: (jsonValue: any) => T +): Promise +export async function memo( + cache: CorpusCache, + text: string, + key: string, + fn: () => Promise, + toJSONValue?: (value: T) => any, + fromJSONValue?: (jsonValue: any) => T +): Promise { + const cid = await contentID(text) + const memoized = await (cache as CorpusCache).get(cid, key) + if (memoized !== null) { + // console.log('HIT') + return fromJSONValue ? fromJSONValue(memoized) : memoized + } + // console.log('MISS') + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const result = await fn() + await (cache as CorpusCache).set(cid, key, toJSONValue ? toJSONValue(result) : result) + return result +} + +export const noopCache: CorpusCache = { + get: async () => Promise.resolve(null), + set: async () => {}, +} diff --git a/provider/docs/src/corpus/cache/fs.ts b/provider/docs/src/corpus/cache/fs.ts new file mode 100644 index 00000000..5e33d18a --- /dev/null +++ b/provider/docs/src/corpus/cache/fs.ts @@ -0,0 +1,32 @@ +import { mkdir, readFile, writeFile } from 'fs/promises' +import path from 'path' +import { type ContentID, type CorpusCache } from './cache' + +/** + * Create a {@link CorpusCache} that stores cache data in the file system. + */ +export function createFileSystemCorpusCache(basePath: string): CorpusCache { + function cacheFilePath(contentID: ContentID, key: string): string { + return path.join(basePath, `${contentID}-${key.replaceAll('/', '_')}.json`) + } + + return { + async get(contentID, key) { + try { + const data = await readFile(cacheFilePath(contentID, key), 'utf8') + return JSON.parse(data) + } catch (error: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + if ('code' in error && error.code === 'ENOENT') { + return null + } + throw error + } + }, + async set(contentID, key, value) { + const filePath = cacheFilePath(contentID, key) + await mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }) + await writeFile(filePath, JSON.stringify(value, null, 2)) + }, + } +} diff --git a/provider/docs/src/corpus/cache/indexedDB.ts b/provider/docs/src/corpus/cache/indexedDB.ts new file mode 100644 index 00000000..d9e297b6 --- /dev/null +++ b/provider/docs/src/corpus/cache/indexedDB.ts @@ -0,0 +1,36 @@ +/// + +import IndexedDBStorage from 'better-localstorage' +import { type ContentID, type CorpusCache } from './cache' + +/** + * Create a {@link CorpusCache} that stores cache data using IndexedDB. + */ +export function createIndexedDBCorpusCache(keyPrefix: string): CorpusCache { + const storage = new IndexedDBStorage(keyPrefix) + + function storageKey(contentID: ContentID, key: string): string { + return `${keyPrefix}:${contentID}:${key}` + } + + return { + async get(contentID, key) { + const k = storageKey(contentID, key) + const data = await storage.getItem(k) + try { + return data ?? null + } catch (error) { + // TODO(sqs): cast because https://github.com/dreamsavior/Better-localStorage/pull/1 + await (storage as any).delete(k) + throw error + } + }, + async set(contentID, key, value) { + try { + await storage.setItem(storageKey(contentID, key), value) + } catch (error) { + console.error(`failed to store data for ${contentID}:${key}`, error) + } + }, + } +} diff --git a/provider/docs/src/corpus/cache/localStorage.ts b/provider/docs/src/corpus/cache/localStorage.ts new file mode 100644 index 00000000..0eb22dc9 --- /dev/null +++ b/provider/docs/src/corpus/cache/localStorage.ts @@ -0,0 +1,34 @@ +/// + +import { type ContentID, type CorpusCache } from './cache' + +/** + * Create a {@link CorpusCache} that stores cache data in localStorage (using the Web Storage API). + */ +export function createWebStorageCorpusCache(storage: Storage, keyPrefix: string): CorpusCache { + function storageKey(contentID: ContentID, key: string): string { + return `${keyPrefix}:${contentID}:${key}` + } + + return { + get(contentID, key) { + const k = storageKey(contentID, key) + const data = storage.getItem(k) + try { + return Promise.resolve(data === null ? null : JSON.parse(data)) + } catch (error) { + storage.removeItem(k) + throw error + } + }, + set(contentID, key, value) { + const valueData = JSON.stringify(value) + try { + storage.setItem(storageKey(contentID, key), valueData) + } catch { + // console.error(`failed to store data for ${contentID}:${key}`, error) + } + return Promise.resolve() + }, + } +} diff --git a/provider/docs/src/corpus/data.ts b/provider/docs/src/corpus/data.ts new file mode 100644 index 00000000..b973294e --- /dev/null +++ b/provider/docs/src/corpus/data.ts @@ -0,0 +1,17 @@ +import { type Doc } from './doc/doc' + +export function corpusData(docs: Doc[]): CorpusData { + const seenIDs = new Set() + for (const doc of docs) { + if (seenIDs.has(doc.id)) { + throw new Error(`duplicate doc ID: ${doc.id}`) + } + seenIDs.add(doc.id) + } + + return { docs } +} + +export interface CorpusData { + docs: Doc[] +} diff --git a/provider/docs/src/corpus/doc/chunks.test.ts b/provider/docs/src/corpus/doc/chunks.test.ts new file mode 100644 index 00000000..acf1a2c0 --- /dev/null +++ b/provider/docs/src/corpus/doc/chunks.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, test } from 'vitest' +import { chunk, type Chunk } from './chunks' + +describe('chunker', () => { + test('empty', () => expect(chunk('', {})).toEqual([])) + + test('fallback', () => expect(chunk('a', {})).toEqual([{ range: { start: 0, end: 1 }, text: 'a' }])) + + describe('Markdown', () => { + test('by section', () => + expect( + chunk( + ` +# Title + +Intro + +## Section 1 + +Body 1 + +## Section 2 + +Body 2 +`.trim(), + { isMarkdown: true } + ) + ).toEqual([ + { + range: { + start: 2, + end: 16, + }, + text: 'Title\n\nIntro', + }, + { + range: { + start: 5, + end: 24, + }, + text: 'Section 1\n\nBody 1', + }, + { + range: { + start: 8, + end: 25, + }, + text: 'Section 2\n\nBody 2', + }, + ])) + }) +}) diff --git a/provider/docs/src/corpus/doc/chunks.ts b/provider/docs/src/corpus/doc/chunks.ts new file mode 100644 index 00000000..3682915f --- /dev/null +++ b/provider/docs/src/corpus/doc/chunks.ts @@ -0,0 +1,60 @@ +/** + * Index of a {@link Chunk} in a {@link StoredDocument}. + */ +export type ChunkIndex = number + +export interface Chunk { + /** + * The text of the chunk, stripped of semantically meaningless markup, punctuation, and content. + * This text need not be present in the original document. + */ + text: string + + /** + * The range in the original document (as character offsets) represented by this chunk. + */ + range: { start: number; end: number } +} + +/** + * Information about the document to help the chunker know how to split the content into logical + * chunks. + */ +export interface ChunkerHints { + isMarkdown?: boolean +} + +/** + * Split text into logical chunks (such as sections in a Markdown document). + */ +export function chunk(text: string, hints: ChunkerHints): Chunk[] { + if (hints.isMarkdown) { + return chunkMarkdown(text) + } + if (text.length === 0) { + return [] + } + return [{ text, range: { start: 0, end: text.length } }] +} + +function chunkMarkdown(text: string): Chunk[] { + const chunks: Chunk[] = [] + + const sections = text.split(/^(#+\s*)/m) + let pos = 0 + for (const section of sections) { + if (section.length === 0) { + continue + } + if (section.startsWith('#')) { + pos += section.length + continue + } + chunks.push({ + text: section.trim(), + range: { start: pos, end: pos + section.length }, + }) + } + + return chunks +} diff --git a/provider/docs/src/corpus/doc/contentExtractor.test.ts b/provider/docs/src/corpus/doc/contentExtractor.test.ts new file mode 100644 index 00000000..08bfcdf1 --- /dev/null +++ b/provider/docs/src/corpus/doc/contentExtractor.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'vitest' +import { extractContentUsingMozillaReadability, type Content } from './contentExtractor' + +describe('extractContentUsingMozillaReadability', () => { + test('extracts content', async () => + expect( + await extractContentUsingMozillaReadability.extractContent({ + id: 1, + text: 'Bar - MySite

Bar

\n

Baz

', + }) + ).toEqual({ + title: 'Bar - MySite', + content: '

Bar

\n

Baz

', + textContent: 'Bar\nBaz', + })) +}) diff --git a/provider/docs/src/corpus/doc/contentExtractor.ts b/provider/docs/src/corpus/doc/contentExtractor.ts new file mode 100644 index 00000000..1b439099 --- /dev/null +++ b/provider/docs/src/corpus/doc/contentExtractor.ts @@ -0,0 +1,43 @@ +import { Readability } from '@mozilla/readability' +import { parseDOM } from '../../dom' +import { type Doc } from './doc' + +export interface Content { + /** + * Title of the document. + */ + title: string + + /** + * Content of the document, including some markup. Omits non-content-related elements (header, + * footer, navigation, etc.). + */ + content: string + + /** + * Text content of the document, with all markup removed. Omits all non-content-related + * elements. + */ + textContent: string +} + +export interface ContentExtractor { + id: string + extractContent(doc: Doc): Promise +} + +export const extractContentUsingMozillaReadability: ContentExtractor = { + id: 'mozillaReadability', + async extractContent(doc) { + const info = new Readability(await parseDOM(doc.text, doc.url), { + charThreshold: 500, + }).parse() + return info + ? { + title: info.title, + content: info.content, + textContent: info.textContent, + } + : null + }, +} diff --git a/provider/docs/src/corpus/doc/doc.ts b/provider/docs/src/corpus/doc/doc.ts new file mode 100644 index 00000000..a4a462a3 --- /dev/null +++ b/provider/docs/src/corpus/doc/doc.ts @@ -0,0 +1,14 @@ +/** + * A unique identifier for a document in a corpus. + */ +export type DocID = number + +/** + * A raw document in a corpus. + */ +export interface Doc { + id: DocID + text: string + + url?: string +} diff --git a/provider/docs/src/corpus/index.test.ts b/provider/docs/src/corpus/index.test.ts new file mode 100644 index 00000000..f68bdb39 --- /dev/null +++ b/provider/docs/src/corpus/index.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from 'vitest' +import { indexCorpus } from '.' +import { corpusData } from './data' +import { type Doc, type DocID } from './doc/doc' + +export function doc(id: DocID, text: string): Doc { + return { id, text } +} + +describe('indexCorpus', () => { + test('#docs', async () => { + expect((await indexCorpus(corpusData([doc(1, 'a'), doc(2, 'b')]))).docs.length).toBe(2) + }) +}) diff --git a/provider/docs/src/corpus/index.ts b/provider/docs/src/corpus/index.ts new file mode 100644 index 00000000..411797ee --- /dev/null +++ b/provider/docs/src/corpus/index.ts @@ -0,0 +1,91 @@ +import { memo, noopCache, type CorpusCache } from './cache/cache' +import { type CorpusData } from './data' +import { chunk, type Chunk, type ChunkIndex } from './doc/chunks' +import { type Content, type ContentExtractor } from './doc/contentExtractor' +import { type Doc, type DocID } from './doc/doc' +import { multiSearch } from './search/multi' + +/** + * An index of a corpus. + */ +export interface CorpusIndex { + data: CorpusData + + docs: IndexedDoc[] + + doc(id: DocID): IndexedDoc + search(query: string): Promise +} + +/** + * An indexed document. + */ +export interface IndexedDoc { + doc: Doc + content: Content | null + chunks: Chunk[] +} + +/** + * A search result from searching a corpus. + */ + +export interface CorpusSearchResult { + doc: DocID + chunk: ChunkIndex + score: number + excerpt: string +} + +/** + * Options for indexing a corpus. + */ +export interface IndexOptions { + cache?: CorpusCache + contentExtractor?: ContentExtractor +} + +/** + * Index a corpus. + */ +export async function indexCorpus( + data: CorpusData, + { cache = noopCache, contentExtractor }: IndexOptions = { cache: noopCache } +): Promise { + const indexedDocs: IndexedDoc[] = [] + + for (const doc of data.docs) { + const content = await cachedExtractContent(cache, contentExtractor, doc) + + const chunks = chunk(content?.content ?? doc.text, { isMarkdown: doc.text.includes('##') }) + + indexedDocs.push({ doc, content, chunks }) + } + + const index: CorpusIndex = { + data, + docs: indexedDocs, + doc(id) { + const doc = indexedDocs.find(d => d.doc.id === id) + if (!doc) { + throw new Error(`no document with id ${id} in corpus`) + } + return doc + }, + search(query) { + return multiSearch(index, query, cache) + }, + } + return index +} + +function cachedExtractContent( + cache: CorpusCache, + extractor: ContentExtractor | undefined, + doc: Doc +): Promise { + if (!extractor) { + return Promise.resolve(null) + } + return memo(cache, `${doc.url}:${doc.text}`, `extractContent:${extractor.id}`, () => extractor.extractContent(doc)) +} diff --git a/provider/docs/src/corpus/search/embeddings.test.ts b/provider/docs/src/corpus/search/embeddings.test.ts new file mode 100644 index 00000000..36198fd9 --- /dev/null +++ b/provider/docs/src/corpus/search/embeddings.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'vitest' +import { indexCorpus, type CorpusSearchResult } from '..' +import { corpusData } from '../data' +import { doc } from '../index.test' +import { embeddingsSearch, embedTextInThisScope, similarity } from './embeddings' + +describe('embeddingsSearch', () => { + test('finds matches', async () => { + expect(await embeddingsSearch(await indexCorpus(corpusData([doc(1, 'a'), doc(2, 'b')])), 'b')).toEqual< + CorpusSearchResult[] + >([{ doc: 2, chunk: 0, score: 1, excerpt: 'b' }]) + }) +}) + +describe('embedText', () => { + test('embeds', async () => { + const s = await embedTextInThisScope('hello world') + expect(s).toBeInstanceOf(Float32Array) + }) +}) + +describe('similarity', () => { + test('works', async () => { + expect(await similarity('what is the current time', 'what time is it')).toBeCloseTo(0.7217, 4) + expect(await similarity('hello world', 'seafood')).toBeCloseTo(0.2025, 4) + }) +}) diff --git a/provider/docs/src/corpus/search/embeddings.ts b/provider/docs/src/corpus/search/embeddings.ts new file mode 100644 index 00000000..c9e5a968 --- /dev/null +++ b/provider/docs/src/corpus/search/embeddings.ts @@ -0,0 +1,95 @@ +import { cos_sim, env, pipeline } from '@xenova/transformers' +import * as onnxWeb from 'onnxruntime-web' +import { type CorpusIndex, type CorpusSearchResult } from '..' +import { useWebWorker } from '../../env' +import { embedTextOnWorker } from '../../mlWorker/webWorkerClient' +import { memo, noopCache, type CorpusCache } from '../cache/cache' + +// TODO(sqs): think we can remove this entirely... +// +// eslint-disable-next-line @typescript-eslint/prefer-optional-chain +if (typeof process !== 'undefined' && process.env.FORCE_WASM) { + // Force use of wasm backend for parity between Node.js and web. + // + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + env.onnx = onnxWeb.env + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + ;(env as any).onnx.wasm.numThreads = 1 +} + +env.allowLocalModels = false + +export async function embeddingsSearch( + index: CorpusIndex, + query: string, + cache: CorpusCache = noopCache +): Promise { + const queryVec = await embedText(query) + + // Compute embeddings in parallel. + const results: CorpusSearchResult[] = await Promise.all( + index.docs.flatMap(({ doc, chunks }) => + chunks.map(async (chunk, i) => { + const chunkVec = await cachedEmbedText(chunk.text, cache) + const score = cos_sim(queryVec, chunkVec) + return { doc: doc.id, chunk: i, score, excerpt: chunk.text } satisfies CorpusSearchResult + }) + ) + ) + + results.sort((a, b) => b.score - a.score) + + return results.slice(0, 1) +} + +function cachedEmbedText(text: string, cache: CorpusCache): Promise { + return memo( + cache, + text, + 'embedText', + () => embedText(text), + f32a => Array.from(f32a), + arr => new Float32Array(arr) + ) +} + +const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {}) + +/** + * Embed the text and return the vector. Run in a worker in some environments. + */ +export const embedText = useWebWorker ? embedTextOnWorker : embedTextInThisScope + +/** + * Embed the text and return the vector. + * + * Run in the current scope (instead of in a worker in some environments). + */ +export async function embedTextInThisScope(text: string): Promise { + try { + console.time('embed') + const out = await pipe(text, { pooling: 'mean', normalize: true }) + console.timeEnd('embed') + return out.data + } catch (error) { + console.log(error) + throw error + } +} + +/** + * Compute the cosine similarity of the two texts' embeddings vectors. + */ +export async function similarity(text1: string, text2: string): Promise { + const emb1 = await embedTextInThisScope(text1) + const emb2 = await embedTextInThisScope(text2) + return cos_sim(emb1, emb2) +} + +declare module '@xenova/transformers' { + // The cos_sim function is declared in the @xenova/transformers module as only accepting + // number[], but it accepts Float32Array as well. + export function cos_sim(a: Float32Array, b: Float32Array): number +} diff --git a/provider/docs/src/corpus/search/keyword.test.ts b/provider/docs/src/corpus/search/keyword.test.ts new file mode 100644 index 00000000..977f5a9e --- /dev/null +++ b/provider/docs/src/corpus/search/keyword.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, test } from 'vitest' +import { indexCorpus, type CorpusSearchResult } from '..' +import { corpusData } from '../data' +import { doc } from '../index.test' +import { keywordSearch } from './keyword' +import { calculateTFIDF } from './tfidf' + +describe('keywordSearch', () => { + test('finds matches', async () => { + expect(keywordSearch(await indexCorpus(corpusData([doc(1, 'aaa'), doc(2, 'bbb')])), 'bbb')).toEqual< + CorpusSearchResult[] + >([ + { + doc: 2, + chunk: 0, + score: calculateTFIDF({ + termOccurrencesInChunk: 1, + chunkTermLength: 1, + totalChunks: 2, + termChunkFrequency: 1, + }), + excerpt: 'bbb', + }, + ]) + }) +}) diff --git a/provider/docs/src/corpus/search/keyword.ts b/provider/docs/src/corpus/search/keyword.ts new file mode 100644 index 00000000..2e015547 --- /dev/null +++ b/provider/docs/src/corpus/search/keyword.ts @@ -0,0 +1,19 @@ +import { type CorpusIndex, type CorpusSearchResult } from '..' +import { terms } from './terms' +import { createIndexForTFIDF } from './tfidf' + +export function keywordSearch(index: CorpusIndex, query: string): CorpusSearchResult[] { + const queryTerms = terms(query).filter(term => term.length >= 3) + const tfidf = createIndexForTFIDF(index.docs) + + const results: CorpusSearchResult[] = [] + for (const { doc, chunks } of index.docs) { + for (const [i, chunk] of chunks.entries()) { + const score = queryTerms.reduce((score, term) => score + tfidf(term, doc.id, i), 0) + if (score > 0) { + results.push({ doc: doc.id, chunk: i, score, excerpt: chunk.text }) + } + } + } + return results +} diff --git a/provider/docs/src/corpus/search/multi.ts b/provider/docs/src/corpus/search/multi.ts new file mode 100644 index 00000000..2de0446f --- /dev/null +++ b/provider/docs/src/corpus/search/multi.ts @@ -0,0 +1,47 @@ +import { type CorpusIndex, type CorpusSearchResult } from '..' +import { scopedCache, type CorpusCache } from '../cache/cache' +import { type ChunkIndex } from '../doc/chunks' +import { type DocID } from '../doc/doc' +import { embeddingsSearch } from './embeddings' +import { keywordSearch } from './keyword' + +/** + * Search using multiple search methods. + */ +export async function multiSearch( + index: CorpusIndex, + query: string, + cache: CorpusCache +): Promise { + const allResults = ( + await Promise.all( + Object.entries(SEARCH_METHODS).map(([name, searchFn]) => searchFn(index, query, scopedCache(cache, name))) + ) + ).flat() + + // Sum scores for each chunk. + const combinedResults = new Map>() + for (const result of allResults) { + let docResults = combinedResults.get(result.doc) + if (!docResults) { + docResults = new Map() + combinedResults.set(result.doc, docResults) + } + + const chunkResult = docResults.get(result.chunk) ?? { + doc: result.doc, + chunk: result.chunk, + score: 0, + excerpt: result.excerpt, + } + docResults.set(result.chunk, { ...chunkResult, score: chunkResult.score + result.score }) + } + + const results = Array.from(combinedResults.values()).flatMap(docResults => Array.from(docResults.values())) + return results.toSorted((a, b) => b.score - a.score) +} + +const SEARCH_METHODS: Record< + string, + (index: CorpusIndex, query: string, cache: CorpusCache) => CorpusSearchResult[] | Promise +> = { keywordSearch, embeddingsSearch } diff --git a/provider/docs/src/corpus/search/terms.test.ts b/provider/docs/src/corpus/search/terms.test.ts new file mode 100644 index 00000000..8c561b6e --- /dev/null +++ b/provider/docs/src/corpus/search/terms.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, test } from 'vitest' +import { terms } from './terms' + +describe('terms', () => { + test('splits, stems, normalizes', () => { + expect(terms('my apples are cooler when stored')).toEqual(['my', 'apple', 'are', 'cool', 'when', 'stor']) + }) +}) diff --git a/provider/docs/src/corpus/search/terms.ts b/provider/docs/src/corpus/search/terms.ts new file mode 100644 index 00000000..014aab36 --- /dev/null +++ b/provider/docs/src/corpus/search/terms.ts @@ -0,0 +1,14 @@ +export type Term = string + +/** + * All terms in the text, with normalization and stemming applied. + */ +export function terms(text: string): Term[] { + return ( + text + .toLowerCase() + .split(/[^\w-]+/) + // TODO(sqs): get a real stemmer + .map(term => term.replace(/(.*)(?:es|ed|ing|s|er)$/, '$1')) + ) +} diff --git a/provider/docs/src/corpus/search/tfidf.test.ts b/provider/docs/src/corpus/search/tfidf.test.ts new file mode 100644 index 00000000..3e461e2b --- /dev/null +++ b/provider/docs/src/corpus/search/tfidf.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from 'vitest' +import { indexCorpus } from '..' +import { corpusData } from '../data' +import { calculateTFIDF, createIndexForTFIDF } from './tfidf' + +describe('createIndexForTFIDF', async () => { + const data = corpusData([ + { id: 1, text: 'a b c c c' }, + { id: 2, text: 'b c d' }, + { id: 3, text: 'c d e' }, + ]) + const docIDs = data.docs.map(({ id }) => id) + const index = await indexCorpus(data) + const tfidf = createIndexForTFIDF(index.docs) + + test('term in 1 doc', () => { + expect(docIDs.map(docID => tfidf('a', docID, 0))).toEqual([ + calculateTFIDF({ termOccurrencesInChunk: 1, chunkTermLength: 5, totalChunks: 3, termChunkFrequency: 1 }), + 0, + 0, + ]) + }) + + test('term in all docs', () => { + expect(docIDs.map(docID => tfidf('c', docID, 0))).toEqual([ + calculateTFIDF({ termOccurrencesInChunk: 3, chunkTermLength: 5, totalChunks: 3, termChunkFrequency: 3 }), + calculateTFIDF({ termOccurrencesInChunk: 1, chunkTermLength: 3, totalChunks: 3, termChunkFrequency: 3 }), + calculateTFIDF({ termOccurrencesInChunk: 1, chunkTermLength: 3, totalChunks: 3, termChunkFrequency: 3 }), + ]) + }) + + test('unknown term', () => { + expect(docIDs.map(docID => tfidf('x', docID, 0))).toEqual([0, 0, 0]) + }) +}) diff --git a/provider/docs/src/corpus/search/tfidf.ts b/provider/docs/src/corpus/search/tfidf.ts new file mode 100644 index 00000000..904b56a5 --- /dev/null +++ b/provider/docs/src/corpus/search/tfidf.ts @@ -0,0 +1,120 @@ +import { type IndexedDoc } from '..' +import { type ChunkIndex } from '../doc/chunks' +import { type DocID } from '../doc/doc' +import { terms, type Term } from './terms' + +/** + * TF-IDF is a way of measuring the relevance of a term to a document in a corpus. See + * https://en.wikipedia.org/wiki/Tf%E2%80%93idf. + * + * TF-IDF = TF * IDF + * - TF = number of occurrences of term in the chunk / number of (non-unique) terms in the chunk + * - IDF = log(number of chunks / number of chunks containing the term) + */ +export type TFIDF = (term: Term, doc: DocID, chunk: ChunkIndex) => number + +/** + * Index the corpus for fast computation of TF-IDF. @see {TFIDF} + */ +export function createIndexForTFIDF(docs: IndexedDoc[]): TFIDF { + /** + * Document -> chunk index -> term -> number of occurrences of term in the chunk. + * + * "TF" in "TF-IDF" (with chunks instead of documents as the unit of analysis). + */ + const termFrequency = new Map[]>() + + /** + * Document -> chunk index -> number of (non-unique) terms in the chunk. + */ + const termLength = new Map() + + /** + * Term -> number of chunks containing the term. + * + * "DF" in "IDF" in "TF-IDF" (with chunks instead of documents as the unit of analysis). + */ + const chunkFrequency = new Map() + + let totalChunks = 0 + + for (const { doc, chunks } of docs) { + const docTermFrequency: Map[] = new Array>(chunks.length) + termFrequency.set(doc.id, docTermFrequency) + + const docTermLength: number[] = new Array(chunks.length) + termLength.set(doc.id, docTermLength) + + for (const [i, chunk] of chunks.entries()) { + const chunkTerms = terms(chunk.text) + + // Set chunk frequencies. + for (const uniqueTerm of new Set(chunkTerms).values()) { + chunkFrequency.set(uniqueTerm, (chunkFrequency.get(uniqueTerm) ?? 0) + 1) + } + + // Set term frequencies. + const chunkTermFrequency = new Map() + docTermFrequency[i] = chunkTermFrequency + for (const term of chunkTerms) { + chunkTermFrequency.set(term, (chunkTermFrequency.get(term) ?? 0) + 1) + } + + // Set term cardinality. + docTermLength[i] = chunkTerms.length + + // Increment total chunks. + totalChunks++ + } + } + + return (termRaw: string, doc: DocID, chunk: ChunkIndex): number => { + const processedTerms = terms(termRaw) + if (processedTerms.length !== 1) { + throw new Error(`term ${JSON.stringify(termRaw)} is not a single term`) + } + const term = processedTerms[0] + + const docTermLength = termLength.get(doc) + if (!docTermLength) { + throw new Error(`doc ${doc} not found in termLength`) + } + if (typeof docTermLength[chunk] !== 'number') { + throw new TypeError(`chunk ${chunk} not found in termLength for doc ${doc}`) + } + + const docTermFrequency = termFrequency.get(doc) + if (!docTermFrequency) { + throw new Error(`doc ${doc} not found in termFrequency`) + } + if (!(docTermFrequency[chunk] instanceof Map)) { + throw new TypeError(`chunk ${chunk} not found in termFrequency for doc ${doc}`) + } + + return calculateTFIDF({ + termOccurrencesInChunk: docTermFrequency[chunk].get(term) ?? 0, + chunkTermLength: docTermLength[chunk], + totalChunks, + termChunkFrequency: chunkFrequency.get(term) ?? 0, + }) + } +} + +/** + * Calculate TF-IDF given the formula inputs. @see {TFIDF} + * + * Use {@link createIndexForTFIDF} instead of calling this directly. + */ +export function calculateTFIDF({ + termOccurrencesInChunk, + chunkTermLength, + totalChunks, + termChunkFrequency, +}: { + termOccurrencesInChunk: number + chunkTermLength: number + totalChunks: number + termChunkFrequency: number +}): number { + return (termOccurrencesInChunk / chunkTermLength) * Math.log((1 + totalChunks) / (1 + termChunkFrequency)) +} diff --git a/provider/docs/src/corpus/source/source.ts b/provider/docs/src/corpus/source/source.ts new file mode 100644 index 00000000..2a9b5073 --- /dev/null +++ b/provider/docs/src/corpus/source/source.ts @@ -0,0 +1,26 @@ +import { type CorpusData } from '../data' +import { type Doc } from '../doc/doc' + +export interface CorpusSource { + docs(): Promise +} + +export function corpusDataSource(data: CorpusData | Promise): CorpusSource { + return { + docs: async () => (await data).docs, + } +} + +export function corpusDataURLSource(url: string): CorpusSource { + return corpusDataSource( + fetch(url).then(resp => { + if (!resp.ok) { + throw new Error(`failed to fetch corpus data from ${url}: ${resp.status} ${resp.statusText}`) + } + if (!resp.headers.get('Content-Type')?.includes('json')) { + throw new Error(`corpus data from ${url} is not JSON`) + } + return resp.json() + }) + ) +} diff --git a/provider/docs/src/corpus/source/web/crawlQueue.ts b/provider/docs/src/corpus/source/web/crawlQueue.ts new file mode 100644 index 00000000..edced5c9 --- /dev/null +++ b/provider/docs/src/corpus/source/web/crawlQueue.ts @@ -0,0 +1,45 @@ +export interface CrawlQueue { + nextURL: () => URL | undefined + enqueueURL: (url: URL) => void + shouldCrawlURL: (url: URL) => boolean +} + +export function createCrawlQueue(isURLInScope: (url: URL) => boolean): CrawlQueue { + const seen = new Set() + const queue: URL[] = [] + + /** Normalize the URL to ignore the query string and hash. */ + function normalizeURLForSeen(url: URL): string { + return url.origin + url.pathname + } + + function seenURL(url: URL): boolean { + return seen.has(normalizeURLForSeen(url)) + } + function addToSeen(url: URL): void { + seen.add(normalizeURLForSeen(url)) + } + + const crawlQueue: CrawlQueue = { + nextURL() { + let url: URL | undefined + while ((url = queue.shift())) { + if (!seenURL(url)) { + addToSeen(url) + return url + } + } + return undefined + }, + enqueueURL(url) { + if (crawlQueue.shouldCrawlURL(url)) { + queue.push(url) + } + }, + shouldCrawlURL(url) { + return !seenURL(url) && isURLInScope(url) + }, + } + + return crawlQueue +} diff --git a/provider/docs/src/corpus/source/web/webCorpusSource.test.ts b/provider/docs/src/corpus/source/web/webCorpusSource.test.ts new file mode 100644 index 00000000..d13c6ea2 --- /dev/null +++ b/provider/docs/src/corpus/source/web/webCorpusSource.test.ts @@ -0,0 +1,112 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from 'vitest' +import createFetchMock from 'vitest-fetch-mock' +import { type Doc } from '../../doc/doc' +import { createWebCorpusSource, urlHasPrefix } from './webCorpusSource' + +describe('createWebCorpusSource', () => { + const fetchMocker = createFetchMock(vi) + beforeAll(() => fetchMocker.enableMocks()) + afterEach(() => fetchMocker.resetMocks()) + afterAll(() => fetchMocker.disableMocks()) + + test('crawls', async () => { + type MockResponseInit = Parameters[1] & { body: string } + const mockPages: { [pathname: string]: MockResponseInit } = { + '/docs': { + url: 'https://example.com/docs/entry', // redirected + body: 'Redirected', + }, + '/docs/entry': { + body: ` +

Docs

+

See foo and bar.

+

See also x. +Docs home +`, + }, + '/docs/foo': { + body: ` +

Foo

+

See foo/a.

+`, + }, + '/docs/foo/a': { + body: ` +

Foo/a

+

See foo and docs.

+`, + }, + '/docs/bar': { + body: ` +

Bar

+

See bar/a.

+`, + }, + '/docs/bar/a': { + status: 404, + body: 'Not Found', + }, + } + fetchMocker.mockResponse(req => { + const url = new URL(req.url) + if (url.protocol !== 'https:' || url.host !== 'example.com') { + throw new Error(`not mocked: ${req.url}`) + } + const resp = mockPages[url.pathname] + if (resp) { + return { url: url.toString(), ...resp } + } + throw new Error(`not mocked: ${req.url}`) + }) + + const source = createWebCorpusSource({ + entryPage: new URL('https://example.com/docs/entry'), + prefix: new URL('https://example.com/docs'), + }) + expect(await source.docs()).toEqual([ + { id: 1, text: mockPages['/docs/entry'].body, url: 'https://example.com/docs/entry' }, + { id: 2, text: mockPages['/docs/foo'].body, url: 'https://example.com/docs/foo' }, + { id: 3, text: mockPages['/docs/bar'].body, url: 'https://example.com/docs/bar' }, + { id: 4, text: mockPages['/docs/foo/a'].body, url: 'https://example.com/docs/foo/a' }, + ]) + }) + + test('respects canonical URLs', async () => { + fetchMocker.mockResponse( + ` + + + + + + no trailing slash + trailing slash + with querystring + + `, + { url: 'https://example.com/a/' } + ) + const source = createWebCorpusSource({ + entryPage: new URL('https://example.com/a/'), + prefix: new URL('https://example.com'), + }) + expect(await source.docs()).toMatchObject[]>([{ id: 1, url: 'https://example.com/a' }]) + }) +}) + +describe('urlHasPrefix', () => { + test('same url', () => + expect(urlHasPrefix(new URL('https://example.com/a/b'), new URL('https://example.com/a/b'))).toBe(true)) + + test('path prefix', () => { + expect(urlHasPrefix(new URL('https://example.com/a/b'), new URL('https://example.com/a'))).toBe(true) + expect(urlHasPrefix(new URL('https://example.com/a/b'), new URL('https://example.com/a/'))).toBe(true) + expect(urlHasPrefix(new URL('https://example.com/a'), new URL('https://example.com/a/'))).toBe(true) + expect(urlHasPrefix(new URL('https://example.com/a-b'), new URL('https://example.com/a'))).toBe(false) + }) + + test('query', () => { + expect(urlHasPrefix(new URL('https://example.com/a?page=b'), new URL('https://example.com/a'))).toBe(true) + expect(urlHasPrefix(new URL('https://example.com/a/b?page=c'), new URL('https://example.com/a'))).toBe(true) + }) +}) diff --git a/provider/docs/src/corpus/source/web/webCorpusSource.ts b/provider/docs/src/corpus/source/web/webCorpusSource.ts new file mode 100644 index 00000000..13c27b2e --- /dev/null +++ b/provider/docs/src/corpus/source/web/webCorpusSource.ts @@ -0,0 +1,126 @@ +import { parseDOM } from '../../../dom' +import { type Logger } from '../../../logger' +import { type Doc } from '../../doc/doc' +import { type CorpusSource } from '../source' +import { createCrawlQueue } from './crawlQueue' + +interface WebCorpusSourceOptions { + /** + * Start crawling from this page. + */ + entryPage: URL + + /** + * Only include pages whose URL starts with this prefix. + */ + prefix: URL + + /** + * Exclude pages whose URL contains any of these strings. + */ + ignore?: string[] + + /** + * Called to print a log message. + */ + logger?: Logger +} + +export function createWebCorpusSource({ entryPage, prefix, ignore, logger }: WebCorpusSourceOptions): CorpusSource { + return { + docs: async () => { + const { nextURL, enqueueURL, shouldCrawlURL } = createCrawlQueue( + url => urlHasPrefix(url, prefix) && !ignore?.some(ignore => url.href.includes(ignore)) + ) + + if (!shouldCrawlURL(entryPage)) { + throw new Error(`web corpus entryPage (${entryPage}) does not start with prefix (${prefix})`) + } + + enqueueURL(entryPage) + + const documents: Doc[] = [] + + let url: URL | undefined + while ((url = nextURL())) { + logger?.(`Crawling URL: ${url.href}`) + + const resp = await fetch(url.href) + logger?.(`- Response: ${resp.status} ${resp.statusText}`) + if (!resp.ok) { + continue + } + + // Handle redirects. + if (resp.redirected || resp.url !== url.href) { + logger?.(`- Got redirect (redirected=${resp.redirected}, resp.url=${resp.url})`) + const wasRedirectedFromEntryPage = entryPage.href === url.href + url = new URL(resp.url) + if (!shouldCrawlURL(url) && !wasRedirectedFromEntryPage) { + logger?.(`- Skipping redirect destination URL: ${url}`) + continue + } + logger?.(`- Continuing with redirect destination URL: ${url}`) + } + + const html = await resp.text() + const dom = await parseDOM(html, resp.url) + + const canonicalURLStr = dom.querySelector("head > link[rel='canonical']")?.href + if (canonicalURLStr && canonicalURLStr !== url.href) { + const canonicalURL = parseURL(canonicalURLStr) + if (canonicalURL) { + // Only trust the canonical URL if it's same-origin, to avoid letting other + // sites pollute this corpus. + if (canonicalURL.origin === url.origin) { + logger?.(`- Found canonical URL: ${canonicalURL}`) + url = canonicalURL + if (!shouldCrawlURL(url)) { + continue + } + } + } + } + + documents.push({ + id: documents.length + 1, + text: html, + url: url.toString(), + }) + + const pageLinks = dom.querySelectorAll('a[href]') + logger?.(`- Found ${pageLinks.length} links on page`) + for (const link of pageLinks) { + const linkURL = parseURL(link.href) + if (linkURL) { + enqueueURL(linkURL) + } + } + } + + return documents + }, + } +} + +function parseURL(urlStr: string): URL | undefined { + try { + return new URL(urlStr) + } catch { + return undefined + } +} + +export function urlHasPrefix(url: URL, prefix: URL): boolean { + // Disallow username and password. + if (url.username || url.password) { + return false + } + return ( + url.protocol === prefix.protocol && + url.host === prefix.host && + (url.pathname === prefix.pathname || + url.pathname.startsWith(prefix.pathname.endsWith('/') ? prefix.pathname : prefix.pathname + '/') || + (prefix.pathname.endsWith('/') && url.pathname === prefix.pathname.slice(0, -1))) + ) +} diff --git a/provider/docs/src/dom.ts b/provider/docs/src/dom.ts new file mode 100644 index 00000000..95efcebb --- /dev/null +++ b/provider/docs/src/dom.ts @@ -0,0 +1,23 @@ +export type ParseDOM = (html: string, url: string | undefined) => Promise + +/** + * Parse DOM (works in both Node and browser). + */ +export const parseDOM: ParseDOM = + typeof DOMParser === 'undefined' + ? async (html, url) => { + const { JSDOM } = await import('jsdom') + return new JSDOM(html, { url }).window.document + } + : (html, url) => { + const document = new DOMParser().parseFromString(html, 'text/html') + + // Set base URL. + if (url && document.head.querySelectorAll('base').length === 0) { + const baseEl = document.createElement('base') + baseEl.setAttribute('href', url) + document.head.append(baseEl) + } + + return Promise.resolve(document) + } diff --git a/provider/docs/src/e2e.test.ts b/provider/docs/src/e2e.test.ts new file mode 100644 index 00000000..1494cd6b --- /dev/null +++ b/provider/docs/src/e2e.test.ts @@ -0,0 +1,30 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { describe, expect, test } from 'vitest' +import { indexCorpus, type CorpusSearchResult } from './corpus' +import { corpusData } from './corpus/data' + +describe('e2e', () => { + test('urlParsing', async () => { + const docFile = await fs.readFile(path.join(__dirname, 'testdata/corpus/urlParsing.md'), 'utf8') + const codeFile = await fs.readFile(path.join(__dirname, 'testdata/code/urlParsing.ts'), 'utf8') + + const corpus = await indexCorpus(corpusData([{ id: 1, text: docFile }])) + const results = await corpus.search(codeFile) + roundScores(results) + expect(results).toEqual([ + { + doc: 1, + chunk: 3, + excerpt: 'Audio URL parsing\n\nTo parse an audio URL, use the `parseAudioURL` function.', + score: 0.755, + }, + ]) + }) +}) + +function roundScores(results: CorpusSearchResult[]) { + for (const result of results) { + result.score = Math.round(result.score * 1000) / 1000 + } +} diff --git a/provider/docs/src/env.ts b/provider/docs/src/env.ts new file mode 100644 index 00000000..f19a9011 --- /dev/null +++ b/provider/docs/src/env.ts @@ -0,0 +1,3 @@ +const isWebWindowRuntime = typeof window !== 'undefined' + +export const useWebWorker = isWebWindowRuntime diff --git a/provider/docs/src/logger.ts b/provider/docs/src/logger.ts new file mode 100644 index 00000000..a42e4737 --- /dev/null +++ b/provider/docs/src/logger.ts @@ -0,0 +1 @@ +export type Logger = (message: string) => void diff --git a/provider/docs/src/mlWorker/api.ts b/provider/docs/src/mlWorker/api.ts new file mode 100644 index 00000000..97c878a7 --- /dev/null +++ b/provider/docs/src/mlWorker/api.ts @@ -0,0 +1,7 @@ +export interface MLWorkerMessagePair { + type: T + request: { id: number; type: T; args: A } + response: { id: number; result: R } +} + +export interface MLWorkerEmbedTextMessage extends MLWorkerMessagePair<'embedText', string, Float32Array> {} diff --git a/provider/docs/src/mlWorker/webWorker.ts b/provider/docs/src/mlWorker/webWorker.ts new file mode 100644 index 00000000..ad3af12d --- /dev/null +++ b/provider/docs/src/mlWorker/webWorker.ts @@ -0,0 +1,27 @@ +/// + +import { embedTextInThisScope } from '../corpus/search/embeddings' +import { type MLWorkerEmbedTextMessage, type MLWorkerMessagePair } from './api' + +declare let self: DedicatedWorkerGlobalScope + +onRequest( + 'embedText', + async (text: string): Promise => embedTextInThisScope(text) +) + +// Tell our host we are ready. +self.postMessage('ready') + +function onRequest

( + type: P['type'], + handler: (args: P['request']['args']) => Promise +): void { + self.addEventListener('message', async event => { + const request = event.data as P['request'] + if (request.type === type) { + const response: P['response'] = { id: request.id, result: await handler(request.args) } + self.postMessage(response) + } + }) +} diff --git a/provider/docs/src/mlWorker/webWorkerClient.ts b/provider/docs/src/mlWorker/webWorkerClient.ts new file mode 100644 index 00000000..288e7683 --- /dev/null +++ b/provider/docs/src/mlWorker/webWorkerClient.ts @@ -0,0 +1,67 @@ +import { type embedTextInThisScope } from '../corpus/search/embeddings' +import { type MLWorkerEmbedTextMessage, type MLWorkerMessagePair } from './api' + +export const embedTextOnWorker: typeof embedTextInThisScope = async (text: string): Promise => + sendMessage('embedText', text) + +async function sendMessage

( + type: P['type'], + args: P['request']['args'] +): Promise { + const worker = await acquireWorker() + const id = nextID() + worker.postMessage({ id, type, args } satisfies P['request']) + return new Promise(resolve => { + const onMessage = (event: MessageEvent): void => { + const response = event.data as P['response'] + if (response.id === id) { + resolve(response.result) + worker.removeEventListener('message', onMessage) + } + } + worker.addEventListener('message', onMessage) + }) +} + +const NUM_WORKERS: number = Math.min( + 8, + (await (async (): Promise => { + if (typeof navigator !== 'undefined') { + return navigator.hardwareConcurrency + } + try { + const os = await import('node:os') + return os.cpus().length + // eslint-disable-next-line no-empty + } catch {} + return 1 + })()) || 1 +) + +const workers: (Promise | undefined)[] = [] +let workerSeq = 0 + +/** + * Acquire a worker from the pool. Currently the acquisition is round-robin. + */ +async function acquireWorker(): Promise { + const workerID = workerSeq++ % NUM_WORKERS + let workerInstance = workers[workerID] + if (!workerInstance) { + workerInstance = new Promise(resolve => { + const worker = new Worker(new URL('./webWorker.ts', import.meta.url), { type: 'module' }) + + // Wait for worker to become ready. It sends a message when it is ready. The actual message + // doesn't matter. + worker.addEventListener('message', () => resolve(worker)) + }) + console.log('worker', workerID, 'is ready') + workers[workerID] = workerInstance + } + return workerInstance +} + +let id = 1 +function nextID(): number { + return id++ +} diff --git a/provider/docs/src/provider/multiplex.ts b/provider/docs/src/provider/multiplex.ts new file mode 100644 index 00000000..d0280f99 --- /dev/null +++ b/provider/docs/src/provider/multiplex.ts @@ -0,0 +1,31 @@ +import { type Provider } from '@opencodegraph/provider' + +/** + * @template S The settings type. + */ +export function multiplex(createProvider: (settings: S) => Promise>): Provider { + const providerCache = new Map>>() + + function getProvider(settings: S): Promise> { + const key = JSON.stringify(settings) + let provider = providerCache.get(key) + if (!provider) { + provider = createProvider(settings) + providerCache.set(key, provider) + + // Prevent accidental memory leaks in case `settings` keeps changing. + // + // TODO(sqs): use an LRU cache or something + const MAX_SIZE = 10 + if (providerCache.size > MAX_SIZE) { + throw new Error(`provider cache is too big (max size ${MAX_SIZE})`) + } + } + return provider + } + + return { + capabilities: (params, settings) => getProvider(settings).then(p => p.capabilities(params, settings)), + annotations: (params, settings) => getProvider(settings).then(p => p.annotations(params, settings)), + } +} diff --git a/provider/docs/src/provider/provider.ts b/provider/docs/src/provider/provider.ts new file mode 100644 index 00000000..5318839c --- /dev/null +++ b/provider/docs/src/provider/provider.ts @@ -0,0 +1,97 @@ +/* eslint-disable import/no-default-export */ +import { + type AnnotationsParams, + type AnnotationsResult, + type CapabilitiesResult, + type Item, +} from '@opencodegraph/provider' +import { indexCorpus } from '../corpus' +import { createIndexedDBCorpusCache } from '../corpus/cache/indexedDb' +import { createWebStorageCorpusCache } from '../corpus/cache/localStorage' +import { corpusData } from '../corpus/data' +import { extractContentUsingMozillaReadability } from '../corpus/doc/contentExtractor' +import { corpusDataURLSource } from '../corpus/source/source' +import { createWebCorpusSource } from '../corpus/source/web/webCorpusSource' +import { multiplex } from './multiplex' + +/** Settings for the docs OpenCodeGraph provider. */ +export interface Settings { + corpus: + | { url: string } + | { + entryPage: string + prefix: string + ignore?: string[] + } +} + +const CORPUS_CACHE = + typeof indexedDB === 'undefined' + ? typeof localStorage === 'undefined' + ? undefined + : createWebStorageCorpusCache(localStorage, 'ocg-provider-docs') + : createIndexedDBCorpusCache('ocg-provider-docs') + +/** + * An [OpenCodeGraph](https://opencodegraph.org) provider that adds contextual documentation to your + * code from an existing documentation corpus. + */ +export default multiplex(async settings => { + const source = + 'url' in settings.corpus + ? corpusDataURLSource(settings.corpus.url) + : createWebCorpusSource({ + entryPage: new URL(settings.corpus.entryPage), + prefix: new URL(settings.corpus.prefix), + ignore: settings.corpus.ignore, + logger: message => console.log(message), + }) + const index = await indexCorpus(corpusData(await source.docs()), { + cache: CORPUS_CACHE, + contentExtractor: extractContentUsingMozillaReadability, + }) + + return { + capabilities(): CapabilitiesResult { + return {} + }, + + async annotations(params: AnnotationsParams): Promise { + console.time('search') + const searchResults = await index.search(params.content) + console.timeEnd('search') + + const result: AnnotationsResult = { items: [], annotations: [] } + for (const [i, sr] of searchResults.entries()) { + const MAX_RESULTS = 4 + if (i >= MAX_RESULTS) { + break + } + + const doc = index.doc(sr.doc) + const item: Item = { + id: i.toString(), + title: truncate(doc.content?.title || doc.doc.url || 'Untitled', 50), + detail: truncate(doc.content?.textContent || sr.excerpt, 100), + url: doc.doc.url, + } + result.items.push(item) + result.annotations.push({ + item: { id: item.id }, + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }, + }) + } + return result + }, + } +}) + +function truncate(text: string, maxLength: number): string { + if (text.length > maxLength) { + return text.slice(0, maxLength) + '...' + } + return text +} diff --git a/provider/docs/src/testdata/code/urlParsing.ts b/provider/docs/src/testdata/code/urlParsing.ts new file mode 100644 index 00000000..8a7c652f --- /dev/null +++ b/provider/docs/src/testdata/code/urlParsing.ts @@ -0,0 +1,6 @@ +// @ts-nocheck + +function getAudio(title: string): URL { + const audioFile = searchAudioFiles(title) + return parseAudioURL(audioFile.url) +} diff --git a/provider/docs/src/testdata/corpus/urlParsing.md b/provider/docs/src/testdata/corpus/urlParsing.md new file mode 100644 index 00000000..864548eb --- /dev/null +++ b/provider/docs/src/testdata/corpus/urlParsing.md @@ -0,0 +1,15 @@ +# URL parsing + +To parse a URL, use the `parseURL` function. + +## Image URL parsing + +To parse an image URL, use the `parseImageURL` function. + +## Video URL parsing + +To parse an image URL, use the `parseVideoURL` function. + +## Audio URL parsing + +To parse an audio URL, use the `parseAudioURL` function. diff --git a/provider/docs/tsconfig.json b/provider/docs/tsconfig.json new file mode 100644 index 00000000..3f1daf72 --- /dev/null +++ b/provider/docs/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../.config/tsconfig.base.json", + "compilerOptions": { + "module": "ESNext", + "rootDir": ".", + "outDir": "dist", + "lib": ["ESNext"], + }, + "include": ["src", "bin"], + "exclude": ["dist", "src/testdata", "vitest.config.ts"], + "references": [{ "path": "../../lib/provider" }], +} diff --git a/provider/docs/vitest.config.ts b/provider/docs/vitest.config.ts new file mode 100644 index 00000000..abed6b21 --- /dev/null +++ b/provider/docs/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({}) diff --git a/tsconfig.json b/tsconfig.json index 3c4c6d67..f5eb76d8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "noEmit": true }, "files": [], - "exclude": ["**/dist", "client/browser"], + "exclude": ["**/dist", "**/testdata", "client/browser"], "references": [ { "path": "lib/client" }, { "path": "lib/protocol" }, @@ -22,6 +22,7 @@ { "path": "client/vscode/dev" }, { "path": "client/vscode/test/integration" }, { "path": "client/web-playground" }, + { "path": "provider/docs" }, { "path": "provider/hello-world" }, { "path": "provider/links" }, { "path": "provider/prometheus" }, diff --git a/web/content/docs/concepts.mdx b/web/content/docs/concepts.mdx index 4f76f41a..5b288dd5 100644 --- a/web/content/docs/concepts.mdx +++ b/web/content/docs/concepts.mdx @@ -63,3 +63,7 @@ How do clients authenticate with providers? Right now, a few hostnames are hard- ### Implementing providers: thin clients or lots-of-logic? Will OpenCodeGraph providers be super lightweight presentation-layer programs that hit some existing system that makes the data easy to fetch? Maybe that existing system is a separate server built for OCG to hit for now, but ideally in the future (years) it would become a common endpoint for dev tools vendors to expose, like oEmbed. + +### Recording additional metadata + +How should annotations record other kinds of metadata to make it searchable and exportable? For example, the Storybook provider could add `{"storybook": true}` metadata in an annotation on all storybook files, which would make it possible to identify all storybook files in a code search application. diff --git a/web/package.json b/web/package.json index 9b0daed5..396020ce 100644 --- a/web/package.json +++ b/web/package.json @@ -25,12 +25,12 @@ "@mapbox/rehype-prism": "^0.9.0", "@mdx-js/react": "^3.0.0", "@mdx-js/rollup": "^3.0.0", + "@opencodegraph/codemirror-extension": "workspace:*", + "@opencodegraph/monaco-editor-extension": "workspace:*", "@opencodegraph/provider-links": "workspace:*", - "@opencodegraph/provider-storybook": "workspace:*", "@opencodegraph/provider-prometheus": "workspace:*", + "@opencodegraph/provider-storybook": "workspace:*", "@opencodegraph/web-playground": "workspace:*", - "@opencodegraph/codemirror-extension": "workspace:*", - "@opencodegraph/monaco-editor-extension": "workspace:*", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-dropdown-menu": "^2.0.6", "@radix-ui/react-navigation-menu": "^1.1.4",