feat(core): support ai network search (#9357)

### What Changed? - Add `PerplexityProvider` in backend. - Update session prompt name if user toggle network search mode in chat panel. - Add experimental flag for AI network search feature. - Add unit tests and e2e tests. Search results are streamed and appear word for word: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov">录屏2024-12-27 18.58.40.mov</video> Click the little globe icon to manually turn on/off Internet search: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov">录屏2024-12-27 19.01.16.mov</video> When there is an image, it will automatically switch to the openai model: <div class='graphite__hidden'> <div>🎥 Video uploaded on Graphite:</div> <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov"> <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov"> </a> </div> <video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov">录屏2024-12-27 19.02.13.mov</video>
toeverything · Jan 9, 2025 · 58ce865 · 58ce865
1 parent 4f10457
commit 58ce865
Show file tree

Hide file tree

Showing 49 changed files with 1,276 additions and 171 deletions.
diff --git a/.github/actions/copilot-test/action.yml b/.github/actions/copilot-test/action.yml
@@ -26,6 +26,7 @@ runs:
         DEV_SERVER_URL: http://localhost:8080
         COPILOT_OPENAI_API_KEY: ${{ inputs.openai-key }}
         COPILOT_FAL_API_KEY: ${{ inputs.fal-key }}
+        COPILOT_PERPLEXITY_API_KEY: ${{ inputs.perplexity-key }}
 
     - name: Upload test results
       if: ${{ failure() }}

diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs
@@ -17,6 +17,7 @@ const {
   METRICS_CUSTOMER_IO_TOKEN,
   COPILOT_OPENAI_API_KEY,
   COPILOT_FAL_API_KEY,
+  COPILOT_PERPLEXITY_API_KEY,
   COPILOT_UNSPLASH_API_KEY,
   MAILER_SENDER,
   MAILER_USER,
@@ -147,6 +148,7 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        graphql.app.copilot.enabled=true`,
     `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`,
     `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`,
+    `--set-string graphql.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`,
     `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`,
     `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`,
     `--set-string graphql.app.mailer.user="${MAILER_USER}"`,

diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml
@@ -157,6 +157,11 @@ spec:
               secretKeyRef:
                 name: "{{ .Values.app.copilot.secretName }}"
                 key: falSecret
+          - name: COPILOT_PERPLEXITY_API_KEY
+            valueFrom:
+              secretKeyRef:
+                name: "{{ .Values.app.copilot.secretName }}"
+                key: perplexitySecret
           - name: COPILOT_UNSPLASH_API_KEY
             valueFrom:
               secretKeyRef:

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -531,6 +531,7 @@ jobs:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+          COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 
       - name: Upload server test coverage results
         if: ${{ steps.check-blocksuite-update.outputs.skip != 'true' || steps.apifilter.outputs.changed == 'true' }}
@@ -619,6 +620,7 @@ jobs:
           script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
+          perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 
   server-e2e-test:
     name: ${{ matrix.tests.name }}
@@ -703,6 +705,7 @@ jobs:
           DEV_SERVER_URL: http://localhost:8080
           COPILOT_OPENAI_API_KEY: 1
           COPILOT_FAL_API_KEY: 1
+          COPILOT_PERPLEXITY_API_KEY: 1
 
       - name: Upload test results
         if: ${{ failure() }}

diff --git a/.github/workflows/copilot-test.yml b/.github/workflows/copilot-test.yml
@@ -84,6 +84,7 @@ jobs:
           CARGO_TARGET_DIR: '${{ github.workspace }}/target'
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+          COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 
       - name: Upload server test coverage results
         uses: codecov/codecov-action@v5
@@ -147,6 +148,7 @@ jobs:
           script: yarn affine @affine-test/affine-cloud-copilot e2e --forbid-only --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
           openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
+          perplexity-key: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
 
   test-done:
     needs:

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -98,6 +98,7 @@ jobs:
           CAPTCHA_TURNSTILE_SECRET: ${{ secrets.CAPTCHA_TURNSTILE_SECRET }}
           COPILOT_OPENAI_API_KEY: ${{ secrets.COPILOT_OPENAI_API_KEY }}
           COPILOT_FAL_API_KEY: ${{ secrets.COPILOT_FAL_API_KEY }}
+          COPILOT_PERPLEXITY_API_KEY: ${{ secrets.COPILOT_PERPLEXITY_API_KEY }}
           COPILOT_UNSPLASH_API_KEY: ${{ secrets.COPILOT_UNSPLASH_API_KEY }}
           METRICS_CUSTOMER_IO_TOKEN: ${{ secrets.METRICS_CUSTOMER_IO_TOKEN }}
           MAILER_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }}

diff --git a/blocksuite/blocks/src/root-block/widgets/ai-panel/ai-panel.ts b/blocksuite/blocks/src/root-block/widgets/ai-panel/ai-panel.ts
@@ -63,7 +63,7 @@ export class AffineAIPanelWidget extends WidgetComponent {
       box-sizing: border-box;
       width: 100%;
       height: fit-content;
-      padding: 8px 0;
+      padding: 10px 0;
     }
 
     .ai-panel-container:not(:has(ai-panel-generating)) {
@@ -474,6 +474,7 @@ export class AffineAIPanelWidget extends WidgetComponent {
             .onBlur=${this.discard}
             .onFinish=${this._inputFinish}
             .onInput=${this.onInput}
+            .networkSearchConfig=${config.networkSearchConfig}
           ></ai-panel-input>`,
       ],
       [

diff --git a/blocksuite/blocks/src/root-block/widgets/ai-panel/components/state/input.ts b/blocksuite/blocks/src/root-block/widgets/ai-panel/components/state/input.ts
@@ -1,11 +1,14 @@
 import { AIStarIcon } from '@blocksuite/affine-components/icons';
+import { unsafeCSSVarV2 } from '@blocksuite/affine-shared/theme';
 import { stopPropagation } from '@blocksuite/affine-shared/utils';
-import { WithDisposable } from '@blocksuite/global/utils';
-import { SendIcon } from '@blocksuite/icons/lit';
+import { SignalWatcher, WithDisposable } from '@blocksuite/global/utils';
+import { PublishIcon, SendIcon } from '@blocksuite/icons/lit';
 import { css, html, LitElement, nothing } from 'lit';
 import { property, query, state } from 'lit/decorators.js';
 
-export class AIPanelInput extends WithDisposable(LitElement) {
+import type { AINetworkSearchConfig } from '../../type';
+
+export class AIPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
   static override styles = css`
     :host {
       width: 100%;
@@ -20,8 +23,9 @@ export class AIPanelInput extends WithDisposable(LitElement) {
       background: var(--affine-background-overlay-panel-color);
     }
 
-    .icon {
+    .star {
       display: flex;
+      padding: 2px;
       align-items: center;
     }
 
@@ -66,22 +70,36 @@ export class AIPanelInput extends WithDisposable(LitElement) {
       display: flex;
       align-items: center;
       padding: 2px;
-      gap: 10px;
+      gap: 4px;
       border-radius: 4px;
-      background: var(--affine-black-10, rgba(0, 0, 0, 0.1));
-
+      background: ${unsafeCSSVarV2('icon/disable')};
       svg {
-        width: 16px;
-        height: 16px;
-        color: var(--affine-pure-white, #fff);
+        width: 20px;
+        height: 20px;
+        color: ${unsafeCSSVarV2('button/pureWhiteText')};
       }
     }
     .arrow[data-active] {
-      background: var(--affine-brand-color, #1e96eb);
+      background: ${unsafeCSSVarV2('icon/activated')};
     }
     .arrow[data-active]:hover {
       cursor: pointer;
     }
+    .network {
+      display: flex;
+      align-items: center;
+      padding: 2px;
+      gap: 4px;
+      cursor: pointer;
+      svg {
+        width: 20px;
+        height: 20px;
+        color: ${unsafeCSSVarV2('icon/primary')};
+      }
+    }
+    .network[data-active='true'] svg {
+      color: ${unsafeCSSVarV2('icon/activated')};
+    }
   `;
 
   private readonly _onInput = () => {
@@ -101,22 +119,32 @@ export class AIPanelInput extends WithDisposable(LitElement) {
 
   private readonly _onKeyDown = (e: KeyboardEvent) => {
     if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) {
-      e.preventDefault();
-      this._sendToAI();
+      this._sendToAI(e);
     }
   };
 
-  private readonly _sendToAI = () => {
+  private readonly _sendToAI = (e: MouseEvent | KeyboardEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+
     const value = this.textarea.value.trim();
     if (value.length === 0) return;
 
     this.onFinish?.(value);
     this.remove();
   };
 
+  private readonly _toggleNetworkSearch = (e: MouseEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    const enable = this.networkSearchConfig.enabled.value;
+    this.networkSearchConfig.setEnabled(!enable);
+  };
+
   override render() {
     return html`<div class="root">
-      <div class="icon">${AIStarIcon}</div>
+      <div class="star">${AIStarIcon}</div>
       <div class="textarea-container">
         <textarea
           placeholder="What are your thoughts?"
@@ -131,6 +159,21 @@ export class AIPanelInput extends WithDisposable(LitElement) {
           @paste=${stopPropagation}
           @keyup=${stopPropagation}
         ></textarea>
+        ${this.networkSearchConfig.visible.value
+          ? html`
+              <div
+                class="network"
+                data-active=${!!this.networkSearchConfig.enabled.value}
+                @click=${this._toggleNetworkSearch}
+                @pointerdown=${stopPropagation}
+              >
+                ${PublishIcon()}
+                <affine-tooltip .offset=${12}
+                  >Toggle Network Search</affine-tooltip
+                >
+              </div>
+            `
+          : nothing}
         <div
           class="arrow"
           @click=${this._sendToAI}
@@ -157,6 +200,9 @@ export class AIPanelInput extends WithDisposable(LitElement) {
   @state()
   private accessor _hasContent = false;
 
+  @property({ attribute: false })
+  accessor networkSearchConfig!: AINetworkSearchConfig;
+
   @property({ attribute: false })
   accessor onFinish: ((input: string) => void) | undefined = undefined;
 

diff --git a/blocksuite/blocks/src/root-block/widgets/ai-panel/type.ts b/blocksuite/blocks/src/root-block/widgets/ai-panel/type.ts
@@ -2,6 +2,7 @@ import type {
   AIError,
   AIItemGroupConfig,
 } from '@blocksuite/affine-components/ai-item';
+import type { Signal } from '@preact/signals-core';
 import type { nothing, TemplateResult } from 'lit';
 
 export interface CopyConfig {
@@ -28,6 +29,12 @@ export interface AIPanelGeneratingConfig {
   stages?: string[];
 }
 
+export interface AINetworkSearchConfig {
+  visible: Signal<boolean | undefined>;
+  enabled: Signal<boolean | undefined>;
+  setEnabled: (state: boolean) => void;
+}
+
 export interface AffineAIPanelWidgetConfig {
   answerRenderer: (
     answer: string,
@@ -44,10 +51,10 @@ export interface AffineAIPanelWidgetConfig {
   finishStateConfig: AIPanelAnswerConfig;
   generatingStateConfig: AIPanelGeneratingConfig;
   errorStateConfig: AIPanelErrorConfig;
+  networkSearchConfig: AINetworkSearchConfig;
   hideCallback?: () => void;
   discardCallback?: () => void;
   inputCallback?: (input: string) => void;
-
   copy?: CopyConfig;
 }
 

diff --git a/packages/backend/server/.env.example b/packages/backend/server/.env.example
@@ -2,6 +2,7 @@
 # REDIS_SERVER_HOST=localhost
 # COPILOT_FAL_API_KEY=YOUR_KEY
 # COPILOT_OPENAI_API_KEY=YOUR_KEY
+# COPILOT_PERPLEXITY_API_KEY=YOUR_KEY
 
 # MAILER_HOST=127.0.0.1
 # MAILER_PORT=1025

diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json
@@ -59,6 +59,7 @@
     "@socket.io/redis-adapter": "^8.3.0",
     "cookie-parser": "^1.4.7",
     "dotenv": "^16.4.7",
+    "eventsource-parser": "^3.0.0",
     "express": "^4.21.2",
     "fast-xml-parser": "^4.5.0",
     "get-stream": "^9.0.1",

diff --git a/packages/backend/server/src/config/affine.env.ts b/packages/backend/server/src/config/affine.env.ts
@@ -28,6 +28,7 @@ AFFiNE.ENV_MAP = {
   CAPTCHA_TURNSTILE_SECRET: ['plugins.captcha.turnstile.secret', 'string'],
   COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey',
   COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey',
+  COPILOT_PERPLEXITY_API_KEY: 'plugins.copilot.perplexity.apiKey',
   COPILOT_UNSPLASH_API_KEY: 'plugins.copilot.unsplashKey',
   REDIS_SERVER_HOST: 'redis.host',
   REDIS_SERVER_PORT: ['redis.port', 'int'],

diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts
@@ -3,10 +3,12 @@ import type { ClientOptions as OpenAIClientOptions } from 'openai';
 import { defineStartupConfig, ModuleConfig } from '../../base/config';
 import { StorageConfig } from '../../base/storage/config';
 import type { FalConfig } from './providers/fal';
+import { PerplexityConfig } from './providers/perplexity';
 
 export interface CopilotStartupConfigurations {
   openai?: OpenAIClientOptions;
   fal?: FalConfig;
+  perplexity?: PerplexityConfig;
   test?: never;
   unsplashKey?: string;
   storage: StorageConfig;

diff --git a/packages/backend/server/src/plugins/copilot/index.ts b/packages/backend/server/src/plugins/copilot/index.ts
@@ -13,6 +13,7 @@ import {
   CopilotProviderService,
   FalProvider,
   OpenAIProvider,
+  PerplexityProvider,
   registerCopilotProvider,
 } from './providers';
 import {
@@ -26,6 +27,7 @@ import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow';
 
 registerCopilotProvider(FalProvider);
 registerCopilotProvider(OpenAIProvider);
+registerCopilotProvider(PerplexityProvider);
 
 @Plugin({
   name: 'copilot',

diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts
@@ -952,6 +952,11 @@ const chat: Prompt[] = [
       },
     ],
   },
+  {
+    name: 'Search With AFFiNE AI',
+    model: 'llama-3.1-sonar-small-128k-online',
+    messages: [],
+  },
   // use for believer plan
   {
     name: 'Chat With AFFiNE AI - Believer',

diff --git a/packages/backend/server/src/plugins/copilot/providers/index.ts b/packages/backend/server/src/plugins/copilot/providers/index.ts
@@ -124,9 +124,7 @@ export class CopilotProviderService {
     if (!this.cachedProviders.has(provider)) {
       this.cachedProviders.set(provider, this.create(provider));
     }
-
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    return this.cachedProviders.get(provider)!;
+    return this.cachedProviders.get(provider) as CopilotProvider;
   }
 
   async getProviderByCapability<C extends CopilotCapability>(
@@ -196,3 +194,4 @@ export class CopilotProviderService {
 
 export { FalProvider } from './fal';
 export { OpenAIProvider } from './openai';
+export { PerplexityProvider } from './perplexity';