fix: AI test

web-infra-dev · Jan 23, 2025 · 8a449f3 · 8a449f3
1 parent 70aef6b
commit 8a449f3
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 51 deletions.
diff --git a/.github/workflows/ai.yml b/.github/workflows/ai.yml
@@ -21,7 +21,7 @@ jobs:
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
-      MIDSCENE_MODEL_NAME: gpt-4o-2024-08-06
+      MIDSCENE_MODEL_NAME: gpt-4o-2024-11-20
       CI: 1
       # MIDSCENE_DEBUG_AI_PROFILE: 1
 

diff --git a/packages/midscene/tests/ai/plan/plan-to-target.test.ts b/packages/midscene/tests/ai/plan/plan-to-target.test.ts
@@ -4,55 +4,58 @@ import { savePositionImg } from '@midscene/shared/img';
 import { expect, test } from 'vitest';
 import { getPageTestData } from '../evaluate/test-suite/util';
 
-test('inspect with quick answer', async () => {
-  const { context } = await getPageTestData(
-    path.join(__dirname, '../evaluate/test-data/todo'),
-  );
+test.skipIf(!process.env.MIDSCENE_USE_VLM_UI_TARS)(
+  'inspect with quick answer',
+  async () => {
+    const { context } = await getPageTestData(
+      path.join(__dirname, '../evaluate/test-data/todo'),
+    );
 
-  const { width, height } = context.size;
-  const startTime = Date.now();
-  const { realActions } = await vlmPlanning({
-    userInstruction: '删除第二条任务',
-    conversationHistory: [
-      {
-        role: 'user',
-        content: [
-          {
-            type: 'image_url',
-            image_url: {
-              url: context.originalScreenshotBase64,
+    const { width, height } = context.size;
+    const startTime = Date.now();
+    const { realActions } = await vlmPlanning({
+      userInstruction: '删除第二条任务',
+      conversationHistory: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image_url',
+              image_url: {
+                url: context.originalScreenshotBase64,
+              },
             },
-          },
-        ],
+          ],
+        },
+      ],
+      size: {
+        width,
+        height,
       },
-    ],
-    size: {
-      width,
-      height,
-    },
-  });
+    });
 
-  const endTime = Date.now();
-  const cost = (endTime - startTime) / 1000;
-  const start_box =
-    'start_box' in realActions[0].action_inputs
-      ? realActions[0].action_inputs.start_box
-      : '[]';
-  const box = JSON.parse(start_box);
-  console.log('plan to target content:', {
-    box,
-    size: {
-      width,
-      height,
-    },
-    cost: `${cost}s`,
-  });
-  // expect(box).toEqual([0.397, 0.218, 0.397, 0.218]);
-  expect(true).toBe(true);
-  await savePositionImg({
-    inputImgBase64: context.originalScreenshotBase64,
-    rect: { x: box[0] * width, y: box[1] * height },
-    outputPath: path.join(__dirname, 'output.png'),
-  });
-  //   expect(cost).toBeLessThan(100);
-});
+    const endTime = Date.now();
+    const cost = (endTime - startTime) / 1000;
+    const start_box =
+      'start_box' in realActions[0].action_inputs
+        ? realActions[0].action_inputs.start_box
+        : '[]';
+    const box = JSON.parse(start_box);
+    console.log('plan to target content:', {
+      box,
+      size: {
+        width,
+        height,
+      },
+      cost: `${cost}s`,
+    });
+    // expect(box).toEqual([0.397, 0.218, 0.397, 0.218]);
+    expect(true).toBe(true);
+    await savePositionImg({
+      inputImgBase64: context.originalScreenshotBase64,
+      rect: { x: box[0] * width, y: box[1] * height },
+      outputPath: path.join(__dirname, 'output.png'),
+    });
+    //   expect(cost).toBeLessThan(100);
+  },
+);
diff --git a/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts
@@ -67,9 +67,9 @@ describe(
       });
     });
 
-    it('find widgets in antd', async () => {
+    it.skipIf(process.env.CI)('find widgets in antd', async () => {
       const { originPage, reset } = await launchPage(
-        'https://ant.design/components/form/',
+        'https://ant.design/components/form/', // will be banned by the website on CI
       );
       resetFn = reset;
       const mid = new PuppeteerAgent(originPage);