Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(e2e): optimize e2e error #307

Merged
merged 2 commits into from
Jan 22, 2025
Merged

fix(e2e): optimize e2e error #307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a35dd83
fix(e2e): optimize e2e error
zhoushaw Jan 22, 2025
58d5460
chore: ignore unless e2e
zhoushaw Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ai.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
MIDSCENE_MODEL_NAME: gpt-4o-2024-11-20
MIDSCENE_MODEL_NAME: gpt-4o-2024-08-06
CI: 1
# MIDSCENE_DEBUG_AI_PROFILE: 1

Expand Down
358 changes: 179 additions & 179 deletions packages/web-integration/tests/ai/web/playwright/plan-to-target/new-ai.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,179 +1,179 @@
import fs from 'node:fs';
import path from 'node:path';
import { generateExtractData, generateTestDataPath } from '@/debug';
import { PlaywrightWebPage } from '@/playwright';
import {
type ChatCompletionMessageParam,
vlmPlanning,
} from '@midscene/core/ai-model';
import { saveBase64Image, savePositionImg } from '@midscene/shared/img';
import type { Page } from 'playwright';
import { test } from '../fixture';

function getPoint(startBox: string, size: { width: number; height: number }) {
const [x, y] = JSON.parse(startBox);
return [x * size.width, y * size.height];
}

function sleep(number: number) {
return new Promise((resolve) => setTimeout(resolve, number));
}
function capitalize(str: string) {
return str.charAt(0).toUpperCase() + str.slice(1);
}

async function loopAgent(
page: Page,
playwrightPage: PlaywrightWebPage,
userInstruction: string,
) {
let isCompleted = false;
let currentActionNumber = 0;
// const targetPrompt =
// '切换语言到中文，下单一个饮品（饮品规格页说明：饮品规格页有很多饮品规格，可能需要滚动完成必选饮品的勾选，最下面有确认下单按钮）';
const conversationHistory: ChatCompletionMessageParam[] = [];

while (!isCompleted && currentActionNumber < 100) {
await page.waitForLoadState('networkidle');
await sleep(1000);
const screenshotBase64 = await playwrightPage.screenshotBase64();
const size = await playwrightPage.size();
const startTime = Date.now();
conversationHistory.push({
role: 'user',
content: [
{
type: 'image_url',
image_url: {
url: screenshotBase64,
},
},
],
});
const { realActions, action_summary } = await vlmPlanning({
userInstruction: userInstruction,
conversationHistory,
size,
});
const endTime = Date.now();
const duration = endTime - startTime;
console.log(`API call duration: ${duration}ms`, realActions);

const action = realActions[0];

if (action.action_type === 'click') {
const point = getPoint(action.action_inputs.start_box, size);
await playwrightPage.mouse.click(point[0], point[1]);
await savePositionImg({
inputImgBase64: screenshotBase64,
rect: {
x: point[0],
y: point[1],
},
outputPath: path.join(
__dirname,
'output',
`plan-target-${currentActionNumber++}.png`,
),
});
} else if (action.action_type === 'type') {
await playwrightPage.keyboard.type(
action.action_inputs.content.trim().replace('\\n', '\n'),
);
} else if (action.action_type === 'hotkey') {
const keys = action.action_inputs.key.split(',');
for (const key of keys) {
await playwrightPage.keyboard.press(capitalize(key) as any);
}
// await playwrightPage.keyboard.press(
// capitalize(action.action_inputs.key) as any,
// );
} else if (action.action_type === 'finished') {
isCompleted = true;
break;
} else if (action.action_type === 'scroll') {
if (action.action_inputs.direction === 'down') {
await playwrightPage.scrollDown();
} else {
await playwrightPage.scrollUp();
}
// await playwrightPage.scrollTo(
}

conversationHistory.push({
role: 'assistant',
content: action_summary,
});

if (currentActionNumber > 10) {
break;
}

// console.log('conversationHistory', conversationHistory);
}

console.log('historyActions', conversationHistory);
}

test('search midscene and star aiTarget', async ({ page, aiAction }) => {
try {
const outputPath = path.join(__dirname, 'output');
await fs.rmSync(outputPath, { recursive: true });
} catch (error) {}
await page.goto('https://google.com');
page.on('popup', async (popup) => {
const url = await popup.url();
console.log(`Popup opened: ${url}`);
await popup.close(); // 关闭新打开的标签页
await page.goto(url);
});
await aiAction('搜索 midscene ，找到 github 地址，点击 star');
});

test('music aiTarget', async ({ page, aiAction }) => {
try {
const outputPath = path.join(__dirname, 'output');
await fs.rmdirSync(outputPath, { recursive: true });
} catch (error) {}
await page.goto('https://google.com');
page.on('popup', async (popup) => {
const url = await popup.url();
console.log(`Popup opened: ${url}`);
await popup.close(); // 关闭新打开的标签页
await page.goto(url);
});
await aiAction('搜索抖音，搜索黑神话悟空，点赞');
});

test('search midscene and star', async ({ page }) => {
try {
const outputPath = path.join(__dirname, 'output');
await fs.rmdirSync(outputPath, { recursive: true });
} catch (error) {}
await page.goto('https://google.com');
page.on('popup', async (popup) => {
const url = await popup.url();
console.log(`Popup opened: ${url}`);
await popup.close(); // 关闭新打开的标签页
await page.goto(url);
});
const playwrightPage = new PlaywrightWebPage(page);

await loopAgent(
page,
playwrightPage,
'搜索 midscene ，找到 github 地址，滚动到下面查看有几个贡献者',
);
});

test('douyin', async ({ page }) => {
try {
const outputPath = path.join(__dirname, 'output');
await fs.rmdirSync(outputPath, { recursive: true });
} catch (error) {}
await page.goto('https://douyin.com');

const playwrightPage = new PlaywrightWebPage(page);

await loopAgent(page, playwrightPage, '搜索 黑神话，看两个视频');
});
// import fs from 'node:fs';
// import path from 'node:path';
// import { generateExtractData, generateTestDataPath } from '@/debug';
// import { PlaywrightWebPage } from '@/playwright';
// import {
// type ChatCompletionMessageParam,
// vlmPlanning,
// } from '@midscene/core/ai-model';
// import { saveBase64Image, savePositionImg } from '@midscene/shared/img';
// import type { Page } from 'playwright';
// import { test } from '../fixture';

// function getPoint(startBox: string, size: { width: number; height: number }) {
// const [x, y] = JSON.parse(startBox);
// return [x * size.width, y * size.height];
// }

// function sleep(number: number) {
// return new Promise((resolve) => setTimeout(resolve, number));
// }
// function capitalize(str: string) {
// return str.charAt(0).toUpperCase() + str.slice(1);
// }

// async function loopAgent(
// page: Page,
// playwrightPage: PlaywrightWebPage,
// userInstruction: string,
// ) {
// let isCompleted = false;
// let currentActionNumber = 0;
// // const targetPrompt =
// // '切换语言到中文，下单一个饮品（饮品规格页说明：饮品规格页有很多饮品规格，可能需要滚动完成必选饮品的勾选，最下面有确认下单按钮）';
// const conversationHistory: ChatCompletionMessageParam[] = [];

// while (!isCompleted && currentActionNumber < 100) {
// await page.waitForLoadState('networkidle');
// await sleep(1000);
// const screenshotBase64 = await playwrightPage.screenshotBase64();
// const size = await playwrightPage.size();
// const startTime = Date.now();
// conversationHistory.push({
// role: 'user',
// content: [
// {
// type: 'image_url',
// image_url: {
// url: screenshotBase64,
// },
// },
// ],
// });
// const { realActions, action_summary } = await vlmPlanning({
// userInstruction: userInstruction,
// conversationHistory,
// size,
// });
// const endTime = Date.now();
// const duration = endTime - startTime;
// console.log(`API call duration: ${duration}ms`, realActions);

// const action = realActions[0];

// if (action.action_type === 'click') {
// const point = getPoint(action.action_inputs.start_box, size);
// await playwrightPage.mouse.click(point[0], point[1]);
// await savePositionImg({
// inputImgBase64: screenshotBase64,
// rect: {
// x: point[0],
// y: point[1],
// },
// outputPath: path.join(
// __dirname,
// 'output',
// `plan-target-${currentActionNumber++}.png`,
// ),
// });
// } else if (action.action_type === 'type') {
// await playwrightPage.keyboard.type(
// action.action_inputs.content.trim().replace('\\n', '\n'),
// );
// } else if (action.action_type === 'hotkey') {
// const keys = action.action_inputs.key.split(',');
// for (const key of keys) {
// await playwrightPage.keyboard.press(capitalize(key) as any);
// }
// // await playwrightPage.keyboard.press(
// // capitalize(action.action_inputs.key) as any,
// // );
// } else if (action.action_type === 'finished') {
// isCompleted = true;
// break;
// } else if (action.action_type === 'scroll') {
// if (action.action_inputs.direction === 'down') {
// await playwrightPage.scrollDown();
// } else {
// await playwrightPage.scrollUp();
// }
// // await playwrightPage.scrollTo(
// }

// conversationHistory.push({
// role: 'assistant',
// content: action_summary,
// });

// if (currentActionNumber > 10) {
// break;
// }

// // console.log('conversationHistory', conversationHistory);
// }

// console.log('historyActions', conversationHistory);
// }

// test('search midscene and star aiTarget', async ({ page, aiAction }) => {
// try {
// const outputPath = path.join(__dirname, 'output');
// await fs.rmSync(outputPath, { recursive: true });
// } catch (error) {}
// await page.goto('https://google.com');
// page.on('popup', async (popup) => {
// const url = await popup.url();
// console.log(`Popup opened: ${url}`);
// await popup.close(); // 关闭新打开的标签页
// await page.goto(url);
// });
// await aiAction('搜索 midscene ，找到 github 地址，点击 star');
// });

// test('music aiTarget', async ({ page, aiAction }) => {
// try {
// const outputPath = path.join(__dirname, 'output');
// await fs.rmdirSync(outputPath, { recursive: true });
// } catch (error) {}
// await page.goto('https://google.com');
// page.on('popup', async (popup) => {
// const url = await popup.url();
// console.log(`Popup opened: ${url}`);
// await popup.close(); // 关闭新打开的标签页
// await page.goto(url);
// });
// await aiAction('搜索抖音，搜索黑神话悟空，点赞');
// });

// test('search midscene and star', async ({ page }) => {
// try {
// const outputPath = path.join(__dirname, 'output');
// await fs.rmdirSync(outputPath, { recursive: true });
// } catch (error) {}
// await page.goto('https://google.com');
// page.on('popup', async (popup) => {
// const url = await popup.url();
// console.log(`Popup opened: ${url}`);
// await popup.close(); // 关闭新打开的标签页
// await page.goto(url);
// });
// const playwrightPage = new PlaywrightWebPage(page);

// await loopAgent(
// page,
// playwrightPage,
// '搜索 midscene ，找到 github 地址，滚动到下面查看有几个贡献者',
// );
// });

// test('douyin', async ({ page }) => {
// try {
// const outputPath = path.join(__dirname, 'output');
// await fs.rmdirSync(outputPath, { recursive: true });
// } catch (error) {}
// await page.goto('https://douyin.com');

// const playwrightPage = new PlaywrightWebPage(page);

// await loopAgent(page, playwrightPage, '搜索 黑神话，看两个视频');
// });
Loading