Skip to content

Commit

Permalink
test(feature dev): E2E test script post approach removal (aws#5596)
Browse files Browse the repository at this point in the history 
## Problem

E2E test is not up to date post approach removal

## Solution

- Removed tests related to approach
- Refactored the E2E test script
  • Loading branch information
@willyyhuang
willyyhuang authored Sep 16, 2024
1 parent 8416271 commit 54f0573
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 171 deletions.
264 changes: 93 additions & 171 deletions packages/amazonq/test/e2e/amazonq/featureDev.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,76 +6,20 @@
import assert from 'assert'
import { qTestingFramework } from './framework/framework'
import sinon from 'sinon'
import { verifyTextOrder } from './framework/text'
import { registerAuthHook, using } from 'aws-core-vscode/test'
import { loginToIdC } from './utils/setup'
import { Messenger } from './framework/messenger'
import { FollowUpTypes, examples } from 'aws-core-vscode/amazonqFeatureDev'
import { ChatItem } from '@aws/mynah-ui'
import { sleep } from 'aws-core-vscode/shared'

describe('Amazon Q Feature Dev', function () {
let framework: qTestingFramework
let tab: Messenger

const maxTestDuration = 600000
const prompt = 'Implement fibonacci in python'
const iterateApproachPrompt = prompt + ' and add a unit test'
const prompt = 'Add blank.txt file with empty content'
const codegenApproachPrompt = prompt + ' and add a readme that describes the changes'
const tooManyRequestsWaitTime = 100000

before(async function () {
/**
* The tests are getting throttled, only run them on stable for now
*
* TODO: Re-enable for all versions once the backend can handle them
*/
const testVersion = process.env['VSCODE_TEST_VERSION']
if (testVersion && testVersion !== 'stable') {
this.skip()
}

await using(registerAuthHook('amazonq-test-account'), async () => {
await loginToIdC()
})
})

beforeEach(() => {
registerAuthHook('amazonq-test-account')
framework = new qTestingFramework('featuredev', true)
tab = framework.createTab()
})

afterEach(() => {
framework.removeTab(tab.tabID)
framework.dispose()
sinon.restore()
})

describe('quick action availability', () => {
it('Shows /dev when feature dev is enabled', async () => {
const command = tab.findCommand('/dev')
if (!command) {
assert.fail('Could not find command')
}

if (command.length > 1) {
assert.fail('Found too many commands with the name /dev')
}
})

it('Does NOT show /dev when feature dev is NOT enabled', () => {
// The beforeEach registers a framework which accepts requests. If we don't dispose before building a new one we have duplicate messages
framework.dispose()
framework = new qTestingFramework('featuredev', false)
const tab = framework.createTab()
const command = tab.findCommand('/dev')
if (command.length > 0) {
assert.fail('Found command when it should not have been found')
}
})
})

function waitForButtons(buttons: FollowUpTypes[]) {
return tab.waitForEvent(() => {
return buttons.every((value) => tab.hasButton(value))
Expand All @@ -94,19 +38,6 @@ describe('Amazon Q Feature Dev', function () {
)
}

function verifyApproachState(chatItems: ChatItem[], expectedResponses: RegExp[]) {
// Verify that all the responses come back in the correct order
verifyTextOrder(chatItems, expectedResponses)

// Check that the UI has the two buttons
assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
{
type: FollowUpTypes.GenerateCode,
disabled: false,
},
])
}

async function iterate(prompt: string) {
tab.addChatMessage({ prompt })

Expand All @@ -115,9 +46,7 @@ describe('Amazon Q Feature Dev', function () {
// Wait for a backend response
await tab.waitForChatFinishesLoading()
},
() => {
tab.addChatMessage({ prompt })
}
() => {}
)
}

Expand Down Expand Up @@ -163,116 +92,79 @@ describe('Amazon Q Feature Dev', function () {
}
}

const functionalTests = () => {
afterEach(async function () {
// currentTest.state is undefined if a beforeEach fails
if (
this.currentTest?.state === undefined ||
this.currentTest?.isFailed() ||
this.currentTest?.isPending()
) {
// Since the tests are long running this may help in diagnosing the issue
console.log('Current chat items at failure')
console.log(JSON.stringify(tab.getChatItems(), undefined, 4))
}
})
before(async function () {
/**
* The tests are getting throttled, only run them on stable for now
*
* TODO: Re-enable for all versions once the backend can handle them
*/
const testVersion = process.env['VSCODE_TEST_VERSION']
if (testVersion && testVersion !== 'stable') {
this.skip()
}

it('Should receive chat response', async () => {
verifyApproachState(tab.getChatItems(), [new RegExp(prompt), /.\S/])
await using(registerAuthHook('amazonq-test-account'), async () => {
await loginToIdC()
})
})

describe('Moves directly from approach to codegen', () => {
codegenTests()
})
beforeEach(() => {
registerAuthHook('amazonq-test-account')
framework = new qTestingFramework('featuredev', true)
tab = framework.createTab()
})

describe('Iterates on approach', () => {
beforeEach(async function () {
this.timeout(maxTestDuration)
await iterate(iterateApproachPrompt)
})
afterEach(() => {
framework.removeTab(tab.tabID)
framework.dispose()
sinon.restore()
})

it('Should iterate successfully', () => {
verifyApproachState(tab.getChatItems(), [new RegExp(prompt), /.\S/])
})
describe('Quick action availability', () => {
it('Shows /dev when feature dev is enabled', async () => {
const command = tab.findCommand('/dev')
if (!command) {
assert.fail('Could not find command')
}

describe('Moves to codegen after iteration', () => {
codegenTests()
})
if (command.length > 1) {
assert.fail('Found too many commands with the name /dev')
}
})

function codegenTests() {
beforeEach(async function () {
this.timeout(maxTestDuration)
tab.clickButton(FollowUpTypes.GenerateCode)
await retryIfRequired(async () => {
await Promise.any([
waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
waitForButtons([FollowUpTypes.Retry]),
])
})
})

describe('Clicks accept code', () => {
insertCodeTests()
})

describe('Iterates on codegen', () => {
beforeEach(async function () {
this.timeout(maxTestDuration)
tab.clickButton(FollowUpTypes.ProvideFeedbackAndRegenerateCode)
await tab.waitForChatFinishesLoading()
await iterate(codegenApproachPrompt)
})

describe('Clicks accept code', () => {
insertCodeTests()
})
})
}

function insertCodeTests() {
beforeEach(async function () {
this.timeout(maxTestDuration)
tab.clickButton(FollowUpTypes.InsertCode)
await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
})

it('clicks new task', async () => {
tab.clickButton(FollowUpTypes.NewTask)
await waitForText('What change would you like to make?')
assert.deepStrictEqual(tab.getChatItems().pop()?.body, 'What change would you like to make?')
})

it('click close session', async () => {
tab.clickButton(FollowUpTypes.CloseSession)
await waitForText('Your session is now closed.')
assert.deepStrictEqual(tab.getPlaceholder(), 'Your session is now closed.')
})
}
}
it('Does NOT show /dev when feature dev is NOT enabled', () => {
// The beforeEach registers a framework which accepts requests. If we don't dispose before building a new one we have duplicate messages
framework.dispose()
framework = new qTestingFramework('featuredev', false)
const tab = framework.createTab()
const command = tab.findCommand('/dev')
if (command.length > 0) {
assert.fail('Found command when it should not have been found')
}
})
})

describe('/dev {msg} entry', async () => {
beforeEach(async function () {
this.timeout(maxTestDuration)
tab.addChatMessage({ command: '/dev', prompt })
describe('/dev entry', () => {
it('Clicks examples', async () => {
const q = framework.createTab()
q.addChatMessage({ command: '/dev' })
await retryIfRequired(
async () => {
await tab.waitForChatFinishesLoading()
await q.waitForChatFinishesLoading()
},
() => {
tab.addChatMessage({ prompt })
q.clickButton(FollowUpTypes.DevExamples)

const lastChatItems = q.getChatItems().pop()
assert.deepStrictEqual(lastChatItems?.body, examples)
}
)
})

functionalTests()
})

describe('/dev entry', () => {
describe('/dev {msg} entry', async () => {
beforeEach(async function () {
this.timeout(maxTestDuration)
tab.addChatMessage({ command: '/dev' })
tab.addChatMessage({ prompt })
tab.addChatMessage({ command: '/dev', prompt })
await retryIfRequired(
async () => {
await tab.waitForChatFinishesLoading()
Expand All @@ -283,15 +175,45 @@ describe('Amazon Q Feature Dev', function () {
)
})

it('Clicks examples', async () => {
const q = framework.createTab()
q.addChatMessage({ command: '/dev' })
q.clickButton(FollowUpTypes.DevExamples)
afterEach(async function () {
// currentTest.state is undefined if a beforeEach fails
if (
this.currentTest?.state === undefined ||
this.currentTest?.isFailed() ||
this.currentTest?.isPending()
) {
// Since the tests are long running this may help in diagnosing the issue
console.log('Current chat items at failure')
console.log(JSON.stringify(tab.getChatItems(), undefined, 4))
}
})

const lastChatItems = q.getChatItems().pop()
assert.deepStrictEqual(lastChatItems?.body, examples)
it('Clicks accept code and click new task', async () => {
await retryIfRequired(async () => {
await Promise.any([
waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
waitForButtons([FollowUpTypes.Retry]),
])
})
tab.clickButton(FollowUpTypes.InsertCode)
await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
tab.clickButton(FollowUpTypes.NewTask)
await waitForText('What new task would you like to work on?')
assert.deepStrictEqual(tab.getChatItems().pop()?.body, 'What new task would you like to work on?')
})

functionalTests()
it('Iterates on codegen', async () => {
await retryIfRequired(async () => {
await Promise.any([
waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
waitForButtons([FollowUpTypes.Retry]),
])
})
tab.clickButton(FollowUpTypes.ProvideFeedbackAndRegenerateCode)
await tab.waitForChatFinishesLoading()
await iterate(codegenApproachPrompt)
tab.clickButton(FollowUpTypes.InsertCode)
await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
})
})
})
6 changes: 6 additions & 0 deletions packages/amazonq/test/e2e/amazonq/framework/jsdomInjector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ export function injectJSDOM() {
global.HTMLElement = dom.window.HTMLElement
global.Node = dom.window.Node

global.ResizeObserver = class ResizeObserver {
observe() {}
unobserve() {}
disconnect() {}
}

// jsdom doesn't have support for innerText: https://github.com/jsdom/jsdom/issues/1245 which mynah ui uses
Object.defineProperty(global.Element.prototype, 'innerText', {
get() {
Expand Down

0 comments on commit 54f0573

Please sign in to comment.