Skip to content

Commit

Permalink
fix(amazonq): unsupported languages test generation query (aws#6363)
Browse files Browse the repository at this point in the history
## Problem
- The test generation for unsupported language files and external files
is unreliable and flaky.
- Telemetry updated to include new metadata for existing metadata.

## Solution
- Fix to add file name as fallback in case of empty prompt or no code
selection.
- Telemetry updated and values passed to it in controller and messenger.
Essentially same PR :
aws#6323 was closed last week
due to build issues.
---
Before:

https://github.com/user-attachments/assets/90ed6629-4753-41e6-9d11-b6d4dd0d6e7c

After fix:


https://github.com/user-attachments/assets/d0993345-cd2c-47ac-a8b5-1e9badf8fbac




- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Co-authored-by: Justin M. Keyes <[email protected]>
  • Loading branch information
ashishrp-aws and justinmk3 authored Jan 15, 2025
1 parent 12256f7 commit 73c0926
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 11 deletions.
8 changes: 4 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"mergeReports": "ts-node ./scripts/mergeReports.ts"
},
"devDependencies": {
"@aws-toolkits/telemetry": "^1.0.289",
"@aws-toolkits/telemetry": "^1.0.293",
"@playwright/browser-chromium": "^1.43.1",
"@stylistic/eslint-plugin": "^2.11.0",
"@types/he": "^1.2.3",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "Bug Fix",
"description": "/test: for unsupported languages was sometimes unreliable"
}
22 changes: 17 additions & 5 deletions packages/core/src/amazonqTest/chat/controller/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ export class TestController {
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
true,
isCancel ? 'Cancelled' : 'Failed',
session.startTestGenerationRequestId,
performance.now() - session.testGenerationStartTime,
Expand Down Expand Up @@ -456,7 +457,14 @@ export class TestController {
unsupportedMessage = `<span style="color: #EE9D28;">&#9888;<b>I'm sorry, but /test only supports Python and Java</b><br></span> I will still generate a suggestion below.`
}
this.messenger.sendMessage(unsupportedMessage, tabID, 'answer')
await this.onCodeGeneration(session, message.prompt, tabID, fileName, filePath)
await this.onCodeGeneration(
session,
message.prompt,
tabID,
fileName,
filePath,
workspaceFolder !== undefined
)
} else {
this.messenger.sendCapabilityCard({ tabID })
this.messenger.sendMessage(testGenSummaryMessage(fileName), message.tabID, 'answer-part')
Expand Down Expand Up @@ -722,6 +730,7 @@ export class TestController {
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
true,
'Succeeded',
session.startTestGenerationRequestId,
session.latencyOfTestGeneration,
Expand Down Expand Up @@ -799,20 +808,21 @@ export class TestController {
message: string,
tabID: string,
fileName: string,
filePath: string
filePath: string,
fileInWorkspace: boolean
) {
try {
// TODO: Write this entire gen response to basiccommands and call here.
const editorText = await fs.readFileText(filePath)

const triggerPayload = {
query: `Generate unit tests for the following part of my code: ${message}`,
query: `Generate unit tests for the following part of my code: ${message?.trim() || fileName}`,
codeSelection: undefined,
trigger: ChatTriggerType.ChatMessage,
fileText: editorText,
fileLanguage: session.fileLanguage,
filePath: filePath,
message: `Generate unit tests for the following part of my code: ${message}`,
message: `Generate unit tests for the following part of my code: ${message?.trim() || fileName}`,
matchPolicy: undefined,
codeQuery: undefined,
userIntent: UserIntent.GENERATE_UNIT_TESTS,
Expand All @@ -827,7 +837,8 @@ export class TestController {
tabID,
randomUUID.toString(),
triggerPayload,
fileName
fileName,
fileInWorkspace
)
} finally {
this.messenger.sendChatInputEnabled(tabID, true)
Expand All @@ -845,6 +856,7 @@ export class TestController {
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
true,
'Succeeded',
session.startTestGenerationRequestId,
session.latencyOfTestGeneration,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,8 @@ export class Messenger {
tabID: string,
triggerID: string,
triggerPayload: TriggerPayload,
fileName: string
fileName: string,
fileInWorkspace: boolean
) {
let message = ''
let messageId = response.$metadata.requestId ?? ''
Expand Down Expand Up @@ -277,6 +278,7 @@ export class Messenger {
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
false,
fileInWorkspace,
'Cancelled',
messageId,
performance.now() - session.testGenerationStartTime,
Expand All @@ -291,6 +293,7 @@ export class Messenger {
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
false,
fileInWorkspace,
'Succeeded',
messageId,
performance.now() - session.testGenerationStartTime
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/codewhisperer/util/telemetryHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export class TelemetryHelper {
public sendTestGenerationToolkitEvent(
session: Session,
isSupportedLanguage: boolean,
isFileInWorkspace: boolean,
result: 'Succeeded' | 'Failed' | 'Cancelled',
requestId?: string,
perfClientLatency?: number,
Expand All @@ -90,6 +91,7 @@ export class TelemetryHelper {
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
hasUserPromptSupplied: session.hasUserPromptSupplied,
isSupportedLanguage: isSupportedLanguage,
isFileInWorkspace: isFileInWorkspace,
result: result,
artifactsUploadDuration: artifactsUploadDuration,
buildPayloadBytes: buildPayloadBytes,
Expand Down

0 comments on commit 73c0926

Please sign in to comment.