feat(amazonq): scan support for non-workspace files aws#5050

Problem
Security scans should support scanning files outside of any workspace folder.

Solution
Include files outside of workspace folder in code artifact

packages/amazonq/.changes/next-release/Feature-0c3c8bb1-444a-4fd4-81d9-19059caad5db.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Security Scan: Support for scanning files outside of workspaces."
+}

packages/core/src/codewhisperer/activation.ts

@@ -315,7 +315,7 @@ export async function activate(context: ExtContext): Promise<void> {
             auth.isConnectionValid() &&
             !auth.isBuilderIdInUse() &&
             editor &&
-            vscode.workspace.getWorkspaceFolder(editor.document.uri) &&
+            editor.document.uri.scheme === 'file' &&
             securityScanLanguageContext.isLanguageSupported(editor.document.languageId)
         )
     }

packages/core/src/codewhisperer/models/constants.ts

@@ -301,9 +301,7 @@ export const FileSizeExceededErrorMessage = `Amazon Q: The selected file exceeds
 
 export const ProjectSizeExceededErrorMessage = `Amazon Q: The selected project exceeds the input artifact limit. Try again with a smaller project. For more information about scan limits, see the [Amazon Q documentation](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/security-scans.html#quotas).`
 
-export const NoWorkspaceFoundErrorMessage = 'Amazon Q: No workspace folders found'
-
-export const InvalidSourceFilesErrorMessage = 'Amazon Q: Project does not contain valid files to scan'
+export const noSourceFilesErrorMessage = 'Amazon Q: Project does not contain valid files to scan'
 
 export const UploadArtifactToS3ErrorMessage = `Amazon Q is unable to upload your workspace artifacts to Amazon S3 for security scans. For more information, see the [Amazon Q documentation](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/security_iam_manage-access-with-policies.html#data-perimeters).`
 

packages/core/src/codewhisperer/models/errors.ts

@@ -6,10 +6,9 @@ import { ToolkitError } from '../../shared/errors'
 import {
     DefaultCodeScanErrorMessage,
     FileSizeExceededErrorMessage,
-    NoWorkspaceFoundErrorMessage,
     ProjectSizeExceededErrorMessage,
-    InvalidSourceFilesErrorMessage,
     UploadArtifactToS3ErrorMessage,
+    noSourceFilesErrorMessage,
 } from './constants'
 
 export class SecurityScanError extends ToolkitError {
@@ -38,19 +37,13 @@ export class DefaultError extends SecurityScanError {
 
 export class InvalidSourceZipError extends SecurityScanError {
     constructor() {
-        super('Failed to create valid source zip', 'InvalidSourceFiles', InvalidSourceFilesErrorMessage)
+        super('Failed to create valid source zip', 'InvalidSourceZip', DefaultCodeScanErrorMessage)
     }
 }
 
-export class NoWorkspaceFolderFoundError extends SecurityScanError {
+export class NoSourceFilesError extends SecurityScanError {
     constructor() {
-        super('No workspace folders found', 'NoWorkspaceFound', NoWorkspaceFoundErrorMessage)
-    }
-}
-
-export class InvalidSourceFilesError extends SecurityScanError {
-    constructor() {
-        super('Project does not contain valid files to scan', 'InvalidSourceZip', DefaultCodeScanErrorMessage)
+        super('Project does not contain valid files to scan', 'NoSourceFilesError', noSourceFilesErrorMessage)
     }
 }
 

packages/core/src/codewhisperer/service/securityScanHandler.ts

@@ -5,7 +5,13 @@
 
 import { DefaultCodeWhispererClient } from '../client/codewhisperer'
 import { getLogger } from '../../shared/logger'
-import { AggregatedCodeScanIssue, CodeScansState, codeScanState, CodeScanStoppedError } from '../models/model'
+import {
+    AggregatedCodeScanIssue,
+    CodeScanIssue,
+    CodeScansState,
+    codeScanState,
+    CodeScanStoppedError,
+} from '../models/model'
 import { sleep } from '../../shared/utilities/timeoutUtils'
 import * as codewhispererClient from '../client/codewhisperer'
 import * as CodeWhispererConstants from '../models/constants'
@@ -67,31 +73,41 @@ export async function listScanResults(
             if (existsSync(filePath) && statSync(filePath).isFile()) {
                 const aggregatedCodeScanIssue: AggregatedCodeScanIssue = {
                     filePath: filePath,
-                    issues: issues.map(issue => {
-                        return {
-                            startLine: issue.startLine - 1 >= 0 ? issue.startLine - 1 : 0,
-                            endLine: issue.endLine,
-                            comment: `${issue.title.trim()}: ${issue.description.text.trim()}`,
-                            title: issue.title,
-                            description: issue.description,
-                            detectorId: issue.detectorId,
-                            detectorName: issue.detectorName,
-                            findingId: issue.findingId,
-                            ruleId: issue.ruleId,
-                            relatedVulnerabilities: issue.relatedVulnerabilities,
-                            severity: issue.severity,
-                            recommendation: issue.remediation.recommendation,
-                            suggestedFixes: issue.remediation.suggestedFixes,
-                        }
-                    }),
+                    issues: issues.map(mapRawToCodeScanIssue),
                 }
                 aggregatedCodeScanIssueList.push(aggregatedCodeScanIssue)
             }
         })
+        const maybeAbsolutePath = `/${key}`
+        if (existsSync(maybeAbsolutePath) && statSync(maybeAbsolutePath).isFile()) {
+            const aggregatedCodeScanIssue: AggregatedCodeScanIssue = {
+                filePath: maybeAbsolutePath,
+                issues: issues.map(mapRawToCodeScanIssue),
+            }
+            aggregatedCodeScanIssueList.push(aggregatedCodeScanIssue)
+        }
     })
     return aggregatedCodeScanIssueList
 }
 
+function mapRawToCodeScanIssue(issue: RawCodeScanIssue): CodeScanIssue {
+    return {
+        startLine: issue.startLine - 1 >= 0 ? issue.startLine - 1 : 0,
+        endLine: issue.endLine,
+        comment: `${issue.title.trim()}: ${issue.description.text.trim()}`,
+        title: issue.title,
+        description: issue.description,
+        detectorId: issue.detectorId,
+        detectorName: issue.detectorName,
+        findingId: issue.findingId,
+        ruleId: issue.ruleId,
+        relatedVulnerabilities: issue.relatedVulnerabilities,
+        severity: issue.severity,
+        recommendation: issue.remediation.recommendation,
+        suggestedFixes: issue.remediation.suggestedFixes,
+    }
+}
+
 function mapToAggregatedList(codeScanIssueMap: Map<string, RawCodeScanIssue[]>, json: string) {
     const codeScanIssues: RawCodeScanIssue[] = JSON.parse(json)
     codeScanIssues.forEach(issue => {

packages/core/src/codewhisperer/util/zipUtil.ts

@@ -14,12 +14,7 @@ import { getLoggerForScope } from '../service/securityScanHandler'
 import { runtimeLanguageContext } from './runtimeLanguageContext'
 import { CodewhispererLanguage } from '../../shared/telemetry/telemetry.gen'
 import { CurrentWsFolders, collectFiles } from '../../shared/utilities/workspaceUtils'
-import {
-    FileSizeExceededError,
-    InvalidSourceFilesError,
-    NoWorkspaceFolderFoundError,
-    ProjectSizeExceededError,
-} from '../models/errors'
+import { FileSizeExceededError, NoSourceFilesError, ProjectSizeExceededError } from '../models/errors'
 
 export interface ZipMetadata {
     rootDir: string
@@ -35,7 +30,6 @@ export interface ZipMetadata {
 export const ZipConstants = {
     newlineRegex: /\r?\n/,
     gitignoreFilename: '.gitignore',
-    javaBuildExt: '.class',
 }
 
 export class ZipUtil {
@@ -61,10 +55,7 @@ export class ZipUtil {
 
     public getProjectPaths() {
         const workspaceFolders = vscode.workspace.workspaceFolders
-        if (!workspaceFolders || workspaceFolders.length === 0) {
-            throw new NoWorkspaceFolderFoundError()
-        }
-        return workspaceFolders.map(folder => folder.uri.fsPath)
+        return workspaceFolders?.map(folder => folder.uri.fsPath) ?? []
     }
 
     protected async getTextContent(uri: vscode.Uri) {
@@ -73,10 +64,6 @@ export class ZipUtil {
         return content
     }
 
-    public isJavaClassFile(uri: vscode.Uri) {
-        return uri.fsPath.endsWith(ZipConstants.javaBuildExt)
-    }
-
     public reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean {
         if (scope === CodeWhispererConstants.CodeAnalysisScope.FILE) {
             return size > this.getFileScanPayloadSizeLimitInBytes()
@@ -99,13 +86,14 @@ export class ZipUtil {
         const content = await this.getTextContent(uri)
 
         const workspaceFolder = vscode.workspace.getWorkspaceFolder(uri)
-        if (!workspaceFolder) {
-            throw Error('No workspace folder found')
+        if (workspaceFolder) {
+            const projectName = workspaceFolder.name
+            const relativePath = vscode.workspace.asRelativePath(uri)
+            const zipEntryPath = this.getZipEntryPath(projectName, relativePath)
+            zip.addFile(zipEntryPath, Buffer.from(content, 'utf-8'))
+        } else {
+            zip.addFile(uri.fsPath, Buffer.from(content, 'utf-8'))
         }
-        const projectName = workspaceFolder.name
-        const relativePath = vscode.workspace.asRelativePath(uri)
-        const zipEntryPath = this.getZipEntryPath(projectName, relativePath)
-        zip.addFile(zipEntryPath, Buffer.from(content, 'utf-8'))
 
         this._pickedSourceFiles.add(uri.fsPath)
         this._totalSize += (await fsCommon.stat(uri.fsPath)).size
@@ -131,56 +119,81 @@ export class ZipUtil {
         const zip = new admZip()
 
         const projectPaths = this.getProjectPaths()
+        const languageCount = new Map<CodewhispererLanguage, number>()
+
+        await this.processSourceFiles(zip, languageCount, projectPaths)
+        this.processOtherFiles(zip, languageCount)
+
+        if (languageCount.size === 0) {
+            throw new NoSourceFilesError()
+        }
+        this._language = [...languageCount.entries()].reduce((a, b) => (b[1] > a[1] ? b : a))[0]
+        const zipFilePath = this.getZipDirPath() + CodeWhispererConstants.codeScanZipExt
+        zip.writeZip(zipFilePath)
+        return zipFilePath
+    }
 
-        const files = await collectFiles(
+    protected async processSourceFiles(
+        zip: admZip,
+        languageCount: Map<CodewhispererLanguage, number>,
+        projectPaths: string[] | undefined
+    ) {
+        if (!projectPaths || projectPaths.length === 0) {
+            return
+        }
+
+        const sourceFiles = await collectFiles(
             projectPaths,
             vscode.workspace.workspaceFolders as CurrentWsFolders,
             true,
-            CodeWhispererConstants.projectScanPayloadSizeLimitBytes
+            this.getProjectScanPayloadSizeLimitInBytes()
         )
-        const languageCount = new Map<CodewhispererLanguage, number>()
-        for (const file of files) {
+        for (const file of sourceFiles) {
             const isFileOpenAndDirty = this.isFileOpenAndDirty(file.fileUri)
             const fileContent = isFileOpenAndDirty ? await this.getTextContent(file.fileUri) : file.fileContent
-
-            const fileSize = Buffer.from(fileContent).length
-            if (this.isJavaClassFile(file.fileUri)) {
-                this._pickedBuildFiles.add(file.fileUri.fsPath)
-                this._totalBuildSize += fileSize
-            } else {
-                if (
-                    this.reachSizeLimit(this._totalSize, CodeWhispererConstants.CodeAnalysisScope.PROJECT) ||
-                    this.willReachSizeLimit(this._totalSize, fileSize)
-                ) {
-                    throw new ProjectSizeExceededError()
-                }
-                this._pickedSourceFiles.add(file.fileUri.fsPath)
-                this._totalSize += fileSize
-                this._totalLines += fileContent.split(ZipConstants.newlineRegex).length
-
-                const fileExtension = path.extname(file.fileUri.fsPath).slice(1)
-                const language = runtimeLanguageContext.getLanguageFromFileExtension(fileExtension)
-                if (language && language !== 'plaintext') {
-                    languageCount.set(language, (languageCount.get(language) || 0) + 1)
-                }
-            }
-
             const zipEntryPath = this.getZipEntryPath(file.workspaceFolder.name, file.zipFilePath)
+            this.processFile(zip, file.fileUri, fileContent, languageCount, zipEntryPath)
+        }
+    }
 
-            if (isFileOpenAndDirty) {
-                zip.addFile(zipEntryPath, Buffer.from(fileContent, 'utf-8'))
-            } else {
-                zip.addLocalFile(file.fileUri.fsPath, path.dirname(zipEntryPath))
-            }
+    protected processOtherFiles(zip: admZip, languageCount: Map<CodewhispererLanguage, number>) {
+        vscode.workspace.textDocuments
+            .filter(document => document.uri.scheme === 'file')
+            .filter(document => vscode.workspace.getWorkspaceFolder(document.uri) === undefined)
+            .forEach(document =>
+                this.processFile(zip, document.uri, document.getText(), languageCount, document.uri.fsPath)
+            )
+    }
+
+    protected processFile(
+        zip: admZip,
+        uri: vscode.Uri,
+        fileContent: string,
+        languageCount: Map<CodewhispererLanguage, number>,
+        zipEntryPath: string
+    ) {
+        const fileSize = Buffer.from(fileContent).length
+
+        if (
+            this.reachSizeLimit(this._totalSize, CodeWhispererConstants.CodeAnalysisScope.PROJECT) ||
+            this.willReachSizeLimit(this._totalSize, fileSize)
+        ) {
+            throw new ProjectSizeExceededError()
         }
+        this._pickedSourceFiles.add(uri.fsPath)
+        this._totalSize += fileSize
+        this._totalLines += fileContent.split(ZipConstants.newlineRegex).length
 
-        if (languageCount.size === 0) {
-            throw new InvalidSourceFilesError()
+        this.incrementCountForLanguage(uri, languageCount)
+        zip.addFile(zipEntryPath, Buffer.from(fileContent, 'utf-8'))
+    }
+
+    protected incrementCountForLanguage(uri: vscode.Uri, languageCount: Map<CodewhispererLanguage, number>) {
+        const fileExtension = path.extname(uri.fsPath).slice(1)
+        const language = runtimeLanguageContext.getLanguageFromFileExtension(fileExtension)
+        if (language && language !== 'plaintext') {
+            languageCount.set(language, (languageCount.get(language) || 0) + 1)
         }
-        this._language = [...languageCount.entries()].reduce((a, b) => (b[1] > a[1] ? b : a))[0]
-        const zipFilePath = this.getZipDirPath() + CodeWhispererConstants.codeScanZipExt
-        zip.writeZip(zipFilePath)
-        return zipFilePath
     }
 
     protected isFileOpenAndDirty(uri: vscode.Uri) {

packages/core/src/test/codewhisperer/commands/startSecurityScan.test.ts

@@ -296,7 +296,6 @@ describe('startSecurityScan', function () {
             CodeAnalysisScope.PROJECT
         )
         assertTelemetry('codewhisperer_securityScan', {
-            codewhispererLanguage: 'yaml',
             codewhispererCodeScanTotalIssues: 1,
             codewhispererCodeScanIssuesWithFixes: 0,
             codewhispererCodeScanScope: 'PROJECT',

packages/core/src/test/codewhisperer/service/securityScanHandler.test.ts

@@ -16,7 +16,7 @@ import fs from 'fs'
 
 const mockCodeScanFindings = JSON.stringify([
     {
-        filePath: '/workspaceFolder/python3.7-plain-sam-app/hello_world/app.py',
+        filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py',
         startLine: 1,
         endLine: 1,
         title: 'title',
@@ -85,8 +85,9 @@ describe('securityScanHandler', function () {
                 CodeAnalysisScope.PROJECT
             )
 
-            assert.equal(aggregatedCodeScanIssueList.length, 1)
+            assert.equal(aggregatedCodeScanIssueList.length, 2)
             assert.equal(aggregatedCodeScanIssueList[0].issues.length, 1)
+            assert.equal(aggregatedCodeScanIssueList[1].issues.length, 1)
         })
 
         it('should handle ListCodeScanFindings request with paginated response', async function () {
@@ -106,7 +107,7 @@ describe('securityScanHandler', function () {
                 CodeAnalysisScope.PROJECT
             )
 
-            assert.equal(aggregatedCodeScanIssueList.length, 1)
+            assert.equal(aggregatedCodeScanIssueList.length, 2)
             assert.equal(aggregatedCodeScanIssueList[0].issues.length, 3)
         })
     })

packages/core/src/test/codewhisperer/util/zipUtil.test.ts

@@ -83,27 +83,6 @@ describe('zipUtil', function () {
             )
         })
 
-        it('Should include java .class files', async function () {
-            const isClassFileStub = sinon
-                .stub(zipUtil, 'isJavaClassFile')
-                .onFirstCall()
-                .returns(true)
-                .onSecondCall()
-                .callsFake((...args) => {
-                    isClassFileStub.restore()
-                    return zipUtil.isJavaClassFile(...args)
-                })
-
-            const zipMetadata = await zipUtil.generateZip(vscode.Uri.file(appCodePath), CodeAnalysisScope.PROJECT)
-            assert.ok(zipMetadata.lines > 0)
-            assert.ok(zipMetadata.rootDir.includes(CodeWhispererConstants.codeScanTruncDirPrefix))
-            assert.ok(zipMetadata.srcPayloadSizeInBytes > 0)
-            assert.ok(zipMetadata.scannedFiles.size > 0)
-            assert.ok(zipMetadata.buildPayloadSizeInBytes > 0)
-            assert.ok(zipMetadata.zipFileSizeInBytes > 0)
-            assert.ok(zipMetadata.zipFilePath.includes(CodeWhispererConstants.codeScanTruncDirPrefix))
-        })
-
         it('Should throw error if scan type is invalid', async function () {
             await assert.rejects(
                 () => zipUtil.generateZip(vscode.Uri.file(appCodePath), 'unknown' as CodeAnalysisScope),