diff --git a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml index 85460cd9dbbc..ff68f1737abc 100644 --- a/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml +++ b/eng/pipelines/templates/stages/archetype-spec-gen-sdk.yml @@ -27,11 +27,13 @@ parameters: extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: + Use1ESOfficial: false stages: - stage: Build displayName: 'SDK Generation' jobs: - job: + timeoutInMinutes: 2400 variables: - template: /eng/pipelines/templates/variables/image.yml @@ -56,12 +58,12 @@ extends: displayName: Publish SDK artifacts to Pipeline Artifacts condition: and(ne(variables['ValidationResult'], ''), eq(variables['HasSDKArtifact'], 'true')) artifactName: $(sdkArtifactName) - targetPath: "$(System.DefaultWorkingDirectory)/out/generatedSdkArtifacts" + targetPath: "$(System.DefaultWorkingDirectory)/out/stagedArtifacts" - output: pipelineArtifact displayName: Publish API View artifacts to Pipeline Artifacts condition: and(ne(variables['ValidationResult'], ''), eq(variables['HasApiViewArtifact'], 'true')) artifactName: $(ArtifactName) - targetPath: "$(System.DefaultWorkingDirectory)/out/sdkApiViewArtifacts" + targetPath: "$(System.DefaultWorkingDirectory)/out/stagedArtifacts" - output: pipelineArtifact displayName: Publish logs to Pipeline Artifacts condition: ne(variables['ValidationResult'], '') @@ -168,7 +170,7 @@ extends: optional_params="" sdk_gen_info="sdk generation from Config : " - if [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then + if [ "$(Build.Reason)" != "PullRequest" ] && [ "${{ parameters.ConfigType }}" = "TypeSpec" ]; then optional_params="$optional_params --tsp-config-relative-path ${{ parameters.ConfigPath }}" sdk_gen_info="$sdk_gen_info '${{ parameters.ConfigPath }}'," elif [ "${{ parameters.ConfigType }}" = "OpenAPI" ]; then @@ -177,7 +179,7 @@ extends: fi if [ "$(Build.Reason)" = "PullRequest" ]; then - optional_params="$optional_params --pr-number=$(System.PullRequest.PullRequestNumber)" + optional_params="$optional_params --pr-number $(System.PullRequest.PullRequestNumber)" specPrUrl="${{ parameters.SpecRepoUrl }}/pull/$(System.PullRequest.PullRequestNumber)" sdk_gen_info="$sdk_gen_info spec PR: $specPrUrl" fi diff --git a/eng/tools/spec-gen-sdk-runner/cmd/spec-gen-sdk-runner.js b/eng/tools/spec-gen-sdk-runner/cmd/spec-gen-sdk-runner.js old mode 100644 new mode 100755 diff --git a/eng/tools/spec-gen-sdk-runner/eslint.config.js b/eng/tools/spec-gen-sdk-runner/eslint.config.js index e2399d13ebaf..d7d2b6b9c373 100644 --- a/eng/tools/spec-gen-sdk-runner/eslint.config.js +++ b/eng/tools/spec-gen-sdk-runner/eslint.config.js @@ -68,27 +68,23 @@ const config = tseslint.config( "@typescript-eslint/restrict-template-expressions": "off", "@typescript-eslint/no-unsafe-assignment": "off", "@typescript-eslint/no-unsafe-argument": "off", + "@typescript-eslint/consistent-indexed-object-style": "off", + "@typescript-eslint/no-unnecessary-condition": "off", + "@typescript-eslint/consistent-type-definitions": "off", + "@typescript-eslint/no-inferrable-types": "off", + "@typescript-eslint/no-dynamic-delete": "off", // We want more flexibility with file names. // https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/filename-case.md "unicorn/filename-case": "off", "unicorn/prefer-ternary": "off", + "unicorn/no-useless-undefined": "off", + "unicorn/prevent-abbreviations": "off", // We prefer to have explicitly import at the top of the file, even if the same element is exported again, // which we do in index.ts files. // https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/prefer-export-from.md "unicorn/prefer-export-from": ["error", { ignoreUsedVariables: true }], - - // We allow some abbreviations that we like. - // https://github.com/sindresorhus/eslint-plugin-unicorn/blob/main/docs/rules/prevent-abbreviations.md - "unicorn/prevent-abbreviations": [ - "error", - { - allowList: { - args: true, - }, - }, - ], }, } ); diff --git a/eng/tools/spec-gen-sdk-runner/src/change-files.ts b/eng/tools/spec-gen-sdk-runner/src/change-files.ts new file mode 100644 index 000000000000..ba9ef68e7ae8 --- /dev/null +++ b/eng/tools/spec-gen-sdk-runner/src/change-files.ts @@ -0,0 +1,184 @@ +import path from "node:path"; +import { + getChangedFiles, + searchRelatedParentFolders, + searchSharedLibrary, + searchRelatedTypeSpecProjectBySharedLibrary, + groupPathsByService, + createCombinedSpecs, + type SpecResults, + type ChangedSpecs, + getLastPathSegment, +} from "./utils.js"; +import { logMessage } from "./log.js"; +import { SpecGenSdkCmdInput } from "./types.js"; + +const readmeMdRegex = /^readme.md$/i; +const typespecProjectRegex = /^tspconfig.yaml$/i; +const typespecProjectSharedLibraryRegex = /[^/]+\.Shared/; + +export function detectChangedSpecConfigFiles(commandInput: SpecGenSdkCmdInput): ChangedSpecs[] { + const prChangedFiles: string[] = getChangedFiles(commandInput.localSpecRepoPath) ?? []; + if (prChangedFiles.length === 0) { + logMessage("No files changed in the PR"); + } + logMessage(`Changed files in the PR: ${prChangedFiles.length}`); + for (const file of prChangedFiles) { + logMessage(`\t${file}`); + } + const fileList = prChangedFiles + .filter((p) => p.startsWith("specification/")) + .filter((p) => !p.includes("/scenarios/")); + + logMessage(`Related readme.md and typespec project list:`); + const changedSpecs: ChangedSpecs[] = []; + + const readmeMDResult = searchRelatedParentFolders(fileList, { + searchFileRegex: readmeMdRegex, + specRepoFolder: commandInput.localSpecRepoPath, + stopAtFolder: "specification", + }); + + const typespecProjectResult = searchRelatedParentFolders(fileList, { + searchFileRegex: typespecProjectRegex, + specRepoFolder: commandInput.localSpecRepoPath, + stopAtFolder: "specification", + }); + + const typespecProjectSharedLibraries = searchSharedLibrary(fileList, { + searchFileRegex: typespecProjectSharedLibraryRegex, + specRepoFolder: commandInput.localSpecRepoPath, + }); + + const typespecProjectResultSearchedBySharedLibrary = searchRelatedTypeSpecProjectBySharedLibrary( + typespecProjectSharedLibraries, + { + searchFileRegex: typespecProjectRegex, + specRepoFolder: commandInput.localSpecRepoPath, + }, + ); + + // Merge typespec project results + for (const folderPath of Object.keys(typespecProjectResultSearchedBySharedLibrary)) { + if (typespecProjectResult[folderPath]) { + typespecProjectResult[folderPath] = [ + ...typespecProjectResult[folderPath], + ...typespecProjectResultSearchedBySharedLibrary[folderPath], + ]; + } else { + typespecProjectResult[folderPath] = typespecProjectResultSearchedBySharedLibrary[folderPath]; + } + } + + // Group paths by service + const serviceMap = groupPathsByService(readmeMDResult, typespecProjectResult); + + const results: SpecResults = { readmeMDResult, typespecProjectResult }; + + // Process each service + for (const [, info] of serviceMap) { + // Case: Resource Manager with .Management + if (info.managementPaths.length > 0) { + if (info.resourceManagerPaths.length === 1) { + // Single resource-manager path - match with all Management paths + const newSpecs = createCombinedSpecs( + info.resourceManagerPaths[0].path, + info.managementPaths, + results, + ); + changedSpecs.push(...newSpecs); + logMessage( + `\t readme folders: ${info.resourceManagerPaths[0].path}, tspconfig folders: ${info.managementPaths}`, + ); + for (const p of info.managementPaths) { + delete typespecProjectResult[p]; + } + delete readmeMDResult[info.resourceManagerPaths[0].path]; + } else { + // Multiple resource-manager paths - match by subfolder name + for (const rmPath of info.resourceManagerPaths) { + const matchingManagements = info.managementPaths.filter((mPath) => { + const rmSubPath = rmPath.subPath; + const managementName = getLastPathSegment(mPath).replace(".Management", ""); + return rmSubPath && rmSubPath === managementName; + }); + if (matchingManagements.length > 0) { + const newSpecs = createCombinedSpecs(rmPath.path, matchingManagements, results); + changedSpecs.push(...newSpecs); + logMessage( + `\t readme folders: ${rmPath.path}, tspconfig folders: ${matchingManagements}`, + ); + for (const p of matchingManagements) { + delete typespecProjectResult[p]; + } + delete readmeMDResult[rmPath.path]; + } + } + } + } + + // Case: Data Plane matching + if (info.dataPlanePaths.length > 0 && info.otherTypeSpecPaths.length > 0) { + if (info.dataPlanePaths.length === 1) { + // Single data-plane path - match with all non-Management TypeSpec paths + const newSpecs = createCombinedSpecs( + info.dataPlanePaths[0].path, + info.otherTypeSpecPaths, + results, + ); + changedSpecs.push(...newSpecs); + logMessage( + `\t readme folders: ${info.dataPlanePaths[0].path}, tspconfig folders: ${info.otherTypeSpecPaths}`, + ); + for (const p of info.otherTypeSpecPaths) { + delete typespecProjectResult[p]; + } + delete readmeMDResult[info.dataPlanePaths[0].path]; + } else { + // Multiple data-plane paths - match by subfolder name + for (const dpPath of info.dataPlanePaths) { + const matchingTypeSpecs = info.otherTypeSpecPaths.filter((tsPath) => { + const dpSubFolder = dpPath.subFolder; + const tsLastSegment = getLastPathSegment(tsPath); + return dpSubFolder && dpSubFolder === tsLastSegment; + }); + if (matchingTypeSpecs.length > 0) { + const newSpecs = createCombinedSpecs(dpPath.path, matchingTypeSpecs, results); + changedSpecs.push(...newSpecs); + logMessage( + `\t readme folders: ${dpPath.path}, tspconfig folders: ${matchingTypeSpecs}`, + ); + for (const p of matchingTypeSpecs) { + delete typespecProjectResult[p]; + } + delete readmeMDResult[dpPath.path]; + } + } + } + } + } + + // Process remaining unmatched paths + for (const folderPath of new Set([ + ...Object.keys(readmeMDResult), + ...Object.keys(typespecProjectResult), + ])) { + const cs: ChangedSpecs = { + specs: [], + }; + + if (typespecProjectResult[folderPath]) { + cs.specs = typespecProjectResult[folderPath]; + cs.typespecProject = path.join(folderPath, "tspconfig.yaml"); + logMessage(`\t tspconfig: ${cs.typespecProject}`); + } else { + cs.readmeMd = path.join(folderPath, "readme.md"); + cs.specs = readmeMDResult[folderPath]; + logMessage(`\t readme: ${cs.readmeMd}`); + } + + changedSpecs.push(cs); + } + + return changedSpecs; +} diff --git a/eng/tools/spec-gen-sdk-runner/src/commands.ts b/eng/tools/spec-gen-sdk-runner/src/commands.ts index 0c6a498a0c49..012a837b66b0 100644 --- a/eng/tools/spec-gen-sdk-runner/src/commands.ts +++ b/eng/tools/spec-gen-sdk-runner/src/commands.ts @@ -6,9 +6,11 @@ import { getArgumentValue, runSpecGenSdkCommand, getAllTypeSpecPaths, + resetGitRepo, } from "./utils.js"; import { LogLevel, logMessage, vsoAddAttachment } from "./log.js"; import { SpecGenSdkCmdInput } from "./types.js"; +import { detectChangedSpecConfigFiles } from "./change-files.js"; export async function generateSdkForSingleSpec(): Promise { // Parse the arguments @@ -44,8 +46,71 @@ export async function generateSdkForSingleSpec(): Promise { return statusCode; } +/* Generate SDKs for spec pull request */ +export async function generateSdkForSpecPr(): Promise { + // Parse the arguments + const commandInput: SpecGenSdkCmdInput = parseArguments(); + // Construct the spec-gen-sdk command + const specGenSdkCommand = prepareSpecGenSdkCommand(commandInput); + // Get the spec paths from the changed files + const changedSpecs = detectChangedSpecConfigFiles(commandInput); + + let statusCode = 0; + let pushedSpecConfigCount; + for (const changedSpec of changedSpecs) { + if (!changedSpec.typespecProject && !changedSpec.readmeMd) { + logMessage("No spec config file found in the changed files", LogLevel.Warn); + continue; + } + pushedSpecConfigCount = 0; + if (changedSpec.typespecProject) { + specGenSdkCommand.push("--tsp-config-relative-path", changedSpec.typespecProject); + pushedSpecConfigCount++; + } + if (changedSpec.readmeMd) { + specGenSdkCommand.push("--readme-relative-path", changedSpec.readmeMd); + pushedSpecConfigCount++; + } + const changedSpecPath = changedSpec.typespecProject ?? changedSpec.readmeMd; + logMessage(`Generating SDK from ${changedSpecPath}`, LogLevel.Group); + logMessage(`Command:${specGenSdkCommand.join(" ")}`); + + try { + await resetGitRepo(commandInput.localSdkRepoPath); + await runSpecGenSdkCommand(specGenSdkCommand); + logMessage("Command executed successfully"); + } catch (error) { + logMessage(`Error executing command:${error}`, LogLevel.Error); + statusCode = 1; + } + // Pop the spec config path from specGenSdkCommand + for (let index = 0; index < pushedSpecConfigCount * 2; index++) { + specGenSdkCommand.pop(); + } + // Read the execution report to determine if the generation was successful + const executionReportPath = path.join( + commandInput.workingFolder, + `${commandInput.sdkRepoName}_tmp/execution-report.json`, + ); + try { + const executionReport = JSON.parse(fs.readFileSync(executionReportPath, "utf8")); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const executionResult = executionReport.executionResult; + logMessage(`Execution Result:${executionResult}`); + } catch (error) { + logMessage( + `Error reading execution report at ${executionReportPath}:${error}`, + LogLevel.Error, + ); + statusCode = 1; + } + logMessage("ending group logging", LogLevel.EndGroup); + } + return statusCode; +} + /** - * Generate SDKs for all specs. + * Generate SDKs for batch specs. */ export async function generateSdkForBatchSpecs(runMode: string): Promise { // Parse the arguments @@ -60,9 +125,9 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise let markdownContent = "\n"; let failedContent = `## Spec Failures in the Generation Process\n`; let succeededContent = `## Successful Specs in the Generation Process\n`; - let undefinedContent = `## Disabled Specs in the Generation Process\n`; + let notEnabledContent = `## Specs with SDK Not Enabled\n`; let failedCount = 0; - let undefinedCount = 0; + let notEnabledCount = 0; let succeededCount = 0; // Generate SDKs for each spec @@ -75,6 +140,7 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise } logMessage(`Command:${specGenSdkCommand.join(" ")}`); try { + await resetGitRepo(commandInput.localSdkRepoPath); await runSpecGenSdkCommand(specGenSdkCommand); logMessage("Command executed successfully"); } catch (error) { @@ -96,12 +162,12 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise const executionResult = executionReport.executionResult; logMessage(`Execution Result:${executionResult}`); - if (executionResult === "succeeded") { + if (executionResult === "succeeded" || executionResult === "warning") { succeededContent += `${specConfigPath},`; succeededCount++; - } else if (executionResult === undefined) { - undefinedContent += `${specConfigPath},`; - undefinedCount++; + } else if (executionResult === "notEnabled") { + notEnabledContent += `${specConfigPath},`; + notEnabledCount++; } else { failedContent += `${specConfigPath},`; failedCount++; @@ -118,15 +184,15 @@ export async function generateSdkForBatchSpecs(runMode: string): Promise if (failedCount > 0) { markdownContent += `${failedContent}\n`; } - if (undefinedCount > 0) { - markdownContent += `${undefinedContent}\n`; + if (notEnabledCount > 0) { + markdownContent += `${notEnabledContent}\n`; } if (succeededCount > 0) { markdownContent += `${succeededContent}\n`; } markdownContent += failedCount ? `## Total Failed Specs\n ${failedCount}\n` : ""; - markdownContent += undefinedCount - ? `## Total Disabled Specs in the Configuration\n ${undefinedCount}\n` + markdownContent += notEnabledCount + ? `## Total Specs with SDK not enabled in the Configuration\n ${notEnabledCount}\n` : ""; markdownContent += succeededCount ? `## Total Successful Specs\n ${succeededCount}\n` : ""; markdownContent += `## Total Specs Count\n ${specConfigPaths.length}\n\n`; @@ -251,7 +317,10 @@ function getSpecPaths(runMode: string, specRepoPath: string): string[] { break; } case "sample-typespecs": { - specConfigPaths.push("specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml"); + specConfigPaths.push( + "specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml", + "specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml", + ); } } return specConfigPaths; diff --git a/eng/tools/spec-gen-sdk-runner/src/index.ts b/eng/tools/spec-gen-sdk-runner/src/index.ts index b25a78a36701..9b1914d17d1e 100644 --- a/eng/tools/spec-gen-sdk-runner/src/index.ts +++ b/eng/tools/spec-gen-sdk-runner/src/index.ts @@ -1,14 +1,23 @@ import { exit } from "node:process"; import { getArgumentValue } from "./utils.js"; -import { generateSdkForBatchSpecs, generateSdkForSingleSpec } from "./commands.js"; +import { + generateSdkForBatchSpecs, + generateSdkForSingleSpec, + generateSdkForSpecPr, +} from "./commands.js"; export async function main() { // Get the arguments passed to the script const args: string[] = process.argv.slice(2); + // Log the arguments to the console + console.log("Arguments passed to the script:", args.join(" ")); const runMode: string = getArgumentValue(args, "--rm", ""); + const pullRequestNumber: string = getArgumentValue(args, "--pr-number", ""); let statusCode = 0; if (runMode) { statusCode = await generateSdkForBatchSpecs(runMode); + } else if (pullRequestNumber) { + statusCode = await generateSdkForSpecPr(); } else { statusCode = await generateSdkForSingleSpec(); } diff --git a/eng/tools/spec-gen-sdk-runner/src/utils.ts b/eng/tools/spec-gen-sdk-runner/src/utils.ts index 2041914a9368..1458c410e1e3 100644 --- a/eng/tools/spec-gen-sdk-runner/src/utils.ts +++ b/eng/tools/spec-gen-sdk-runner/src/utils.ts @@ -1,11 +1,36 @@ -import { spawn, spawnSync } from "node:child_process"; +import { spawn, spawnSync, exec } from "node:child_process"; import path from "node:path"; import fs from "node:fs"; import { LogLevel, logMessage } from "./log.js"; +import { promisify } from "node:util"; type Dirent = fs.Dirent; -// Common function to find files recursively with case-insensitive matching +export const execAsync = promisify(exec); + +/** + * Reset unstaged changes in a git repository + * @param repoPath The path to the git repository + * @returns A promise that resolves when the reset is complete + */ +export async function resetGitRepo(repoPath: string): Promise { + try { + const { stderr } = await execAsync("git clean -fdx && git reset --hard HEAD", { + cwd: repoPath, + }); + if (stderr) { + logMessage(`Warning during git reset: ${stderr}`, LogLevel.Warn); + } else { + logMessage(`Successfully reset git repo at ${repoPath}`, LogLevel.Info); + } + } catch (error) { + throw new Error(`Failed to reset git repo at ${repoPath}: ${error}`); + } +} + +/* + * Common function to find files recursively with case-insensitive matching + */ export function findFilesRecursive(directory: string, fileName: string): string[] { let results: string[] = []; const list: Dirent[] = fs.readdirSync(directory, { withFileTypes: true }); @@ -31,6 +56,9 @@ export function getArgumentValue(args: string[], flag: string, defaultValue: str return index !== -1 && args[index + 1] ? args[index + 1] : defaultValue; } +/* + * Get the relative path from the specification folder + */ export function getRelativePathFromSpecification(absolutePath: string): string { const specificationIndex = absolutePath.indexOf("specification/"); if (specificationIndex !== -1) { @@ -39,6 +67,9 @@ export function getRelativePathFromSpecification(absolutePath: string): string { return absolutePath; } +/* + * Run the spec-gen-sdk command + */ export async function runSpecGenSdkCommand(specGenSdkCommand: string[]): Promise { return new Promise((resolve, reject) => { const childProcess = spawn("npx", specGenSdkCommand, { @@ -59,6 +90,9 @@ export async function runSpecGenSdkCommand(specGenSdkCommand: string[]): Promise }); } +/* + * Get the list of all type spec project folder paths + */ export function getAllTypeSpecPaths(specRepoPath: string): string[] { const scriptPath = path.resolve(specRepoPath, "eng/scripts/Get-TypeSpec-Folders.ps1"); const args = [ @@ -85,6 +119,10 @@ export function getAllTypeSpecPaths(specRepoPath: string): string[] { return []; } } + +/* + * Run the PowerShell script + */ export function runPowerShellScript(args: string[]): string | undefined { const result = spawnSync("/usr/bin/pwsh", args, { encoding: "utf8" }); if (result.error) { @@ -94,5 +132,289 @@ export function runPowerShellScript(args: string[]): string | undefined { if (result.stderr) { logMessage(`PowerShell script error output:${result.stderr}`, LogLevel.Error); } - return result.stdout.trim(); + return result.stdout?.trim(); +} + +// Function to call Get-ChangedFiles from PowerShell script +export function getChangedFiles( + specRepoPath: string, + baseCommitish: string = "HEAD^", + targetCommitish: string = "HEAD", + diffFilter: string = "d", +): string[] | undefined { + const scriptPath = path.resolve(specRepoPath, "eng/scripts/ChangedFiles-Functions.ps1"); + const args = [ + "-Command", + `& { . '${scriptPath}'; Get-ChangedFiles '${baseCommitish}' '${targetCommitish}' '${diffFilter}' }`, + ]; + + const output = runPowerShellScript(args); + if (output) { + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + } + return undefined; +} + +/** + * Searches upward from a starting path to find the nearest parent directory containing a file matching the given pattern + * @param startPath - The directory path to start searching from + * @param searchFile - Regular expression pattern to match the target file name + * @param specRepoFolder - The root folder of the repository + * @param stopAtFolder - Optional boundary directory where the search should stop + * @returns The path of the directory containing the matching file, or undefined if not found + */ +export function findParentWithFile( + startPath: string, + searchFile: RegExp, + specRepoFolder: string, + stopAtFolder?: string, +): string | undefined { + let currentPath = startPath; + + while (currentPath) { + try { + const absolutePath = path.resolve(specRepoFolder, currentPath); + const files = fs.readdirSync(absolutePath); + if (files.some((file) => searchFile.test(file.toLowerCase()))) { + return currentPath; + } + } catch (error) { + logMessage(`Error reading directory: ${currentPath} with ${error}`, LogLevel.Warn); + return undefined; + } + currentPath = path.dirname(currentPath); + if (stopAtFolder && currentPath === stopAtFolder) { + return undefined; + } + } + return undefined; +} + +/** + * Searches for parent directories containing specific files for a list of files + * Optimizes the search by grouping files in the same directory to avoid redundant searches + * @param files - Array of file paths to process + * @param options - Search configuration options + * @returns Object mapping parent directory paths to arrays of related files + */ +export function searchRelatedParentFolders( + files: string[], + options: { searchFileRegex: RegExp; specRepoFolder: string; stopAtFolder?: string }, +): { [folderPath: string]: string[] } { + const result: { [folderPath: string]: string[] } = {}; + + // Group files by their directory path to avoid redundant searches + // Example: for files ["dir1/a.ts", "dir1/b.ts", "dir2/c.ts"] + // Creates: { "dir1": ["dir1/a.ts", "dir1/b.ts"], "dir2": ["dir2/c.ts"] } + // eslint-disable-next-line unicorn/no-array-reduce + const filesByDir = files.reduce<{ [dir: string]: string[] }>((acc, file) => { + const dir = path.dirname(file); + if (!acc[dir]) { + acc[dir] = []; + } + acc[dir].push(file); + return acc; + }, {}); + + // Search parent folder only once per unique directory + for (const [dir, dirFiles] of Object.entries(filesByDir)) { + const parentFolder = findParentWithFile( + dir, + options.searchFileRegex, + options.specRepoFolder, + options.stopAtFolder, + ); + if (parentFolder) { + if (!result[parentFolder]) { + result[parentFolder] = []; + } + result[parentFolder].push(...dirFiles); + } + } + + return result; +} + +/** + * Identifies files that are part of a shared library based on their directory names + * @param files - Array of file paths to check + * @param options - Search configuration options + * @returns Array of files that belong to shared libraries + */ +export function searchSharedLibrary( + files: string[], + options: { searchFileRegex: RegExp; specRepoFolder: string }, +): string[] { + return files.filter((file) => { + const dirname = path.dirname(file); + return options.searchFileRegex.test(dirname); + }); +} + +/** + * Finds peer TypeSpec projects for shared libraries and maps them to their source libraries + * Assumes all shared libraries are from the same parent folder + * @param sharedLibraries - Array of shared library file paths (all from same parent) + * @param options - Search configuration options + * @returns Object mapping project directory paths to arrays of related shared library files + */ +export function searchRelatedTypeSpecProjectBySharedLibrary( + sharedLibraries: string[], + options: { searchFileRegex: RegExp; specRepoFolder: string }, +): { [folderPath: string]: string[] } { + if (sharedLibraries.length === 0) { + return {}; + } + + const result: { [folderPath: string]: string[] } = {}; + const managementSuffix = ".Management"; + + // Get parent directory from first library (all libraries share same parent) + const parentDir = path.dirname(path.dirname(sharedLibraries[0])); + const sourceLibDir = path.dirname(sharedLibraries[0]); + + try { + const absoluteParentPath = path.resolve(options.specRepoFolder, parentDir); + const peerDirs = fs.readdirSync(absoluteParentPath, { withFileTypes: true }); + + // Find all peer directories containing TypeSpec projects + for (const peerDir of peerDirs) { + if ( + !peerDir.isDirectory() || + peerDir.name.endsWith(managementSuffix) || + path.join(parentDir, peerDir.name) === sourceLibDir + ) { + continue; + } + + const peerPath = path.join(parentDir, peerDir.name); + try { + const peerFiles = fs.readdirSync(path.resolve(options.specRepoFolder, peerPath)); + if (peerFiles.some((file) => options.searchFileRegex.test(file.toLowerCase()))) { + result[peerPath] = sharedLibraries; + } + } catch { + logMessage(`Error reading directory: ${peerPath}`, LogLevel.Warn); + } + } + } catch { + logMessage(`Error reading directory: ${parentDir}`, LogLevel.Warn); + } + + return result; +} + +export function extractServiceName(path: string): string { + const match = /specification\/([^/]*)\//.exec(path); + return match ? match[1] : ""; +} + +export type ServiceFolderInfo = { + resourceManagerPaths: { path: string; subPath?: string }[]; + dataPlanePaths: { path: string; subFolder?: string }[]; + managementPaths: string[]; + otherTypeSpecPaths: string[]; +}; + +export function groupPathsByService( + readmeMDResult: { [folderPath: string]: string[] }, + typespecProjectResult: { [folderPath: string]: string[] }, +): Map { + const serviceMap = new Map(); + + // Process readme paths + for (const folderPath of Object.keys(readmeMDResult)) { + const serviceName = extractServiceName(folderPath); + if (!serviceName) continue; + + if (!serviceMap.has(serviceName)) { + serviceMap.set(serviceName, { + resourceManagerPaths: [], + dataPlanePaths: [], + managementPaths: [], + otherTypeSpecPaths: [], + }); + } + + const info = serviceMap.get(serviceName)!; + if (folderPath.includes("resource-manager")) { + const subPathMatch = /resource-manager\/([^/]+)/.exec(folderPath); + info.resourceManagerPaths.push({ + path: folderPath, + subPath: subPathMatch ? subPathMatch[1] : undefined, + }); + } else if (folderPath.includes("data-plane")) { + const subFolderMatch = /data-plane\/([^/]+)/.exec(folderPath); + info.dataPlanePaths.push({ + path: folderPath, + subFolder: subFolderMatch ? subFolderMatch[1] : undefined, + }); + } + } + + // Process typespec paths + for (const folderPath of Object.keys(typespecProjectResult)) { + const serviceName = extractServiceName(folderPath); + if (!serviceName) continue; + + if (!serviceMap.has(serviceName)) { + serviceMap.set(serviceName, { + resourceManagerPaths: [], + dataPlanePaths: [], + managementPaths: [], + otherTypeSpecPaths: [], + }); + } + + const info = serviceMap.get(serviceName)!; + if (folderPath.endsWith(".Management")) { + info.managementPaths.push(folderPath); + } else { + info.otherTypeSpecPaths.push(folderPath); + } + } + + return serviceMap; +} + +export function getLastPathSegment(path: string): string { + const segments = path.split("/"); + // eslint-disable-next-line unicorn/prefer-at + return segments[segments.length - 1]; +} + +export type SpecResults = { + readmeMDResult: { [folderPath: string]: string[] }; + typespecProjectResult: { [folderPath: string]: string[] }; +}; + +export type ChangedSpecs = { + [K in "readmeMd" | "typespecProject"]?: string; +} & { + specs: string[]; +}; + +/** + * Creates combined specs from readme and typespec paths + * @param readmePath - Path to the readme file + * @param typespecPaths - Array of typespec paths to combine with + * @param results - Current state of readme and typespec results + * @returns Array of specs with combined files + */ +export function createCombinedSpecs( + readmePath: string, + typespecPaths: string[], + results: SpecResults, +): ChangedSpecs[] { + return typespecPaths.map((tsPath) => ({ + specs: [ + ...(results.readmeMDResult[readmePath] || []), + ...(results.typespecProjectResult[tsPath] || []), + ], + readmeMd: path.join(readmePath, "readme.md"), + typespecProject: path.join(tsPath, "tspconfig.yaml"), + })); } diff --git a/specification/contosowidgetmanager/Contoso.WidgetManager.Shared/main.tsp b/specification/contosowidgetmanager/Contoso.WidgetManager.Shared/main.tsp index 1b94bb705031..12e86d537f79 100644 --- a/specification/contosowidgetmanager/Contoso.WidgetManager.Shared/main.tsp +++ b/specification/contosowidgetmanager/Contoso.WidgetManager.Shared/main.tsp @@ -1,6 +1,6 @@ @doc("Faked shared model") model FakedSharedModel { - @doc("The tag.") + @doc("The tag. test") tag: string; @doc("The created date.") diff --git a/specification/contosowidgetmanager/Contoso.WidgetManager/main.tsp b/specification/contosowidgetmanager/Contoso.WidgetManager/main.tsp index 7e5e683af7f9..c64c7ca26af2 100644 --- a/specification/contosowidgetmanager/Contoso.WidgetManager/main.tsp +++ b/specification/contosowidgetmanager/Contoso.WidgetManager/main.tsp @@ -23,7 +23,7 @@ enum Versions { v2022_11_01_Preview: "2022-11-01-preview", } -@doc("A widget.") +@doc("A widget. test") @resource("widgets") model WidgetSuite { @key("widgetName") diff --git a/specification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/widgets.json b/specification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/widgets.json index 22be02d8821e..a1d24d7ba85c 100644 --- a/specification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/widgets.json +++ b/specification/contosowidgetmanager/data-plane/Azure.Contoso.WidgetManager/preview/2022-11-01-preview/widgets.json @@ -447,7 +447,7 @@ "properties": { "tag": { "type": "string", - "description": "The tag." + "description": "The tag. test" }, "createdAt": { "type": "string", @@ -466,7 +466,7 @@ "properties": { "tag": { "type": "string", - "description": "The tag." + "description": "The tag. test" }, "createdAt": { "type": "string", @@ -499,7 +499,7 @@ }, "WidgetSuite": { "type": "object", - "description": "A widget.", + "description": "A widget. test", "properties": { "name": { "type": "string", @@ -522,7 +522,7 @@ }, "WidgetSuiteCreateOrUpdate": { "type": "object", - "description": "A widget.", + "description": "A widget. test", "properties": { "manufacturerId": { "type": "string",