diff --git a/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json b/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json new file mode 100644 index 00000000000..136490cc78e --- /dev/null +++ b/common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "comment": "Simplifies the process of going from operation to build cache ID.", + "type": "none", + "packageName": "@microsoft/rush" + } + ], + "packageName": "@microsoft/rush", + "email": "aramissennyeydd@users.noreply.github.com" +} \ No newline at end of file diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index cbba6950d83..9b3e495c70d 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -13,33 +13,37 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; +import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord'; -export interface IProjectBuildCacheOptions { +export interface IOperationBuildCacheOptions { /** * The repo-wide configuration for the build cache. */ buildCacheConfiguration: BuildCacheConfiguration; /** - * The project to be cached. + * The terminal to use for logging. */ - project: RushConfigurationProject; + terminal: ITerminal; +} + +export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & { /** * Value from rush-project.json */ projectOutputFolderNames: ReadonlyArray; /** - * The hash of all relevant inputs and configuration that uniquely identifies this execution. + * The project to be cached. */ - operationStateHash: string; + project: RushConfigurationProject; /** - * The terminal to use for logging. + * The hash of all relevant inputs and configuration that uniquely identifies this execution. */ - terminal: ITerminal; + operationStateHash: string; /** * The name of the phase that is being cached. */ phaseName: string; -} +}; interface IPathsToCache { filteredOutputFolderNames: string[]; @@ -94,6 +98,31 @@ export class ProjectBuildCache { return new ProjectBuildCache(cacheId, options); } + public static forOperation( + operation: OperationExecutionRecord, + options: IOperationBuildCacheOptions + ): ProjectBuildCache { + if (!operation.associatedProject) { + throw new InternalError('Operation must have an associated project'); + } + if (!operation.associatedPhase) { + throw new InternalError('Operation must have an associated phase'); + } + const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])]; + if (operation.metadataFolderPath) { + outputFolders.push(operation.metadataFolderPath); + } + const buildCacheOptions: IProjectBuildCacheOptions = { + ...options, + project: operation.associatedProject, + phaseName: operation.associatedPhase.name, + projectOutputFolderNames: outputFolders, + operationStateHash: operation.stateHash + }; + const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions); + return new ProjectBuildCache(cacheId, buildCacheOptions); + } + public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 5632236f735..37ba2d626e7 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -35,7 +35,6 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10; @@ -88,8 +87,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options; - const { cacheHashSalt } = buildCacheConfiguration; - hooks.beforeExecuteOperations.tap( PLUGIN_NAME, ( @@ -104,76 +101,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { ); } - // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure. - const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot; - const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled ? new DisjointSet() : undefined; - const hashByOperation: Map = new Map(); - // Build cache hashes are computed up front to ensure stability and to catch configuration errors early. - function getOrCreateOperationHash(operation: Operation): string { - const cachedHash: string | undefined = hashByOperation.get(operation); - if (cachedHash !== undefined) { - return cachedHash; - } - - // Examples of data in the config hash: - // - CLI parameters (ShellOperationRunner) - const configHash: string | undefined = operation.runner?.getConfigHash(); - - const { associatedProject, associatedPhase } = operation; - // Examples of data in the local state hash: - // - Environment variables specified in `dependsOnEnvVars` - // - Git hashes of tracked files in the associated project - // - Git hash of the shrinkwrap file for the project - // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) - const localStateHash: string | undefined = - associatedProject && - definitelyDefinedInputsSnapshot.getOperationOwnStateHash( - associatedProject, - associatedPhase?.name - ); - - // The final state hashes of operation dependencies are factored into the hash to ensure that any - // state changes in dependencies will invalidate the cache. - const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort(); - - const hasher: crypto.Hash = crypto.createHash('sha1'); - // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content - // of the build cache. - hasher.update(`${RushConstants.buildCacheVersion}`); - - if (cacheHashSalt !== undefined) { - // This allows repository owners to force a cache bust by changing the salt. - // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. - hasher.update(cacheHashSalt); - } - - for (const dependencyHash of dependencyHashes) { - hasher.update(dependencyHash); - } - - if (localStateHash) { - hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); - } - - if (configHash) { - hasher.update(`${RushConstants.hashDelimiter}${configHash}`); - } - - const hashString: string = hasher.digest('hex'); - - hashByOperation.set(operation, hashString); - return hashString; - } - - function getDependencyHash(operation: Operation): string { - return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`; - } - for (const [operation, record] of recordByOperation) { + const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({ + inputsSnapshot, + buildCacheConfiguration + }); const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; if (!associatedProject || !associatedPhase || !runner) { return; @@ -188,7 +124,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // depending on the selected phase. const fileHashes: ReadonlyMap | undefined = inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); - const stateHash: string = getOrCreateOperationHash(operation); const cacheDisabledReason: string | undefined = projectConfiguration ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp) @@ -325,10 +260,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({ buildCacheContext, buildCacheConfiguration, - rushProject: project, - phase, terminal: buildCacheTerminal, - operation: operation + record }); // Try to acquire the cobuild lock @@ -639,39 +572,30 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private _tryGetProjectBuildCache({ buildCacheConfiguration, buildCacheContext, - rushProject, - phase, terminal, - operation + record }: { buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; - rushProject: RushConfigurationProject; - phase: IPhase; terminal: ITerminal; - operation: Operation; + record: OperationExecutionRecord; }): ProjectBuildCache | undefined { if (!buildCacheContext.operationBuildCache) { const { cacheDisabledReason } = buildCacheContext; - if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) { + if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) { terminal.writeVerboseLine(cacheDisabledReason); return; } - const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext; - if (!outputFolderNames || !buildCacheConfiguration) { + if (!buildCacheConfiguration) { // Unreachable, since this will have set `cacheDisabledReason`. return; } // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent - buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({ - project: rushProject, - projectOutputFolderNames: outputFolderNames, + buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(record, { buildCacheConfiguration, - terminal, - operationStateHash, - phaseName: phase.name + terminal }); } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index d9c4a2b82a9..54ca0331531 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as crypto from 'crypto'; import { type ITerminal, @@ -29,6 +30,9 @@ import { initializeProjectLogFilesAsync } from './ProjectLogWritable'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { RushConstants } from '../RushConstants'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -114,6 +118,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera private _collatedWriter: CollatedWriter | undefined = undefined; private _status: OperationStatus; + private _stateHash: string | undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { const { runner, associatedPhase, associatedProject } = operation; @@ -206,6 +211,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return !this.operation.enabled || this.runner.silent; } + public get stateHash(): string { + if (!this._stateHash) { + throw new Error( + 'Operation state hash is not calculated yet, you must call `calculateStateHash` first.' + ); + } + return this._stateHash; + } + /** * {@inheritdoc IOperationRunnerContext.runWithTerminalAsync} */ @@ -335,4 +349,61 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } } } + + public calculateStateHash(options: { + inputsSnapshot: IInputsSnapshot; + buildCacheConfiguration: BuildCacheConfiguration; + }): string { + if (!this._stateHash) { + const { + inputsSnapshot, + buildCacheConfiguration: { cacheHashSalt } + } = options; + + // Examples of data in the config hash: + // - CLI parameters (ShellOperationRunner) + const configHash: string = this.runner.getConfigHash(); + + const { associatedProject, associatedPhase } = this; + // Examples of data in the local state hash: + // - Environment variables specified in `dependsOnEnvVars` + // - Git hashes of tracked files in the associated project + // - Git hash of the shrinkwrap file for the project + // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) + const localStateHash: string | undefined = + associatedProject && + inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name); + + // The final state hashes of operation dependencies are factored into the hash to ensure that any + // state changes in dependencies will invalidate the cache. + const dependencyHashes: string[] = Array.from(this.dependencies, (record) => { + return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`; + }).sort(); + + const hasher: crypto.Hash = crypto.createHash('sha1'); + // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content + // of the build cache. + hasher.update(`${RushConstants.buildCacheVersion}`); + + if (cacheHashSalt !== undefined) { + // This allows repository owners to force a cache bust by changing the salt. + // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output. + hasher.update(cacheHashSalt); + } + + for (const dependencyHash of dependencyHashes) { + hasher.update(dependencyHash); + } + + if (localStateHash) { + hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`); + } + + hasher.update(`${RushConstants.hashDelimiter}${configHash}`); + + const hash: string = hasher.digest('hex'); + this._stateHash = hash; + } + return this._stateHash; + } }