[rush] feat(buildcache): improve access to operation build cache

common/changes/@microsoft/rush/sennyeya-cache-id_2024-12-26-23-12.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "comment": "Simplifies the process of going from operation to build cache ID.",
+      "type": "none",
+      "packageName": "@microsoft/rush"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "[email protected]"
+}

libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

@@ -13,33 +13,37 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider';
 import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider';
 import { TarExecutable } from '../../utilities/TarExecutable';
 import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
+import type { OperationExecutionRecord } from '../operations/OperationExecutionRecord';
 
-export interface IProjectBuildCacheOptions {
+export interface IOperationBuildCacheOptions {
   /**
    * The repo-wide configuration for the build cache.
    */
   buildCacheConfiguration: BuildCacheConfiguration;
   /**
-   * The project to be cached.
+   * The terminal to use for logging.
    */
-  project: RushConfigurationProject;
+  terminal: ITerminal;
+}
+
+export type IProjectBuildCacheOptions = IOperationBuildCacheOptions & {
   /**
    * Value from rush-project.json
    */
   projectOutputFolderNames: ReadonlyArray<string>;
   /**
-   * The hash of all relevant inputs and configuration that uniquely identifies this execution.
+   * The project to be cached.
    */
-  operationStateHash: string;
+  project: RushConfigurationProject;
   /**
-   * The terminal to use for logging.
+   * The hash of all relevant inputs and configuration that uniquely identifies this execution.
    */
-  terminal: ITerminal;
+  operationStateHash: string;
   /**
    * The name of the phase that is being cached.
    */
   phaseName: string;
-}
+};
 
 interface IPathsToCache {
   filteredOutputFolderNames: string[];
@@ -94,6 +98,31 @@ export class ProjectBuildCache {
     return new ProjectBuildCache(cacheId, options);
   }
 
+  public static forOperation(
+    operation: OperationExecutionRecord,
+    options: IOperationBuildCacheOptions
+  ): ProjectBuildCache {
+    if (!operation.associatedProject) {
+      throw new InternalError('Operation must have an associated project');
+    }
+    if (!operation.associatedPhase) {
+      throw new InternalError('Operation must have an associated phase');
+    }
+    const outputFolders: string[] = [...(operation.operation.settings?.outputFolderNames ?? [])];
+    if (operation.metadataFolderPath) {
+      outputFolders.push(operation.metadataFolderPath);
+    }
+    const buildCacheOptions: IProjectBuildCacheOptions = {
+      ...options,
+      project: operation.associatedProject,
+      phaseName: operation.associatedPhase.name,
+      projectOutputFolderNames: outputFolders,
+      operationStateHash: operation.stateHash
+    };
+    const cacheId: string | undefined = ProjectBuildCache._getCacheId(buildCacheOptions);
+    return new ProjectBuildCache(cacheId, buildCacheOptions);
+  }
+
   public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean> {
     const cacheId: string | undefined = specifiedCacheId || this._cacheId;
     if (!cacheId) {

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -35,7 +35,6 @@ import type { IPhase } from '../../api/CommandLineConfiguration';
 import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
 import type { IOperationExecutionResult } from './IOperationExecutionResult';
 import type { OperationExecutionRecord } from './OperationExecutionRecord';
-import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
 
 const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin';
 const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10;
@@ -88,8 +87,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
   public apply(hooks: PhasedCommandHooks): void {
     const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration } = this._options;
 
-    const { cacheHashSalt } = buildCacheConfiguration;
-
     hooks.beforeExecuteOperations.tap(
       PLUGIN_NAME,
       (
@@ -104,76 +101,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           );
         }
 
-        // This redefinition is necessary due to limitations in TypeScript's control flow analysis, due to the nested closure.
-        const definitelyDefinedInputsSnapshot: IInputsSnapshot = inputsSnapshot;
-
         const disjointSet: DisjointSet<Operation> | undefined = cobuildConfiguration?.cobuildFeatureEnabled
           ? new DisjointSet()
           : undefined;
 
-        const hashByOperation: Map<Operation, string> = new Map();
-        // Build cache hashes are computed up front to ensure stability and to catch configuration errors early.
-        function getOrCreateOperationHash(operation: Operation): string {
-          const cachedHash: string | undefined = hashByOperation.get(operation);
-          if (cachedHash !== undefined) {
-            return cachedHash;
-          }
-
-          // Examples of data in the config hash:
-          // - CLI parameters (ShellOperationRunner)
-          const configHash: string | undefined = operation.runner?.getConfigHash();
-
-          const { associatedProject, associatedPhase } = operation;
-          // Examples of data in the local state hash:
-          // - Environment variables specified in `dependsOnEnvVars`
-          // - Git hashes of tracked files in the associated project
-          // - Git hash of the shrinkwrap file for the project
-          // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
-          const localStateHash: string | undefined =
-            associatedProject &&
-            definitelyDefinedInputsSnapshot.getOperationOwnStateHash(
-              associatedProject,
-              associatedPhase?.name
-            );
-
-          // The final state hashes of operation dependencies are factored into the hash to ensure that any
-          // state changes in dependencies will invalidate the cache.
-          const dependencyHashes: string[] = Array.from(operation.dependencies, getDependencyHash).sort();
-
-          const hasher: crypto.Hash = crypto.createHash('sha1');
-          // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
-          // of the build cache.
-          hasher.update(`${RushConstants.buildCacheVersion}`);
-
-          if (cacheHashSalt !== undefined) {
-            // This allows repository owners to force a cache bust by changing the salt.
-            // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
-            hasher.update(cacheHashSalt);
-          }
-
-          for (const dependencyHash of dependencyHashes) {
-            hasher.update(dependencyHash);
-          }
-
-          if (localStateHash) {
-            hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
-          }
-
-          if (configHash) {
-            hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
-          }
-
-          const hashString: string = hasher.digest('hex');
-
-          hashByOperation.set(operation, hashString);
-          return hashString;
-        }
-
-        function getDependencyHash(operation: Operation): string {
-          return `${RushConstants.hashDelimiter}${operation.name}=${getOrCreateOperationHash(operation)}`;
-        }
-
         for (const [operation, record] of recordByOperation) {
+          const stateHash: string = (record as OperationExecutionRecord).calculateStateHash({
+            inputsSnapshot,
+            buildCacheConfiguration
+          });
           const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation;
           if (!associatedProject || !associatedPhase || !runner) {
             return;
@@ -188,7 +124,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           // depending on the selected phase.
           const fileHashes: ReadonlyMap<string, string> | undefined =
             inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName);
-          const stateHash: string = getOrCreateOperationHash(operation);
 
           const cacheDisabledReason: string | undefined = projectConfiguration
             ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp)
@@ -325,10 +260,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           let projectBuildCache: ProjectBuildCache | undefined = this._tryGetProjectBuildCache({
             buildCacheContext,
             buildCacheConfiguration,
-            rushProject: project,
-            phase,
             terminal: buildCacheTerminal,
-            operation: operation
+            record
           });
 
           // Try to acquire the cobuild lock
@@ -639,40 +572,34 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
   private _tryGetProjectBuildCache({
     buildCacheConfiguration,
     buildCacheContext,
-    rushProject,
-    phase,
     terminal,
-    operation
+    record
   }: {
     buildCacheContext: IOperationBuildCacheContext;
     buildCacheConfiguration: BuildCacheConfiguration | undefined;
-    rushProject: RushConfigurationProject;
-    phase: IPhase;
     terminal: ITerminal;
-    operation: Operation;
+    record: OperationExecutionRecord;
   }): ProjectBuildCache | undefined {
     if (!buildCacheContext.operationBuildCache) {
       const { cacheDisabledReason } = buildCacheContext;
-      if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) {
+      if (cacheDisabledReason && !record.operation.settings?.allowCobuildWithoutCache) {
         terminal.writeVerboseLine(cacheDisabledReason);
         return;
       }
 
-      const { outputFolderNames, stateHash: operationStateHash } = buildCacheContext;
-      if (!outputFolderNames || !buildCacheConfiguration) {
+      if (!buildCacheConfiguration) {
         // Unreachable, since this will have set `cacheDisabledReason`.
         return;
       }
 
       // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent
-      buildCacheContext.operationBuildCache = ProjectBuildCache.getProjectBuildCache({
-        project: rushProject,
-        projectOutputFolderNames: outputFolderNames,
-        buildCacheConfiguration,
-        terminal,
-        operationStateHash,
-        phaseName: phase.name
-      });
+      buildCacheContext.operationBuildCache = ProjectBuildCache.forOperation(
+        record as OperationExecutionRecord,
+        {
+          buildCacheConfiguration,
+          terminal
+        }
+      );
     }
 
     return buildCacheContext.operationBuildCache;

libraries/rush-lib/src/logic/operations/Operation.ts

@@ -107,6 +107,8 @@ export class Operation {
    */
   public enabled: boolean;
 
+  private _stateHash: string | undefined;
+
   public constructor(options: IOperationOptions) {
     const { phase, project, runner, settings, logFilenameIdentifier } = options;
     this.associatedPhase = phase;

libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
+import * as crypto from 'crypto';
 
 import {
   type ITerminal,
@@ -29,6 +30,9 @@ import {
   initializeProjectLogFilesAsync
 } from './ProjectLogWritable';
 import type { IOperationExecutionResult } from './IOperationExecutionResult';
+import type { IInputsSnapshot } from '../incremental/InputsSnapshot';
+import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
+import { RushConstants } from '../RushConstants';
 
 export interface IOperationExecutionRecordContext {
   streamCollator: StreamCollator;
@@ -114,6 +118,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
 
   private _collatedWriter: CollatedWriter | undefined = undefined;
   private _status: OperationStatus;
+  private _stateHash: string | undefined;
 
   public constructor(operation: Operation, context: IOperationExecutionRecordContext) {
     const { runner, associatedPhase, associatedProject } = operation;
@@ -206,6 +211,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
     return !this.operation.enabled || this.runner.silent;
   }
 
+  public get stateHash(): string {
+    if (!this._stateHash) {
+      throw new Error(
+        'Operation state hash is not calculated yet, you must call `calculateStateHash` first.'
+      );
+    }
+    return this._stateHash;
+  }
+
   /**
    * {@inheritdoc IOperationRunnerContext.runWithTerminalAsync}
    */
@@ -335,4 +349,60 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
       }
     }
   }
+
+  public calculateStateHash(options: {
+    inputsSnapshot: IInputsSnapshot;
+    buildCacheConfiguration: BuildCacheConfiguration;
+  }): string {
+    if (this._stateHash) {
+      return this._stateHash;
+    }
+    const { inputsSnapshot, buildCacheConfiguration } = options;
+    const { cacheHashSalt } = buildCacheConfiguration;
+
+    // Examples of data in the config hash:
+    // - CLI parameters (ShellOperationRunner)
+    const configHash: string = this.runner.getConfigHash();
+
+    const { associatedProject, associatedPhase } = this;
+    // Examples of data in the local state hash:
+    // - Environment variables specified in `dependsOnEnvVars`
+    // - Git hashes of tracked files in the associated project
+    // - Git hash of the shrinkwrap file for the project
+    // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project)
+    const localStateHash: string | undefined =
+      associatedProject && inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase?.name);
+
+    // The final state hashes of operation dependencies are factored into the hash to ensure that any
+    // state changes in dependencies will invalidate the cache.
+    const dependencyHashes: string[] = Array.from(this.dependencies, (record) => {
+      return `${RushConstants.hashDelimiter}${record.name}=${record.calculateStateHash(options)}`;
+    }).sort();
+
+    const hasher: crypto.Hash = crypto.createHash('sha1');
+    // This property is used to force cache bust when version changes, e.g. when fixing bugs in the content
+    // of the build cache.
+    hasher.update(`${RushConstants.buildCacheVersion}`);
+
+    if (cacheHashSalt !== undefined) {
+      // This allows repository owners to force a cache bust by changing the salt.
+      // A common use case is to invalidate the cache when adding/removing/updating rush plugins that alter the build output.
+      hasher.update(cacheHashSalt);
+    }
+
+    for (const dependencyHash of dependencyHashes) {
+      hasher.update(dependencyHash);
+    }
+
+    if (localStateHash) {
+      hasher.update(`${RushConstants.hashDelimiter}${localStateHash}`);
+    }
+
+    hasher.update(`${RushConstants.hashDelimiter}${configHash}`);
+
+    const hash: string = hasher.digest('hex');
+    this._stateHash = hash;
+
+    return this.stateHash;
+  }
 }