Stacks: values generation (#3914)

* Pre allocate checked paths

* Stack value generation

* Add loading of stack values

* add stack values generation

* Add generation of terragrunt values

* Output fix

* Add stack values generaiton

* add experiment flag

* Config update

* add stack tests

* Add auto generated token

* Updated values header

* Stack lint issues

* improved error handling

* code rabbit comment

* Permissions simplification

* Permissions constants

* Added errors printing

* Update documentation

cli/commands/stack/action.go

@@ -5,9 +5,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/gruntwork-io/terragrunt/config"
-	"github.com/zclconf/go-cty/cty"
-
 	runall "github.com/gruntwork-io/terragrunt/cli/commands/run-all"
 	"github.com/gruntwork-io/terragrunt/internal/cli"
 
@@ -20,7 +17,6 @@ import (
 const (
 	stackDir         = ".terragrunt-stack"
 	defaultStackFile = "terragrunt.stack.hcl"
-	dirPerm          = 0755
 )
 
 // RunGenerate runs the stack command.
@@ -29,10 +25,6 @@ func RunGenerate(ctx context.Context, opts *options.TerragruntOptions) error {
 		return err
 	}
 
-	if err := populateStackValues(ctx, opts); err != nil {
-		return errors.New(err)
-	}
-
 	return generateStack(ctx, opts)
 }
 
@@ -42,10 +34,6 @@ func Run(ctx context.Context, opts *options.TerragruntOptions) error {
 		return err
 	}
 
-	if err := populateStackValues(ctx, opts); err != nil {
-		return errors.New(err)
-	}
-
 	if err := RunGenerate(ctx, opts); err != nil {
 		return err
 	}
@@ -61,10 +49,6 @@ func RunOutput(ctx context.Context, opts *options.TerragruntOptions, index strin
 		return err
 	}
 
-	if err := populateStackValues(ctx, opts); err != nil {
-		return errors.New(err)
-	}
-
 	// collect outputs
 	outputs, err := generateOutput(ctx, opts)
 	if err != nil {
@@ -118,23 +102,3 @@ func checkStackExperiment(opts *options.TerragruntOptions) error {
 
 	return nil
 }
-
-func populateStackValues(ctx context.Context, opts *options.TerragruntOptions) error {
-	opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
-
-	stackFile, err := config.ReadStackConfigFile(ctx, opts)
-	if err != nil {
-		return errors.Errorf("Failed to read stack file from %s %v", opts.WorkingDir, err)
-	}
-
-	unitValues := map[string]*cty.Value{}
-
-	for _, unit := range stackFile.Units {
-		path := filepath.Join(opts.WorkingDir, stackDir, unit.Path)
-		unitValues[path] = unit.Values
-	}
-
-	opts.StackValues = options.NewStackValues(&cty.NilVal, unitValues)
-
-	return nil
-}

cli/commands/stack/generate.go

@@ -22,6 +22,7 @@ const (
 
 func generateStack(ctx context.Context, opts *options.TerragruntOptions) error {
 	opts.Logger.Infof("Generating stack from %s", opts.TerragruntStackConfigPath)
+	opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
 	stackFile, err := config.ReadStackConfigFile(ctx, opts)
 
 	if err != nil {
@@ -37,7 +38,7 @@ func generateStack(ctx context.Context, opts *options.TerragruntOptions) error {
 
 func processStackFile(ctx context.Context, opts *options.TerragruntOptions, stackFile *config.StackConfigFile) error {
 	baseDir := filepath.Join(opts.WorkingDir, stackDir)
-	if err := os.MkdirAll(baseDir, dirPerm); err != nil {
+	if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
 		return errors.New(fmt.Errorf("failed to create base directory: %w", err))
 	}
 
@@ -69,14 +70,19 @@ func processStackFile(ctx context.Context, opts *options.TerragruntOptions, stac
 				return errors.New(err)
 			}
 		} else {
-			if err := os.MkdirAll(dest, dirPerm); err != nil {
+			if err := os.MkdirAll(dest, os.ModePerm); err != nil {
 				return errors.New(err)
 			}
 
 			if _, err := getter.GetAny(ctx, dest, src); err != nil {
 				return errors.New(err)
 			}
 		}
+
+		// generate unit values file
+		if err := config.WriteUnitValues(opts, unit, dest); err != nil {
+			return errors.New(err)
+		}
 	}
 
 	return nil

cli/commands/stack/output.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"path/filepath"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2/hclwrite"
@@ -20,6 +21,7 @@ import (
 
 func generateOutput(ctx context.Context, opts *options.TerragruntOptions) (map[string]map[string]cty.Value, error) {
 	opts.Logger.Debugf("Generating output from %s", opts.TerragruntStackConfigPath)
+	opts.TerragruntStackConfigPath = filepath.Join(opts.WorkingDir, defaultStackFile)
 	stackFile, err := config.ReadStackConfigFile(ctx, opts)
 
 	if err != nil {

config/config.go

@@ -42,6 +42,7 @@ const (
 	DefaultTerragruntConfigPath     = "terragrunt.hcl"
 	DefaultTerragruntJSONConfigPath = "terragrunt.hcl.json"
 	RecommendedParentConfigName     = "root.hcl"
+	StackValuesFile                 = "terragrunt.values.hcl"
 
 	FoundInFile = "found_in_file"
 
@@ -78,7 +79,6 @@ const (
 	MetadataErrors                      = "errors"
 	MetadataRetry                       = "retry"
 	MetadataIgnore                      = "ignore"
-	MetadataUnit                        = "unit"
 	MetadataValues                      = "values"
 )
 
@@ -895,6 +895,16 @@ func ParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *Inc
 		return nil, err
 	}
 
+	// read unit files and add to context
+	if ctx.TerragruntOptions.Experiments.Evaluate(experiment.Stacks) {
+		unitValues, err := ReadUnitValues(ctx.Context, ctx.TerragruntOptions, filepath.Dir(file.ConfigPath))
+		if err != nil {
+			return nil, err
+		}
+
+		ctx = ctx.WithValues(unitValues)
+	}
+
 	// Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are.
 	baseBlocks, err := DecodeBaseBlocks(ctx, file, includeFromChild)
 	if err != nil {

config/config_helpers.go

@@ -210,6 +210,10 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E
 		evalCtx.Variables[MetadataFeatureFlag] = *ctx.Features
 	}
 
+	if ctx.Values != nil {
+		evalCtx.Variables[MetadataValues] = *ctx.Values
+	}
+
 	if ctx.DecodedDependencies != nil {
 		evalCtx.Variables[MetadataDependency] = *ctx.DecodedDependencies
 	}
@@ -225,16 +229,6 @@ func createTerragruntEvalContext(ctx *ParsingContext, configPath string) (*hcl.E
 		evalCtx.Variables[MetadataInclude] = exposedInclude
 	}
 
-	// Add to context unit values
-	path := filepath.Dir(configPath)
-	unitValues := ctx.TerragruntOptions.StackValues.UnitValues(path)
-
-	if unitValues != nil {
-		evalCtx.Variables[MetadataUnit] = cty.ObjectVal(map[string]cty.Value{
-			MetadataValues: *unitValues,
-		})
-	}
-
 	return evalCtx, nil
 }
 

config/config_partial.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/gruntwork-io/terragrunt/internal/experiment"
+
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/huandu/go-clone"
 
@@ -346,6 +348,16 @@ func PartialParseConfigString(ctx *ParsingContext, configPath, configString stri
 func PartialParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
 	ctx = ctx.WithTrackInclude(nil)
 
+	// read unit files and add to context
+	if ctx.TerragruntOptions.Experiments.Evaluate(experiment.Stacks) {
+		unitValues, err := ReadUnitValues(ctx.Context, ctx.TerragruntOptions, filepath.Dir(file.ConfigPath))
+		if err != nil {
+			return nil, err
+		}
+
+		ctx = ctx.WithValues(unitValues)
+	}
+
 	// Decode just the Base blocks. See the function docs for DecodeBaseBlocks for more info on what base blocks are.
 	// Initialize evaluation ctx extensions from base blocks.
 	baseBlocks, err := DecodeBaseBlocks(ctx, file, includeFromChild)

config/locals.go

@@ -182,8 +182,8 @@ func canEvaluateLocals(expression hcl.Expression, evaluatedLocals map[string]cty
 		case rootName == MetadataFeatureFlag:
 			// If the variable is `feature`
 
-		case rootName == MetadataUnit:
-			// If the variable is `unit`
+		case rootName == MetadataValues:
+			// If the variable is `values`
 
 		case rootName != "local":
 			// We can't evaluate any variable other than `local`

config/parsing_context.go

@@ -27,6 +27,9 @@ type ParsingContext struct {
 	// Features are the feature flags that are enabled for the current terragrunt config.
 	Features *cty.Value
 
+	// Values of the unit
+	Values *cty.Value
+
 	// DecodedDependencies are references of other terragrunt config. This contains the following attributes that map to
 	// various fields related to that config:
 	// - outputs: The map of outputs from the terraform state obtained by running `terragrunt output` on that target config.
@@ -72,6 +75,11 @@ func (ctx ParsingContext) WithLocals(locals *cty.Value) *ParsingContext {
 	return &ctx
 }
 
+func (ctx ParsingContext) WithValues(values *cty.Value) *ParsingContext {
+	ctx.Values = values
+	return &ctx
+}
+
 // WithFeatures sets the feature flags to be used in evaluation context.
 func (ctx ParsingContext) WithFeatures(features *cty.Value) *ParsingContext {
 	ctx.Features = features

config/stack.go

@@ -3,9 +3,15 @@ package config
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+
+	"github.com/gruntwork-io/terragrunt/util"
+	"github.com/hashicorp/hcl/v2/hclwrite"
+
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/gruntwork-io/terragrunt/config/hclparse"
@@ -14,7 +20,10 @@ import (
 )
 
 const (
-	stackDir = ".terragrunt-stack"
+	stackDir       = ".terragrunt-stack"
+	unitValuesFile = "terragrunt.values.hcl"
+	unitDirPerm    = 0755
+	valueFilePerm  = 0644
 )
 
 // StackConfigFile represents the structure of terragrunt.stack.hcl stack file.
@@ -87,6 +96,88 @@ func ReadStackConfigFile(ctx context.Context, opts *options.TerragruntOptions) (
 	return config, nil
 }
 
+// WriteUnitValues generates and writes unit values to a terragrunt.values.hcl file in the specified unit directory.
+// If the unit has no values (Values is nil), the function logs a debug message and returns.
+// Parameters:
+//   - opts: TerragruntOptions containing logger and other configuration
+//   - unit: Unit containing the values to write
+//   - unitDirectory: Target directory where the values file will be created
+//
+// Returns an error if the directory creation or file writing fails.
+func WriteUnitValues(opts *options.TerragruntOptions, unit *Unit, unitDirectory string) error {
+	if unitDirectory == "" {
+		return errors.New("WriteUnitValues: unit directory path cannot be empty")
+	}
+
+	if err := os.MkdirAll(unitDirectory, unitDirPerm); err != nil {
+		return errors.Errorf("failed to create directory %s: %w", unitDirectory, err)
+	}
+
+	filePath := filepath.Join(unitDirectory, unitValuesFile)
+	if unit.Values == nil {
+		opts.Logger.Debugf("No values to write for unit %s in %s", unit.Name, filePath)
+		return nil
+	}
+
+	opts.Logger.Debugf("Writing values for unit %s in %s", unit.Name, filePath)
+
+	file := hclwrite.NewEmptyFile()
+	body := file.Body()
+	body.AppendUnstructuredTokens([]*hclwrite.Token{
+		{
+			Type:  hclsyntax.TokenComment,
+			Bytes: []byte("# Auto-generated by the terragrunt.stack.hcl file by Terragrunt. Do not edit manually\n"),
+		},
+	})
+
+	for key, val := range unit.Values.AsValueMap() {
+		body.SetAttributeValue(key, val)
+	}
+
+	if err := os.WriteFile(filePath, file.Bytes(), valueFilePerm); err != nil {
+		return errors.Errorf("failed to write values file %s: %w", filePath, err)
+	}
+
+	return nil
+}
+
+// ReadUnitValues reads the unit values from the terragrunt.values.hcl file.
+func ReadUnitValues(ctx context.Context, opts *options.TerragruntOptions, unitDirectory string) (*cty.Value, error) {
+	if unitDirectory == "" {
+		return nil, errors.New("ReadUnitValues: unit directory path cannot be empty")
+	}
+
+	filePath := filepath.Join(unitDirectory, unitValuesFile)
+
+	if util.FileNotExists(filePath) {
+		return nil, nil
+	}
+
+	opts.Logger.Debugf("Reading Terragrunt stack values file at %s", filePath)
+	parser := NewParsingContext(ctx, opts)
+	file, err := hclparse.NewParser(parser.ParserOptions...).ParseFromFile(filePath)
+
+	if err != nil {
+		return nil, errors.New(err)
+	}
+	//nolint:contextcheck
+	evalParsingContext, err := createTerragruntEvalContext(parser, file.ConfigPath)
+
+	if err != nil {
+		return nil, errors.New(err)
+	}
+
+	values := map[string]cty.Value{}
+
+	if err := file.Decode(&values, evalParsingContext); err != nil {
+		return nil, errors.New(err)
+	}
+
+	result := cty.ObjectVal(values)
+
+	return &result, nil
+}
+
 // ValidateStackConfig validates a StackConfigFile instance according to the rules:
 // - Unit name, source, and path shouldn't be empty
 // - Unit names should be unique
@@ -98,8 +189,9 @@ func ValidateStackConfig(config *StackConfigFile) error {
 
 	validationErrors := &errors.MultiError{}
 
-	names := make(map[string]bool)
-	paths := make(map[string]bool)
+	// Pre-allocate maps with known capacity to avoid resizing
+	names := make(map[string]bool, len(config.Units))
+	paths := make(map[string]bool, len(config.Units))
 
 	for i, unit := range config.Units {
 		name := strings.TrimSpace(unit.Name)