Merge pull request gruntwork-io#1062 from gruntwork-io/yori-fix-depen…

…dency-exclude

Make include dirs consistent when dependencies are involved

cli/args.go

@@ -104,6 +104,8 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer)
 		return nil, err
 	}
 
+	strictInclude := parseBooleanArg(args, OPT_TERRAGRUNT_STRICT_INCLUDE, false)
+
 	opts, err := options.NewTerragruntOptions(filepath.ToSlash(terragruntConfigPath))
 	if err != nil {
 		return nil, err
@@ -131,6 +133,7 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer)
 	opts.IamRole = iamRole
 	opts.ExcludeDirs = excludeDirs
 	opts.IncludeDirs = includeDirs
+	opts.StrictInclude = strictInclude
 	opts.Check = parseBooleanArg(args, OPT_TERRAGRUNT_CHECK, os.Getenv("TERRAGRUNT_CHECK") == "false")
 
 	return opts, nil

cli/cli_app.go

@@ -38,10 +38,31 @@ const OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES = "terragrunt-ignore-external-
 const OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES = "terragrunt-include-external-dependencies"
 const OPT_TERRAGRUNT_EXCLUDE_DIR = "terragrunt-exclude-dir"
 const OPT_TERRAGRUNT_INCLUDE_DIR = "terragrunt-include-dir"
+const OPT_TERRAGRUNT_STRICT_INCLUDE = "terragrunt-strict-include"
 const OPT_TERRAGRUNT_CHECK = "terragrunt-check"
 
-var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ORDER, OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES, OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES, OPT_TERRAGRUNT_NO_AUTO_INIT, OPT_TERRAGRUNT_NO_AUTO_RETRY, OPT_TERRAGRUNT_CHECK}
-var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_DOWNLOAD_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE, OPT_TERRAGRUNT_EXCLUDE_DIR, OPT_TERRAGRUNT_INCLUDE_DIR}
+var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{
+	OPT_NON_INTERACTIVE,
+	OPT_TERRAGRUNT_SOURCE_UPDATE,
+	OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS,
+	OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ORDER,
+	OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES,
+	OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES,
+	OPT_TERRAGRUNT_NO_AUTO_INIT,
+	OPT_TERRAGRUNT_NO_AUTO_RETRY,
+	OPT_TERRAGRUNT_CHECK,
+	OPT_TERRAGRUNT_STRICT_INCLUDE,
+}
+var ALL_TERRAGRUNT_STRING_OPTS = []string{
+	OPT_TERRAGRUNT_CONFIG,
+	OPT_TERRAGRUNT_TFPATH,
+	OPT_WORKING_DIR,
+	OPT_DOWNLOAD_DIR,
+	OPT_TERRAGRUNT_SOURCE,
+	OPT_TERRAGRUNT_IAM_ROLE,
+	OPT_TERRAGRUNT_EXCLUDE_DIR,
+	OPT_TERRAGRUNT_INCLUDE_DIR,
+}
 
 const CMD_PLAN_ALL = "plan-all"
 const CMD_APPLY_ALL = "apply-all"

configstack/module.go

@@ -181,17 +181,23 @@ func flagIncludedDirs(modules []*TerraformModule, terragruntOptions *options.Ter
 
 	for _, module := range modules {
 		if findModuleinPath(module, canonicalIncludeDirs) {
-			// Mark module itself as included
 			module.FlagExcluded = false
-			// Mark all affected dependencies as included
-			for _, dependency := range module.Dependencies {
-				dependency.FlagExcluded = false
-			}
 		} else {
 			module.FlagExcluded = true
 		}
 	}
 
+	// Mark all affected dependencies as included before proceeding if not in strict include mode.
+	if !terragruntOptions.StrictInclude {
+		for _, module := range modules {
+			if !module.FlagExcluded {
+				for _, dependency := range module.Dependencies {
+					dependency.FlagExcluded = false
+				}
+			}
+		}
+	}
+
 	return modules, nil
 }
 

docs/_docs/04_reference/cli-options.md

@@ -26,6 +26,7 @@ Terragrunt forwards all arguments and options to Terraform. The only exceptions
 - [terragrunt-iam-role](#terragrunt-iam-role)
 - [terragrunt-exclude-dir](#terragrunt-exclude-dir)
 - [terragrunt-include-dir](#terragrunt-include-dir)
+- [terragrunt-strict-include](#terragrunt-strict-include)
 - [terragrunt-ignore-dependency-order](#terragrunt-ignore-dependency-order)
 - [terragrunt-ignore-external-dependencies](#terragrunt-ignore-external-dependencies)
 - [terragrunt-include-external-dependencies](#terragrunt-include-external-dependencies)
@@ -167,6 +168,15 @@ dependent modules) will be included during execution of the commands. If a relat
 relative from `--terragrunt-working-dir`. Flag can be specified multiple times.
 
 
+## terragrunt-strict-include
+
+**CLI Arg**: `--terragrunt-strict-include`
+
+When passed in, only modules under the directories passed in with [--terragrunt-include-dir](#terragrunt-include-dir)
+will be included. All dependencies of the included directories will be excluded if they are not in the included
+directories.
+
+
 ## terragrunt-ignore-dependency-order
 
 **CLI Arg**: `--terragrunt-ignore-dependency-order`
@@ -178,14 +188,19 @@ When passed in, ignore the depedencies between modules when running `*-all` comm
 
 **CLI Arg**: `--terragrunt-ignore-external-dependencies`
 
-When passed in, don't attempt to include any external dependencies when running `*-all` commands
+When passed in, don't attempt to include any external dependencies when running `*-all` commands. Note that an external
+dependency is a dependency that is outside the current terragrunt working directory, and is not respective to the
+included directories with `terragrunt-include-dir`.
 
 
 ## terragrunt-include-external-dependencies
 
 **CLI Arg**: `--terragrunt-include-external-dependencies`
 
-When passed in, include any external dependencies when running `*-all` without asking.
+When passed in, include any external dependencies when running `*-all` without asking. Note that an external
+dependency is a dependency that is outside the current terragrunt working directory, and is not respective to the
+included directories with `terragrunt-include-dir`.
+
 
 
 ## terragrunt-check

options/options.go

@@ -111,6 +111,9 @@ type TerragruntOptions struct {
 	// Unix-style glob of directories to include when running *-all commands
 	IncludeDirs []string
 
+	// If set to true, do not include dependencies when processing IncludeDirs (unless they are in the included dirs)
+	StrictInclude bool
+
 	// Enable check mode, by default it's disabled.
 	Check bool
 
@@ -157,6 +160,7 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro
 		RetryableErrors:             util.CloneStringList(RETRYABLE_ERRORS),
 		ExcludeDirs:                 []string{},
 		IncludeDirs:                 []string{},
+		StrictInclude:               false,
 		Check:                       false,
 		RunTerragrunt: func(terragruntOptions *TerragruntOptions) error {
 			return errors.WithStackTrace(RunTerragruntCommandNotSet)
@@ -227,6 +231,7 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *
 		RetryableErrors:             util.CloneStringList(terragruntOptions.RetryableErrors),
 		ExcludeDirs:                 terragruntOptions.ExcludeDirs,
 		IncludeDirs:                 terragruntOptions.IncludeDirs,
+		StrictInclude:               terragruntOptions.StrictInclude,
 		RunTerragrunt:               terragruntOptions.RunTerragrunt,
 	}
 }

test/fixture-regressions/exclude-dependency/amazing-app/k8s/terragrunt.hcl

@@ -0,0 +1,19 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_terragrunt_dir()}/../../modules/k8s"
+}
+
+dependency "eks" {
+  config_path  = "${get_terragrunt_dir()}/../../clusters/eks"
+  skip_outputs = true
+  mock_outputs = {
+    random_string = "foo"
+  }
+}
+
+inputs = {
+  cluster = dependency.eks.outputs.random_string
+}

test/fixture-regressions/exclude-dependency/clusters/eks/terragrunt.hcl

@@ -0,0 +1,7 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_terragrunt_dir()}/../../modules/eks"
+}

test/fixture-regressions/exclude-dependency/modules/eks/main.tf

@@ -0,0 +1,7 @@
+resource "random_string" "random" {
+  length = 16
+}
+
+output "random_string" {
+  value = random_string.random.result
+}

test/fixture-regressions/exclude-dependency/modules/k8s/main.tf

@@ -0,0 +1,2 @@
+variable "cluster" {}
+output "cluster" { value = var.cluster }

test/fixture-regressions/exclude-dependency/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty

test/fixture-regressions/exclude-dependency/testapp/k8s/terragrunt.hcl

@@ -0,0 +1,19 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_terragrunt_dir()}/../../modules/k8s"
+}
+
+dependency "eks" {
+  config_path = "${get_terragrunt_dir()}/../../clusters/eks"
+  skip_outputs = true
+  mock_outputs = {
+    random_string = "foo"
+  }
+}
+
+inputs = {
+  cluster = dependency.eks.outputs.random_string
+}

test/integration_test.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -1537,6 +1538,50 @@ func TestIncludeDirs(t *testing.T) {
 	}
 }
 
+func TestIncludeDirsDependencyConsistencyRegression(t *testing.T) {
+	t.Parallel()
+
+	modulePaths := []string{
+		"amazing-app/k8s",
+		"clusters/eks",
+		"testapp/k8s",
+	}
+
+	testPath := filepath.Join(TEST_FIXTURE_REGRESSIONS, "exclude-dependency")
+	cleanupTerragruntFolder(t, testPath)
+	for _, modulePath := range modulePaths {
+		cleanupTerragruntFolder(t, filepath.Join(testPath, modulePath))
+	}
+
+	includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s", false)
+	assert.Equal(t, includedModulesWithAmzApp, []string{"amazing-app/k8s", "clusters/eks"})
+
+	includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s", false)
+	assert.Equal(t, includedModulesWithTestApp, []string{"clusters/eks", "testapp/k8s"})
+}
+
+func TestIncludeDirsStrict(t *testing.T) {
+	t.Parallel()
+
+	modulePaths := []string{
+		"amazing-app/k8s",
+		"clusters/eks",
+		"testapp/k8s",
+	}
+
+	testPath := filepath.Join(TEST_FIXTURE_REGRESSIONS, "exclude-dependency")
+	cleanupTerragruntFolder(t, testPath)
+	for _, modulePath := range modulePaths {
+		cleanupTerragruntFolder(t, filepath.Join(testPath, modulePath))
+	}
+
+	includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s", true)
+	assert.Equal(t, includedModulesWithAmzApp, []string{"amazing-app/k8s"})
+
+	includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s", true)
+	assert.Equal(t, includedModulesWithTestApp, []string{"testapp/k8s"})
+}
+
 func TestTerragruntExternalDependencies(t *testing.T) {
 	t.Parallel()
 
@@ -3039,3 +3084,46 @@ func fileIsInFolder(t *testing.T, name string, path string) bool {
 	require.NoError(t, err)
 	return found
 }
+
+func runValidateAllWithIncludeAndGetIncludedModules(t *testing.T, rootModulePath string, includeModulePath string, strictInclude bool) []string {
+	cmd := fmt.Sprintf(
+		"terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-include-dir %s",
+		rootModulePath,
+		includeModulePath,
+	)
+	if strictInclude {
+		cmd = cmd + " --terragrunt-strict-include"
+	}
+	validateAllStdout := bytes.Buffer{}
+	validateAllStderr := bytes.Buffer{}
+	err := runTerragruntCommand(
+		t,
+		cmd,
+		&validateAllStdout,
+		&validateAllStderr,
+	)
+	logBufferContentsLineByLine(t, validateAllStdout, "validate-all stdout")
+	logBufferContentsLineByLine(t, validateAllStderr, "validate-all stderr")
+	require.NoError(t, err)
+
+	currentDir, err := os.Getwd()
+	require.NoError(t, err)
+
+	includedModulesRegexp, err := regexp.Compile(
+		fmt.Sprintf(
+			`=> Module %s/%s/(.+) \(excluded: (true|false)`,
+			currentDir,
+			rootModulePath,
+		),
+	)
+	require.NoError(t, err)
+	matches := includedModulesRegexp.FindAllStringSubmatch(string(validateAllStderr.Bytes()), -1)
+	includedModules := []string{}
+	for _, match := range matches {
+		if match[2] == "false" {
+			includedModules = append(includedModules, match[1])
+		}
+	}
+	sort.Strings(includedModules)
+	return includedModules
+}