From 043a254c42d4a94cf45a293e07d966a5629a4e9a Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 20 Feb 2020 06:13:59 -0800 Subject: [PATCH 1/2] Make behavior of dependency inclusion consistent --- configstack/module.go | 12 ++-- docs/_docs/04_reference/cli-options.md | 9 ++- .../amazing-app/k8s/terragrunt.hcl | 19 ++++++ .../clusters/eks/terragrunt.hcl | 7 +++ .../exclude-dependency/modules/eks/main.tf | 7 +++ .../exclude-dependency/modules/k8s/main.tf | 2 + .../exclude-dependency/terragrunt.hcl | 1 + .../testapp/k8s/terragrunt.hcl | 19 ++++++ test/integration_test.go | 62 +++++++++++++++++++ 9 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 test/fixture-regressions/exclude-dependency/amazing-app/k8s/terragrunt.hcl create mode 100644 test/fixture-regressions/exclude-dependency/clusters/eks/terragrunt.hcl create mode 100644 test/fixture-regressions/exclude-dependency/modules/eks/main.tf create mode 100644 test/fixture-regressions/exclude-dependency/modules/k8s/main.tf create mode 100644 test/fixture-regressions/exclude-dependency/terragrunt.hcl create mode 100644 test/fixture-regressions/exclude-dependency/testapp/k8s/terragrunt.hcl diff --git a/configstack/module.go b/configstack/module.go index 0bb174c3bb..c30e9224ed 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -181,14 +181,18 @@ func flagIncludedDirs(modules []*TerraformModule, terragruntOptions *options.Ter for _, module := range modules { if findModuleinPath(module, canonicalIncludeDirs) { - // Mark module itself as included module.FlagExcluded = false - // Mark all affected dependencies as included + } else { + module.FlagExcluded = true + } + } + + // Mark all affected dependencies as included before proceeding + for _, module := range modules { + if !module.FlagExcluded { for _, dependency := range module.Dependencies { dependency.FlagExcluded = false } - } else { - module.FlagExcluded = true } } diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index cc0a3f3904..94d80d3f22 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -178,14 +178,19 @@ When passed in, ignore the depedencies between modules when running `*-all` comm **CLI Arg**: `--terragrunt-ignore-external-dependencies` -When passed in, don't attempt to include any external dependencies when running `*-all` commands +When passed in, don't attempt to include any external dependencies when running `*-all` commands. Note that an external +dependency is a dependency that is outside the current terragrunt working directory, and is not respective to the +included directories with `terragrunt-include-dir`. ## terragrunt-include-external-dependencies **CLI Arg**: `--terragrunt-include-external-dependencies` -When passed in, include any external dependencies when running `*-all` without asking. +When passed in, include any external dependencies when running `*-all` without asking. Note that an external +dependency is a dependency that is outside the current terragrunt working directory, and is not respective to the +included directories with `terragrunt-include-dir`. + ## terragrunt-check diff --git a/test/fixture-regressions/exclude-dependency/amazing-app/k8s/terragrunt.hcl b/test/fixture-regressions/exclude-dependency/amazing-app/k8s/terragrunt.hcl new file mode 100644 index 0000000000..0d35b13186 --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/amazing-app/k8s/terragrunt.hcl @@ -0,0 +1,19 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "${get_terragrunt_dir()}/../../modules/k8s" +} + +dependency "eks" { + config_path = "${get_terragrunt_dir()}/../../clusters/eks" + skip_outputs = true + mock_outputs = { + random_string = "foo" + } +} + +inputs = { + cluster = dependency.eks.outputs.random_string +} diff --git a/test/fixture-regressions/exclude-dependency/clusters/eks/terragrunt.hcl b/test/fixture-regressions/exclude-dependency/clusters/eks/terragrunt.hcl new file mode 100644 index 0000000000..7acbc0a237 --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/clusters/eks/terragrunt.hcl @@ -0,0 +1,7 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "${get_terragrunt_dir()}/../../modules/eks" +} diff --git a/test/fixture-regressions/exclude-dependency/modules/eks/main.tf b/test/fixture-regressions/exclude-dependency/modules/eks/main.tf new file mode 100644 index 0000000000..d7414aefef --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/modules/eks/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "random" { + length = 16 +} + +output "random_string" { + value = random_string.random.result +} diff --git a/test/fixture-regressions/exclude-dependency/modules/k8s/main.tf b/test/fixture-regressions/exclude-dependency/modules/k8s/main.tf new file mode 100644 index 0000000000..7077ac6f0b --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/modules/k8s/main.tf @@ -0,0 +1,2 @@ +variable "cluster" {} +output "cluster" { value = var.cluster } diff --git a/test/fixture-regressions/exclude-dependency/terragrunt.hcl b/test/fixture-regressions/exclude-dependency/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-regressions/exclude-dependency/testapp/k8s/terragrunt.hcl b/test/fixture-regressions/exclude-dependency/testapp/k8s/terragrunt.hcl new file mode 100644 index 0000000000..b43099351b --- /dev/null +++ b/test/fixture-regressions/exclude-dependency/testapp/k8s/terragrunt.hcl @@ -0,0 +1,19 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "${get_terragrunt_dir()}/../../modules/k8s" +} + +dependency "eks" { + config_path = "${get_terragrunt_dir()}/../../clusters/eks" + skip_outputs = true + mock_outputs = { + random_string = "foo" + } +} + +inputs = { + cluster = dependency.eks.outputs.random_string +} diff --git a/test/integration_test.go b/test/integration_test.go index fc299d4679..a8af406ea9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "testing" "time" @@ -1537,6 +1538,28 @@ func TestIncludeDirs(t *testing.T) { } } +func TestIncludeDirsDependencyConsistencyRegression(t *testing.T) { + t.Parallel() + + modulePaths := []string{ + "amazing-app/k8s", + "clusters/eks", + "testapp/k8s", + } + + testPath := filepath.Join(TEST_FIXTURE_REGRESSIONS, "exclude-dependency") + cleanupTerragruntFolder(t, testPath) + for _, modulePath := range modulePaths { + cleanupTerragruntFolder(t, filepath.Join(testPath, modulePath)) + } + + includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s") + assert.Equal(t, includedModulesWithAmzApp, []string{"amazing-app/k8s", "clusters/eks"}) + + includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s") + assert.Equal(t, includedModulesWithTestApp, []string{"clusters/eks", "testapp/k8s"}) +} + func TestTerragruntExternalDependencies(t *testing.T) { t.Parallel() @@ -3039,3 +3062,42 @@ func fileIsInFolder(t *testing.T, name string, path string) bool { require.NoError(t, err) return found } + +func runValidateAllWithIncludeAndGetIncludedModules(t *testing.T, rootModulePath string, includeModulePath string) []string { + validateAllStdout := bytes.Buffer{} + validateAllStderr := bytes.Buffer{} + err := runTerragruntCommand( + t, + fmt.Sprintf( + "terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-include-dir %s", + rootModulePath, + includeModulePath, + ), + &validateAllStdout, + &validateAllStderr, + ) + logBufferContentsLineByLine(t, validateAllStdout, "validate-all stdout") + logBufferContentsLineByLine(t, validateAllStderr, "validate-all stderr") + require.NoError(t, err) + + currentDir, err := os.Getwd() + require.NoError(t, err) + + includedModulesRegexp, err := regexp.Compile( + fmt.Sprintf( + `=> Module %s/%s/(.+) \(excluded: (true|false)`, + currentDir, + rootModulePath, + ), + ) + require.NoError(t, err) + matches := includedModulesRegexp.FindAllStringSubmatch(string(validateAllStderr.Bytes()), -1) + includedModules := []string{} + for _, match := range matches { + if match[2] == "false" { + includedModules = append(includedModules, match[1]) + } + } + sort.Strings(includedModules) + return includedModules +} From 1d8dfb025bbedd7270345653542e3bfc4dae1733 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 20 Feb 2020 06:30:15 -0800 Subject: [PATCH 2/2] Introduce new flag to ignore dependencies for include dir --- cli/args.go | 3 ++ cli/cli_app.go | 25 +++++++++++++-- configstack/module.go | 12 +++++--- docs/_docs/04_reference/cli-options.md | 10 ++++++ options/options.go | 5 +++ test/integration_test.go | 42 +++++++++++++++++++++----- 6 files changed, 82 insertions(+), 15 deletions(-) diff --git a/cli/args.go b/cli/args.go index 2043149ccf..8440b0873e 100644 --- a/cli/args.go +++ b/cli/args.go @@ -104,6 +104,8 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer) return nil, err } + strictInclude := parseBooleanArg(args, OPT_TERRAGRUNT_STRICT_INCLUDE, false) + opts, err := options.NewTerragruntOptions(filepath.ToSlash(terragruntConfigPath)) if err != nil { return nil, err @@ -131,6 +133,7 @@ func parseTerragruntOptionsFromArgs(args []string, writer, errWriter io.Writer) opts.IamRole = iamRole opts.ExcludeDirs = excludeDirs opts.IncludeDirs = includeDirs + opts.StrictInclude = strictInclude opts.Check = parseBooleanArg(args, OPT_TERRAGRUNT_CHECK, os.Getenv("TERRAGRUNT_CHECK") == "false") return opts, nil diff --git a/cli/cli_app.go b/cli/cli_app.go index c0cfab5c8f..eda5d59c15 100644 --- a/cli/cli_app.go +++ b/cli/cli_app.go @@ -38,10 +38,31 @@ const OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES = "terragrunt-ignore-external- const OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES = "terragrunt-include-external-dependencies" const OPT_TERRAGRUNT_EXCLUDE_DIR = "terragrunt-exclude-dir" const OPT_TERRAGRUNT_INCLUDE_DIR = "terragrunt-include-dir" +const OPT_TERRAGRUNT_STRICT_INCLUDE = "terragrunt-strict-include" const OPT_TERRAGRUNT_CHECK = "terragrunt-check" -var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{OPT_NON_INTERACTIVE, OPT_TERRAGRUNT_SOURCE_UPDATE, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ORDER, OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES, OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES, OPT_TERRAGRUNT_NO_AUTO_INIT, OPT_TERRAGRUNT_NO_AUTO_RETRY, OPT_TERRAGRUNT_CHECK} -var ALL_TERRAGRUNT_STRING_OPTS = []string{OPT_TERRAGRUNT_CONFIG, OPT_TERRAGRUNT_TFPATH, OPT_WORKING_DIR, OPT_DOWNLOAD_DIR, OPT_TERRAGRUNT_SOURCE, OPT_TERRAGRUNT_IAM_ROLE, OPT_TERRAGRUNT_EXCLUDE_DIR, OPT_TERRAGRUNT_INCLUDE_DIR} +var ALL_TERRAGRUNT_BOOLEAN_OPTS = []string{ + OPT_NON_INTERACTIVE, + OPT_TERRAGRUNT_SOURCE_UPDATE, + OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ERRORS, + OPT_TERRAGRUNT_IGNORE_DEPENDENCY_ORDER, + OPT_TERRAGRUNT_IGNORE_EXTERNAL_DEPENDENCIES, + OPT_TERRAGRUNT_INCLUDE_EXTERNAL_DEPENDENCIES, + OPT_TERRAGRUNT_NO_AUTO_INIT, + OPT_TERRAGRUNT_NO_AUTO_RETRY, + OPT_TERRAGRUNT_CHECK, + OPT_TERRAGRUNT_STRICT_INCLUDE, +} +var ALL_TERRAGRUNT_STRING_OPTS = []string{ + OPT_TERRAGRUNT_CONFIG, + OPT_TERRAGRUNT_TFPATH, + OPT_WORKING_DIR, + OPT_DOWNLOAD_DIR, + OPT_TERRAGRUNT_SOURCE, + OPT_TERRAGRUNT_IAM_ROLE, + OPT_TERRAGRUNT_EXCLUDE_DIR, + OPT_TERRAGRUNT_INCLUDE_DIR, +} const CMD_PLAN_ALL = "plan-all" const CMD_APPLY_ALL = "apply-all" diff --git a/configstack/module.go b/configstack/module.go index c30e9224ed..b20e55991d 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -187,11 +187,13 @@ func flagIncludedDirs(modules []*TerraformModule, terragruntOptions *options.Ter } } - // Mark all affected dependencies as included before proceeding - for _, module := range modules { - if !module.FlagExcluded { - for _, dependency := range module.Dependencies { - dependency.FlagExcluded = false + // Mark all affected dependencies as included before proceeding if not in strict include mode. + if !terragruntOptions.StrictInclude { + for _, module := range modules { + if !module.FlagExcluded { + for _, dependency := range module.Dependencies { + dependency.FlagExcluded = false + } } } } diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md index 94d80d3f22..ab1ca77f74 100644 --- a/docs/_docs/04_reference/cli-options.md +++ b/docs/_docs/04_reference/cli-options.md @@ -26,6 +26,7 @@ Terragrunt forwards all arguments and options to Terraform. The only exceptions - [terragrunt-iam-role](#terragrunt-iam-role) - [terragrunt-exclude-dir](#terragrunt-exclude-dir) - [terragrunt-include-dir](#terragrunt-include-dir) +- [terragrunt-strict-include](#terragrunt-strict-include) - [terragrunt-ignore-dependency-order](#terragrunt-ignore-dependency-order) - [terragrunt-ignore-external-dependencies](#terragrunt-ignore-external-dependencies) - [terragrunt-include-external-dependencies](#terragrunt-include-external-dependencies) @@ -167,6 +168,15 @@ dependent modules) will be included during execution of the commands. If a relat relative from `--terragrunt-working-dir`. Flag can be specified multiple times. +## terragrunt-strict-include + +**CLI Arg**: `--terragrunt-strict-include` + +When passed in, only modules under the directories passed in with [--terragrunt-include-dir](#terragrunt-include-dir) +will be included. All dependencies of the included directories will be excluded if they are not in the included +directories. + + ## terragrunt-ignore-dependency-order **CLI Arg**: `--terragrunt-ignore-dependency-order` diff --git a/options/options.go b/options/options.go index c3c3383a18..eec912e383 100644 --- a/options/options.go +++ b/options/options.go @@ -111,6 +111,9 @@ type TerragruntOptions struct { // Unix-style glob of directories to include when running *-all commands IncludeDirs []string + // If set to true, do not include dependencies when processing IncludeDirs (unless they are in the included dirs) + StrictInclude bool + // Enable check mode, by default it's disabled. Check bool @@ -157,6 +160,7 @@ func NewTerragruntOptions(terragruntConfigPath string) (*TerragruntOptions, erro RetryableErrors: util.CloneStringList(RETRYABLE_ERRORS), ExcludeDirs: []string{}, IncludeDirs: []string{}, + StrictInclude: false, Check: false, RunTerragrunt: func(terragruntOptions *TerragruntOptions) error { return errors.WithStackTrace(RunTerragruntCommandNotSet) @@ -227,6 +231,7 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) * RetryableErrors: util.CloneStringList(terragruntOptions.RetryableErrors), ExcludeDirs: terragruntOptions.ExcludeDirs, IncludeDirs: terragruntOptions.IncludeDirs, + StrictInclude: terragruntOptions.StrictInclude, RunTerragrunt: terragruntOptions.RunTerragrunt, } } diff --git a/test/integration_test.go b/test/integration_test.go index a8af406ea9..b932c70f79 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1553,13 +1553,35 @@ func TestIncludeDirsDependencyConsistencyRegression(t *testing.T) { cleanupTerragruntFolder(t, filepath.Join(testPath, modulePath)) } - includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s") + includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s", false) assert.Equal(t, includedModulesWithAmzApp, []string{"amazing-app/k8s", "clusters/eks"}) - includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s") + includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s", false) assert.Equal(t, includedModulesWithTestApp, []string{"clusters/eks", "testapp/k8s"}) } +func TestIncludeDirsStrict(t *testing.T) { + t.Parallel() + + modulePaths := []string{ + "amazing-app/k8s", + "clusters/eks", + "testapp/k8s", + } + + testPath := filepath.Join(TEST_FIXTURE_REGRESSIONS, "exclude-dependency") + cleanupTerragruntFolder(t, testPath) + for _, modulePath := range modulePaths { + cleanupTerragruntFolder(t, filepath.Join(testPath, modulePath)) + } + + includedModulesWithAmzApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "amazing-app/k8s", true) + assert.Equal(t, includedModulesWithAmzApp, []string{"amazing-app/k8s"}) + + includedModulesWithTestApp := runValidateAllWithIncludeAndGetIncludedModules(t, testPath, "testapp/k8s", true) + assert.Equal(t, includedModulesWithTestApp, []string{"testapp/k8s"}) +} + func TestTerragruntExternalDependencies(t *testing.T) { t.Parallel() @@ -3063,16 +3085,20 @@ func fileIsInFolder(t *testing.T, name string, path string) bool { return found } -func runValidateAllWithIncludeAndGetIncludedModules(t *testing.T, rootModulePath string, includeModulePath string) []string { +func runValidateAllWithIncludeAndGetIncludedModules(t *testing.T, rootModulePath string, includeModulePath string, strictInclude bool) []string { + cmd := fmt.Sprintf( + "terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-include-dir %s", + rootModulePath, + includeModulePath, + ) + if strictInclude { + cmd = cmd + " --terragrunt-strict-include" + } validateAllStdout := bytes.Buffer{} validateAllStderr := bytes.Buffer{} err := runTerragruntCommand( t, - fmt.Sprintf( - "terragrunt validate-all --terragrunt-non-interactive --terragrunt-working-dir %s --terragrunt-include-dir %s", - rootModulePath, - includeModulePath, - ), + cmd, &validateAllStdout, &validateAllStderr, )