add plugin tar inclusion

Signed-off-by: Clément Legrand <[email protected]>

USERS.md

@@ -121,6 +121,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Grupo MasMovil](https://grupomasmovil.com/en/)
 1. [Handelsbanken](https://www.handelsbanken.se)
 1. [Healy](https://www.healyworld.net)
+1. [Heetch](https://www.heetch.com)
 1. [Helio](https://helio.exchange)
 1. [Hetki](https://hetki.ai)
 1. [hipages](https://hipages.com.au/)

cmd/argocd-repo-server/commands/argocd_repo_server.go

@@ -64,6 +64,7 @@ func NewCommand() *cobra.Command {
 		disableTLS                        bool
 		maxCombinedDirectoryManifestsSize string
 		cmpTarExcludedGlobs               []string
+		cmpTarIncludedGlobs               []string
 		allowOutOfBoundsSymlinks          bool
 		streamedManifestMaxTarSize        string
 		streamedManifestMaxExtractedSize  string
@@ -121,6 +122,7 @@ func NewCommand() *cobra.Command {
 				SubmoduleEnabled:                             gitSubmoduleEnabled,
 				MaxCombinedDirectoryManifestsSize:            maxCombinedDirectoryManifestsQuantity,
 				CMPTarExcludedGlobs:                          cmpTarExcludedGlobs,
+				CMPTarIncludedGlobs:                          cmpTarIncludedGlobs,
 				AllowOutOfBoundsSymlinks:                     allowOutOfBoundsSymlinks,
 				StreamedManifestMaxExtractedSize:             streamedManifestMaxExtractedSizeQuantity.ToDec().Value(),
 				StreamedManifestMaxTarSize:                   streamedManifestMaxTarSizeQuantity.ToDec().Value(),
@@ -204,6 +206,7 @@ func NewCommand() *cobra.Command {
 	command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS on the gRPC endpoint")
 	command.Flags().StringVar(&maxCombinedDirectoryManifestsSize, "max-combined-directory-manifests-size", env.StringFromEnv("ARGOCD_REPO_SERVER_MAX_COMBINED_DIRECTORY_MANIFESTS_SIZE", "10M"), "Max combined size of manifest files in a directory-type Application")
 	command.Flags().StringArrayVar(&cmpTarExcludedGlobs, "plugin-tar-exclude", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS", []string{}, ";"), "Globs to filter when sending tarballs to plugins.")
+	command.Flags().StringArrayVar(&cmpTarIncludedGlobs, "plugin-tar-include", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS", []string{}, ";"), "Globs to include files when sending tarballs to plugins.")
 	command.Flags().BoolVar(&allowOutOfBoundsSymlinks, "allow-oob-symlinks", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS", false), "Allow out-of-bounds symlinks in repositories (not recommended)")
 	command.Flags().StringVar(&streamedManifestMaxTarSize, "streamed-manifest-max-tar-size", env.StringFromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE", "100M"), "Maximum size of streamed manifest archives")
 	command.Flags().StringVar(&streamedManifestMaxExtractedSize, "streamed-manifest-max-extracted-size", env.StringFromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of streamed manifest archives when extracted")

cmd/argocd/commands/app.go

@@ -1136,7 +1136,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 	command.Flags().StringVar(&revision, "revision", "", "Compare live app to a particular revision")
 	command.Flags().StringVar(&localRepoRoot, "local-repo-root", "/", "Path to the repository root. Used together with --local allows setting the repository root")
 	command.Flags().BoolVar(&serverSideGenerate, "server-side-generate", false, "Used with --local, this will send your manifests to the server for diffing")
-	command.Flags().StringArrayVar(&localIncludes, "local-include", []string{"*.yaml", "*.yml", "*.json"}, "Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path.")
+	command.Flags().StringArrayVar(&localIncludes, "local-include", []string{"**/*.yaml", "**/*.yml", "**/*.json"}, "Used with --server-side-generate, specify patterns of filenames to send. It does support glob patterns")
 	return command
 }
 

cmpserver/plugin/plugin_test.go

@@ -541,7 +541,7 @@ type MockGenerateManifestStream struct {
 }
 
 func NewMockGenerateManifestStream(repoPath, appPath string, env []string) (*MockGenerateManifestStream, error) {
-	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil)
+	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -615,7 +615,7 @@ type MockMatchRepositoryStream struct {
 }
 
 func NewMockMatchRepositoryStream(repoPath, appPath string, env []string) (*MockMatchRepositoryStream, error) {
-	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil)
+	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -688,7 +688,7 @@ type MockParametersAnnouncementStream struct {
 }
 
 func NewMockParametersAnnouncementStream(repoPath, appPath string, env []string) (*MockParametersAnnouncementStream, error) {
-	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil)
+	tgz, mr, err := cmp.GetCompressedRepoAndMetadata(repoPath, appPath, env, nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}

docs/operator-manual/config-management-plugins.md

@@ -344,7 +344,7 @@ If you are actively developing a sidecar-installed CMP, keep a few things in min
 | -- | -- |
 | `no matches for kind "ConfigManagementPlugin" in version "argoproj.io/v1alpha1"` | The `ConfigManagementPlugin` CRD was deprecated in Argo CD 2.4 and removed in 2.8. This error means you've tried to put the configuration for your plugin directly into Kubernetes as a CRD. Refer to this [section of documentation](#write-the-plugin-configuration-file) for how to write the plugin configuration file and place it properly in the sidecar. |
 
-## Plugin tar stream exclusions
+## Plugin tar stream exclusions or inclusions
 
 In order to increase the speed of manifest generation, certain files and folders can be excluded from being sent to your
 plugin. We recommend excluding your `.git` folder if it isn't necessary. Use Go's
@@ -356,6 +356,16 @@ You can set it one of three ways:
 2. The `reposerver.plugin.tar.exclusions` key if you are using `argocd-cmd-params-cm`
 3. Directly setting `ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS` environment variable on the repo server.
 
+In addition, if you're using a monorepo mixed with code and GitOps resources, you probably want to only include files 
+involved in manifests generation in the tar archive. Use Go's [filepatch.Match](https://pkg.go.dev/path/filepath#Match) syntax to include
+desired files. For example, `*.y*ml` to include `.yaml` or `.yml` files.
+
+You can set it one of three ways:
+
+1. The `--plugin-tar-include` argument on the repo server.
+2. The `reposerver.plugin.tar.inclusions` key if you are using `argocd-cmd-params-cm`
+3. Directly setting `ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS` environment variable on the repo server.
+
 For option 1, the flag can be repeated multiple times. For option 2 and 3, you can specify multiple globs by separating
 them with semicolons.
 

docs/operator-manual/server-commands/argocd-repo-server.md

@@ -33,6 +33,7 @@ argocd-repo-server [flags]
       --otlp-insecure                                  OpenTelemetry collector insecure mode (default true)
       --parallelismlimit int                           Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.
       --plugin-tar-exclude stringArray                 Globs to filter when sending tarballs to plugins.
+      --plugin-tar-include stringArray                 Globs to include files when sending tarballs to plugins.
       --port int                                       Listen on given port for incoming connections (default 8081)
       --redis string                                   Redis server hostname and port (e.g. argocd-redis:6379). 
       --redis-ca-certificate string                    Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.

docs/user-guide/commands/argocd_app_diff.md

@@ -21,7 +21,7 @@ argocd app diff APPNAME [flags]
       --hard-refresh                Refresh application data as well as target manifests cache
   -h, --help                        help for diff
       --local string                Compare live app to a local manifests
-      --local-include stringArray   Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path. (default [*.yaml,*.yml,*.json])
+      --local-include stringArray   Used with --server-side-generate, specify patterns of filenames to send. It does support glob patterns (default [**/*.yaml,**/*.yml,**/*.json])
       --local-repo-root string      Path to the repository root. Used together with --local allows setting the repository root (default "/")
       --refresh                     Refresh application data when retrieving
       --revision string             Compare live app to a particular revision

go.sum

@@ -753,6 +753,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
+github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
 github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM=

manifests/base/repo-server/argocd-repo-server-deployment.yaml

@@ -144,6 +144,12 @@ spec:
                 name: argocd-cmd-params-cm
                 key: reposerver.plugin.tar.exclusions
                 optional: true
+          - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+            valueFrom:
+              configMapKeyRef:
+                key: reposerver.plugin.tar.inclusions
+                name: argocd-cmd-params-cm
+                optional: true
           - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
             valueFrom:
               configMapKeyRef:

manifests/core-install.yaml

@@ -21309,6 +21309,12 @@ spec:
               key: reposerver.plugin.tar.exclusions
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.plugin.tar.inclusions
+              name: argocd-cmd-params-cm
+              optional: true
         - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
           valueFrom:
             configMapKeyRef:

manifests/ha/install.yaml

@@ -22789,6 +22789,12 @@ spec:
               key: reposerver.plugin.tar.exclusions
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.plugin.tar.inclusions
+              name: argocd-cmd-params-cm
+              optional: true
         - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
           valueFrom:
             configMapKeyRef:

manifests/ha/namespace-install.yaml

@@ -2174,6 +2174,12 @@ spec:
               key: reposerver.plugin.tar.exclusions
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.plugin.tar.inclusions
+              name: argocd-cmd-params-cm
+              optional: true
         - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
           valueFrom:
             configMapKeyRef:

manifests/install.yaml

@@ -21835,6 +21835,12 @@ spec:
               key: reposerver.plugin.tar.exclusions
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.plugin.tar.inclusions
+              name: argocd-cmd-params-cm
+              optional: true
         - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
           valueFrom:
             configMapKeyRef:

manifests/namespace-install.yaml

@@ -1220,6 +1220,12 @@ spec:
               key: reposerver.plugin.tar.exclusions
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_PLUGIN_TAR_INCLUSIONS
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.plugin.tar.inclusions
+              name: argocd-cmd-params-cm
+              optional: true
         - name: ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS
           valueFrom:
             configMapKeyRef: