Net 1783 Add PostStart hook to connect inject

charts/consul/templates/connect-inject-deployment.yaml

@@ -256,7 +256,8 @@ spec:
                 -default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds={{ .Values.connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds }} \
                 -default-sidecar-proxy-lifecycle-graceful-port={{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulPort }} \
                 -default-sidecar-proxy-lifecycle-graceful-shutdown-path="{{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulShutdownPath }}" \
-
+                -default-sidecar-proxy-lifecycle-startup-grace-period-seconds={{ .Values.connectInject.sidecarProxy.lifecycle.defaultStartupGracePeriodSeconds }} \
+                -default-sidecar-proxy-lifecycle-graceful-startup-path={{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulStartupPath | quote }} \
                 {{- if .Values.connectInject.initContainer }}
                 {{- $initResources := .Values.connectInject.initContainer.resources }}
                 {{- if not (kindIs "invalid" $initResources.limits.memory) }}

charts/consul/values.yaml

@@ -2684,6 +2684,8 @@ connectInject:
     # - `consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds`
     # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port`
     # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path`
+    # - `consul.hashicorp.com/sidecar-proxy-lifecycle-startup-grace-period-seconds`
+    # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-startup-path`
     # @type: map
     lifecycle:
       # @type: boolean
@@ -2696,6 +2698,10 @@ connectInject:
       defaultGracefulPort: 20600
       # @type: string
       defaultGracefulShutdownPath: "/graceful_shutdown"
+      # @type: integer
+      defaultStartupGracePeriodSeconds: 0
+      # @type: string
+      defaultGracefulStartupPath: "/graceful_startup"
 
   # The resource settings for the Connect injected init container. If null, the resources
   # won't be set for the initContainer. The defaults are optimized for developer instances of

cli/helm/values.go

@@ -438,6 +438,8 @@ type Lifecycle struct {
 	DefaultShutdownGracePeriodSeconds   int    `yaml:"defaultShutdownGracePeriodSeconds"`
 	DefaultGracefulPort                 int    `yaml:"defaultGracefulPort"`
 	DefaultGracefulShutdownPath         string `yaml:"defaultGracefulShutdownPath"`
+	DefaultStartupGracePeriodSeconds    int    `yaml:"defaultStartupGracePeriodSeconds"`
+	DefaultGracefulStartupPath          string `yaml:"defaultGracefulStartupPath"`
 }
 
 type ConnectInject struct {

control-plane/connect-inject/constants/annotations_and_labels.go

@@ -106,6 +106,8 @@ const (
 	AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds   = "consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds"
 	AnnotationSidecarProxyLifecycleGracefulPort                 = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port"
 	AnnotationSidecarProxyLifecycleGracefulShutdownPath         = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path"
+	AnnotationSidecarProxyLifecycleGracefulStartupPath          = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-startup-path"
+	AnnotationSidecarProxyLifecycleStartupGracePeriodSeconds    = "consul.hashicorp.com/sidecar-proxy-lifecycle-startup-grace-period-seconds"
 
 	// annotations for sidecar volumes.
 	AnnotationConsulSidecarUserVolume      = "consul.hashicorp.com/consul-sidecar-user-volume"

control-plane/connect-inject/constants/constants.go

@@ -58,6 +58,12 @@ const (
 	// DefaultGracefulShutdownPath is the default path that consul-dataplane uses for graceful shutdown.
 	DefaultGracefulShutdownPath = "/graceful_shutdown"
 
+	// DefaultGracefulStartupPath is the default path that consul-dataplane uses for graceful startup.
+	DefaultGracefulStartupPath = "/graceful_startup"
+
+	// DefaultStartupGracePeriodSeconds is the default number of seconds to block containers waiting for dataplane to start.
+	DefaultStartupGracePeriodSeconds = 0
+
 	// ConsulKubernetesCheckType is the type of health check in Consul for Kubernetes readiness status.
 	ConsulKubernetesCheckType = "kubernetes-readiness"
 

control-plane/connect-inject/lifecycle/lifecycle_configuration.go

@@ -19,6 +19,8 @@ type Config struct {
 	DefaultShutdownGracePeriodSeconds   int
 	DefaultGracefulPort                 string
 	DefaultGracefulShutdownPath         string
+	DefaultGracefulStartupPath          string
+	DefaultStartupGracePeriodSeconds    int
 }
 
 // EnableProxyLifecycle returns whether proxy lifecycle management is enabled either via the default value in the meshWebhook, or if it's been
@@ -93,3 +95,31 @@ func (lc Config) GracefulShutdownPath(pod corev1.Pod) string {
 
 	return lc.DefaultGracefulShutdownPath
 }
+
+// StartupGracePeriodSeconds returns how long in seconds the graceful_startup request should block while waiting for the dataplane to start.
+func (lc Config) StartupGracePeriodSeconds(pod corev1.Pod) (int, error) {
+
+	startupGracePeriodSeconds := lc.DefaultStartupGracePeriodSeconds
+	if startupGracePeriodSecondsAnnotation, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleStartupGracePeriodSeconds]; ok {
+		val, err := strconv.ParseUint(startupGracePeriodSecondsAnnotation, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("unable to parse annotation %q: %w", constants.AnnotationSidecarProxyLifecycleStartupGracePeriodSeconds, err)
+		}
+		startupGracePeriodSeconds = int(val)
+	}
+	return startupGracePeriodSeconds, nil
+}
+
+// GracefulStartupPath returns the path on which consul-dataplane should serve the graceful startup HTTP endpoint, either via the default value in the meshWebhook, or
+// if it's been overridden via the annotation.
+func (lc Config) GracefulStartupPath(pod corev1.Pod) string {
+	if raw, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulStartupPath]; ok && raw != "" {
+		return raw
+	}
+
+	if lc.DefaultGracefulStartupPath == "" {
+		return constants.DefaultGracefulStartupPath
+	}
+
+	return lc.DefaultGracefulStartupPath
+}

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -143,6 +143,19 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		ReadinessProbe: probe,
 	}
 
+	if lifecycleEnabled, err := w.LifecycleConfig.EnableProxyLifecycle(pod); lifecycleEnabled && err == nil {
+		// gracefulPort, _ := w.LifecycleConfig.GracefulPort(pod)
+		container.Lifecycle = &corev1.Lifecycle{
+			PostStart: &corev1.LifecycleHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{
+						"/usr/local/bin/consul-dataplane", "graceful-startup",
+					},
+				},
+			},
+		}
+	}
+
 	if w.AuthMethod != "" {
 		container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount)
 	}
@@ -327,6 +340,16 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, mpi mu
 			return nil, fmt.Errorf("unable to determine proxy lifecycle graceful shutdown path: %w", err)
 		}
 		args = append(args, fmt.Sprintf("-graceful-shutdown-path=%s", gracefulShutdownPath))
+
+		gracefulStartupPath := w.LifecycleConfig.GracefulStartupPath(pod)
+		args = append(args, fmt.Sprintf("-graceful-startup-path=%s", gracefulStartupPath))
+
+		startupGracePeriodSeconds, err := w.LifecycleConfig.StartupGracePeriodSeconds(pod)
+		if err != nil {
+			return nil, fmt.Errorf("unable to determine proxy lifecycle startup grace period: %w", err)
+		}
+		args = append(args, fmt.Sprintf("-startup-grace-period-seconds=%d", startupGracePeriodSeconds))
+
 	}
 
 	// Set a default scrape path that can be overwritten by the annotation.

control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go

@@ -1363,6 +1363,8 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 	gracefulShutdownSeconds := 10
 	gracefulPort := "20307"
 	gracefulShutdownPath := "/exit"
+	gracefulStartupPath := "/start"
+	gracefulStartupSeconds := 10
 
 	cases := []struct {
 		name        string
@@ -1386,10 +1388,12 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 					DefaultShutdownGracePeriodSeconds:   gracefulShutdownSeconds,
 					DefaultGracefulPort:                 gracefulPort,
 					DefaultGracefulShutdownPath:         gracefulShutdownPath,
+					DefaultGracefulStartupPath:          gracefulStartupPath,
+					DefaultStartupGracePeriodSeconds:    gracefulStartupSeconds,
 				},
 			},
 			annotations: nil,
-			expCmdArgs:  "graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit",
+			expCmdArgs:  "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit -graceful-startup-path=/start -startup-grace-period-seconds=10",
 		},
 		{
 			name:    "no defaults, all annotations",
@@ -1400,8 +1404,10 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 				constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds:   fmt.Sprint(gracefulShutdownSeconds),
 				constants.AnnotationSidecarProxyLifecycleGracefulPort:                 gracefulPort,
 				constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath:         gracefulShutdownPath,
+				constants.AnnotationSidecarProxyLifecycleGracefulStartupPath:          gracefulStartupPath,
+				constants.AnnotationSidecarProxyLifecycleStartupGracePeriodSeconds:    fmt.Sprint(gracefulStartupSeconds),
 			},
-			expCmdArgs: "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit",
+			expCmdArgs: "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit -graceful-startup-path=/start -startup-grace-period-seconds=10",
 		},
 		{
 			name: "annotations override defaults",
@@ -1420,8 +1426,10 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 				constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds:   fmt.Sprint(gracefulShutdownSeconds + 5),
 				constants.AnnotationSidecarProxyLifecycleGracefulPort:                 "20317",
 				constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath:         "/foo",
+				constants.AnnotationSidecarProxyLifecycleGracefulStartupPath:          "/bar",
+				constants.AnnotationSidecarProxyLifecycleStartupGracePeriodSeconds:    fmt.Sprint(gracefulStartupSeconds + 5),
 			},
-			expCmdArgs: "-graceful-port=20317 -shutdown-grace-period-seconds=15 -graceful-shutdown-path=/foo",
+			expCmdArgs: "-graceful-port=20317 -shutdown-grace-period-seconds=15 -graceful-shutdown-path=/foo -graceful-startup-path=/bar -startup-grace-period-seconds=15",
 		},
 		{
 			name: "lifecycle disabled, no annotations",
@@ -1432,6 +1440,8 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 					DefaultShutdownGracePeriodSeconds:   gracefulShutdownSeconds,
 					DefaultGracefulPort:                 gracefulPort,
 					DefaultGracefulShutdownPath:         gracefulShutdownPath,
+					DefaultGracefulStartupPath:          gracefulStartupPath,
+					DefaultStartupGracePeriodSeconds:    gracefulStartupSeconds,
 				},
 			},
 			annotations: nil,
@@ -1456,6 +1466,8 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 					DefaultShutdownGracePeriodSeconds:   gracefulShutdownSeconds,
 					DefaultGracefulPort:                 gracefulPort,
 					DefaultGracefulShutdownPath:         gracefulShutdownPath,
+					DefaultGracefulStartupPath:          gracefulStartupPath,
+					DefaultStartupGracePeriodSeconds:    gracefulStartupSeconds,
 				},
 			},
 			annotations: map[string]string{
@@ -1479,6 +1491,18 @@ func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) {
 			},
 			expCmdArgs: "",
 		},
+		{
+			name: "annotations skip graceful startup",
+			webhook: MeshWebhook{
+				LifecycleConfig: lifecycle.Config{
+					DefaultEnableProxyLifecycle: false,
+				},
+			},
+			annotations: map[string]string{
+				constants.AnnotationEnableSidecarProxyLifecycle: "false",
+			},
+			expCmdArgs: "",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

control-plane/subcommand/inject-connect/command.go

@@ -87,6 +87,8 @@ type Command struct {
 	flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds   int
 	flagDefaultSidecarProxyLifecycleGracefulPort                 string
 	flagDefaultSidecarProxyLifecycleGracefulShutdownPath         string
+	flagDefaultSidecarProxyLifecycleStartupGracePeriodSeconds    int
+	flagDefaultSidecarProxyLifecycleGracefulStartupPath          string
 
 	// Metrics settings.
 	flagDefaultEnableMetrics        bool
@@ -248,7 +250,8 @@ func (c *Command) init() {
 	c.flagSet.IntVar(&c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, "default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds", 0, "Default sidecar proxy shutdown grace period in seconds.")
 	c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulPort, "default-sidecar-proxy-lifecycle-graceful-port", strconv.Itoa(constants.DefaultGracefulPort), "Default port for sidecar proxy lifecycle management HTTP endpoints.")
 	c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, "default-sidecar-proxy-lifecycle-graceful-shutdown-path", "/graceful_shutdown", "Default sidecar proxy lifecycle management graceful shutdown path.")
-
+	c.flagSet.IntVar(&c.flagDefaultSidecarProxyLifecycleStartupGracePeriodSeconds, "default-sidecar-proxy-lifecycle-startup-grace-period-seconds", 0, "Default sidecar proxy startup grace period in seconds.")
+	c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulStartupPath, "default-sidecar-proxy-lifecycle-graceful-startup-path", "/graceful_startup", "Default sidecar proxy lifecycle management graceful startup path.")
 	// Metrics setting flags.
 	c.flagSet.BoolVar(&c.flagDefaultEnableMetrics, "default-enable-metrics", false, "Default for enabling connect service metrics.")
 	c.flagSet.BoolVar(&c.flagEnableGatewayMetrics, "enable-gateway-metrics", false, "Allows enabling Consul gateway metrics.")
@@ -396,6 +399,7 @@ func (c *Command) Run(args []string) int {
 	} else {
 		err = c.configureV1Controllers(ctx, mgr, watcher)
 	}
+
 	if err != nil {
 		setupLog.Error(err, fmt.Sprintf("could not configure controllers: %s", err.Error()))
 		return 1

control-plane/subcommand/inject-connect/v1controllers.go

@@ -41,6 +41,8 @@ func (c *Command) configureV1Controllers(ctx context.Context, mgr manager.Manage
 		DefaultShutdownGracePeriodSeconds:   c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds,
 		DefaultGracefulPort:                 c.flagDefaultSidecarProxyLifecycleGracefulPort,
 		DefaultGracefulShutdownPath:         c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath,
+		DefaultStartupGracePeriodSeconds:    c.flagDefaultSidecarProxyLifecycleStartupGracePeriodSeconds,
+		DefaultGracefulStartupPath:          c.flagDefaultSidecarProxyLifecycleGracefulStartupPath,
 	}
 
 	metricsConfig := metrics.Config{

control-plane/subcommand/inject-connect/v2controllers.go

@@ -63,6 +63,16 @@ func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manage
 		DefaultPrometheusScrapePath: c.flagDefaultPrometheusScrapePath,
 	}
 
+	// lifecycleConfig := lifecycle.Config{
+	// 	DefaultEnableProxyLifecycle:         c.flagDefaultEnableSidecarProxyLifecycle,
+	// 	DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners,
+	// 	DefaultShutdownGracePeriodSeconds:   c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds,
+	// 	DefaultGracefulPort:                 c.flagDefaultSidecarProxyLifecycleGracefulPort,
+	// 	DefaultGracefulShutdownPath:         c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath,
+	// 	DefaultStartupGracePeriodSeconds:    c.flagDefaultSidecarProxyLifecycleStartupGracePeriodSeconds,
+	// 	DefaultGracefulStartupPath:          c.flagDefaultSidecarProxyLifecycleGracefulStartupPath,
+	// }
+
 	if err := (&pod.Controller{
 		Client:                   mgr.GetClient(),
 		ConsulClientConfig:       consulConfig,