diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index 43c9974e1a..9e02599c65 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "time" "github.com/spf13/cobra" "go.uber.org/zap" @@ -22,23 +23,11 @@ const ( gatewayClassFlag = "gatewayclass" gatewayClassNameUsage = `The name of the GatewayClass resource. ` + `Every NGINX Gateway Fabric must have a unique corresponding GatewayClass resource.` - gatewayCtrlNameFlag = "gateway-ctlr-name" - gatewayCtrlNameUsageFmt = `The name of the Gateway controller. ` + + gatewayCtlrNameFlag = "gateway-ctlr-name" + gatewayCtlrNameUsageFmt = `The name of the Gateway controller. ` + `The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'` ) -var ( - // Backing values for common cli flags shared among all subcommands - // The values are managed by the Root command. - gatewayCtlrName = stringValidatingValue{ - validator: validateGatewayControllerName, - } - - gatewayClassName = stringValidatingValue{ - validator: validateResourceName, - } -) - func createRootCommand() *cobra.Command { rootCmd := &cobra.Command{ Use: "gateway", @@ -47,20 +36,6 @@ func createRootCommand() *cobra.Command { }, } - rootCmd.PersistentFlags().Var( - &gatewayCtlrName, - gatewayCtrlNameFlag, - fmt.Sprintf(gatewayCtrlNameUsageFmt, domain), - ) - utilruntime.Must(rootCmd.MarkPersistentFlagRequired(gatewayCtrlNameFlag)) - - rootCmd.PersistentFlags().Var( - &gatewayClassName, - gatewayClassFlag, - gatewayClassNameUsage, - ) - utilruntime.Must(rootCmd.MarkPersistentFlagRequired(gatewayClassFlag)) - return rootCmd } @@ -82,6 +57,14 @@ func createStaticModeCommand() *cobra.Command { // flag values var ( + gatewayCtlrName = stringValidatingValue{ + validator: validateGatewayControllerName, + } + + gatewayClassName = stringValidatingValue{ + validator: validateResourceName, + } + updateGCStatus bool gateway = namespacedNameValue{} configName = stringValidatingValue{ @@ -185,6 +168,20 @@ func createStaticModeCommand() *cobra.Command { }, } + cmd.Flags().Var( + &gatewayCtlrName, + gatewayCtlrNameFlag, + fmt.Sprintf(gatewayCtlrNameUsageFmt, domain), + ) + utilruntime.Must(cmd.MarkFlagRequired(gatewayCtlrNameFlag)) + + cmd.Flags().Var( + &gatewayClassName, + gatewayClassFlag, + gatewayClassNameUsage, + ) + utilruntime.Must(cmd.MarkFlagRequired(gatewayClassFlag)) + cmd.Flags().Var( &gateway, gatewayFlag, @@ -271,7 +268,16 @@ func createStaticModeCommand() *cobra.Command { } func createProvisionerModeCommand() *cobra.Command { - return &cobra.Command{ + var ( + gatewayCtlrName = stringValidatingValue{ + validator: validateGatewayControllerName, + } + gatewayClassName = stringValidatingValue{ + validator: validateResourceName, + } + ) + + cmd := &cobra.Command{ Use: "provisioner-mode", Short: "Provision a static-mode NGINX Gateway Fabric Deployment per Gateway resource", Hidden: true, @@ -291,4 +297,50 @@ func createProvisionerModeCommand() *cobra.Command { }) }, } + + cmd.Flags().Var( + &gatewayCtlrName, + gatewayCtlrNameFlag, + fmt.Sprintf(gatewayCtlrNameUsageFmt, domain), + ) + utilruntime.Must(cmd.MarkFlagRequired(gatewayCtlrNameFlag)) + + cmd.Flags().Var( + &gatewayClassName, + gatewayClassFlag, + gatewayClassNameUsage, + ) + utilruntime.Must(cmd.MarkFlagRequired(gatewayClassFlag)) + + return cmd +} + +// FIXME(pleshakov): Remove this command once NGF min supported Kubernetes version supports sleep action in +// preStop hook. +// nolint:lll +// See https://github.com/kubernetes/enhancements/tree/4ec371d92dcd4f56a2ab18c8ba20bb85d8d20efe/keps/sig-node/3960-pod-lifecycle-sleep-action +func createSleepCommand() *cobra.Command { + // flag names + const durationFlag = "duration" + // flag values + var duration time.Duration + + cmd := &cobra.Command{ + Use: "sleep", + Short: "Sleep for specified duration and exit", + Run: func(cmd *cobra.Command, args []string) { + // It is expected that this command is run from lifecycle hook. + // Because logs from hooks are not visible in the container logs, we don't log here at all. + time.Sleep(duration) + }, + } + + cmd.Flags().DurationVar( + &duration, + durationFlag, + 30*time.Second, + "Set the duration of sleep. Must be parsable by https://pkg.go.dev/time#ParseDuration", + ) + + return cmd } diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index 1b8302def4..120788d4e3 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -37,7 +37,17 @@ func testFlag(t *testing.T, cmd *cobra.Command, test flagTestCase) { } } -func TestRootCmdFlagValidation(t *testing.T) { +func TestRootCmd(t *testing.T) { + testCase := flagTestCase{ + name: "no flags", + args: nil, + wantErr: false, + } + + testFlag(t, createRootCommand(), testCase) +} + +func TestCommonFlagsValidation(t *testing.T) { tests := []flagTestCase{ { name: "valid flags", @@ -103,9 +113,11 @@ func TestRootCmdFlagValidation(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rootCmd := createRootCommand() - testFlag(t, rootCmd, test) + t.Run(test.name+"_static_mode", func(t *testing.T) { + testFlag(t, createStaticModeCommand(), test) + }) + t.Run(test.name+"_provisioner_mode", func(t *testing.T) { + testFlag(t, createProvisionerModeCommand(), test) }) } } @@ -115,6 +127,8 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { { name: "valid flags", args: []string{ + "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", // common and required flag + "--gatewayclass=nginx", // common and required flag "--gateway=nginx-gateway/nginx", "--config=nginx-gateway-config", "--service=nginx-gateway", @@ -130,8 +144,11 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { wantErr: false, }, { - name: "valid flags, not set", - args: nil, + name: "valid flags, non-required not set", + args: []string{ + "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", // common and required flag + "--gatewayclass=nginx", // common and required flag, + }, wantErr: false, }, { @@ -280,6 +297,8 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { }, } + // common flags validation is tested separately + for _, test := range tests { t.Run(test.name, func(t *testing.T) { cmd := createStaticModeCommand() @@ -287,3 +306,58 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { }) } } + +func TestProvisionerModeCmdFlagValidation(t *testing.T) { + testCase := flagTestCase{ + name: "valid flags", + args: []string{ + "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", // common and required flag + "--gatewayclass=nginx", // common and required flag + }, + wantErr: false, + } + + // common flags validation is tested separately + + testFlag(t, createProvisionerModeCommand(), testCase) +} + +func TestSleepCmdFlagValidation(t *testing.T) { + tests := []flagTestCase{ + { + name: "valid flags", + args: []string{ + "--duration=1s", + }, + wantErr: false, + }, + { + name: "omitted flags", + args: nil, + wantErr: false, + }, + { + name: "duration is set to empty string", + args: []string{ + "--duration=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--duration" flag: time: invalid duration ""`, + }, + { + name: "duration is invalid", + args: []string{ + "--duration=invalid", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "invalid" for "--duration" flag: time: invalid duration "invalid"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := createSleepCommand() + testFlag(t, cmd, test) + }) + } +} diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 12b2a89517..79509340c9 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -18,6 +18,7 @@ func main() { rootCmd.AddCommand( createStaticModeCommand(), createProvisionerModeCommand(), + createSleepCommand(), ) if err := rootCmd.Execute(); err != nil { diff --git a/conformance/provisioner/static-deployment.yaml b/conformance/provisioner/static-deployment.yaml index 8d2b4c68a6..c6b2326b04 100644 --- a/conformance/provisioner/static-deployment.yaml +++ b/conformance/provisioner/static-deployment.yaml @@ -101,6 +101,7 @@ spec: mountPath: /var/cache/nginx - name: nginx-lib mountPath: /var/lib/nginx + terminationGracePeriodSeconds: 30 serviceAccountName: nginx-gateway shareProcessNamespace: true securityContext: diff --git a/deploy/helm-chart/README.md b/deploy/helm-chart/README.md index 2f76629d03..b428944bfc 100644 --- a/deploy/helm-chart/README.md +++ b/deploy/helm-chart/README.md @@ -138,6 +138,7 @@ The following tables lists the configurable parameters of the NGINX Gateway Fabr | `nginxGateway.image.repository` | The repository for the NGINX Gateway Fabric image. | ghcr.io/nginxinc/nginx-gateway-fabric | | `nginxGateway.image.tag` | The tag for the NGINX Gateway Fabric image. | edge | | `nginxGateway.image.pullPolicy` | The `imagePullPolicy` for the NGINX Gateway Fabric image. | Always | +| `nginxGateway.lifecycle` | The `lifecycle` of the nginx-gateway container. | {} | | `nginxGateway.gatewayClassName` | The name of the GatewayClass for the NGINX Gateway Fabric deployment. | nginx | | `nginxGateway.gatewayControllerName` | The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is gateway.nginx.org. | gateway.nginx.org/nginx-gateway-controller | | `nginxGateway.kind` | The kind of the NGINX Gateway Fabric installation - currently, only Deployment is supported. | deployment | @@ -151,6 +152,9 @@ The following tables lists the configurable parameters of the NGINX Gateway Fabr | `nginx.image.repository` | The repository for the NGINX image. | ghcr.io/nginxinc/nginx-gateway-fabric/nginx | | `nginx.image.tag` | The tag for the NGINX image. | edge | | `nginx.image.pullPolicy` | The `imagePullPolicy` for the NGINX image. | Always | +| `nginx.lifecycle` | The `lifecycle` of the nginx container. | {} | +| `terminationGracePeriodSeconds` | The termination grace period of the NGINX Gateway Fabric pod. | 30 | +| `affinity` | The `affinity` of the NGINX Gateway Fabric pod. | {} | | `serviceAccount.annotations` | The `annotations` for the ServiceAccount used by the NGINX Gateway Fabric deployment. | {} | | `serviceAccount.name` | Name of the ServiceAccount used by the NGINX Gateway Fabric deployment. | Autogenerated | | `service.create` | Creates a service to expose the NGINX Gateway Fabric pods. | true | diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index bc7195b478..b576f793a0 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -65,6 +65,10 @@ spec: image: {{ .Values.nginxGateway.image.repository }}:{{ .Values.nginxGateway.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} name: nginx-gateway + {{- if .Values.nginxGateway.lifecycle }} + lifecycle: + {{- toYaml .Values.nginxGateway.lifecycle | nindent 10 }} + {{- end }} ports: {{- if .Values.metrics.enable }} - name: metrics @@ -100,6 +104,10 @@ spec: - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} name: nginx + {{- if .Values.nginx.lifecycle }} + lifecycle: + {{- toYaml .Values.nginx.lifecycle | nindent 10 }} + {{- end }} ports: - containerPort: 80 name: http @@ -125,6 +133,11 @@ spec: mountPath: /var/cache/nginx - name: nginx-lib mountPath: /var/lib/nginx + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }} shareProcessNamespace: true securityContext: diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index fd16c1c21a..6380f61e43 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -45,6 +45,9 @@ nginxGateway: ## Some environments may need this set to true in order for the control plane to successfully reload NGINX. allowPrivilegeEscalation: false + ## The lifecycle of the nginx-gateway container. + lifecycle: {} + nginx: ## The NGINX image to use image: @@ -52,6 +55,15 @@ nginx: tag: edge pullPolicy: Always + ## The lifecycle of the nginx container. + lifecycle: {} + +## The termination grace period of the NGINX Gateway Fabric pod. +terminationGracePeriodSeconds: 30 + +## The affinity of the NGINX Gateway Fabric pod. +affinity: {} + serviceAccount: annotations: {} ## The name of the service account of the NGINX Gateway Fabric pods. Used for RBAC. diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml index 28c26b86b2..045c81c679 100644 --- a/deploy/manifests/nginx-gateway.yaml +++ b/deploy/manifests/nginx-gateway.yaml @@ -215,6 +215,7 @@ spec: mountPath: /var/cache/nginx - name: nginx-lib mountPath: /var/lib/nginx + terminationGracePeriodSeconds: 30 serviceAccountName: nginx-gateway shareProcessNamespace: true securityContext: diff --git a/docs/cli-help.md b/docs/cli-help.md index 7c20ae892e..e3b795ae23 100644 --- a/docs/cli-help.md +++ b/docs/cli-help.md @@ -29,3 +29,21 @@ Flags: | `health-port` | `int` | Set the port where the health probe server is exposed. Format: `[1024 - 65535]` (default `8081`) | | `leader-election-disable` | `bool` | Disable leader election. Leader election is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If disabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources. (default false) | | `leader-election-lock-name` | `string` | The name of the leader election lock. A Lease object with this name will be created in the same Namespace as the controller. (default "nginx-gateway-leader-election-lock") | + +## Sleep + +This command sleeps for specified duration and exits. + +Usage: + +```text +Usage: + gateway sleep [flags] +``` + +| Name | Type | Description | +|----------|-----------------|-------------------------------------------------------------------------------------------------------| +| duration | `time.Duration` | Set the duration of sleep. Must be parsable by [`time.ParseDuration`][parseDuration]. (default `30s`) | + + +[parseDuration]:https://pkg.go.dev/time#ParseDuration diff --git a/tests/zero-downtime-upgrades/results/1.0.0-special/1.0.0-special.md b/tests/zero-downtime-upgrades/results/1.0.0-special/1.0.0-special.md new file mode 100644 index 0000000000..07830e4979 --- /dev/null +++ b/tests/zero-downtime-upgrades/results/1.0.0-special/1.0.0-special.md @@ -0,0 +1,175 @@ +# Results + +## Why is it named Special? + +A proper zero downtime upgrade test tests an upgrade from the latest stable release +to a new version. + +At the time of the test, the latest stable release 0.6.0 doesn't support features +to achieve zero downtime upgrades. As a result, we can't run a proper test. + +Instead, we will do an upgrade from the same version to the same version to test +that the new zero downtime upgrade features ([PR-1159](https://github.com/nginxinc/nginx-gateway-fabric/pull/1159)) +achieve zero downtime upgrades. + +## Versions + +Kubernetes: + +```text +Server Version: version.Info{Major:"1", Minor:"27", GitVersion:"v1.27.3-gke.100", GitCommit:"6466b51b762a5c49ae3fb6c2c7233ffe1c96e48c", GitTreeState:"clean", BuildDate:"2023-06-23T09:27:28Z", GoVersion:"go1.20.5 X:boringcrypto", Compiler:"gc", Platform:"linux/amd64"} +``` + +NGF - version from https://github.com/nginxinc/nginx-gateway-fabric/pull/1159 + +## Start + +To deploy NGF, use a helm command: + +```command +helm install my-release ../../deploy/helm-chart --create-namespace --wait -n nginx-gateway -f values.yaml +``` + +Deployed Pods: + +```text +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +my-release-nginx-gateway-fabric-66cb67bf9c-4nvms 2/2 Running 0 100s 10.12.6.10 gke-michael-3-default-pool-78305034-bkqs +my-release-nginx-gateway-fabric-66cb67bf9c-nnr95 2/2 Running 0 101s 10.12.8.10 gke-michael-3-default-pool-78305034-fl2c +``` + +Logs check: + +- my-release-nginx-gateway-fabric-66cb67bf9c-4nvms + - NGINX logs - no errors or warnings. + - NGF logs - no errors +- my-release-nginx-gateway-fabric-66cb67bf9c-nnr95 + - NGINX logs - no errors or warnings. + - NGF logs - no errors + +## Upgrades + +To simulate an upgrade, run: + +```text +kubectl -n nginx-gateway rollout restart deployment/my-release-nginx-gateway-fabric +``` + +New Pods: + +```text +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +my-release-nginx-gateway-fabric-8497f944-4sz2x 2/2 Running 0 3m47s 10.12.9.8 gke-michael-3-default-pool-78305034-7nqm +my-release-nginx-gateway-fabric-8497f944-6rjlb 2/2 Running 0 3m41s 10.12.7.5 gke-michael-3-default-pool-78305034-srqf +``` + +Note: the new Pods were scheduled on different nodes from the old Pods' nodes, as we wanted. + +Check that one of the NGF Pods became the leader: + +```text +kubectl -n nginx-gateway get lease +NAME HOLDER AGE +my-release-nginx-gateway-fabric-leader-election my-release-nginx-gateway-fabric-8497f944-6rjlb 22h +``` + +Pod my-release-nginx-gateway-fabric-8497f944-6rjlb is the leader. + +Gateway status has been updated with the new listener. + +### Analyze + +#### Tester VMs + +Tester 1 wrk output: + +```text +Running 1m test @ http://cafe.example.com/coffee + 2 threads and 100 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 26.09ms 23.97ms 366.19ms 86.57% + Req/Sec 2.26k 740.51 4.57k 69.33% + Latency Distribution + 50% 19.93ms + 75% 32.78ms + 90% 53.08ms + 99% 120.73ms + 269508 requests in 1.00m, 95.87MB read + Socket errors: connect 0, read 4, write 0, timeout 0 +Requests/sec: 4490.57 +Transfer/sec: 1.60MB +``` + +There are 4 read socket errors. +See https://github.com/nginxinc/nginx-gateway-fabric/issues/1147 + +Tester 1 graph: + +![http.png](http.png) + +All curl requests succeeded. + +Tester 2 wrk output: + +```text +Running 1m test @ https://cafe.example.com/tea + 2 threads and 100 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 28.03ms 25.31ms 369.99ms 86.64% + Req/Sec 2.07k 618.73 3.64k 63.83% + Latency Distribution + 50% 21.33ms + 75% 35.10ms + 90% 56.07ms + 99% 127.31ms + 247847 requests in 1.00m, 86.51MB read + Socket errors: connect 0, read 3, write 0, timeout 0 +Requests/sec: 4129.21 +Transfer/sec: 1.44MB +``` + +There are 3 read socket errors. +See https://github.com/nginxinc/nginx-gateway-fabric/issues/1147 + +Tester 2 graph: + +![https.png](https.png) + +All curl requests succeeded. + +#### Old Pods + +- my-release-nginx-gateway-fabric-66cb67bf9c-4nvms + - NGF - no errors + - NGINX + - Access logs - all responses are 200. + - Error logs - no errors or warnings. +- my-release-nginx-gateway-fabric-66cb67bf9c-nnr95 + - NGF - no errors + - NGINX + - Access logs - only 200 responses. + - Error logs - no errors or warnings. + +#### New Pods + +- my-release-nginx-gateway-fabric-8497f944-4sz2x + - NGF - no errors + - NGINX + - Access logs - 44 responses like below: + + ```text + INFO 2023-10-19T20:03:26.362441989Z [resource.labels.containerName: nginx] 10.128.0.9 - - [19/Oct/2023:20:03:26 +0000] "GET /coffee HTTP/1.1" 499 0 "-" "-" + . . . + INFO 2023-10-19T20:03:29.469971066Z [resource.labels.containerName: nginx] 10.128.15.241 - - [19/Oct/2023:20:03:29 +0000] "GET /tea HTTP/1.1" 499 0 "-" "-" + ``` + + Meaning clients closed connection (499 status code). + Those requests belong to wrk (curl requests have `curl` user agent in the logs). + All requests for coffee are during the same time. Same for tea. + That's probably how wrk closed connections before it exited after 60s. + - Error logs - No errors or warnings. +- my-release-nginx-gateway-fabric-8497f944-6rjlb + - NGF - no errors. + - NGINX + - Access logs - 44 responses similar to the first Pod. Same conclusion as above. + - Error logs - No errors or warnings. diff --git a/tests/zero-downtime-upgrades/results/1.0.0-special/http.csv b/tests/zero-downtime-upgrades/results/1.0.0-special/http.csv new file mode 100644 index 0000000000..3068dc1224 --- /dev/null +++ b/tests/zero-downtime-upgrades/results/1.0.0-special/http.csv @@ -0,0 +1,600 @@ +2023-10-19 20:02:27.727659325+00:00,1 +2023-10-19 20:02:27.859479883+00:00,1 +2023-10-19 20:02:27.992781511+00:00,1 +2023-10-19 20:02:28.122892971+00:00,1 +2023-10-19 20:02:28.259462404+00:00,1 +2023-10-19 20:02:28.392042403+00:00,1 +2023-10-19 20:02:28.527798434+00:00,1 +2023-10-19 20:02:28.665750024+00:00,1 +2023-10-19 20:02:28.795235722+00:00,1 +2023-10-19 20:02:28.929885176+00:00,1 +2023-10-19 20:02:29.072444322+00:00,1 +2023-10-19 20:02:29.208397013+00:00,1 +2023-10-19 20:02:29.340441697+00:00,1 +2023-10-19 20:02:29.470839061+00:00,1 +2023-10-19 20:02:29.625046065+00:00,1 +2023-10-19 20:02:29.783801047+00:00,1 +2023-10-19 20:02:29.950180290+00:00,1 +2023-10-19 20:02:30.093141289+00:00,1 +2023-10-19 20:02:30.246054095+00:00,1 +2023-10-19 20:02:30.389539499+00:00,1 +2023-10-19 20:02:30.533237345+00:00,1 +2023-10-19 20:02:30.690752941+00:00,1 +2023-10-19 20:02:30.829967904+00:00,1 +2023-10-19 20:02:30.983468317+00:00,1 +2023-10-19 20:02:31.178824163+00:00,1 +2023-10-19 20:02:31.374262259+00:00,1 +2023-10-19 20:02:31.521108917+00:00,1 +2023-10-19 20:02:31.680439895+00:00,1 +2023-10-19 20:02:31.842498544+00:00,1 +2023-10-19 20:02:31.982671150+00:00,1 +2023-10-19 20:02:32.120447254+00:00,1 +2023-10-19 20:02:32.266644523+00:00,1 +2023-10-19 20:02:32.410258814+00:00,1 +2023-10-19 20:02:32.621467052+00:00,1 +2023-10-19 20:02:32.767604516+00:00,1 +2023-10-19 20:02:32.907503654+00:00,1 +2023-10-19 20:02:33.059918406+00:00,1 +2023-10-19 20:02:33.202708917+00:00,1 +2023-10-19 20:02:33.354277024+00:00,1 +2023-10-19 20:02:33.518310997+00:00,1 +2023-10-19 20:02:33.666880799+00:00,1 +2023-10-19 20:02:33.821911902+00:00,1 +2023-10-19 20:02:33.962308319+00:00,1 +2023-10-19 20:02:34.219678061+00:00,1 +2023-10-19 20:02:34.385815827+00:00,1 +2023-10-19 20:02:34.540296767+00:00,1 +2023-10-19 20:02:34.681823891+00:00,1 +2023-10-19 20:02:34.835850325+00:00,1 +2023-10-19 20:02:34.974488218+00:00,1 +2023-10-19 20:02:35.118968818+00:00,1 +2023-10-19 20:02:35.272271196+00:00,1 +2023-10-19 20:02:35.415590813+00:00,1 +2023-10-19 20:02:35.564216215+00:00,1 +2023-10-19 20:02:35.697207731+00:00,1 +2023-10-19 20:02:35.875787049+00:00,1 +2023-10-19 20:02:36.114894675+00:00,1 +2023-10-19 20:02:36.272439282+00:00,1 +2023-10-19 20:02:36.480285978+00:00,1 +2023-10-19 20:02:36.629686238+00:00,1 +2023-10-19 20:02:36.778089715+00:00,1 +2023-10-19 20:02:36.921490998+00:00,1 +2023-10-19 20:02:37.061193350+00:00,1 +2023-10-19 20:02:37.208389498+00:00,1 +2023-10-19 20:02:37.352089117+00:00,1 +2023-10-19 20:02:37.508584215+00:00,1 +2023-10-19 20:02:37.657127064+00:00,1 +2023-10-19 20:02:37.800176509+00:00,1 +2023-10-19 20:02:37.952351242+00:00,1 +2023-10-19 20:02:38.170627612+00:00,1 +2023-10-19 20:02:38.310390069+00:00,1 +2023-10-19 20:02:38.464630600+00:00,1 +2023-10-19 20:02:38.613195214+00:00,1 +2023-10-19 20:02:38.785676801+00:00,1 +2023-10-19 20:02:38.926502517+00:00,1 +2023-10-19 20:02:39.112178209+00:00,1 +2023-10-19 20:02:39.259636787+00:00,1 +2023-10-19 20:02:39.422587262+00:00,1 +2023-10-19 20:02:39.572021664+00:00,1 +2023-10-19 20:02:39.722869100+00:00,1 +2023-10-19 20:02:39.881833097+00:00,1 +2023-10-19 20:02:40.024088925+00:00,1 +2023-10-19 20:02:40.187150300+00:00,1 +2023-10-19 20:02:40.334559257+00:00,1 +2023-10-19 20:02:40.530692589+00:00,1 +2023-10-19 20:02:40.701215421+00:00,1 +2023-10-19 20:02:40.943995495+00:00,1 +2023-10-19 20:02:41.106798296+00:00,1 +2023-10-19 20:02:41.248608047+00:00,1 +2023-10-19 20:02:41.431589893+00:00,1 +2023-10-19 20:02:41.598303443+00:00,1 +2023-10-19 20:02:41.774385063+00:00,1 +2023-10-19 20:02:41.920599307+00:00,1 +2023-10-19 20:02:42.058175634+00:00,1 +2023-10-19 20:02:42.206955949+00:00,1 +2023-10-19 20:02:42.349721269+00:00,1 +2023-10-19 20:02:42.498033378+00:00,1 +2023-10-19 20:02:42.636672475+00:00,1 +2023-10-19 20:02:42.775639865+00:00,1 +2023-10-19 20:02:42.937237998+00:00,1 +2023-10-19 20:02:43.075607427+00:00,1 +2023-10-19 20:02:43.219386061+00:00,1 +2023-10-19 20:02:43.383491288+00:00,1 +2023-10-19 20:02:43.507964924+00:00,1 +2023-10-19 20:02:43.628071651+00:00,1 +2023-10-19 20:02:43.747299816+00:00,1 +2023-10-19 20:02:43.866719610+00:00,1 +2023-10-19 20:02:43.986070200+00:00,1 +2023-10-19 20:02:44.115560503+00:00,1 +2023-10-19 20:02:44.236357015+00:00,1 +2023-10-19 20:02:44.396617709+00:00,1 +2023-10-19 20:02:44.516038634+00:00,1 +2023-10-19 20:02:44.662556471+00:00,1 +2023-10-19 20:02:44.809382521+00:00,1 +2023-10-19 20:02:44.966359544+00:00,1 +2023-10-19 20:02:45.109511955+00:00,1 +2023-10-19 20:02:45.229295951+00:00,1 +2023-10-19 20:02:45.373651868+00:00,1 +2023-10-19 20:02:45.492676938+00:00,1 +2023-10-19 20:02:45.611019811+00:00,1 +2023-10-19 20:02:45.751631597+00:00,1 +2023-10-19 20:02:45.869449311+00:00,1 +2023-10-19 20:02:45.987001239+00:00,1 +2023-10-19 20:02:46.157843905+00:00,1 +2023-10-19 20:02:46.279586825+00:00,1 +2023-10-19 20:02:46.442036633+00:00,1 +2023-10-19 20:02:46.592363660+00:00,1 +2023-10-19 20:02:46.711588802+00:00,1 +2023-10-19 20:02:46.853506120+00:00,1 +2023-10-19 20:02:46.997817565+00:00,1 +2023-10-19 20:02:47.147262310+00:00,1 +2023-10-19 20:02:47.266782250+00:00,1 +2023-10-19 20:02:47.386779980+00:00,1 +2023-10-19 20:02:47.506233015+00:00,1 +2023-10-19 20:02:47.663913235+00:00,1 +2023-10-19 20:02:47.807984493+00:00,1 +2023-10-19 20:02:47.976458542+00:00,1 +2023-10-19 20:02:48.139543918+00:00,1 +2023-10-19 20:02:48.295043976+00:00,1 +2023-10-19 20:02:48.452842412+00:00,1 +2023-10-19 20:02:48.608408722+00:00,1 +2023-10-19 20:02:48.763431563+00:00,1 +2023-10-19 20:02:48.889231820+00:00,1 +2023-10-19 20:02:49.008680742+00:00,1 +2023-10-19 20:02:49.138635549+00:00,1 +2023-10-19 20:02:49.304444297+00:00,1 +2023-10-19 20:02:49.438658487+00:00,1 +2023-10-19 20:02:49.560949255+00:00,1 +2023-10-19 20:02:49.684700846+00:00,1 +2023-10-19 20:02:49.840582729+00:00,1 +2023-10-19 20:02:49.981900785+00:00,1 +2023-10-19 20:02:50.129788277+00:00,1 +2023-10-19 20:02:50.254996427+00:00,1 +2023-10-19 20:02:50.404514273+00:00,1 +2023-10-19 20:02:50.539377466+00:00,1 +2023-10-19 20:02:50.695990902+00:00,1 +2023-10-19 20:02:50.842154459+00:00,1 +2023-10-19 20:02:50.964137290+00:00,1 +2023-10-19 20:02:51.122268173+00:00,1 +2023-10-19 20:02:51.244062007+00:00,1 +2023-10-19 20:02:51.381544181+00:00,1 +2023-10-19 20:02:51.503722533+00:00,1 +2023-10-19 20:02:51.689941384+00:00,1 +2023-10-19 20:02:51.825121905+00:00,1 +2023-10-19 20:02:51.947771555+00:00,1 +2023-10-19 20:02:52.070232507+00:00,1 +2023-10-19 20:02:52.245196501+00:00,1 +2023-10-19 20:02:52.395267697+00:00,1 +2023-10-19 20:02:52.515183313+00:00,1 +2023-10-19 20:02:52.645265616+00:00,1 +2023-10-19 20:02:52.794878633+00:00,1 +2023-10-19 20:02:52.915262681+00:00,1 +2023-10-19 20:02:53.071524470+00:00,1 +2023-10-19 20:02:53.238559134+00:00,1 +2023-10-19 20:02:53.367519444+00:00,1 +2023-10-19 20:02:53.490086310+00:00,1 +2023-10-19 20:02:53.615096186+00:00,1 +2023-10-19 20:02:53.748375277+00:00,1 +2023-10-19 20:02:53.871962857+00:00,1 +2023-10-19 20:02:54.028774568+00:00,1 +2023-10-19 20:02:54.173826100+00:00,1 +2023-10-19 20:02:54.332523316+00:00,1 +2023-10-19 20:02:54.492107045+00:00,1 +2023-10-19 20:02:54.612941958+00:00,1 +2023-10-19 20:02:54.735696960+00:00,1 +2023-10-19 20:02:54.860067642+00:00,1 +2023-10-19 20:02:54.988693128+00:00,1 +2023-10-19 20:02:55.128860338+00:00,1 +2023-10-19 20:02:55.254126136+00:00,1 +2023-10-19 20:02:55.411576496+00:00,1 +2023-10-19 20:02:55.541608565+00:00,1 +2023-10-19 20:02:55.696621113+00:00,1 +2023-10-19 20:02:55.875257314+00:00,1 +2023-10-19 20:02:56.011269864+00:00,1 +2023-10-19 20:02:56.137539664+00:00,1 +2023-10-19 20:02:56.264600842+00:00,1 +2023-10-19 20:02:56.399972517+00:00,1 +2023-10-19 20:02:56.524853445+00:00,1 +2023-10-19 20:02:56.661208857+00:00,1 +2023-10-19 20:02:56.825493479+00:00,1 +2023-10-19 20:02:56.988940557+00:00,1 +2023-10-19 20:02:57.128745014+00:00,1 +2023-10-19 20:02:57.264757492+00:00,1 +2023-10-19 20:02:57.394673220+00:00,1 +2023-10-19 20:02:57.518844307+00:00,1 +2023-10-19 20:02:57.657166722+00:00,1 +2023-10-19 20:02:57.806795485+00:00,1 +2023-10-19 20:02:57.940720072+00:00,1 +2023-10-19 20:02:58.082345093+00:00,1 +2023-10-19 20:02:58.213392149+00:00,1 +2023-10-19 20:02:58.344225991+00:00,1 +2023-10-19 20:02:58.487746274+00:00,1 +2023-10-19 20:02:58.617472897+00:00,1 +2023-10-19 20:02:58.760081055+00:00,1 +2023-10-19 20:02:58.881976158+00:00,1 +2023-10-19 20:02:59.025299534+00:00,1 +2023-10-19 20:02:59.161287947+00:00,1 +2023-10-19 20:02:59.280899114+00:00,1 +2023-10-19 20:02:59.421901192+00:00,1 +2023-10-19 20:02:59.542688611+00:00,1 +2023-10-19 20:02:59.677157731+00:00,1 +2023-10-19 20:02:59.810176392+00:00,1 +2023-10-19 20:02:59.956494540+00:00,1 +2023-10-19 20:03:00.084932885+00:00,1 +2023-10-19 20:03:00.205305193+00:00,1 +2023-10-19 20:03:00.327974455+00:00,1 +2023-10-19 20:03:00.462355958+00:00,1 +2023-10-19 20:03:00.616514171+00:00,1 +2023-10-19 20:03:00.752629402+00:00,1 +2023-10-19 20:03:00.872300827+00:00,1 +2023-10-19 20:03:01.022549498+00:00,1 +2023-10-19 20:03:01.144726282+00:00,1 +2023-10-19 20:03:01.268683570+00:00,1 +2023-10-19 20:03:01.397383814+00:00,1 +2023-10-19 20:03:01.544092566+00:00,1 +2023-10-19 20:03:01.692530511+00:00,1 +2023-10-19 20:03:01.823901205+00:00,1 +2023-10-19 20:03:01.951285995+00:00,1 +2023-10-19 20:03:02.085576257+00:00,1 +2023-10-19 20:03:02.244916622+00:00,1 +2023-10-19 20:03:02.380043687+00:00,1 +2023-10-19 20:03:02.512238395+00:00,1 +2023-10-19 20:03:02.649820640+00:00,1 +2023-10-19 20:03:02.783960213+00:00,1 +2023-10-19 20:03:02.909404293+00:00,1 +2023-10-19 20:03:03.041119035+00:00,1 +2023-10-19 20:03:03.178405000+00:00,1 +2023-10-19 20:03:03.305948424+00:00,1 +2023-10-19 20:03:03.444787811+00:00,1 +2023-10-19 20:03:03.592102100+00:00,1 +2023-10-19 20:03:03.714693399+00:00,1 +2023-10-19 20:03:03.840404597+00:00,1 +2023-10-19 20:03:03.985102289+00:00,1 +2023-10-19 20:03:04.123586171+00:00,1 +2023-10-19 20:03:04.306261819+00:00,1 +2023-10-19 20:03:04.475548734+00:00,1 +2023-10-19 20:03:04.608743353+00:00,1 +2023-10-19 20:03:04.762964917+00:00,1 +2023-10-19 20:03:04.901134258+00:00,1 +2023-10-19 20:03:05.114988407+00:00,1 +2023-10-19 20:03:05.282641420+00:00,1 +2023-10-19 20:03:05.421630675+00:00,1 +2023-10-19 20:03:05.570421651+00:00,1 +2023-10-19 20:03:05.716330644+00:00,1 +2023-10-19 20:03:05.860941553+00:00,1 +2023-10-19 20:03:06.008496652+00:00,1 +2023-10-19 20:03:06.144462171+00:00,1 +2023-10-19 20:03:06.357903800+00:00,1 +2023-10-19 20:03:06.489422192+00:00,1 +2023-10-19 20:03:06.642112515+00:00,1 +2023-10-19 20:03:06.784476711+00:00,1 +2023-10-19 20:03:06.919350728+00:00,1 +2023-10-19 20:03:07.050491073+00:00,1 +2023-10-19 20:03:07.178555944+00:00,1 +2023-10-19 20:03:07.304198286+00:00,1 +2023-10-19 20:03:07.433026755+00:00,1 +2023-10-19 20:03:07.575631467+00:00,1 +2023-10-19 20:03:07.708770176+00:00,1 +2023-10-19 20:03:07.838572965+00:00,1 +2023-10-19 20:03:07.963160972+00:00,1 +2023-10-19 20:03:08.125876326+00:00,1 +2023-10-19 20:03:08.252047515+00:00,1 +2023-10-19 20:03:08.385137901+00:00,1 +2023-10-19 20:03:08.505668729+00:00,1 +2023-10-19 20:03:08.626695971+00:00,1 +2023-10-19 20:03:08.759022122+00:00,1 +2023-10-19 20:03:08.891773842+00:00,1 +2023-10-19 20:03:09.029509184+00:00,1 +2023-10-19 20:03:09.164722360+00:00,1 +2023-10-19 20:03:09.319925349+00:00,1 +2023-10-19 20:03:09.464977277+00:00,1 +2023-10-19 20:03:09.604291815+00:00,1 +2023-10-19 20:03:09.739068361+00:00,1 +2023-10-19 20:03:09.876517190+00:00,1 +2023-10-19 20:03:10.008681518+00:00,1 +2023-10-19 20:03:10.146778721+00:00,1 +2023-10-19 20:03:10.278864041+00:00,1 +2023-10-19 20:03:10.414030784+00:00,1 +2023-10-19 20:03:10.572397078+00:00,1 +2023-10-19 20:03:10.704416403+00:00,1 +2023-10-19 20:03:10.860495741+00:00,1 +2023-10-19 20:03:11.010792847+00:00,1 +2023-10-19 20:03:11.185857647+00:00,1 +2023-10-19 20:03:11.319044283+00:00,1 +2023-10-19 20:03:11.448287249+00:00,1 +2023-10-19 20:03:11.590277302+00:00,1 +2023-10-19 20:03:11.716543625+00:00,1 +2023-10-19 20:03:11.842250242+00:00,1 +2023-10-19 20:03:11.976879183+00:00,1 +2023-10-19 20:03:12.107941067+00:00,1 +2023-10-19 20:03:12.242570633+00:00,1 +2023-10-19 20:03:12.389346344+00:00,1 +2023-10-19 20:03:12.527150612+00:00,1 +2023-10-19 20:03:12.661889159+00:00,1 +2023-10-19 20:03:12.807872522+00:00,1 +2023-10-19 20:03:12.951496344+00:00,1 +2023-10-19 20:03:13.081394525+00:00,1 +2023-10-19 20:03:13.235398081+00:00,1 +2023-10-19 20:03:13.376174906+00:00,1 +2023-10-19 20:03:13.515963135+00:00,1 +2023-10-19 20:03:13.643225561+00:00,1 +2023-10-19 20:03:13.788748036+00:00,1 +2023-10-19 20:03:13.967175011+00:00,1 +2023-10-19 20:03:14.100010280+00:00,1 +2023-10-19 20:03:14.229973948+00:00,1 +2023-10-19 20:03:14.384434769+00:00,1 +2023-10-19 20:03:14.525127816+00:00,1 +2023-10-19 20:03:14.647514751+00:00,1 +2023-10-19 20:03:14.805861659+00:00,1 +2023-10-19 20:03:14.945658158+00:00,1 +2023-10-19 20:03:15.069380168+00:00,1 +2023-10-19 20:03:15.207050760+00:00,1 +2023-10-19 20:03:15.341598429+00:00,1 +2023-10-19 20:03:15.468849035+00:00,1 +2023-10-19 20:03:15.604889903+00:00,1 +2023-10-19 20:03:15.740729775+00:00,1 +2023-10-19 20:03:15.929640101+00:00,1 +2023-10-19 20:03:16.058994114+00:00,1 +2023-10-19 20:03:16.199874496+00:00,1 +2023-10-19 20:03:16.332974919+00:00,1 +2023-10-19 20:03:16.459741353+00:00,1 +2023-10-19 20:03:16.588060280+00:00,1 +2023-10-19 20:03:16.721097068+00:00,1 +2023-10-19 20:03:16.848763222+00:00,1 +2023-10-19 20:03:16.999353294+00:00,1 +2023-10-19 20:03:17.129905764+00:00,1 +2023-10-19 20:03:17.268071258+00:00,1 +2023-10-19 20:03:17.417110053+00:00,1 +2023-10-19 20:03:17.544967399+00:00,1 +2023-10-19 20:03:17.697582364+00:00,1 +2023-10-19 20:03:17.824818234+00:00,1 +2023-10-19 20:03:17.990534061+00:00,1 +2023-10-19 20:03:18.112976522+00:00,1 +2023-10-19 20:03:18.272150960+00:00,1 +2023-10-19 20:03:18.413137696+00:00,1 +2023-10-19 20:03:18.558298345+00:00,1 +2023-10-19 20:03:18.709375421+00:00,1 +2023-10-19 20:03:18.872840179+00:00,1 +2023-10-19 20:03:18.998636694+00:00,1 +2023-10-19 20:03:19.205994967+00:00,1 +2023-10-19 20:03:19.432956419+00:00,1 +2023-10-19 20:03:19.617162708+00:00,1 +2023-10-19 20:03:19.786356969+00:00,1 +2023-10-19 20:03:19.950133733+00:00,1 +2023-10-19 20:03:20.090259804+00:00,1 +2023-10-19 20:03:20.269719202+00:00,1 +2023-10-19 20:03:20.414143431+00:00,1 +2023-10-19 20:03:20.568876113+00:00,1 +2023-10-19 20:03:20.727484107+00:00,1 +2023-10-19 20:03:20.889803340+00:00,1 +2023-10-19 20:03:21.025587759+00:00,1 +2023-10-19 20:03:21.174902081+00:00,1 +2023-10-19 20:03:21.324477472+00:00,1 +2023-10-19 20:03:21.468079999+00:00,1 +2023-10-19 20:03:21.614028105+00:00,1 +2023-10-19 20:03:21.765227884+00:00,1 +2023-10-19 20:03:21.925728780+00:00,1 +2023-10-19 20:03:22.066160140+00:00,1 +2023-10-19 20:03:22.267471602+00:00,1 +2023-10-19 20:03:22.414303937+00:00,1 +2023-10-19 20:03:22.567634990+00:00,1 +2023-10-19 20:03:22.714141696+00:00,1 +2023-10-19 20:03:22.868936504+00:00,1 +2023-10-19 20:03:23.026298477+00:00,1 +2023-10-19 20:03:23.180186181+00:00,1 +2023-10-19 20:03:23.327101931+00:00,1 +2023-10-19 20:03:23.468082992+00:00,1 +2023-10-19 20:03:23.608915524+00:00,1 +2023-10-19 20:03:23.756172784+00:00,1 +2023-10-19 20:03:23.902236316+00:00,1 +2023-10-19 20:03:24.052582271+00:00,1 +2023-10-19 20:03:24.359294392+00:00,1 +2023-10-19 20:03:24.521608626+00:00,1 +2023-10-19 20:03:24.726275711+00:00,1 +2023-10-19 20:03:24.882802475+00:00,1 +2023-10-19 20:03:25.046127527+00:00,1 +2023-10-19 20:03:25.187536822+00:00,1 +2023-10-19 20:03:25.343652256+00:00,1 +2023-10-19 20:03:25.501964301+00:00,1 +2023-10-19 20:03:25.633894282+00:00,1 +2023-10-19 20:03:25.802811165+00:00,1 +2023-10-19 20:03:25.964326498+00:00,1 +2023-10-19 20:03:26.143200137+00:00,1 +2023-10-19 20:03:26.302394655+00:00,1 +2023-10-19 20:03:26.526404892+00:00,1 +2023-10-19 20:03:26.649671499+00:00,1 +2023-10-19 20:03:26.796741490+00:00,1 +2023-10-19 20:03:26.922139527+00:00,1 +2023-10-19 20:03:27.115813214+00:00,1 +2023-10-19 20:03:27.238996813+00:00,1 +2023-10-19 20:03:27.359538785+00:00,1 +2023-10-19 20:03:27.485061737+00:00,1 +2023-10-19 20:03:27.616765106+00:00,1 +2023-10-19 20:03:27.740803917+00:00,1 +2023-10-19 20:03:27.865346350+00:00,1 +2023-10-19 20:03:27.987734299+00:00,1 +2023-10-19 20:03:28.120010805+00:00,1 +2023-10-19 20:03:28.264771165+00:00,1 +2023-10-19 20:03:28.403514181+00:00,1 +2023-10-19 20:03:28.550868282+00:00,1 +2023-10-19 20:03:28.688355478+00:00,1 +2023-10-19 20:03:28.823643775+00:00,1 +2023-10-19 20:03:28.951549874+00:00,1 +2023-10-19 20:03:29.081485684+00:00,1 +2023-10-19 20:03:29.223428714+00:00,1 +2023-10-19 20:03:29.345027110+00:00,1 +2023-10-19 20:03:29.470822347+00:00,1 +2023-10-19 20:03:29.587628770+00:00,1 +2023-10-19 20:03:29.704738190+00:00,1 +2023-10-19 20:03:29.822364123+00:00,1 +2023-10-19 20:03:29.938989321+00:00,1 +2023-10-19 20:03:30.055990911+00:00,1 +2023-10-19 20:03:30.172956576+00:00,1 +2023-10-19 20:03:30.289587883+00:00,1 +2023-10-19 20:03:30.406957028+00:00,1 +2023-10-19 20:03:30.523674673+00:00,1 +2023-10-19 20:03:30.640385192+00:00,1 +2023-10-19 20:03:30.757909481+00:00,1 +2023-10-19 20:03:30.875452017+00:00,1 +2023-10-19 20:03:31.000223298+00:00,1 +2023-10-19 20:03:31.117962637+00:00,1 +2023-10-19 20:03:31.234130527+00:00,1 +2023-10-19 20:03:31.350601161+00:00,1 +2023-10-19 20:03:31.467691779+00:00,1 +2023-10-19 20:03:31.585092549+00:00,1 +2023-10-19 20:03:31.702783777+00:00,1 +2023-10-19 20:03:31.820491155+00:00,1 +2023-10-19 20:03:31.937891918+00:00,1 +2023-10-19 20:03:32.055536838+00:00,1 +2023-10-19 20:03:32.172173797+00:00,1 +2023-10-19 20:03:32.289072436+00:00,1 +2023-10-19 20:03:32.406691735+00:00,1 +2023-10-19 20:03:32.523601819+00:00,1 +2023-10-19 20:03:32.640640260+00:00,1 +2023-10-19 20:03:32.758106636+00:00,1 +2023-10-19 20:03:32.874931402+00:00,1 +2023-10-19 20:03:32.991925232+00:00,1 +2023-10-19 20:03:33.109509841+00:00,1 +2023-10-19 20:03:33.227315481+00:00,1 +2023-10-19 20:03:33.344154528+00:00,1 +2023-10-19 20:03:33.461080269+00:00,1 +2023-10-19 20:03:33.577617394+00:00,1 +2023-10-19 20:03:33.694084942+00:00,1 +2023-10-19 20:03:33.811761083+00:00,1 +2023-10-19 20:03:33.929076344+00:00,1 +2023-10-19 20:03:34.046728552+00:00,1 +2023-10-19 20:03:34.164403210+00:00,1 +2023-10-19 20:03:34.282989004+00:00,1 +2023-10-19 20:03:34.400647612+00:00,1 +2023-10-19 20:03:34.517520810+00:00,1 +2023-10-19 20:03:34.634131707+00:00,1 +2023-10-19 20:03:34.751199590+00:00,1 +2023-10-19 20:03:34.868174252+00:00,1 +2023-10-19 20:03:34.985277420+00:00,1 +2023-10-19 20:03:35.102178556+00:00,1 +2023-10-19 20:03:35.218227101+00:00,1 +2023-10-19 20:03:35.334898759+00:00,1 +2023-10-19 20:03:35.452274585+00:00,1 +2023-10-19 20:03:35.568833465+00:00,1 +2023-10-19 20:03:35.685738303+00:00,1 +2023-10-19 20:03:35.803172718+00:00,1 +2023-10-19 20:03:35.920079081+00:00,1 +2023-10-19 20:03:36.036706986+00:00,1 +2023-10-19 20:03:36.153026234+00:00,1 +2023-10-19 20:03:36.269978350+00:00,1 +2023-10-19 20:03:36.385994262+00:00,1 +2023-10-19 20:03:36.503245370+00:00,1 +2023-10-19 20:03:36.620397811+00:00,1 +2023-10-19 20:03:36.736868255+00:00,1 +2023-10-19 20:03:36.852802361+00:00,1 +2023-10-19 20:03:36.969649393+00:00,1 +2023-10-19 20:03:37.086320116+00:00,1 +2023-10-19 20:03:37.202612050+00:00,1 +2023-10-19 20:03:37.319680622+00:00,1 +2023-10-19 20:03:37.437046284+00:00,1 +2023-10-19 20:03:37.557464457+00:00,1 +2023-10-19 20:03:37.674275279+00:00,1 +2023-10-19 20:03:37.791074778+00:00,1 +2023-10-19 20:03:37.907912036+00:00,1 +2023-10-19 20:03:38.024044794+00:00,1 +2023-10-19 20:03:38.140680057+00:00,1 +2023-10-19 20:03:38.257045485+00:00,1 +2023-10-19 20:03:38.374320444+00:00,1 +2023-10-19 20:03:38.491175090+00:00,1 +2023-10-19 20:03:38.608017143+00:00,1 +2023-10-19 20:03:38.724758069+00:00,1 +2023-10-19 20:03:38.842344777+00:00,1 +2023-10-19 20:03:38.958844666+00:00,1 +2023-10-19 20:03:39.075443820+00:00,1 +2023-10-19 20:03:39.191952692+00:00,1 +2023-10-19 20:03:39.308005294+00:00,1 +2023-10-19 20:03:39.424139843+00:00,1 +2023-10-19 20:03:39.541152087+00:00,1 +2023-10-19 20:03:39.657844929+00:00,1 +2023-10-19 20:03:39.774969582+00:00,1 +2023-10-19 20:03:39.891810251+00:00,1 +2023-10-19 20:03:40.008824780+00:00,1 +2023-10-19 20:03:40.126738804+00:00,1 +2023-10-19 20:03:40.243521011+00:00,1 +2023-10-19 20:03:40.360388007+00:00,1 +2023-10-19 20:03:40.476700158+00:00,1 +2023-10-19 20:03:40.593667641+00:00,1 +2023-10-19 20:03:40.710172005+00:00,1 +2023-10-19 20:03:40.826989377+00:00,1 +2023-10-19 20:03:40.944800680+00:00,1 +2023-10-19 20:03:41.061989932+00:00,1 +2023-10-19 20:03:41.178833522+00:00,1 +2023-10-19 20:03:41.295717220+00:00,1 +2023-10-19 20:03:41.412172637+00:00,1 +2023-10-19 20:03:41.528173432+00:00,1 +2023-10-19 20:03:41.644650897+00:00,1 +2023-10-19 20:03:41.762110488+00:00,1 +2023-10-19 20:03:41.878960654+00:00,1 +2023-10-19 20:03:41.996182195+00:00,1 +2023-10-19 20:03:42.113971768+00:00,1 +2023-10-19 20:03:42.230784207+00:00,1 +2023-10-19 20:03:42.347994035+00:00,1 +2023-10-19 20:03:42.465034409+00:00,1 +2023-10-19 20:03:42.581277859+00:00,1 +2023-10-19 20:03:42.697985270+00:00,1 +2023-10-19 20:03:42.815579909+00:00,1 +2023-10-19 20:03:42.932841188+00:00,1 +2023-10-19 20:03:43.049666981+00:00,1 +2023-10-19 20:03:43.166996729+00:00,1 +2023-10-19 20:03:43.283706178+00:00,1 +2023-10-19 20:03:43.400904392+00:00,1 +2023-10-19 20:03:43.517231850+00:00,1 +2023-10-19 20:03:43.633720519+00:00,1 +2023-10-19 20:03:43.751553960+00:00,1 +2023-10-19 20:03:43.868489901+00:00,1 +2023-10-19 20:03:43.986188101+00:00,1 +2023-10-19 20:03:44.103037333+00:00,1 +2023-10-19 20:03:44.220577560+00:00,1 +2023-10-19 20:03:44.337499066+00:00,1 +2023-10-19 20:03:44.455012093+00:00,1 +2023-10-19 20:03:44.571401283+00:00,1 +2023-10-19 20:03:44.687479934+00:00,1 +2023-10-19 20:03:44.804228260+00:00,1 +2023-10-19 20:03:44.920169295+00:00,1 +2023-10-19 20:03:45.037056103+00:00,1 +2023-10-19 20:03:45.153889595+00:00,1 +2023-10-19 20:03:45.270303730+00:00,1 +2023-10-19 20:03:45.387825034+00:00,1 +2023-10-19 20:03:45.504917944+00:00,1 +2023-10-19 20:03:45.621690750+00:00,1 +2023-10-19 20:03:45.738333795+00:00,1 +2023-10-19 20:03:45.854716593+00:00,1 +2023-10-19 20:03:45.971637283+00:00,1 +2023-10-19 20:03:46.088677701+00:00,1 +2023-10-19 20:03:46.204975620+00:00,1 +2023-10-19 20:03:46.322094772+00:00,1 +2023-10-19 20:03:46.438670256+00:00,1 +2023-10-19 20:03:46.555489047+00:00,1 +2023-10-19 20:03:46.672354437+00:00,1 +2023-10-19 20:03:46.789515645+00:00,1 +2023-10-19 20:03:46.906195982+00:00,1 +2023-10-19 20:03:47.022937493+00:00,1 +2023-10-19 20:03:47.140179418+00:00,1 +2023-10-19 20:03:47.256761105+00:00,1 +2023-10-19 20:03:47.373279598+00:00,1 +2023-10-19 20:03:47.489961010+00:00,1 +2023-10-19 20:03:47.605870534+00:00,1 +2023-10-19 20:03:47.722461171+00:00,1 +2023-10-19 20:03:47.839262111+00:00,1 +2023-10-19 20:03:47.956425757+00:00,1 +2023-10-19 20:03:48.073302519+00:00,1 +2023-10-19 20:03:48.190698468+00:00,1 +2023-10-19 20:03:48.307495429+00:00,1 +2023-10-19 20:03:48.423609917+00:00,1 +2023-10-19 20:03:48.540726134+00:00,1 +2023-10-19 20:03:48.657542506+00:00,1 +2023-10-19 20:03:48.774704532+00:00,1 +2023-10-19 20:03:48.891487633+00:00,1 +2023-10-19 20:03:49.008075664+00:00,1 +2023-10-19 20:03:49.124799729+00:00,1 +2023-10-19 20:03:49.241858249+00:00,1 +2023-10-19 20:03:49.358467803+00:00,1 +2023-10-19 20:03:49.476164868+00:00,1 +2023-10-19 20:03:49.592795001+00:00,1 +2023-10-19 20:03:49.709613929+00:00,1 +2023-10-19 20:03:49.825579644+00:00,1 diff --git a/tests/zero-downtime-upgrades/results/1.0.0-special/http.png b/tests/zero-downtime-upgrades/results/1.0.0-special/http.png new file mode 100644 index 0000000000..80019cedb5 Binary files /dev/null and b/tests/zero-downtime-upgrades/results/1.0.0-special/http.png differ diff --git a/tests/zero-downtime-upgrades/results/1.0.0-special/https.csv b/tests/zero-downtime-upgrades/results/1.0.0-special/https.csv new file mode 100644 index 0000000000..210e948d0e --- /dev/null +++ b/tests/zero-downtime-upgrades/results/1.0.0-special/https.csv @@ -0,0 +1,600 @@ +2023-10-19 20:02:30.841325755+00:00,1 +2023-10-19 20:02:31.003576969+00:00,1 +2023-10-19 20:02:31.212437621+00:00,1 +2023-10-19 20:02:31.446205975+00:00,1 +2023-10-19 20:02:31.628345603+00:00,1 +2023-10-19 20:02:31.785030326+00:00,1 +2023-10-19 20:02:31.984870777+00:00,1 +2023-10-19 20:02:32.192090815+00:00,1 +2023-10-19 20:02:32.349552835+00:00,1 +2023-10-19 20:02:32.544690454+00:00,1 +2023-10-19 20:02:32.715916784+00:00,1 +2023-10-19 20:02:32.879360847+00:00,1 +2023-10-19 20:02:33.034619987+00:00,1 +2023-10-19 20:02:33.241585300+00:00,1 +2023-10-19 20:02:33.399808620+00:00,1 +2023-10-19 20:02:33.553901982+00:00,1 +2023-10-19 20:02:33.723307478+00:00,1 +2023-10-19 20:02:33.893365647+00:00,1 +2023-10-19 20:02:34.071635301+00:00,1 +2023-10-19 20:02:34.230403524+00:00,1 +2023-10-19 20:02:34.394201836+00:00,1 +2023-10-19 20:02:34.585251385+00:00,1 +2023-10-19 20:02:34.745949394+00:00,1 +2023-10-19 20:02:34.920973450+00:00,1 +2023-10-19 20:02:35.083872644+00:00,1 +2023-10-19 20:02:35.234763293+00:00,1 +2023-10-19 20:02:35.394301826+00:00,1 +2023-10-19 20:02:35.557384390+00:00,1 +2023-10-19 20:02:35.712952662+00:00,1 +2023-10-19 20:02:35.875056183+00:00,1 +2023-10-19 20:02:36.096915413+00:00,1 +2023-10-19 20:02:36.258154830+00:00,1 +2023-10-19 20:02:36.453343169+00:00,1 +2023-10-19 20:02:36.610376578+00:00,1 +2023-10-19 20:02:36.806721161+00:00,1 +2023-10-19 20:02:36.970435244+00:00,1 +2023-10-19 20:02:37.121078818+00:00,1 +2023-10-19 20:02:37.282973829+00:00,1 +2023-10-19 20:02:37.450345283+00:00,1 +2023-10-19 20:02:37.613201073+00:00,1 +2023-10-19 20:02:37.762166703+00:00,1 +2023-10-19 20:02:37.919621324+00:00,1 +2023-10-19 20:02:38.117563138+00:00,1 +2023-10-19 20:02:38.286199851+00:00,1 +2023-10-19 20:02:38.450278454+00:00,1 +2023-10-19 20:02:38.602065402+00:00,1 +2023-10-19 20:02:38.776199874+00:00,1 +2023-10-19 20:02:38.926426808+00:00,1 +2023-10-19 20:02:39.077551306+00:00,1 +2023-10-19 20:02:39.233779983+00:00,1 +2023-10-19 20:02:39.395588106+00:00,1 +2023-10-19 20:02:39.559300857+00:00,1 +2023-10-19 20:02:39.751729286+00:00,1 +2023-10-19 20:02:39.924888694+00:00,1 +2023-10-19 20:02:40.081251178+00:00,1 +2023-10-19 20:02:40.242388013+00:00,1 +2023-10-19 20:02:40.442292533+00:00,1 +2023-10-19 20:02:40.624767075+00:00,1 +2023-10-19 20:02:40.784553186+00:00,1 +2023-10-19 20:02:41.071688614+00:00,1 +2023-10-19 20:02:41.223760446+00:00,1 +2023-10-19 20:02:41.434810780+00:00,1 +2023-10-19 20:02:41.593319565+00:00,1 +2023-10-19 20:02:41.746642557+00:00,1 +2023-10-19 20:02:41.946333506+00:00,1 +2023-10-19 20:02:42.093600309+00:00,1 +2023-10-19 20:02:42.243360526+00:00,1 +2023-10-19 20:02:42.400913083+00:00,1 +2023-10-19 20:02:42.587471004+00:00,1 +2023-10-19 20:02:42.830335461+00:00,1 +2023-10-19 20:02:42.986560635+00:00,1 +2023-10-19 20:02:43.142534684+00:00,1 +2023-10-19 20:02:43.275739493+00:00,1 +2023-10-19 20:02:43.434290227+00:00,1 +2023-10-19 20:02:43.567605496+00:00,1 +2023-10-19 20:02:43.728427447+00:00,1 +2023-10-19 20:02:43.890398938+00:00,1 +2023-10-19 20:02:44.056064145+00:00,1 +2023-10-19 20:02:44.206680209+00:00,1 +2023-10-19 20:02:44.367413467+00:00,1 +2023-10-19 20:02:44.526577184+00:00,1 +2023-10-19 20:02:44.681246098+00:00,1 +2023-10-19 20:02:44.814815639+00:00,1 +2023-10-19 20:02:44.981226198+00:00,1 +2023-10-19 20:02:45.194979483+00:00,1 +2023-10-19 20:02:45.323572625+00:00,1 +2023-10-19 20:02:45.581571946+00:00,1 +2023-10-19 20:02:45.710811534+00:00,1 +2023-10-19 20:02:45.884837212+00:00,1 +2023-10-19 20:02:46.042915987+00:00,1 +2023-10-19 20:02:46.235132102+00:00,1 +2023-10-19 20:02:46.389811715+00:00,1 +2023-10-19 20:02:46.573353426+00:00,1 +2023-10-19 20:02:46.701918240+00:00,1 +2023-10-19 20:02:46.864166440+00:00,1 +2023-10-19 20:02:47.024665941+00:00,1 +2023-10-19 20:02:47.155774122+00:00,1 +2023-10-19 20:02:47.304864305+00:00,1 +2023-10-19 20:02:47.474913464+00:00,1 +2023-10-19 20:02:47.641528836+00:00,1 +2023-10-19 20:02:47.778477630+00:00,1 +2023-10-19 20:02:47.960802664+00:00,1 +2023-10-19 20:02:48.150605430+00:00,1 +2023-10-19 20:02:48.315668450+00:00,1 +2023-10-19 20:02:48.480363938+00:00,1 +2023-10-19 20:02:48.640754951+00:00,1 +2023-10-19 20:02:48.787770025+00:00,1 +2023-10-19 20:02:48.956969660+00:00,1 +2023-10-19 20:02:49.163546518+00:00,1 +2023-10-19 20:02:49.314886716+00:00,1 +2023-10-19 20:02:49.509095010+00:00,1 +2023-10-19 20:02:49.654013713+00:00,1 +2023-10-19 20:02:49.813476728+00:00,1 +2023-10-19 20:02:49.958872391+00:00,1 +2023-10-19 20:02:50.090760998+00:00,1 +2023-10-19 20:02:50.243914269+00:00,1 +2023-10-19 20:02:50.410345572+00:00,1 +2023-10-19 20:02:50.542462302+00:00,1 +2023-10-19 20:02:50.703423933+00:00,1 +2023-10-19 20:02:50.851006059+00:00,1 +2023-10-19 20:02:50.997122800+00:00,1 +2023-10-19 20:02:51.204414461+00:00,1 +2023-10-19 20:02:51.405990616+00:00,1 +2023-10-19 20:02:51.568878890+00:00,1 +2023-10-19 20:02:51.706255500+00:00,1 +2023-10-19 20:02:51.891269844+00:00,1 +2023-10-19 20:02:52.039592909+00:00,1 +2023-10-19 20:02:52.219622723+00:00,1 +2023-10-19 20:02:52.437520202+00:00,1 +2023-10-19 20:02:52.581511374+00:00,1 +2023-10-19 20:02:52.746378512+00:00,1 +2023-10-19 20:02:52.907131542+00:00,1 +2023-10-19 20:02:53.073375026+00:00,1 +2023-10-19 20:02:53.219654282+00:00,1 +2023-10-19 20:02:53.401657644+00:00,1 +2023-10-19 20:02:53.542948159+00:00,1 +2023-10-19 20:02:53.697019811+00:00,1 +2023-10-19 20:02:53.875533620+00:00,1 +2023-10-19 20:02:54.044140240+00:00,1 +2023-10-19 20:02:54.229099418+00:00,1 +2023-10-19 20:02:54.400096403+00:00,1 +2023-10-19 20:02:54.535522456+00:00,1 +2023-10-19 20:02:54.673563161+00:00,1 +2023-10-19 20:02:54.808787110+00:00,1 +2023-10-19 20:02:54.975941525+00:00,1 +2023-10-19 20:02:55.220160865+00:00,1 +2023-10-19 20:02:55.416274735+00:00,1 +2023-10-19 20:02:55.559872636+00:00,1 +2023-10-19 20:02:55.713243347+00:00,1 +2023-10-19 20:02:55.928340026+00:00,1 +2023-10-19 20:02:56.079667574+00:00,1 +2023-10-19 20:02:56.593463215+00:00,1 +2023-10-19 20:02:56.734756105+00:00,1 +2023-10-19 20:02:56.888749539+00:00,1 +2023-10-19 20:02:57.131723727+00:00,1 +2023-10-19 20:02:57.286952153+00:00,1 +2023-10-19 20:02:57.445429111+00:00,1 +2023-10-19 20:02:57.627661288+00:00,1 +2023-10-19 20:02:57.807943161+00:00,1 +2023-10-19 20:02:57.966575341+00:00,1 +2023-10-19 20:02:58.141443982+00:00,1 +2023-10-19 20:02:58.298159521+00:00,1 +2023-10-19 20:02:58.441254622+00:00,1 +2023-10-19 20:02:58.597731036+00:00,1 +2023-10-19 20:02:58.752230624+00:00,1 +2023-10-19 20:02:58.916003577+00:00,1 +2023-10-19 20:02:59.070977487+00:00,1 +2023-10-19 20:02:59.239073752+00:00,1 +2023-10-19 20:02:59.376747189+00:00,1 +2023-10-19 20:02:59.527137161+00:00,1 +2023-10-19 20:02:59.688137492+00:00,1 +2023-10-19 20:02:59.824405338+00:00,1 +2023-10-19 20:02:59.975524507+00:00,1 +2023-10-19 20:03:00.120447365+00:00,1 +2023-10-19 20:03:00.270246408+00:00,1 +2023-10-19 20:03:00.414354972+00:00,1 +2023-10-19 20:03:00.590458823+00:00,1 +2023-10-19 20:03:00.747494267+00:00,1 +2023-10-19 20:03:00.906834495+00:00,1 +2023-10-19 20:03:01.062425437+00:00,1 +2023-10-19 20:03:01.211513803+00:00,1 +2023-10-19 20:03:01.394301955+00:00,1 +2023-10-19 20:03:01.566208726+00:00,1 +2023-10-19 20:03:01.723976725+00:00,1 +2023-10-19 20:03:01.884577907+00:00,1 +2023-10-19 20:03:02.029560030+00:00,1 +2023-10-19 20:03:02.167944297+00:00,1 +2023-10-19 20:03:02.334793226+00:00,1 +2023-10-19 20:03:02.498677147+00:00,1 +2023-10-19 20:03:02.644326136+00:00,1 +2023-10-19 20:03:02.791388165+00:00,1 +2023-10-19 20:03:02.969368023+00:00,1 +2023-10-19 20:03:03.107310141+00:00,1 +2023-10-19 20:03:03.266446519+00:00,1 +2023-10-19 20:03:03.419181376+00:00,1 +2023-10-19 20:03:03.575781469+00:00,1 +2023-10-19 20:03:03.775119440+00:00,1 +2023-10-19 20:03:03.922009964+00:00,1 +2023-10-19 20:03:04.090363089+00:00,1 +2023-10-19 20:03:04.383698946+00:00,1 +2023-10-19 20:03:04.538177369+00:00,1 +2023-10-19 20:03:04.706300126+00:00,1 +2023-10-19 20:03:04.869933007+00:00,1 +2023-10-19 20:03:05.019017042+00:00,1 +2023-10-19 20:03:05.168383033+00:00,1 +2023-10-19 20:03:05.331789014+00:00,1 +2023-10-19 20:03:05.486050261+00:00,1 +2023-10-19 20:03:05.633191540+00:00,1 +2023-10-19 20:03:05.772746634+00:00,1 +2023-10-19 20:03:05.921702795+00:00,1 +2023-10-19 20:03:06.065208723+00:00,1 +2023-10-19 20:03:06.252096586+00:00,1 +2023-10-19 20:03:06.390014061+00:00,1 +2023-10-19 20:03:06.577172048+00:00,1 +2023-10-19 20:03:06.756621512+00:00,1 +2023-10-19 20:03:06.918470659+00:00,1 +2023-10-19 20:03:07.075030149+00:00,1 +2023-10-19 20:03:07.254488226+00:00,1 +2023-10-19 20:03:07.414686012+00:00,1 +2023-10-19 20:03:07.651746896+00:00,1 +2023-10-19 20:03:07.798817388+00:00,1 +2023-10-19 20:03:07.972775060+00:00,1 +2023-10-19 20:03:08.124836758+00:00,1 +2023-10-19 20:03:08.322862883+00:00,1 +2023-10-19 20:03:08.479429889+00:00,1 +2023-10-19 20:03:08.638893217+00:00,1 +2023-10-19 20:03:08.797249602+00:00,1 +2023-10-19 20:03:08.945292456+00:00,1 +2023-10-19 20:03:09.098530331+00:00,1 +2023-10-19 20:03:09.253456269+00:00,1 +2023-10-19 20:03:09.417868260+00:00,1 +2023-10-19 20:03:09.612322429+00:00,1 +2023-10-19 20:03:09.755356570+00:00,1 +2023-10-19 20:03:09.929426723+00:00,1 +2023-10-19 20:03:10.109718079+00:00,1 +2023-10-19 20:03:10.251035039+00:00,1 +2023-10-19 20:03:10.424032796+00:00,1 +2023-10-19 20:03:10.592750776+00:00,1 +2023-10-19 20:03:10.745936571+00:00,1 +2023-10-19 20:03:10.892651666+00:00,1 +2023-10-19 20:03:11.099998744+00:00,1 +2023-10-19 20:03:11.248557313+00:00,1 +2023-10-19 20:03:11.514779292+00:00,1 +2023-10-19 20:03:11.667619765+00:00,1 +2023-10-19 20:03:11.830731026+00:00,1 +2023-10-19 20:03:12.002983649+00:00,1 +2023-10-19 20:03:12.182400707+00:00,1 +2023-10-19 20:03:12.322523358+00:00,1 +2023-10-19 20:03:12.463508167+00:00,1 +2023-10-19 20:03:12.613429268+00:00,1 +2023-10-19 20:03:12.773753905+00:00,1 +2023-10-19 20:03:12.940500841+00:00,1 +2023-10-19 20:03:13.110183634+00:00,1 +2023-10-19 20:03:13.255755023+00:00,1 +2023-10-19 20:03:13.407316171+00:00,1 +2023-10-19 20:03:13.566823088+00:00,1 +2023-10-19 20:03:13.719591252+00:00,1 +2023-10-19 20:03:13.916537595+00:00,1 +2023-10-19 20:03:14.069755435+00:00,1 +2023-10-19 20:03:14.225211638+00:00,1 +2023-10-19 20:03:14.403597354+00:00,1 +2023-10-19 20:03:14.543892503+00:00,1 +2023-10-19 20:03:14.680258389+00:00,1 +2023-10-19 20:03:14.825292441+00:00,1 +2023-10-19 20:03:15.010303398+00:00,1 +2023-10-19 20:03:15.192761848+00:00,1 +2023-10-19 20:03:15.340195657+00:00,1 +2023-10-19 20:03:15.489060676+00:00,1 +2023-10-19 20:03:15.640916168+00:00,1 +2023-10-19 20:03:15.783635222+00:00,1 +2023-10-19 20:03:16.021501054+00:00,1 +2023-10-19 20:03:16.182193845+00:00,1 +2023-10-19 20:03:16.347754640+00:00,1 +2023-10-19 20:03:16.500548732+00:00,1 +2023-10-19 20:03:16.699802229+00:00,1 +2023-10-19 20:03:16.838858605+00:00,1 +2023-10-19 20:03:17.018283632+00:00,1 +2023-10-19 20:03:17.168217150+00:00,1 +2023-10-19 20:03:17.344185111+00:00,1 +2023-10-19 20:03:17.636729807+00:00,1 +2023-10-19 20:03:17.804377384+00:00,1 +2023-10-19 20:03:18.015315227+00:00,1 +2023-10-19 20:03:18.181348692+00:00,1 +2023-10-19 20:03:18.360149457+00:00,1 +2023-10-19 20:03:18.531585287+00:00,1 +2023-10-19 20:03:18.731883089+00:00,1 +2023-10-19 20:03:18.935083470+00:00,1 +2023-10-19 20:03:19.136816445+00:00,1 +2023-10-19 20:03:19.425338385+00:00,1 +2023-10-19 20:03:19.629936676+00:00,1 +2023-10-19 20:03:19.830195969+00:00,1 +2023-10-19 20:03:19.971475501+00:00,1 +2023-10-19 20:03:20.129869850+00:00,1 +2023-10-19 20:03:20.311956096+00:00,1 +2023-10-19 20:03:20.531773542+00:00,1 +2023-10-19 20:03:20.717346119+00:00,1 +2023-10-19 20:03:20.893607396+00:00,1 +2023-10-19 20:03:21.054162962+00:00,1 +2023-10-19 20:03:21.217317454+00:00,1 +2023-10-19 20:03:21.443602581+00:00,1 +2023-10-19 20:03:21.599622991+00:00,1 +2023-10-19 20:03:21.762991537+00:00,1 +2023-10-19 20:03:22.019479690+00:00,1 +2023-10-19 20:03:22.204188637+00:00,1 +2023-10-19 20:03:22.409783579+00:00,1 +2023-10-19 20:03:22.646447446+00:00,1 +2023-10-19 20:03:22.807145837+00:00,1 +2023-10-19 20:03:22.967815977+00:00,1 +2023-10-19 20:03:23.177190018+00:00,1 +2023-10-19 20:03:23.336872891+00:00,1 +2023-10-19 20:03:23.494938513+00:00,1 +2023-10-19 20:03:23.645465024+00:00,1 +2023-10-19 20:03:23.803727763+00:00,1 +2023-10-19 20:03:24.082028289+00:00,1 +2023-10-19 20:03:24.236699166+00:00,1 +2023-10-19 20:03:24.383583687+00:00,1 +2023-10-19 20:03:24.543849483+00:00,1 +2023-10-19 20:03:24.815899728+00:00,1 +2023-10-19 20:03:24.987352819+00:00,1 +2023-10-19 20:03:25.144750820+00:00,1 +2023-10-19 20:03:25.316157609+00:00,1 +2023-10-19 20:03:25.473303287+00:00,1 +2023-10-19 20:03:25.621571898+00:00,1 +2023-10-19 20:03:25.788716919+00:00,1 +2023-10-19 20:03:25.959605295+00:00,1 +2023-10-19 20:03:26.148459942+00:00,1 +2023-10-19 20:03:26.306454859+00:00,1 +2023-10-19 20:03:26.533742849+00:00,1 +2023-10-19 20:03:26.671520793+00:00,1 +2023-10-19 20:03:26.824907052+00:00,1 +2023-10-19 20:03:26.982535411+00:00,1 +2023-10-19 20:03:27.129248810+00:00,1 +2023-10-19 20:03:27.479076329+00:00,1 +2023-10-19 20:03:27.647455652+00:00,1 +2023-10-19 20:03:27.813105550+00:00,1 +2023-10-19 20:03:27.969215066+00:00,1 +2023-10-19 20:03:28.139532029+00:00,1 +2023-10-19 20:03:28.348087888+00:00,1 +2023-10-19 20:03:28.503878884+00:00,1 +2023-10-19 20:03:28.664236232+00:00,1 +2023-10-19 20:03:28.830194234+00:00,1 +2023-10-19 20:03:28.974652307+00:00,1 +2023-10-19 20:03:29.124570884+00:00,1 +2023-10-19 20:03:29.541011833+00:00,1 +2023-10-19 20:03:29.667442894+00:00,1 +2023-10-19 20:03:29.793063420+00:00,1 +2023-10-19 20:03:29.918968378+00:00,1 +2023-10-19 20:03:30.044334560+00:00,1 +2023-10-19 20:03:30.170416950+00:00,1 +2023-10-19 20:03:30.296268831+00:00,1 +2023-10-19 20:03:30.423013607+00:00,1 +2023-10-19 20:03:30.549138167+00:00,1 +2023-10-19 20:03:30.674718787+00:00,1 +2023-10-19 20:03:30.800361940+00:00,1 +2023-10-19 20:03:30.925350184+00:00,1 +2023-10-19 20:03:31.051264313+00:00,1 +2023-10-19 20:03:31.178171674+00:00,1 +2023-10-19 20:03:31.305769342+00:00,1 +2023-10-19 20:03:31.431307595+00:00,1 +2023-10-19 20:03:31.556670538+00:00,1 +2023-10-19 20:03:31.682982215+00:00,1 +2023-10-19 20:03:31.809257890+00:00,1 +2023-10-19 20:03:31.935194748+00:00,1 +2023-10-19 20:03:32.060740761+00:00,1 +2023-10-19 20:03:32.187627488+00:00,1 +2023-10-19 20:03:32.314200096+00:00,1 +2023-10-19 20:03:32.440265412+00:00,1 +2023-10-19 20:03:32.565956831+00:00,1 +2023-10-19 20:03:32.691494664+00:00,1 +2023-10-19 20:03:32.819045519+00:00,1 +2023-10-19 20:03:32.946132318+00:00,1 +2023-10-19 20:03:33.072602722+00:00,1 +2023-10-19 20:03:33.200409271+00:00,1 +2023-10-19 20:03:33.327104115+00:00,1 +2023-10-19 20:03:33.452815076+00:00,1 +2023-10-19 20:03:33.578989043+00:00,1 +2023-10-19 20:03:33.705385312+00:00,1 +2023-10-19 20:03:33.830914066+00:00,1 +2023-10-19 20:03:33.956942700+00:00,1 +2023-10-19 20:03:34.082934246+00:00,1 +2023-10-19 20:03:34.208488060+00:00,1 +2023-10-19 20:03:34.334314313+00:00,1 +2023-10-19 20:03:34.459956755+00:00,1 +2023-10-19 20:03:34.586142425+00:00,1 +2023-10-19 20:03:34.710740877+00:00,1 +2023-10-19 20:03:34.836280491+00:00,1 +2023-10-19 20:03:34.963373411+00:00,1 +2023-10-19 20:03:35.089018844+00:00,1 +2023-10-19 20:03:35.215518701+00:00,1 +2023-10-19 20:03:35.341256834+00:00,1 +2023-10-19 20:03:35.467321113+00:00,1 +2023-10-19 20:03:35.592863679+00:00,1 +2023-10-19 20:03:35.718758470+00:00,1 +2023-10-19 20:03:35.845545135+00:00,1 +2023-10-19 20:03:35.971187624+00:00,1 +2023-10-19 20:03:36.097164394+00:00,1 +2023-10-19 20:03:36.222562287+00:00,1 +2023-10-19 20:03:36.348388180+00:00,1 +2023-10-19 20:03:36.473757626+00:00,1 +2023-10-19 20:03:36.599021482+00:00,1 +2023-10-19 20:03:36.725278420+00:00,1 +2023-10-19 20:03:36.850963374+00:00,1 +2023-10-19 20:03:36.977259007+00:00,1 +2023-10-19 20:03:37.103225517+00:00,1 +2023-10-19 20:03:37.228867364+00:00,1 +2023-10-19 20:03:37.354257386+00:00,1 +2023-10-19 20:03:37.479604099+00:00,1 +2023-10-19 20:03:37.604592838+00:00,1 +2023-10-19 20:03:37.729423660+00:00,1 +2023-10-19 20:03:37.855094133+00:00,1 +2023-10-19 20:03:37.980385394+00:00,1 +2023-10-19 20:03:38.105814876+00:00,1 +2023-10-19 20:03:38.232039513+00:00,1 +2023-10-19 20:03:38.358100820+00:00,1 +2023-10-19 20:03:38.484133341+00:00,1 +2023-10-19 20:03:38.610907306+00:00,1 +2023-10-19 20:03:38.736731712+00:00,1 +2023-10-19 20:03:38.862549143+00:00,1 +2023-10-19 20:03:38.988210394+00:00,1 +2023-10-19 20:03:39.113393449+00:00,1 +2023-10-19 20:03:39.239052321+00:00,1 +2023-10-19 20:03:39.363851806+00:00,1 +2023-10-19 20:03:39.489593565+00:00,1 +2023-10-19 20:03:39.615466041+00:00,1 +2023-10-19 20:03:39.740469697+00:00,1 +2023-10-19 20:03:39.865383825+00:00,1 +2023-10-19 20:03:39.989363564+00:00,1 +2023-10-19 20:03:40.114697003+00:00,1 +2023-10-19 20:03:40.239925476+00:00,1 +2023-10-19 20:03:40.366355970+00:00,1 +2023-10-19 20:03:40.492160033+00:00,1 +2023-10-19 20:03:40.617514779+00:00,1 +2023-10-19 20:03:40.743108587+00:00,1 +2023-10-19 20:03:40.867880484+00:00,1 +2023-10-19 20:03:40.992976609+00:00,1 +2023-10-19 20:03:41.117822326+00:00,1 +2023-10-19 20:03:41.242797141+00:00,1 +2023-10-19 20:03:41.368544188+00:00,1 +2023-10-19 20:03:41.496373405+00:00,1 +2023-10-19 20:03:41.622175058+00:00,1 +2023-10-19 20:03:41.747201147+00:00,1 +2023-10-19 20:03:41.872615546+00:00,1 +2023-10-19 20:03:41.998159300+00:00,1 +2023-10-19 20:03:42.123665945+00:00,1 +2023-10-19 20:03:42.249553311+00:00,1 +2023-10-19 20:03:42.375533995+00:00,1 +2023-10-19 20:03:42.500895244+00:00,1 +2023-10-19 20:03:42.627359509+00:00,1 +2023-10-19 20:03:42.753292879+00:00,1 +2023-10-19 20:03:42.879831271+00:00,1 +2023-10-19 20:03:43.006141985+00:00,1 +2023-10-19 20:03:43.131938647+00:00,1 +2023-10-19 20:03:43.258704091+00:00,1 +2023-10-19 20:03:43.383979402+00:00,1 +2023-10-19 20:03:43.509780706+00:00,1 +2023-10-19 20:03:43.636033903+00:00,1 +2023-10-19 20:03:43.762470991+00:00,1 +2023-10-19 20:03:43.889309943+00:00,1 +2023-10-19 20:03:44.015755763+00:00,1 +2023-10-19 20:03:44.142019122+00:00,1 +2023-10-19 20:03:44.267874094+00:00,1 +2023-10-19 20:03:44.392567203+00:00,1 +2023-10-19 20:03:44.517889743+00:00,1 +2023-10-19 20:03:44.643712540+00:00,1 +2023-10-19 20:03:44.769609557+00:00,1 +2023-10-19 20:03:44.894991034+00:00,1 +2023-10-19 20:03:45.020581264+00:00,1 +2023-10-19 20:03:45.146494534+00:00,1 +2023-10-19 20:03:45.272631321+00:00,1 +2023-10-19 20:03:45.398704128+00:00,1 +2023-10-19 20:03:45.524490669+00:00,1 +2023-10-19 20:03:45.650927339+00:00,1 +2023-10-19 20:03:45.777254690+00:00,1 +2023-10-19 20:03:45.902746956+00:00,1 +2023-10-19 20:03:46.028563129+00:00,1 +2023-10-19 20:03:46.154384640+00:00,1 +2023-10-19 20:03:46.281288753+00:00,1 +2023-10-19 20:03:46.407684480+00:00,1 +2023-10-19 20:03:46.533714072+00:00,1 +2023-10-19 20:03:46.659181716+00:00,1 +2023-10-19 20:03:46.785944723+00:00,1 +2023-10-19 20:03:46.910772172+00:00,1 +2023-10-19 20:03:47.037213852+00:00,1 +2023-10-19 20:03:47.163516924+00:00,1 +2023-10-19 20:03:47.289699726+00:00,1 +2023-10-19 20:03:47.416607837+00:00,1 +2023-10-19 20:03:47.541457421+00:00,1 +2023-10-19 20:03:47.667540974+00:00,1 +2023-10-19 20:03:47.793408451+00:00,1 +2023-10-19 20:03:47.918864882+00:00,1 +2023-10-19 20:03:48.045160245+00:00,1 +2023-10-19 20:03:48.171436871+00:00,1 +2023-10-19 20:03:48.297084496+00:00,1 +2023-10-19 20:03:48.421898756+00:00,1 +2023-10-19 20:03:48.547742889+00:00,1 +2023-10-19 20:03:48.673851022+00:00,1 +2023-10-19 20:03:48.800164623+00:00,1 +2023-10-19 20:03:48.926549055+00:00,1 +2023-10-19 20:03:49.053124625+00:00,1 +2023-10-19 20:03:49.179570821+00:00,1 +2023-10-19 20:03:49.305470980+00:00,1 +2023-10-19 20:03:49.431309991+00:00,1 +2023-10-19 20:03:49.557210841+00:00,1 +2023-10-19 20:03:49.682818266+00:00,1 +2023-10-19 20:03:49.808817763+00:00,1 +2023-10-19 20:03:49.934704605+00:00,1 +2023-10-19 20:03:50.061372940+00:00,1 +2023-10-19 20:03:50.187529683+00:00,1 +2023-10-19 20:03:50.313932510+00:00,1 +2023-10-19 20:03:50.439870056+00:00,1 +2023-10-19 20:03:50.564791783+00:00,1 +2023-10-19 20:03:50.690724927+00:00,1 +2023-10-19 20:03:50.815953583+00:00,1 +2023-10-19 20:03:50.942256158+00:00,1 +2023-10-19 20:03:51.068491535+00:00,1 +2023-10-19 20:03:51.194368777+00:00,1 +2023-10-19 20:03:51.321342229+00:00,1 +2023-10-19 20:03:51.447335277+00:00,1 +2023-10-19 20:03:51.574122506+00:00,1 +2023-10-19 20:03:51.699566261+00:00,1 +2023-10-19 20:03:51.825229496+00:00,1 +2023-10-19 20:03:51.949990650+00:00,1 +2023-10-19 20:03:52.077028274+00:00,1 +2023-10-19 20:03:52.203313676+00:00,1 +2023-10-19 20:03:52.329586195+00:00,1 +2023-10-19 20:03:52.455734307+00:00,1 +2023-10-19 20:03:52.581655146+00:00,1 +2023-10-19 20:03:52.707965953+00:00,1 +2023-10-19 20:03:52.833774085+00:00,1 +2023-10-19 20:03:52.959901560+00:00,1 +2023-10-19 20:03:53.086211812+00:00,1 +2023-10-19 20:03:53.212129446+00:00,1 +2023-10-19 20:03:53.339767226+00:00,1 +2023-10-19 20:03:53.465887175+00:00,1 +2023-10-19 20:03:53.592274410+00:00,1 +2023-10-19 20:03:53.717903478+00:00,1 +2023-10-19 20:03:53.843722996+00:00,1 +2023-10-19 20:03:53.969529685+00:00,1 +2023-10-19 20:03:54.094474005+00:00,1 +2023-10-19 20:03:54.219569461+00:00,1 +2023-10-19 20:03:54.345725124+00:00,1 +2023-10-19 20:03:54.472267404+00:00,1 +2023-10-19 20:03:54.597951981+00:00,1 +2023-10-19 20:03:54.723007397+00:00,1 +2023-10-19 20:03:54.849058693+00:00,1 +2023-10-19 20:03:54.974669642+00:00,1 +2023-10-19 20:03:55.099935422+00:00,1 +2023-10-19 20:03:55.225188947+00:00,1 +2023-10-19 20:03:55.351063750+00:00,1 +2023-10-19 20:03:55.477317749+00:00,1 +2023-10-19 20:03:55.603836548+00:00,1 +2023-10-19 20:03:55.729248991+00:00,1 +2023-10-19 20:03:55.855592458+00:00,1 +2023-10-19 20:03:55.980920547+00:00,1 +2023-10-19 20:03:56.107688061+00:00,1 +2023-10-19 20:03:56.233570069+00:00,1 +2023-10-19 20:03:56.359527681+00:00,1 +2023-10-19 20:03:56.485253022+00:00,1 +2023-10-19 20:03:56.611909870+00:00,1 +2023-10-19 20:03:56.738279581+00:00,1 +2023-10-19 20:03:56.864793067+00:00,1 +2023-10-19 20:03:56.991135150+00:00,1 +2023-10-19 20:03:57.117649689+00:00,1 +2023-10-19 20:03:57.242806854+00:00,1 +2023-10-19 20:03:57.369069253+00:00,1 +2023-10-19 20:03:57.494868342+00:00,1 +2023-10-19 20:03:57.620537588+00:00,1 +2023-10-19 20:03:57.746733121+00:00,1 +2023-10-19 20:03:57.872193554+00:00,1 +2023-10-19 20:03:57.998163538+00:00,1 +2023-10-19 20:03:58.124111995+00:00,1 +2023-10-19 20:03:58.250004798+00:00,1 +2023-10-19 20:03:58.376110570+00:00,1 +2023-10-19 20:03:58.501572189+00:00,1 +2023-10-19 20:03:58.627497912+00:00,1 +2023-10-19 20:03:58.753029068+00:00,1 +2023-10-19 20:03:58.877922147+00:00,1 +2023-10-19 20:03:59.003911261+00:00,1 +2023-10-19 20:03:59.129999831+00:00,1 +2023-10-19 20:03:59.257587323+00:00,1 +2023-10-19 20:03:59.383328885+00:00,1 +2023-10-19 20:03:59.509152441+00:00,1 +2023-10-19 20:03:59.635163813+00:00,1 +2023-10-19 20:03:59.760215033+00:00,1 +2023-10-19 20:03:59.886190662+00:00,1 +2023-10-19 20:04:00.012314168+00:00,1 +2023-10-19 20:04:00.137882774+00:00,1 +2023-10-19 20:04:00.263761628+00:00,1 +2023-10-19 20:04:00.388560077+00:00,1 +2023-10-19 20:04:00.514360657+00:00,1 +2023-10-19 20:04:00.640242246+00:00,1 +2023-10-19 20:04:00.766135606+00:00,1 +2023-10-19 20:04:00.891904630+00:00,1 +2023-10-19 20:04:01.018688145+00:00,1 +2023-10-19 20:04:01.145230213+00:00,1 +2023-10-19 20:04:01.270731671+00:00,1 +2023-10-19 20:04:01.397180505+00:00,1 +2023-10-19 20:04:01.523442947+00:00,1 +2023-10-19 20:04:01.648963466+00:00,1 +2023-10-19 20:04:01.775126825+00:00,1 diff --git a/tests/zero-downtime-upgrades/results/1.0.0-special/https.png b/tests/zero-downtime-upgrades/results/1.0.0-special/https.png new file mode 100644 index 0000000000..6adff7750b Binary files /dev/null and b/tests/zero-downtime-upgrades/results/1.0.0-special/https.png differ diff --git a/tests/zero-downtime-upgrades/values.yaml b/tests/zero-downtime-upgrades/values.yaml new file mode 100644 index 0000000000..679e200cb2 --- /dev/null +++ b/tests/zero-downtime-upgrades/values.yaml @@ -0,0 +1,35 @@ +nginxGateway: + replicaCount: 2 + image: + repository: gcr.io//nginx-gateway-fabric + tag: + pullPolicy: Always + lifecycle: + preStop: + exec: + command: + - /usr/bin/gateway + - sleep + - --duration=40s + +nginx: + lifecycle: + preStop: + exec: + command: + - /bin/sleep + - "40" + +terminationGracePeriodSeconds: 50 + +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: nginx-gateway + +service: + annotations: + networking.gke.io/load-balancer-type: "Internal" diff --git a/tests/zero-downtime-upgrades/zero-downtime-upgrades.md b/tests/zero-downtime-upgrades/zero-downtime-upgrades.md index 06c7bada30..c37d50c311 100644 --- a/tests/zero-downtime-upgrades/zero-downtime-upgrades.md +++ b/tests/zero-downtime-upgrades/zero-downtime-upgrades.md @@ -202,6 +202,7 @@ Notes: ## Results - [1.0.0](results/1.0.0/1.0.0.md) +- [1.0.0-special](results/1.0.0-special/1.0.0-special.md) ## Appendix