From 67cdb7460eedf7fb68dd785e95c8fea04b272895 Mon Sep 17 00:00:00 2001 From: Guido R Date: Sun, 22 May 2022 19:03:32 +0200 Subject: [PATCH] Serve instance metrics --- deploy/crownlabs/values.yaml | 1 + operators/cmd/instance-operator/main.go | 1 + .../templates/deployment.yaml | 1 + .../deploy/instance-operator/values.yaml | 1 + operators/pkg/forge/containers.go | 49 +++- operators/pkg/forge/containers_test.go | 38 ++- operators/pkg/forge/ingresses.go | 24 +- operators/pkg/forge/ingresses_test.go | 18 +- operators/pkg/instctrl/controller_test.go | 4 +- operators/pkg/instctrl/exposition_test.go | 13 +- .../containers/gui-common/websockify/go.mod | 12 +- .../containers/gui-common/websockify/go.sum | 134 +++++++++- .../gui-common/websockify/instmetrics.go | 212 ++++++++++++++++ .../websockify/instmetrics/instmetrics.pb.go | 237 ++++++++++++++++++ .../websockify/instmetrics/instmetrics.proto | 21 ++ .../instmetrics/instmetrics_grpc.pb.go | 109 ++++++++ .../containers/gui-common/websockify/main.go | 53 +++- .../gui-common/websockify/metrics.go | 5 +- .../novnc-overrides/app/images/stats-img.svg | 2 + .../novnc-overrides/app/styles/metrics.css | 153 +++++++++++ .../websockify/novnc-overrides/app/ui.js | 216 +++++++++++++++- .../containers/gui-common/websockify/novnc.go | 65 ++++- .../websockify/remote_instmetrics.go | 71 ++++++ .../gui-common/websockify/websockify.go | 41 ++- 24 files changed, 1391 insertions(+), 90 deletions(-) create mode 100644 provisioning/containers/gui-common/websockify/instmetrics.go create mode 100644 provisioning/containers/gui-common/websockify/instmetrics/instmetrics.pb.go create mode 100644 provisioning/containers/gui-common/websockify/instmetrics/instmetrics.proto create mode 100644 provisioning/containers/gui-common/websockify/instmetrics/instmetrics_grpc.pb.go create mode 100644 provisioning/containers/gui-common/websockify/novnc-overrides/app/images/stats-img.svg create mode 100644 provisioning/containers/gui-common/websockify/novnc-overrides/app/styles/metrics.css create mode 100644 provisioning/containers/gui-common/websockify/remote_instmetrics.go diff --git a/deploy/crownlabs/values.yaml b/deploy/crownlabs/values.yaml index b46079186..664326710 100644 --- a/deploy/crownlabs/values.yaml +++ b/deploy/crownlabs/values.yaml @@ -75,6 +75,7 @@ instance-operator: novncImage: crownlabs/novnc filebrowserImage: filebrowser/filebrowser filebrowserImageTag: latest + instmetricsServerEndpoint: crownlabs-instmetrics.crownlabs-production:9090 containerVmSnapshots: kanikoImage: gcr.io/kaniko-project/executor exportImage: "crownlabs/img-exporter" diff --git a/operators/cmd/instance-operator/main.go b/operators/cmd/instance-operator/main.go index ad6068751..d95fc690d 100644 --- a/operators/cmd/instance-operator/main.go +++ b/operators/cmd/instance-operator/main.go @@ -86,6 +86,7 @@ func main() { flag.StringVar(&containerEnvOpts.ContentDownloaderImg, "container-env-content-downloader-img", "latest", "The image name for the init-container to download and unarchive initial content to the instance volume.") flag.StringVar(&containerEnvOpts.ContentUploaderImg, "container-env-content-uploader-img", "latest", "The image name for the job to compress and upload instance content from a persistent instance.") flag.StringVar(&containerEnvOpts.MyDriveImgAndTag, "container-env-mydrive-img-and-tag", "filebrowser/filebrowser:latest", "The image name and tag for the filebrowser image (sidecar for gui-based file manager)") + flag.StringVar(&containerEnvOpts.InstMetricsEndpoint, "container-env-instmetrics-server-endpoint", "instmetrics:9090", "The endpoint of the InstMetrics gRPC server") flag.StringVar(&instSnapOpts.VMRegistry, "vm-registry", "", "The registry where VMs should be uploaded") flag.StringVar(&instSnapOpts.RegistrySecretName, "vm-registry-secret", "", "The name of the secret for the VM registry") diff --git a/operators/deploy/instance-operator/templates/deployment.yaml b/operators/deploy/instance-operator/templates/deployment.yaml index 6aea66dda..a7f223420 100644 --- a/operators/deploy/instance-operator/templates/deployment.yaml +++ b/operators/deploy/instance-operator/templates/deployment.yaml @@ -45,6 +45,7 @@ spec: - "--container-env-content-downloader-img={{ .Values.configurations.containerEnvironmentOptions.contentDownloaderImage }}" - "--container-env-content-uploader-img={{ .Values.configurations.containerEnvironmentOptions.contentUploaderImage }}" - "--container-env-mydrive-img-and-tag={{ .Values.configurations.containerEnvironmentOptions.mydriveImageAndTag }}" + - "--container-env-instmetrics-server-endpoint={{ .Values.configurations.containerEnvironmentOptions.instmetricsServerEndpoint }}" - "--vm-registry={{ .Values.configurations.privateContainerRegistry.url }}" - "--vm-registry-secret={{ .Values.configurations.privateContainerRegistry.secretName }}" - "--container-export-img={{ .Values.configurations.containerVmSnapshots.exportImage }}:{{ include "instance-operator.containerExportImageTag" . }}" diff --git a/operators/deploy/instance-operator/values.yaml b/operators/deploy/instance-operator/values.yaml index 0d36d464e..0cf72d2be 100644 --- a/operators/deploy/instance-operator/values.yaml +++ b/operators/deploy/instance-operator/values.yaml @@ -19,6 +19,7 @@ configurations: contentDownloaderImage: crownlabs/content-downloader contentUploaderImage: crownlabs/content-uploader mydriveImageAndTag: filebrowser/filebrowser:latest + instmetricsServerEndpoint: crownlabs-instmetrics.crownlabs-production:9090 containerVmSnapshots: kanikoImage: gcr.io/kaniko-project/executor:latest exportImage: "crownlabs/img-exporter" diff --git a/operators/pkg/forge/containers.go b/operators/pkg/forge/containers.go index a0da509eb..7ce41731d 100644 --- a/operators/pkg/forge/containers.go +++ b/operators/pkg/forge/containers.go @@ -53,10 +53,23 @@ const ( CrownLabsUserID = int64(1010) // SubmissionJobMaxRetries -> max number of retries for submission jobs. SubmissionJobMaxRetries = 10 + // AppCPULimitsEnvName -> name of the env variable containing AppContainer CPU limits. + AppCPULimitsEnvName = "APP_CPU_LIMITS" + // AppMEMLimitsEnvName -> name of the env variable containing AppContainer memory limits. + AppMEMLimitsEnvName = "APP_MEM_LIMITS" + // PodNameEnvName -> name of the env variable containing the Pod Name. + PodNameEnvName = "POD_NAME" containersTerminationGracePeriod = 10 ) +var ( + // DefaultDivisor -> "0". + DefaultDivisor = *resource.NewQuantity(0, "") + // MilliDivisor -> "1m". + MilliDivisor = *resource.NewMilliQuantity(1, resource.DecimalSI) +) + // ContainerEnvOpts contains images name and tag for container environment. type ContainerEnvOpts struct { ImagesTag string @@ -65,6 +78,7 @@ type ContainerEnvOpts struct { MyDriveImgAndTag string ContentDownloaderImg string ContentUploaderImg string + InstMetricsEndpoint string } // PVCSpec forges a ReadWriteOnce PersistentVolumeClaimSpec @@ -166,7 +180,7 @@ func ContainersSpec(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir volumeMountPath := MyDriveMountPath(environment) switch environment.EnvironmentType { case clv1alpha2.ClassContainer: - containers = append(containers, WebsockifyContainer(opts, environment), XVncContainer(opts), AppContainer(instance, environment, volumeMountPath)) + containers = append(containers, WebsockifyContainer(opts, environment, instance), XVncContainer(opts), AppContainer(instance, environment, volumeMountPath)) case clv1alpha2.ClassStandalone: containers = append(containers, StandaloneContainer(instance, environment, volumeMountPath)) default: @@ -179,14 +193,22 @@ func ContainersSpec(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir // WebsockifyContainer forges the sidecar container to proxy requests from websocket // to the VNC server. -func WebsockifyContainer(opts *ContainerEnvOpts, environment *clv1alpha2.Environment) corev1.Container { +func WebsockifyContainer(opts *ContainerEnvOpts, environment *clv1alpha2.Environment, instance *clv1alpha2.Instance) corev1.Container { websockifyContainer := GenericContainer(WebsockifyName, fmt.Sprintf("%s:%s", opts.WebsockifyImg, opts.ImagesTag)) SetContainerResources(&websockifyContainer, 0.01, 0.1, 30, 100) + AddEnvVariableFromFieldToContainer(&websockifyContainer, PodNameEnvName, "metadata.name") + AddEnvVariableFromResourcesToContainer(&websockifyContainer, AppCPULimitsEnvName, environment.Name, corev1.ResourceLimitsCPU, MilliDivisor) + AddEnvVariableFromResourcesToContainer(&websockifyContainer, AppMEMLimitsEnvName, environment.Name, corev1.ResourceLimitsMemory, DefaultDivisor) AddTCPPortToContainer(&websockifyContainer, GUIPortName, GUIPortNumber) AddTCPPortToContainer(&websockifyContainer, MetricsPortName, MetricsPortNumber) AddContainerArg(&websockifyContainer, "http-addr", fmt.Sprintf(":%d", GUIPortNumber)) + AddContainerArg(&websockifyContainer, "base-path", IngressGUICleanPath(instance)) AddContainerArg(&websockifyContainer, "metrics-addr", fmt.Sprintf(":%d", MetricsPortNumber)) AddContainerArg(&websockifyContainer, "show-controls", fmt.Sprint(environment.Mode == clv1alpha2.ModeStandard)) + AddContainerArg(&websockifyContainer, "instmetrics-server-endpoint", opts.InstMetricsEndpoint) + AddContainerArg(&websockifyContainer, "pod-name", fmt.Sprintf("$(%s)", PodNameEnvName)) + AddContainerArg(&websockifyContainer, "cpu-limit", fmt.Sprintf("$(%s)", AppCPULimitsEnvName)) + AddContainerArg(&websockifyContainer, "memory-limit", fmt.Sprintf("$(%s)", AppMEMLimitsEnvName)) SetContainerReadinessTCPProbe(&websockifyContainer, GUIPortName) return websockifyContainer } @@ -236,8 +258,8 @@ func StandaloneContainer(instance *clv1alpha2.Instance, environment *clv1alpha2. func AppContainer(instance *clv1alpha2.Instance, environment *clv1alpha2.Environment, volumeMountPath string) corev1.Container { appContainer := GenericContainer(environment.Name, environment.Image) SetContainerResourcesFromEnvironment(&appContainer, environment) - AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU) - AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU) + AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_REQUESTS", appContainer.Name, corev1.ResourceRequestsCPU, DefaultDivisor) + AddEnvVariableFromResourcesToContainer(&appContainer, "CROWNLABS_CPU_LIMITS", appContainer.Name, corev1.ResourceLimitsCPU, DefaultDivisor) if NeedsContainerVolume(instance, environment) { AddContainerVolumeMount(&appContainer, MyDriveName, volumeMountPath) } @@ -316,14 +338,27 @@ func AddEnvVariableToContainer(c *corev1.Container, name, value string) { }) } -// AddEnvVariableFromResourcesToContainer appends an environment variable to the given container's env with a resource-referenced value. -func AddEnvVariableFromResourcesToContainer(c *corev1.Container, name string, resName corev1.ResourceName) { +// AddEnvVariableFromFieldToContainer appends an environment variable to the given container's env with a generic field-referenced value. +func AddEnvVariableFromFieldToContainer(c *corev1.Container, name, value string) { c.Env = append(c.Env, corev1.EnvVar{ Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: value, + }, + }, + }) +} + +// AddEnvVariableFromResourcesToContainer appends an environment variable to the given container's env with a resource-referenced value from the source container. +func AddEnvVariableFromResourcesToContainer(c *corev1.Container, envVarName, srcContainerName string, resName corev1.ResourceName, divisor resource.Quantity) { + c.Env = append(c.Env, corev1.EnvVar{ + Name: envVarName, ValueFrom: &corev1.EnvVarSource{ ResourceFieldRef: &corev1.ResourceFieldSelector{ - ContainerName: c.Name, + ContainerName: srcContainerName, Resource: resName.String(), + Divisor: divisor, }, }, }) diff --git a/operators/pkg/forge/containers_test.go b/operators/pkg/forge/containers_test.go index 2d9614a7e..7e57b876e 100644 --- a/operators/pkg/forge/containers_test.go +++ b/operators/pkg/forge/containers_test.go @@ -276,7 +276,7 @@ var _ = Describe("Containers and Deployment spec forging", func() { EnvironmentType: clv1alpha2.ClassContainer, ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container { return []corev1.Container{ - forge.WebsockifyContainer(&opts, &environment), + forge.WebsockifyContainer(&opts, &environment, i), forge.XVncContainer(&opts), forge.MyDriveContainer(i, &opts, myDriveMountPath), forge.AppContainer(i, e, myDriveMountPath), @@ -289,7 +289,7 @@ var _ = Describe("Containers and Deployment spec forging", func() { EnvironmentType: clv1alpha2.ClassContainer, ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container { return []corev1.Container{ - forge.WebsockifyContainer(&opts, &environment), + forge.WebsockifyContainer(&opts, &environment, i), forge.XVncContainer(&opts), forge.AppContainer(i, e, myDriveMountPath), } @@ -301,7 +301,7 @@ var _ = Describe("Containers and Deployment spec forging", func() { EnvironmentType: clv1alpha2.ClassContainer, ExpectedOutput: func(i *clv1alpha2.Instance, e *clv1alpha2.Environment) []corev1.Container { return []corev1.Container{ - forge.WebsockifyContainer(&opts, &environment), + forge.WebsockifyContainer(&opts, &environment, i), forge.XVncContainer(&opts), forge.AppContainer(i, e, myDriveMountPath), } @@ -360,8 +360,8 @@ var _ = Describe("Containers and Deployment spec forging", func() { expected.Name = envName forge.AddEnvVariableToContainer(&expected, "CROWNLABS_BASE_PATH", forge.IngressGUICleanPath(&instance)) forge.AddEnvVariableToContainer(&expected, "CROWNLABS_LISTEN_PORT", "6080") - forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU) - forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU) + forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", expected.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor) + forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", expected.Name, corev1.ResourceLimitsCPU, forge.DefaultDivisor) Expect(actual.Env).To(ConsistOf(expected.Env)) }) }) @@ -370,7 +370,7 @@ var _ = Describe("Containers and Deployment spec forging", func() { var actual, expected corev1.Container JustBeforeEach(func() { expected = corev1.Container{} - actual = forge.WebsockifyContainer(&opts, &environment) + actual = forge.WebsockifyContainer(&opts, &environment, &instance) }) It("Should set the correct container name and image", func() { @@ -387,13 +387,17 @@ var _ = Describe("Containers and Deployment spec forging", func() { forge.AddTCPPortToContainer(&expected, "metrics", 9090) Expect(actual.Ports).To(Equal(expected.Ports)) }) - It("Should set no environment variables", func() { - Expect(actual.Env).To(BeEmpty()) - }) It("Should set the readiness probe", func() { forge.SetContainerReadinessTCPProbe(&expected, "gui") Expect(actual.ReadinessProbe).To(Equal(expected.ReadinessProbe)) }) + It("Should set the env varibles", func() { + expected.Name = forge.WebsockifyName + forge.AddEnvVariableFromFieldToContainer(&expected, forge.PodNameEnvName, "metadata.name") + forge.AddEnvVariableFromResourcesToContainer(&expected, forge.AppCPULimitsEnvName, environment.Name, corev1.ResourceLimitsCPU, forge.MilliDivisor) + forge.AddEnvVariableFromResourcesToContainer(&expected, forge.AppMEMLimitsEnvName, environment.Name, corev1.ResourceLimitsMemory, forge.DefaultDivisor) + Expect(actual.Env).To(ConsistOf(expected.Env)) + }) When("the environment mode is Standard", func() { BeforeEach(func() { @@ -403,8 +407,13 @@ var _ = Describe("Containers and Deployment spec forging", func() { It("Should set the correct arguments", func() { Expect(actual.Args).To(ConsistOf([]string{ fmt.Sprintf("--http-addr=:%d", forge.GUIPortNumber), + fmt.Sprintf("--base-path=%s", forge.IngressGUICleanPath(&instance)), fmt.Sprintf("--metrics-addr=:%d", forge.MetricsPortNumber), "--show-controls=true", + fmt.Sprintf("--instmetrics-server-endpoint=%s", opts.InstMetricsEndpoint), + fmt.Sprintf("--pod-name=$(%s)", forge.PodNameEnvName), + fmt.Sprintf("--cpu-limit=$(%s)", forge.AppCPULimitsEnvName), + fmt.Sprintf("--memory-limit=$(%s)", forge.AppMEMLimitsEnvName), })) }) }) @@ -417,8 +426,13 @@ var _ = Describe("Containers and Deployment spec forging", func() { It("Should set the correct arguments", func() { Expect(actual.Args).To(ConsistOf([]string{ fmt.Sprintf("--http-addr=:%d", forge.GUIPortNumber), + fmt.Sprintf("--base-path=%s", forge.IngressGUICleanPath(&instance)), fmt.Sprintf("--metrics-addr=:%d", forge.MetricsPortNumber), "--show-controls=false", + fmt.Sprintf("--instmetrics-server-endpoint=%s", opts.InstMetricsEndpoint), + fmt.Sprintf("--pod-name=$(%s)", forge.PodNameEnvName), + fmt.Sprintf("--cpu-limit=$(%s)", forge.AppCPULimitsEnvName), + fmt.Sprintf("--memory-limit=$(%s)", forge.AppMEMLimitsEnvName), })) }) }) @@ -516,8 +530,8 @@ var _ = Describe("Containers and Deployment spec forging", func() { }) It("Should set the env varibles", func() { expected.Name = envName - forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", corev1.ResourceRequestsCPU) - forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", corev1.ResourceLimitsCPU) + forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_REQUESTS", expected.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor) + forge.AddEnvVariableFromResourcesToContainer(&expected, "CROWNLABS_CPU_LIMITS", expected.Name, corev1.ResourceLimitsCPU, forge.DefaultDivisor) Expect(actual.Env).To(ConsistOf(expected.Env)) }) }) @@ -779,7 +793,7 @@ var _ = Describe("Containers and Deployment spec forging", func() { Describe("The forge.AddEnvVariableFromResourcesToContainer", func() { JustBeforeEach(func() { container.Name = envName - forge.AddEnvVariableFromResourcesToContainer(&container, envVarName, corev1.ResourceRequestsCPU) + forge.AddEnvVariableFromResourcesToContainer(&container, envVarName, container.Name, corev1.ResourceRequestsCPU, forge.DefaultDivisor) }) It("Should add a single env entry with the specified parameters", func() { Expect(container.Env).To(ConsistOf(corev1.EnvVar{ diff --git a/operators/pkg/forge/ingresses.go b/operators/pkg/forge/ingresses.go index 911a28a6d..3183abd8f 100644 --- a/operators/pkg/forge/ingresses.go +++ b/operators/pkg/forge/ingresses.go @@ -31,8 +31,8 @@ const ( IngressGUINameSuffix = "gui" // IngressMyDriveNameSuffix -> the suffix added to the name of the ingress targeting the environment "MyDrive". IngressMyDriveNameSuffix = "mydrive" - // IngressStandaloneSuffix -> the suffix added to the path of the ingress targeting the standalone application. - IngressStandaloneSuffix = "app" + // IngressAppSuffix -> the suffix added to the path of the ingress targeting standalone and container environments. + IngressAppSuffix = "app" // IngressDefaultCertificateName -> the name of the secret containing the crownlabs certificate. IngressDefaultCertificateName = "crownlabs-ingress-secret" @@ -81,8 +81,6 @@ func IngressGUIAnnotations(environment *clv1alpha2.Environment, annotations map[ } if environment.EnvironmentType == clv1alpha2.ClassStandalone && environment.RewriteURL { annotations["nginx.ingress.kubernetes.io/rewrite-target"] = StandaloneRewriteEndpoint - } else if environment.EnvironmentType == clv1alpha2.ClassContainer { - annotations["nginx.ingress.kubernetes.io/rewrite-target"] = WebsockifyRewriteEndpoint } annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = "3600" annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = "3600" @@ -142,10 +140,12 @@ func IngressGUIPath(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir switch environment.EnvironmentType { case clv1alpha2.ClassStandalone: if environment.RewriteURL { - return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix+"(/|$)(.*)"), "/") + return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix+"(/|$)(.*)"), "/") } - return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix), "/") - case clv1alpha2.ClassContainer, clv1alpha2.ClassCloudVM, clv1alpha2.ClassVM: + return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/") + case clv1alpha2.ClassContainer: + return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/") + case clv1alpha2.ClassCloudVM, clv1alpha2.ClassVM: return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressVNCGUIPathSuffix), "/") } return "" @@ -153,15 +153,15 @@ func IngressGUIPath(instance *clv1alpha2.Instance, environment *clv1alpha2.Envir // IngressGUICleanPath returns the path of the ingress targeting the environment GUI vnc or Standalone, without the url-rewrite's regex. func IngressGUICleanPath(instance *clv1alpha2.Instance) string { - return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressStandaloneSuffix), "/") + return strings.TrimRight(fmt.Sprintf("%v/%v/%v", IngressInstancePrefix, instance.UID, IngressAppSuffix), "/") } // IngressGuiStatusURL returns the path of the ingress targeting the environment. func IngressGuiStatusURL(host string, environment *clv1alpha2.Environment, instance *clv1alpha2.Instance) string { switch environment.EnvironmentType { - case clv1alpha2.ClassStandalone: - return fmt.Sprintf("https://%v%v/%v/%v/", host, IngressInstancePrefix, instance.UID, IngressStandaloneSuffix) - case clv1alpha2.ClassContainer, clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM: + case clv1alpha2.ClassStandalone, clv1alpha2.ClassContainer: + return fmt.Sprintf("https://%v%v/%v/%v/", host, IngressInstancePrefix, instance.UID, IngressAppSuffix) + case clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM: return fmt.Sprintf("https://%v%v/%v/", host, IngressInstancePrefix, instance.UID) } return "" @@ -171,7 +171,7 @@ func IngressGuiStatusURL(host string, environment *clv1alpha2.Environment, insta func IngressGUIName(environment *clv1alpha2.Environment) string { switch environment.EnvironmentType { case clv1alpha2.ClassStandalone: - return IngressStandaloneSuffix + return IngressAppSuffix case clv1alpha2.ClassContainer, clv1alpha2.ClassVM, clv1alpha2.ClassCloudVM: return IngressGUINameSuffix } diff --git a/operators/pkg/forge/ingresses_test.go b/operators/pkg/forge/ingresses_test.go index 4510bc740..2aab77e44 100644 --- a/operators/pkg/forge/ingresses_test.go +++ b/operators/pkg/forge/ingresses_test.go @@ -155,20 +155,16 @@ var _ = Describe("Ingresses", func() { Expect(forge.IngressGUIAnnotations(&c.Environment, c.Annotations)).To(Equal(c.ExpectedOutput)) }, Entry("When the input annotations map is nil", InstanceGUIAnnotationsCase{ - Annotations: nil, - Environment: clv1alpha2.Environment{EnvironmentType: clv1alpha2.ClassContainer}, - ExpectedOutput: addNginxProxyTimeoutAnnotations(map[string]string{ - "nginx.ingress.kubernetes.io/rewrite-target": "/websockify", - }, "3600"), + Annotations: nil, + Environment: clv1alpha2.Environment{EnvironmentType: clv1alpha2.ClassContainer}, + ExpectedOutput: addNginxProxyTimeoutAnnotations(map[string]string{}, "3600"), }), Entry("When the input labels map already contains the expected values", InstanceGUIAnnotationsCase{ Annotations: addNginxProxyTimeoutAnnotations(map[string]string{ - "nginx.ingress.kubernetes.io/rewrite-target": "/websockify", "user/key": "user/value", }, "3600"), Environment: clv1alpha2.Environment{EnvironmentType: clv1alpha2.ClassContainer}, ExpectedOutput: addNginxProxyTimeoutAnnotations(map[string]string{ - "nginx.ingress.kubernetes.io/rewrite-target": "/websockify", "user/key": "user/value", }, "3600"), }), @@ -178,7 +174,6 @@ var _ = Describe("Ingresses", func() { }, "3600"), Environment: clv1alpha2.Environment{EnvironmentType: clv1alpha2.ClassContainer}, ExpectedOutput: addNginxProxyTimeoutAnnotations(map[string]string{ - "nginx.ingress.kubernetes.io/rewrite-target": "/websockify", "user/key": "user/value", }, "3600"), }), @@ -362,7 +357,7 @@ var _ = Describe("Ingresses", func() { }) Context("The instance has no special configurations", func() { It("Should generate a path based on the instance UID", func() { - Expect(path).To(BeIdenticalTo("/instance/" + instanceUID + "/vnc")) + Expect(path).To(BeIdenticalTo("/instance/" + instanceUID + "/app")) }) }) }) @@ -396,8 +391,8 @@ var _ = Describe("Ingresses", func() { BeforeEach(func() { environment.EnvironmentType = clv1alpha2.ClassContainer }) - It("Should generate a path based on the instance UID", func() { - Expect(statusPath).To(BeIdenticalTo("https://" + host + "/instance/" + instanceUID + "/")) + It("Should generate a path based on the instance UID and /app at the end", func() { + Expect(statusPath).To(BeIdenticalTo("https://" + host + "/instance/" + instanceUID + "/app/")) }) }) }) @@ -410,7 +405,6 @@ var _ = Describe("Ingresses", func() { BeforeEach(func() { environment.EnvironmentType = clv1alpha2.ClassStandalone }) - It("Should generate a path based on the instance UID and /app at the end", func() { Expect(GUIName).To(BeIdenticalTo("app")) }) diff --git a/operators/pkg/instctrl/controller_test.go b/operators/pkg/instctrl/controller_test.go index ff9c606fe..325798441 100644 --- a/operators/pkg/instctrl/controller_test.go +++ b/operators/pkg/instctrl/controller_test.go @@ -178,7 +178,7 @@ var _ = Describe("The instance-controller Reconcile method", func() { runInstance = false }) - StandaloneContainerIt(forge.IngressStandaloneSuffix, true) + StandaloneContainerIt(forge.IngressAppSuffix, true) }) When("the environment is NOT persistent", func() { BeforeEach(func() { @@ -186,7 +186,7 @@ var _ = Describe("The instance-controller Reconcile method", func() { environment.EnvironmentType = clv1alpha2.ClassStandalone runInstance = false }) - StandaloneContainerIt(forge.IngressStandaloneSuffix, false) + StandaloneContainerIt(forge.IngressAppSuffix, false) }) }) diff --git a/operators/pkg/instctrl/exposition_test.go b/operators/pkg/instctrl/exposition_test.go index 461362438..04188329b 100644 --- a/operators/pkg/instctrl/exposition_test.go +++ b/operators/pkg/instctrl/exposition_test.go @@ -161,6 +161,17 @@ var _ = Describe("Generation of the exposition environment", func() { InstanceStatusExpected: fmt.Sprintf("https://%v/instance/%v/", host, instanceUID), } + DescribeBodyParametersIngressGUIContainer := DescribeBodyParameters{ + NamespacedName: &ingressGUIName, Object: &ingress, GroupResource: netv1.Resource("ingresses"), + ExpectedSpecForger: func(inst *clv1alpha2.Instance, _ *clv1alpha2.Environment) interface{} { + return forge.IngressSpec(host, forge.IngressGUIPath(inst, &environment), + forge.IngressDefaultCertificateName, serviceName.Name, forge.GUIPortName) + }, + EmptySpec: netv1.IngressSpec{}, + InstanceStatusGetter: func(inst *clv1alpha2.Instance) string { return inst.Status.URL }, + InstanceStatusExpected: fmt.Sprintf("https://%v/instance/%v/app/", host, instanceUID), + } + DescribeBodyParametersIngressMD := DescribeBodyParameters{ NamespacedName: &ingressMDName, Object: &ingress, GroupResource: netv1.Resource("ingresses"), ExpectedSpecForger: func(inst *clv1alpha2.Instance, _ *clv1alpha2.Environment) interface{} { @@ -312,7 +323,7 @@ var _ = Describe("Generation of the exposition environment", func() { }) Describe("Assessing the service presence", func() { DescribeBodyPresent(DescribeBodyParametersService) }) - Describe("Assessing the GUI ingress presence", func() { DescribeBodyPresent(DescribeBodyParametersIngressGUI) }) + Describe("Assessing the GUI ingress presence", func() { DescribeBodyPresent(DescribeBodyParametersIngressGUIContainer) }) Describe("Assessing the MyDrive ingress presence", func() { DescribeBodyPresent(DescribeBodyParametersIngressMD) }) }) }) diff --git a/provisioning/containers/gui-common/websockify/go.mod b/provisioning/containers/gui-common/websockify/go.mod index 7dec3064c..8cca8d9e5 100644 --- a/provisioning/containers/gui-common/websockify/go.mod +++ b/provisioning/containers/gui-common/websockify/go.mod @@ -3,18 +3,26 @@ module github.com/novnc/websockify-other/websockify go 1.17 require ( + github.com/google/uuid v1.1.2 github.com/gorilla/websocket v1.4.2 github.com/prometheus/client_golang v1.12.1 + google.golang.org/grpc v1.46.0 + google.golang.org/protobuf v1.27.1 + k8s.io/apimachinery v0.24.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect - google.golang.org/protobuf v1.26.0 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/provisioning/containers/gui-common/websockify/go.sum b/provisioning/containers/gui-common/websockify/go.sum index d695d1661..345f5ae6c 100644 --- a/provisioning/containers/gui-common/websockify/go.sum +++ b/provisioning/containers/gui-common/websockify/go.sum @@ -33,11 +33,17 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -51,12 +57,28 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -66,8 +88,17 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -98,6 +129,7 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -106,9 +138,12 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -119,12 +154,17 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -135,22 +175,40 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -177,23 +235,33 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -230,8 +298,10 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -244,6 +314,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -254,11 +325,16 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -273,9 +349,12 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -286,7 +365,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -301,23 +383,34 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -355,12 +448,16 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -411,12 +508,15 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 h1:bFFRpT+e8JJVY7lMMfvezL1ZIwqiwmPl2bsE2yx4HqM= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -429,6 +529,10 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -440,19 +544,30 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -460,6 +575,19 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/provisioning/containers/gui-common/websockify/instmetrics.go b/provisioning/containers/gui-common/websockify/instmetrics.go new file mode 100644 index 000000000..215e58039 --- /dev/null +++ b/provisioning/containers/gui-common/websockify/instmetrics.go @@ -0,0 +1,212 @@ +// Copyright 2020-2022 Politecnico di Torino +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "math" + "net/http" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + "k8s.io/apimachinery/pkg/api/resource" +) + +// Resources utilization derived from ContainerMetrics. +type Resources struct { + Connections []ConnInfo `json:"connections"` + ConnCount uint32 `json:"connectionsCount"` + CPUPerc uint16 `json:"cpu"` + MemPerc uint16 `json:"mem"` + Net int64 `json:"net"` + Timestamp time.Time `json:"timestamp,omitempty"` + ErrorMsg string `json:"error,omitempty"` +} + +// InstanceMetricsHandler is the main handler for instance metrics. +type InstanceMetricsHandler struct { + instanceMetricsClient *InstanceMetricsClient + // Application container resources.limits.cpu. + cpuLimit string + // Application container resources.limits.memory. + memoryLimit string + // PodName where application container is running. + podName string + // Last Resources extracted from CustomMetricsServer + cachedResources *Resources + cachedResourcesMutex sync.RWMutex + // map + connectionsTracking *sync.Map + connectionsCount uint32 +} + +// ServeHTTP serves WS server. +func (h *InstanceMetricsHandler) serveWs(w http.ResponseWriter, r *http.Request) { + var err error + var updatePeriod int64 = 2 + + if h.instanceMetricsClient == nil { + http.Error(w, "Instmetrics server unavailable", http.StatusServiceUnavailable) + return + } + + if up, ok := r.URL.Query()["updatePeriod"]; ok { + updatePeriod, err = strconv.ParseInt(up[0], 10, 16) + if err != nil { + http.Error(w, "Invalid updatePeriod: it must be Integer (update seconds)", http.StatusBadRequest) + return + } + } + updatePeriodD := time.Duration(updatePeriod) * time.Second + + var connUID *string + if uid, ok := r.URL.Query()["connUid"]; ok { + // noVNC page request + uids := uid[0] + connUID = &uids + atomic.AddUint32(&h.connectionsCount, 1) + } else { + // metricsDashboard request + connUID = nil + } + + ip := r.Header.Get(headerXForwardedFor) + log.Printf("Incoming websocket connection on path /usages with update period %d from IP=%s", updatePeriod, ip) + + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println("upgrade error:", err) + return + } + + go func() { + ticker := time.NewTicker(updatePeriodD) + for range ticker.C { + if err := h.sendInstMetrics(ws, updatePeriodD, connUID); err != nil { + log.Printf("[sendCustomMetricsCycle] Error sending data on websocket. Closing WebSocket connection. Error: %v", err) + _ = ws.Close() + ticker.Stop() + return + } + } + }() +} + +func (h *InstanceMetricsHandler) sendInstMetrics(ws *websocket.Conn, updatePeriod time.Duration, connUID *string) error { + ctx, cancel := context.WithTimeout(context.Background(), updatePeriod) + defer cancel() + + if !h.cachedResourcesIsUpdated(updatePeriod) { + if err := h.updateCachedResources(ctx, updatePeriod); err != nil { + return err + } + } + if err := h.sendCachedResources(ws, updatePeriod, connUID); err != nil { + return err + } + + return nil +} + +// cachedResourcesIsUpdated returns true if chachedResources is no older than updatePeriod. +func (h *InstanceMetricsHandler) cachedResourcesIsUpdated(updatePeriod time.Duration) bool { + h.cachedResourcesMutex.RLock() + defer h.cachedResourcesMutex.RUnlock() + if cr := h.cachedResources; cr == nil { + return false + } + return time.Since(h.cachedResources.Timestamp) < updatePeriod +} + +// updateCachedResources retrieves updated ContainerMetrics from instMetrics server. +func (h *InstanceMetricsHandler) updateCachedResources(ctx context.Context, updatePeriod time.Duration) error { + h.cachedResourcesMutex.RLock() + cr := h.cachedResources + h.cachedResourcesMutex.RUnlock() + if cr != nil && time.Since(cr.Timestamp) < updatePeriod { + return nil + } + + if h.podName == "" { + return errors.New("[InstanceMetricsHandler] podName is required") + } + + response, err := h.instanceMetricsClient.ContainerMetrics(ctx, &h.podName) + if err != nil { + return err + } + log.Println("InstMetrics Response: ", response) + + resources := percentagesFromContainerMetrics(response.CpuPerc, response.MemBytes, h.cpuLimit, h.memoryLimit) + resources.Timestamp = time.Now() + + h.cachedResources = resources + return nil +} + +func (h *InstanceMetricsHandler) sendCachedResources(ws *websocket.Conn, updatePeriod time.Duration, connUID *string) error { + h.cachedResourcesMutex.RLock() + cr := Resources{ + CPUPerc: h.cachedResources.CPUPerc, + MemPerc: h.cachedResources.MemPerc, + Timestamp: time.Now(), + } + h.cachedResourcesMutex.RUnlock() + + if connUID != nil { + // send noVNC page latency. + if ci, ok := h.connectionsTracking.Load(*connUID); ok { + cr.Net = ci.(ConnInfo).Latency + } + } else { + // send all existing connections info and connections count. + h.connectionsTracking.Range(func(k, v interface{}) bool { + cr.Connections = append(cr.Connections, v.(ConnInfo)) + return true + }) + cr.ConnCount = h.connectionsCount + } + + message, err := json.Marshal(cr) + if err != nil { + log.Println("Error marhaling cachedResources") + return err + } + + if err = ws.WriteMessage(websocket.TextMessage, message); err != nil { + log.Println("InstMetrics WebSocket error:", err) + return err + } + return nil +} + +func percentagesFromContainerMetrics(cpuBasePerc float32, memoryBytes uint64, cpuLimit, memoryLimit string) *Resources { + memoryLimitQuantity := resource.MustParse(memoryLimit) + memoryLimitBytes := memoryLimitQuantity.Value() + + floatCPULimit, _ := strconv.ParseFloat(cpuLimit, 32) + cpuLimitCores := float32(floatCPULimit / 1000) + + memoryPerc := memoryBytes * 100 / uint64(memoryLimitBytes) + cpuPerc := math.Round(float64(cpuBasePerc / cpuLimitCores)) + + return &Resources{CPUPerc: uint16(cpuPerc), MemPerc: uint16(memoryPerc)} +} diff --git a/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.pb.go b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.pb.go new file mode 100644 index 000000000..4cd69c964 --- /dev/null +++ b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.pb.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.6.1 +// source: instmetrics.proto + +package instmetrics + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ContainerMetricsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CpuPerc float32 `protobuf:"fixed32,1,opt,name=cpu_perc,json=cpuPerc,proto3" json:"cpu_perc,omitempty"` + MemBytes uint64 `protobuf:"varint,2,opt,name=mem_bytes,json=memBytes,proto3" json:"mem_bytes,omitempty"` + DiskBytes uint64 `protobuf:"varint,3,opt,name=disk_bytes,json=diskBytes,proto3" json:"disk_bytes,omitempty"` +} + +func (x *ContainerMetricsResponse) Reset() { + *x = ContainerMetricsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_instmetrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerMetricsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerMetricsResponse) ProtoMessage() {} + +func (x *ContainerMetricsResponse) ProtoReflect() protoreflect.Message { + mi := &file_instmetrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerMetricsResponse.ProtoReflect.Descriptor instead. +func (*ContainerMetricsResponse) Descriptor() ([]byte, []int) { + return file_instmetrics_proto_rawDescGZIP(), []int{0} +} + +func (x *ContainerMetricsResponse) GetCpuPerc() float32 { + if x != nil { + return x.CpuPerc + } + return 0 +} + +func (x *ContainerMetricsResponse) GetMemBytes() uint64 { + if x != nil { + return x.MemBytes + } + return 0 +} + +func (x *ContainerMetricsResponse) GetDiskBytes() uint64 { + if x != nil { + return x.DiskBytes + } + return 0 +} + +type ContainerMetricsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Filter needed to find target "application container" + PodName string `protobuf:"bytes,1,opt,name=pod_name,json=podName,proto3" json:"pod_name,omitempty"` +} + +func (x *ContainerMetricsRequest) Reset() { + *x = ContainerMetricsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_instmetrics_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ContainerMetricsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContainerMetricsRequest) ProtoMessage() {} + +func (x *ContainerMetricsRequest) ProtoReflect() protoreflect.Message { + mi := &file_instmetrics_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContainerMetricsRequest.ProtoReflect.Descriptor instead. +func (*ContainerMetricsRequest) Descriptor() ([]byte, []int) { + return file_instmetrics_proto_rawDescGZIP(), []int{1} +} + +func (x *ContainerMetricsRequest) GetPodName() string { + if x != nil { + return x.PodName + } + return "" +} + +var File_instmetrics_proto protoreflect.FileDescriptor + +var file_instmetrics_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x69, 0x6e, 0x73, 0x74, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x22, 0x71, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x70, 0x75, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x07, + 0x63, 0x70, 0x75, 0x50, 0x65, 0x72, 0x63, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x65, 0x6d, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6d, 0x65, 0x6d, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x62, 0x79, 0x74, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x42, 0x79, + 0x74, 0x65, 0x73, 0x22, 0x34, 0x0a, 0x17, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x32, 0x74, 0x0a, 0x0f, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x61, 0x0a, 0x10, + 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x12, 0x24, 0x2e, 0x69, 0x6e, 0x73, 0x74, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x69, 0x6e, 0x73, 0x74, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x69, 0x6e, 0x73, 0x74, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_instmetrics_proto_rawDescOnce sync.Once + file_instmetrics_proto_rawDescData = file_instmetrics_proto_rawDesc +) + +func file_instmetrics_proto_rawDescGZIP() []byte { + file_instmetrics_proto_rawDescOnce.Do(func() { + file_instmetrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_instmetrics_proto_rawDescData) + }) + return file_instmetrics_proto_rawDescData +} + +var file_instmetrics_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_instmetrics_proto_goTypes = []interface{}{ + (*ContainerMetricsResponse)(nil), // 0: instmetrics.ContainerMetricsResponse + (*ContainerMetricsRequest)(nil), // 1: instmetrics.ContainerMetricsRequest +} +var file_instmetrics_proto_depIdxs = []int32{ + 1, // 0: instmetrics.InstanceMetrics.ContainerMetrics:input_type -> instmetrics.ContainerMetricsRequest + 0, // 1: instmetrics.InstanceMetrics.ContainerMetrics:output_type -> instmetrics.ContainerMetricsResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_instmetrics_proto_init() } +func file_instmetrics_proto_init() { + if File_instmetrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_instmetrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerMetricsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_instmetrics_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerMetricsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_instmetrics_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_instmetrics_proto_goTypes, + DependencyIndexes: file_instmetrics_proto_depIdxs, + MessageInfos: file_instmetrics_proto_msgTypes, + }.Build() + File_instmetrics_proto = out.File + file_instmetrics_proto_rawDesc = nil + file_instmetrics_proto_goTypes = nil + file_instmetrics_proto_depIdxs = nil +} diff --git a/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.proto b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.proto new file mode 100644 index 000000000..d06a4582a --- /dev/null +++ b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package instmetrics; +option go_package = "./instmetrics"; + +service InstanceMetrics { + // ContainerMetrics returns metrics of the "application container" related to the required PodName. + // If the container does not exist, the call returns an error. + rpc ContainerMetrics(ContainerMetricsRequest) returns (ContainerMetricsResponse) {} +} + +message ContainerMetricsResponse { + float cpu_perc = 1; + uint64 mem_bytes = 2; + uint64 disk_bytes = 3; +} + +message ContainerMetricsRequest { + // Filter needed to find target "application container" + string pod_name = 1; +} diff --git a/provisioning/containers/gui-common/websockify/instmetrics/instmetrics_grpc.pb.go b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics_grpc.pb.go new file mode 100644 index 000000000..6d0cb8fed --- /dev/null +++ b/provisioning/containers/gui-common/websockify/instmetrics/instmetrics_grpc.pb.go @@ -0,0 +1,109 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.6.1 +// source: instmetrics.proto + +package instmetrics + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// InstanceMetricsClient is the client API for InstanceMetrics service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type InstanceMetricsClient interface { + // ContainerMetrics returns metrics of the "application container" related to the required PodName. + // If the container does not exist, the call returns an error. + ContainerMetrics(ctx context.Context, in *ContainerMetricsRequest, opts ...grpc.CallOption) (*ContainerMetricsResponse, error) +} + +type instanceMetricsClient struct { + cc grpc.ClientConnInterface +} + +func NewInstanceMetricsClient(cc grpc.ClientConnInterface) InstanceMetricsClient { + return &instanceMetricsClient{cc} +} + +func (c *instanceMetricsClient) ContainerMetrics(ctx context.Context, in *ContainerMetricsRequest, opts ...grpc.CallOption) (*ContainerMetricsResponse, error) { + out := new(ContainerMetricsResponse) + err := c.cc.Invoke(ctx, "/instmetrics.InstanceMetrics/ContainerMetrics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// InstanceMetricsServer is the server API for InstanceMetrics service. +// All implementations must embed UnimplementedInstanceMetricsServer +// for forward compatibility +type InstanceMetricsServer interface { + // ContainerMetrics returns metrics of the "application container" related to the required PodName. + // If the container does not exist, the call returns an error. + ContainerMetrics(context.Context, *ContainerMetricsRequest) (*ContainerMetricsResponse, error) + mustEmbedUnimplementedInstanceMetricsServer() +} + +// UnimplementedInstanceMetricsServer must be embedded to have forward compatible implementations. +type UnimplementedInstanceMetricsServer struct { +} + +func (UnimplementedInstanceMetricsServer) ContainerMetrics(context.Context, *ContainerMetricsRequest) (*ContainerMetricsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ContainerMetrics not implemented") +} +func (UnimplementedInstanceMetricsServer) mustEmbedUnimplementedInstanceMetricsServer() {} + +// UnsafeInstanceMetricsServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to InstanceMetricsServer will +// result in compilation errors. +type UnsafeInstanceMetricsServer interface { + mustEmbedUnimplementedInstanceMetricsServer() +} + +func RegisterInstanceMetricsServer(s grpc.ServiceRegistrar, srv InstanceMetricsServer) { + s.RegisterService(&InstanceMetrics_ServiceDesc, srv) +} + +func _InstanceMetrics_ContainerMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ContainerMetricsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InstanceMetricsServer).ContainerMetrics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/instmetrics.InstanceMetrics/ContainerMetrics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InstanceMetricsServer).ContainerMetrics(ctx, req.(*ContainerMetricsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// InstanceMetrics_ServiceDesc is the grpc.ServiceDesc for InstanceMetrics service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var InstanceMetrics_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "instmetrics.InstanceMetrics", + HandlerType: (*InstanceMetricsServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ContainerMetrics", + Handler: _InstanceMetrics_ContainerMetrics_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "instmetrics.proto", +} diff --git a/provisioning/containers/gui-common/websockify/main.go b/provisioning/containers/gui-common/websockify/main.go index 6e0710461..9ec9d7c6a 100644 --- a/provisioning/containers/gui-common/websockify/main.go +++ b/provisioning/containers/gui-common/websockify/main.go @@ -15,11 +15,14 @@ package main import ( + "context" "embed" "flag" "log" "net/http" + "strconv" "strings" + "sync" "time" ) @@ -30,31 +33,61 @@ var ( func main() { httpAddr := flag.String("http-addr", "0.0.0.0:8080", "websockify server listen address:port") - metricsAddr := flag.String("metrics-addr", "0.0.0.0:9090", "metrics server listen address:port") + metricsAddr := flag.String("metrics-addr", "0.0.0.0:9090", "prometheus metrics server listen address:port") targetAddr := flag.String("target", "127.0.0.1:5900", "vnc service address:port") basePath := flag.String("base-path", "", "base path on which listening for connections") - pingInterval := flag.Int("ping-interval", 5, "ping interval in seconds") + pingInterval := flag.Int("ping-interval", 2, "ping interval in seconds") showBar := flag.Bool("show-controls", false, "show novnc control bar") + instMetricsServerEndpoint := flag.String("instmetrics-server-endpoint", "localhost:9090", "endpoint for connection to instance metrics server") + instMetricsConnectionTimeout := flag.Duration("connection-timeout", 5*time.Second, "timeout of connection to instmetrics server") + podName := flag.String("pod-name", "", "instance podName") + cpuLimit := flag.String("cpu-limit", "", "application container resources.limits.cpu") + memLimit := flag.String("memory-limit", "", "application container resources.limits.memory") - flag.Parse() log.SetFlags(0) + flag.Parse() + + if _, err := strconv.ParseFloat(*cpuLimit, 64); err != nil { + log.Fatal("error parsing cpuLimit to float", err) + } if !strings.HasPrefix(*basePath, "/") { *basePath = "/" + *basePath } *basePath = strings.TrimSuffix(*basePath, "/") + ctx := context.Background() + instMetricsClient, err := GetInstanceMetricsClient(ctx, *instMetricsConnectionTimeout, *instMetricsServerEndpoint) + if err != nil { + log.Println("Failed connecting to instmetrics server", err) + log.Println("Instance metrics will not be available") + } + go runMetricsServer(*metricsAddr, "/metrics") log.Printf("Websockify listening on %s%s", *httpAddr, *basePath) - if err := http.ListenAndServe(*httpAddr, &NoVncHandler{ - BasePath: *basePath, - NoVncFS: http.FileServer(http.FS(novncFS)), - ShowNoVncBar: *showBar, - TargetSocket: *targetAddr, - PingInterval: time.Second * time.Duration(*pingInterval), - }); err != nil { + // SyncMap shared between NoVncHandler and InstanceMetricsHandler + var connectionsTracking sync.Map + + http.Handle("/", &NoVncHandler{ + BasePath: *basePath, + NoVncFS: http.FileServer(http.FS(novncFS)), + ShowNoVncBar: *showBar, + TargetSocket: *targetAddr, + PingInterval: time.Second * time.Duration(*pingInterval), + connectionsTracking: &connectionsTracking, + MetricsHandler: &InstanceMetricsHandler{ + cpuLimit: *cpuLimit, + memoryLimit: *memLimit, + podName: *podName, + connectionsTracking: &connectionsTracking, + cachedResourcesMutex: sync.RWMutex{}, + instanceMetricsClient: instMetricsClient, + }, + }) + + if err := http.ListenAndServe(*httpAddr, nil); err != nil { log.Fatal("failed starting websockify server", err) } } diff --git a/provisioning/containers/gui-common/websockify/metrics.go b/provisioning/containers/gui-common/websockify/metrics.go index 9e67d2820..426f907c7 100644 --- a/provisioning/containers/gui-common/websockify/metrics.go +++ b/provisioning/containers/gui-common/websockify/metrics.go @@ -34,8 +34,9 @@ var ( ) ) -func runMetricsServer(addr, endpoint string) { - http.Handle(endpoint, promhttp.Handler()) +func runMetricsServer(addr, metricsEndpoint string) { + http.Handle(metricsEndpoint, promhttp.Handler()) + if err := http.ListenAndServe(addr, nil); err != nil { log.Fatal("failed starting metrics server", err) } diff --git a/provisioning/containers/gui-common/websockify/novnc-overrides/app/images/stats-img.svg b/provisioning/containers/gui-common/websockify/novnc-overrides/app/images/stats-img.svg new file mode 100644 index 000000000..d14423c6b --- /dev/null +++ b/provisioning/containers/gui-common/websockify/novnc-overrides/app/images/stats-img.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/provisioning/containers/gui-common/websockify/novnc-overrides/app/styles/metrics.css b/provisioning/containers/gui-common/websockify/novnc-overrides/app/styles/metrics.css new file mode 100644 index 000000000..1fb854875 --- /dev/null +++ b/provisioning/containers/gui-common/websockify/novnc-overrides/app/styles/metrics.css @@ -0,0 +1,153 @@ +.usage-btn, +.usage-btn * { + user-select: none; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; +} + +.usage-btn { + background-color: #4caf50; + padding: 15px 0; + box-sizing: border-box; + text-align: center; + font-size: 16px; + position: fixed; + height: 60px; + width: 60px; + border-radius: 50%; + top: 50vh; + left: 0; + opacity: 30%; + z-index: 99997; + transform: scale(0.5); + transition: all 0.4s; +} + +.usage-btn .usage-img { + position: absolute; + top: 0; + left: 2px; + padding: 0; + z-index: 99998; + height: 55px; + width: 55px; +} + +.usage-btn .usage-img-transparent { + position: absolute; + top: 0; + left: 0; + padding: 0; + height: 60px; + width: 60px; + z-index: 99999; + opacity: 0; +} + +.usage-btn .btn { + font-size: 14px; + position: absolute; + box-sizing: border-box; + transform: scale(0.8); + border-radius: 50%; + pointer-events: none; + transition: all 0.4s; + transition-delay: 0.1s; + opacity: 0; + height: 40px; + width: 40px; + z-index: 99995; + padding: 12px 0; + left: 10px; +} + +.usage-btn .btn .usage-stat-txt { + word-break: break-word; + white-space: pre-wrap; + -moz-white-space: pre-wrap; +} + +.usage-cpu { + top: -20px; +} + +.usage-mem { + top: -70px; +} + +.usage-net { + top: -120px; +} + +.usage-btn .btn::after { + content: attr(data-cont); + position: absolute; + background: inherit; + opacity: 0.5; + left: 37px; + width: 40px; + padding: 2px 4px; + display: block; + top: 30%; +} + +.usage-btn:hover { + opacity: 1; + cursor: grab; + transform: scale(1); +} + +.usage-btn.active { + cursor: grabbing; + transition: all 0s; +} + +.usage-btn:hover .btn { + transform: scale(1); + opacity: 1; +} + +.usage-btn:hover .usage-cpu { + top: -45px; +} + +.usage-btn:hover .usage-mem { + top: -95px; +} + +.usage-btn:hover .usage-net { + top: -145px; +} + +.btn-grn { + background: linear-gradient(0deg, rgb(22, 161, 0) 0%, rgb(51, 255, 0) 100%); +} + +.btn-yel { + background: linear-gradient(0deg, rgba(217, 175, 0, 1) 0%, rgba(255, 206, 0, 1) 100%); +} + +.btn-red { + background: linear-gradient(0deg, rgba(217, 0, 0, 1) 0%, rgba(255, 0, 0, 1) 100%); +} + +.dim-screen { + position: fixed; + padding: 0; + margin: 0; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(255, 255, 255, 0.1); + pointer-events: none; + z-index: 999; + opacity: 0; + transition: all 0.2s; +} + +.dim-screen.active { + pointer-events: all; + opacity: 1; +} diff --git a/provisioning/containers/gui-common/websockify/novnc-overrides/app/ui.js b/provisioning/containers/gui-common/websockify/novnc-overrides/app/ui.js index d7587776c..c9fd23ed9 100644 --- a/provisioning/containers/gui-common/websockify/novnc-overrides/app/ui.js +++ b/provisioning/containers/gui-common/websockify/novnc-overrides/app/ui.js @@ -12,7 +12,10 @@ UI.initSettings = () => { UI.forceSetting('path', window.websockifyTargetUrl); } else { let { pathname } = window.location; - pathname = pathname.split('/').filter(e => e).join('/'); + pathname = pathname + .split('/') + .filter((e) => e) + .join('/'); if (pathname) UI.forceSetting('path', pathname + '/vnc'); } UI.forceSetting('show_dot', true); @@ -27,8 +30,217 @@ UI.openConnectPanel = () => { if (connAttempts < 5) { setTimeout(UI.connect, 100); } else { - document.querySelector('#noVNC_connect_dlg .noVNC_logo').innerText = 'Connection failed' + document.querySelector('#noVNC_connect_dlg .noVNC_logo').innerText = 'Connection failed'; connAttempts = 0; oocp(); } +}; + +/** + * STATS VIEW + */ +const link = document.createElement('link'); +link.rel = 'stylesheet'; +link.href = 'app/styles/metrics.css'; +document.head.appendChild(link); + +class Resources { + constructor(cpu, mem, net, ip, timestamp, error) { + if (cpu) this.cpu = Number(cpu); + if (mem) this.mem = Number(mem); + if (net) this.net = Number(net); //ms + if (ip) this.ip = ip; + if (timestamp) this.timestamp = timestamp; + if (error) this.error = error; + } + + static from(json) { + if (json.cpu) json.cpu = Number(json.cpu) > 100 ? 100 : Number(json.cpu); + if (json.net == undefined) json.net = 0; + const resources = Object.assign(new Resources(), json); + return resources; + } +} + +class ResourcesHistory { + constructor(historyLen = 5) { + this.lastResources = []; + this.historyLen = historyLen; + } + + /** + * Save last resources + */ + addResources(resources) { + this.lastResources.push(resources); + if (this.lastResources.length > this.historyLen) this.lastResources.shift(); + } + + getAvg(stat) { + return this.lastResources.map((e) => e[stat]).reduce((a, b) => a + b, 0) / this.lastResources.length; + } + + getCurrent(stat) { + return this.lastResources.at(-1)[stat]; + } } + +/** + * Register actions needed when DOM content is loaded: + * - Button drag and drop logic + */ +document.addEventListener('DOMContentLoaded', () => { + /** + * Button element + */ + document.body.insertAdjacentHTML( + 'beforeend', + ` +
+ + Stats +
+
+ ? % +
+
+ ? % +
+
+ ? ms +
+
+
` + ); + + const button = document.getElementById('usage-btn'); + const buttonNet = document.getElementById('usage-net'); + const buttonText = document.getElementById('usage-btn-txt'); + const overlay = document.getElementById('page-element'); + + let resourcesHistory = new ResourcesHistory(); + + const thresholds = { + cpu: { + warn: 80, + crit: 90, + el: document.getElementById('usage-cpu'), + txt: document.getElementById('usage-cpu-txt'), + format: (v) => `${v}%`, + }, + mem: { + warn: 85, + crit: 90, + el: document.getElementById('usage-mem'), + txt: document.getElementById('usage-mem-txt'), + format: (v) => `${v}%`, + }, + net: { + warn: 200, + crit: 800, + el: document.getElementById('usage-net'), + txt: document.getElementById('usage-net-txt'), + format: (v) => `${v}ms`, + }, + }; + + /** + * DOM manupulation + */ + let worstAvgColor = 'grn'; + const updateViewUsages = () => { + button.classList.remove(`btn-${worstAvgColor}`); + let worstColor = 'grn'; + worstAvgColor = 'grn'; + Object.keys(thresholds).forEach((stat) => { + const { crit, warn, el, txt, format } = thresholds[stat]; + const value = resourcesHistory.getCurrent(stat); + const avgValue = resourcesHistory.getAvg(stat); + txt.innerHTML = format(value); + let color = 'grn'; + if (value > crit) { + color = 'red'; + worstColor = 'red'; + } else if (value > warn) { + color = 'yel'; + if (worstColor === 'grn') worstColor = 'yel'; + } + + if (avgValue > crit) { + worstAvgColor = 'red'; + } else if (avgValue > warn && worstAvgColor === 'grn') { + worstAvgColor = 'yel'; + } + + el.className = `usage-${stat} btn btn-${color}`; + }); + + if ( + !((worstAvgColor === 'red' && worstColor === 'red') || + (worstAvgColor === 'yel' && (worstColor === 'yel' || worstColor === 'red')) || + worstAvgColor === 'grn') + ) { + worstAvgColor = worstColor; + } + button.classList.add(`btn-${worstAvgColor}`); + }; + + let isMoving = false; + let initialX, initialY; + + const proto = window.location.protocol.replace('http','ws') + const updatePeriod = 2; //seconds + + // WS metrics connection + let metricsPath = `${proto}//${window.location.host}/${window.metricsTargetUrl}&updatePeriod=${updatePeriod}`; + console.log('connceting usages ws ' + metricsPath); + const conn = new WebSocket(metricsPath); + conn.onclose = (evt) => { + button.remove(); + Log.Error('Usages WS connection closed: ', evt); + }; + conn.onerror = (err) => { + button.remove(); + Log.Error('Usages WS connection error: ' + err); + }; + conn.onmessage = (evt) => { + try { + const resourcesJson = JSON.parse(evt.data.toString()); + resourcesHistory.addResources(Resources.from(resourcesJson)); + updateViewUsages(); + } catch (error) { + Log.Error('Error on JSON parsing from WS data: ', error); + buttonText.innerHTML = 'WS Error'; + } + }; + + button.addEventListener('mousedown', (e) => { + isMoving = true; + initialX = e.clientX - button.offsetLeft; + initialY = e.clientY - button.offsetTop; + overlay.classList.add('active'); + button.classList.add('active'); + }); + + document.addEventListener('mousemove', (e) => { + if (isMoving) { + let { clientX, clientY } = e; + + if (clientX - initialX < 0) clientX = initialX; + if (clientX + button.offsetWidth - initialX > window.innerWidth) + clientX = window.innerWidth - button.offsetWidth + initialX; + if (clientY - initialY < -buttonNet.offsetTop) clientY = initialY - buttonNet.offsetTop; + if (clientY > window.innerHeight - button.offsetHeight + initialY) + clientY = window.innerHeight - button.offsetHeight + initialY; + + button.style.left = clientX - initialX + 'px'; + button.style.top = clientY - initialY + 'px'; + } + }); + + document.addEventListener('mouseup', () => { + isMoving = false; + overlay.classList.remove('active'); + button.classList.remove('active'); + }); +}); diff --git a/provisioning/containers/gui-common/websockify/novnc.go b/provisioning/containers/gui-common/websockify/novnc.go index 728c232f7..1b937d1b3 100644 --- a/provisioning/containers/gui-common/websockify/novnc.go +++ b/provisioning/containers/gui-common/websockify/novnc.go @@ -20,28 +20,45 @@ import ( "log" "net/http" "strings" + "sync" "time" + + "github.com/google/uuid" ) const ( headerXForwardedFor = "X-Forwarded-For" searchStringHead = "" hideNovncBarStyle = "\n" - websockifyPath = "/websockify" + websockifyPath = "/websockify" + usagesPath = "/usages" ) var ( searchStringHeadBytes = []byte(searchStringHead) ) +// ConnInfo stores useful information related to the WS connection. +type ConnInfo struct { + // latency in ms + Latency int64 `json:"latency"` + IP string `json:"ip,omitempty"` + UID string `json:"connUid,omitempty"` + ConnTime time.Time `json:"connTime,omitempty"` + DisconnTime time.Time `json:"disconnTime,omitempty"` + Active bool `json:"active"` +} + // NoVncHandler is the main handler for the noVNC server. type NoVncHandler struct { - BasePath string - PingInterval time.Duration - NoVncFS http.Handler - ShowNoVncBar bool - TargetSocket string - connectionsCount uint32 + // + connectionsTracking *sync.Map + BasePath string + PingInterval time.Duration + NoVncFS http.Handler + ShowNoVncBar bool + TargetSocket string + MetricsHandler *InstanceMetricsHandler } // ServeHTTP handles the HTTP request. @@ -56,12 +73,24 @@ func (h *NoVncHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // serve index on root switch cleanedUpPath { - case "": // enforce slash terminated path + case "": // enforce slash terminated path. http.Redirect(w, r, r.URL.Path+"/", http.StatusFound) - case "/": // serve vnc index on root + case "/": // serve vnc index on root. h.serveNoVncHome(w, r) case websockifyPath: - h.serveWs(w, r) + // connUid is required for websockify connections. + if connUID, ok := r.URL.Query()["connUid"]; ok { + if _, ok := h.connectionsTracking.Load(connUID[0]); !ok { + http.Error(w, "received connUid queryParam was never assigned", http.StatusBadRequest) + } else { + log.Println("Connection UID: ", connUID[0]) + h.serveWs(w, r) + } + } else { + http.Error(w, "connUid queryParam required", http.StatusBadRequest) + } + case usagesPath: + h.MetricsHandler.serveWs(w, r) default: r.URL.Path = "novnc" + strings.Replace(r.URL.Path, h.BasePath, "/", 1) h.NoVncFS.ServeHTTP(w, r) @@ -75,7 +104,13 @@ func (h *NoVncHandler) serveNoVncHome(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - log.Printf("Requested novnc gui from IP=%s", r.Header.Get(headerXForwardedFor)) + + ip := r.Header.Get(headerXForwardedFor) + if ip == "" { + ip = "unknown" + } + + log.Printf("Requested novnc gui from IP=%s", ip) injectStr := "\n" @@ -84,7 +119,13 @@ func (h *NoVncHandler) serveNoVncHome(w http.ResponseWriter, r *http.Request) { } vncEndpoint := strings.TrimPrefix(h.BasePath+websockifyPath, "/") - injectStr += fmt.Sprintf("\n", vncEndpoint) + usagesEndpoint := strings.TrimPrefix(h.BasePath+usagesPath, "/") + uid := strings.ReplaceAll(uuid.New().String(), "-", "") + injectStr += fmt.Sprintf(``, vncEndpoint, uid, usagesEndpoint, uid) + + connectionInfo := &ConnInfo{IP: ip, UID: uid, Latency: 0, ConnTime: time.Now(), DisconnTime: time.Now(), Active: false} + h.connectionsTracking.Store(connectionInfo.UID, *connectionInfo) data = bytes.ReplaceAll(data, searchStringHeadBytes, []byte(injectStr)) diff --git a/provisioning/containers/gui-common/websockify/remote_instmetrics.go b/provisioning/containers/gui-common/websockify/remote_instmetrics.go new file mode 100644 index 000000000..d23d0322f --- /dev/null +++ b/provisioning/containers/gui-common/websockify/remote_instmetrics.go @@ -0,0 +1,71 @@ +// Copyright 2020-2022 Politecnico di Torino +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "time" + + pb "github.com/novnc/websockify-other/websockify/instmetrics" + "google.golang.org/grpc" +) + +// InstanceMetricsClient is a gRPC implementation of pb.InstanceMetricsClient. +type InstanceMetricsClient struct { + client pb.InstanceMetricsClient +} + +// GetInstanceMetricsClient creates and returns a new InstanceMetricsClient. +func GetInstanceMetricsClient(ctx context.Context, connectionTimeout time.Duration, instanceMetricsEndpoint string) (*InstanceMetricsClient, error) { + ctx, cancel := context.WithTimeout(ctx, connectionTimeout) + defer cancel() + + log.Println("Initializing connection to remote runtime service") + + res, err := newInstanceMetricsClient(ctx, instanceMetricsEndpoint) + if err != nil { + log.Println(err, "Error creating new InstanceMetricsClient") + return nil, err + } + return res, nil +} + +// newRemoteRuntimeServiceClient creates a new InstanceMetricsClient. +func newInstanceMetricsClient(ctx context.Context, endpoint string) (*InstanceMetricsClient, error) { + log.Println("Connecting to instance metrics server", "endpoint", endpoint) + + connection, err := grpc.DialContext(ctx, endpoint, grpc.WithInsecure(), grpc.WithBlock()) + if err != nil { + log.Printf("An error occurred while creating instance metrics server client: %v", err) + return nil, err + } + log.Println("Successfully connected to runtime service") + + instMetricsClient := &InstanceMetricsClient{client: pb.NewInstanceMetricsClient(connection)} + return instMetricsClient, nil +} + +// ContainerMetrics returns the containerMetrics given a podName. +func (c *InstanceMetricsClient) ContainerMetrics(ctx context.Context, podName *string) (*pb.ContainerMetricsResponse, error) { + resp, err := c.client.ContainerMetrics(ctx, &pb.ContainerMetricsRequest{ + PodName: *podName, + }) + if err != nil { + log.Printf("ContainerMetrics with podName filter '%s' failed. Error: %v", *podName, err) + return nil, err + } + return resp, nil +} diff --git a/provisioning/containers/gui-common/websockify/websockify.go b/provisioning/containers/gui-common/websockify/websockify.go index 810b01215..2fe144931 100644 --- a/provisioning/containers/gui-common/websockify/websockify.go +++ b/provisioning/containers/gui-common/websockify/websockify.go @@ -18,11 +18,9 @@ package main // Original version at https://github.com/novnc/websockify-other/tree/master/golang import ( - "fmt" "log" "net" "net/http" - "sync/atomic" "time" "github.com/gorilla/websocket" @@ -86,23 +84,34 @@ func (h *NoVncHandler) serveWs(w http.ResponseWriter, r *http.Request) { } ip := r.Header.Get(headerXForwardedFor) - connID := atomic.AddUint32(&h.connectionsCount, 1) - - metric := makeLatencyObserver(ip, fmt.Sprint(connID)) + if ip == "" { + ip = "unknown" + } + connUID := r.URL.Query().Get("connUid") - log.Printf("Incoming websocket connection from IP=%s", ip) - go forwardtcp(ws, vnc) - go forwardweb(ws, vnc) - go h.pingCycle(ws, metric) + if ci, ok := h.connectionsTracking.Load(connUID); ok { + connectionInfo := ci.(ConnInfo) + metric := makeLatencyObserver(ip, connUID) + log.Printf("Incoming websocket connection on path /websockify from IP=%s", ip) + go forwardtcp(ws, vnc) + go forwardweb(ws, vnc) + go h.pingCycle(ws, metric, &connectionInfo) + } else { + log.Println("received connUid queryParam was never assigned") + _ = ws.Close() + return + } } -func (h *NoVncHandler) pingCycle(wsconn *websocket.Conn, metric prometheus.Observer) { +func (h *NoVncHandler) pingCycle(wsconn *websocket.Conn, metric prometheus.Observer, connInfo *ConnInfo) { // set pong handler lastPing := time.Now() wsconn.SetPongHandler(func(data string) error { - latency := time.Since(lastPing) - metric.Observe(float64(latency.Milliseconds())) - log.Printf("ping latency: %s", latency) + connInfo.Latency = time.Since(lastPing).Milliseconds() + connInfo.Active = true + metric.Observe(float64(connInfo.Latency)) + h.connectionsTracking.Store(connInfo.UID, *connInfo) + log.Printf("ping latency: %dms, connUid: %s", connInfo.Latency, connInfo.UID) return nil }) @@ -111,6 +120,12 @@ func (h *NoVncHandler) pingCycle(wsconn *websocket.Conn, metric prometheus.Obser err := wsconn.WriteControl(websocket.PingMessage, nil, time.Now().Add(h.PingInterval)) if err != nil { log.Println("ping error:", err) + log.Printf("stopping connection tracking for ", connInfo.IP, connInfo.UID) + + connInfo.Latency = 0 + connInfo.Active = false + connInfo.DisconnTime = time.Now() + h.connectionsTracking.Store(connInfo.UID, *connInfo) ticker.Stop() } }