Merge pull request loft-sh#810 from matskiv/misc-improvements

Misc improvements - metrics flag, job.securityContext helm value, docs updates

charts/eks/templates/pre-install-hook-job.yaml

@@ -56,14 +56,7 @@ spec:
             - --service-cidr={{ .Values.serviceCIDR }}
             {{- end }}
           securityContext:
-            capabilities:
-              drop:
-                - ALL
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 1000
-            runAsGroup: 1000
+{{ toYaml .Values.job.securityContext | indent 12 }}
           volumeMounts:
             - name: cert-storage
               mountPath: /certs

charts/eks/values.yaml

@@ -302,6 +302,16 @@ job:
   resources: {}
   podAnnotations: {}
   podLabels: {}
+  # Security context configuration of the container
+  securityContext:
+    capabilities:
+      drop:
+        - ALL
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 10001
+    runAsGroup: 10001
 
 # Configure the ingress resource that allows you to access the vcluster
 ingress:

charts/k8s/templates/pre-install-hook-job.yaml

@@ -56,14 +56,7 @@ spec:
             - --service-cidr={{ .Values.serviceCIDR }}
             {{- end }}
           securityContext:
-            capabilities:
-              drop:
-                - ALL
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 1000
-            runAsGroup: 1000
+{{ toYaml .Values.job.securityContext | indent 12 }}
           volumeMounts:
             - name: cert-storage
               mountPath: /certs

charts/k8s/values.yaml

@@ -306,6 +306,16 @@ job:
   resources: {}
   podAnnotations: {}
   podLabels: {}
+  # Security context configuration of the container
+  securityContext:
+    capabilities:
+      drop:
+        - ALL
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 10001
+    runAsGroup: 10001
 
 # Configure the ingress resource that allows you to access the vcluster
 ingress:

cmd/vcluster/cmd/start.go

@@ -140,6 +140,9 @@ func NewStartCommand() *cobra.Command {
 	cmd.Flags().StringSliceVar(&options.MapVirtualServices, "map-virtual-service", []string{}, "Maps a given service inside the virtual cluster to a service inside the host cluster. E.g. default/test=physical-service")
 	cmd.Flags().StringSliceVar(&options.MapHostServices, "map-host-service", []string{}, "Maps a given service inside the host cluster to a service inside the virtual cluster. E.g. other-namespace/my-service=my-vcluster-namespace/my-service")
 
+	cmd.Flags().StringVar(&options.HostMetricsBindAddress, "host-metrics-bind-address", "0", "If set, metrics for the controller manager for the resources managed in the host cluster will be exposed at this address")
+	cmd.Flags().StringVar(&options.VirtualMetricsBindAddress, "virtual-metrics-bind-address", "0", "If set, metrics for the controller manager for the resources managed in the virtual cluster will be exposed at this address")
+
 	// Deprecated Flags
 	cmd.Flags().BoolVar(&options.DeprecatedSyncNodeChanges, "sync-node-changes", false, "If enabled and --fake-nodes is false, the virtual cluster will proxy node updates from the virtual cluster to the host cluster. This is not recommended and should only be used if you know what you are doing.")
 	cmd.Flags().BoolVar(&options.DeprecatedUseFakeKubelets, "fake-kubelets", true, "DEPRECATED: use --disable-fake-kubelets instead")
@@ -309,7 +312,7 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error {
 	klog.Info("Using physical cluster at " + inClusterConfig.Host)
 	localManager, err := ctrl.NewManager(inClusterConfig, ctrl.Options{
 		Scheme:             scheme,
-		MetricsBindAddress: "0",
+		MetricsBindAddress: options.HostMetricsBindAddress,
 		LeaderElection:     false,
 		Namespace:          options.TargetNamespace,
 		NewClient:          pluginhookclient.NewPhysicalPluginClientFactory(blockingcacheclient.NewCacheClient),
@@ -319,7 +322,7 @@ func ExecuteStart(options *context2.VirtualClusterOptions) error {
 	}
 	virtualClusterManager, err := ctrl.NewManager(virtualClusterConfig, ctrl.Options{
 		Scheme:             scheme,
-		MetricsBindAddress: "0",
+		MetricsBindAddress: options.VirtualMetricsBindAddress,
 		LeaderElection:     false,
 		NewClient:          pluginhookclient.NewVirtualPluginClientFactory(blockingcacheclient.NewCacheClient),
 	})

cmd/vcluster/context/context.go

@@ -77,6 +77,9 @@ type VirtualClusterOptions struct {
 	VirtualContainerLogsPath string
 	VirtualKubeletPodPath    string
 
+	HostMetricsBindAddress    string `json:"hostMetricsBindAddress,omitempty"`
+	VirtualMetricsBindAddress string `json:"virtualMetricsBindAddress,omitempty"`
+
 	// DEPRECATED FLAGS
 	DeprecatedSyncNodeChanges          bool `json:"syncNodeChanges"`
 	DeprecatedDisableSyncResources     string

docs/pages/architecture/storage.mdx

@@ -80,12 +80,6 @@ Kubernetes VolumeSnapshot resource represents a snapshot of a volume on a storag
 
 By default, VolumeSnapshot syncing is disabled, and creating a VolumeSnapshot custom resource in the vcluster will have no effect. Following chapters describe how to enable this feature in the vcluster.
 
-:::caution Tech-preview feature
-This vcluster feature is still in the early state of maturity, so it is highly recommended to perform testing with your specific storage configurations and use cases.
-
-We offer this feature to the community as a tech-preview to gather feedback and information about compatibility with various CSI implementations and Kubernetes distributions. Please share your experience with us via *#vcluster* channel in [our Slack instance](https://slack.loft.sh/) or via [GitHub issues](https://github.com/loft-sh/vcluster/issues).
-:::
-
 #### Host prerequisites
 Vcluster relies fully on the volume snapshot capabilities of the host cluster, which has to fullfil certain criteria.
 

docs/pages/config-reference.mdx

@@ -4,24 +4,36 @@ sidebar_label: Config Reference
 ---
 
 ### Syncer Flags
+Before using any particular flag mentioned below, we recommend making yourself familiar with the documentation pages that are related to the topic addressed by the flag and default to using the flags and helm variables as described in the documentation. 
 
 ```
       --bind-address string                       The address to bind the server to (default "0.0.0.0")
       --client-ca-cert string                     The path to the client ca certificate (default "/data/server/tls/client-ca.crt")
       --cluster-domain string                     The cluster domain ending that should be used for the virtual cluster (default "cluster.local")
+      --default-image-registry string             This address will be prepended to all deployed system images by vcluster
       --disable-fake-kubelets                     If disabled, the virtual cluster will not create fake kubelet endpoints to support metrics-servers
+      --disable-plugins                           If enabled, vcluster will not load any plugins
+      --enable-scheduler                          If enabled, will expect a scheduler running in the virtual cluster
       --enforce-node-selector                     If enabled and --node-selector is set then the virtual cluster will ensure that no pods are scheduled outside of the node selector (default true)
-  -h, --help                                      help for start
+      --enforce-pod-security-standard string      This can be set to privileged, baseline, restricted and vcluster would make sure during translation that these policies are enforced.
+      --enforce-toleration strings                If set will apply the provided tolerations to all pods in the vcluster
+      -h, --help                                      help for start
+      --host-metrics-bind-address string          If set, metrics for the controller manager for the resources managed in the host cluster will be exposed at this address
       --kube-config string                        The path to the virtual cluster admin kube config (default "/data/server/cred/admin.kubeconfig")
+      --kube-config-context-name string           If set, will override the context name of the generated virtual cluster kube config with this name
       --leader-elect                              If enabled, syncer will use leader election
       --lease-duration int                        Lease duration of the leader election in seconds (default 60)
+      --map-host-service strings                  Maps a given service inside the host cluster to a service inside the virtual cluster. E.g. other-namespace/my-service=my-vcluster-namespace/my-service
+      --map-virtual-service strings               Maps a given service inside the virtual cluster to a service inside the host cluster. E.g. default/test=physical-service
       --name string                               The name of the virtual cluster
-      --node-selector string                      If set, nodes with the given node selector will be synced to the virtual cluster. This will implicitly set --fake-nodes=false
+      --node-selector string                      If nodes sync is enabled, nodes with the given node selector will be synced to the virtual cluster. If fake nodes are used, and --enforce-node-selector flag is set, then vcluster will ensure that no pods are scheduled outside of the node selector.
       --out-kube-config-secret string             If specified, the virtual cluster will write the generated kube config to the given secret
       --out-kube-config-secret-namespace string   If specified, the virtual cluster will write the generated kube config in the given namespace
       --out-kube-config-server string             If specified, the virtual cluster will use this server for the generated kube config (e.g. https://my-vcluster.domain.com)
       --override-hosts                            If enabled, vcluster will override a containers /etc/hosts file if there is a subdomain specified for the pod (spec.subdomain). (default true)
       --override-hosts-container-image string     The image for the init container that is used for creating the override hosts file. (default "library/alpine:3.13.1")
+      --plugin-listen-address string              The plugin address to listen to. If this is changed, you'll need to configure your plugins to connect to the updated port (default "localhost:10099")
+      --plugins strings                           The plugins to wait for during startup
       --port int                                  The port to bind to (default 8443)
       --renew-deadline int                        Renew deadline of the leader election in seconds (default 40)
       --request-header-ca-cert string             The path to the request header ca certificate (default "/data/server/tls/request-header-ca.crt")
@@ -31,12 +43,14 @@ sidebar_label: Config Reference
       --service-account string                    If set, will set this host service account on the synced pods
       --service-name string                       The service name where the vcluster proxy will be available
       --set-owner                                 If true, will set the same owner the currently running syncer pod has on the synced resources (default true)
-      --sync string                               A list of sync controllers to enable. 'foo' enables the sync controller named 'foo', '-foo' disables the sync controller named 'foo'
+      --sync strings                              A list of sync controllers to enable. 'foo' enables the sync controller named 'foo', '-foo' disables the sync controller named 'foo'
       --sync-all-nodes                            If enabled and --fake-nodes is false, the virtual cluster will sync all nodes instead of only the needed ones
+      --sync-labels strings                       The specified labels will be synced to physical resources, in addition to their vcluster translated versions.
       --sync-node-changes                         If enabled and --fake-nodes is false, the virtual cluster will proxy node updates from the virtual cluster to the host cluster. This is not recommended and should only be used if you know what you are doing.
       --target-namespace string                   The namespace to run the virtual cluster in (defaults to current namespace)
       --tls-san strings                           Add additional hostname or IP as a Subject Alternative Name in the TLS cert
       --translate-image strings                   Translates image names from the virtual pod to the physical pod (e.g. coredns/coredns=mirror.io/coredns/coredns)
+      --virtual-metrics-bind-address string       If set, metrics for the controller manager for the resources managed in the virtual cluster will be exposed at this address
 ```
 
 All of these syncer flags can be set and configured through the values override file when creating the vcluster. In particular these syncer flags go into the `extraArgs` section of the values

docs/pages/operator/cluster-api-provider.mdx

@@ -6,33 +6,10 @@ sidebar_label: Cluster API Provider
 [Cluster API (CAPI)](https://cluster-api.sigs.k8s.io/) is a Kubernetes sub-project focused on providing declarative APIs and tooling to simplify provisioning, upgrading, and operating multiple Kubernetes clusters. Kubernetes service providers and distribution maintainers implement CAPI providers which allow users to create Kubernetes clusters of their choice with the CAPI tooling. For the vcluster project, we have also implemented a CAPI provider - [cluster-api-provider-vcluster](https://github.com/loft-sh/cluster-api-provider-vcluster). Instructions below will describe how to instantiate a vcluster instance using CAPI tooling.
 
 ## Prerequisites 
-[clusterctl](https://cluster-api.sigs.k8s.io/clusterctl/overview.html) CLI tool is used to install the vcluster provider and to generate a custom resource that will serve as a declarative definition of the cluster. It is possible to create the custom resource without clusterctl, but in this guide, we will be using it because it provides access to the templates for the `VCluster` custom resource. Please follow [the official installation instructions for clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl).
+[clusterctl (v1.1.5+)](https://cluster-api.sigs.k8s.io/clusterctl/overview.html) CLI tool is used to install the vcluster provider and to generate a custom resource that will serve as a declarative definition of the cluster. It is possible to create the custom resource without clusterctl, but in this guide, we will be using it because it provides access to the templates for the `VCluster` custom resource. Please follow [the official installation instructions for clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl).
 
 You will need cluster-admin permissions in a Kubernetes cluster where you wish to install the CAPI stack with the vcluster provider. This is required because vcluster instances that will be created by the vcluster provider may have arbitrary RBAC permission requirements. The vcluster provider installation with a more limited permission scope will be introduced in a future version.
 
-### Provider configuration - only clusterctl v1.1.4 and earlier
-If you are using clusterctl version 1.1.4 or earlier, you will need to add the vcluster provider to your local clusterctl configuration. To check which version you have installed look for the value of `GitVersion` field in the output of the `clusterctl version` command. You may skip to the next chapter if you are using a newer version.
-
-```console
-$ clusterctl version
-clusterctl version: &version.Info{Major:"1", Minor:"1", GitVersion:"v1.1.4", GitCommit:"31146bd17a220ef6214c4c7a21f1aa57380b6b1f", GitTreeState:"clean", BuildDate:"2022-03-08T18:52:05Z", GoVersion:"go1.17.3", Compiler:"gc", Platform:"linux/amd64"}
-```
-
-Open your local clusterctl configuration file `~/.cluster-api/clusterctl.yaml` and add vcluster to the list of provider as follows:
-
-```yaml
-providers:
-  - name: vcluster
-    url: https://github.com/loft-sh/cluster-api-provider-vcluster/releases/latest/infrastructure-components.yaml
-    type: InfrastructureProvider                                                           
-```
-
-Verify that the vcluster provider was added successfully using the `clusterctl config repositories` command, e.g.:
-```shell
-clusterctl config repositories | grep vcluster
-vcluster   InfrastructureProvider   https://github.com/loft-sh/cluster-api-provider-vcluster/releases/latest/                    infrastructure-components.yaml
-```
-
 ## Installing the CAPI stack with the vcluster provider
 Switch your kubectl context to the cluster where you wish to install the CAPI stack with the vcluster provider and run the following command:
 ```shell

docs/pages/operator/pausing-vcluster.mdx

@@ -3,10 +3,6 @@ title: Pausing & Resuming vcluster
 sidebar_label: Pausing & Resuming vcluster
 ---
 
-:::warning
-This is an alpha feature and only available in the preview `v0.6.0` versions of vcluster
-:::
-
 Pausing a vcluster means to temporarily scale down the vcluster and delete all its created workloads on the host cluster. This can be useful to save computing resources used by vcluster workloads in the host cluster. 
 
 ## Pausing a vcluster

docs/pages/operator/security.mdx

@@ -15,11 +15,6 @@ Besides these benefits, vcluster by default will **not** provide any workload or
 In general, we recommend to deploy a single vcluster into a namespace and then isolate the namespace, which is far easier than isolating multiple vclusters from each other in a single namespace.
 
 ## Isolated Mode
-
-:::warning
-This is an alpha feature and only available in v0.7.0 and newer vcluster versions
-:::
-
 vcluster offers a feature called isolated mode to automatically isolate workloads in a virtual cluster. Isolated mode can be enabled via the `--isolate` flag in `vcluster create` or through the helm value `isolation.enabled: true`:
 
 ```
@@ -150,7 +145,9 @@ If you want more control over this, you can also use an admission controller, th
 
 ### Advanced Isolation
 
-Besides this basic workload isolation, you could also dive into more advanced isolation methods, such as isolating the workloads on separate nodes or through an other container runtime. Using different nodes for your vcluster workloads can be accomplished through the [--node-selector flag](../architecture/nodes.mdx) on vcluster syncer.
+Besides this basic workload isolation, you could also dive into more advanced isolation methods, such as isolating the workloads on separate nodes or through another container runtime. Using different nodes for your vcluster workloads can be accomplished through the [--node-selector flag](../architecture/nodes.mdx) on vcluster syncer.
+
+You should also be aware that pods created in the vcluster will set their [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/), which will affect scheduling decisions. To prevent the pods from being scheduled to the undesirable nodes you can use the [--node-selector flag](../architecture/nodes.mdx) or admission controller as mentioned above.
 
 ## Network Isolation
 

docs/pages/plugins/overview.mdx

@@ -3,10 +3,6 @@ title: Overview
 sidebar_label: Overview
 ---
 
-:::warning
-This is currently still an alpha feature
-:::
-
 Plugins are a feature to extend the capabilities of vcluster. They allow you to add custom functionality, such as:
 
 1. Syncing specific resources from or to the virtual clusters, including cluster scoped resources like cluster roles