fix: kubelet checks via config resource

Signed-off-by: chenk <[email protected]>

examples/trivy.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"encoding/json"
+	//"encoding/json"
 	"fmt"
 	"log"
 
@@ -32,61 +32,61 @@ func main() {
 
 	trivyk8s := tk.New(cluster, logger.Sugar(), tk.WithExcludeOwned(true))
 	fmt.Println("Scanning cluster")
+	/*
+		//trivy k8s #cluster
+		artifacts, err := trivyk8s.ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
 
-	//trivy k8s #cluster
-	artifacts, err := trivyk8s.ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
-
-	fmt.Println("Scanning kind 'pods' with exclude-owned=true")
-	artifacts, err = trivyk8s.Resources("pod").AllNamespaces().ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
-
-	fmt.Println("Scanning namespace 'default'")
-	//trivy k8s --namespace default
-	artifacts, err = trivyk8s.Namespace("default").ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
-	fmt.Println("Scanning all namespaces ")
-	artifacts, err = trivyk8s.AllNamespaces().ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
+		fmt.Println("Scanning kind 'pods' with exclude-owned=true")
+		artifacts, err = trivyk8s.Resources("pod").AllNamespaces().ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
 
-	fmt.Println("Scanning namespace 'default', resource 'deployment/orion'")
+		fmt.Println("Scanning namespace 'default'")
+		//trivy k8s --namespace default
+		artifacts, err = trivyk8s.Namespace("default").ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
+		fmt.Println("Scanning all namespaces ")
+		artifacts, err = trivyk8s.AllNamespaces().ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
 
-	//trivy k8s --namespace default deployment/orion
-	artifact, err := trivyk8s.Namespace("default").GetArtifact(ctx, "deploy", "orion")
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifact(artifact)
+		fmt.Println("Scanning namespace 'default', resource 'deployment/orion'")
 
-	fmt.Println("Scanning 'deployments'")
+		//trivy k8s --namespace default deployment/orion
+		artifact, err := trivyk8s.Namespace("default").GetArtifact(ctx, "deploy", "orion")
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifact(artifact)
 
-	//trivy k8s deployment
-	artifacts, err = trivyk8s.Namespace("default").Resources("deployment").ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
+		fmt.Println("Scanning 'deployments'")
 
-	fmt.Println("Scanning 'cm,pods'")
-	//trivy k8s clusterroles,pods
-	artifacts, err = trivyk8s.Namespace("default").Resources("cm,pods").ListArtifacts(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	printArtifacts(artifacts)
+		//trivy k8s deployment
+		artifacts, err = trivyk8s.Namespace("default").Resources("deployment").ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
 
+		fmt.Println("Scanning 'cm,pods'")
+		//trivy k8s clusterroles,pods
+		artifacts, err = trivyk8s.Namespace("default").Resources("cm,pods").ListArtifacts(ctx)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printArtifacts(artifacts)
+	*/
 	tolerations := []corev1.Toleration{
 		{
 			Effect:   corev1.TaintEffectNoSchedule,
@@ -124,16 +124,21 @@ func main() {
 		}
 		fmt.Println(a.RawResource)
 	}
+	/*
+	   bi, err := trivyk8s.ListClusterBomInfo(ctx)
 
-	bi, err := trivyk8s.ListClusterBomInfo(ctx)
-	if err != nil {
-		log.Fatal(err)
-	}
-	bb, err := json.Marshal(bi)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Print(string(bb))
+	   	if err != nil {
+	   		log.Fatal(err)
+	   	}
+
+	   bb, err := json.Marshal(bi)
+
+	   	if err != nil {
+	   		log.Fatal(err)
+	   	}
+
+	   fmt.Print(string(bb))
+	*/
 }
 
 func printArtifacts(artifacts []*artifacts.Artifact) {

pkg/jobs/auth-builder.go

@@ -0,0 +1,63 @@
+package jobs
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	clusterRole    = "node-collector-cr"
+	roleBinding    = "node-collector-rb"
+	serviceAccount = "node-collector-sa"
+)
+
+type AuthOption func(*AuthBuilder)
+
+func WithServiceAccountNamespace(namespace string) AuthOption {
+	return func(a *AuthBuilder) {
+		a.namespace = namespace
+	}
+}
+
+func GetAuth(opts ...AuthOption) (*rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding, *corev1.ServiceAccount, error) {
+	ab := &AuthBuilder{}
+	for _, opt := range opts {
+		opt(ab)
+	}
+	return ab.build()
+}
+
+type AuthBuilder struct {
+	namespace string
+}
+
+func (b *AuthBuilder) build() (*rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding, *corev1.ServiceAccount, error) {
+	// load ClusterRole, ClusterRoleBinding, ServiceAccount
+	template := getTemplate(clusterRole)
+	var cr rbacv1.ClusterRole
+	err := yaml.Unmarshal([]byte(template), &cr)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	template = getTemplate(roleBinding)
+	var rb rbacv1.ClusterRoleBinding
+	err = yaml.Unmarshal([]byte(template), &rb)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	if len(b.namespace) > 0 {
+		rb.Subjects[0].Namespace = b.namespace
+	}
+	template = getTemplate(serviceAccount)
+	var sa corev1.ServiceAccount
+	err = yaml.Unmarshal([]byte(template), &sa)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	if len(b.namespace) > 0 {
+		sa.Namespace = b.namespace
+	}
+	return &cr, &rb, &sa, nil
+
+}

pkg/jobs/builder.go

@@ -143,7 +143,9 @@ func (b *JobBuilder) build() (*batchv1.Job, error) {
 	if len(b.imageRef) > 0 {
 		job.Spec.Template.Spec.Containers[0].Image = b.imageRef
 	}
-
+	if len(b.nodeSelector) > 0 {
+		job.Spec.Template.Spec.Containers[0].Args = append(job.Spec.Template.Spec.Containers[0].Args, "--node", b.nodeSelector)
+	}
 	if b.nodeSelector != "" {
 		job.Spec.Template.Spec.NodeSelector = map[string]string{
 			corev1.LabelHostname: b.nodeSelector,

pkg/jobs/collector.go

@@ -185,6 +185,34 @@ type ObjectRef struct {
 // ApplyAndCollect deploy k8s job by template to  specific node  and namespace, it read pod logs
 // cleaning up job and returning it output (for cli use-case)
 func (jb *jobCollector) ApplyAndCollect(ctx context.Context, nodeName string) (string, error) {
+
+	_, err := jb.getTrivyNamespace(ctx)
+	if err != nil {
+		if k8sapierror.IsNotFound(err) {
+			trivyNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: jb.namespace}}
+			_, err = jb.cluster.GetK8sClientSet().CoreV1().Namespaces().Create(ctx, trivyNamespace, metav1.CreateOptions{})
+			if err != nil {
+				return "", err
+			}
+		}
+	}
+	cr, rb, sa, err := GetAuth(WithServiceAccountNamespace(jb.namespace))
+	if err != nil {
+		return "", fmt.Errorf("running node-collector job: %w", err)
+	}
+	_, err = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{})
+	if err != nil {
+		return "", fmt.Errorf("creating cluster role: %w", err)
+	}
+	_, err = jb.cluster.GetK8sClientSet().CoreV1().ServiceAccounts(jb.namespace).Create(ctx, sa, metav1.CreateOptions{})
+	if err != nil {
+		return "", fmt.Errorf("creating service account: %w", err)
+	}
+	_, err = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoleBindings().Create(ctx, rb, metav1.CreateOptions{})
+	if err != nil {
+		return "", fmt.Errorf("creating role binding: %w", err)
+	}
+
 	job, err := GetJob(
 		WithTemplate(jb.templateName),
 		WithNamespace(jb.namespace),
@@ -211,22 +239,21 @@ func (jb *jobCollector) ApplyAndCollect(ctx context.Context, nodeName string) (s
 		return "", fmt.Errorf("running node-collector job: %w", err)
 	}
 
-	_, err = jb.getTrivyNamespace(ctx)
-	if err != nil {
-		if k8sapierror.IsNotFound(err) {
-			trivyNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: jb.namespace}}
-			_, err = jb.cluster.GetK8sClientSet().CoreV1().Namespaces().Create(ctx, trivyNamespace, metav1.CreateOptions{})
-			if err != nil {
-				return "", err
-			}
-		}
-	}
 	err = New(WithTimeout(jb.timeout)).Run(ctx, NewRunnableJob(jb.cluster.GetK8sClientSet(), job))
 	if err != nil {
 		return "", fmt.Errorf("running node-collector job: %w", err)
 	}
 	defer func() {
 		background := metav1.DeletePropagationBackground
+		_ = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoleBindings().Delete(ctx, roleBinding, metav1.DeleteOptions{
+			PropagationPolicy: &background,
+		})
+		_ = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoles().Delete(ctx, clusterRole, metav1.DeleteOptions{
+			PropagationPolicy: &background,
+		})
+		_ = jb.cluster.GetK8sClientSet().CoreV1().ServiceAccounts(job.Namespace).Delete(ctx, serviceAccount, metav1.DeleteOptions{
+			PropagationPolicy: &background,
+		})
 		_ = jb.cluster.GetK8sClientSet().BatchV1().Jobs(job.Namespace).Delete(ctx, job.Name, metav1.DeleteOptions{
 			PropagationPolicy: &background,
 		})
@@ -255,6 +282,7 @@ func (jb *jobCollector) Apply(ctx context.Context, nodeName string) (*batchv1.Jo
 		withSecurityContext(jb.securityContext),
 		WithTolerations(jb.tolerations),
 		WithJobServiceAccount(jb.serviceAccount),
+		WithNodeSelector(nodeName),
 		WithNodeCollectorImageRef(jb.imageRef),
 		WithAnnotation(jb.annotation),
 		WithTemplate(jb.templateName),

pkg/jobs/template/cluster-role.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-collector-cr
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/proxy
+    verbs:
+      - get

pkg/jobs/template/node-collector.yaml

@@ -14,7 +14,7 @@ spec:
       automountServiceAccountToken: true
       containers:
         - name: node-collector
-          image: ghcr.io/aquasecurity/node-collector:0.0.9
+          image: ghcr.io/aquasecurity/node-collector:0.1.1
           command:
             - node-collector
           args:

pkg/jobs/template/role-binding.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-collector-rb
+  labels:
+    app.kubernetes.io/version: 0.17.1
+    app.kubernetes.io/managed-by: kubectl
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-collector-cr
+subjects:
+  - kind: ServiceAccount
+    name: node-collector-sa
+    namespace: trivy-temp

pkg/jobs/template/service-account.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-collector-sa
+  namespace: trivy-temp
+  labels:
+    app.kubernetes.io/managed-by: kubectl