From e41a7700cf20994673a3f2d8c41be03d455ce3ae Mon Sep 17 00:00:00 2001 From: chenk Date: Thu, 21 Dec 2023 14:35:51 +0200 Subject: [PATCH] fix: kubelet checks via config resource (#270) * fix: kubelet checks via config resource Signed-off-by: chenk * fix: kubelet checks via config resource Signed-off-by: chenk * fix: kubelet checks via config resource Signed-off-by: chenk * fix: kubelet checks via config resource Signed-off-by: chenk * fix: kubelet checks via config resource Signed-off-by: chenk --------- Signed-off-by: chenk --- pkg/jobs/auth-builder.go | 63 ++++++++++++++++++++++++++ pkg/jobs/builder.go | 4 +- pkg/jobs/builder_test.go | 2 +- pkg/jobs/collector.go | 50 +++++++++++++++----- pkg/jobs/template/cluster-role.yaml | 12 +++++ pkg/jobs/template/node-collector.yaml | 2 +- pkg/jobs/template/role-binding.yaml | 16 +++++++ pkg/jobs/template/service-account.yaml | 8 ++++ 8 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 pkg/jobs/auth-builder.go create mode 100644 pkg/jobs/template/cluster-role.yaml create mode 100644 pkg/jobs/template/role-binding.yaml create mode 100644 pkg/jobs/template/service-account.yaml diff --git a/pkg/jobs/auth-builder.go b/pkg/jobs/auth-builder.go new file mode 100644 index 0000000..b73f495 --- /dev/null +++ b/pkg/jobs/auth-builder.go @@ -0,0 +1,63 @@ +package jobs + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/yaml" +) + +const ( + clusterRole = "node-collector-cr" + roleBinding = "node-collector-rb" + serviceAccount = "node-collector-sa" +) + +type AuthOption func(*AuthBuilder) + +func WithServiceAccountNamespace(namespace string) AuthOption { + return func(a *AuthBuilder) { + a.namespace = namespace + } +} + +func GetAuth(opts ...AuthOption) (*rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding, *corev1.ServiceAccount, error) { + ab := &AuthBuilder{} + for _, opt := range opts { + opt(ab) + } + return ab.build() +} + +type AuthBuilder struct { + namespace string +} + +func (b *AuthBuilder) build() (*rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding, *corev1.ServiceAccount, error) { + // load ClusterRole, ClusterRoleBinding, ServiceAccount + template := getTemplate(clusterRole) + var cr rbacv1.ClusterRole + err := yaml.Unmarshal([]byte(template), &cr) + if err != nil { + return nil, nil, nil, err + } + template = getTemplate(roleBinding) + var rb rbacv1.ClusterRoleBinding + err = yaml.Unmarshal([]byte(template), &rb) + if err != nil { + return nil, nil, nil, err + } + if len(b.namespace) > 0 { + rb.Subjects[0].Namespace = b.namespace + } + template = getTemplate(serviceAccount) + var sa corev1.ServiceAccount + err = yaml.Unmarshal([]byte(template), &sa) + if err != nil { + return nil, nil, nil, err + } + if len(b.namespace) > 0 { + sa.Namespace = b.namespace + } + return &cr, &rb, &sa, nil + +} diff --git a/pkg/jobs/builder.go b/pkg/jobs/builder.go index 2bc525e..337cc62 100644 --- a/pkg/jobs/builder.go +++ b/pkg/jobs/builder.go @@ -143,7 +143,9 @@ func (b *JobBuilder) build() (*batchv1.Job, error) { if len(b.imageRef) > 0 { job.Spec.Template.Spec.Containers[0].Image = b.imageRef } - + if len(b.nodeSelector) > 0 { + job.Spec.Template.Spec.Containers[0].Args = append(job.Spec.Template.Spec.Containers[0].Args, "--node", b.nodeSelector) + } if b.nodeSelector != "" { job.Spec.Template.Spec.NodeSelector = map[string]string{ corev1.LabelHostname: b.nodeSelector, diff --git a/pkg/jobs/builder_test.go b/pkg/jobs/builder_test.go index a6e4b36..e83640a 100644 --- a/pkg/jobs/builder_test.go +++ b/pkg/jobs/builder_test.go @@ -63,7 +63,7 @@ func TestLoadBuilder(t *testing.T) { ReadOnlyRootFilesystem: pointer.Bool(true), }, Name: "node-collector", - Image: "ghcr.io/aquasecurity/node-collector:0.0.9", + Image: "ghcr.io/aquasecurity/node-collector:0.1.1", Command: []string{"node-collector"}, Args: []string{"k8s"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/pkg/jobs/collector.go b/pkg/jobs/collector.go index dfce8c5..aa3a7ee 100644 --- a/pkg/jobs/collector.go +++ b/pkg/jobs/collector.go @@ -185,12 +185,40 @@ type ObjectRef struct { // ApplyAndCollect deploy k8s job by template to specific node and namespace, it read pod logs // cleaning up job and returning it output (for cli use-case) func (jb *jobCollector) ApplyAndCollect(ctx context.Context, nodeName string) (string, error) { + + _, err := jb.getTrivyNamespace(ctx) + if err != nil { + if k8sapierror.IsNotFound(err) { + trivyNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: jb.namespace}} + _, err = jb.cluster.GetK8sClientSet().CoreV1().Namespaces().Create(ctx, trivyNamespace, metav1.CreateOptions{}) + if err != nil { + return "", err + } + } + } + cr, rb, sa, err := GetAuth(WithServiceAccountNamespace(jb.namespace)) + if err != nil { + return "", fmt.Errorf("running node-collector job: %w", err) + } + _, err = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("creating cluster role: %w", err) + } + _, err = jb.cluster.GetK8sClientSet().CoreV1().ServiceAccounts(jb.namespace).Create(ctx, sa, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("creating service account: %w", err) + } + _, err = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoleBindings().Create(ctx, rb, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("creating role binding: %w", err) + } + job, err := GetJob( WithTemplate(jb.templateName), WithNamespace(jb.namespace), WithNodeSelector(nodeName), WithAnnotation(jb.annotation), - WithJobServiceAccount(jb.serviceAccount), + WithJobServiceAccount(serviceAccount), WithLabels(jb.labels), withSecurityContext(jb.securityContext), withPodSecurityContext(jb.podSecurityContext), @@ -211,22 +239,21 @@ func (jb *jobCollector) ApplyAndCollect(ctx context.Context, nodeName string) (s return "", fmt.Errorf("running node-collector job: %w", err) } - _, err = jb.getTrivyNamespace(ctx) - if err != nil { - if k8sapierror.IsNotFound(err) { - trivyNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: jb.namespace}} - _, err = jb.cluster.GetK8sClientSet().CoreV1().Namespaces().Create(ctx, trivyNamespace, metav1.CreateOptions{}) - if err != nil { - return "", err - } - } - } err = New(WithTimeout(jb.timeout)).Run(ctx, NewRunnableJob(jb.cluster.GetK8sClientSet(), job)) if err != nil { return "", fmt.Errorf("running node-collector job: %w", err) } defer func() { background := metav1.DeletePropagationBackground + _ = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoleBindings().Delete(ctx, roleBinding, metav1.DeleteOptions{ + PropagationPolicy: &background, + }) + _ = jb.cluster.GetK8sClientSet().RbacV1().ClusterRoles().Delete(ctx, clusterRole, metav1.DeleteOptions{ + PropagationPolicy: &background, + }) + _ = jb.cluster.GetK8sClientSet().CoreV1().ServiceAccounts(job.Namespace).Delete(ctx, serviceAccount, metav1.DeleteOptions{ + PropagationPolicy: &background, + }) _ = jb.cluster.GetK8sClientSet().BatchV1().Jobs(job.Namespace).Delete(ctx, job.Name, metav1.DeleteOptions{ PropagationPolicy: &background, }) @@ -255,6 +282,7 @@ func (jb *jobCollector) Apply(ctx context.Context, nodeName string) (*batchv1.Jo withSecurityContext(jb.securityContext), WithTolerations(jb.tolerations), WithJobServiceAccount(jb.serviceAccount), + WithNodeSelector(nodeName), WithNodeCollectorImageRef(jb.imageRef), WithAnnotation(jb.annotation), WithTemplate(jb.templateName), diff --git a/pkg/jobs/template/cluster-role.yaml b/pkg/jobs/template/cluster-role.yaml new file mode 100644 index 0000000..e4781f2 --- /dev/null +++ b/pkg/jobs/template/cluster-role.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: node-collector-cr +rules: + - apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get diff --git a/pkg/jobs/template/node-collector.yaml b/pkg/jobs/template/node-collector.yaml index a034289..9893879 100644 --- a/pkg/jobs/template/node-collector.yaml +++ b/pkg/jobs/template/node-collector.yaml @@ -14,7 +14,7 @@ spec: automountServiceAccountToken: true containers: - name: node-collector - image: ghcr.io/aquasecurity/node-collector:0.0.9 + image: ghcr.io/aquasecurity/node-collector:0.1.1 command: - node-collector args: diff --git a/pkg/jobs/template/role-binding.yaml b/pkg/jobs/template/role-binding.yaml new file mode 100644 index 0000000..26982cd --- /dev/null +++ b/pkg/jobs/template/role-binding.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: node-collector-rb + labels: + app.kubernetes.io/version: 0.17.1 + app.kubernetes.io/managed-by: kubectl +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-collector-cr +subjects: + - kind: ServiceAccount + name: node-collector-sa + namespace: trivy-temp diff --git a/pkg/jobs/template/service-account.yaml b/pkg/jobs/template/service-account.yaml new file mode 100644 index 0000000..431edb0 --- /dev/null +++ b/pkg/jobs/template/service-account.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-collector-sa + namespace: trivy-temp + labels: + app.kubernetes.io/managed-by: kubectl