From 63a3f251bfbc8e8312fd76c3f103160efd44abf8 Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Mon, 13 May 2019 17:10:32 -0700 Subject: [PATCH 01/22] Consume new kubernetes releases [#165551713] Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan --- manifests/cfcr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 85cba014..3183f925 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -373,7 +373,7 @@ instance_groups: store_dir: /var/vcap/data release: docker - name: kubernetes-dependencies - release: kubo + release: kubernetes - name: kubelet properties: api-token: ((kubelet-password)) @@ -454,6 +454,8 @@ releases: sha1: c3376e70d6a080054012afa45fae1e9249b2a6d9 url: https://storage.googleapis.com/kubo-precompiled-releases/bpm-1.0.4-ubuntu-xenial-315.64-20190703-011222-636424609.tgz version: 1.0.4 +- name: kubernetes + version: latest stemcells: - alias: default os: ubuntu-xenial From 7b4fbc7b3bb5d2002352dd523d9b7d719038efa9 Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Tue, 14 May 2019 12:12:36 -0700 Subject: [PATCH 02/22] Add kubernetes-dependencies job to apply-specs and master VM [#165551713] --- manifests/cfcr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 3183f925..179fe9be 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -27,6 +27,8 @@ instance_groups: kubernetes-dashboard: ((tls-kubernetes-dashboard)) metrics-server: ((tls-metrics-server)) release: kubo + - name: kubernetes-dependencies + release: kubernetes lifecycle: errand name: apply-addons networks: @@ -49,6 +51,8 @@ instance_groups: certificate: ((tls-etcdctl-flanneld.certificate)) private_key: ((tls-etcdctl-flanneld.private_key)) release: kubo + - name: kubernetes-dependencies + release: kubernetes - name: kube-apiserver properties: admin-password: ((kubo-admin-password)) From 8b17ca88038e34701170c6f9b36ad4290d70fb06 Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Wed, 15 May 2019 11:50:13 -0700 Subject: [PATCH 03/22] Pass addons spec using opsfile [#165551713] --- .../{ => addon-specs}/addons-spec.yml | 0 .../ops-files/addon-specs/coredns-spec.yml | 183 ++++++++++++++++++ .../addon-specs/kubernetes-dashboard-spec.yml | 174 +++++++++++++++++ .../addon-specs/metrics-server-spec.yml | 177 +++++++++++++++++ 4 files changed, 534 insertions(+) rename manifests/ops-files/{ => addon-specs}/addons-spec.yml (100%) create mode 100644 manifests/ops-files/addon-specs/coredns-spec.yml create mode 100644 manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml create mode 100644 manifests/ops-files/addon-specs/metrics-server-spec.yml diff --git a/manifests/ops-files/addons-spec.yml b/manifests/ops-files/addon-specs/addons-spec.yml similarity index 100% rename from manifests/ops-files/addons-spec.yml rename to manifests/ops-files/addon-specs/addons-spec.yml diff --git a/manifests/ops-files/addon-specs/coredns-spec.yml b/manifests/ops-files/addon-specs/coredns-spec.yml new file mode 100644 index 00000000..117b4e3c --- /dev/null +++ b/manifests/ops-files/addon-specs/coredns-spec.yml @@ -0,0 +1,183 @@ +- type: replace + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/coredns-spec? + value: | + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: | + .:53 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf { + policy sequential # needed for workloads to be able to use BOSH-DNS + } + cache 30 + loop + reload + loadbalance + } + --- + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical # Added for Guaranteed Scheduling + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.3.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + # NOTE: Security Context is denied unless privileged containers + # are enabled. Once security context can be separated from + # allow-privileged in the release, then this should become + # conditional. + # securityContext: + # allowPrivilegeEscalation: false + # capabilities: + # add: + # - NET_BIND_SERVICE + # drop: + # - all + # readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + selector: + k8s-app: kube-dns + clusterIP: ((kubedns_service_ip)) + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml b/manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml new file mode 100644 index 00000000..4bf583e0 --- /dev/null +++ b/manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml @@ -0,0 +1,174 @@ +- type: replace + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/kubernetes-dashboard-spec? + value: | + # Copyright 2017 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # Configuration to deploy release version of the Dashboard UI compatible with + # Kubernetes 1.8. + # + # Example usage: kubectl create -f + + --- + # ------------------- Dashboard Service Account ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Role & Role Binding ------------------- # + + kind: Role + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-minimal + namespace: kube-system + rules: + # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. Added separately from Addons Spec + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: kubernetes-dashboard-minimal + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard-minimal + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Config ------------------- # + + apiVersion: v1 + kind: ConfigMap + metadata: + name: kubernetes-dashboard-settings + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + --- + # ------------------- Dashboard Deployment ------------------- # + + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical + containers: + - name: kubernetes-dashboard + image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 50m + memory: 100Mi + ports: + - containerPort: 8443 + protocol: TCP + args: + - --default-cert-dir=/certs + - --tls-cert-file=kubernetes-dashboard-cert + - --tls-key-file=kubernetes-dashboard-key + # Uncomment the following line + # Create on-disk volume to store exec logs + volumeMounts: + - mountPath: /tmp + name: tmp-volume + - mountPath: /certs + name: kubernetes-dashboard-certs + readOnly: true + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + # Comment the following tolerations if Dashboard must not be deployed on master + # tolerations: + # - key: node-role.kubernetes.io/master + # effect: NoSchedule + + --- + # ------------------- Dashboard Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 443 + targetPort: 8443 + type: NodePort diff --git a/manifests/ops-files/addon-specs/metrics-server-spec.yml b/manifests/ops-files/addon-specs/metrics-server-spec.yml new file mode 100644 index 00000000..8b7861f1 --- /dev/null +++ b/manifests/ops-files/addon-specs/metrics-server-spec.yml @@ -0,0 +1,177 @@ +- type: replace + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/metrics-server-spec? + value: | + --- + # ------------------- Auth Delegator ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Auth Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Metrics APIService ------------------- # + + apiVersion: apiregistration.k8s.io/v1beta1 + kind: APIService + metadata: + name: v1beta1.metrics.k8s.io + spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 + + --- + # ------------------- Metrics Server Deployment ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: metrics-server + namespace: kube-system + --- + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server + version: v0.3.1 + spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + - name: metrics-server-secrets + secret: + secretName: metrics-server-certs + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.2 + imagePullPolicy: IfNotPresent + command: + - /metrics-server + - --kubelet-preferred-address-types=InternalIP + - --kubelet-insecure-tls + - --client-ca-file=/var/run/kubernetes/client-ca.crt + - --requestheader-client-ca-file=/var/run/kubernetes/requestheader-client-ca.crt + - --tls-cert-file=/var/run/kubernetes/client.crt + - --tls-private-key-file=/var/run/kubernetes/client.key + ports: + - containerPort: 443 + name: https + protocol: TCP + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: metrics-server-secrets + mountPath: /var/run/kubernetes + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + + --- + # ------------------- Metrics Server Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: https + + --- + # ------------------- Resource Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - create + - get + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system From 4b1e9bd146637fe865e0fd95bfd144c341aca956 Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Wed, 15 May 2019 15:20:06 -0700 Subject: [PATCH 04/22] Move all kubernetes-roles policies from kubo-release to opsfile [#165551713] --- .../{addons-spec.yml => addons.yml} | 2 +- .../{coredns-spec.yml => coredns.yml} | 2 +- ...oard-spec.yml => kubernetes-dashboard.yml} | 2 +- ...ics-server-spec.yml => metrics-server.yml} | 2 +- manifests/ops-files/change-cidrs.yml | 4 - .../ops-files/iaas/azure/cloud-provider.yml | 26 ++++++ .../ops-files/iaas/vsphere/cloud-provider.yml | 17 ++++ .../ops-files/role-policies/cluster-admin.yml | 16 ++++ .../ops-files/role-policies/kube-proxy.yml | 16 ++++ .../kube-system-podsecuritypolicy.yml | 79 +++++++++++++++++++ .../ops-files/role-policies/kubelet-drain.yml | 43 ++++++++++ manifests/ops-files/role-policies/kubelet.yml | 16 ++++ 12 files changed, 217 insertions(+), 8 deletions(-) rename manifests/ops-files/addon-specs/{addons-spec.yml => addons.yml} (79%) rename manifests/ops-files/addon-specs/{coredns-spec.yml => coredns.yml} (99%) rename manifests/ops-files/addon-specs/{kubernetes-dashboard-spec.yml => kubernetes-dashboard.yml} (99%) rename manifests/ops-files/addon-specs/{metrics-server-spec.yml => metrics-server.yml} (99%) create mode 100644 manifests/ops-files/role-policies/cluster-admin.yml create mode 100644 manifests/ops-files/role-policies/kube-proxy.yml create mode 100644 manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml create mode 100644 manifests/ops-files/role-policies/kubelet-drain.yml create mode 100644 manifests/ops-files/role-policies/kubelet.yml diff --git a/manifests/ops-files/addon-specs/addons-spec.yml b/manifests/ops-files/addon-specs/addons.yml similarity index 79% rename from manifests/ops-files/addon-specs/addons-spec.yml rename to manifests/ops-files/addon-specs/addons.yml index 1e767ae2..a62d3eea 100644 --- a/manifests/ops-files/addon-specs/addons-spec.yml +++ b/manifests/ops-files/addon-specs/addons.yml @@ -1,3 +1,3 @@ - type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/addons-spec? + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/addons? value: ((addons-spec)) diff --git a/manifests/ops-files/addon-specs/coredns-spec.yml b/manifests/ops-files/addon-specs/coredns.yml similarity index 99% rename from manifests/ops-files/addon-specs/coredns-spec.yml rename to manifests/ops-files/addon-specs/coredns.yml index 117b4e3c..49122ea8 100644 --- a/manifests/ops-files/addon-specs/coredns-spec.yml +++ b/manifests/ops-files/addon-specs/coredns.yml @@ -1,5 +1,5 @@ - type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/coredns-spec? + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/coredns? value: | --- apiVersion: v1 diff --git a/manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml b/manifests/ops-files/addon-specs/kubernetes-dashboard.yml similarity index 99% rename from manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml rename to manifests/ops-files/addon-specs/kubernetes-dashboard.yml index 4bf583e0..c6f7b9ce 100644 --- a/manifests/ops-files/addon-specs/kubernetes-dashboard-spec.yml +++ b/manifests/ops-files/addon-specs/kubernetes-dashboard.yml @@ -1,5 +1,5 @@ - type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/kubernetes-dashboard-spec? + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/kubernetes-dashboard? value: | # Copyright 2017 The Kubernetes Authors. # diff --git a/manifests/ops-files/addon-specs/metrics-server-spec.yml b/manifests/ops-files/addon-specs/metrics-server.yml similarity index 99% rename from manifests/ops-files/addon-specs/metrics-server-spec.yml rename to manifests/ops-files/addon-specs/metrics-server.yml index 8b7861f1..f4789cb1 100644 --- a/manifests/ops-files/addon-specs/metrics-server-spec.yml +++ b/manifests/ops-files/addon-specs/metrics-server.yml @@ -1,5 +1,5 @@ - type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/metrics-server-spec? + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/metrics-server? value: | --- # ------------------- Auth Delegator ------------------- # diff --git a/manifests/ops-files/change-cidrs.yml b/manifests/ops-files/change-cidrs.yml index c8bbcffc..cf431c81 100644 --- a/manifests/ops-files/change-cidrs.yml +++ b/manifests/ops-files/change-cidrs.yml @@ -1,8 +1,4 @@ --- -- type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/kubedns-service-ip? - value: ((kubedns_service_ip)) - - type: replace path: /instance_groups/name=master/jobs/name=kube-apiserver/properties/k8s-args/service-cluster-ip-range? value: ((service_cluster_cidr)) diff --git a/manifests/ops-files/iaas/azure/cloud-provider.yml b/manifests/ops-files/iaas/azure/cloud-provider.yml index 644ddc07..a39c36a8 100644 --- a/manifests/ops-files/iaas/azure/cloud-provider.yml +++ b/manifests/ops-files/iaas/azure/cloud-provider.yml @@ -80,3 +80,29 @@ - type: replace path: /instance_groups/name=worker/jobs/name=kube-proxy/properties/cloud-provider? value: azure + +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cloud-provider? + value: | + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system diff --git a/manifests/ops-files/iaas/vsphere/cloud-provider.yml b/manifests/ops-files/iaas/vsphere/cloud-provider.yml index 3e5fa472..2e2818f6 100644 --- a/manifests/ops-files/iaas/vsphere/cloud-provider.yml +++ b/manifests/ops-files/iaas/vsphere/cloud-provider.yml @@ -62,3 +62,20 @@ path: /instance_groups/name=apply-addons/jobs/name=apply-specs/consumes? value: cloud-provider: {from: master-cloud-provider} + +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cloud-provider? + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:vsphere-cloud-provider + subjects: + - kind: ServiceAccount + name: vsphere-cloud-provider + namespace: kube-system + roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/cluster-admin.yml b/manifests/ops-files/role-policies/cluster-admin.yml new file mode 100644 index 00000000..fbc3d962 --- /dev/null +++ b/manifests/ops-files/role-policies/cluster-admin.yml @@ -0,0 +1,16 @@ +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cluster-admin? + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:admin + subjects: + - kind: User + name: admin + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kube-proxy.yml b/manifests/ops-files/role-policies/kube-proxy.yml new file mode 100644 index 00000000..b695ee60 --- /dev/null +++ b/manifests/ops-files/role-policies/kube-proxy.yml @@ -0,0 +1,16 @@ +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kube-proxy? + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kube-proxy + subjects: + - kind: User + name: kube-proxy + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml b/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml new file mode 100644 index 00000000..1dfc94af --- /dev/null +++ b/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml @@ -0,0 +1,79 @@ +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kube-system-podsecuritypolicy? + value: | + --- + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: kube-system-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + allowedCapabilities: + - '*' + # Allow core volume types. + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + # rule: 'MustRunAs' + # ranges: + # # Forbid adding the root group. + # - min: 1 + # max: 65535 + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: psp:kube-system-psp + namespace: kube-system + rules: + - apiGroups: + - extensions + resourceNames: + - kube-system-psp + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: psp:kube-system-psp + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: psp:kube-system-psp + subjects: + - kind: ServiceAccount + name: coredns + - kind: ServiceAccount + name: metrics-server + - kind: ServiceAccount + name: kubernetes-dashboard diff --git a/manifests/ops-files/role-policies/kubelet-drain.yml b/manifests/ops-files/role-policies/kubelet-drain.yml new file mode 100644 index 00000000..86dca900 --- /dev/null +++ b/manifests/ops-files/role-policies/kubelet-drain.yml @@ -0,0 +1,43 @@ +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kubelet-drain? + value: | + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "delete"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["statefulsets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get"] + - apiGroups: [""] + resources: ["replicationcontrollers"] + verbs: ["get"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + subjects: + - kind: User + name: kubelet-drain + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: kubo:internal:kubelet-drain + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kubelet.yml b/manifests/ops-files/role-policies/kubelet.yml new file mode 100644 index 00000000..c43bd11f --- /dev/null +++ b/manifests/ops-files/role-policies/kubelet.yml @@ -0,0 +1,16 @@ +- type: replace + path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kubelet? + value: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet + subjects: + - kind: User + name: kubelet + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io From 438caf24faf6b53f690ed63791d8af2b71dbc88d Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Wed, 15 May 2019 15:43:57 -0700 Subject: [PATCH 05/22] Move policies in to cfcr.yml [#165551713] --- manifests/cfcr.yml | 161 ++++++++++++++++++ .../ops-files/role-policies/cluster-admin.yml | 16 -- .../ops-files/role-policies/kube-proxy.yml | 16 -- .../kube-system-podsecuritypolicy.yml | 79 --------- .../ops-files/role-policies/kubelet-drain.yml | 43 ----- manifests/ops-files/role-policies/kubelet.yml | 16 -- 6 files changed, 161 insertions(+), 170 deletions(-) delete mode 100644 manifests/ops-files/role-policies/cluster-admin.yml delete mode 100644 manifests/ops-files/role-policies/kube-proxy.yml delete mode 100644 manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml delete mode 100644 manifests/ops-files/role-policies/kubelet-drain.yml delete mode 100644 manifests/ops-files/role-policies/kubelet.yml diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 179fe9be..b190e0f2 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -300,6 +300,167 @@ instance_groups: admin-username: admin tls: kubernetes: ((tls-kubernetes)) + specs: + cluster-admin: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:admin + subjects: + - kind: User + name: admin + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + kube-proxy: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kube-proxy + subjects: + - kind: User + name: kube-proxy + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io + kube-system-podsecuritypolicy: | + --- + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: kube-system-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + allowedCapabilities: + - '*' + # Allow core volume types. + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + runAsUser: + # Require the container to run without root privileges. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + # rule: 'MustRunAs' + # ranges: + # # Forbid adding the root group. + # - min: 1 + # max: 65535 + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: psp:kube-system-psp + namespace: kube-system + rules: + - apiGroups: + - extensions + resourceNames: + - kube-system-psp + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: psp:kube-system-psp + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: psp:kube-system-psp + subjects: + - kind: ServiceAccount + name: coredns + - kind: ServiceAccount + name: metrics-server + - kind: ServiceAccount + name: kubernetes-dashboard + kubelet-drain: | + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "delete"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["statefulsets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get"] + - apiGroups: [""] + resources: ["replicationcontrollers"] + verbs: ["get"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet-drain + subjects: + - kind: User + name: kubelet-drain + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: kubo:internal:kubelet-drain + apiGroup: rbac.authorization.k8s.io + kubelet: | + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubo:internal:kubelet + subjects: + - kind: User + name: kubelet + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:node + apiGroup: rbac.authorization.k8s.io release: kubo - name: etcd properties: diff --git a/manifests/ops-files/role-policies/cluster-admin.yml b/manifests/ops-files/role-policies/cluster-admin.yml deleted file mode 100644 index fbc3d962..00000000 --- a/manifests/ops-files/role-policies/cluster-admin.yml +++ /dev/null @@ -1,16 +0,0 @@ -- type: replace - path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/cluster-admin? - value: | - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: kubo:internal:admin - subjects: - - kind: User - name: admin - apiGroup: rbac.authorization.k8s.io - roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kube-proxy.yml b/manifests/ops-files/role-policies/kube-proxy.yml deleted file mode 100644 index b695ee60..00000000 --- a/manifests/ops-files/role-policies/kube-proxy.yml +++ /dev/null @@ -1,16 +0,0 @@ -- type: replace - path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kube-proxy? - value: | - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: kubo:internal:kube-proxy - subjects: - - kind: User - name: kube-proxy - apiGroup: rbac.authorization.k8s.io - roleRef: - kind: ClusterRole - name: system:node-proxier - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml b/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml deleted file mode 100644 index 1dfc94af..00000000 --- a/manifests/ops-files/role-policies/kube-system-podsecuritypolicy.yml +++ /dev/null @@ -1,79 +0,0 @@ -- type: replace - path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kube-system-podsecuritypolicy? - value: | - --- - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: kube-system-psp - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' - apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' - apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' - spec: - privileged: false - # Required to prevent escalations to root. - allowPrivilegeEscalation: false - allowedCapabilities: - - '*' - # Allow core volume types. - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - volumes: - - 'configMap' - - 'emptyDir' - - 'projected' - - 'secret' - - 'downwardAPI' - runAsUser: - # Require the container to run without root privileges. - rule: 'RunAsAny' - seLinux: - # This policy assumes the nodes are using AppArmor rather than SELinux. - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - # rule: 'MustRunAs' - # ranges: - # # Forbid adding the root group. - # - min: 1 - # max: 65535 - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: psp:kube-system-psp - namespace: kube-system - rules: - - apiGroups: - - extensions - resourceNames: - - kube-system-psp - resources: - - podsecuritypolicies - verbs: - - use - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: psp:kube-system-psp - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: psp:kube-system-psp - subjects: - - kind: ServiceAccount - name: coredns - - kind: ServiceAccount - name: metrics-server - - kind: ServiceAccount - name: kubernetes-dashboard diff --git a/manifests/ops-files/role-policies/kubelet-drain.yml b/manifests/ops-files/role-policies/kubelet-drain.yml deleted file mode 100644 index 86dca900..00000000 --- a/manifests/ops-files/role-policies/kubelet-drain.yml +++ /dev/null @@ -1,43 +0,0 @@ -- type: replace - path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kubelet-drain? - value: | - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: kubo:internal:kubelet-drain - rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list", "get", "patch", "delete"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["list", "delete"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: ["apps"] - resources: ["statefulsets", "daemonsets"] - verbs: ["get"] - - apiGroups: ["extensions"] - resources: ["replicasets", "daemonsets"] - verbs: ["get"] - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get"] - - apiGroups: [""] - resources: ["replicationcontrollers"] - verbs: ["get"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: kubo:internal:kubelet-drain - subjects: - - kind: User - name: kubelet-drain - apiGroup: rbac.authorization.k8s.io - roleRef: - kind: ClusterRole - name: kubo:internal:kubelet-drain - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/ops-files/role-policies/kubelet.yml b/manifests/ops-files/role-policies/kubelet.yml deleted file mode 100644 index c43bd11f..00000000 --- a/manifests/ops-files/role-policies/kubelet.yml +++ /dev/null @@ -1,16 +0,0 @@ -- type: replace - path: /instance_groups/name=master/jobs/name=kubernetes-roles/properties/specs?/kubelet? - value: | - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: kubo:internal:kubelet - subjects: - - kind: User - name: kubelet - apiGroup: rbac.authorization.k8s.io - roleRef: - kind: ClusterRole - name: system:node - apiGroup: rbac.authorization.k8s.io From 51aa1381ad697f9a7157e9a45809292c9c75c81d Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Thu, 16 May 2019 17:54:23 -0700 Subject: [PATCH 06/22] Fix tests [#165551713] --- bin/run_tests | 2 +- bin/test-standard-ops.sh | 7 ++++++- manifests/README.md | 10 +++++++++- manifests/cfcr.yml | 18 +++++++++++------- .../iaas/gcp/use-vm-extensions.yml | 1 - .../{addon-specs => addons-spec}/addons.yml | 0 .../{addon-specs => addons-spec}/coredns.yml | 0 .../kubernetes-dashboard.yml | 0 .../metrics-server.yml | 0 manifests/ops-files/kubo-local-release.yml | 3 +-- manifests/ops-files/misc/scale-to-one-az.yml | 4 ++-- 11 files changed, 30 insertions(+), 15 deletions(-) rename manifests/ops-files/{addon-specs => addons-spec}/addons.yml (100%) rename manifests/ops-files/{addon-specs => addons-spec}/coredns.yml (100%) rename manifests/ops-files/{addon-specs => addons-spec}/kubernetes-dashboard.yml (100%) rename manifests/ops-files/{addon-specs => addons-spec}/metrics-server.yml (100%) diff --git a/bin/run_tests b/bin/run_tests index e21c84c3..c804a423 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -132,7 +132,7 @@ main() { echo echo -e "${LIGHT_GREEN} ***** Begin affirmative readme operations tests ***** ${NOCOLOR}" local ops_files; - ops_files=$(ls ${home}/manifests/ops-files/*.yml ${home}/manifests/ops-files/iaas/{aws,azure,gcp,openstack,vsphere,virtualbox}/*.yml) + ops_files=$(ls ${home}/manifests/ops-files/*.yml ${home}/manifests/ops-files/iaas/{aws,azure,gcp,openstack,vsphere,virtualbox}/*.yml ${home}/manifests/ops-files/addons-spec/*.yml) ensure_opsfiles_in_readme "$home/manifests/README.md" "$ops_files" echo diff --git a/bin/test-standard-ops.sh b/bin/test-standard-ops.sh index 6808e19d..cbf9e326 100755 --- a/bin/test-standard-ops.sh +++ b/bin/test-standard-ops.sh @@ -58,7 +58,6 @@ test_standard_ops() { check_interpolation "add-hostname-to-master-certificate.yml" "-v api-hostname=example.com" check_interpolation "add-oidc-endpoint.yml" "-l example-vars-files/misc/oidc.yml" check_interpolation "change-audit-log-flags.yml" "-l example-vars-files/change-audit-log-flags.yml" - check_interpolation "addons-spec.yml" "-v addons-spec={}" check_interpolation "allow-privileged-containers.yml" check_interpolation "change-cidrs.yml" "-l example-vars-files/new-cidrs.yml" check_interpolation "disable-anonymous-auth.yml" @@ -70,6 +69,12 @@ test_standard_ops() { check_interpolation "use-hostgw.yml" check_interpolation "set-fs-inotify-limit.yml" "-l example-vars-files/fs-inotify-limit.yml" + ## Addons + check_interpolation "addons-spec/addons.yml" "-v addons-spec={}" + check_interpolation "addons-spec/coredns.yml" "-v kubedns_service_ip=192.168.20.50" + check_interpolation "addons-spec/kubernetes-dashboard.yml" + check_interpolation "addons-spec/metrics-server.yml" + # Etcd check_interpolation "change-etcd-metrics-url.yml" "-v etcd_metrics_protocol=http -v etcd_metrics_port=2378" diff --git a/manifests/README.md b/manifests/README.md index 2b7d2e4d..5c3a9378 100644 --- a/manifests/README.md +++ b/manifests/README.md @@ -83,7 +83,6 @@ For deeper documentation to deploy CFCR go [here](https://github.com/cloudfoundr | Name | Purpose | Notes | |:--- |:--- |:--- | -| [`ops-files/addons-spec.yml`](ops-files/addons-spec.yml) | Addons to be deployed into the Kubernetes cluster | - | | [`ops-files/allow-privileged-containers.yml`](ops-files/allow-privileged-containers.yml) | Allows privileged containers for the Kubernetes cluster. | It is not recommended to use privileged containers however some workloads require it. Container privileges can be limited with the SecurityContextDeny admission plugin (set by default in CFCR). See kubernetes documentation for more information | | [`ops-files/disable-anonymous-auth.yml`](ops-files/disable-anonymous-auth.yml) | Disable `anonymous-auth` on the API server | - | | [`ops-files/add-oidc-endpoint.yml`](ops-files/add-oidc-endpoint.yml) | Enable OIDC authentication for the Kubernetes cluster | - | @@ -98,6 +97,15 @@ For deeper documentation to deploy CFCR go [here](https://github.com/cloudfoundr | [`ops-files/use-hostgw.yml`](ops-files/use-hostgw.yml) | Sets the cluster to use host-gw backend in flannel. Necessary for Windows workers. | - | | [`ops-files/set-fs-inotify-limit.yml`](ops-files/set-fs-inotify-limit.yml) | Configure fs.inotify.max_user_watches.| Extra Vars Required:
- **fs_inotify_max_user_watches:** Required for configuring the max inotify user watches. | +### Addons + +| Name | Purpose | Notes | +|:--- |:--- |:--- | +| [`ops-files/addons-spec/addons.yml`](ops-files/addons-spec/addons.yml) | Addons to be deployed into the Kubernetes cluster | - | +| [`ops-files/addons-spec/coredns.yml`](ops-files/addons-spec/coredns.yml) | Coredns to be deployed into the Kubernetes cluster | `kubedns_service_ip` variable is needed, for example: `10.100.200.10` | +| [`ops-files/addons-spec/kubernetes-dashboard.yml`](ops-files/addons-spec/kubernetes-dashboard.yml) | Kubernetes dashboard to be deployed into the Kubernetes cluster | - | +| [`ops-files/addons-spec/metrics-server.yml`](ops-files/addons-spec/metrics-server.yml) | Metrics server to be deployed into the Kubernetes cluster | - | + ### Etcd | Name | Purpose | Notes| diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index b190e0f2..6a8149bc 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -297,11 +297,11 @@ instance_groups: - name: kubernetes-roles properties: admin-password: ((kubo-admin-password)) - admin-username: admin tls: kubernetes: ((tls-kubernetes)) - specs: - cluster-admin: | + post-start-policies: + - name: cluster-admin + value: | --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -315,7 +315,8 @@ instance_groups: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io - kube-proxy: | + - name: kube-proxy + value: | --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -329,7 +330,8 @@ instance_groups: kind: ClusterRole name: system:node-proxier apiGroup: rbac.authorization.k8s.io - kube-system-podsecuritypolicy: | + - name: kube-system-podsecuritypolicy + value: | --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -406,7 +408,8 @@ instance_groups: name: metrics-server - kind: ServiceAccount name: kubernetes-dashboard - kubelet-drain: | + - name: kubelet-drain + value: | --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -447,7 +450,8 @@ instance_groups: kind: ClusterRole name: kubo:internal:kubelet-drain apiGroup: rbac.authorization.k8s.io - kubelet: | + - name: kubelet + value: | --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml index 4ac7a7e0..3ba17ad9 100644 --- a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml +++ b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml @@ -1,6 +1,5 @@ vm_extensions: - cloud_properties: - backend_service: ((cfcr_backend_service)) service_account: ((cfcr_master_service_account_address)) name: ((deployment_name))-master-cloud-properties - cloud_properties: diff --git a/manifests/ops-files/addon-specs/addons.yml b/manifests/ops-files/addons-spec/addons.yml similarity index 100% rename from manifests/ops-files/addon-specs/addons.yml rename to manifests/ops-files/addons-spec/addons.yml diff --git a/manifests/ops-files/addon-specs/coredns.yml b/manifests/ops-files/addons-spec/coredns.yml similarity index 100% rename from manifests/ops-files/addon-specs/coredns.yml rename to manifests/ops-files/addons-spec/coredns.yml diff --git a/manifests/ops-files/addon-specs/kubernetes-dashboard.yml b/manifests/ops-files/addons-spec/kubernetes-dashboard.yml similarity index 100% rename from manifests/ops-files/addon-specs/kubernetes-dashboard.yml rename to manifests/ops-files/addons-spec/kubernetes-dashboard.yml diff --git a/manifests/ops-files/addon-specs/metrics-server.yml b/manifests/ops-files/addons-spec/metrics-server.yml similarity index 100% rename from manifests/ops-files/addon-specs/metrics-server.yml rename to manifests/ops-files/addons-spec/metrics-server.yml diff --git a/manifests/ops-files/kubo-local-release.yml b/manifests/ops-files/kubo-local-release.yml index f0cd2d61..4983a8dc 100644 --- a/manifests/ops-files/kubo-local-release.yml +++ b/manifests/ops-files/kubo-local-release.yml @@ -2,5 +2,4 @@ path: /releases/name=kubo value: name: kubo - version: create - url: file://../kubo-release + version: latest diff --git a/manifests/ops-files/misc/scale-to-one-az.yml b/manifests/ops-files/misc/scale-to-one-az.yml index 9a3faef5..5308efa7 100644 --- a/manifests/ops-files/misc/scale-to-one-az.yml +++ b/manifests/ops-files/misc/scale-to-one-az.yml @@ -4,10 +4,10 @@ # in a single Availability Zone. - type: replace path: /instance_groups/name=master/instances - value: 1 + value: ((master_instance)) - type: replace path: /instance_groups/name=worker/instances - value: 1 + value: ((worker_instance)) - type: replace path: /instance_groups/name=apply-addons/instances value: 1 From abef3d4ea50dcead5d49171b399f9e3b01012ea5 Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Fri, 17 May 2019 12:17:03 -0700 Subject: [PATCH 07/22] Remove opsfile of addons-spec and split coredns spec [#165551713] --- bin/run_tests | 2 +- bin/test-standard-ops.sh | 7 +- manifests/README.md | 10 +- manifests/cfcr.yml | 535 ++++++++++++++++++ .../iaas/gcp/use-vm-extensions.yml | 3 +- .../addons.yml => addons-spec.yml} | 0 manifests/ops-files/addons-spec/coredns.yml | 183 ------ .../addons-spec/kubernetes-dashboard.yml | 174 ------ .../ops-files/addons-spec/metrics-server.yml | 177 ------ manifests/ops-files/change-cidrs.yml | 4 + manifests/ops-files/kubo-local-release.yml | 3 +- manifests/ops-files/misc/scale-to-one-az.yml | 4 +- 12 files changed, 548 insertions(+), 554 deletions(-) rename manifests/ops-files/{addons-spec/addons.yml => addons-spec.yml} (100%) delete mode 100644 manifests/ops-files/addons-spec/coredns.yml delete mode 100644 manifests/ops-files/addons-spec/kubernetes-dashboard.yml delete mode 100644 manifests/ops-files/addons-spec/metrics-server.yml diff --git a/bin/run_tests b/bin/run_tests index c804a423..e21c84c3 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -132,7 +132,7 @@ main() { echo echo -e "${LIGHT_GREEN} ***** Begin affirmative readme operations tests ***** ${NOCOLOR}" local ops_files; - ops_files=$(ls ${home}/manifests/ops-files/*.yml ${home}/manifests/ops-files/iaas/{aws,azure,gcp,openstack,vsphere,virtualbox}/*.yml ${home}/manifests/ops-files/addons-spec/*.yml) + ops_files=$(ls ${home}/manifests/ops-files/*.yml ${home}/manifests/ops-files/iaas/{aws,azure,gcp,openstack,vsphere,virtualbox}/*.yml) ensure_opsfiles_in_readme "$home/manifests/README.md" "$ops_files" echo diff --git a/bin/test-standard-ops.sh b/bin/test-standard-ops.sh index cbf9e326..6808e19d 100755 --- a/bin/test-standard-ops.sh +++ b/bin/test-standard-ops.sh @@ -58,6 +58,7 @@ test_standard_ops() { check_interpolation "add-hostname-to-master-certificate.yml" "-v api-hostname=example.com" check_interpolation "add-oidc-endpoint.yml" "-l example-vars-files/misc/oidc.yml" check_interpolation "change-audit-log-flags.yml" "-l example-vars-files/change-audit-log-flags.yml" + check_interpolation "addons-spec.yml" "-v addons-spec={}" check_interpolation "allow-privileged-containers.yml" check_interpolation "change-cidrs.yml" "-l example-vars-files/new-cidrs.yml" check_interpolation "disable-anonymous-auth.yml" @@ -69,12 +70,6 @@ test_standard_ops() { check_interpolation "use-hostgw.yml" check_interpolation "set-fs-inotify-limit.yml" "-l example-vars-files/fs-inotify-limit.yml" - ## Addons - check_interpolation "addons-spec/addons.yml" "-v addons-spec={}" - check_interpolation "addons-spec/coredns.yml" "-v kubedns_service_ip=192.168.20.50" - check_interpolation "addons-spec/kubernetes-dashboard.yml" - check_interpolation "addons-spec/metrics-server.yml" - # Etcd check_interpolation "change-etcd-metrics-url.yml" "-v etcd_metrics_protocol=http -v etcd_metrics_port=2378" diff --git a/manifests/README.md b/manifests/README.md index 5c3a9378..2b7d2e4d 100644 --- a/manifests/README.md +++ b/manifests/README.md @@ -83,6 +83,7 @@ For deeper documentation to deploy CFCR go [here](https://github.com/cloudfoundr | Name | Purpose | Notes | |:--- |:--- |:--- | +| [`ops-files/addons-spec.yml`](ops-files/addons-spec.yml) | Addons to be deployed into the Kubernetes cluster | - | | [`ops-files/allow-privileged-containers.yml`](ops-files/allow-privileged-containers.yml) | Allows privileged containers for the Kubernetes cluster. | It is not recommended to use privileged containers however some workloads require it. Container privileges can be limited with the SecurityContextDeny admission plugin (set by default in CFCR). See kubernetes documentation for more information | | [`ops-files/disable-anonymous-auth.yml`](ops-files/disable-anonymous-auth.yml) | Disable `anonymous-auth` on the API server | - | | [`ops-files/add-oidc-endpoint.yml`](ops-files/add-oidc-endpoint.yml) | Enable OIDC authentication for the Kubernetes cluster | - | @@ -97,15 +98,6 @@ For deeper documentation to deploy CFCR go [here](https://github.com/cloudfoundr | [`ops-files/use-hostgw.yml`](ops-files/use-hostgw.yml) | Sets the cluster to use host-gw backend in flannel. Necessary for Windows workers. | - | | [`ops-files/set-fs-inotify-limit.yml`](ops-files/set-fs-inotify-limit.yml) | Configure fs.inotify.max_user_watches.| Extra Vars Required:
- **fs_inotify_max_user_watches:** Required for configuring the max inotify user watches. | -### Addons - -| Name | Purpose | Notes | -|:--- |:--- |:--- | -| [`ops-files/addons-spec/addons.yml`](ops-files/addons-spec/addons.yml) | Addons to be deployed into the Kubernetes cluster | - | -| [`ops-files/addons-spec/coredns.yml`](ops-files/addons-spec/coredns.yml) | Coredns to be deployed into the Kubernetes cluster | `kubedns_service_ip` variable is needed, for example: `10.100.200.10` | -| [`ops-files/addons-spec/kubernetes-dashboard.yml`](ops-files/addons-spec/kubernetes-dashboard.yml) | Kubernetes dashboard to be deployed into the Kubernetes cluster | - | -| [`ops-files/addons-spec/metrics-server.yml`](ops-files/addons-spec/metrics-server.yml) | Metrics server to be deployed into the Kubernetes cluster | - | - ### Etcd | Name | Purpose | Notes| diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 6a8149bc..fcfb3ede 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -26,6 +26,541 @@ instance_groups: kubernetes: ((tls-kubernetes)) kubernetes-dashboard: ((tls-kubernetes-dashboard)) metrics-server: ((tls-metrics-server)) + specs: + coredns: + - name: service-account + value: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + - name: cluster-role + value: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - name: cluster-role-binding + value: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + - name: config-map + value: + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: | + .:53 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf { + policy sequential # needed for workloads to be able to use BOSH-DNS + } + cache 30 + loop + reload + loadbalance + } + - name: deployment + value: + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical # Added for Guaranteed Scheduling + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.3.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + # NOTE: Security Context is denied unless privileged containers + # are enabled. Once security context can be separated from + # allow-privileged in the release, then this should become + # conditional. + # securityContext: + # allowPrivilegeEscalation: false + # capabilities: + # add: + # - NET_BIND_SERVICE + # drop: + # - all + # readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - name: service + value: + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + selector: + k8s-app: kube-dns + clusterIP: 10.100.200.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + kubernetes-dashboard: | + # Copyright 2017 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # Configuration to deploy release version of the Dashboard UI compatible with + # Kubernetes 1.8. + # + # Example usage: kubectl create -f + + --- + # ------------------- Dashboard Service Account ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Role & Role Binding ------------------- # + + kind: Role + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-minimal + namespace: kube-system + rules: + # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. Added separately from Addons Spec + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: kubernetes-dashboard-minimal + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard-minimal + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + # ------------------- Dashboard Config ------------------- # + + apiVersion: v1 + kind: ConfigMap + metadata: + name: kubernetes-dashboard-settings + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + --- + # ------------------- Dashboard Deployment ------------------- # + + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: + priorityClassName: system-cluster-critical + containers: + - name: kubernetes-dashboard + image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 50m + memory: 100Mi + ports: + - containerPort: 8443 + protocol: TCP + args: + - --default-cert-dir=/certs + - --tls-cert-file=kubernetes-dashboard-cert + - --tls-key-file=kubernetes-dashboard-key + # Uncomment the following line + # Create on-disk volume to store exec logs + volumeMounts: + - mountPath: /tmp + name: tmp-volume + - mountPath: /certs + name: kubernetes-dashboard-certs + readOnly: true + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + # Comment the following tolerations if Dashboard must not be deployed on master + # tolerations: + # - key: node-role.kubernetes.io/master + # effect: NoSchedule + + --- + # ------------------- Dashboard Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 443 + targetPort: 8443 + type: NodePort + metrics-server: | + --- + # ------------------- Auth Delegator ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Auth Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + + --- + # ------------------- Metrics APIService ------------------- # + + apiVersion: apiregistration.k8s.io/v1beta1 + kind: APIService + metadata: + name: v1beta1.metrics.k8s.io + spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 + + --- + # ------------------- Metrics Server Deployment ------------------- # + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: metrics-server + namespace: kube-system + --- + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server + version: v0.3.1 + spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + - name: metrics-server-secrets + secret: + secretName: metrics-server-certs + containers: + - name: metrics-server + image: k8s.gcr.io/metrics-server-amd64:v0.3.2 + imagePullPolicy: IfNotPresent + command: + - /metrics-server + - --kubelet-preferred-address-types=InternalIP + - --kubelet-insecure-tls + - --client-ca-file=/var/run/kubernetes/client-ca.crt + - --requestheader-client-ca-file=/var/run/kubernetes/requestheader-client-ca.crt + - --tls-cert-file=/var/run/kubernetes/client.crt + - --tls-private-key-file=/var/run/kubernetes/client.key + ports: + - containerPort: 443 + name: https + protocol: TCP + volumeMounts: + - name: tmp-dir + mountPath: /tmp + - name: metrics-server-secrets + mountPath: /var/run/kubernetes + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + + --- + # ------------------- Metrics Server Service ------------------- # + + apiVersion: v1 + kind: Service + metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: https + + --- + # ------------------- Resource Reader ------------------- # + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - create + - get + - apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system release: kubo - name: kubernetes-dependencies release: kubernetes diff --git a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml index 3ba17ad9..1d7d3a78 100644 --- a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml +++ b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml @@ -1,7 +1,8 @@ vm_extensions: - cloud_properties: + backend_service: ((cfcr_backend_service)) service_account: ((cfcr_master_service_account_address)) name: ((deployment_name))-master-cloud-properties - cloud_properties: - service_account: ((cfcr_worker_service_account_address)) + service_accoqunt: ((cfcr_worker_service_account_address)) name: ((deployment_name))-worker-cloud-properties diff --git a/manifests/ops-files/addons-spec/addons.yml b/manifests/ops-files/addons-spec.yml similarity index 100% rename from manifests/ops-files/addons-spec/addons.yml rename to manifests/ops-files/addons-spec.yml diff --git a/manifests/ops-files/addons-spec/coredns.yml b/manifests/ops-files/addons-spec/coredns.yml deleted file mode 100644 index 49122ea8..00000000 --- a/manifests/ops-files/addons-spec/coredns.yml +++ /dev/null @@ -1,183 +0,0 @@ -- type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/coredns? - value: | - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: coredns - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns - subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: coredns - namespace: kube-system - data: - Corefile: | - .:53 { - errors - health - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - upstream - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - proxy . /etc/resolv.conf { - policy sequential # needed for workloads to be able to use BOSH-DNS - } - cache 30 - loop - reload - loadbalance - } - --- - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" - spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - annotations: - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - priorityClassName: system-cluster-critical # Added for Guaranteed Scheduling - serviceAccountName: coredns - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - containers: - - name: coredns - image: coredns/coredns:1.3.1 - imagePullPolicy: IfNotPresent - resources: - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - # NOTE: Security Context is denied unless privileged containers - # are enabled. Once security context can be separated from - # allow-privileged in the release, then this should become - # conditional. - # securityContext: - # allowPrivilegeEscalation: false - # capabilities: - # add: - # - NET_BIND_SERVICE - # drop: - # - all - # readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile - --- - apiVersion: v1 - kind: Service - metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - spec: - selector: - k8s-app: kube-dns - clusterIP: ((kubedns_service_ip)) - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP diff --git a/manifests/ops-files/addons-spec/kubernetes-dashboard.yml b/manifests/ops-files/addons-spec/kubernetes-dashboard.yml deleted file mode 100644 index c6f7b9ce..00000000 --- a/manifests/ops-files/addons-spec/kubernetes-dashboard.yml +++ /dev/null @@ -1,174 +0,0 @@ -- type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/kubernetes-dashboard? - value: | - # Copyright 2017 The Kubernetes Authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # Configuration to deploy release version of the Dashboard UI compatible with - # Kubernetes 1.8. - # - # Example usage: kubectl create -f - - --- - # ------------------- Dashboard Service Account ------------------- # - - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kube-system - - --- - # ------------------- Dashboard Role & Role Binding ------------------- # - - kind: Role - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-minimal - namespace: kube-system - rules: - # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. Added separately from Addons Spec - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create"] - - # Allow Dashboard to get, update and delete Dashboard exclusive secrets. - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] - verbs: ["get", "update", "delete"] - # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. - - apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["kubernetes-dashboard-settings"] - verbs: ["get", "update"] - - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: kubernetes-dashboard-minimal - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubernetes-dashboard-minimal - subjects: - - kind: ServiceAccount - name: kubernetes-dashboard - namespace: kube-system - - --- - # ------------------- Dashboard Config ------------------- # - - apiVersion: v1 - kind: ConfigMap - metadata: - name: kubernetes-dashboard-settings - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - --- - # ------------------- Dashboard Deployment ------------------- # - - apiVersion: apps/v1 - kind: Deployment - metadata: - name: kubernetes-dashboard - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - spec: - selector: - matchLabels: - k8s-app: kubernetes-dashboard - template: - metadata: - labels: - k8s-app: kubernetes-dashboard - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - seccomp.security.alpha.kubernetes.io/pod: 'docker/default' - spec: - priorityClassName: system-cluster-critical - containers: - - name: kubernetes-dashboard - image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1 - resources: - limits: - cpu: 100m - memory: 300Mi - requests: - cpu: 50m - memory: 100Mi - ports: - - containerPort: 8443 - protocol: TCP - args: - - --default-cert-dir=/certs - - --tls-cert-file=kubernetes-dashboard-cert - - --tls-key-file=kubernetes-dashboard-key - # Uncomment the following line - # Create on-disk volume to store exec logs - volumeMounts: - - mountPath: /tmp - name: tmp-volume - - mountPath: /certs - name: kubernetes-dashboard-certs - readOnly: true - livenessProbe: - httpGet: - scheme: HTTPS - path: / - port: 8443 - initialDelaySeconds: 30 - timeoutSeconds: 30 - volumes: - - name: kubernetes-dashboard-certs - secret: - secretName: kubernetes-dashboard-certs - - name: tmp-volume - emptyDir: {} - serviceAccountName: kubernetes-dashboard - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - # Comment the following tolerations if Dashboard must not be deployed on master - # tolerations: - # - key: node-role.kubernetes.io/master - # effect: NoSchedule - - --- - # ------------------- Dashboard Service ------------------- # - - apiVersion: v1 - kind: Service - metadata: - name: kubernetes-dashboard - namespace: kube-system - labels: - k8s-app: kubernetes-dashboard - spec: - selector: - k8s-app: kubernetes-dashboard - ports: - - port: 443 - targetPort: 8443 - type: NodePort diff --git a/manifests/ops-files/addons-spec/metrics-server.yml b/manifests/ops-files/addons-spec/metrics-server.yml deleted file mode 100644 index f4789cb1..00000000 --- a/manifests/ops-files/addons-spec/metrics-server.yml +++ /dev/null @@ -1,177 +0,0 @@ -- type: replace - path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs?/metrics-server? - value: | - --- - # ------------------- Auth Delegator ------------------- # - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: metrics-server:system:auth-delegator - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - - --- - # ------------------- Auth Reader ------------------- # - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: metrics-server-auth-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - - --- - # ------------------- Metrics APIService ------------------- # - - apiVersion: apiregistration.k8s.io/v1beta1 - kind: APIService - metadata: - name: v1beta1.metrics.k8s.io - spec: - service: - name: metrics-server - namespace: kube-system - group: metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 - - --- - # ------------------- Metrics Server Deployment ------------------- # - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: metrics-server - namespace: kube-system - --- - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - name: metrics-server - namespace: kube-system - labels: - k8s-app: metrics-server - version: v0.3.1 - spec: - selector: - matchLabels: - k8s-app: metrics-server - template: - metadata: - name: metrics-server - labels: - k8s-app: metrics-server - spec: - serviceAccountName: metrics-server - volumes: - # mount in tmp so we can safely use from-scratch images and/or read-only containers - - name: tmp-dir - emptyDir: {} - - name: metrics-server-secrets - secret: - secretName: metrics-server-certs - containers: - - name: metrics-server - image: k8s.gcr.io/metrics-server-amd64:v0.3.2 - imagePullPolicy: IfNotPresent - command: - - /metrics-server - - --kubelet-preferred-address-types=InternalIP - - --kubelet-insecure-tls - - --client-ca-file=/var/run/kubernetes/client-ca.crt - - --requestheader-client-ca-file=/var/run/kubernetes/requestheader-client-ca.crt - - --tls-cert-file=/var/run/kubernetes/client.crt - - --tls-private-key-file=/var/run/kubernetes/client.key - ports: - - containerPort: 443 - name: https - protocol: TCP - volumeMounts: - - name: tmp-dir - mountPath: /tmp - - name: metrics-server-secrets - mountPath: /var/run/kubernetes - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - - --- - # ------------------- Metrics Server Service ------------------- # - - apiVersion: v1 - kind: Service - metadata: - name: metrics-server - namespace: kube-system - labels: - kubernetes.io/name: "Metrics-server" - spec: - selector: - k8s-app: metrics-server - ports: - - port: 443 - protocol: TCP - targetPort: https - - --- - # ------------------- Resource Reader ------------------- # - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:metrics-server - rules: - - apiGroups: - - "" - resources: - - nodes/stats - verbs: - - create - - get - - apiGroups: - - "" - resources: - - pods - - nodes - - namespaces - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - deployments - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: system:metrics-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system diff --git a/manifests/ops-files/change-cidrs.yml b/manifests/ops-files/change-cidrs.yml index cf431c81..422695a8 100644 --- a/manifests/ops-files/change-cidrs.yml +++ b/manifests/ops-files/change-cidrs.yml @@ -7,6 +7,10 @@ path: /instance_groups/name=worker/jobs/name=kubelet/properties/kubelet-configuration/clusterDNS? value: [((kubedns_service_ip))] +- type: replace + path: /instance_groups/name=apply-addons/jobs/name=apply-specs/properties/specs/coredns/name=service/value/spec/clusterIP + value: ((kubedns_service_ip)) + - type: replace path: /instance_groups/name=worker/jobs/name=flanneld/properties?/pod-network-cidr? value: ((pod_network_cidr)) diff --git a/manifests/ops-files/kubo-local-release.yml b/manifests/ops-files/kubo-local-release.yml index 4983a8dc..f0cd2d61 100644 --- a/manifests/ops-files/kubo-local-release.yml +++ b/manifests/ops-files/kubo-local-release.yml @@ -2,4 +2,5 @@ path: /releases/name=kubo value: name: kubo - version: latest + version: create + url: file://../kubo-release diff --git a/manifests/ops-files/misc/scale-to-one-az.yml b/manifests/ops-files/misc/scale-to-one-az.yml index 5308efa7..9a3faef5 100644 --- a/manifests/ops-files/misc/scale-to-one-az.yml +++ b/manifests/ops-files/misc/scale-to-one-az.yml @@ -4,10 +4,10 @@ # in a single Availability Zone. - type: replace path: /instance_groups/name=master/instances - value: ((master_instance)) + value: 1 - type: replace path: /instance_groups/name=worker/instances - value: ((worker_instance)) + value: 1 - type: replace path: /instance_groups/name=apply-addons/instances value: 1 From 8c57b814c0c800833c35214230d80899659634ef Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Fri, 17 May 2019 15:04:36 -0700 Subject: [PATCH 08/22] Bump metrics server --- manifests/cfcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index fcfb3ede..c5a185a4 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -474,7 +474,7 @@ instance_groups: secretName: metrics-server-certs containers: - name: metrics-server - image: k8s.gcr.io/metrics-server-amd64:v0.3.2 + image: k8s.gcr.io/metrics-server-amd64:v0.3.3 imagePullPolicy: IfNotPresent command: - /metrics-server From 81985fbb5c42ba7bb6681bc2fe37548672d100e3 Mon Sep 17 00:00:00 2001 From: Carlo Colombo Date: Mon, 20 May 2019 15:34:05 -0700 Subject: [PATCH 09/22] Hardcode a url for kubernetes-release [#165551713] Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan Signed-off-by: Sagar Muchhal --- manifests/cfcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index c5a185a4..824e6057 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -1160,6 +1160,7 @@ releases: version: 1.0.4 - name: kubernetes version: latest + url: a stemcells: - alias: default os: ubuntu-xenial From 4d922f0065e82573d4a708fea8da64fddd05fc01 Mon Sep 17 00:00:00 2001 From: Sagar Muchhal Date: Mon, 20 May 2019 15:42:45 -0700 Subject: [PATCH 10/22] Add kubernetes releases to non-precompiled-releases.yml Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan Signed-off-by: Carlo Colombo --- manifests/ops-files/non-precompiled-releases.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index fe3d434b..82833e42 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -17,3 +17,6 @@ sha1: 41df19697d6a69d2552bc2c132928157fa91abe0 url: https://bosh.io/d/github.com/cloudfoundry-incubator/bpm-release?v=1.0.4 version: 1.0.4 + - name: kubernetes + url: hardcode + version: 1.0.0 From 859b5cfcbf525261b7fb25664bfad2ef365a4a94 Mon Sep 17 00:00:00 2001 From: Carlo Colombo Date: Mon, 20 May 2019 15:54:44 -0700 Subject: [PATCH 11/22] Fix kubernetes release url and version Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan Signed-off-by: Sagar Muchhal --- manifests/ops-files/non-precompiled-releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index 82833e42..61b2a4d0 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -18,5 +18,5 @@ url: https://bosh.io/d/github.com/cloudfoundry-incubator/bpm-release?v=1.0.4 version: 1.0.4 - name: kubernetes - url: hardcode - version: 1.0.0 + url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz + version: 0+dev.2 From 97bbfd60d22673170179b69c678a5059a562f7ad Mon Sep 17 00:00:00 2001 From: Sagar Muchhal Date: Mon, 20 May 2019 16:02:22 -0700 Subject: [PATCH 12/22] Fix version Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan Signed-off-by: Carlo Colombo --- manifests/ops-files/non-precompiled-releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index 61b2a4d0..a1cd2901 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -19,4 +19,4 @@ version: 1.0.4 - name: kubernetes url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz - version: 0+dev.2 + version: 0.0.0-dev.1 From ba63d02086b70cdd7f24a9fb945b033e3346b04a Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Mon, 20 May 2019 17:06:15 -0700 Subject: [PATCH 13/22] Fix url Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan Signed-off-by: Lubron Zhan --- manifests/cfcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index 824e6057..df30eba0 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -1160,7 +1160,7 @@ releases: version: 1.0.4 - name: kubernetes version: latest - url: a + url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz stemcells: - alias: default os: ubuntu-xenial From 82e095729ca43d9879fd13d9e544f999fcef896a Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Tue, 28 May 2019 13:44:40 -0700 Subject: [PATCH 14/22] Update kubernetes-release version Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan --- manifests/ops-files/non-precompiled-releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index a1cd2901..e3cdf990 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -19,4 +19,4 @@ version: 1.0.4 - name: kubernetes url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz - version: 0.0.0-dev.1 + version: 0.0.0-dev.2 From 6bf323b55ab22e14becb49290ab922a20a175d9e Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Tue, 28 May 2019 16:03:42 -0700 Subject: [PATCH 15/22] Update kubernetes release version to 0.0.0-dev.3 Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan --- manifests/ops-files/non-precompiled-releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/ops-files/non-precompiled-releases.yml b/manifests/ops-files/non-precompiled-releases.yml index e3cdf990..c6c30bc8 100644 --- a/manifests/ops-files/non-precompiled-releases.yml +++ b/manifests/ops-files/non-precompiled-releases.yml @@ -19,4 +19,4 @@ version: 1.0.4 - name: kubernetes url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz - version: 0.0.0-dev.2 + version: 0.0.0-dev.3 From 23e47be2895c63cc0747eec64ca70947a32d63fc Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Tue, 28 May 2019 18:09:31 -0700 Subject: [PATCH 16/22] Add using latest kubernetes release to misc/dev.yml Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan --- manifests/ops-files/misc/dev.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manifests/ops-files/misc/dev.yml b/manifests/ops-files/misc/dev.yml index f8ebb609..c3a280bf 100644 --- a/manifests/ops-files/misc/dev.yml +++ b/manifests/ops-files/misc/dev.yml @@ -3,3 +3,9 @@ value: name: kubo version: latest + +- type: replace + path: /releases/name=kubernetes? + value: + name: kubernetes + version: latest From ff6b26bd39fe10d92b5dd47cce9c095a53b10717 Mon Sep 17 00:00:00 2001 From: lubronzhan Date: Wed, 29 May 2019 00:42:35 -0700 Subject: [PATCH 17/22] Try another opsfile formate for updating kubo --- manifests/ops-files/misc/dev.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/manifests/ops-files/misc/dev.yml b/manifests/ops-files/misc/dev.yml index c3a280bf..587e05ed 100644 --- a/manifests/ops-files/misc/dev.yml +++ b/manifests/ops-files/misc/dev.yml @@ -1,11 +1,17 @@ -- type: replace +- type: remove path: /releases/name=kubo + +- type: replace + path: /releases/- value: name: kubo version: latest +- type: remove + path: /releases/name=kubernetes + - type: replace - path: /releases/name=kubernetes? + path: /releases/- value: name: kubernetes version: latest From d2a133eca05cdd6a0f5e562193e95759d4628e99 Mon Sep 17 00:00:00 2001 From: lubronzhan Date: Wed, 29 May 2019 10:01:21 -0700 Subject: [PATCH 18/22] Fix type --- manifests/cloud-config/iaas/gcp/use-vm-extensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml index 1d7d3a78..4ac7a7e0 100644 --- a/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml +++ b/manifests/cloud-config/iaas/gcp/use-vm-extensions.yml @@ -4,5 +4,5 @@ vm_extensions: service_account: ((cfcr_master_service_account_address)) name: ((deployment_name))-master-cloud-properties - cloud_properties: - service_accoqunt: ((cfcr_worker_service_account_address)) + service_account: ((cfcr_worker_service_account_address)) name: ((deployment_name))-worker-cloud-properties From 9ded54f9fe7bb6a696c6c254731924aeb6e1bd68 Mon Sep 17 00:00:00 2001 From: lubronzhan Date: Wed, 29 May 2019 10:48:53 -0700 Subject: [PATCH 19/22] Revert "Try another opsfile formate for updating kubo" This reverts commit cafdd2f46eb4ddae2bd460ece5f430651a1b7d06. --- manifests/ops-files/misc/dev.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/manifests/ops-files/misc/dev.yml b/manifests/ops-files/misc/dev.yml index 587e05ed..c3a280bf 100644 --- a/manifests/ops-files/misc/dev.yml +++ b/manifests/ops-files/misc/dev.yml @@ -1,17 +1,11 @@ -- type: remove - path: /releases/name=kubo - - type: replace - path: /releases/- + path: /releases/name=kubo value: name: kubo version: latest -- type: remove - path: /releases/name=kubernetes - - type: replace - path: /releases/- + path: /releases/name=kubernetes? value: name: kubernetes version: latest From 264c608bba1f5d8933ef77f91934efe6961538d2 Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Mon, 13 May 2019 17:10:32 -0700 Subject: [PATCH 20/22] Consume new kubernetes releases [#165551713] Co-authored-by: Winnie Kwon Co-authored-by: Lubron Zhan --- manifests/cfcr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/cfcr.yml b/manifests/cfcr.yml index df30eba0..c5a185a4 100644 --- a/manifests/cfcr.yml +++ b/manifests/cfcr.yml @@ -1160,7 +1160,6 @@ releases: version: 1.0.4 - name: kubernetes version: latest - url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-release.tar.gz stemcells: - alias: default os: ubuntu-xenial From b7ff0df58eb43e36e0289f2a744eb510ae727139 Mon Sep 17 00:00:00 2001 From: Winnie Kwon Date: Wed, 17 Jul 2019 11:57:20 +0800 Subject: [PATCH 21/22] Add worker should also add kubernetes-windows release Signed-off-by: Lubron Zhan --- manifests/ops-files/windows/add-worker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manifests/ops-files/windows/add-worker.yml b/manifests/ops-files/windows/add-worker.yml index b15193c1..53258970 100644 --- a/manifests/ops-files/windows/add-worker.yml +++ b/manifests/ops-files/windows/add-worker.yml @@ -21,6 +21,12 @@ url: "https://storage.googleapis.com/kubo-precompiled-releases/kubo-windows-0.31.0-windows2019-2019.2-20190325-131732-878123.tgz" sha1: "05ead5f098611e25a6fc6e5cfb33825cf1c9b8ae" +- type: replace + path: /releases/- + value: + name: "kubernetes-windows" + version: latest + - type: replace path: /addons/- value: From 8d1dbee8776f6d19d25f22f63629790ec26cfdc3 Mon Sep 17 00:00:00 2001 From: Lubron Zhan Date: Wed, 17 Jul 2019 22:31:31 +0800 Subject: [PATCH 22/22] Add dev url to kubernetes-windows release [#165551713] Signed-off-by: Winnie Kwon --- manifests/ops-files/windows/add-worker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/ops-files/windows/add-worker.yml b/manifests/ops-files/windows/add-worker.yml index 53258970..e3b4789e 100644 --- a/manifests/ops-files/windows/add-worker.yml +++ b/manifests/ops-files/windows/add-worker.yml @@ -25,7 +25,8 @@ path: /releases/- value: name: "kubernetes-windows" - version: latest + url: https://storage.googleapis.com/kubo-pipeline-store-test/kubernetes-windows-release.tar.gz + version: 0.0.0-dev.7 - type: replace path: /addons/-