Bump v1.0.0 (#16)

* update to support kfp v1.0.0 * update readme * remove unused var * add region configs workflow controller * fix argo configs * add executorImage
e2fyi · Oct 14, 2020 · 06a8b84 · 06a8b84
1 parent 4738900
commit 06a8b84
Show file tree

Hide file tree

Showing 11 changed files with 68 additions and 28 deletions.
diff --git a/pipelines/README.md b/pipelines/README.md
@@ -1,9 +1,9 @@
 # Kubeflow Pipelines
 
-[![0.5.1](https://img.shields.io/badge/kubeflow--pipelines-0.5.1-blue.svg)](https://github.com/kubeflow/pipelines/tree/0.5.1)
+[![1.0.0](https://img.shields.io/badge/kubeflow--pipelines-1.0.0-blue.svg)](https://github.com/kubeflow/pipelines/tree/1.0.0)
 
 [Kubeflow pipelines (kfp)](https://github.com/kubeflow/pipelines) are reusable end-to-end ML workflows.
-As of release `0.5.1`, kfp comprises of the following components:
+As of release `1.0.0`, kfp comprises of the following components:
 
 | service                                                    | description                                                                                                                                                                                                    |
 | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |

diff --git a/pipelines/base/aws-configurations-patch.yaml b/pipelines/base/aws-configurations-patch.yaml
@@ -5,14 +5,16 @@ metadata:
 data:
   config: |
     {
-    namespace: $(NAMESPACE),
+    namespace: $(kfp-namespace),
+    executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
     artifactRepository:
     {
         archiveLogs: true,
         s3: {
             bucket: $(artifactRepositoryBucket),
             keyPrefix: $(artifactRepositoryKeyPrefix),
             endpoint: s3.amazonaws.com,
+            region: $(awsRegion),
             insecure: false
         }
     }
@@ -62,10 +64,19 @@ spec:
       containers:
         - name: ml-pipeline-api-server
           env:
-            - name: OBJECTSTORECONFIG_SECURE
-              value: "true"
-            - name: OBJECTSTORECONFIG_BUCKETNAME
-              value: $(pipelineBucket)
+            # we do not use minio related access key options, delete these
+            # env vars as it interferes with config.json settings due to viper config
+            # management taking precedence of env vars over config.json
+            # currently, minio client will init a static client if access keys and secret
+            # are not empty.
+            - $patch: delete
+              name: OBJECTSTORECONFIG_BUCKETNAME
+            - $patch: delete
+              name: OBJECTSTORECONFIG_ACCESSKEY
+            - $patch: delete
+              name: OBJECTSTORECONFIG_SECRETACCESSKEY
+            - $patch: delete
+              name: OBJECTSTORECONFIG_SECURE
             - name: DBCONFIG_USER
               valueFrom:
                 secretKeyRef:

diff --git a/pipelines/base/kustomization.yaml b/pipelines/base/kustomization.yaml
@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 bases:
-  - github.com/kubeflow/pipelines//manifests/kustomize/base?ref=0.5.1
+  - github.com/kubeflow/pipelines/manifests/kustomize/base?ref=1.0.0
 
 resources:
   # use a custom podTemplate definition for tensorboard viewer
@@ -20,7 +20,7 @@ patchesStrategicMerge:
 
 configMapGenerator:
   # Note: pipeline-install-config is used by official kubeflow for installation parameters.
-  # See: https://github.com/kubeflow/pipelines/blob/0.5.1/manifests/kustomize/base/kustomization.yaml
+  # See: https://github.com/kubeflow/pipelines/blob/1.0.0/manifests/kustomize/base/kustomization.yaml
   # Create your own if you want to overwrite those parameters
   - name: pipeline-aws-parameters
     env: params.env    # All parameters can be set inside this file
@@ -62,13 +62,6 @@ vars:
       apiVersion: v1
     fieldref:
       fieldpath: data.artifactRepositoryKeyPrefix
-  - name: pipelineBucket
-    objref:
-      kind: ConfigMap
-      name: pipeline-aws-parameters
-      apiVersion: v1
-    fieldref:
-      fieldpath: data.pipelineBucket
   - name: pipelinePath
     objref:
       kind: ConfigMap
@@ -111,6 +104,41 @@ vars:
       apiVersion: v1
     fieldref:
       fieldpath: data.groupConcatMaxLen
+  - name: dbHost
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.dbHost
+  - name: dbPort
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.dbPort
+  - name: mlmdDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.mlmdDb
+  - name: cacheDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.cacheDb
+  - name: pipelineDb
+    objref:
+      kind: ConfigMap
+      name: pipeline-install-config
+      apiVersion: v1
+    fieldref:
+      fieldpath: data.pipelineDb
 
 configurations:
   - params.yaml
diff --git a/pipelines/base/manifest/ml-pipeline-apiserver-configmap.yaml b/pipelines/base/manifest/ml-pipeline-apiserver-configmap.yaml
@@ -8,20 +8,22 @@ data:
   config.json: |
     {
       "DBConfig": {
-        "Host": "$(DBSERVICE_HOST)",
+        "Host": "$(dbHost)",
         "Port": "3306",
         "DriverName": "mysql",
         "DataSourceName": "",
-        "DBName": "$(DBNAME_PIPELINE)",
+        "DBName": "$(pipelineDb)",
         "GroupConcatMaxLen": "$(groupConcatMaxLen)"
       },
       "ObjectStoreConfig": {
         "Host": "s3.amazonaws.com",
         "Region": "$(awsRegion)",
         "Secure": true,
-        "BucketName": "$(pipelineBucket)",
+        "BucketName": "$(artifactRepositoryBucket)",
         "PipelineFolder": "$(pipelinePath)",
-        "PipelinePath": "$(pipelinePath)"
+        "PipelinePath": "$(pipelinePath)",
+        "AccessKey": "",
+        "SecretAccessKey": ""
       },
       "InitConnectionTimeout": "6m",
       "DefaultPipelineRunnerServiceAccount": "pipeline-runner"

diff --git a/pipelines/base/manifest/mysql.yaml b/pipelines/base/manifest/mysql.yaml
@@ -21,7 +21,7 @@ spec:
         - name: mysql
           env:
             - name: MYSQL_DATABASES
-              value: "$(DBNAME_PIPELINE),$(DBNAME_MLMD),$(DBNAME_CACHE)"
+              value: "$(pipelineDb),$(mlmdDb),$(cacheDb)"
             - name: MYSQL_ROOT_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -64,7 +64,7 @@ kind: Service
 metadata:
   labels:
     app: mysql
-  name: $(DBSERVICE_HOST)
+  name: $(dbHost)
 spec:
   selector:
     app: mysql

diff --git a/pipelines/base/params.env b/pipelines/base/params.env
@@ -5,6 +5,5 @@ mysqlStorageSize=20Gi
 awsRegion=ap-southeast-1
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines
 groupConcatMaxLen=4194304
diff --git a/pipelines/base/params.yaml b/pipelines/base/params.yaml
@@ -27,3 +27,5 @@ varReference:
   kind: StorageClass
 - path: metadata/name
   kind: Service
+- path: spec/ports/port
+  kind: Service
diff --git a/pipelines/cluster-scoped-resources/kustomization.yaml b/pipelines/cluster-scoped-resources/kustomization.yaml
@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 bases:
-  - github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=0.5.1
+  - github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=1.0.0
diff --git a/pipelines/overlay/accesskey/aws-configurations-patch.yaml b/pipelines/overlay/accesskey/aws-configurations-patch.yaml
@@ -2,18 +2,18 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: workflow-controller-configmap
-  namespace: kubeflow
 data:
   config: |
     {
-    executorImage: $(executorImage),
+    namespace: $(kfp-namespace),
     artifactRepository:
     {
         archiveLogs: true,
         s3: {
             bucket: $(artifactRepositoryBucket),
             keyPrefix: $(artifactRepositoryKeyPrefix),
             endpoint: s3.amazonaws.com,
+            region: $(awsRegion),
             insecure: false,
             accessKeySecret: {
                 name: $(awsSecretName),

diff --git a/pipelines/overlay/accesskey/params.env b/pipelines/overlay/accesskey/params.env
@@ -8,5 +8,4 @@ awsAccessKeySecretKey=accesskey
 awsSecretKeySecretKey=secretkey
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines
diff --git a/pipelines/overlay/iam/params.env b/pipelines/overlay/iam/params.env
@@ -7,5 +7,4 @@ awsIAMAnnotationKey=iam.amazonaws.com/role
 awsIAMRole=kube2iam-role
 artifactRepositoryBucket=mlpipeline
 artifactRepositoryKeyPrefix=artifacts
-pipelineBucket=mlpipeline
 pipelinePath=pipelines