pipeline.yaml

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: my-pipeline-
  annotations:
    pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
    pipelines.kubeflow.org/pipeline_compilation_time: '2024-01-10T13:17:55.128150'
    pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"name": "url", "type": "String"},
      {"default": "", "name": "pipeline-root"}, {"default": "pipeline/my-pipeline",
      "name": "pipeline-name"}], "name": "my-pipeline"}'
    pipelines.kubeflow.org/v2_pipeline: "true"
  labels:
    pipelines.kubeflow.org/v2_pipeline: "true"
    pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
spec:
  entrypoint: my-pipeline
  templates:
  - name: download-data
    container:
      args:
      - sh
      - -c
      - |2

        if ! [ -x "$(command -v pip)" ]; then
            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
        fi

        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'pandas' 'openpyxl' 'kfp==1.8.22' && "$0" "$@"
      - sh
      - -ec
      - |
        program_path=$(mktemp -d)
        printf "%s" "$0" > "$program_path/ephemeral_component.py"
        python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
      - |2+

        import kfp
        from kfp.v2 import dsl
        from kfp.v2.dsl import *
        from typing import *

        def download_data(url:str, output_csv:Output[Dataset]):
            import pandas as pd

            # Use pandas excel reader
            df = pd.read_excel(url)
            df = df.sample(frac=1).reset_index(drop=True)
            df.to_csv(output_csv.path, index=False)

      - --executor_input
      - '{{$}}'
      - --function_to_execute
      - download_data
      command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST),
        --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO),
        --container_image, $(KFP_V2_IMAGE), --task_name, download-data, --pipeline_name,
        '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource,
        workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name,
        $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}',
        --enable_caching, $(ENABLE_CACHING), --, 'url={{inputs.parameters.url}}',
        --]
      env:
      - name: KFP_POD_NAME
        valueFrom:
          fieldRef: {fieldPath: metadata.name}
      - name: KFP_POD_UID
        valueFrom:
          fieldRef: {fieldPath: metadata.uid}
      - name: KFP_NAMESPACE
        valueFrom:
          fieldRef: {fieldPath: metadata.namespace}
      - name: WORKFLOW_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
      - name: KFP_RUN_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'}
      - name: ENABLE_CACHING
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
      - {name: KFP_V2_IMAGE, value: 'python:3.7'}
      - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"url": {"type": "STRING"}},
          "inputArtifacts": {}, "outputParameters": {}, "outputArtifacts": {"output_csv":
          {"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion":
          "0.0.1", "metadataPath": "/tmp/outputs/output_csv/data"}}}'}
      envFrom:
      - configMapRef: {name: metadata-grpc-configmap, optional: true}
      image: python:3.7
      volumeMounts:
      - {mountPath: /kfp-launcher, name: kfp-launcher}
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      - {name: url}
    outputs:
      artifacts:
      - {name: download-data-output_csv, path: /tmp/outputs/output_csv/data}
    metadata:
      annotations:
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/component_ref: '{}'
        pipelines.kubeflow.org/arguments.parameters: '{"url": "{{inputs.parameters.url}}"}'
      labels:
        pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
        pipelines.kubeflow.org/pipeline-sdk-type: kfp
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/enable_caching: "true"
    initContainers:
    - command: [launcher, --copy, /kfp-launcher/launch]
      image: gcr.io/ml-pipeline/kfp-launcher:1.8.7
      name: kfp-launcher
      mirrorVolumeMounts: true
    volumes:
    - {name: kfp-launcher}
  - name: eval-model
    container:
      args:
      - sh
      - -c
      - |2

        if ! [ -x "$(command -v pip)" ]; then
            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
        fi

        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'tensorflow' 'pandas' 'kfp==1.8.22' && "$0" "$@"
      - sh
      - -ec
      - |
        program_path=$(mktemp -d)
        printf "%s" "$0" > "$program_path/ephemeral_component.py"
        python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
      - |2+

        import kfp
        from kfp.v2 import dsl
        from kfp.v2.dsl import *
        from typing import *

        def eval_model(input_model: Input[Model], input_history: Input[Artifact],
                       input_test_x: Input[Dataset], input_test_y: Input[Artifact],
                       MLPipeline_Metrics: Output[Metrics]):
            import pandas as pd
            import tensorflow as tf
            import pickle

            model = tf.keras.models.load_model(input_model.path)

            norm_test_X = pd.read_csv(input_test_x.path)

            with open(input_test_y.path, "rb") as file:
                test_Y = pickle.load(file)

            # Test the model and print loss and mse for both outputs
            loss, Y1_loss, Y2_loss, Y1_rmse, Y2_rmse = model.evaluate(x=norm_test_X, y=test_Y)
            print("Loss = {}, Y1_loss = {}, Y1_mse = {}, Y2_loss = {}, Y2_mse = {}".format(loss, Y1_loss, Y1_rmse, Y2_loss, Y2_rmse))

            MLPipeline_Metrics.log_metric("loss", loss)
            MLPipeline_Metrics.log_metric("Y1_loss", Y1_loss)
            MLPipeline_Metrics.log_metric("Y2_loss", Y2_loss)
            MLPipeline_Metrics.log_metric("Y1_rmse", Y1_rmse)
            MLPipeline_Metrics.log_metric("Y2_rmse", Y2_rmse)

      - --executor_input
      - '{{$}}'
      - --function_to_execute
      - eval_model
      command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST),
        --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO),
        --container_image, $(KFP_V2_IMAGE), --task_name, eval-model, --pipeline_name,
        '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource,
        workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name,
        $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}',
        --enable_caching, $(ENABLE_CACHING), --, --]
      env:
      - name: KFP_POD_NAME
        valueFrom:
          fieldRef: {fieldPath: metadata.name}
      - name: KFP_POD_UID
        valueFrom:
          fieldRef: {fieldPath: metadata.uid}
      - name: KFP_NAMESPACE
        valueFrom:
          fieldRef: {fieldPath: metadata.namespace}
      - name: WORKFLOW_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
      - name: KFP_RUN_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'}
      - name: ENABLE_CACHING
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
      - {name: KFP_V2_IMAGE, value: 'python:3.7'}
      - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
          {"input_history": {"metadataPath": "/tmp/inputs/input_history/data", "schemaTitle":
          "system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}, "input_model":
          {"metadataPath": "/tmp/inputs/input_model/data", "schemaTitle": "system.Model",
          "instanceSchema": "", "schemaVersion": "0.0.1"}, "input_test_x": {"metadataPath":
          "/tmp/inputs/input_test_x/data", "schemaTitle": "system.Dataset", "instanceSchema":
          "", "schemaVersion": "0.0.1"}, "input_test_y": {"metadataPath": "/tmp/inputs/input_test_y/data",
          "schemaTitle": "system.Artifact", "instanceSchema": "", "schemaVersion":
          "0.0.1"}}, "outputParameters": {}, "outputArtifacts": {"MLPipeline_Metrics":
          {"schemaTitle": "system.Metrics", "instanceSchema": "", "schemaVersion":
          "0.0.1", "metadataPath": "/tmp/outputs/MLPipeline_Metrics/data"}}}'}
      envFrom:
      - configMapRef: {name: metadata-grpc-configmap, optional: true}
      image: python:3.7
      volumeMounts:
      - {mountPath: /kfp-launcher, name: kfp-launcher}
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      artifacts:
      - {name: train-model-output_history, path: /tmp/inputs/input_history/data}
      - {name: train-model-output_model, path: /tmp/inputs/input_model/data}
      - {name: preprocess-data-output_test_x, path: /tmp/inputs/input_test_x/data}
      - {name: preprocess-data-output_test_y, path: /tmp/inputs/input_test_y/data}
    outputs:
      artifacts:
      - {name: eval-model-MLPipeline_Metrics, path: /tmp/outputs/MLPipeline_Metrics/data}
    metadata:
      annotations:
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/component_ref: '{}'
      labels:
        pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
        pipelines.kubeflow.org/pipeline-sdk-type: kfp
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/enable_caching: "true"
    initContainers:
    - command: [launcher, --copy, /kfp-launcher/launch]
      image: gcr.io/ml-pipeline/kfp-launcher:1.8.7
      name: kfp-launcher
      mirrorVolumeMounts: true
    volumes:
    - {name: kfp-launcher}
  - name: my-pipeline
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      - {name: url}
    dag:
      tasks:
      - name: download-data
        template: download-data
        arguments:
          parameters:
          - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'}
          - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'}
          - {name: url, value: '{{inputs.parameters.url}}'}
      - name: eval-model
        template: eval-model
        dependencies: [preprocess-data, train-model]
        arguments:
          parameters:
          - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'}
          - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'}
          artifacts:
          - {name: preprocess-data-output_test_x, from: '{{tasks.preprocess-data.outputs.artifacts.preprocess-data-output_test_x}}'}
          - {name: preprocess-data-output_test_y, from: '{{tasks.preprocess-data.outputs.artifacts.preprocess-data-output_test_y}}'}
          - {name: train-model-output_history, from: '{{tasks.train-model.outputs.artifacts.train-model-output_history}}'}
          - {name: train-model-output_model, from: '{{tasks.train-model.outputs.artifacts.train-model-output_model}}'}
      - name: preprocess-data
        template: preprocess-data
        dependencies: [split-data]
        arguments:
          parameters:
          - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'}
          - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'}
          artifacts:
          - {name: split-data-test_csv, from: '{{tasks.split-data.outputs.artifacts.split-data-test_csv}}'}
          - {name: split-data-train_csv, from: '{{tasks.split-data.outputs.artifacts.split-data-train_csv}}'}
      - name: split-data
        template: split-data
        dependencies: [download-data]
        arguments:
          parameters:
          - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'}
          - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'}
          artifacts:
          - {name: download-data-output_csv, from: '{{tasks.download-data.outputs.artifacts.download-data-output_csv}}'}
      - name: train-model
        template: train-model
        dependencies: [preprocess-data]
        arguments:
          parameters:
          - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'}
          - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'}
          artifacts:
          - {name: preprocess-data-output_train_x, from: '{{tasks.preprocess-data.outputs.artifacts.preprocess-data-output_train_x}}'}
          - {name: preprocess-data-output_train_y, from: '{{tasks.preprocess-data.outputs.artifacts.preprocess-data-output_train_y}}'}
  - name: preprocess-data
    container:
      args:
      - sh
      - -c
      - |2

        if ! [ -x "$(command -v pip)" ]; then
            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
        fi

        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'pandas' 'numpy' 'kfp==1.8.22' && "$0" "$@"
      - sh
      - -ec
      - |
        program_path=$(mktemp -d)
        printf "%s" "$0" > "$program_path/ephemeral_component.py"
        python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
      - |2+

        import kfp
        from kfp.v2 import dsl
        from kfp.v2.dsl import *
        from typing import *

        def preprocess_data(input_train_csv: Input[Dataset], input_test_csv: Input[Dataset],
                            output_train_x: Output[Dataset], output_test_x: Output[Dataset],
                            output_train_y: Output[Artifact], output_test_y: Output[Artifact]):

            import pandas as pd
            import numpy as np
            import pickle

            def format_output(data):
                y1 = data.pop('Y1')
                y1 = np.array(y1)
                y2 = data.pop('Y2')
                y2 = np.array(y2)
                return y1, y2

            def norm(x, train_stats):
                return (x - train_stats['mean']) / train_stats['std']

            train = pd.read_csv(input_train_csv.path)
            test = pd.read_csv(input_test_csv.path)

            train_stats = train.describe()

            # Get Y1 and Y2 as the 2 outputs and format them as np arrays
            train_stats.pop('Y1')
            train_stats.pop('Y2')
            train_stats = train_stats.transpose()

            train_Y = format_output(train)
            with open(output_train_y.path, "wb") as file:
              pickle.dump(train_Y, file)

            test_Y = format_output(test)
            with open(output_test_y.path, "wb") as file:
              pickle.dump(test_Y, file)

            # Normalize the training and test data
            norm_train_X = norm(train, train_stats)
            norm_test_X = norm(test, train_stats)

            norm_train_X.to_csv(output_train_x.path, index=False)
            norm_test_X.to_csv(output_test_x.path, index=False)

      - --executor_input
      - '{{$}}'
      - --function_to_execute
      - preprocess_data
      command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST),
        --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO),
        --container_image, $(KFP_V2_IMAGE), --task_name, preprocess-data, --pipeline_name,
        '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource,
        workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name,
        $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}',
        --enable_caching, $(ENABLE_CACHING), --, --]
      env:
      - name: KFP_POD_NAME
        valueFrom:
          fieldRef: {fieldPath: metadata.name}
      - name: KFP_POD_UID
        valueFrom:
          fieldRef: {fieldPath: metadata.uid}
      - name: KFP_NAMESPACE
        valueFrom:
          fieldRef: {fieldPath: metadata.namespace}
      - name: WORKFLOW_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
      - name: KFP_RUN_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'}
      - name: ENABLE_CACHING
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
      - {name: KFP_V2_IMAGE, value: 'python:3.7'}
      - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
          {"input_test_csv": {"metadataPath": "/tmp/inputs/input_test_csv/data", "schemaTitle":
          "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}, "input_train_csv":
          {"metadataPath": "/tmp/inputs/input_train_csv/data", "schemaTitle": "system.Dataset",
          "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": {},
          "outputArtifacts": {"output_test_x": {"schemaTitle": "system.Dataset", "instanceSchema":
          "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_test_x/data"},
          "output_test_y": {"schemaTitle": "system.Artifact", "instanceSchema": "",
          "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_test_y/data"},
          "output_train_x": {"schemaTitle": "system.Dataset", "instanceSchema": "",
          "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_train_x/data"},
          "output_train_y": {"schemaTitle": "system.Artifact", "instanceSchema": "",
          "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_train_y/data"}}}'}
      envFrom:
      - configMapRef: {name: metadata-grpc-configmap, optional: true}
      image: python:3.7
      volumeMounts:
      - {mountPath: /kfp-launcher, name: kfp-launcher}
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      artifacts:
      - {name: split-data-test_csv, path: /tmp/inputs/input_test_csv/data}
      - {name: split-data-train_csv, path: /tmp/inputs/input_train_csv/data}
    outputs:
      artifacts:
      - {name: preprocess-data-output_test_x, path: /tmp/outputs/output_test_x/data}
      - {name: preprocess-data-output_test_y, path: /tmp/outputs/output_test_y/data}
      - {name: preprocess-data-output_train_x, path: /tmp/outputs/output_train_x/data}
      - {name: preprocess-data-output_train_y, path: /tmp/outputs/output_train_y/data}
    metadata:
      annotations:
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/component_ref: '{}'
      labels:
        pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
        pipelines.kubeflow.org/pipeline-sdk-type: kfp
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/enable_caching: "true"
    initContainers:
    - command: [launcher, --copy, /kfp-launcher/launch]
      image: gcr.io/ml-pipeline/kfp-launcher:1.8.7
      name: kfp-launcher
      mirrorVolumeMounts: true
    volumes:
    - {name: kfp-launcher}
  - name: split-data
    container:
      args:
      - sh
      - -c
      - |2

        if ! [ -x "$(command -v pip)" ]; then
            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
        fi

        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'pandas' 'scikit-learn' 'kfp==1.8.22' && "$0" "$@"
      - sh
      - -ec
      - |
        program_path=$(mktemp -d)
        printf "%s" "$0" > "$program_path/ephemeral_component.py"
        python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
      - |2+

        import kfp
        from kfp.v2 import dsl
        from kfp.v2.dsl import *
        from typing import *

        def split_data(input_csv: Input[Dataset], train_csv: Output[Dataset], test_csv: Output[Dataset]):
            import pandas as pd
            from sklearn.model_selection import train_test_split

            df = pd.read_csv(input_csv.path)
            train, test = train_test_split(df, test_size=0.2)

            train.to_csv(train_csv.path, index=False)
            test.to_csv(test_csv.path, index=False)

      - --executor_input
      - '{{$}}'
      - --function_to_execute
      - split_data
      command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST),
        --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO),
        --container_image, $(KFP_V2_IMAGE), --task_name, split-data, --pipeline_name,
        '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource,
        workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name,
        $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}',
        --enable_caching, $(ENABLE_CACHING), --, --]
      env:
      - name: KFP_POD_NAME
        valueFrom:
          fieldRef: {fieldPath: metadata.name}
      - name: KFP_POD_UID
        valueFrom:
          fieldRef: {fieldPath: metadata.uid}
      - name: KFP_NAMESPACE
        valueFrom:
          fieldRef: {fieldPath: metadata.namespace}
      - name: WORKFLOW_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
      - name: KFP_RUN_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'}
      - name: ENABLE_CACHING
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
      - {name: KFP_V2_IMAGE, value: 'python:3.7'}
      - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
          {"input_csv": {"metadataPath": "/tmp/inputs/input_csv/data", "schemaTitle":
          "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters":
          {}, "outputArtifacts": {"test_csv": {"schemaTitle": "system.Dataset", "instanceSchema":
          "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/test_csv/data"},
          "train_csv": {"schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion":
          "0.0.1", "metadataPath": "/tmp/outputs/train_csv/data"}}}'}
      envFrom:
      - configMapRef: {name: metadata-grpc-configmap, optional: true}
      image: python:3.7
      volumeMounts:
      - {mountPath: /kfp-launcher, name: kfp-launcher}
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      artifacts:
      - {name: download-data-output_csv, path: /tmp/inputs/input_csv/data}
    outputs:
      artifacts:
      - {name: split-data-test_csv, path: /tmp/outputs/test_csv/data}
      - {name: split-data-train_csv, path: /tmp/outputs/train_csv/data}
    metadata:
      annotations:
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/component_ref: '{}'
      labels:
        pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
        pipelines.kubeflow.org/pipeline-sdk-type: kfp
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/enable_caching: "true"
    initContainers:
    - command: [launcher, --copy, /kfp-launcher/launch]
      image: gcr.io/ml-pipeline/kfp-launcher:1.8.7
      name: kfp-launcher
      mirrorVolumeMounts: true
    volumes:
    - {name: kfp-launcher}
  - name: train-model
    container:
      args:
      - sh
      - -c
      - |2

        if ! [ -x "$(command -v pip)" ]; then
            python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
        fi

        PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet     --no-warn-script-location 'tensorflow' 'pandas' 'kfp==1.8.22' && "$0" "$@"
      - sh
      - -ec
      - |
        program_path=$(mktemp -d)
        printf "%s" "$0" > "$program_path/ephemeral_component.py"
        python3 -m kfp.v2.components.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
      - |2+

        import kfp
        from kfp.v2 import dsl
        from kfp.v2.dsl import *
        from typing import *

        def train_model(input_train_x: Input[Dataset], input_train_y: Input[Artifact],
                        output_model: Output[Model], output_history: Output[Artifact]):
            import pandas as pd
            import tensorflow as tf
            import pickle

            from tensorflow.keras.models import Model
            from tensorflow.keras.layers import Dense, Input

            norm_train_X = pd.read_csv(input_train_x.path)

            with open(input_train_y.path, "rb") as file:
                train_Y = pickle.load(file)

            def model_builder(train_X):

              # Define model layers.
              input_layer = Input(shape=(len(train_X.columns),))
              first_dense = Dense(units='128', activation='relu')(input_layer)
              second_dense = Dense(units='128', activation='relu')(first_dense)

              # Y1 output will be fed directly from the second dense
              y1_output = Dense(units='1', name='y1_output')(second_dense)
              third_dense = Dense(units='64', activation='relu')(second_dense)

              # Y2 output will come via the third dense
              y2_output = Dense(units='1', name='y2_output')(third_dense)

              # Define the model with the input layer and a list of output layers
              model = Model(inputs=input_layer, outputs=[y1_output, y2_output])

              print(model.summary())

              return model

            model = model_builder(norm_train_X)

            # Specify the optimizer, and compile the model with loss functions for both outputs
            optimizer = tf.keras.optimizers.SGD(learning_rate=0.001)
            model.compile(optimizer=optimizer,
                          loss={'y1_output': 'mse', 'y2_output': 'mse'},
                          metrics={'y1_output': tf.keras.metrics.RootMeanSquaredError(),
                                  'y2_output': tf.keras.metrics.RootMeanSquaredError()})
            # Train the model for 500 epochs
            history = model.fit(norm_train_X, train_Y, epochs=100, batch_size=10)
            model.save(output_model.path)

            with open(output_history.path, "wb") as file:
                train_Y = pickle.dump(history.history, file)

      - --executor_input
      - '{{$}}'
      - --function_to_execute
      - train_model
      command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST),
        --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO),
        --container_image, $(KFP_V2_IMAGE), --task_name, train-model, --pipeline_name,
        '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource,
        workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name,
        $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}',
        --enable_caching, $(ENABLE_CACHING), --, --]
      env:
      - name: KFP_POD_NAME
        valueFrom:
          fieldRef: {fieldPath: metadata.name}
      - name: KFP_POD_UID
        valueFrom:
          fieldRef: {fieldPath: metadata.uid}
      - name: KFP_NAMESPACE
        valueFrom:
          fieldRef: {fieldPath: metadata.namespace}
      - name: WORKFLOW_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'}
      - name: KFP_RUN_ID
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'}
      - name: ENABLE_CACHING
        valueFrom:
          fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'}
      - {name: KFP_V2_IMAGE, value: 'python:3.7'}
      - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts":
          {"input_train_x": {"metadataPath": "/tmp/inputs/input_train_x/data", "schemaTitle":
          "system.Dataset", "instanceSchema": "", "schemaVersion": "0.0.1"}, "input_train_y":
          {"metadataPath": "/tmp/inputs/input_train_y/data", "schemaTitle": "system.Artifact",
          "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": {},
          "outputArtifacts": {"output_history": {"schemaTitle": "system.Artifact",
          "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_history/data"},
          "output_model": {"schemaTitle": "system.Model", "instanceSchema": "", "schemaVersion":
          "0.0.1", "metadataPath": "/tmp/outputs/output_model/data"}}}'}
      envFrom:
      - configMapRef: {name: metadata-grpc-configmap, optional: true}
      image: python:3.7
      volumeMounts:
      - {mountPath: /kfp-launcher, name: kfp-launcher}
    inputs:
      parameters:
      - {name: pipeline-name}
      - {name: pipeline-root}
      artifacts:
      - {name: preprocess-data-output_train_x, path: /tmp/inputs/input_train_x/data}
      - {name: preprocess-data-output_train_y, path: /tmp/inputs/input_train_y/data}
    outputs:
      artifacts:
      - {name: train-model-output_history, path: /tmp/outputs/output_history/data}
      - {name: train-model-output_model, path: /tmp/outputs/output_model/data}
    metadata:
      annotations:
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/component_ref: '{}'
      labels:
        pipelines.kubeflow.org/kfp_sdk_version: 1.8.22
        pipelines.kubeflow.org/pipeline-sdk-type: kfp
        pipelines.kubeflow.org/v2_component: "true"
        pipelines.kubeflow.org/enable_caching: "true"
    initContainers:
    - command: [launcher, --copy, /kfp-launcher/launch]
      image: gcr.io/ml-pipeline/kfp-launcher:1.8.7
      name: kfp-launcher
      mirrorVolumeMounts: true
    volumes:
    - {name: kfp-launcher}
  arguments:
    parameters:
    - {name: url}
    - {name: pipeline-root, value: ''}
    - {name: pipeline-name, value: pipeline/my-pipeline}
  serviceAccountName: pipeline-runner